From 293bec1a4133faa4b4ccaf1e604193e74a5c1bb6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 14:23:31 -0400 Subject: [PATCH 01/20] Let transport specs declare pre-calibration stages An optional precalibration_stages resource lists the rules bindings whose outputs may feed calibration. A listed unit-entity stage admits its own nodes, engine reference and outputs; every listed stage, person or unit, lets only its declared outputs flow on; a stage's module and outputs must stay clear of the hold-out measures. With the resource absent or empty, the calibration ancestry guard is unchanged, byte for byte. Built from S5-SPEC revision 2 on a GPT-6.1 Sol build lane. Co-Authored-By: Claude Opus 5.5 --- .../transport-precalibration-stages.added.md | 1 + .../src/microcosm/build/transport/compose.py | 437 +++++- .../shared/test_transport_composed_graph.py | 1333 ++++++++++++++++- 3 files changed, 1758 insertions(+), 13 deletions(-) create mode 100644 changelog.d/transport-precalibration-stages.added.md diff --git a/changelog.d/transport-precalibration-stages.added.md b/changelog.d/transport-precalibration-stages.added.md new file mode 100644 index 0000000000..b29800e40c --- /dev/null +++ b/changelog.d/transport-precalibration-stages.added.md @@ -0,0 +1 @@ +Allow transport specs to declare pre-calibration rules stages with explicit outputs, recursive ancestry checks and holdout isolation for person and unit bindings. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index b9d957799f..8c0f710ed5 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -175,6 +175,8 @@ ) _ENCODINGS = frozenset({"value", "json", "sha256"}) _PREPARED = frozenset({"engine_refs"}) +_PRECALIBRATION_SCHEMA_VERSION = 1 +_PRECALIBRATION_PURPOSES = frozenset({"calibration_measure", "classification"}) #: Typed outputs each skeleton kernel must declare, when the declaration #: leaves them implicit. _OUTPUT_TYPES: Mapping[str, Mapping[str, ArtifactType]] = MappingProxyType( @@ -601,10 +603,18 @@ def _embeds_engine_ref( value: object, references: set[str], unit_modules: frozenset[str] = frozenset(), + allowed_references: frozenset[str] = frozenset(), ) -> bool: if not references and not unit_modules: return False + if isinstance(value, str) and value in allowed_references: + return False if isinstance(value, Mapping): + if ( + allowed_references + and canonical_json(value).decode("utf-8") in allowed_references + ): + return False if unit_modules: from microcosm.frame.adapters.axiom import _ENGINE_REF_SCHEMA @@ -620,12 +630,15 @@ def _embeds_engine_ref( if isinstance(module, str) and module in unit_modules: return True return any( - _embeds_engine_ref(item, references, unit_modules) + _embeds_engine_ref(item, references, unit_modules, allowed_references) for pair in value.items() for item in pair ) if isinstance(value, list | tuple): - return any(_embeds_engine_ref(item, references, unit_modules) for item in value) + return any( + _embeds_engine_ref(item, references, unit_modules, allowed_references) + for item in value + ) if not isinstance(value, str): return False if any( @@ -644,10 +657,410 @@ def _embeds_engine_ref( embedded, _ = decoder.raw_decode(value, index) except ValueError: continue - if _embeds_engine_ref(embedded, references, unit_modules): + if _embeds_engine_ref( + embedded, references, unit_modules, allowed_references + ): return True return False - return decoded != value and _embeds_engine_ref(decoded, references, unit_modules) + return decoded != value and _embeds_engine_ref( + decoded, references, unit_modules, allowed_references + ) + + +def _unit_entities(spec: Mapping[str, object]) -> frozenset[str]: + return frozenset({transport_resource(spec, "benefit_unit_rule").get("entity")}) + + +def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: + """Validate stage declarations before inspecting either graph.""" + + if not _has_resource(spec, "precalibration_stages"): + return {} + resource = transport_resource(spec, "precalibration_stages") + if set(resource) != {"schema_version", "stages"} or ( + type(resource.get("schema_version")) is not int + or resource["schema_version"] != _PRECALIBRATION_SCHEMA_VERSION + ): + raise ValueError( + "precalibration_stages must declare exactly schema_version 1 and stages." + ) + stages = _sequence(resource["stages"], "precalibration_stages.stages") + if not stages: + return {} + bindings = { + row["id"]: row + for row in transport_resource(spec, "axiom_rules_bindings")["bindings"] + } + holdout = ( + transport_resource(spec, "holdout_references").get("target_references", ()) + if _has_resource(spec, "holdout_references") + else () + ) + holdout_columns = {(row["entity"], row["measure"]) for row in holdout} + holdout_modules = { + row["rulespec_path"] + for row in bindings.values() + if any( + (row["entity"], variable) in holdout_columns + for variable in row["variables"] + ) + } + result = {} + identifiers = set() + for item in stages: + stage = _mapping(item, "precalibration_stages stage") + if set(stage) != { + "id", + "rules_binding", + "rulespec_path", + "outputs", + "citation", + }: + raise ValueError( + "precalibration_stages stage has missing or unknown fields." + ) + try: + identifier = _safe_node_id(stage["id"]) + except ValueError as error: + raise ValueError(f"precalibration_stages stage id: {error}") from error + if identifier in identifiers: + raise ValueError("precalibration_stages stage ids must be distinct.") + identifiers.add(identifier) + name = stage["rules_binding"] + if not isinstance(name, str) or name not in bindings: + raise ValueError(f"precalibration_stages names unknown binding {name!r}.") + if name in result: + raise ValueError("precalibration_stages bindings must be distinct.") + binding = bindings[name] + if stage["rulespec_path"] != binding["rulespec_path"]: + raise ValueError( + "precalibration_stages rulespec_path must match its binding." + ) + outputs = _sequence(stage["outputs"], "precalibration_stages outputs") + if not outputs: + raise ValueError( + "precalibration_stages outputs must be nonempty and distinct." + ) + columns = set() + for item in outputs: + output = _mapping(item, "precalibration_stages output") + if set(output) != {"entity", "column", "purpose"}: + raise ValueError( + "precalibration_stages output has missing or unknown fields." + ) + if ( + output["entity"] != binding["entity"] + or output["column"] not in binding["variables"] + ): + raise ValueError( + "precalibration_stages output must name a binding variable on its entity." + ) + if ( + not isinstance(output["purpose"], str) + or output["purpose"] not in _PRECALIBRATION_PURPOSES + ): + raise ValueError("precalibration_stages output has unknown purpose.") + column = (output["entity"], output["column"]) + if column in columns: + raise ValueError( + "precalibration_stages outputs must be nonempty and distinct." + ) + columns.add(column) + if columns & holdout_columns: + raise ValueError("precalibration_stages output is a holdout measure.") + if stage["rulespec_path"] in holdout_modules: + raise ValueError("precalibration_stages module produces a holdout measure.") + result[name] = stage + return result + + +def _precalibration_selections(value: object) -> Iterator[Mapping]: + """Walk nested stage parameters without interpreting selected resource data.""" + + for selection in _selections(value): + yield selection + for item in selection.values(): + yield from _precalibration_selections(item) + + +def _validate_precalibration_ancestry( + spec: Mapping[str, object], + stages: Mapping[str, Mapping], + graph: Graph | None, + config: TransportGraphConfig | None, +) -> None: + document = transport_graph_document(spec) + rows = {row["id"]: row for row in document["nodes"]} + bindings = { + row["id"]: row + for row in transport_resource(spec, "axiom_rules_bindings")["bindings"] + } + unit_entities = _unit_entities(spec) + unit_bindings = { + name: row for name, row in bindings.items() if row["entity"] in unit_entities + } + binding_columns = { + name: {(row["entity"], variable) for variable in row["variables"]} + for name, row in bindings.items() + } + stage_columns = { + name: {(row["entity"], row["column"]) for row in stage["outputs"]} + for name, stage in stages.items() + } + binding_refs = { + name: { + reference + for reference in ( + row.get("engine_ref"), + None if config is None else config.engine_refs.get(name), + ) + if isinstance(reference, str) and reference + } + for name, row in bindings.items() + } + # A prepared reference inventory is bookkeeping, not a stage invocation. + prepared_refs = ( + frozenset( + {canonical_json({"engine_refs": dict(config.engine_refs)}).decode("utf-8")} + ) + if config is not None + else frozenset() + ) + roots = { + name for name, row in rows.items() if row["kernel"].startswith("calibrate.") + } + endpoint = document["endpoints"].get("calibration") + if isinstance(endpoint, str): + roots.add(endpoint) + predecessors = {name: set() for name in rows} + producers = { + (row.get("population"), output["entity"], output["column"]): name + for name, row in rows.items() + if isinstance(row.get("population"), str) + for output in row.get("outputs", ()) + } + for name, row in rows.items(): + base = row.get("base") + if isinstance(base, str): + predecessors[name].add(base) + predecessors[name].update( + item["id"] for item in rows.values() if item.get("population") == base + ) + elif isinstance(row.get("population"), str): + predecessors[name].add(row["population"]) + predecessors[name].update( + edge["producer"] for edge in row.get("artifact_inputs", ()) + ) + predecessors[name].update( + producer + for item in row.get("inputs", ()) + for column in item["columns"] + if ( + producer := producers.get( + (row.get("population"), item["entity"], column) + ) + ) + is not None + and producer != name + ) + + declarations = rows + + def check( + rows: Mapping[str, Mapping], predecessors: Mapping, *, composed: bool + ) -> None: + node_bindings = {} + for name, row in rows.items(): + if composed: + node_bindings[name] = { + binding + for binding in bindings + if config is not None + and binding in config.engine_refs + and _embeds_engine_ref( + row.get("params", {}), + {config.engine_refs[binding]}, + allowed_references=prepared_refs, + ) + } + else: + node_bindings[name] = { + selection["binding"] + for value in row.get("params", {}).values() + for selection in _precalibration_selections(value) + if isinstance(selection.get("binding"), str) + and selection["binding"] in bindings + } + if row.get("rules_binding") in bindings: + node_bindings[name].add(row["rules_binding"]) + stage_nodes = { + name for name in rows if node_bindings[name].intersection(stages) + } + ancestors = _ancestor_closure(predecessors, roots | stage_nodes) + inputs = { + name: { + (item["entity"], column) + for item in row.get("inputs", ()) + for column in item["columns"] + } + for name, row in rows.items() + } + encoded = { + (output["entity"], output["column"]) + for row in rows.values() + if row["kernel"] == "concepts.encode_groups@1" + for output in row.get("outputs", ()) + if output["entity"] in unit_entities + } + unit_stages = set(stages).intersection(unit_bindings) + declared_columns = set().union(*(stage_columns[name] for name in unit_stages)) + unit_columns = set().union( + *(binding_columns[binding] for binding in unit_bindings) + ) + for name, row in rows.items(): + if name not in ancestors: + continue + named = node_bindings[name] + own = named.intersection(stages) + own_units = own.intersection(unit_bindings) + forbidden = set(unit_bindings) - own_units + forbidden_refs = set().union( + *(binding_refs[binding] for binding in forbidden) + ) + forbidden_modules = frozenset( + bindings[binding]["rulespec_path"] for binding in forbidden + ) + allowed_refs = frozenset().union( + *(binding_refs[binding] for binding in own_units) + ) + reason = None + if composed and _embeds_engine_ref( + canonical_json(row.get("params", {})).decode("utf-8"), + set(prepared_refs), + ): + reason = ( + "prepared engine_refs map" + if own_units + else "not a declared pre-calibration stage" + ) + if named.intersection(unit_bindings) - own_units: + reason = ( + "mixes listed and unlisted unit bindings" + if own_units + else "not a declared pre-calibration stage" + ) + params = ( + declarations.get(name, {}).get("params", {}) + if composed + else row.get("params", {}) + ) + selections = ( + selection + for value in params.values() + for selection in _precalibration_selections(value) + ) + for selection in selections: + if selection.get("prepared") == "engine_refs": + reason = ( + "prepared engine_refs map" + if own_units + else "not a declared pre-calibration stage" + ) + if isinstance(selection.get("resource"), str): + resource = _stem(selection["resource"]) + if resource == "holdout_references": + reason = "holdout_references selection" + if ( + resource == GRAPH_RESOURCE + and selection.get("encoding") == "sha256" + ): + reason = "graph resource's own digest" + if selection.get("encoding", "value") != "sha256" and _has_resource( + spec, resource + ): + selected = transport_resource(spec, resource) + for component in selection.get("path", ()): + if ( + not isinstance(selected, Mapping) + or component not in selected + ): + selected = None + break + selected = selected[component] + if _embeds_engine_ref( + selected, + forbidden_refs, + forbidden_modules, + allowed_refs, + ): + reason = reason or "not a declared pre-calibration stage" + outputs = { + (item["entity"], item["column"]) for item in row.get("outputs", ()) + } + own_columns = set().union( + *(binding_columns[binding] for binding in own_units) + ) + if ( + inputs[name] & unit_columns - own_columns - declared_columns + or outputs & unit_columns - own_columns + or (inputs[name] & encoded and not own_units) + ): + reason = reason or "not a declared pre-calibration stage" + if row["kernel"] == "concepts.encode_groups@1": + for column in outputs & encoded: + consumers = [ + consumer + for consumer in rows + if consumer in ancestors and column in inputs[consumer] + ] + if not consumers: + reason = reason or "not a declared pre-calibration stage" + elif outputs & encoded: + reason = reason or "not a declared pre-calibration stage" + for binding in stages: + if binding not in own and inputs[name] & ( + binding_columns[binding] - stage_columns[binding] + ): + reason = "undeclared pre-calibration stage output" + if _embeds_engine_ref( + row.get("params", {}), + forbidden_refs, + forbidden_modules, + allowed_refs, + ): + reason = reason or "not a declared pre-calibration stage" + if any( + isinstance(source, str) and "holdout" in source.lower() + for source in row.get("sources", ()) + ): + reason = "hold-out source" + if reason: + raise ValueError( + f"Transport calibration ancestry: node {name!r} carries {reason}." + ) + + check(rows, predecessors, composed=False) + if graph is None: + return + compiled = compile_graph(graph) + roots.update( + node.id for node in graph.nodes if node.kernel.startswith("calibrate.") + ) + compiled_rows = { + node.id: { + "kernel": node.kernel, + "inputs": [ + {"entity": item.entity, "columns": item.columns} for item in node.inputs + ], + "outputs": [ + {"entity": item.entity, "column": item.column} for item in node.outputs + ], + "params": node.params, + "sources": node.sources, + } + for node in graph.nodes + } + check(compiled_rows, compiled.predecessors, composed=True) def validate_transport_calibration_ancestry( @@ -666,9 +1079,15 @@ def validate_transport_calibration_ancestry( prepared references, including references embedded in JSON text. Benefit-unit attributes remain allowed; excluded columns are the unit rules bindings' outputs and the group encoder's declared outputs. + Declared pre-calibration stages admit their own unit semantics, restrict + outputs flowing onward and recursively check each stage's ancestry. """ + stages = _precalibration_stages(spec) if not _has_resource(spec, GRAPH_RESOURCE): return + if stages: + _validate_precalibration_ancestry(spec, stages, graph, config) + return document = transport_graph_document(spec) rows = { row["id"]: row @@ -677,12 +1096,12 @@ def validate_transport_calibration_ancestry( for item in _sequence(document.get("nodes"), "transport nodes") ) } - unit_entity = transport_resource(spec, "benefit_unit_rule").get("entity") + unit_entities = _unit_entities(spec) bindings = transport_resource(spec, "axiom_rules_bindings").get("bindings", ()) unit_bindings = { row["id"]: row for row in _sequence(bindings, "axiom_rules_bindings.bindings") - if isinstance(row, Mapping) and row.get("entity") == unit_entity + if isinstance(row, Mapping) and row.get("entity") in unit_entities } unit_modules = frozenset( row["rulespec_path"] @@ -690,7 +1109,7 @@ def validate_transport_calibration_ancestry( if isinstance(row.get("rulespec_path"), str) ) forbidden_columns = { - (unit_entity, variable) + (row["entity"], variable) for row in unit_bindings.values() for variable in row.get("variables", ()) } @@ -699,7 +1118,7 @@ def validate_transport_calibration_ancestry( forbidden_columns.update( (output["entity"], output["column"]) for output in row.get("outputs", ()) - if output.get("entity") == unit_entity + if output.get("entity") in unit_entities ) references = { row["engine_ref"] @@ -797,7 +1216,7 @@ def validate_transport_calibration_ancestry( forbidden_columns.update( (item.entity, item.column) for item in node.outputs - if item.entity == unit_entity + if item.entity in unit_entities ) for name in _ancestor_closure(compiled.predecessors, roots): node = graph.node(name) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index b5cdd2856c..7123676985 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -42,6 +42,7 @@ import re import shutil import time +from collections.abc import Iterator, Mapping, Sequence from dataclasses import replace from pathlib import Path from unittest.mock import Mock @@ -75,9 +76,11 @@ from microcosm.graph import ( ArtifactInput, ArtifactOutput, + ArtifactType, Capabilities, ContentStore, Determinism, + Graph, GraphError, KernelBase, KernelRegistry, @@ -87,6 +90,7 @@ Slice, SourceRef, StoreMissError, + StructuralDelta, compile_graph, graph_to_json, run_graph, @@ -239,6 +243,269 @@ def _selectors(spec: dict, resource: str) -> set[str]: return found +# The pre-S5 implementation is intentionally retained for differential checks. +def _precalibration_reference_selections(value: object) -> Iterator[Mapping]: + """Every explicit selection inside a declared parameter value.""" + + if isinstance(value, Mapping): + yield value + elif isinstance(value, list | tuple): + for item in value: + yield from _precalibration_reference_selections(item) + + +def _precalibration_reference_ancestors( + predecessors: Mapping[str, Sequence[str]], roots: set[str] +) -> set[str]: + ancestors = set() + pending = list(roots) + while pending: + name = pending.pop() + if name in ancestors: + continue + if name not in predecessors: + raise ValueError(f"Calibration endpoint {name!r} is not a declared node.") + ancestors.add(name) + pending.extend(predecessors[name]) + return ancestors + + +def _precalibration_reference_embeds( + value: object, + references: set[str], + unit_modules: frozenset[str] = frozenset(), +) -> bool: + if not references and not unit_modules: + return False + if isinstance(value, Mapping): + if unit_modules: + from microcosm.build.transport.registry import PYTHON_ENGINE_REF_FORMAT + from microcosm.frame.adapters.axiom import _ENGINE_REF_SCHEMA + + module = None + if value.get("format") == _ENGINE_REF_SCHEMA: + module = value.get("module") + elif value.get("format") == PYTHON_ENGINE_REF_FORMAT: + binding = value.get("binding") + if isinstance(binding, Mapping): + module = binding.get("rulespec_path") + if isinstance(module, str) and module in unit_modules: + return True + return any( + _precalibration_reference_embeds(item, references, unit_modules) + for pair in value.items() + for item in pair + ) + if isinstance(value, list | tuple): + return any( + _precalibration_reference_embeds(item, references, unit_modules) + for item in value + ) + if not isinstance(value, str): + return False + if any( + reference in value or json.dumps(reference, ensure_ascii=False)[1:-1] in value + for reference in references + ): + return True + try: + decoded = json.loads(value) + except (ValueError, TypeError): + decoder = json.JSONDecoder() + for index, character in enumerate(value): + if character not in {"{", '"'}: + continue + try: + embedded, _ = decoder.raw_decode(value, index) + except ValueError: + continue + if _precalibration_reference_embeds(embedded, references, unit_modules): + return True + return False + return decoded != value and _precalibration_reference_embeds( + decoded, references, unit_modules + ) + + +def _precalibration_reference_guard( + spec: Mapping[str, object], + *, + graph: Graph | None = None, + config: TransportGraphConfig | None = None, +) -> None: + """Refuse hold-out or benefit-unit engine semantics upstream of calibration. + + Without a graph, check the declaration's structural-base membership and + artifact ancestry. This permits a packaged declaration check before donor + preparation. Canonical engine-reference documents are matched to the + benefit-unit bindings' module paths even before references are prepared. + A composed graph also checks its exact compiled predecessors and the + prepared references, including references embedded in JSON text. + Benefit-unit attributes remain allowed; excluded columns are the unit + rules bindings' outputs and the group encoder's declared outputs. + """ + if not compose_module._has_resource(spec, compose_module.GRAPH_RESOURCE): + return + document = compose_module.transport_graph_document(spec) + rows = { + row["id"]: row + for row in ( + compose_module._mapping(item, "transport node") + for item in compose_module._sequence( + document.get("nodes"), "transport nodes" + ) + ) + } + unit_entity = compose_module.transport_resource(spec, "benefit_unit_rule").get( + "entity" + ) + bindings = compose_module.transport_resource(spec, "axiom_rules_bindings").get( + "bindings", () + ) + unit_bindings = { + row["id"]: row + for row in compose_module._sequence(bindings, "axiom_rules_bindings.bindings") + if isinstance(row, Mapping) and row.get("entity") == unit_entity + } + unit_modules = frozenset( + row["rulespec_path"] + for row in unit_bindings.values() + if isinstance(row.get("rulespec_path"), str) + ) + forbidden_columns = { + (unit_entity, variable) + for row in unit_bindings.values() + for variable in row.get("variables", ()) + } + for row in rows.values(): + if row.get("kernel") == "concepts.encode_groups@1": + forbidden_columns.update( + (output["entity"], output["column"]) + for output in row.get("outputs", ()) + if output.get("entity") == unit_entity + ) + references = { + row["engine_ref"] + for row in unit_bindings.values() + if isinstance(row.get("engine_ref"), str) and row["engine_ref"] + } + if config is not None: + references.update( + config.engine_refs[name] + for name in unit_bindings + if name in config.engine_refs + ) + endpoints = compose_module._mapping( + document.get("endpoints"), "transport endpoints" + ) + roots = { + row["id"] + for row in rows.values() + if isinstance(row.get("kernel"), str) and row["kernel"].startswith("calibrate.") + } + if isinstance(endpoints.get("calibration"), str): + roots.add(endpoints["calibration"]) + predecessors = {name: set() for name in rows} + for name, row in rows.items(): + base = row.get("base") + if isinstance(base, str): + predecessors[name].add(base) + predecessors[name].update( + item["id"] for item in rows.values() if item.get("population") == base + ) + elif isinstance(row.get("population"), str): + predecessors[name].add(row["population"]) + predecessors[name].update( + edge["producer"] for edge in row.get("artifact_inputs", ()) + ) + for name in _precalibration_reference_ancestors(predecessors, roots): + row = rows[name] + reason = None + if row.get("rules_binding") in unit_bindings: + reason = "benefit-unit rules binding" + for value in compose_module._mapping( + row.get("params", {}), "node params" + ).values(): + for selection in _precalibration_reference_selections(value): + if selection.get("prepared") == "engine_refs": + reason = "prepared engine_refs map" + if isinstance(selection.get("resource"), str): + resource = compose_module._stem(selection["resource"]) + if resource == "holdout_references": + reason = "holdout_references selection" + if ( + resource == compose_module.GRAPH_RESOURCE + and selection.get("encoding") == "sha256" + ): + reason = "graph resource's own digest" + if selection.get( + "encoding", "value" + ) != "sha256" and compose_module._has_resource(spec, resource): + selected = compose_module.transport_resource(spec, resource) + for component in selection.get("path", ()): + if ( + not isinstance(selected, Mapping) + or component not in selected + ): + selected = None + break + selected = selected[component] + if _precalibration_reference_embeds( + selected, references, unit_modules + ): + reason = "selected data embedding a benefit-unit engine reference" + columns = { + (item["entity"], column) + for item in row.get("inputs", ()) + for column in item["columns"] + } + columns.update( + (item["entity"], item["column"]) for item in row.get("outputs", ()) + ) + if columns & forbidden_columns: + reason = "benefit-unit engine input or output column" + if any( + isinstance(source, str) and "holdout" in source.lower() + for source in row.get("sources", ()) + ): + reason = "hold-out source" + if _precalibration_reference_embeds( + row.get("params", {}), references, unit_modules + ): + reason = "embedded benefit-unit engine reference" + if reason: + raise ValueError( + f"Transport calibration ancestry: node {name!r} carries {reason}." + ) + if graph is None: + return + compiled = compile_graph(graph) + roots.update( + node.id for node in graph.nodes if node.kernel.startswith("calibrate.") + ) + for node in graph.nodes: + if node.kernel == "concepts.encode_groups@1": + forbidden_columns.update( + (item.entity, item.column) + for item in node.outputs + if item.entity == unit_entity + ) + for name in _precalibration_reference_ancestors(compiled.predecessors, roots): + node = graph.node(name) + columns = { + (item.entity, column) for item in node.inputs for column in item.columns + } + columns.update((item.entity, item.column) for item in node.outputs) + if ( + any("holdout" in source.lower() for source in node.sources) + or _precalibration_reference_embeds(node.params, references, unit_modules) + or columns & forbidden_columns + ): + raise ValueError( + f"Transport calibration ancestry: node {name!r} carries hold-out or benefit-unit engine semantics." + ) + + # --------------------------------------------------------------------------- # Shared fixtures (built once; tests that edit inputs copy what they change) # --------------------------------------------------------------------------- @@ -249,6 +516,16 @@ def composed(tmp_path_factory): return make_composed_fixture(tmp_path_factory.mktemp("composed")) +@pytest.fixture(scope="module") +def calibration_ancestry_composed(tmp_path_factory): + """Prepare real engine references without unrelated gate binding identities.""" + from microcosm.build.transport import registry as registry_module + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(registry_module, "register_gate_kernel", lambda *args: None) + return make_composed_fixture(tmp_path_factory.mktemp("calibration-ancestry")) + + @pytest.fixture(scope="module") def feed_compilation_config(composed): """Compose every feed compiler, with an optional explicit gap pin.""" @@ -633,7 +910,10 @@ def test_c2_gate_and_holdout_edits_move_no_calibration_key( assert calibration.isdisjoint(expected) -def test_holdout_and_family_semantics_have_no_calibration_ancestry(composed): +def test_holdout_and_family_semantics_have_no_calibration_ancestry( + calibration_ancestry_composed, +): + composed = calibration_ancestry_composed graph = composed.graph(extended=True) family_ref = composed.registry.engine_refs["family_housing"] family_columns = set(FAMILY_INPUTS) | set(PROGRAMS["family_housing"]["outputs"]) @@ -658,7 +938,11 @@ def test_holdout_and_family_semantics_have_no_calibration_ancestry(composed): "violation", ["prepared", "embedded_ref", "holdout_resource", "holdout_source", "graph_digest"], ) -def test_compose_refuses_contaminated_calibration_ancestry(composed, violation): +@pytest.mark.parametrize("guard", ["reference", "absent", "empty"]) +def test_compose_refuses_contaminated_calibration_ancestry( + calibration_ancestry_composed, monkeypatch, violation, guard +): + composed = calibration_ancestry_composed spec = copy.deepcopy(composed.spec) row = next( row @@ -679,10 +963,578 @@ def test_compose_refuses_contaminated_calibration_ancestry(composed, violation): }, "graph_digest": {"resource": "transport_graph", "encoding": "sha256"}, }[violation] - with pytest.raises(ValueError, match="calibration ancestry|graph resource.*digest"): + with monkeypatch.context() as reference: + reference.setattr( + compose_module, + "validate_transport_calibration_ancestry", + _precalibration_reference_guard, + ) + expected = _precalibration_outcome( + lambda document: compose_transport_graph(document, composed.config), spec + ) + _precalibration_guard_variant(spec, guard, monkeypatch) + with pytest.raises( + ValueError, match="calibration ancestry|graph resource.*digest" + ) as error: + compose_transport_graph(spec, composed.config) + assert (type(error.value), str(error.value)) == expected + + +def _precalibration_stage(spec, binding, columns=None): + row = next( + item + for item in spec["resources"]["axiom_rules_bindings"]["bindings"] + if item["id"] == binding + ) + return { + "id": f"toy.precal.{binding}", + "rules_binding": binding, + "rulespec_path": row["rulespec_path"], + "outputs": [ + { + "entity": row["entity"], + "column": column, + "purpose": "classification" + if column.endswith("eligible") + else "calibration_measure", + } + for column in (row["variables"] if columns is None else columns) + ], + "citation": "Synthetic pre-calibration method card.", + } + + +def _precalibration_spec(composed, binding="family_housing"): + spec = copy.deepcopy(composed.spec) + nodes = spec["resources"]["transport_graph"]["nodes"] + calibration = next(row for row in nodes if row["id"] == CALIBRATION) + if binding == "family_housing": + encoder = next(row for row in nodes if row["id"] == "nz.as.inputs") + encoder["population"] = "nz.open" + nodes.append( + { + "id": "nz.precal.family", + "kernel": RULES, + "population": "nz.open", + "sources": ["rulespec_nz"], + "inputs": [{"entity": "family", "columns": list(FAMILY_INPUTS)}], + "outputs": [ + {"entity": "family", "column": name, "dtype": output["dtype"]} + for name, output in PROGRAMS[binding]["outputs"].items() + ], + "rules_binding": binding, + } + ) + columns = ["toy_family_weekly_amount"] + entity = "family" + else: + columns = list(PROGRAMS[binding]["outputs"]) + entity = "person" + measure = next(column for column in columns if not column.endswith("eligible")) + calibration["inputs"].append({"entity": entity, "columns": [measure]}) + spec["resources"]["precalibration_stages"] = { + "schema_version": 1, + "stages": [_precalibration_stage(spec, binding, columns)], + } + return spec + + +def _precalibration_outcome(guard, spec, **kwargs): + try: + guard(spec, **kwargs) + except (TypeError, ValueError) as error: + return type(error), str(error) + return None + + +def _precalibration_guard_variant(spec, guard, monkeypatch): + if guard == "empty": + spec["resources"]["precalibration_stages"] = {"schema_version": 1, "stages": []} + if guard == "reference": + monkeypatch.setattr( + compose_module, + "validate_transport_calibration_ancestry", + _precalibration_reference_guard, + ) + + +@pytest.mark.parametrize("guard", ["reference", "absent", "empty"]) +def test_precalibration_ancestry_unlisted_person_differential( + calibration_ancestry_composed, monkeypatch, guard +): + composed = calibration_ancestry_composed + spec = copy.deepcopy(composed.spec) + original = graph_to_json(composed.graph()) + _precalibration_guard_variant(spec, guard, monkeypatch) + graph = compose_transport_graph(spec, composed.config) + assert graph_to_json(graph) == original + assert ( + graph.node("nz.rules.main_benefits").params["engine_ref"] + == (composed.config.engine_refs["main_benefit"]) + ) + assert _precalibration_outcome( + compose_module.validate_transport_calibration_ancestry, + spec, + graph=graph, + config=composed.config, + ) == _precalibration_outcome( + _precalibration_reference_guard, spec, graph=graph, config=composed.config + ) + + +def test_precalibration_ancestry_listed_unit_stage_composes( + calibration_ancestry_composed, +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + graph = compose_transport_graph(spec, composed.config) + declared = {("family", "toy_family_weekly_amount")} + actual = { + (item.entity, column) + for item in graph.node(CALIBRATION).inputs + for column in item.columns + if item.entity == "family" + } + assert actual == declared + assert {item.column for item in graph.node("nz.precal.family").outputs} == set( + PROGRAMS["family_housing"]["outputs"] + ) + assert len(declared) < len(graph.node("nz.precal.family").outputs) + + +def test_precalibration_ancestry_listed_person_stage_composes( + calibration_ancestry_composed, +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed, "main_benefit") + graph = compose_transport_graph(spec, composed.config) + assert Slice("person", ("toy_main_payment",)) in graph.node(CALIBRATION).inputs + assert ( + Slice( + "person", + ( + "take_up_seed", + "toy_main_eligible", + "toy_main_payment", + "toy_pension_eligible", + "toy_pension_payment", + ), + ) + in graph.node("nz.takeup.assign").inputs + ) + + +@pytest.mark.parametrize("nested", [False, True]) +def test_precalibration_ancestry_selector_stage_in_both_passes( + calibration_ancestry_composed, nested +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + graph = compose_transport_graph(spec, composed.config) + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == "nz.precal.family" + ) + del row["rules_binding"] + row["kernel"] = "simulate.solve_zero@1" + selector = {"binding": "family_housing", "field": "engine_ref"} + row["params"] = {"engine_ref": [{"nested": selector}] if nested else selector} + compose_module.validate_transport_calibration_ancestry(spec, config=composed.config) + reference = composed.config.engine_refs["family_housing"] + graph = replace( + graph, + nodes=tuple( + replace( + node, + kernel="simulate.solve_zero@1", + params={ + "engine_ref": json.dumps({"nested": [reference]}) + if nested + else reference + }, + ) + if node.id == row["id"] + else node + for node in graph.nodes + ), + ) + compose_module.validate_transport_calibration_ancestry( + spec, graph=graph, config=composed.config + ) + + +def test_precalibration_ancestry_unlisted_unit_stage_refuses( + calibration_ancestry_composed, +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + spec["resources"]["precalibration_stages"]["stages"] = [ + _precalibration_stage(spec, "main_benefit") + ] + with pytest.raises(ValueError, match="not a declared pre-calibration stage"): compose_transport_graph(spec, composed.config) +@pytest.mark.parametrize("contamination", ["holdout_source", "holdout_selection"]) +@pytest.mark.parametrize("compiled", [False, True]) +def test_precalibration_ancestry_holdout_fed_stage_refuses( + calibration_ancestry_composed, contamination, compiled +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + graph = compose_transport_graph(spec, composed.config) + if compiled: + graph = replace( + graph, + nodes=tuple( + replace( + node, + sources=("nz_holdout_facts",) + if contamination == "holdout_source" + else node.sources, + ) + if node.id == "nz.as.inputs" + else node + for node in graph.nodes + ), + ) + if contamination == "holdout_selection": + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == "nz.as.inputs" + ) + # Its declaration lies after calibration, while the compiled + # population makes it a predecessor. Selection provenance belongs + # to the declaration; resolved parameter objects stay literal. + row["population"] = "nz.as" + row["params"]["holdout"] = { + "resource": "holdout_references", + "encoding": "json", + } + else: + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == "nz.as.inputs" + ) + if contamination == "holdout_source": + row["sources"] = ["nz_holdout_facts"] + else: + row["params"]["holdout"] = { + "resource": "holdout_references", + "encoding": "json", + } + reason = "hold-out source|holdout_references selection" + with pytest.raises(ValueError, match=f"node 'nz.as.inputs'.*({reason})"): + compose_module.validate_transport_calibration_ancestry( + spec, graph=graph if compiled else None, config=composed.config + ) + + +@pytest.mark.parametrize("module", [False, True]) +def test_precalibration_ancestry_holdout_output_or_module_refuses( + calibration_ancestry_composed, module +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + holdout = spec["resources"]["holdout_references"]["target_references"][0] + holdout["entity"] = "family" + if module: + holdout["measure"] = "toy_family_holdout" + bindings = spec["resources"]["axiom_rules_bindings"]["bindings"] + extra = copy.deepcopy( + next(row for row in bindings if row["id"] == "family_housing") + ) + extra.update(id="family_evaluation", variables=[holdout["measure"]]) + bindings.append(extra) + else: + holdout["measure"] = "toy_family_weekly_amount" + with pytest.raises(ValueError, match="precalibration_stages.*holdout"): + compose_module.validate_transport_calibration_ancestry(spec) + + +@pytest.mark.parametrize("binding", ["family_housing", "main_benefit"]) +@pytest.mark.parametrize("compiled", [False, True]) +def test_precalibration_ancestry_undeclared_stage_variable_refuses( + calibration_ancestry_composed, binding, compiled +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed, binding) + graph = compose_transport_graph(spec, composed.config) + stage = spec["resources"]["precalibration_stages"]["stages"][0] + if binding == "main_benefit": + stage["outputs"] = [ + row for row in stage["outputs"] if row["column"] != "toy_main_eligible" + ] + entity, variable = "person", "toy_main_eligible" + else: + entity, variable = "family", "toy_family_eligible" + if compiled: + # Keep the declaration clean: only this extension consumes the omitted + # variable, so refusal must come from the compiled predecessor pass. + if binding == "main_benefit": + nodes = spec["resources"]["transport_graph"]["nodes"] + for row in nodes: + if "inputs" in row: + row["inputs"] = [ + { + **item, + "columns": [ + column + for column in item["columns"] + if column != variable + ], + } + for item in row["inputs"] + if any(column != variable for column in item["columns"]) + ] + graph = replace( + graph, + nodes=tuple( + replace( + node, + inputs=tuple( + Slice( + item.entity, + tuple( + column + for column in item.columns + if column != variable + ), + ) + for item in node.inputs + if any(column != variable for column in item.columns) + ), + ) + for node in graph.nodes + ), + ) + consumer = Node( + "nz.precal.extension.reader", + "transport.probe@1", + population="nz.open", + inputs=(Slice(entity, (variable,)),), + ) + graph = replace(graph, nodes=(*graph.nodes, consumer)) + name = consumer.id + else: + name = "nz.takeup.assign" if binding == "main_benefit" else "nz.targets.problem" + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == name + ) + if binding == "family_housing": + row["inputs"].append({"entity": entity, "columns": [variable]}) + with pytest.raises( + ValueError, + match=f"node '{name}' carries undeclared pre-calibration stage output", + ): + compose_module.validate_transport_calibration_ancestry( + spec, graph=graph if compiled else None, config=composed.config + ) + + +@pytest.mark.parametrize("compiled", [False, True]) +def test_precalibration_ancestry_shared_encode_output_refuses( + calibration_ancestry_composed, compiled +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + graph = compose_transport_graph(spec, composed.config) + if compiled: + reader = Node( + "nz.precal.extension.reader", + "transport.probe@1", + population="nz.open", + inputs=(Slice("family", ("input_rent",)),), + ) + graph = replace(graph, nodes=(*graph.nodes, reader)) + name = reader.id + else: + name = "nz.targets.problem" + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == name + ) + row["inputs"].append({"entity": "family", "columns": ["input_rent"]}) + with pytest.raises(ValueError, match=f"node '{name}' carries"): + compose_module.validate_transport_calibration_ancestry( + spec, graph=graph if compiled else None, config=composed.config + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "resource_key", + "stage_key", + "output_key", + "duplicate_id", + "empty_id", + "unsafe_id", + "missing_binding", + "path_mismatch", + "duplicate_binding", + "schema_bool", + "schema_float", + "schema_version", + "purpose", + "variable", + "wrong_entity", + "empty_outputs", + "duplicate_output", + "stages_type", + ], +) +def test_precalibration_ancestry_stage_schema_refuses( + calibration_ancestry_composed, mutation +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + resource = spec["resources"]["precalibration_stages"] + stage = resource["stages"][0] + output = stage["outputs"][0] + if mutation == "resource_key": + resource["unknown"] = True + elif mutation == "stage_key": + stage["unknown"] = True + elif mutation == "output_key": + output["unknown"] = True + elif mutation in {"duplicate_id", "duplicate_binding"}: + other = copy.deepcopy(stage) + if mutation == "duplicate_binding": + other["id"] += ".again" + resource["stages"].append(other) + elif mutation in {"empty_id", "unsafe_id"}: + stage["id"] = "" if mutation == "empty_id" else "../unsafe" + elif mutation == "missing_binding": + stage["rules_binding"] = "absent" + elif mutation == "path_mismatch": + stage["rulespec_path"] = "toy/other.yaml" + elif mutation.startswith("schema_"): + resource["schema_version"] = { + "schema_bool": True, + "schema_float": 1.0, + "schema_version": 2, + }[mutation] + elif mutation == "purpose": + output["purpose"] = "evaluation" + elif mutation == "variable": + output["column"] = "not_a_binding_variable" + elif mutation == "wrong_entity": + output["entity"] = "person" + elif mutation == "empty_outputs": + stage["outputs"] = [] + elif mutation == "duplicate_output": + stage["outputs"].append(dict(output)) + else: + resource["stages"] = {} + with pytest.raises((TypeError, ValueError), match="precalibration_stages"): + compose_module.validate_transport_calibration_ancestry(spec) + + +@pytest.mark.parametrize("compiled", [False, True]) +def test_precalibration_ancestry_unlisted_selector_refuses( + calibration_ancestry_composed, compiled +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed, "main_benefit") + graph = compose_transport_graph(spec, composed.config) + name = "nz.precal.unlisted.solve" + if compiled: + graph = replace( + graph, + nodes=( + *graph.nodes, + Node( + name, + "simulate.solve_zero@1", + population="nz.open", + params={ + "engine_ref": composed.config.engine_refs["family_housing"] + }, + ), + ), + ) + else: + spec["resources"]["transport_graph"]["nodes"].append( + { + "id": name, + "kernel": "simulate.solve_zero@1", + "population": "nz.open", + "params": { + "engine_ref": {"binding": "family_housing", "field": "engine_ref"} + }, + } + ) + with pytest.raises( + ValueError, match=f"node '{name}' carries not a declared pre-calibration stage" + ): + compose_module.validate_transport_calibration_ancestry( + spec, graph=graph if compiled else None, config=composed.config + ) + + +@pytest.mark.parametrize("compiled", [False, True]) +def test_precalibration_ancestry_mixed_unit_bindings_refuses( + calibration_ancestry_composed, compiled +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + bindings = spec["resources"]["axiom_rules_bindings"]["bindings"] + other = copy.deepcopy( + next(row for row in bindings if row["id"] == "family_housing") + ) + other.update( + id="other_family", + rulespec_path="toy/other_family.yaml", + variables=["other_result"], + ) + bindings.append(other) + config = replace( + composed.config, + engine_refs={ + **composed.config.engine_refs, + "other_family": "synthetic-other-family-engine-ref", + }, + ) + graph = compose_transport_graph(spec, config) + name = "nz.precal.family" + if compiled: + graph = replace( + graph, + nodes=tuple( + replace( + node, + params={**node.params, "other": config.engine_refs["other_family"]}, + ) + if node.id == name + else node + for node in graph.nodes + ), + ) + else: + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == name + ) + row["params"] = { + "components": [{"binding": "other_family", "field": "engine_ref"}] + } + with pytest.raises( + ValueError, + match=f"node '{name}' carries mixes listed and unlisted unit bindings", + ): + compose_module.validate_transport_calibration_ancestry( + spec, graph=graph if compiled else None, config=config + ) + + @pytest.mark.parametrize( "literal", [ @@ -1195,7 +2047,10 @@ def test_cli_main_refuses_the_packaged_spec_and_writes_nothing(tmp_path, capsys) assert not out.exists() -def test_packaged_holdout_references_are_disjoint_from_calibration_references(): +@pytest.mark.parametrize("guard", ["reference", "absent", "empty"]) +def test_packaged_holdout_references_are_disjoint_from_calibration_references( + monkeypatch, guard +): spec = load_transport_spec("nz") def names(resource): @@ -1204,6 +2059,8 @@ def names(resource): calibration = names("target_references") | names("precal_references") assert names("holdout_references").isdisjoint(calibration) + _precalibration_guard_variant(spec, guard, monkeypatch) + assert _precalibration_outcome(_precalibration_reference_guard, spec) is None compose_module.validate_transport_calibration_ancestry(spec) @@ -1924,3 +2781,471 @@ def unavailable(*args, **kwargs): "transport build refused: rules engine unavailable" in capsys.readouterr().err ) assert not (out / EXPORT_FILENAME).exists() + + +@st.composite +def _precalibration_ancestry_dags(draw): + """Small DAGs with independent binding declarations and column edges.""" + kinds = draw( + st.lists( + st.sampled_from(("encode", "unit", "person", "selector", "holdout")), + min_size=1, + max_size=7, + ) + ) + nodes = [] + bindings = {} + for index, kind in enumerate(kinds): + binding = None + entity = "family" if kind in {"encode", "unit"} else "person" + if kind in {"unit", "person", "selector"}: + if kind == "selector": + entity = draw(st.sampled_from(("person", "family"))) + available = [ + name for name, row in bindings.items() if row["entity"] == entity + ] + if kind == "selector" and available and draw(st.booleans()): + binding = draw(st.sampled_from(available)) + else: + binding = f"binding_{index}" + bindings[binding] = {"entity": entity, "variables": []} + outputs = [f"value_{index}"] + if binding is not None: + outputs.append(f"private_{index}") + bindings[binding]["variables"].extend(outputs) + reads = [] + dependencies = [] + for predecessor, earlier in enumerate(nodes): + edge = draw(st.integers(min_value=0, max_value=3)) + if edge: + dependencies.append(predecessor) + if edge > 1: + columns = earlier["outputs"] if edge == 3 else earlier["outputs"][:1] + reads.extend((predecessor, column) for column in columns) + nodes.append( + { + "kind": kind, + "binding": binding, + "entity": entity, + "outputs": outputs, + "reads": reads, + "dependencies": dependencies, + "holdout_selection": kind == "holdout" and draw(st.booleans()), + } + ) + declared = {} + for binding, row in bindings.items(): + if draw(st.booleans()): + declared[binding] = draw( + st.sets(st.sampled_from(row["variables"]), min_size=1) + ) + dependencies = [] + reads = [] + for predecessor, earlier in enumerate(nodes): + edge = draw(st.integers(min_value=0, max_value=3)) + if edge: + dependencies.append(predecessor) + if edge > 1: + columns = earlier["outputs"] if edge == 3 else earlier["outputs"][:1] + reads.extend((predecessor, column) for column in columns) + nodes.append( + { + "kind": "calibrate", + "binding": None, + "entity": "person", + "outputs": [], + "reads": reads, + "dependencies": dependencies, + "holdout_selection": False, + } + ) + return nodes, bindings, declared + + +def _precalibration_reference_accepts(model): + """The stage rules, expressed on generated kinds and column-use edges.""" + nodes, _, declared = model + relevant = {len(nodes) - 1} + relevant.update( + index for index, node in enumerate(nodes) if node["binding"] in declared + ) + for index in range(len(nodes) - 1, -1, -1): + if index in relevant: + relevant.update(nodes[index]["dependencies"]) + for index in relevant: + node = nodes[index] + binding = node["binding"] + if node["kind"] == "holdout": + return False + if ( + binding is not None + and node["entity"] == "family" + and binding not in declared + ): + return False + if node["kind"] == "encode": + consumers = [ + nodes[consumer] + for consumer in relevant + if any(producer == index for producer, _ in nodes[consumer]["reads"]) + ] + if not consumers or any( + consumer["binding"] not in declared or consumer["entity"] != "family" + for consumer in consumers + ): + return False + for producer, column in node["reads"]: + source_binding = nodes[producer]["binding"] + if ( + source_binding in declared + and source_binding != binding + and column not in declared[source_binding] + ): + return False + return True + + +def _precalibration_synthetic_graph(model): + """Render the same model as declarations and compiled extension nodes.""" + nodes, bindings, declared = model + references = {name: f"synthetic-engine:{name}:end" for name in bindings} + seed = Owned("person", "seed", "float64") + trace = ArtifactType("synthetic.precalibration.trace", 1) + graph_nodes = [ + Node( + "p1.create", + "synthetic.create@1", + structural=StructuralDelta.CREATE, + outputs=(seed,), + sources=("synthetic_donor",), + ) + ] + rows = [ + { + "id": "p1.create", + "kernel": "synthetic.create@1", + "outputs": [{"entity": "person", "column": "seed"}], + } + ] + for index, item in enumerate(nodes): + name = "p1.calibrate" if item["kind"] == "calibrate" else f"p1.node_{index}" + kernel = { + "encode": "concepts.encode_groups@1", + "unit": RULES, + "person": RULES, + "selector": "simulate.solve_zero@1", + "holdout": "synthetic.holdout@1", + "calibrate": "calibrate.synthetic@1", + }[item["kind"]] + inputs = {} + for producer, column in item["reads"]: + inputs.setdefault(nodes[producer]["entity"], []).append(column) + binding = item["binding"] + params = {} + prepared_params = {} + if binding is not None: + prepared_params["engine_ref"] = references[binding] + if item["kind"] == "selector": + params["engine_ref"] = {"binding": binding, "field": "engine_ref"} + sources = () + if item["kind"] == "holdout": + if item["holdout_selection"]: + params["selection"] = {"resource": "holdout_references"} + # Resource selectors disappear during parameter preparation. + prepared_params["selection"] = "{}" + else: + sources = ("synthetic_holdout_facts",) + edges = tuple( + ArtifactInput(f"edge_{producer}", f"p1.node_{producer}", "trace", trace) + for producer in item["dependencies"] + ) + graph_nodes.append( + Node( + name, + kernel, + population="p1.create", + inputs=tuple( + Slice(entity, tuple(columns)) for entity, columns in inputs.items() + ), + outputs=tuple( + Owned(item["entity"], column, "float64") + for column in item["outputs"] + ), + params=prepared_params, + sources=sources, + artifact_inputs=edges, + artifact_outputs=(ArtifactOutput("trace", trace),), + ) + ) + row = { + "id": name, + "kernel": kernel, + "population": "p1.create", + "inputs": [ + {"entity": entity, "columns": columns} + for entity, columns in inputs.items() + ], + "outputs": [ + {"entity": item["entity"], "column": column} + for column in item["outputs"] + ], + "params": params, + "sources": list(sources), + "artifact_inputs": [ + {"producer": f"p1.node_{producer}"} for producer in item["dependencies"] + ], + } + if binding is not None and item["kind"] != "selector": + row["rules_binding"] = binding + rows.append(row) + spec = { + "country": "synthetic", + "resources": { + "benefit_unit_rule": {"entity": "family"}, + "axiom_rules_bindings": { + "bindings": [ + {"id": name, "rulespec_path": f"synthetic/{name}.yaml", **row} + for name, row in bindings.items() + ] + }, + "holdout_references": {"target_references": []}, + "precalibration_stages": { + "schema_version": 1, + "stages": [ + { + "id": f"stage.{name}", + "rules_binding": name, + "rulespec_path": f"synthetic/{name}.yaml", + "outputs": [ + { + "entity": bindings[name]["entity"], + "column": column, + "purpose": "calibration_measure", + } + for column in bindings[name]["variables"] + if column in columns + ], + "citation": "Synthetic stage contract.", + } + for name, columns in declared.items() + ], + }, + "transport_graph": { + "schema_version": 1, + "country": "synthetic", + "nodes": rows, + "endpoints": {"calibration": "p1.calibrate"}, + }, + }, + } + graph = Graph( + "synthetic", + ( + SourceRef("synthetic_donor", "synthetic@1"), + SourceRef("synthetic_holdout_facts", "synthetic@1"), + ), + tuple(graph_nodes), + ) + config = TransportGraphConfig(engine_refs=references, create_outputs=(seed,)) + return spec, graph, config + + +@settings(max_examples=2000, deadline=None, database=None) +@given(model=_precalibration_ancestry_dags()) +def test_precalibration_stages_calibration_ancestry_property(model): + expected = _precalibration_reference_accepts(model) + spec, graph, config = _precalibration_synthetic_graph(model) + compile_graph(graph) + extension_spec = copy.deepcopy(spec) + extension_spec["resources"]["transport_graph"]["nodes"] = [ + spec["resources"]["transport_graph"]["nodes"][0], + { + "id": "p1.calibrate", + "kernel": "calibrate.synthetic@1", + "population": "p1.create", + }, + ] + # A selected hold-out resource leaves no marker in compiled params. Keep + # those source declarations so this path still checks their provenance. + extension_spec["resources"]["transport_graph"]["nodes"].extend( + row + for row, node in zip( + spec["resources"]["transport_graph"]["nodes"][1:], model[0], strict=True + ) + if node["holdout_selection"] + ) + if not model[2]: + extension_spec = spec + for label, document, compiled in ( + ("declaration", spec, None), + ("compiled extensions", extension_spec, graph), + ): + try: + compose_module.validate_transport_calibration_ancestry( + document, graph=compiled, config=config + ) + except ValueError: + actual = False + else: + actual = True + assert actual == expected, (label, model) + + +@pytest.mark.parametrize("shared_module", [False, True]) +def test_precalibration_ancestry_stage_own_selected_reference_is_allowed( + calibration_ancestry_composed, shared_module +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + reference = composed.config.engine_refs["family_housing"] + config = composed.config + if shared_module: + bindings = spec["resources"]["axiom_rules_bindings"]["bindings"] + other = copy.deepcopy( + next(item for item in bindings if item["id"] == "family_housing") + ) + other.update(id="other_family", variables=["other_result"]) + bindings.append(other) + other_reference = json.loads(reference) + other_reference["binding"].update(id="other_family", variables=["other_result"]) + config = replace( + config, + engine_refs={ + **config.engine_refs, + "other_family": canonical_json(other_reference).decode(), + }, + ) + graph = compose_transport_graph(spec, config) + spec["resources"]["stage_engine_data"] = {"nested": [reference]} + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == "nz.precal.family" + ) + row["params"] = {"context": {"resource": "stage_engine_data", "encoding": "json"}} + compose_module.validate_transport_calibration_ancestry(spec, config=config) + compose_module.validate_transport_calibration_ancestry( + spec, graph=graph, config=config + ) + + +@pytest.mark.parametrize( + "literal", + [ + {"resource": "holdout_references", "encoding": "sha256"}, + {"prepared": "engine_refs"}, + {"binding": "family_housing", "field": "engine_ref"}, + ], +) +def test_precalibration_ancestry_person_stage_preserves_selected_literal_objects( + calibration_ancestry_composed, literal +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed, "main_benefit") + spec["resources"]["calibration"]["literal"] = {"items": [[literal]]} + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == "nz.targets.problem" + ) + row["params"]["literal"] = { + "resource": "calibration", + "path": ["literal"], + "encoding": "json", + } + graph = compose_transport_graph(spec, composed.config) + assert json.loads(graph.node(row["id"]).params["literal"]) == {"items": [[literal]]} + + +@pytest.mark.parametrize("first", ["nz.targets.problem", "nz.as.inputs"]) +def test_precalibration_ancestry_refusal_follows_declaration_order( + calibration_ancestry_composed, first +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + document = spec["resources"]["transport_graph"] + for row in document["nodes"]: + if row["id"] in {"nz.targets.problem", "nz.as.inputs"}: + row["sources"] = ["nz_holdout_facts"] + document["nodes"].sort(key=lambda row: row["id"] != first) + outcomes = [ + _precalibration_outcome( + compose_module.validate_transport_calibration_ancestry, + spec, + config=composed.config, + ) + for _ in range(3) + ] + assert outcomes == [ + ( + ValueError, + f"Transport calibration ancestry: node {first!r} carries hold-out source.", + ) + ] * len(outcomes) + + +def test_precalibration_ancestry_schema_checked_before_graph( + calibration_ancestry_composed, +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + del spec["resources"]["transport_graph"] + spec["resources"]["precalibration_stages"]["stages"][0]["outputs"][0]["purpose"] = ( + "evaluation" + ) + with pytest.raises(ValueError, match="precalibration_stages"): + compose_module.validate_transport_calibration_ancestry(spec) + + +def test_precalibration_ancestry_postcalibration_stage_checks_sibling_producer( + calibration_ancestry_composed, +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + nodes = spec["resources"]["transport_graph"]["nodes"] + for row in nodes: + if row["id"] in {"nz.as.inputs", "nz.precal.family"}: + row["population"] = "nz.as" + if row["id"] == "nz.as.inputs": + row["sources"] = ["nz_holdout_facts"] + if row["id"] == CALIBRATION: + row["inputs"] = [ + item for item in row["inputs"] if item["entity"] != "family" + ] + with pytest.raises(ValueError, match="node 'nz.as.inputs' carries hold-out source"): + compose_module.validate_transport_calibration_ancestry( + spec, config=composed.config + ) + + +@pytest.mark.parametrize("name", ["nz.targets.problem", "nz.precal.family"]) +@pytest.mark.parametrize("encoding", ["string", "tuple"]) +def test_precalibration_ancestry_compiled_prepared_map_still_refuses( + calibration_ancestry_composed, name, encoding +): + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + graph = compose_transport_graph(spec, composed.config) + prepared = canonical_json( + {"engine_refs": dict(sorted(composed.config.engine_refs.items()))} + ).decode() + if encoding == "tuple": + prepared = (prepared,) + graph = replace( + graph, + nodes=tuple( + replace(node, params={**node.params, "inventory": prepared}) + if node.id == name + else node + for node in graph.nodes + ), + ) + reason = ( + "prepared engine_refs map" + if name == "nz.precal.family" + else "not a declared pre-calibration stage" + ) + with pytest.raises(ValueError, match=f"node '{name}' carries {reason}"): + compose_module.validate_transport_calibration_ancestry( + spec, graph=graph, config=composed.config + ) From 7b933e9fb223603b0f5ee51452223d212f0bf2a9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 14:27:52 -0400 Subject: [PATCH 02/20] Check only stage nodes upstream of calibration The build checked the ancestry of every node of a listed stage, including nodes that run after calibration. Extension checkpoints run after calibration and may sit beside hold-out data or unit rules, so listing a binding that also runs there would have refused a sound graph. A stage node upstream of calibration already brings its ancestors into the calibration closure, so that closure alone carries rule 4. The property test now compares the guard with the frozen guard without stages (equal with no stages; person stages only restrict) and checks the corrected monotonicity: dropping a person stage never refuses what it admitted, and dropping a unit stage never admits what it refused. Both new example tests and the property fail against the old roots. Documents the resource in docs/transport-precalibration-stages.md. Co-Authored-By: Claude Opus 5.5 --- .../transport-precalibration-stages.added.md | 2 +- docs/agent-guide.md | 3 + docs/transport-precalibration-stages.md | 86 +++++++++++ .../src/microcosm/build/transport/compose.py | 9 +- .../shared/test_transport_composed_graph.py | 139 +++++++++++++++--- 5 files changed, 212 insertions(+), 27 deletions(-) create mode 100644 docs/transport-precalibration-stages.md diff --git a/changelog.d/transport-precalibration-stages.added.md b/changelog.d/transport-precalibration-stages.added.md index b29800e40c..18b641b670 100644 --- a/changelog.d/transport-precalibration-stages.added.md +++ b/changelog.d/transport-precalibration-stages.added.md @@ -1 +1 @@ -Allow transport specs to declare pre-calibration rules stages with explicit outputs, recursive ancestry checks and holdout isolation for person and unit bindings. +Allow transport specs to declare pre-calibration rules stages with explicit outputs, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes upstream of calibration are checked, so a stage binding's nodes after calibration are unaffected. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index 455aad6cbb..0ebed720a8 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -520,6 +520,9 @@ An optional `entitlement_graph.json` legacy resource installs the entitlement extension in both composition and the local driver. Its scenario templates, bridge parameters, comparison references and receipt connections are described in [the entitlement declaration contract](transport-entitlement.md). +An optional `precalibration_stages.json` legacy resource lists the rules +bindings whose declared outputs may feed calibration; see +[pre-calibration stages](transport-precalibration-stages.md). The zero solver requires independent, non-increasing row residuals and verifies the returned answers after its fixed iterations. Its receipt counts the final verification. The local differential driver validates call and output identities diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md new file mode 100644 index 0000000000..d1fd2a8578 --- /dev/null +++ b/docs/transport-precalibration-stages.md @@ -0,0 +1,86 @@ +# Pre-calibration stages + +A transport graph's calibration may read only what the calibration ancestry +guard admits (`validate_transport_calibration_ancestry` in +`microcosm.build.transport.compose`). Without stages, the guard refuses every +benefit-unit engine semantic upstream of a `calibrate.*` node or the +calibration endpoint, and every hold-out source, hold-out selection and the +graph resource's own digest. Person-entity rules bindings are admitted. + +The optional `precalibration_stages` resource declares the rules bindings whose +outputs feed calibration, and exactly which of their outputs flow on: + +```json +{ + "schema_version": 1, + "stages": [ + { + "id": "nz.precal.main_benefit_rates", + "rules_binding": "main_benefits_rates", + "rulespec_path": "", + "outputs": [ + {"entity": "person", "column": "", "purpose": "calibration_measure"} + ], + "citation": "" + } + ] +} +``` + +## Validation + +Key sets are closed at every level, and `schema_version` is the integer 1. +Stage ids are distinct, safe node ids. Each stage names an existing binding, +repeats its `rulespec_path` exactly, and no binding appears in two stages. +Outputs are nonempty and distinct; each is on the binding's entity, names one +of its `variables`, and has a `purpose` of `calibration_measure` or +`classification` (a judgment another pre-calibration node reads). An output may +not be a hold-out `(entity, measure)`, and a stage's module may not produce a +hold-out measure. The resource is checked before either graph pass. + +## Which nodes belong to a stage + +In the declaration pass, a node belongs to a stage when its `rules_binding` is +the stage's binding or any `{"binding": id}` selector in its parameters names +it. In the composed pass, a node belongs to a stage when its parameters carry +the binding's prepared engine reference. A node that names a listed and an +unlisted benefit-unit binding refuses. A parameter carrying the whole prepared +`engine_refs` inventory is bookkeeping, not a stage invocation, and still +refuses upstream of calibration. + +## Rules + +Only nodes upstream of calibration are checked. A stage binding's nodes after +calibration, such as extension checkpoints, are outputs and are unaffected. + +- Absent, or `"stages": []`, is the guard without stages, outcomes and messages + alike. +- A listed benefit-unit stage admits its own nodes, their engine reference and + outputs, and group-encoder outputs read upstream only by listed benefit-unit + stage nodes. Any other benefit-unit semantic upstream refuses as + "not a declared pre-calibration stage". +- A listed person stage admits nothing new; it only adds the next rule. +- A node outside a listed stage that reads one of the stage binding's variables + not in the stage's `outputs` refuses as "undeclared pre-calibration stage + output". Variables a stage's own nodes read do not count. +- The hold-out, selection, digest and prepared-map checks still apply to every + upstream node, stage nodes included, so each stage's ancestry is clean. + +Both passes apply these rules; the composed pass covers compiled predecessors +and extension nodes. No node parameter or hash changes. + +## Invariants + +These are tested, as properties and examples, in +`tests/engine_free/shared/test_transport_composed_graph.py`. + +- With no stages, the guard accepts and refuses exactly as the frozen copy of + the guard without stages does. +- Person stages only restrict: a graph accepted with only person stages listed + is accepted without them. +- Dropping a person stage never refuses a graph its listing admitted; dropping + a benefit-unit stage never admits a graph its listing refused. +- No hold-out source or selection is upstream of calibration, with or without + stages, and no stage output or stage module carries a hold-out measure. +- With stages listed, a refusal names the first offending node in declaration + order. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 8c0f710ed5..a94b608c32 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -893,10 +893,11 @@ def check( } if row.get("rules_binding") in bindings: node_bindings[name].add(row["rules_binding"]) - stage_nodes = { - name for name in rows if node_bindings[name].intersection(stages) - } - ancestors = _ancestor_closure(predecessors, roots | stage_nodes) + # Only nodes upstream of calibration are pre-calibration. A stage node + # there brings its own ancestors into this closure, so each listed + # stage's ancestry passes the same checks; a stage binding's nodes + # after calibration (extension checkpoints) are outputs, not inputs. + ancestors = _ancestor_closure(predecessors, roots) inputs = { name: { (item["entity"], column) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 7123676985..6aa35fd970 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -2863,12 +2863,9 @@ def _precalibration_ancestry_dags(draw): def _precalibration_reference_accepts(model): - """The stage rules, expressed on generated kinds and column-use edges.""" + """The stage rules on generated kinds and column-use edges, upstream of calibration.""" nodes, _, declared = model relevant = {len(nodes) - 1} - relevant.update( - index for index, node in enumerate(nodes) if node["binding"] in declared - ) for index in range(len(nodes) - 1, -1, -1): if index in relevant: relevant.update(nodes[index]["dependencies"]) @@ -3050,10 +3047,8 @@ def _precalibration_synthetic_graph(model): return spec, graph, config -@settings(max_examples=2000, deadline=None, database=None) -@given(model=_precalibration_ancestry_dags()) -def test_precalibration_stages_calibration_ancestry_property(model): - expected = _precalibration_reference_accepts(model) +def _precalibration_property_outcomes(model): + """Acceptance by the guard and by today's guard, in both passes.""" spec, graph, config = _precalibration_synthetic_graph(model) compile_graph(graph) extension_spec = copy.deepcopy(spec) @@ -3076,19 +3071,68 @@ def test_precalibration_stages_calibration_ancestry_property(model): ) if not model[2]: extension_spec = spec + outcomes = {} for label, document, compiled in ( ("declaration", spec, None), ("compiled extensions", extension_spec, graph), ): - try: - compose_module.validate_transport_calibration_ancestry( - document, graph=compiled, config=config + for guard_name, guard in ( + ("guard", compose_module.validate_transport_calibration_ancestry), + ("today", _precalibration_reference_guard), + ): + try: + guard(document, graph=compiled, config=config) + except ValueError: + outcomes[label, guard_name] = False + else: + outcomes[label, guard_name] = True + return outcomes + + +@settings(max_examples=2000, deadline=None, database=None) +@given(model=_precalibration_ancestry_dags(), data=st.data()) +def test_precalibration_stages_calibration_ancestry_property(model, data): + nodes, bindings, declared = model + expected = _precalibration_reference_accepts(model) + outcomes = _precalibration_property_outcomes(model) + labels = ("declaration", "compiled extensions") + for label in labels: + actual = outcomes[label, "guard"] + assert actual == expected, (label, model) + if not declared: + # I1 with no stages: exactly today's guard. + assert actual == outcomes[label, "today"], (label, model) + elif all(bindings[name]["entity"] == "person" for name in declared): + # I1: person stages only restrict today's guard. + assert outcomes[label, "today"] or not actual, (label, model) + if not declared: + return + dropped = data.draw(st.sampled_from(sorted(declared))) + smaller = ( + nodes, + bindings, + {name: columns for name, columns in declared.items() if name != dropped}, + ) + reduced = _precalibration_property_outcomes(smaller) + for label in labels: + assert reduced[label, "guard"] == _precalibration_reference_accepts(smaller), ( + label, + smaller, + ) + if bindings[dropped]["entity"] == "person": + # I2: dropping a person stage never refuses what listing it admitted. + assert reduced[label, "guard"] or not outcomes[label, "guard"], ( + label, + model, + dropped, ) - except ValueError: - actual = False else: - actual = True - assert actual == expected, (label, model) + # I2: dropping a unit stage never admits what listing it refused. + assert outcomes[label, "guard"] or not reduced[label, "guard"], ( + label, + model, + dropped, + ) @pytest.mark.parametrize("shared_module", [False, True]) @@ -3197,22 +3241,73 @@ def test_precalibration_ancestry_schema_checked_before_graph( compose_module.validate_transport_calibration_ancestry(spec) -def test_precalibration_ancestry_postcalibration_stage_checks_sibling_producer( - calibration_ancestry_composed, +@pytest.mark.parametrize("compiled", [False, True]) +def test_precalibration_ancestry_postcalibration_stage_nodes_are_not_checked( + calibration_ancestry_composed, compiled ): + """A stage node after calibration is an output; its hold-out sibling is allowed.""" composed = calibration_ancestry_composed spec = _precalibration_spec(composed) - nodes = spec["resources"]["transport_graph"]["nodes"] - for row in nodes: + for row in spec["resources"]["transport_graph"]["nodes"]: if row["id"] in {"nz.as.inputs", "nz.precal.family"}: row["population"] = "nz.as" - if row["id"] == "nz.as.inputs": - row["sources"] = ["nz_holdout_facts"] if row["id"] == CALIBRATION: row["inputs"] = [ item for item in row["inputs"] if item["entity"] != "family" ] - with pytest.raises(ValueError, match="node 'nz.as.inputs' carries hold-out source"): + graph = compose_transport_graph(spec, composed.config) + if compiled: + graph = replace( + graph, + nodes=tuple( + replace(node, sources=("nz_holdout_facts",)) + if node.id == "nz.as.inputs" + else node + for node in graph.nodes + ), + ) + else: + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == "nz.as.inputs" + ) + row["sources"] = ["nz_holdout_facts"] + compose_module.validate_transport_calibration_ancestry( + spec, graph=graph if compiled else None, config=composed.config + ) + + +def test_precalibration_ancestry_stage_binding_after_calibration_is_not_checked( + calibration_ancestry_composed, +): + """A binding listed before calibration may also run after it, beside hold-out data.""" + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + nodes = spec["resources"]["transport_graph"]["nodes"] + holdout = { + "id": "nz.postcal.holdout", + "kernel": "transport.probe@1", + "population": "nz.as", + "sources": ["nz_holdout_facts"], + "params": {"holdout": {"resource": "holdout_references", "encoding": "json"}}, + } + nodes.extend( + [ + holdout, + { + "id": "nz.postcal.family", + "kernel": RULES, + "population": "nz.as", + "rules_binding": "family_housing", + "artifact_inputs": [{"producer": holdout["id"]}], + }, + ] + ) + compose_module.validate_transport_calibration_ancestry(spec, config=composed.config) + # The same node before calibration refuses: the check is not vacuous. + holdout["population"] = "nz.open" + with pytest.raises(ValueError, match="node 'nz.postcal.holdout' carries"): compose_module.validate_transport_calibration_ancestry( spec, config=composed.config ) From 062d89cda5f5ef28921d0c11cc80ce5158305009 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 17:37:54 -0400 Subject: [PATCH 03/20] Attribute stage outputs to their producers and declare derived outputs Fixes the four findings of the GPT-6.1 Sol review of 7b933e9fb: - Recognise the prepared engine_refs inventory by decoded structure, so any JSON spelling (spaced, nested, embedded) refuses as "prepared engine_refs map". - Restrict rule 3 to columns a stage's own nodes produce upstream of calibration, so another binding that declares the same variable name is unaffected. - Cover hold-out sources on stage nodes in the property, with a rules-only domain that exercises producer attribution in every run. - Compare benefit-unit entities without hashing, so absent or empty stages give today's outcome for malformed entities too. Adds optional derived_outputs (NZ's option b) for columns a stage node computes that are not binding variables, such as a solve_zero gross-up. Each must be produced by a stage node upstream of calibration. Co-Authored-By: Claude Opus 5.5 --- .../transport-precalibration-stages.added.md | 2 +- docs/transport-precalibration-stages.md | 37 +- .../src/microcosm/build/transport/compose.py | 182 ++++-- .../shared/test_transport_composed_graph.py | 524 ++++++++++++++++-- 4 files changed, 660 insertions(+), 85 deletions(-) diff --git a/changelog.d/transport-precalibration-stages.added.md b/changelog.d/transport-precalibration-stages.added.md index 18b641b670..539894dc25 100644 --- a/changelog.d/transport-precalibration-stages.added.md +++ b/changelog.d/transport-precalibration-stages.added.md @@ -1 +1 @@ -Allow transport specs to declare pre-calibration rules stages with explicit outputs, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes upstream of calibration are checked, so a stage binding's nodes after calibration are unaffected. +Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes upstream of calibration are checked, so a stage binding's nodes after calibration are unaffected. diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index d1fd2a8578..430c89268a 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -21,6 +21,9 @@ outputs feed calibration, and exactly which of their outputs flow on: "outputs": [ {"entity": "person", "column": "", "purpose": "calibration_measure"} ], + "derived_outputs": [ + {"entity": "person", "column": "", "purpose": "calibration_measure"} + ], "citation": "" } ] @@ -38,6 +41,15 @@ of its `variables`, and has a `purpose` of `calibration_measure` or not be a hold-out `(entity, measure)`, and a stage's module may not produce a hold-out measure. The resource is checked before either graph pass. +`derived_outputs` is optional. It lists columns that a stage node computes +from the binding but that are not binding variables, such as a gross-up +`simulate.solve_zero@1` node's own output. Each row has the same closed shape +and purposes as an `outputs` row. It names any entity, may not repeat a binding +variable on the binding's entity (that belongs in `outputs`), and may not +repeat any other output of the stage or a hold-out measure. Each derived output +must be produced by one of the stage's nodes upstream of calibration. The +composed graph is checked when there is one, otherwise the declaration. + ## Which nodes belong to a stage In the declaration pass, a node belongs to a stage when its `rules_binding` is @@ -60,11 +72,18 @@ calibration, such as extension checkpoints, are outputs and are unaffected. stage nodes. Any other benefit-unit semantic upstream refuses as "not a declared pre-calibration stage". - A listed person stage admits nothing new; it only adds the next rule. -- A node outside a listed stage that reads one of the stage binding's variables - not in the stage's `outputs` refuses as "undeclared pre-calibration stage - output". Variables a stage's own nodes read do not count. -- The hold-out, selection, digest and prepared-map checks still apply to every - upstream node, stage nodes included, so each stage's ancestry is clean. +- A node outside a listed stage that reads a column one of the stage's own + nodes produces upstream of calibration, other than its `outputs` and + `derived_outputs`, refuses as "undeclared pre-calibration stage output". + Reads among a stage's own nodes do not count. The rule follows the producing + node, not the column name, so another binding that declares the same variable + name is unaffected. +- A parameter that carries the prepared `engine_refs` inventory refuses as + "prepared engine_refs map", on stage nodes too. The inventory may be a + mapping, a JSON string in any spelling, nested in lists or mappings, or JSON + embedded in a longer string. It is bookkeeping, not a stage invocation. +- The hold-out, selection and digest checks still apply to every upstream node, + stage nodes included, so each stage's ancestry is clean. Both passes apply these rules; the composed pass covers compiled predecessors and extension nodes. No node parameter or hash changes. @@ -75,11 +94,13 @@ These are tested, as properties and examples, in `tests/engine_free/shared/test_transport_composed_graph.py`. - With no stages, the guard accepts and refuses exactly as the frozen copy of - the guard without stages does. + the guard without stages does, malformed benefit-unit entities included. - Person stages only restrict: a graph accepted with only person stages listed is accepted without them. -- Dropping a person stage never refuses a graph its listing admitted; dropping - a benefit-unit stage never admits a graph its listing refused. +- Dropping a person stage never refuses a graph its listing admitted. Dropping + a benefit-unit stage never admits a graph its listing refused, except when + the refusal was that stage's own declaration error: its schema, a hold-out + measure, or a derived output that no upstream stage node produces. - No hold-out source or selection is upstream of calibration, with or without stages, and no stage output or stage module carries a hold-out measure. - With stages listed, a refusal names the first offending node in declaration diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index a94b608c32..26b93d651b 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -177,6 +177,9 @@ _PREPARED = frozenset({"engine_refs"}) _PRECALIBRATION_SCHEMA_VERSION = 1 _PRECALIBRATION_PURPOSES = frozenset({"calibration_measure", "classification"}) +_PRECALIBRATION_STAGE_FIELDS = frozenset( + {"id", "rules_binding", "rulespec_path", "outputs", "citation"} +) #: Typed outputs each skeleton kernel must declare, when the declaration #: leaves them implicit. _OUTPUT_TYPES: Mapping[str, Mapping[str, ArtifactType]] = MappingProxyType( @@ -667,8 +670,44 @@ def _embeds_engine_ref( ) -def _unit_entities(spec: Mapping[str, object]) -> frozenset[str]: - return frozenset({transport_resource(spec, "benefit_unit_rule").get("entity")}) +def _unit_entities(spec: Mapping[str, object]) -> tuple[object, ...]: + """The declared benefit-unit entities (S2-d PR 1 adds every unit kind).""" + return (transport_resource(spec, "benefit_unit_rule").get("entity"),) + + +def _is_unit_entity(entity: object, unit_entities: tuple[object, ...]) -> bool: + # Compare, never hash, so a malformed entity behaves as it always did. + return any(entity == unit for unit in unit_entities) + + +def _carries_engine_ref_inventory(value: object) -> bool: + """Whether a value carries a prepared ``engine_refs`` map, in any JSON spelling.""" + + if isinstance(value, Mapping): + return "engine_refs" in value or any( + _carries_engine_ref_inventory(item) + for pair in value.items() + for item in pair + ) + if isinstance(value, list | tuple): + return any(_carries_engine_ref_inventory(item) for item in value) + if not isinstance(value, str) or "{" not in value: + return False + try: + decoded = json.loads(value) + except (ValueError, TypeError): + decoder = json.JSONDecoder() + for index, character in enumerate(value): + if character not in {"{", '"'}: + continue + try: + embedded, _ = decoder.raw_decode(value, index) + except ValueError: + continue + if _carries_engine_ref_inventory(embedded): + return True + return False + return decoded != value and _carries_engine_ref_inventory(decoded) def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: @@ -687,6 +726,10 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: stages = _sequence(resource["stages"], "precalibration_stages.stages") if not stages: return {} + if any( + not isinstance(entity, str) or not entity for entity in _unit_entities(spec) + ): + raise ValueError("precalibration_stages needs a named benefit-unit entity.") bindings = { row["id"]: row for row in transport_resource(spec, "axiom_rules_bindings")["bindings"] @@ -709,13 +752,9 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: identifiers = set() for item in stages: stage = _mapping(item, "precalibration_stages stage") - if set(stage) != { - "id", - "rules_binding", - "rulespec_path", - "outputs", - "citation", - }: + if not _PRECALIBRATION_STAGE_FIELDS <= set(stage) or set(stage) - ( + _PRECALIBRATION_STAGE_FIELDS | {"derived_outputs"} + ): raise ValueError( "precalibration_stages stage has missing or unknown fields." ) @@ -766,6 +805,43 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: "precalibration_stages outputs must be nonempty and distinct." ) columns.add(column) + derived = _sequence( + stage.get("derived_outputs", []), "precalibration_stages derived_outputs" + ) + for item in derived: + output = _mapping(item, "precalibration_stages derived output") + if set(output) != {"entity", "column", "purpose"}: + raise ValueError( + "precalibration_stages derived output has missing or unknown fields." + ) + if any( + not isinstance(output[key], str) or not output[key] + for key in ("entity", "column") + ): + raise ValueError( + "precalibration_stages derived output must name an entity and a column." + ) + if ( + output["entity"] == binding["entity"] + and output["column"] in binding["variables"] + ): + raise ValueError( + "precalibration_stages derived output is a binding variable; " + "list it in outputs." + ) + if ( + not isinstance(output["purpose"], str) + or output["purpose"] not in _PRECALIBRATION_PURPOSES + ): + raise ValueError( + "precalibration_stages derived output has unknown purpose." + ) + column = (output["entity"], output["column"]) + if column in columns: + raise ValueError( + "precalibration_stages outputs must be nonempty and distinct." + ) + columns.add(column) if columns & holdout_columns: raise ValueError("precalibration_stages output is a holdout measure.") if stage["rulespec_path"] in holdout_modules: @@ -807,6 +883,12 @@ def _validate_precalibration_ancestry( name: {(row["entity"], row["column"]) for row in stage["outputs"]} for name, stage in stages.items() } + derived_columns = { + name: { + (row["entity"], row["column"]) for row in stage.get("derived_outputs", ()) + } + for name, stage in stages.items() + } binding_refs = { name: { reference @@ -818,14 +900,6 @@ def _validate_precalibration_ancestry( } for name, row in bindings.items() } - # A prepared reference inventory is bookkeeping, not a stage invocation. - prepared_refs = ( - frozenset( - {canonical_json({"engine_refs": dict(config.engine_refs)}).decode("utf-8")} - ) - if config is not None - else frozenset() - ) roots = { name for name, row in rows.items() if row["kernel"].startswith("calibrate.") } @@ -867,7 +941,11 @@ def _validate_precalibration_ancestry( declarations = rows def check( - rows: Mapping[str, Mapping], predecessors: Mapping, *, composed: bool + rows: Mapping[str, Mapping], + predecessors: Mapping, + *, + composed: bool, + derived: bool, ) -> None: node_bindings = {} for name, row in rows.items(): @@ -878,9 +956,7 @@ def check( if config is not None and binding in config.engine_refs and _embeds_engine_ref( - row.get("params", {}), - {config.engine_refs[binding]}, - allowed_references=prepared_refs, + row.get("params", {}), {config.engine_refs[binding]} ) } else: @@ -898,6 +974,33 @@ def check( # stage's ancestry passes the same checks; a stage binding's nodes # after calibration (extension checkpoints) are outputs, not inputs. ancestors = _ancestor_closure(predecessors, roots) + # A stage restricts only what its own upstream nodes produce: a column + # of the same name from elsewhere is not the stage's output. + produced = { + binding: { + (item["entity"], item["column"]) + for name, row in rows.items() + if name in ancestors and binding in node_bindings[name] + for item in row.get("outputs", ()) + } + for binding in stages + } + if derived: + for binding, stage in stages.items(): + for entity, column in sorted( + derived_columns[binding] - produced[binding] + ): + raise ValueError( + f"precalibration_stages derived output {entity}.{column} of " + f"{stage['id']!r} is not produced by a stage node upstream " + "of calibration." + ) + restricted = { + binding: produced[binding] + - stage_columns[binding] + - derived_columns[binding] + for binding in stages + } inputs = { name: { (item["entity"], column) @@ -935,21 +1038,15 @@ def check( *(binding_refs[binding] for binding in own_units) ) reason = None - if composed and _embeds_engine_ref( - canonical_json(row.get("params", {})).decode("utf-8"), - set(prepared_refs), - ): - reason = ( - "prepared engine_refs map" - if own_units - else "not a declared pre-calibration stage" - ) if named.intersection(unit_bindings) - own_units: reason = ( "mixes listed and unlisted unit bindings" if own_units else "not a declared pre-calibration stage" ) + # The prepared map carries every engine: never a stage's own. + if _carries_engine_ref_inventory(row.get("params", {})): + reason = "prepared engine_refs map" params = ( declarations.get(name, {}).get("params", {}) if composed @@ -962,11 +1059,7 @@ def check( ) for selection in selections: if selection.get("prepared") == "engine_refs": - reason = ( - "prepared engine_refs map" - if own_units - else "not a declared pre-calibration stage" - ) + reason = "prepared engine_refs map" if isinstance(selection.get("resource"), str): resource = _stem(selection["resource"]) if resource == "holdout_references": @@ -1019,9 +1112,7 @@ def check( elif outputs & encoded: reason = reason or "not a declared pre-calibration stage" for binding in stages: - if binding not in own and inputs[name] & ( - binding_columns[binding] - stage_columns[binding] - ): + if binding not in own and inputs[name] & restricted[binding]: reason = "undeclared pre-calibration stage output" if _embeds_engine_ref( row.get("params", {}), @@ -1040,7 +1131,9 @@ def check( f"Transport calibration ancestry: node {name!r} carries {reason}." ) - check(rows, predecessors, composed=False) + # A derived output is checked against the fullest view: the composed graph + # when there is one, otherwise the declaration. + check(rows, predecessors, composed=False, derived=graph is None) if graph is None: return compiled = compile_graph(graph) @@ -1061,7 +1154,7 @@ def check( } for node in graph.nodes } - check(compiled_rows, compiled.predecessors, composed=True) + check(compiled_rows, compiled.predecessors, composed=True, derived=True) def validate_transport_calibration_ancestry( @@ -1102,7 +1195,8 @@ def validate_transport_calibration_ancestry( unit_bindings = { row["id"]: row for row in _sequence(bindings, "axiom_rules_bindings.bindings") - if isinstance(row, Mapping) and row.get("entity") in unit_entities + if isinstance(row, Mapping) + and _is_unit_entity(row.get("entity"), unit_entities) } unit_modules = frozenset( row["rulespec_path"] @@ -1110,7 +1204,7 @@ def validate_transport_calibration_ancestry( if isinstance(row.get("rulespec_path"), str) ) forbidden_columns = { - (row["entity"], variable) + (row.get("entity"), variable) for row in unit_bindings.values() for variable in row.get("variables", ()) } @@ -1119,7 +1213,7 @@ def validate_transport_calibration_ancestry( forbidden_columns.update( (output["entity"], output["column"]) for output in row.get("outputs", ()) - if output.get("entity") in unit_entities + if _is_unit_entity(output.get("entity"), unit_entities) ) references = { row["engine_ref"] @@ -1217,7 +1311,7 @@ def validate_transport_calibration_ancestry( forbidden_columns.update( (item.entity, item.column) for item in node.outputs - if item.entity in unit_entities + if _is_unit_entity(item.entity, unit_entities) ) for name in _ancestor_closure(compiled.predecessors, roots): node = graph.node(name) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 6aa35fd970..befbc62516 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -2783,16 +2783,26 @@ def unavailable(*args, **kwargs): assert not (out / EXPORT_FILENAME).exists() +_PRECALIBRATION_PROPERTY_DOMAINS = { + "mixed": { + "kinds": ("encode", "unit", "person", "selector", "holdout"), + "holdout_sources": True, + }, + # Without hold-out data most graphs pass the other checks, so the + # producer attribution of rule 3 is exercised in every run. + "rules_only": {"kinds": ("person", "selector"), "holdout_sources": False}, +} + + @st.composite -def _precalibration_ancestry_dags(draw): - """Small DAGs with independent binding declarations and column edges.""" - kinds = draw( - st.lists( - st.sampled_from(("encode", "unit", "person", "selector", "holdout")), - min_size=1, - max_size=7, - ) - ) +def _precalibration_ancestry_dags(draw, kinds, holdout_sources): + """Small DAGs with independent binding declarations and column edges. + + A new binding may also declare another binding's variable name, which only + that other binding's node produces; selector nodes add a derived column + that is not a binding variable; any binding node may carry hold-out data. + """ + kinds = draw(st.lists(st.sampled_from(kinds), min_size=1, max_size=7)) nodes = [] bindings = {} for index, kind in enumerate(kinds): @@ -2808,11 +2818,22 @@ def _precalibration_ancestry_dags(draw): binding = draw(st.sampled_from(available)) else: binding = f"binding_{index}" - bindings[binding] = {"entity": entity, "variables": []} + shared = [ + variable + for row in bindings.values() + if row["entity"] == entity + for variable in row["variables"] + ] + bindings[binding] = {"entity": entity, "variables": [], "derived": []} + if shared and draw(st.booleans()): + bindings[binding]["variables"].append(draw(st.sampled_from(shared))) outputs = [f"value_{index}"] if binding is not None: outputs.append(f"private_{index}") bindings[binding]["variables"].extend(outputs) + if kind == "selector": + outputs.append(f"derived_{index}") + bindings[binding]["derived"].append(f"derived_{index}") reads = [] dependencies = [] for predecessor, earlier in enumerate(nodes): @@ -2831,6 +2852,9 @@ def _precalibration_ancestry_dags(draw): "reads": reads, "dependencies": dependencies, "holdout_selection": kind == "holdout" and draw(st.booleans()), + "holdout_source": holdout_sources + and binding is not None + and draw(st.sampled_from((False, False, False, True))), } ) declared = {} @@ -2838,6 +2862,10 @@ def _precalibration_ancestry_dags(draw): if draw(st.booleans()): declared[binding] = draw( st.sets(st.sampled_from(row["variables"]), min_size=1) + ) | ( + draw(st.sets(st.sampled_from(row["derived"]))) + if row["derived"] + else set() ) dependencies = [] reads = [] @@ -2857,22 +2885,44 @@ def _precalibration_ancestry_dags(draw): "reads": reads, "dependencies": dependencies, "holdout_selection": False, + "holdout_source": False, } ) return nodes, bindings, declared -def _precalibration_reference_accepts(model): - """The stage rules on generated kinds and column-use edges, upstream of calibration.""" - nodes, _, declared = model +def _precalibration_reference_relevant(nodes): + """Calibration and its ancestors: the nodes the stage rules govern.""" relevant = {len(nodes) - 1} for index in range(len(nodes) - 1, -1, -1): if index in relevant: relevant.update(nodes[index]["dependencies"]) + return relevant + + +def _precalibration_reference_missing_derived(model, binding): + """Declared derived columns that no node of the stage upstream produces.""" + nodes, bindings, declared = model + relevant = _precalibration_reference_relevant(nodes) + produced = { + column + for index in relevant + if nodes[index]["binding"] == binding + for column in nodes[index]["outputs"] + } + return declared[binding] & set(bindings[binding]["derived"]) - produced + + +def _precalibration_reference_accepts(model): + """The stage rules on generated kinds and column-use edges, upstream of calibration.""" + nodes, _, declared = model + relevant = _precalibration_reference_relevant(nodes) + if any(_precalibration_reference_missing_derived(model, name) for name in declared): + return False for index in relevant: node = nodes[index] binding = node["binding"] - if node["kind"] == "holdout": + if node["kind"] == "holdout" or node["holdout_source"]: return False if ( binding is not None @@ -2944,7 +2994,7 @@ def _precalibration_synthetic_graph(model): prepared_params["engine_ref"] = references[binding] if item["kind"] == "selector": params["engine_ref"] = {"binding": binding, "field": "engine_ref"} - sources = () + sources = ("synthetic_holdout_facts",) if item["holdout_source"] else () if item["kind"] == "holdout": if item["holdout_selection"]: params["selection"] = {"resource": "holdout_references"} @@ -3001,7 +3051,12 @@ def _precalibration_synthetic_graph(model): "benefit_unit_rule": {"entity": "family"}, "axiom_rules_bindings": { "bindings": [ - {"id": name, "rulespec_path": f"synthetic/{name}.yaml", **row} + { + "id": name, + "rulespec_path": f"synthetic/{name}.yaml", + "entity": row["entity"], + "variables": row["variables"], + } for name, row in bindings.items() ] }, @@ -3023,6 +3078,21 @@ def _precalibration_synthetic_graph(model): if column in columns ], "citation": "Synthetic stage contract.", + **( + { + "derived_outputs": [ + { + "entity": bindings[name]["entity"], + "column": column, + "purpose": "calibration_measure", + } + for column in bindings[name]["derived"] + if column in columns + ] + } + if columns & set(bindings[name]["derived"]) + else {} + ), } for name, columns in declared.items() ], @@ -3089,9 +3159,13 @@ def _precalibration_property_outcomes(model): return outcomes +@pytest.mark.parametrize("domain", sorted(_PRECALIBRATION_PROPERTY_DOMAINS)) @settings(max_examples=2000, deadline=None, database=None) -@given(model=_precalibration_ancestry_dags(), data=st.data()) -def test_precalibration_stages_calibration_ancestry_property(model, data): +@given(data=st.data()) +def test_precalibration_stages_calibration_ancestry_property(domain, data): + model = data.draw( + _precalibration_ancestry_dags(**_PRECALIBRATION_PROPERTY_DOMAINS[domain]) + ) nodes, bindings, declared = model expected = _precalibration_reference_accepts(model) outcomes = _precalibration_property_outcomes(model) @@ -3126,8 +3200,9 @@ def test_precalibration_stages_calibration_ancestry_property(model, data): model, dropped, ) - else: - # I2: dropping a unit stage never admits what listing it refused. + elif not _precalibration_reference_missing_derived(model, dropped): + # I2: dropping a unit stage never admits what listing it refused, + # unless the refusal is that stage's own declaration error. assert outcomes[label, "guard"] or not reduced[label, "guard"], ( label, model, @@ -3313,19 +3388,37 @@ def test_precalibration_ancestry_stage_binding_after_calibration_is_not_checked( ) +def _precalibration_inventory(engine_refs, encoding): + """The prepared engine_refs map, spelled the ways a parameter can carry it.""" + inventory = {"engine_refs": dict(sorted(engine_refs.items()))} + canonical = canonical_json(inventory).decode() + return { + "string": canonical, + "tuple": (canonical,), + "spaced": json.dumps(inventory, indent=2), + "nested": json.dumps({"receipt": json.dumps(inventory)}), + "embedded": f"inventory={json.dumps(inventory)};", + }[encoding] + + +_PRECALIBRATION_INVENTORY_ENCODINGS = [ + "string", + "tuple", + "spaced", + "nested", + "embedded", +] + + @pytest.mark.parametrize("name", ["nz.targets.problem", "nz.precal.family"]) -@pytest.mark.parametrize("encoding", ["string", "tuple"]) +@pytest.mark.parametrize("encoding", _PRECALIBRATION_INVENTORY_ENCODINGS) def test_precalibration_ancestry_compiled_prepared_map_still_refuses( calibration_ancestry_composed, name, encoding ): composed = calibration_ancestry_composed spec = _precalibration_spec(composed) graph = compose_transport_graph(spec, composed.config) - prepared = canonical_json( - {"engine_refs": dict(sorted(composed.config.engine_refs.items()))} - ).decode() - if encoding == "tuple": - prepared = (prepared,) + prepared = _precalibration_inventory(composed.config.engine_refs, encoding) graph = replace( graph, nodes=tuple( @@ -3335,12 +3428,379 @@ def test_precalibration_ancestry_compiled_prepared_map_still_refuses( for node in graph.nodes ), ) - reason = ( - "prepared engine_refs map" - if name == "nz.precal.family" - else "not a declared pre-calibration stage" - ) - with pytest.raises(ValueError, match=f"node '{name}' carries {reason}"): + with pytest.raises( + ValueError, match=f"node '{name}' carries prepared engine_refs map" + ): compose_module.validate_transport_calibration_ancestry( spec, graph=graph, config=composed.config ) + + +@pytest.mark.parametrize("encoding", ["string", "spaced", "nested", "embedded"]) +def test_precalibration_ancestry_declared_prepared_map_refuses( + calibration_ancestry_composed, encoding +): + """A listed stage node may not carry every engine in its declaration either.""" + composed = calibration_ancestry_composed + spec = _precalibration_spec(composed) + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == "nz.precal.family" + ) + row["params"] = { + "inventory": _precalibration_inventory(composed.config.engine_refs, encoding) + } + with pytest.raises( + ValueError, match="node 'nz.precal.family' carries prepared engine_refs map" + ): + compose_module.validate_transport_calibration_ancestry( + spec, config=composed.config + ) + + +_SYNTHETIC_SEED = Owned("person", "seed", "float64") + + +def _synthetic_stage_spec(bindings, stages, rows, *, holdout=()): + return { + "country": "synthetic", + "resources": { + "benefit_unit_rule": {"entity": "family"}, + "axiom_rules_bindings": {"bindings": bindings}, + "holdout_references": {"target_references": list(holdout)}, + "precalibration_stages": {"schema_version": 1, "stages": stages}, + "transport_graph": { + "schema_version": 1, + "country": "synthetic", + "nodes": [{"id": "create", "kernel": "synthetic.create@1"}, *rows], + "endpoints": {"calibration": "cal"}, + }, + }, + } + + +def _synthetic_binding(name, entity, variables): + return { + "id": name, + "entity": entity, + "rulespec_path": f"synthetic/{name}.yaml", + "variables": list(variables), + } + + +def _synthetic_stage(binding, outputs, derived=None): + stage = { + "id": f"stage.{binding['id']}", + "rules_binding": binding["id"], + "rulespec_path": binding["rulespec_path"], + "outputs": [ + { + "entity": binding["entity"], + "column": column, + "purpose": "calibration_measure", + } + for column in outputs + ], + "citation": "Synthetic stage contract.", + } + if derived is not None: + stage["derived_outputs"] = [ + {"entity": entity, "column": column, "purpose": "calibration_measure"} + for entity, column in derived + ] + return stage + + +def _synthetic_outcomes(spec, nodes, engine_refs): + """The guard's outcome in the declaration pass and the composed pass.""" + graph = Graph( + "synthetic", + ( + SourceRef("donor", "synthetic@1"), + SourceRef("synthetic_holdout_facts", "synthetic@1"), + ), + ( + Node( + "create", + "synthetic.create@1", + structural=StructuralDelta.CREATE, + sources=("donor",), + outputs=(_SYNTHETIC_SEED,), + ), + *nodes, + ), + ) + config = TransportGraphConfig( + engine_refs=engine_refs, create_outputs=(_SYNTHETIC_SEED,) + ) + return { + label: _precalibration_outcome( + compose_module.validate_transport_calibration_ancestry, + spec, + graph=graph if label == "composed" else None, + config=config, + ) + for label in ("declaration", "composed") + } + + +def _overlap_case(a_produces_shared): + """Bindings A and B both declare family.shared; each stage exposes only its public column.""" + bindings = [ + _synthetic_binding(name, "family", ["shared", f"{name}_public"]) + for name in ("A", "B") + ] + stages = [_synthetic_stage(row, [f"{row['id']}_public"]) for row in bindings] + rows = [ + { + "id": "b_source", + "kernel": RULES, + "population": "create", + "rules_binding": "B", + "outputs": [{"entity": "family", "column": "shared"}], + }, + { + "id": "b", + "kernel": "simulate.solve_zero@1", + "population": "create", + "params": {"ref": {"binding": "B", "field": "engine_ref"}}, + "inputs": [{"entity": "family", "columns": ["shared"]}], + "outputs": [{"entity": "family", "column": "B_public"}], + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": "create", + "inputs": [{"entity": "family", "columns": ["B_public"]}], + }, + ] + nodes = [ + Node( + "b_source", + RULES, + population="create", + params={"engine_ref": "B-ref"}, + outputs=(Owned("family", "shared", "float64"),), + ), + Node( + "b", + "simulate.solve_zero@1", + population="create", + params={"engine_ref": "B-ref"}, + inputs=(Slice("family", ("shared",)),), + outputs=(Owned("family", "B_public", "float64"),), + ), + Node( + "cal", + "calibrate.synthetic@1", + population="create", + inputs=(Slice("family", ("B_public",)),), + ), + ] + if a_produces_shared: + # A also runs upstream and owns a column of the same name, read by B. + rows[0] = {**rows[0], "rules_binding": "A"} + nodes[0] = replace(nodes[0], params={"engine_ref": "A-ref"}) + return _synthetic_stage_spec(bindings, stages, rows), nodes + + +@pytest.mark.parametrize("drop_unused", [False, True]) +def test_precalibration_ancestry_unused_stage_does_not_restrict_by_name(drop_unused): + """Rule 3 restricts what a stage's own nodes produce, not every matching name.""" + spec, nodes = _overlap_case(a_produces_shared=False) + if drop_unused: + spec["resources"]["precalibration_stages"]["stages"].pop(0) + outcomes = _synthetic_outcomes(spec, nodes, {"A": "A-ref", "B": "B-ref"}) + assert outcomes == {"declaration": None, "composed": None} + + +def test_precalibration_ancestry_stage_output_read_by_another_stage_refuses(): + spec, nodes = _overlap_case(a_produces_shared=True) + outcomes = _synthetic_outcomes(spec, nodes, {"A": "A-ref", "B": "B-ref"}) + message = ( + "Transport calibration ancestry: node 'b' carries undeclared " + "pre-calibration stage output." + ) + assert outcomes == { + "declaration": (ValueError, message), + "composed": (ValueError, message), + } + + +def _derived_case(*, derived, producer_upstream=True): + """A person-binding gross-up solve whose output is not a binding variable.""" + binding = _synthetic_binding("tax", "person", ["tax_before_credits"]) + stage = _synthetic_stage(binding, ["tax_before_credits"], derived) + population = "create" if producer_upstream else "after" + rows = [ + { + "id": "gross", + "kernel": "simulate.solve_zero@1", + "population": population, + "params": {"engine_ref": {"binding": "tax", "field": "engine_ref"}}, + "outputs": [{"entity": "person", "column": "benefit_gross"}], + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": "create", + "inputs": [ + { + "entity": "person", + "columns": ["benefit_gross" if producer_upstream else "seed"], + } + ], + }, + ] + nodes = [ + Node( + "gross", + "simulate.solve_zero@1", + population="create", + params={"engine_ref": "tax-ref"}, + outputs=(Owned("person", "benefit_gross", "float64"),), + ), + Node( + "cal", + "calibrate.synthetic@1", + population="create", + inputs=(Slice("person", (rows[1]["inputs"][0]["columns"][0],)),), + ), + ] + if not producer_upstream: + rows.insert( + 1, + { + "id": "after", + "kernel": "transport.boundary@1", + "structural": "filter", + "base": "cal", + }, + ) + nodes = [nodes[1]] + return _synthetic_stage_spec([binding], [stage], rows), nodes + + +def test_precalibration_ancestry_derived_output_flows_to_calibration(): + """NZ's w2b gross-up: a selector stage node's own column is a declared output.""" + spec, nodes = _derived_case(derived=[("person", "benefit_gross")]) + assert _synthetic_outcomes(spec, nodes, {"tax": "tax-ref"}) == { + "declaration": None, + "composed": None, + } + + +def test_precalibration_ancestry_undeclared_derived_column_refuses(): + spec, nodes = _derived_case(derived=None) + message = ( + "Transport calibration ancestry: node 'cal' carries undeclared " + "pre-calibration stage output." + ) + assert _synthetic_outcomes(spec, nodes, {"tax": "tax-ref"}) == { + "declaration": (ValueError, message), + "composed": (ValueError, message), + } + + +@pytest.mark.parametrize("producer_upstream", [False, True]) +def test_precalibration_ancestry_derived_output_needs_an_upstream_producer( + producer_upstream, +): + """A derived row naming a column no upstream stage node produces refuses.""" + # Upstream, the real column is declared too, so only the phantom row fails. + column = "never_produced" if producer_upstream else "benefit_gross" + derived = [("person", "benefit_gross")] if producer_upstream else [] + spec, nodes = _derived_case( + derived=[*derived, ("person", column)], producer_upstream=producer_upstream + ) + message = ( + f"precalibration_stages derived output person.{column} of 'stage.tax' is " + "not produced by a stage node upstream of calibration." + ) + assert _synthetic_outcomes(spec, nodes, {"tax": "tax-ref"}) == { + "declaration": (ValueError, message), + "composed": (ValueError, message), + } + + +@pytest.mark.parametrize( + "mutation", + ["key", "binding_variable", "purpose", "duplicate", "holdout", "empty_column"], +) +def test_precalibration_ancestry_derived_output_schema_refuses(mutation): + spec, _ = _derived_case(derived=[("person", "benefit_gross")]) + stage = spec["resources"]["precalibration_stages"]["stages"][0] + row = stage["derived_outputs"][0] + if mutation == "key": + row["unknown"] = True + elif mutation == "binding_variable": + row["column"] = "tax_before_credits" + elif mutation == "purpose": + row["purpose"] = "evaluation" + elif mutation == "duplicate": + stage["derived_outputs"].append(dict(row)) + elif mutation == "holdout": + spec["resources"]["holdout_references"]["target_references"].append( + {"entity": "person", "measure": "benefit_gross"} + ) + else: + row["column"] = "" + with pytest.raises(ValueError, match="precalibration_stages"): + compose_module.validate_transport_calibration_ancestry(spec) + + +@pytest.mark.parametrize("compiled", [False, True]) +def test_precalibration_ancestry_stage_node_holdout_source_refuses(compiled): + """The hold-out check covers a stage's own nodes, not only their ancestors.""" + spec, nodes = _derived_case(derived=[("person", "benefit_gross")]) + rows = spec["resources"]["transport_graph"]["nodes"] + if compiled: + nodes = [ + replace(node, sources=("synthetic_holdout_facts",)) + if node.id == "gross" + else node + for node in nodes + ] + else: + next(row for row in rows if row["id"] == "gross")["sources"] = [ + "synthetic_holdout_facts" + ] + outcomes = _synthetic_outcomes(spec, nodes, {"tax": "tax-ref"}) + label = "composed" if compiled else "declaration" + assert outcomes[label] == ( + ValueError, + "Transport calibration ancestry: node 'gross' carries hold-out source.", + ) + + +@pytest.mark.parametrize("guard", ["absent", "empty"]) +@pytest.mark.parametrize("malformed", ["unit_entity", "binding_entity", "missing"]) +def test_precalibration_ancestry_malformed_entities_match_today( + monkeypatch, guard, malformed +): + """Without stages, even a malformed entity gets today's exact outcome.""" + spec = _synthetic_stage_spec( + [], [], [{"id": "cal", "kernel": "calibrate.synthetic@1"}] + ) + resources = spec["resources"] + if malformed == "unit_entity": + resources["benefit_unit_rule"]["entity"] = [] + elif malformed == "binding_entity": + resources["axiom_rules_bindings"]["bindings"] = [ + {"id": "odd", "entity": ["family"], "variables": ["x"]} + ] + else: + del resources["benefit_unit_rule"]["entity"] + resources["axiom_rules_bindings"]["bindings"] = [ + {"id": "odd", "variables": ["x"], "rulespec_path": "odd.yaml"} + ] + if guard == "absent": + del resources["precalibration_stages"] + expected = _precalibration_outcome(_precalibration_reference_guard, spec) + assert ( + _precalibration_outcome( + compose_module.validate_transport_calibration_ancestry, spec + ) + == expected + ) From ead5c9c689553ac2756a8b1754942fd8a248927d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 17:46:51 -0400 Subject: [PATCH 04/20] Let a stage declare only derived outputs NZ's Schedule 1 stage keeps the tax inside its gross-up solve, so only the solve's own column flows on. A stage may now list "outputs": [] when it declares derived outputs; a stage with neither still refuses. The derived tests use that shape, and the property draws derived-only declarations. Co-Authored-By: Claude Opus 5.5 --- docs/transport-precalibration-stages.md | 19 +++++++----- .../src/microcosm/build/transport/compose.py | 11 ++++--- .../shared/test_transport_composed_graph.py | 31 +++++++++++++------ 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index 430c89268a..a7f2b54563 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -35,20 +35,23 @@ outputs feed calibration, and exactly which of their outputs flow on: Key sets are closed at every level, and `schema_version` is the integer 1. Stage ids are distinct, safe node ids. Each stage names an existing binding, repeats its `rulespec_path` exactly, and no binding appears in two stages. -Outputs are nonempty and distinct; each is on the binding's entity, names one -of its `variables`, and has a `purpose` of `calibration_measure` or +A stage declares at least one output or derived output, and they are all +distinct. Each output is on the binding's entity, names one of its +`variables`, and has a `purpose` of `calibration_measure` or `classification` (a judgment another pre-calibration node reads). An output may not be a hold-out `(entity, measure)`, and a stage's module may not produce a hold-out measure. The resource is checked before either graph pass. `derived_outputs` is optional. It lists columns that a stage node computes from the binding but that are not binding variables, such as a gross-up -`simulate.solve_zero@1` node's own output. Each row has the same closed shape -and purposes as an `outputs` row. It names any entity, may not repeat a binding -variable on the binding's entity (that belongs in `outputs`), and may not -repeat any other output of the stage or a hold-out measure. Each derived output -must be produced by one of the stage's nodes upstream of calibration. The -composed graph is checked when there is one, otherwise the declaration. +`simulate.solve_zero@1` node's own output. A stage whose binding variables stay +inside its nodes lists `"outputs": []` and only derived outputs. Each row has +the same closed shape and purposes as an `outputs` row. It names any entity, +may not repeat a binding variable on the binding's entity (that belongs in +`outputs`), and may not repeat any other output of the stage or a hold-out +measure. Each derived output must be produced by one of the stage's nodes +upstream of calibration. The composed graph is checked when there is one, +otherwise the declaration. ## Which nodes belong to a stage diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 26b93d651b..e8f1e74ad2 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -776,9 +776,13 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: "precalibration_stages rulespec_path must match its binding." ) outputs = _sequence(stage["outputs"], "precalibration_stages outputs") - if not outputs: + derived = _sequence( + stage.get("derived_outputs", []), "precalibration_stages derived_outputs" + ) + # A stage whose binding variables stay internal lists only derived ones. + if not outputs and not derived: raise ValueError( - "precalibration_stages outputs must be nonempty and distinct." + "precalibration_stages stage declares no outputs or derived outputs." ) columns = set() for item in outputs: @@ -805,9 +809,6 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: "precalibration_stages outputs must be nonempty and distinct." ) columns.add(column) - derived = _sequence( - stage.get("derived_outputs", []), "precalibration_stages derived_outputs" - ) for item in derived: output = _mapping(item, "precalibration_stages derived output") if set(output) != {"entity", "column", "purpose"}: diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index befbc62516..a36d890203 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -2860,12 +2860,9 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources): declared = {} for binding, row in bindings.items(): if draw(st.booleans()): + # Derived columns alone are a complete declaration. declared[binding] = draw( - st.sets(st.sampled_from(row["variables"]), min_size=1) - ) | ( - draw(st.sets(st.sampled_from(row["derived"]))) - if row["derived"] - else set() + st.sets(st.sampled_from(row["variables"] + row["derived"]), min_size=1) ) dependencies = [] reads = [] @@ -3628,10 +3625,14 @@ def test_precalibration_ancestry_stage_output_read_by_another_stage_refuses(): } -def _derived_case(*, derived, producer_upstream=True): - """A person-binding gross-up solve whose output is not a binding variable.""" +def _derived_case(*, derived, outputs=(), producer_upstream=True): + """A person-binding gross-up solve whose output is not a binding variable. + + By default the stage lists no binding variable, as NZ's Schedule 1 stage + does: the tax stays inside the solve and only the gross-up flows on. + """ binding = _synthetic_binding("tax", "person", ["tax_before_credits"]) - stage = _synthetic_stage(binding, ["tax_before_credits"], derived) + stage = _synthetic_stage(binding, outputs, derived) population = "create" if producer_upstream else "after" rows = [ { @@ -3692,7 +3693,7 @@ def test_precalibration_ancestry_derived_output_flows_to_calibration(): def test_precalibration_ancestry_undeclared_derived_column_refuses(): - spec, nodes = _derived_case(derived=None) + spec, nodes = _derived_case(derived=None, outputs=["tax_before_credits"]) message = ( "Transport calibration ancestry: node 'cal' carries undeclared " "pre-calibration stage output." @@ -3726,7 +3727,15 @@ def test_precalibration_ancestry_derived_output_needs_an_upstream_producer( @pytest.mark.parametrize( "mutation", - ["key", "binding_variable", "purpose", "duplicate", "holdout", "empty_column"], + [ + "key", + "binding_variable", + "purpose", + "duplicate", + "holdout", + "empty_column", + "no_outputs", + ], ) def test_precalibration_ancestry_derived_output_schema_refuses(mutation): spec, _ = _derived_case(derived=[("person", "benefit_gross")]) @@ -3744,6 +3753,8 @@ def test_precalibration_ancestry_derived_output_schema_refuses(mutation): spec["resources"]["holdout_references"]["target_references"].append( {"entity": "person", "measure": "benefit_gross"} ) + elif mutation == "no_outputs": + stage["derived_outputs"] = [] else: row["column"] = "" with pytest.raises(ValueError, match="precalibration_stages"): From 16c35df4e24a742faf0eb43907e5e6732654af0e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 17:52:53 -0400 Subject: [PATCH 05/20] Check nodes after the central fit that feed a variant calibration Roots are every calibrate.* node and the calibration endpoint, so a node after the central fit that a variant calibration reads is in that variant's ancestry. Add the regression the NZ hub asked for: a hold-out source or an unlisted unit rules node there refuses in both passes, and the same node feeding no calibration is accepted. The doc now says "outside the ancestry of every calibration node". Co-Authored-By: Claude Opus 5.5 --- .../transport-precalibration-stages.added.md | 2 +- docs/transport-precalibration-stages.md | 8 +- .../shared/test_transport_composed_graph.py | 82 +++++++++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/changelog.d/transport-precalibration-stages.added.md b/changelog.d/transport-precalibration-stages.added.md index 539894dc25..904155fd70 100644 --- a/changelog.d/transport-precalibration-stages.added.md +++ b/changelog.d/transport-precalibration-stages.added.md @@ -1 +1 @@ -Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes upstream of calibration are checked, so a stage binding's nodes after calibration are unaffected. +Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index a7f2b54563..19bb60a5ce 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -65,8 +65,12 @@ refuses upstream of calibration. ## Rules -Only nodes upstream of calibration are checked. A stage binding's nodes after -calibration, such as extension checkpoints, are outputs and are unaffected. +Only nodes in the ancestry of a calibration node are checked. The calibration +nodes are every `calibrate.*` node and the calibration endpoint, in the +declaration and in the composed graph. A node outside the ancestry of every +calibration node, such as a stage binding's extension checkpoint after the +fit, is an output and is unaffected. A node after the central fit that a +variant calibration reads is in that variant's ancestry, so it is checked. - Absent, or `"stages": []`, is the guard without stages, outcomes and messages alike. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index a36d890203..2ba7b98007 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -3385,6 +3385,88 @@ def test_precalibration_ancestry_stage_binding_after_calibration_is_not_checked( ) +def _variant_calibration_case(post, feeds_variant): + """A node after the central fit, and a variant calibration that may read it.""" + tax = _synthetic_binding("tax", "person", ["tax_out"]) + family = _synthetic_binding("fam", "family", ["fam_out"]) + fit = ArtifactType("synthetic.fit", 1) + holdout = post == "holdout" + entity, column = ("person", "post_value") if holdout else ("family", "fam_out") + post_row = { + "id": "post", + "kernel": "synthetic.holdout@1" if holdout else RULES, + "population": "create", + "outputs": [{"entity": entity, "column": column}], + "artifact_inputs": [{"producer": "cal"}], + **( + {"sources": ["synthetic_holdout_facts"]} + if holdout + else {"rules_binding": "fam"} + ), + } + rows = [ + {"id": "cal", "kernel": "calibrate.synthetic@1", "population": "create"}, + post_row, + { + "id": "variant", + "kernel": "calibrate.variant@1", + "population": "create", + "artifact_inputs": [{"producer": "post"}] if feeds_variant else [], + }, + ] + nodes = [ + Node( + "cal", + "calibrate.synthetic@1", + population="create", + artifact_outputs=(ArtifactOutput("fit", fit),), + ), + Node( + "post", + post_row["kernel"], + population="create", + params={} if holdout else {"engine_ref": "fam-ref"}, + sources=("synthetic_holdout_facts",) if holdout else (), + outputs=(Owned(entity, column, "float64"),), + artifact_inputs=(ArtifactInput("fit", "cal", "fit", fit),), + artifact_outputs=(ArtifactOutput("fit", fit),), + ), + Node( + "variant", + "calibrate.variant@1", + population="create", + artifact_inputs=( + (ArtifactInput("fit", "post", "fit", fit),) if feeds_variant else () + ), + ), + ] + stages = [_synthetic_stage(tax, ["tax_out"])] + return _synthetic_stage_spec([tax, family], stages, rows), nodes + + +@pytest.mark.parametrize("feeds_variant", [False, True]) +@pytest.mark.parametrize("post", ["holdout", "unlisted_unit"]) +def test_precalibration_ancestry_node_after_central_fit_feeding_a_variant_is_checked( + post, feeds_variant +): + """After the central fit is not after calibration when a variant reads it.""" + spec, nodes = _variant_calibration_case(post, feeds_variant) + reason = ( + "hold-out source" + if post == "holdout" + else ("not a declared pre-calibration stage") + ) + expected = ( + (ValueError, f"Transport calibration ancestry: node 'post' carries {reason}.") + if feeds_variant + else None + ) + assert _synthetic_outcomes(spec, nodes, {"tax": "tax-ref", "fam": "fam-ref"}) == { + "declaration": expected, + "composed": expected, + } + + def _precalibration_inventory(engine_refs, encoding): """The prepared engine_refs map, spelled the ways a parameter can carry it.""" inventory = {"engine_refs": dict(sorted(engine_refs.items()))} From 3d8e9f9ddbd3e4d6988033ac66f7cd7fbf0249a9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 18:35:53 -0400 Subject: [PATCH 06/20] Treat a stage node's artifacts as undeclared outputs Stage outputs are columns, so rule 3 checked only column reads, and a node that names a listed stage could pass an artifact straight to calibration. For a unit stage, today's guard refuses that graph and the stage path admitted it. Now a node outside the stage that reads one of its artifacts upstream of calibration refuses as "undeclared pre-calibration stage output", in both passes; the composed rows carry artifact edges for this. The property now draws artifact edges apart from column reads. The guard without stages follows column reads only in its compiled predecessors, and a resource selection leaves no marker there, so the no-stage reference is the frozen guard. The doc scopes unit-stage I2 to drops that leave a stage listed, and says the no-stage guard still misses a hold-out selection reached only through column reads. Co-Authored-By: Claude Opus 5.5 --- docs/transport-precalibration-stages.md | 21 ++- .../src/microcosm/build/transport/compose.py | 20 ++- .../shared/test_transport_composed_graph.py | 143 ++++++++++++++++-- 3 files changed, 164 insertions(+), 20 deletions(-) diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index 19bb60a5ce..3302e72b98 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -85,6 +85,9 @@ variant calibration reads is in that variant's ancestry, so it is checked. Reads among a stage's own nodes do not count. The rule follows the producing node, not the column name, so another binding that declares the same variable name is unaffected. +- Outputs are columns, so a stage's artifacts are never declared. A node + outside the stage that reads an artifact one of the stage's nodes produces + upstream of calibration refuses the same way. - A parameter that carries the prepared `engine_refs` inventory refuses as "prepared engine_refs map", on stage nodes too. The inventory may be a mapping, a JSON string in any spelling, nested in lists or mappings, or JSON @@ -104,11 +107,17 @@ These are tested, as properties and examples, in the guard without stages does, malformed benefit-unit entities included. - Person stages only restrict: a graph accepted with only person stages listed is accepted without them. -- Dropping a person stage never refuses a graph its listing admitted. Dropping - a benefit-unit stage never admits a graph its listing refused, except when - the refusal was that stage's own declaration error: its schema, a hold-out - measure, or a derived output that no upstream stage node produces. -- No hold-out source or selection is upstream of calibration, with or without - stages, and no stage output or stage module carries a hold-out measure. +- Dropping a person stage never refuses a graph its listing admitted. While + another stage stays listed, dropping a benefit-unit stage never admits a + graph its listing refused, except when the refusal was that stage's own + declaration error: its schema, a hold-out measure, or a derived output that + no upstream stage node produces. Dropping the last stage gives exactly the + guard without stages, which can admit more: its declaration pass follows + structural and artifact edges, not column reads, and a resource selection + leaves no marker in the composed graph. +- With stages listed, no hold-out source or selection is upstream of + calibration, and no stage output or stage module carries a hold-out measure. + The guard without stages is unchanged, so it still misses a hold-out + selection that calibration reaches only through column reads. - With stages listed, a refusal names the first offending node in declaration order. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index e8f1e74ad2..9e4f5f777a 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -1010,6 +1010,18 @@ def check( } for name, row in rows.items() } + # Outputs are columns, so a stage's artifacts are never declared: only + # the stage's own nodes may read them. + artifact_stages = { + name: { + binding + for edge in row.get("artifact_inputs", ()) + if edge["producer"] in ancestors + for binding in node_bindings.get(edge["producer"], ()) + if binding in stages + } + for name, row in rows.items() + } encoded = { (output["entity"], output["column"]) for row in rows.values() @@ -1113,7 +1125,10 @@ def check( elif outputs & encoded: reason = reason or "not a declared pre-calibration stage" for binding in stages: - if binding not in own and inputs[name] & restricted[binding]: + if binding not in own and ( + inputs[name] & restricted[binding] + or binding in artifact_stages[name] + ): reason = "undeclared pre-calibration stage output" if _embeds_engine_ref( row.get("params", {}), @@ -1152,6 +1167,9 @@ def check( ], "params": node.params, "sources": node.sources, + "artifact_inputs": [ + {"producer": edge.producer} for edge in node.artifact_inputs + ], } for node in graph.nodes } diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 2ba7b98007..8f2d35951e 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -2836,10 +2836,13 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources): bindings[binding]["derived"].append(f"derived_{index}") reads = [] dependencies = [] + artifacts = [] for predecessor, earlier in enumerate(nodes): edge = draw(st.integers(min_value=0, max_value=3)) if edge: dependencies.append(predecessor) + if edge == 1: + artifacts.append(predecessor) if edge > 1: columns = earlier["outputs"] if edge == 3 else earlier["outputs"][:1] reads.extend((predecessor, column) for column in columns) @@ -2851,6 +2854,7 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources): "outputs": outputs, "reads": reads, "dependencies": dependencies, + "artifacts": artifacts, "holdout_selection": kind == "holdout" and draw(st.booleans()), "holdout_source": holdout_sources and binding is not None @@ -2866,10 +2870,13 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources): ) dependencies = [] reads = [] + artifacts = [] for predecessor, earlier in enumerate(nodes): edge = draw(st.integers(min_value=0, max_value=3)) if edge: dependencies.append(predecessor) + if edge == 1: + artifacts.append(predecessor) if edge > 1: columns = earlier["outputs"] if edge == 3 else earlier["outputs"][:1] reads.extend((predecessor, column) for column in columns) @@ -2881,6 +2888,7 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources): "outputs": [], "reads": reads, "dependencies": dependencies, + "artifacts": artifacts, "holdout_selection": False, "holdout_source": False, } @@ -2911,7 +2919,7 @@ def _precalibration_reference_missing_derived(model, binding): def _precalibration_reference_accepts(model): - """The stage rules on generated kinds and column-use edges, upstream of calibration.""" + """The stage rules on generated kinds, column reads and artifact edges, upstream of calibration.""" nodes, _, declared = model relevant = _precalibration_reference_relevant(nodes) if any(_precalibration_reference_missing_derived(model, name) for name in declared): @@ -2946,6 +2954,11 @@ def _precalibration_reference_accepts(model): and column not in declared[source_binding] ): return False + # A stage's artifacts are never declared outputs. + for producer in node["artifacts"]: + source_binding = nodes[producer]["binding"] + if source_binding in declared and source_binding != binding: + return False return True @@ -2999,9 +3012,10 @@ def _precalibration_synthetic_graph(model): prepared_params["selection"] = "{}" else: sources = ("synthetic_holdout_facts",) + # A column read is its own edge; an artifact carries the rest. edges = tuple( ArtifactInput(f"edge_{producer}", f"p1.node_{producer}", "trace", trace) - for producer in item["dependencies"] + for producer in item["artifacts"] ) graph_nodes.append( Node( @@ -3036,7 +3050,7 @@ def _precalibration_synthetic_graph(model): "params": params, "sources": list(sources), "artifact_inputs": [ - {"producer": f"p1.node_{producer}"} for producer in item["dependencies"] + {"producer": f"p1.node_{producer}"} for producer in item["artifacts"] ], } if binding is not None and item["kind"] != "selector": @@ -3164,16 +3178,17 @@ def test_precalibration_stages_calibration_ancestry_property(domain, data): _precalibration_ancestry_dags(**_PRECALIBRATION_PROPERTY_DOMAINS[domain]) ) nodes, bindings, declared = model - expected = _precalibration_reference_accepts(model) outcomes = _precalibration_property_outcomes(model) labels = ("declaration", "compiled extensions") for label in labels: actual = outcomes[label, "guard"] - assert actual == expected, (label, model) if not declared: - # I1 with no stages: exactly today's guard. + # I1 with no stages: exactly today's guard. Its declaration pass + # follows artifact and structural edges only, not column reads. assert actual == outcomes[label, "today"], (label, model) - elif all(bindings[name]["entity"] == "person" for name in declared): + continue + assert actual == _precalibration_reference_accepts(model), (label, model) + if all(bindings[name]["entity"] == "person" for name in declared): # I1: person stages only restrict today's guard. assert outcomes[label, "today"] or not actual, (label, model) if not declared: @@ -3186,10 +3201,11 @@ def test_precalibration_stages_calibration_ancestry_property(domain, data): ) reduced = _precalibration_property_outcomes(smaller) for label in labels: - assert reduced[label, "guard"] == _precalibration_reference_accepts(smaller), ( - label, - smaller, - ) + assert reduced[label, "guard"] == ( + _precalibration_reference_accepts(smaller) + if smaller[2] + else reduced[label, "today"] + ), (label, smaller) if bindings[dropped]["entity"] == "person": # I2: dropping a person stage never refuses what listing it admitted. assert reduced[label, "guard"] or not outcomes[label, "guard"], ( @@ -3197,9 +3213,14 @@ def test_precalibration_stages_calibration_ancestry_property(domain, data): model, dropped, ) - elif not _precalibration_reference_missing_derived(model, dropped): + elif smaller[2] and not _precalibration_reference_missing_derived( + model, dropped + ): # I2: dropping a unit stage never admits what listing it refused, - # unless the refusal is that stage's own declaration error. + # unless the refusal is that stage's own declaration error. Dropping + # the last stage restores today's guard exactly (checked above), + # and today's guard does not follow column-only paths to a + # resource selection or, in its declaration pass, at all. assert outcomes[label, "guard"] or not reduced[label, "guard"], ( label, model, @@ -3385,6 +3406,102 @@ def test_precalibration_ancestry_stage_binding_after_calibration_is_not_checked( ) +def _stage_artifact_case(entity, reader): + """A stage node writes an artifact; calibration or another stage node reads it.""" + binding = _synthetic_binding("u", entity, ["u_out"]) + trace = ArtifactType("synthetic.trace", 1) + selector = {"engine_ref": {"binding": "u", "field": "engine_ref"}} + # Calibration reads the artifact itself, or a second stage node turns it + # into the declared column that calibration reads. + writer = "probe" if reader == "calibration" else "own" + rows = [ + { + "id": "probe", + "kernel": "synthetic.probe@1", + "population": "create", + "params": selector, + **( + {"outputs": [{"entity": entity, "column": "u_out"}]} + if writer == "probe" + else {} + ), + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": "create", + "inputs": [{"entity": entity, "columns": ["u_out"]}], + }, + ] + edge = ArtifactInput("trace", "probe", "trace", trace) + nodes = [ + Node( + "probe", + "synthetic.probe@1", + population="create", + params={"engine_ref": "u-ref"}, + outputs=(Owned(entity, "u_out", "float64"),) if writer == "probe" else (), + artifact_outputs=(ArtifactOutput("trace", trace),), + ) + ] + if writer == "probe": + rows[1]["artifact_inputs"] = [{"name": "trace", "producer": "probe"}] + else: + rows.insert( + 1, + { + "id": "own", + "kernel": "synthetic.probe@1", + "population": "create", + "params": selector, + "outputs": [{"entity": entity, "column": "u_out"}], + "artifact_inputs": [{"name": "trace", "producer": "probe"}], + }, + ) + nodes.append( + Node( + "own", + "synthetic.probe@1", + population="create", + params={"engine_ref": "u-ref"}, + outputs=(Owned(entity, "u_out", "float64"),), + artifact_inputs=(edge,), + ) + ) + nodes.append( + Node( + "cal", + "calibrate.synthetic@1", + population="create", + inputs=(Slice(entity, ("u_out",)),), + artifact_inputs=(edge,) if writer == "probe" else (), + ) + ) + stage = _synthetic_stage(binding, ["u_out"]) + return _synthetic_stage_spec([binding], [stage], rows), nodes + + +@pytest.mark.parametrize("reader", ["calibration", "own_stage"]) +@pytest.mark.parametrize("entity", ["family", "person"]) +def test_precalibration_ancestry_stage_artifact_is_not_a_declared_output( + entity, reader +): + """Outputs are columns, so only a stage's own nodes may read its artifacts.""" + spec, nodes = _stage_artifact_case(entity, reader) + outcomes = _synthetic_outcomes(spec, nodes, {"u": "u-ref"}) + if reader == "own_stage": + assert outcomes == {"declaration": None, "composed": None} + return + message = ( + "Transport calibration ancestry: node 'cal' carries undeclared " + "pre-calibration stage output." + ) + assert outcomes == { + "declaration": (ValueError, message), + "composed": (ValueError, message), + } + + def _variant_calibration_case(post, feeds_variant): """A node after the central fit, and a variant calibration that may read it.""" tax = _synthetic_binding("tax", "person", ["tax_out"]) From c4e23bd1298e0527fca3c567cff2cebb01c2e8b5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 19:10:30 -0400 Subject: [PATCH 07/20] Refuse malformed declarations in the pre-calibration stage checks With stages listed, the declaration pass now validates every field it reads, on every declared node and rules binding, before either pass uses it. A malformed field refuses as "node 'x' has malformed " instead of raising KeyError or TypeError, or being read character by character. Read character by character, three spellings were accepted that their list form refuses: - a string `columns` hid a read of an undeclared stage output; - a string selection `path` hid selected data embedding an engine ref; - a string `sources` hid a hold-out source. A repeated node id hid the first node's reads, and a string binding `variables` matched stage outputs by substring. Resource selections follow activation's rules: text resource, list path, known encoding. Binding selections (binding and field) name both as text; engine-reference documents carry no field and are untouched. The guard without stages is unchanged, so I1 identity holds, malformed declarations included. Adds an example test that isolates the composed pass's artifact edges, which only the property test covered before. Co-Authored-By: Claude Opus 5.5 --- .../transport-precalibration-stages.added.md | 2 +- docs/transport-precalibration-stages.md | 22 ++ .../src/microcosm/build/transport/compose.py | 151 +++++++++++- .../shared/test_transport_composed_graph.py | 217 ++++++++++++++++++ 4 files changed, 380 insertions(+), 12 deletions(-) diff --git a/changelog.d/transport-precalibration-stages.added.md b/changelog.d/transport-precalibration-stages.added.md index 904155fd70..6abda401e7 100644 --- a/changelog.d/transport-precalibration-stages.added.md +++ b/changelog.d/transport-precalibration-stages.added.md @@ -1 +1 @@ -Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. +Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. With stages listed, a malformed node or binding field the checks read refuses, naming the node, instead of being read character by character. diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index 3302e72b98..cbfa1f3a44 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -53,6 +53,26 @@ measure. Each derived output must be produced by one of the stage's nodes upstream of calibration. The composed graph is checked when there is one, otherwise the declaration. +With stages listed, every field the checks read is validated before either +pass uses it, on every declared node and rules binding: + +- **Nodes.** `id` and `kernel` are text, and ids are distinct. `population`, + `base` and `rules_binding` are text or null. `params` is an object. Each + resource selection in it names a text resource, a list `path` and a known + encoding, as activation requires. Each binding selection (an object with + `binding` and `field`) names both as text. `inputs`, `outputs` and + `artifact_inputs` are lists of objects with a text `entity`, a list of text + `columns`, a text `column` and a text `producer` respectively. `sources` is + a list of text. +- **Rules bindings.** Each names a distinct text `id`, a text `entity` and + `rulespec_path`, and a list of text `variables`. +- **The calibration endpoint**, when given, is text. + +A malformed field refuses, naming the node: for example "node 'mid' has +malformed inputs". Read character by character, a string column, source or +path list would otherwise drop a read, a hold-out source or a selection, and +a repeated id would hide a node. The guard without stages is unchanged. + ## Which nodes belong to a stage In the declaration pass, a node belongs to a stage when its `rules_binding` is @@ -121,3 +141,5 @@ These are tested, as properties and examples, in selection that calibration reaches only through column reads. - With stages listed, a refusal names the first offending node in declaration order. +- With stages listed, a malformed declaration never admits. Every field the + checks read is validated first, and a malformed one refuses naming its node. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 9e4f5f777a..f1d30b934b 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -710,6 +710,135 @@ def _carries_engine_ref_inventory(value: object) -> bool: return decoded != value and _carries_engine_ref_inventory(decoded) +def _is_text(value: object) -> bool: + return isinstance(value, str) and bool(value) + + +def _is_list(value: object, item: object = None) -> bool: + """A JSON list (never a string), each item passing ``item`` when given.""" + return ( + not isinstance(value, str | bytes) + and isinstance(value, Sequence) + and (item is None or all(item(entry) for entry in value)) + ) + + +def _is_rows(value: object, **fields: object) -> bool: + """A list of objects whose named fields each pass their check.""" + return _is_list( + value, + lambda row: isinstance(row, Mapping) + and all(check(row.get(key)) for key, check in fields.items()), + ) + + +def _precalibration_params(params: object) -> bool: + """Whether every resource or binding selection the stage checks read is well formed. + + A resource selection names text, a list path and a known encoding, as + activation requires. A binding selection (an object with binding and + field) names both as text; engine-reference documents carry no field. + """ + if not isinstance(params, Mapping): + return False + for value in params.values(): + for selection in _precalibration_selections(value): + encoding = selection.get("encoding", "value") + if "resource" in selection and not ( + _is_text(selection["resource"]) + and _is_list(selection.get("path", ())) + and isinstance(encoding, str) + and encoding in _ENCODINGS + ): + return False + if ( + "binding" in selection + and "field" in selection + and not ( + _is_text(selection["binding"]) and _is_text(selection["field"]) + ) + ): + return False + return True + + +def _precalibration_bindings(spec: Mapping[str, object]) -> dict[str, Mapping]: + """The rules bindings, refusing any field the stage checks read malformed. + + The registry validates bindings in full only when it builds engines, after + these checks run; a string ``variables`` would otherwise be read character + by character. + """ + + bindings: dict[str, Mapping] = {} + for item in _sequence( + transport_resource(spec, "axiom_rules_bindings").get("bindings"), + "axiom_rules_bindings.bindings", + ): + row = _mapping(item, "rules binding") + name = row.get("id") + if not _is_text(name) or name in bindings: + raise ValueError("precalibration_stages needs distinct rules binding ids.") + if not ( + _is_text(row.get("entity")) + and _is_text(row.get("rulespec_path")) + and _is_list(row.get("variables"), _is_text) + ): + raise ValueError( + f"precalibration_stages needs rules binding {name!r} to name its " + "entity, rulespec_path and variables." + ) + bindings[name] = row + return bindings + + +def _precalibration_rows(document: Mapping) -> dict[str, Mapping]: + """The declared nodes, refusing any field the stage checks read malformed. + + The declaration pass runs before the nodes are parsed. Read character by + character, a string column, source or path list would drop a read, a + hold-out source or a selection, and a repeated id would hide a node. + """ + + rows: dict[str, Mapping] = {} + for item in _sequence(document.get("nodes"), "transport nodes"): + row = _mapping(item, "transport node") + name = row.get("id") + if not _is_text(name) or name in rows: + raise ValueError( + "Transport calibration ancestry: node ids must be distinct text." + ) + checks = { + "kernel": _is_text(row.get("kernel")), + # Null is the same as absent for the optional links. + **{ + field: row.get(field) is None or _is_text(row[field]) + for field in ("population", "base", "rules_binding") + }, + "params": _precalibration_params(row.get("params", {})), + "inputs": _is_rows( + row.get("inputs", ()), + entity=_is_text, + columns=lambda columns: _is_list(columns, _is_text), + ), + "outputs": _is_rows( + row.get("outputs", ()), entity=_is_text, column=_is_text + ), + "artifact_inputs": _is_rows( + row.get("artifact_inputs", ()), producer=_is_text + ), + "sources": _is_list(row.get("sources", ()), _is_text), + } + malformed = [field for field, valid in checks.items() if not valid] + if malformed: + raise ValueError( + f"Transport calibration ancestry: node {name!r} has malformed " + f"{', '.join(malformed)}." + ) + rows[name] = row + return rows + + def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: """Validate stage declarations before inspecting either graph.""" @@ -730,10 +859,7 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: not isinstance(entity, str) or not entity for entity in _unit_entities(spec) ): raise ValueError("precalibration_stages needs a named benefit-unit entity.") - bindings = { - row["id"]: row - for row in transport_resource(spec, "axiom_rules_bindings")["bindings"] - } + bindings = _precalibration_bindings(spec) holdout = ( transport_resource(spec, "holdout_references").get("target_references", ()) if _has_resource(spec, "holdout_references") @@ -867,11 +993,8 @@ def _validate_precalibration_ancestry( config: TransportGraphConfig | None, ) -> None: document = transport_graph_document(spec) - rows = {row["id"]: row for row in document["nodes"]} - bindings = { - row["id"]: row - for row in transport_resource(spec, "axiom_rules_bindings")["bindings"] - } + rows = _precalibration_rows(document) + bindings = _precalibration_bindings(spec) unit_entities = _unit_entities(spec) unit_bindings = { name: row for name, row in bindings.items() if row["entity"] in unit_entities @@ -904,8 +1027,14 @@ def _validate_precalibration_ancestry( roots = { name for name, row in rows.items() if row["kernel"].startswith("calibrate.") } - endpoint = document["endpoints"].get("calibration") - if isinstance(endpoint, str): + endpoint = _mapping(document.get("endpoints"), "transport endpoints").get( + "calibration" + ) + if endpoint is not None and not _is_text(endpoint): + raise ValueError( + "Transport calibration ancestry: the calibration endpoint must name a node." + ) + if endpoint is not None: roots.add(endpoint) predecessors = {name: set() for name in rows} producers = { diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 8f2d35951e..67a0c8924a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -3502,6 +3502,22 @@ def test_precalibration_ancestry_stage_artifact_is_not_a_declared_output( } +@pytest.mark.parametrize("entity", ["family", "person"]) +def test_precalibration_ancestry_composed_stage_artifact_read_refuses(entity): + """Only the composed graph reads the stage artifact, as an extension node can.""" + spec, nodes = _stage_artifact_case(entity, "calibration") + rows = spec["resources"]["transport_graph"]["nodes"] + del next(row for row in rows if row["id"] == "cal")["artifact_inputs"] + assert _synthetic_outcomes(spec, nodes, {"u": "u-ref"}) == { + "declaration": None, + "composed": ( + ValueError, + "Transport calibration ancestry: node 'cal' carries undeclared " + "pre-calibration stage output.", + ), + } + + def _variant_calibration_case(post, feeds_variant): """A node after the central fit, and a variant calibration that may read it.""" tax = _synthetic_binding("tax", "person", ["tax_out"]) @@ -4014,3 +4030,204 @@ def test_precalibration_ancestry_malformed_entities_match_today( ) == expected ) + + +def _shape_case(): + """A listed unit stage and one reader of its declared output, well formed.""" + binding = _synthetic_binding("u", "family", ["u_out", "u_priv"]) + rows = [ + { + "id": "probe", + "kernel": "synthetic.probe@1", + "population": "create", + "params": {"engine_ref": {"binding": "u", "field": "engine_ref"}}, + "outputs": [ + {"entity": "family", "column": "u_out"}, + {"entity": "family", "column": "u_priv"}, + ], + }, + { + "id": "mid", + "kernel": "synthetic.mid@1", + "population": "create", + "inputs": [{"entity": "family", "columns": ["u_out"]}], + "outputs": [{"entity": "person", "column": "mid_out"}], + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": "create", + "inputs": [{"entity": "person", "columns": ["mid_out"]}], + }, + ] + binding["engine_ref"] = "u-ref" + stage = _synthetic_stage(binding, ["u_out"]) + spec = _synthetic_stage_spec([binding], [stage], rows) + # Data a non-stage node may select; it embeds the stage's engine reference. + spec["resources"]["probe_refs"] = {"refs": {"u": "u-ref"}} + return spec + + +def _shape_rows(spec): + return {row["id"]: row for row in spec["resources"]["transport_graph"]["nodes"]} + + +def _shape_outcome(guard, spec): + """Any refusal, so today's guard compares even where it raises KeyError.""" + try: + guard(spec) + except Exception as error: + return type(error), str(error) + return None + + +#: A malformed field the declaration pass reads, by the node that carries it. +_MALFORMED_NODE_FIELDS = { + "string_columns": ("mid", "inputs", [{"entity": "family", "columns": "u_priv"}]), + "missing_columns": ("mid", "inputs", [{"entity": "family"}]), + "string_inputs": ("mid", "inputs", "u_out"), + "output_without_entity": ("mid", "outputs", [{"column": "mid_out"}]), + "edge_without_producer": ("cal", "artifact_inputs", [{"name": "trace"}]), + "string_edges": ("cal", "artifact_inputs", "probe"), + "missing_kernel": ("probe", "kernel", None), + "list_population": ("mid", "population", ["create"]), + "list_rules_binding": ("mid", "rules_binding", ["u"]), + "list_params": ("probe", "params", ["engine_ref"]), + "string_path": ( + "mid", + "params", + {"rows": {"resource": "probe_refs", "path": "refs"}}, + ), + "string_sources": ("mid", "sources", "synthetic_holdout_facts"), + "list_resource": ("mid", "params", {"rows": {"resource": ["holdout_references"]}}), + "unknown_encoding": ( + "mid", + "params", + {"rows": {"resource": "probe_refs", "encoding": ["sha256"]}}, + ), + "list_binding_selector": ( + "mid", + "params", + {"ref": {"binding": ["u"], "field": "engine_ref"}}, + ), +} + + +def test_precalibration_ancestry_shape_case_is_accepted(): + assert ( + _shape_outcome( + compose_module.validate_transport_calibration_ancestry, _shape_case() + ) + is None + ) + + +@pytest.mark.parametrize("case", sorted(_MALFORMED_NODE_FIELDS)) +def test_precalibration_ancestry_malformed_node_field_refuses_naming_the_node(case): + """With stages, a malformed field refuses before it can drop a read or edge.""" + node, field, value = _MALFORMED_NODE_FIELDS[case] + spec = _shape_case() + _shape_rows(spec)[node][field] = value + guard = compose_module.validate_transport_calibration_ancestry + assert _shape_outcome(guard, spec) == ( + ValueError, + f"Transport calibration ancestry: node {node!r} has malformed {field}.", + ) + # Without stages the guard is today's, malformed declarations included. + spec["resources"]["precalibration_stages"]["stages"] = [] + assert _shape_outcome(guard, spec) == _shape_outcome( + _precalibration_reference_guard, spec + ) + + +@pytest.mark.parametrize( + ("case", "value", "reason"), + [ + ( + "string_columns", + [{"entity": "family", "columns": ["u_priv"]}], + "undeclared pre-calibration stage output", + ), + ( + "string_path", + {"rows": {"resource": "probe_refs", "path": ["refs"]}}, + "not a declared pre-calibration stage", + ), + ("string_sources", ["synthetic_holdout_facts"], "hold-out source"), + ( + "list_resource", + {"rows": {"resource": "holdout_references"}}, + "holdout_references selection", + ), + ], +) +def test_precalibration_ancestry_malformed_spelling_hid_a_refusal(case, value, reason): + """Read character by character, these spellings used to be accepted.""" + node, field, _ = _MALFORMED_NODE_FIELDS[case] + spec = _shape_case() + _shape_rows(spec)[node][field] = value + assert _shape_outcome( + compose_module.validate_transport_calibration_ancestry, spec + ) == ( + ValueError, + f"Transport calibration ancestry: node {node!r} carries {reason}.", + ) + + +def test_precalibration_ancestry_repeated_node_id_refuses(): + """A repeated id would hide the first node's read of an undeclared output.""" + spec = _shape_case() + rows = spec["resources"]["transport_graph"]["nodes"] + _shape_rows(spec)["mid"]["inputs"] = [{"entity": "family", "columns": ["u_priv"]}] + rows.insert( + rows.index(_shape_rows(spec)["cal"]), + {"id": "mid", "kernel": "synthetic.mid@1", "population": "create"}, + ) + assert _shape_outcome( + compose_module.validate_transport_calibration_ancestry, spec + ) == ( + ValueError, + "Transport calibration ancestry: node ids must be distinct text.", + ) + + +@pytest.mark.parametrize( + ("case", "change"), + [ + ("string_variables", {"variables": "u_out_total"}), + ("list_entity", {"entity": ["family"]}), + ("missing_rulespec_path", {"rulespec_path": None}), + ], +) +def test_precalibration_ancestry_malformed_binding_refuses(case, change): + """A string variable list would match stage outputs by substring.""" + spec = _shape_case() + (binding,) = spec["resources"]["axiom_rules_bindings"]["bindings"] + binding.update(change) + assert _shape_outcome( + compose_module.validate_transport_calibration_ancestry, spec + ) == ( + ValueError, + "precalibration_stages needs rules binding 'u' to name its entity, " + "rulespec_path and variables.", + ) + + +def test_precalibration_ancestry_repeated_binding_id_refuses(): + spec = _shape_case() + bindings = spec["resources"]["axiom_rules_bindings"]["bindings"] + bindings.append(_synthetic_binding("u", "person", ["other"])) + assert _shape_outcome( + compose_module.validate_transport_calibration_ancestry, spec + ) == (ValueError, "precalibration_stages needs distinct rules binding ids.") + + +def test_precalibration_ancestry_malformed_calibration_endpoint_refuses(): + spec = _shape_case() + spec["resources"]["transport_graph"]["endpoints"]["calibration"] = ["cal"] + assert _shape_outcome( + compose_module.validate_transport_calibration_ancestry, spec + ) == ( + ValueError, + "Transport calibration ancestry: the calibration endpoint must name a node.", + ) From 20dfb3e21f2266dcf2ddc742cecfd3667693dcc2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 22:15:43 -0400 Subject: [PATCH 08/20] Attribute stage reads through population versions; check extension selections Fixes the five findings of the round-2 review of c4e23bd12. - Rule 3 follows each read to its producer, as the compiler resolves it: the owner in the reader's population version, or the nearest base version for an inherited column. A structural node reads its base, and a rewrite reads its incumbent from below its version. A second stage's own column of the same name in a later version no longer refuses as the first stage's private output. The composed pass resolves exactly; the declaration pass falls back to every declared producer where a version is unknown (no population, an undeclared version, a base cycle). - The composed pass reads entitlement rows' declared selections (entitlement._rows), so a variant target compiler selecting holdout_references refuses. Those rows need distinct text ids and well-formed params. - The prepared engine_refs inventory is detected in every string (escaped braces included) and in selected resource data. - An explicit null calibration endpoint refuses. - The doc states that shape refusals come first, and that the guard without stages misses entitlement-row hold-out selections. The P1 generator now draws population versions with shared column names. All 32 mutants are killed by the example tests alone, including rule 3 without version attribution. Co-Authored-By: Claude Opus 5.5 --- .../transport-precalibration-stages.added.md | 2 +- docs/transport-precalibration-stages.md | 64 +- .../src/microcosm/build/transport/compose.py | 223 +++++-- .../shared/test_transport_composed_graph.py | 563 ++++++++++++++++-- 4 files changed, 755 insertions(+), 97 deletions(-) diff --git a/changelog.d/transport-precalibration-stages.added.md b/changelog.d/transport-precalibration-stages.added.md index 6abda401e7..b1bb8960ec 100644 --- a/changelog.d/transport-precalibration-stages.added.md +++ b/changelog.d/transport-precalibration-stages.added.md @@ -1 +1 @@ -Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. With stages listed, a malformed node or binding field the checks read refuses, naming the node, instead of being read character by character. +Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. A stage's undeclared outputs are restricted by producer: each read follows the reader's population-version lineage, so a column of the same name that another node owns in another version is not the stage's. With stages listed, a malformed node or binding field the checks read refuses, naming the node, instead of being read character by character; entitlement rows' selections are checked too, and the prepared engine-reference map is refused in any JSON spelling and in selected resource data. diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index cbfa1f3a44..dde1d79b03 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -2,10 +2,11 @@ A transport graph's calibration may read only what the calibration ancestry guard admits (`validate_transport_calibration_ancestry` in -`microcosm.build.transport.compose`). Without stages, the guard refuses every -benefit-unit engine semantic upstream of a `calibrate.*` node or the -calibration endpoint, and every hold-out source, hold-out selection and the -graph resource's own digest. Person-entity rules bindings are admitted. +`microcosm.build.transport.compose`). Without stages, the guard refuses +benefit-unit engine semantics upstream of a `calibrate.*` node or the +calibration endpoint, hold-out sources, `holdout_references` selections in the +`transport_graph` rows it reaches, and the graph resource's own digest. +Person-entity rules bindings are admitted. The optional `precalibration_stages` resource declares the rules bindings whose outputs feed calibration, and exactly which of their outputs flow on: @@ -66,7 +67,12 @@ pass uses it, on every declared node and rules binding: a list of text. - **Rules bindings.** Each names a distinct text `id`, a text `entity` and `rulespec_path`, and a list of text `variables`. -- **The calibration endpoint**, when given, is text. +- **The calibration endpoint**, when present, is text. An explicit null + refuses too: it would drop a calibration node whose kernel is not + `calibrate.*`. +- **Entitlement rows**, in the composed pass. Each row the entitlement + extension instantiates has a text id that no skeleton row or other + entitlement row uses, and its `params` follow the rules above. A malformed field refuses, naming the node: for example "node 'mid' has malformed inputs". Read character by character, a string column, source or @@ -102,18 +108,35 @@ variant calibration reads is in that variant's ancestry, so it is checked. - A node outside a listed stage that reads a column one of the stage's own nodes produces upstream of calibration, other than its `outputs` and `derived_outputs`, refuses as "undeclared pre-calibration stage output". - Reads among a stage's own nodes do not count. The rule follows the producing - node, not the column name, so another binding that declares the same variable - name is unaffected. + Reads among a stage's own nodes do not count. +- The rule follows each read to its producer, as the compiler resolves it: + the owner of the column in the reader's population version or, for an + inherited column, the owner in the nearest base version. A structural node + reads its base, and a rewrite receives the incumbent from below its version, + so both count as reads. Another binding that declares the same variable + name, or another node that owns a column of the same name in another + version, is therefore unaffected. +- The composed pass resolves reads exactly. Where a declaration leaves the + version unknown, the declaration pass counts every declared producer of the + column instead: a reader with no population, a version that is not declared + and a cycle of bases all fall back this way. A declared producer with no + population counts in every version. - Outputs are columns, so a stage's artifacts are never declared. A node outside the stage that reads an artifact one of the stage's nodes produces upstream of calibration refuses the same way. - A parameter that carries the prepared `engine_refs` inventory refuses as "prepared engine_refs map", on stage nodes too. The inventory may be a - mapping, a JSON string in any spelling, nested in lists or mappings, or JSON - embedded in a longer string. It is bookkeeping, not a stage invocation. + mapping, a JSON string in any spelling (braces written as escapes + included), nested in lists or mappings, JSON embedded in a longer string, + or the data a resource selection names. It is bookkeeping, not a stage + invocation. - The hold-out, selection and digest checks still apply to every upstream node, - stage nodes included, so each stage's ancestry is clean. + stage nodes included, so each stage's ancestry is clean. They read each + node's declared selections: its `transport_graph` row or, in the composed + pass, the entitlement row the extension instantiated it from, whose + parameters are resolved before the graph exists. A code extension's node has + no declaration, so its compiled parameters get the engine-reference and + inventory checks but not the selection checks. Both passes apply these rules; the composed pass covers compiled predecessors and extension nodes. No node parameter or hash changes. @@ -135,11 +158,18 @@ These are tested, as properties and examples, in guard without stages, which can admit more: its declaration pass follows structural and artifact edges, not column reads, and a resource selection leaves no marker in the composed graph. -- With stages listed, no hold-out source or selection is upstream of - calibration, and no stage output or stage module carries a hold-out measure. - The guard without stages is unchanged, so it still misses a hold-out - selection that calibration reaches only through column reads. -- With stages listed, a refusal names the first offending node in declaration - order. +- With stages listed, no hold-out source and no declared hold-out selection + (in a `transport_graph` or entitlement row) is upstream of calibration, and + no stage output or stage module carries a hold-out measure. The guard + without stages is unchanged, so it still misses a hold-out selection that + calibration reaches only through column reads, and any hold-out selection + in an entitlement row. +- With stages listed, the outcome is deterministic. Shape checks run first, + over every declared node and binding, so a malformed declaration refuses + naming its first malformed node even when an earlier node would refuse on + substance. Otherwise a refusal names the first offending node in + declaration order. The composed pass runs only after the declaration pass + admits: it checks the entitlement rows' shapes, then names the first + offending node in graph order. - With stages listed, a malformed declaration never admits. Every field the checks read is validated first, and a malformed one refuses naming its node. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index f1d30b934b..15a9c96b79 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -74,6 +74,7 @@ ArtifactInput, ArtifactOutput, ArtifactType, + CompiledGraph, Graph, Node, Owned, @@ -691,7 +692,8 @@ def _carries_engine_ref_inventory(value: object) -> bool: ) if isinstance(value, list | tuple): return any(_carries_engine_ref_inventory(item) for item in value) - if not isinstance(value, str) or "{" not in value: + # Every string is decoded: a JSON string can spell its braces as escapes. + if not isinstance(value, str): return False try: decoded = json.loads(value) @@ -986,6 +988,144 @@ def _precalibration_selections(value: object) -> Iterator[Mapping]: yield from _precalibration_selections(item) +def _precalibration_extension_rows( + spec: Mapping[str, object], rows: Mapping[str, Mapping] +) -> dict[str, Mapping]: + """The entitlement rows the extension instantiates, by node id. + + Their parameters are resolved before the composed graph exists, so the + composed pass reads their selections here, as it reads a skeleton row's. + """ + + if not ( + _has_resource(spec, "entitlement_graph") and _has_resource(spec, "scenarios") + ): + return {} + from .entitlement import _rows + + result: dict[str, Mapping] = {} + for row, _ in _rows(spec)[0]: + name = row.get("id") + if not _is_text(name) or name in rows or name in result: + raise ValueError( + "Transport calibration ancestry: node ids must be distinct text." + ) + if not _precalibration_params(row.get("params", {})): + raise ValueError( + f"Transport calibration ancestry: node {name!r} has malformed params." + ) + result[name] = row + return result + + +def _precalibration_declared_reads( + rows: Mapping[str, Mapping], +) -> dict[str, set[tuple[str, str, str]]]: + """Each declared node's column reads, as (entity, column, producer). + + A read resolves in the reader's population version and then down its base + chain, as the compiler resolves it: a structural node reads its base, and + a rewrite receives the incumbent from below its version. Where the + declaration leaves the version unknown (no population, an undeclared + version or a cycle), every declared producer of the column counts, and a + producer without a population counts in every version. + """ + + owners: dict[tuple[object, str, str], set[str]] = {} + producers: dict[tuple[str, str], set[str]] = {} + for name, row in rows.items(): + version = row.get("population") if row.get("base") is None else None + for output in row.get("outputs", ()): + key = (output["entity"], output["column"]) + producers.setdefault(key, set()).add(name) + owners.setdefault((version, *key), set()).add(name) + + def resolve(version: object, key: tuple[str, str]) -> set[str]: + if not isinstance(version, str): + return producers.get(key, set()) + floating = owners.get((None, *key), set()) + seen = set() + while isinstance(version, str): + if version in seen or version not in rows: + return producers.get(key, set()) + seen.add(version) + if (version, *key) in owners: + return owners[(version, *key)] | floating + version = rows[version].get("base") + return floating + + reads: dict[str, set[tuple[str, str, str]]] = {} + for name, row in rows.items(): + base = row.get("base") + start = row.get("population") if base is None else base + holder = rows.get(row.get("population")) + # A rewrite's incumbent comes from below its version. + incumbent = None if holder is None else holder.get("base") + rewritten = { + (output["entity"], output["column"]) + for output in row.get("outputs", ()) + if output.get("rewrite") + } + keys = { + (item["entity"], column) + for item in row.get("inputs", ()) + for column in item["columns"] + } + reads[name] = { + (*key, producer) + for key in keys | rewritten + for producer in resolve(incumbent if key in rewritten else start, key) + if producer != name + } + return reads + + +def _precalibration_compiled_reads( + graph: Graph, compiled: CompiledGraph +) -> dict[str, set[tuple[str, str, str]]]: + """Each compiled node's column reads, as (entity, column, producer). + + The producer is the owner in the reader's version or, for an inherited + column, in the nearest base version that owns it. A structural node reads + its base; a rewrite receives the incumbent from below its version. + """ + + by_id = {node.id: node for node in graph.nodes} + + def resolve(version: str | None, entity: str, column: str) -> str | None: + while version is not None: + owner = compiled.owners.get((version, entity, column)) + if owner is not None: + return owner + version = by_id[version].base + return None + + reads: dict[str, set[tuple[str, str, str]]] = {} + for node in graph.nodes: + version = compiled.versions[node.id] + start = version if node.structural is StructuralDelta.NONE else node.base + rewritten = { + (item.entity, item.column) for item in node.outputs if item.rewrite + } + keys = { + (item.entity, column) for item in node.inputs for column in item.columns + } + reads[node.id] = { + (entity, column, producer) + for entity, column in keys | rewritten + if ( + producer := resolve( + by_id[version].base if (entity, column) in rewritten else start, + entity, + column, + ) + ) + is not None + and producer != node.id + } + return reads + + def _validate_precalibration_ancestry( spec: Mapping[str, object], stages: Mapping[str, Mapping], @@ -1027,22 +1167,17 @@ def _validate_precalibration_ancestry( roots = { name for name, row in rows.items() if row["kernel"].startswith("calibrate.") } - endpoint = _mapping(document.get("endpoints"), "transport endpoints").get( - "calibration" - ) - if endpoint is not None and not _is_text(endpoint): - raise ValueError( - "Transport calibration ancestry: the calibration endpoint must name a node." - ) - if endpoint is not None: - roots.add(endpoint) + endpoints = _mapping(document.get("endpoints"), "transport endpoints") + # Present means a node: an explicit null would drop the only root. + if "calibration" in endpoints: + if not _is_text(endpoints["calibration"]): + raise ValueError( + "Transport calibration ancestry: the calibration endpoint must " + "name a node." + ) + roots.add(endpoints["calibration"]) predecessors = {name: set() for name in rows} - producers = { - (row.get("population"), output["entity"], output["column"]): name - for name, row in rows.items() - if isinstance(row.get("population"), str) - for output in row.get("outputs", ()) - } + reads = _precalibration_declared_reads(rows) for name, row in rows.items(): base = row.get("base") if isinstance(base, str): @@ -1055,24 +1190,14 @@ def _validate_precalibration_ancestry( predecessors[name].update( edge["producer"] for edge in row.get("artifact_inputs", ()) ) - predecessors[name].update( - producer - for item in row.get("inputs", ()) - for column in item["columns"] - if ( - producer := producers.get( - (row.get("population"), item["entity"], column) - ) - ) - is not None - and producer != name - ) + predecessors[name].update(producer for _, _, producer in reads[name]) declarations = rows def check( rows: Mapping[str, Mapping], predecessors: Mapping, + reads: Mapping[str, set[tuple[str, str, str]]], *, composed: bool, derived: bool, @@ -1104,8 +1229,6 @@ def check( # stage's ancestry passes the same checks; a stage binding's nodes # after calibration (extension checkpoints) are outputs, not inputs. ancestors = _ancestor_closure(predecessors, roots) - # A stage restricts only what its own upstream nodes produce: a column - # of the same name from elsewhere is not the stage's output. produced = { binding: { (item["entity"], item["column"]) @@ -1125,12 +1248,23 @@ def check( f"{stage['id']!r} is not produced by a stage node upstream " "of calibration." ) - restricted = { - binding: produced[binding] - - stage_columns[binding] - - derived_columns[binding] + # A stage restricts only what its own upstream nodes produce: each read + # follows its producer in the reader's version lineage, so a column of + # the same name from another node or version is not the stage's output. + public = { + binding: stage_columns[binding] | derived_columns[binding] for binding in stages } + consumed = { + name: { + binding + for entity, column, producer in reads[name] + if producer in ancestors + for binding in node_bindings.get(producer, ()) + if binding in stages and (entity, column) not in public[binding] + } + for name in rows + } inputs = { name: { (item["entity"], column) @@ -1223,6 +1357,8 @@ def check( selected = None break selected = selected[component] + if _carries_engine_ref_inventory(selected): + reason = "prepared engine_refs map" if _embeds_engine_ref( selected, forbidden_refs, @@ -1253,12 +1389,8 @@ def check( reason = reason or "not a declared pre-calibration stage" elif outputs & encoded: reason = reason or "not a declared pre-calibration stage" - for binding in stages: - if binding not in own and ( - inputs[name] & restricted[binding] - or binding in artifact_stages[name] - ): - reason = "undeclared pre-calibration stage output" + if (consumed[name] | artifact_stages[name]) - own: + reason = "undeclared pre-calibration stage output" if _embeds_engine_ref( row.get("params", {}), forbidden_refs, @@ -1278,9 +1410,10 @@ def check( # A derived output is checked against the fullest view: the composed graph # when there is one, otherwise the declaration. - check(rows, predecessors, composed=False, derived=graph is None) + check(rows, predecessors, reads, composed=False, derived=graph is None) if graph is None: return + declarations = {**rows, **_precalibration_extension_rows(spec, rows)} compiled = compile_graph(graph) roots.update( node.id for node in graph.nodes if node.kernel.startswith("calibrate.") @@ -1302,7 +1435,13 @@ def check( } for node in graph.nodes } - check(compiled_rows, compiled.predecessors, composed=True, derived=True) + check( + compiled_rows, + compiled.predecessors, + _precalibration_compiled_reads(graph, compiled), + composed=True, + derived=True, + ) def validate_transport_calibration_ancestry( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 67a0c8924a..4ea2c89d1a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -106,6 +106,9 @@ make_composed_fixture, toy_extension, ) +from test_support.microcosm_build.transport_entitlement import ( + make_entitlement_fixture, +) from test_support.microcosm_build.transport_graph import descendants from test_support.paths import paths_for @@ -2785,15 +2788,44 @@ def unavailable(*args, **kwargs): _PRECALIBRATION_PROPERTY_DOMAINS = { "mixed": { - "kinds": ("encode", "unit", "person", "selector", "holdout"), + "kinds": ("encode", "unit", "person", "selector", "holdout", "filter"), "holdout_sources": True, }, # Without hold-out data most graphs pass the other checks, so the # producer attribution of rule 3 is exercised in every run. - "rules_only": {"kinds": ("person", "selector"), "holdout_sources": False}, + "rules_only": { + "kinds": ("person", "selector", "filter"), + "holdout_sources": False, + }, } +def _precalibration_edges(draw, nodes, version, visible): + """Dependencies, reads and artifact edges from earlier nodes. + + A read names the producer the column resolves to in ``version``. + """ + dependencies, reads, artifacts = [], [], [] + for predecessor, earlier in enumerate(nodes): + if earlier["kind"] == "filter": + continue + edge = draw(st.integers(min_value=0, max_value=3)) + if edge == 1: + dependencies.append(predecessor) + artifacts.append(predecessor) + if edge > 1: + columns = [ + column + for column in earlier["outputs"] + if visible(version, column) == predecessor + ] + columns = columns if edge == 3 else columns[:1] + if columns: + dependencies.append(predecessor) + reads.extend((predecessor, column) for column in columns) + return dependencies, reads, artifacts + + @st.composite def _precalibration_ancestry_dags(draw, kinds, holdout_sources): """Small DAGs with independent binding declarations and column edges. @@ -2801,11 +2833,66 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources): A new binding may also declare another binding's variable name, which only that other binding's node produces; selector nodes add a derived column that is not a binding variable; any binding node may carry hold-out data. + A filter node opens a population version on an earlier one. A binding node + may own a ``shared`` column that a node in another version also owns, and + each read names the producer it resolves to in the reader's version. """ kinds = draw(st.lists(st.sampled_from(kinds), min_size=1, max_size=7)) nodes = [] bindings = {} + # Version None is CREATE; a filter's index names its version. A version a + # filter branches from takes no new members, so structural edges point back. + bases = {None: None} + closed = set() + owned = {} + # Versions a read resolved through: a later owner there would re-route it. + frozen = set() + + def visible(version, column): + while (version, column) not in owned: + if version is None: + return None + version = bases[version] + return owned[version, column] + + def freeze(version, column): + while (version, column) not in owned: + frozen.add((version, column)) + version = bases[version] + + def member_version(): + return draw(st.sampled_from(sorted(set(bases) - closed, key=str))) + for index, kind in enumerate(kinds): + if kind == "filter": + base = draw(st.sampled_from(sorted(bases, key=str))) + bases[index] = base + closed.add(base) + nodes.append( + { + "kind": kind, + "binding": None, + "entity": "person", + "version": None, + "base": base, + "outputs": [], + "reads": [], + # A version depends on its base and every member of it. + "dependencies": [ + *([] if base is None else [base]), + *( + member + for member, row in enumerate(nodes) + if row["kind"] != "filter" and row["version"] == base + ), + ], + "artifacts": [], + "holdout_selection": False, + "holdout_source": False, + } + ) + continue + version = member_version() binding = None entity = "family" if kind in {"encode", "unit"} else "person" if kind in {"unit", "person", "selector"}: @@ -2827,33 +2914,41 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources): bindings[binding] = {"entity": entity, "variables": [], "derived": []} if shared and draw(st.booleans()): bindings[binding]["variables"].append(draw(st.sampled_from(shared))) + dependencies, reads, artifacts = _precalibration_edges( + draw, nodes, version, visible + ) + for _, column in reads: + freeze(version, column) outputs = [f"value_{index}"] if binding is not None: outputs.append(f"private_{index}") - bindings[binding]["variables"].extend(outputs) + # A node may not own a column it reads, one its version owns, or + # one an earlier read in its version resolved below it. + common = f"shared_{entity}" + if (version, common) not in owned.keys() | frozen and draw(st.booleans()): + outputs.append(common) + bindings[binding]["variables"].extend( + column + for column in outputs + if column not in bindings[binding]["variables"] + ) if kind == "selector": outputs.append(f"derived_{index}") bindings[binding]["derived"].append(f"derived_{index}") - reads = [] - dependencies = [] - artifacts = [] - for predecessor, earlier in enumerate(nodes): - edge = draw(st.integers(min_value=0, max_value=3)) - if edge: - dependencies.append(predecessor) - if edge == 1: - artifacts.append(predecessor) - if edge > 1: - columns = earlier["outputs"] if edge == 3 else earlier["outputs"][:1] - reads.extend((predecessor, column) for column in columns) + for column in outputs: + owned[version, column] = index nodes.append( { "kind": kind, "binding": binding, "entity": entity, + "version": version, "outputs": outputs, "reads": reads, - "dependencies": dependencies, + "dependencies": [ + *([] if version is None else [version]), + *dependencies, + ], "artifacts": artifacts, "holdout_selection": kind == "holdout" and draw(st.booleans()), "holdout_source": holdout_sources @@ -2868,26 +2963,19 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources): declared[binding] = draw( st.sets(st.sampled_from(row["variables"] + row["derived"]), min_size=1) ) - dependencies = [] - reads = [] - artifacts = [] - for predecessor, earlier in enumerate(nodes): - edge = draw(st.integers(min_value=0, max_value=3)) - if edge: - dependencies.append(predecessor) - if edge == 1: - artifacts.append(predecessor) - if edge > 1: - columns = earlier["outputs"] if edge == 3 else earlier["outputs"][:1] - reads.extend((predecessor, column) for column in columns) + version = member_version() + dependencies, reads, artifacts = _precalibration_edges( + draw, nodes, version, visible + ) nodes.append( { "kind": "calibrate", "binding": None, "entity": "person", + "version": version, "outputs": [], "reads": reads, - "dependencies": dependencies, + "dependencies": [*([] if version is None else [version]), *dependencies], "artifacts": artifacts, "holdout_selection": False, "holdout_source": False, @@ -2984,8 +3072,31 @@ def _precalibration_synthetic_graph(model): "outputs": [{"entity": "person", "column": "seed"}], } ] + + def version_id(version): + return "p1.create" if version is None else f"p1.node_{version}" + for index, item in enumerate(nodes): name = "p1.calibrate" if item["kind"] == "calibrate" else f"p1.node_{index}" + if item["kind"] == "filter": + base = version_id(item["base"]) + graph_nodes.append( + Node( + name, + "synthetic.filter@1", + structural=StructuralDelta.FILTER, + base=base, + ) + ) + rows.append( + { + "id": name, + "kernel": "synthetic.filter@1", + "structural": "filter", + "base": base, + } + ) + continue kernel = { "encode": "concepts.encode_groups@1", "unit": RULES, @@ -3021,7 +3132,7 @@ def _precalibration_synthetic_graph(model): Node( name, kernel, - population="p1.create", + population=version_id(item["version"]), inputs=tuple( Slice(entity, tuple(columns)) for entity, columns in inputs.items() ), @@ -3038,7 +3149,7 @@ def _precalibration_synthetic_graph(model): row = { "id": name, "kernel": kernel, - "population": "p1.create", + "population": version_id(item["version"]), "inputs": [ {"entity": entity, "columns": columns} for entity, columns in inputs.items() @@ -3141,8 +3252,10 @@ def _precalibration_property_outcomes(model): "population": "p1.create", }, ] - # A selected hold-out resource leaves no marker in compiled params. Keep - # those source declarations so this path still checks their provenance. + # These compiled nodes stand for a code extension's, which have no + # declaration, and a selected hold-out resource leaves no marker in + # compiled params. Keep those rows so this path still checks their + # provenance. (Entitlement rows are declarations: see the example tests.) extension_spec["resources"]["transport_graph"]["nodes"].extend( row for row, node in zip( @@ -3610,6 +3723,8 @@ def _precalibration_inventory(engine_refs, encoding): "spaced": json.dumps(inventory, indent=2), "nested": json.dumps({"receipt": json.dumps(inventory)}), "embedded": f"inventory={json.dumps(inventory)};", + # A JSON string spelling every brace as an escape: no literal brace. + "escaped": json.dumps(canonical).replace("{", "\\u007b"), }[encoding] @@ -3619,6 +3734,7 @@ def _precalibration_inventory(engine_refs, encoding): "spaced", "nested", "embedded", + "escaped", ] @@ -3648,7 +3764,9 @@ def test_precalibration_ancestry_compiled_prepared_map_still_refuses( ) -@pytest.mark.parametrize("encoding", ["string", "spaced", "nested", "embedded"]) +@pytest.mark.parametrize( + "encoding", ["string", "spaced", "nested", "embedded", "escaped"] +) def test_precalibration_ancestry_declared_prepared_map_refuses( calibration_ancestry_composed, encoding ): @@ -3840,6 +3958,228 @@ def test_precalibration_ancestry_stage_output_read_by_another_stage_refuses(): } +def _version_member(name, kernel, version, *, binding=None, inputs=(), outputs=()): + """A declaration row and its node in a population version. + + ``outputs`` are ``(column, rewrite)`` pairs on the person entity. + """ + row = {"id": name, "kernel": kernel, "population": version} + params = {} + if binding is not None: + row["rules_binding"] = binding + params["engine_ref"] = f"{binding}-ref" + if inputs: + row["inputs"] = [{"entity": "person", "columns": list(inputs)}] + if outputs: + row["outputs"] = [ + {"entity": "person", "column": column, "rewrite": rewrite} + for column, rewrite in outputs + ] + node = Node( + name, + kernel, + population=version, + params=params, + inputs=(Slice("person", tuple(inputs)),) if inputs else (), + outputs=tuple( + Owned("person", column, "float64", rewrite=rewrite) + for column, rewrite in outputs + ), + ) + return row, node + + +def _version_filter(name, base, *, inputs=()): + row = {"id": name, "kernel": "synthetic.filter@1", "structural": "filter"} + row["base"] = base + if inputs: + row["inputs"] = [{"entity": "person", "columns": list(inputs)}] + node = Node( + name, + "synthetic.filter@1", + structural=StructuralDelta.FILTER, + base=base, + inputs=(Slice("person", tuple(inputs)),) if inputs else (), + ) + return row, node + + +def _version_case(shape): + """Stages A and B each own a person.shared, in different population versions. + + A produces its private ``shared`` in CREATE. A FILTER makes ``v2``, where + B's ``b`` reads ``shared`` and publishes only ``B_public`` to calibration. + The shape says where the ``shared`` that ``b`` reads comes from. + """ + bindings = [ + _synthetic_binding(name, "person", ["shared", f"{name}_public"]) + for name in ("A", "B") + ] + stages = [_synthetic_stage(row, [f"{row['id']}_public"]) for row in bindings] + deep = shape.startswith("deep_") + reader = "v3" if deep else "v2" + members = [ + _version_member( + "a_source", + RULES, + "create", + binding="A", + outputs=(("shared", False), ("A_public", False)), + ), + _version_filter( + "v2", + "create", + inputs=("shared",) if shape.startswith("structural") else (), + ), + ] + if shape in {"own", "structural", "deep_own"}: + # B's own column of the same name, in a version above A's. + members.append( + _version_member( + "b_source", RULES, "v2", binding="B", outputs=(("shared", False),) + ) + ) + if shape in {"rewrite", "structural_shadowed"}: + # A node outside both stages owns v2's shared: as a rewrite, its + # incumbent is A's private column; otherwise it shadows A's. + members.append( + _version_member( + "adjust", + "synthetic.adjust@1", + "v2", + outputs=(("shared", shape == "rewrite"),), + ) + ) + if deep: + members.append(_version_filter("v3", "v2")) + members.extend( + [ + _version_member( + "b", + RULES, + reader, + binding="B", + inputs=("shared",), + outputs=(("B_public", False),), + ), + _version_member( + "cal", "calibrate.synthetic@1", reader, inputs=("B_public",) + ), + ] + ) + rows, nodes = zip(*members, strict=True) + return _synthetic_stage_spec(bindings, stages, list(rows)), list(nodes) + + +@pytest.mark.parametrize("declared", [True, False], ids=["declared", "compiled_only"]) +@pytest.mark.parametrize( + ("shape", "refuses"), + [ + ("own", None), + ("deep_own", None), + ("inherited", "b"), + ("deep_inherited", "b"), + ("rewrite", "adjust"), + ("structural", "v2"), + ("structural_shadowed", "v2"), + ], +) +def test_precalibration_ancestry_reads_follow_population_versions( + shape, refuses, declared +): + """Rule 3 follows each read to its producer in the reader's version lineage. + + B's own ``shared`` in a later version is not A's private column of the same + name. A read that resolves to A's column refuses: an inherited read (also + two versions up), a rewrite's incumbent and a structural node's input, which + comes from its base even when its own version owns a column of that name. + ``compiled_only`` declares just calibration, as for extension nodes, so the + composed pass decides alone. + """ + spec, nodes = _version_case(shape) + if not declared: + spec["resources"]["transport_graph"]["nodes"][1:] = [ + {"id": "cal", "kernel": "calibrate.synthetic@1", "population": "create"} + ] + outcome = ( + None + if refuses is None + else ( + ValueError, + f"Transport calibration ancestry: node {refuses!r} carries undeclared " + "pre-calibration stage output.", + ) + ) + outcomes = _synthetic_outcomes(spec, nodes, {"A": "A-ref", "B": "B-ref"}) + assert outcomes == { + "declaration": outcome if declared else None, + "composed": outcome, + } + + +@pytest.mark.parametrize("shape", ["inherited", "own"]) +def test_precalibration_ancestry_producer_without_a_version_counts_everywhere(shape): + """A declared producer with no population may be in the reader's version. + + It counts beside the reader's own version's owner too (``own``): the + declaration alone cannot place it. + """ + spec, nodes = _version_case(shape) + rows = {row["id"]: row for row in spec["resources"]["transport_graph"]["nodes"]} + del rows["a_source"]["population"] + message = ( + "Transport calibration ancestry: node 'b' carries undeclared " + "pre-calibration stage output." + ) + outcomes = _synthetic_outcomes(spec, nodes, {"A": "A-ref", "B": "B-ref"}) + assert outcomes == { + "declaration": (ValueError, message), + "composed": (ValueError, message), + } + + +def test_precalibration_ancestry_cyclic_versions_count_every_producer(): + """A declared base cycle never resolves, so every producer counts.""" + spec, _ = _version_case("inherited") + rows = spec["resources"]["transport_graph"]["nodes"] + {row["id"]: row for row in rows}["v2"]["base"] = "v3" + rows.append( + { + "id": "v3", + "kernel": "synthetic.filter@1", + "structural": "filter", + "base": "v2", + } + ) + assert _precalibration_outcome( + compose_module.validate_transport_calibration_ancestry, spec + ) == ( + ValueError, + "Transport calibration ancestry: node 'b' carries undeclared " + "pre-calibration stage output.", + ) + + +def test_precalibration_ancestry_unresolved_version_counts_every_producer(): + """A declared read with no version counts every producer of its column. + + The composed graph still places ``b`` in ``v2``; the declaration alone + cannot, so it refuses (and a composed run checks the declaration first). + """ + spec, nodes = _version_case("own") + rows = {row["id"]: row for row in spec["resources"]["transport_graph"]["nodes"]} + del rows["b"]["population"] + message = ( + "Transport calibration ancestry: node 'b' carries undeclared " + "pre-calibration stage output." + ) + outcomes = _synthetic_outcomes(spec, nodes, {"A": "A-ref", "B": "B-ref"}) + assert outcomes == { + "declaration": (ValueError, message), + "composed": (ValueError, message), + } + + def _derived_case(*, derived, outputs=(), producer_upstream=True): """A person-binding gross-up solve whose output is not a binding variable. @@ -4222,12 +4562,161 @@ def test_precalibration_ancestry_repeated_binding_id_refuses(): ) == (ValueError, "precalibration_stages needs distinct rules binding ids.") -def test_precalibration_ancestry_malformed_calibration_endpoint_refuses(): +@pytest.mark.parametrize("endpoint", [["cal"], None, ""]) +def test_precalibration_ancestry_malformed_calibration_endpoint_refuses(endpoint): + """A present endpoint names a node: null would drop a non-calibrate root.""" spec = _shape_case() - spec["resources"]["transport_graph"]["endpoints"]["calibration"] = ["cal"] + rows = _shape_rows(spec) + rows["cal"]["kernel"] = "synthetic.fit@1" + rows["mid"]["sources"] = ["synthetic_holdout_facts"] + guard = compose_module.validate_transport_calibration_ancestry + assert _shape_outcome(guard, spec) == ( + ValueError, + "Transport calibration ancestry: node 'mid' carries hold-out source.", + ) + spec["resources"]["transport_graph"]["endpoints"]["calibration"] = endpoint + assert _shape_outcome(guard, spec) == ( + ValueError, + "Transport calibration ancestry: the calibration endpoint must name a node.", + ) + + +@pytest.mark.parametrize("node", ["probe", "mid"]) +def test_precalibration_ancestry_selected_prepared_map_refuses(node): + """Selected resource data carrying the inventory is the inventory.""" + spec = _shape_case() + spec["resources"]["probe_inventory"] = {"engine_refs": {"u": "u-ref"}} + _shape_rows(spec)[node].setdefault("params", {})["inventory"] = { + "resource": "probe_inventory", + "encoding": "json", + } assert _shape_outcome( compose_module.validate_transport_calibration_ancestry, spec ) == ( ValueError, - "Transport calibration ancestry: the calibration endpoint must name a node.", + f"Transport calibration ancestry: node {node!r} carries prepared " + "engine_refs map.", + ) + + +@pytest.fixture(scope="module") +def precalibration_entitlement(tmp_path_factory): + """The entitlement fixture, with real extension rows and variant fits.""" + from microcosm.build.transport import registry as registry_module + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(registry_module, "register_gate_kernel", lambda *args: None) + return make_entitlement_fixture( + tmp_path_factory.mktemp("precalibration-entitlement") + ) + + +def _entitlement_stage_spec(fixture, *, listed=True): + """The fixture's spec, listing its person main-benefit binding as a stage.""" + spec = copy.deepcopy(fixture.spec) + resources = spec["resources"] + binding = next( + row + for row in resources["axiom_rules_bindings"]["bindings"] + if row["id"] == "main_benefit" + ) + resources["precalibration_stages"] = { + "schema_version": 1, + "stages": [ + { + "id": "stage.main", + "rules_binding": binding["id"], + "rulespec_path": binding["rulespec_path"], + "outputs": [ + { + "entity": binding["entity"], + "column": variable, + "purpose": "calibration_measure", + } + for variable in binding["variables"] + ], + "citation": "Synthetic test stage.", + } + ] + if listed + else [], + } + return spec + + +def _entitlement_variant_targets(spec): + """V1's target compiler: an extension row feeding the V1 variant fit.""" + scenario = next( + row for row in spec["resources"]["scenarios"]["scenarios"] if row["id"] == "V1" ) + return next(row for row in scenario["nodes"] if row["id"] == "nz.v.V1.targets") + + +def _select_holdout(row, encoding): + key = "references" if encoding == "json" else "references_sha256" + row["params"][key] = {"resource": "holdout_references", "encoding": encoding} + row["inputs"] = [{"entity": "person", "columns": ["receives_main"]}] + + +@pytest.mark.parametrize("encoding", ["json", "sha256"]) +def test_precalibration_ancestry_extension_holdout_selection_refuses( + precalibration_entitlement, encoding +): + """An entitlement row's selection is read from its declaration. + + Its parameters are resolved before the composed graph exists, so the + resolved node carries no marker of the hold-out resource it selected. + """ + fixture = precalibration_entitlement + spec = _entitlement_stage_spec(fixture) + compose_transport_graph(spec, fixture.config) + _select_holdout(_entitlement_variant_targets(spec), encoding) + with pytest.raises( + ValueError, + match="node 'nz.v.V1.targets' carries holdout_references selection", + ): + compose_transport_graph(spec, fixture.config) + + +def test_precalibration_ancestry_extension_selection_without_stages_is_todays( + precalibration_entitlement, +): + fixture = precalibration_entitlement + spec = _entitlement_stage_spec(fixture, listed=False) + _select_holdout(_entitlement_variant_targets(spec), "json") + graph = compose_transport_graph(spec, fixture.config) + assert _precalibration_outcome( + compose_module.validate_transport_calibration_ancestry, + spec, + graph=graph, + config=fixture.config, + ) == _precalibration_outcome( + _precalibration_reference_guard, spec, graph=graph, config=fixture.config + ) + + +@pytest.mark.parametrize( + ("change", "message"), + [ + ( + {"params": {"references": {"resource": ["holdout_references"]}}}, + "node 'nz.v.V1.targets' has malformed params.", + ), + ({"id": "nz.targets.compile"}, "node ids must be distinct text."), + ], + ids=["malformed_params", "repeated_id"], +) +def test_precalibration_ancestry_malformed_extension_row_refuses( + precalibration_entitlement, change, message +): + """A standalone check refuses an extension row it cannot read.""" + fixture = precalibration_entitlement + spec = _entitlement_stage_spec(fixture) + graph = compose_transport_graph(spec, fixture.config) + _entitlement_variant_targets(spec).update(change) + assert _precalibration_outcome( + compose_module.validate_transport_calibration_ancestry, + spec, + graph=graph, + config=fixture.config, + ) == (ValueError, f"Transport calibration ancestry: {message}") From c05685cdef41e42f476d30a9b737fd8f6d1f27ed Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 22:28:09 -0400 Subject: [PATCH 09/20] Name the node in id refusals; correct the stage doc's field and scope lines From the NZ hub's review of c4e23bd12 (m2), the two doc points still open at 20dfb3e21: - A non-text or repeated node id now refuses naming it ("node id None is not text", "node 'x' is declared twice"), in the declaration and in the entitlement rows, so "a malformed field refuses, naming the node" holds. Rules binding ids refuse the same way. - The doc's validation list maps each list to its own fields, and its intro states the guard without stages checks selections only in the rows its declaration pass reaches through population, base and artifact links. - The guard's docstring says which view the derived-output check uses. Co-Authored-By: Claude Opus 5.5 --- docs/transport-precalibration-stages.md | 17 ++++++---- .../src/microcosm/build/transport/compose.py | 32 +++++++++++++++---- .../shared/test_transport_composed_graph.py | 30 ++++++++++++++--- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index dde1d79b03..b03d1ca637 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -4,8 +4,10 @@ A transport graph's calibration may read only what the calibration ancestry guard admits (`validate_transport_calibration_ancestry` in `microcosm.build.transport.compose`). Without stages, the guard refuses benefit-unit engine semantics upstream of a `calibrate.*` node or the -calibration endpoint, hold-out sources, `holdout_references` selections in the -`transport_graph` rows it reaches, and the graph resource's own digest. +calibration endpoint, hold-out sources, and `holdout_references` selections +and the graph resource's own digest in the `transport_graph` rows its +declaration pass reaches through population, base and artifact links. It does +not follow column reads, and its composed pass checks no selections. Person-entity rules bindings are admitted. The optional `precalibration_stages` resource declares the rules bindings whose @@ -61,10 +63,10 @@ pass uses it, on every declared node and rules binding: `base` and `rules_binding` are text or null. `params` is an object. Each resource selection in it names a text resource, a list `path` and a known encoding, as activation requires. Each binding selection (an object with - `binding` and `field`) names both as text. `inputs`, `outputs` and - `artifact_inputs` are lists of objects with a text `entity`, a list of text - `columns`, a text `column` and a text `producer` respectively. `sources` is - a list of text. + `binding` and `field`) names both as text. `inputs` is a list of objects + with a text `entity` and a list of text `columns`; `outputs` is a list of + objects with a text `entity` and a text `column`; `artifact_inputs` is a + list of objects with a text `producer`. `sources` is a list of text. - **Rules bindings.** Each names a distinct text `id`, a text `entity` and `rulespec_path`, and a list of text `variables`. - **The calibration endpoint**, when present, is text. An explicit null @@ -75,7 +77,8 @@ pass uses it, on every declared node and rules binding: entitlement row uses, and its `params` follow the rules above. A malformed field refuses, naming the node: for example "node 'mid' has -malformed inputs". Read character by character, a string column, source or +malformed inputs", "node 'mid' is declared twice" or "node id None is not +text". Read character by character, a string column, source or path list would otherwise drop a read, a hold-out source or a selection, and a repeated id would hide a node. The guard without stages is unchanged. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 15a9c96b79..8d962acf12 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -779,8 +779,15 @@ def _precalibration_bindings(spec: Mapping[str, object]) -> dict[str, Mapping]: ): row = _mapping(item, "rules binding") name = row.get("id") - if not _is_text(name) or name in bindings: - raise ValueError("precalibration_stages needs distinct rules binding ids.") + if not _is_text(name): + raise ValueError( + f"precalibration_stages needs text rules binding ids, not {name!r}." + ) + if name in bindings: + raise ValueError( + f"precalibration_stages needs distinct rules binding ids; {name!r} " + "repeats." + ) if not ( _is_text(row.get("entity")) and _is_text(row.get("rulespec_path")) @@ -806,9 +813,13 @@ def _precalibration_rows(document: Mapping) -> dict[str, Mapping]: for item in _sequence(document.get("nodes"), "transport nodes"): row = _mapping(item, "transport node") name = row.get("id") - if not _is_text(name) or name in rows: + if not _is_text(name): raise ValueError( - "Transport calibration ancestry: node ids must be distinct text." + f"Transport calibration ancestry: node id {name!r} is not text." + ) + if name in rows: + raise ValueError( + f"Transport calibration ancestry: node {name!r} is declared twice." ) checks = { "kernel": _is_text(row.get("kernel")), @@ -1006,9 +1017,13 @@ def _precalibration_extension_rows( result: dict[str, Mapping] = {} for row, _ in _rows(spec)[0]: name = row.get("id") - if not _is_text(name) or name in rows or name in result: + if not _is_text(name): + raise ValueError( + f"Transport calibration ancestry: node id {name!r} is not text." + ) + if name in rows or name in result: raise ValueError( - "Transport calibration ancestry: node ids must be distinct text." + f"Transport calibration ancestry: node {name!r} is declared twice." ) if not _precalibration_params(row.get("params", {})): raise ValueError( @@ -1461,7 +1476,10 @@ def validate_transport_calibration_ancestry( Benefit-unit attributes remain allowed; excluded columns are the unit rules bindings' outputs and the group encoder's declared outputs. Declared pre-calibration stages admit their own unit semantics, restrict - outputs flowing onward and recursively check each stage's ancestry. + outputs flowing onward and recursively check each stage's ancestry. With + stages and a graph, each derived output must be produced in the composed + graph; without a graph, in the declaration. Composition always runs the + graph-less check first. """ stages = _precalibration_stages(spec) if not _has_resource(spec, GRAPH_RESOURCE): diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 4ea2c89d1a..67d8474f73 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -4527,7 +4527,19 @@ def test_precalibration_ancestry_repeated_node_id_refuses(): compose_module.validate_transport_calibration_ancestry, spec ) == ( ValueError, - "Transport calibration ancestry: node ids must be distinct text.", + "Transport calibration ancestry: node 'mid' is declared twice.", + ) + + +@pytest.mark.parametrize("name", [None, ["mid"], 3]) +def test_precalibration_ancestry_non_text_node_id_refuses(name): + spec = _shape_case() + _shape_rows(spec)["mid"]["id"] = name + assert _shape_outcome( + compose_module.validate_transport_calibration_ancestry, spec + ) == ( + ValueError, + f"Transport calibration ancestry: node id {name!r} is not text.", ) @@ -4553,13 +4565,21 @@ def test_precalibration_ancestry_malformed_binding_refuses(case, change): ) -def test_precalibration_ancestry_repeated_binding_id_refuses(): +@pytest.mark.parametrize( + ("name", "message"), + [ + ("u", "needs distinct rules binding ids; 'u' repeats."), + (None, "needs text rules binding ids, not None."), + ], + ids=["repeated", "non_text"], +) +def test_precalibration_ancestry_binding_id_refuses(name, message): spec = _shape_case() bindings = spec["resources"]["axiom_rules_bindings"]["bindings"] - bindings.append(_synthetic_binding("u", "person", ["other"])) + bindings.append({**_synthetic_binding("u", "person", ["other"]), "id": name}) assert _shape_outcome( compose_module.validate_transport_calibration_ancestry, spec - ) == (ValueError, "precalibration_stages needs distinct rules binding ids.") + ) == (ValueError, f"precalibration_stages {message}") @pytest.mark.parametrize("endpoint", [["cal"], None, ""]) @@ -4702,7 +4722,7 @@ def test_precalibration_ancestry_extension_selection_without_stages_is_todays( {"params": {"references": {"resource": ["holdout_references"]}}}, "node 'nz.v.V1.targets' has malformed params.", ), - ({"id": "nz.targets.compile"}, "node ids must be distinct text."), + ({"id": "nz.targets.compile"}, "node 'nz.targets.compile' is declared twice."), ], ids=["malformed_params", "repeated_id"], ) From 3cb81726d388df30232e1b20effd4454d4ffa378 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 00:03:03 -0400 Subject: [PATCH 10/20] Draw rewrites, filter reads and shadowed names in the stage property A property-only mutant pass at 20dfb3e21 killed 9 of 32 mutants, and missed rule 3 without version attribution: the generator drew a graph that tells it from the reference about once in 2,000 examples. It never drew a rewrite or a structural node's inputs. The generator now draws: - plain nodes, outside any stage, that may own a shared column; - a random single column for a one-column read, not the first; - rewrites by person nodes in a filter version of a person column visible from below, which the reference counts as reads of the incumbent; - filter inputs, resolved in the filter's base; - a third domain, "versions" (person, filter and plain nodes), where shared names, rewrites and stage listings are likely, so version attribution decides most outcomes. The property alone now kills 14 of 32 mutants. In ten property-only runs it killed the version-blind rule 3, a compiled rewrite resolved at its own version and compiled structural reads at their own version every time, and rewrites ignored in either pass nine times. The example tests still kill all 32. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_transport_composed_graph.py | 131 +++++++++++++++--- 1 file changed, 110 insertions(+), 21 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 67d8474f73..1ed6c45084 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -2788,15 +2788,23 @@ def unavailable(*args, **kwargs): _PRECALIBRATION_PROPERTY_DOMAINS = { "mixed": { - "kinds": ("encode", "unit", "person", "selector", "holdout", "filter"), + "kinds": ("encode", "unit", "person", "selector", "holdout", "filter", "plain"), "holdout_sources": True, }, # Without hold-out data most graphs pass the other checks, so the # producer attribution of rule 3 is exercised in every run. "rules_only": { - "kinds": ("person", "selector", "filter"), + "kinds": ("person", "selector", "filter", "plain"), "holdout_sources": False, }, + # Population versions with shared names: a column a stage leaves + # undeclared is often owned again in another version, read there or + # rewritten, so version attribution decides most outcomes. + "versions": { + "kinds": ("person", "filter", "plain"), + "holdout_sources": False, + "eager": True, + }, } @@ -2819,7 +2827,8 @@ def _precalibration_edges(draw, nodes, version, visible): for column in earlier["outputs"] if visible(version, column) == predecessor ] - columns = columns if edge == 3 else columns[:1] + if columns and edge == 2: + columns = [draw(st.sampled_from(columns))] if columns: dependencies.append(predecessor) reads.extend((predecessor, column) for column in columns) @@ -2827,17 +2836,23 @@ def _precalibration_edges(draw, nodes, version, visible): @st.composite -def _precalibration_ancestry_dags(draw, kinds, holdout_sources): +def _precalibration_ancestry_dags(draw, kinds, holdout_sources, eager=False): """Small DAGs with independent binding declarations and column edges. A new binding may also declare another binding's variable name, which only that other binding's node produces; selector nodes add a derived column that is not a binding variable; any binding node may carry hold-out data. - A filter node opens a population version on an earlier one. A binding node - may own a ``shared`` column that a node in another version also owns, and - each read names the producer it resolves to in the reader's version. + A filter node opens a population version on an earlier one and may read + columns from its base. A binding or plain node may own a ``shared`` column + that a node in another version also owns, and a person node in a filter + version may rewrite a person column it sees from below. Each read and + rewrite names the producer it resolves to: in the reader's version, in a + filter's base, or below a rewrite's version. """ - kinds = draw(st.lists(st.sampled_from(kinds), min_size=1, max_size=7)) + kinds = draw( + st.lists(st.sampled_from(kinds), min_size=1, max_size=9 if eager else 7) + ) + likely = st.sampled_from((True, True, True, False)) if eager else st.booleans() nodes = [] bindings = {} # Version None is CREATE; a filter's index names its version. A version a @@ -2845,6 +2860,7 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources): bases = {None: None} closed = set() owned = {} + entities = {} # Versions a read resolved through: a later owner there would re-route it. frozen = set() @@ -2863,9 +2879,30 @@ def freeze(version, column): def member_version(): return draw(st.sampled_from(sorted(set(bases) - closed, key=str))) + def seen(version): + return sorted( + {column for _, column in owned if visible(version, column) is not None} + ) + for index, kind in enumerate(kinds): if kind == "filter": base = draw(st.sampled_from(sorted(bases, key=str))) + # A structural node reads its inputs from its base. + columns = seen(base) + chosen = ( + draw( + st.lists( + st.sampled_from(columns), + unique=True, + min_size=1 if eager else 0, + max_size=2, + ) + ) + if columns + else [] + ) + for column in chosen: + freeze(base, column) bases[index] = base closed.add(base) nodes.append( @@ -2876,7 +2913,8 @@ def member_version(): "version": None, "base": base, "outputs": [], - "reads": [], + "reads": [(visible(base, column), column) for column in chosen], + "rewrites": [], # A version depends on its base and every member of it. "dependencies": [ *([] if base is None else [base]), @@ -2922,11 +2960,42 @@ def member_version(): outputs = [f"value_{index}"] if binding is not None: outputs.append(f"private_{index}") - # A node may not own a column it reads, one its version owns, or - # one an earlier read in its version resolved below it. - common = f"shared_{entity}" - if (version, common) not in owned.keys() | frozen and draw(st.booleans()): - outputs.append(common) + # A node may not own a column it reads, one its version owns, or one an + # earlier read in its version resolved below it. + taken = owned.keys() | frozen + common = f"shared_{entity}" + if ( + (binding is not None or kind == "plain") + and (version, common) not in taken + and draw(likely) + ): + outputs.append(common) + rewrites = [] + if version is not None and entity == "person": + # The incumbent comes from below the version; CREATE has none. A + # rewritten column joins the node's binding variables, which a + # derived output may never be. + columns = [ + column + for column in seen(bases[version]) + if entities[column] == "person" + and not column.startswith("derived_") + and (version, column) not in taken + and column not in outputs + and column not in {read for _, read in reads} + ] + if columns and draw(likely): + for column in draw( + st.lists( + st.sampled_from(columns), + unique=True, + min_size=1, + max_size=2 if eager else 1, + ) + ): + rewrites.append((visible(bases[version], column), column)) + outputs.append(column) + if binding is not None: bindings[binding]["variables"].extend( column for column in outputs @@ -2937,6 +3006,7 @@ def member_version(): bindings[binding]["derived"].append(f"derived_{index}") for column in outputs: owned[version, column] = index + entities.setdefault(column, entity) nodes.append( { "kind": kind, @@ -2945,6 +3015,7 @@ def member_version(): "version": version, "outputs": outputs, "reads": reads, + "rewrites": rewrites, "dependencies": [ *([] if version is None else [version]), *dependencies, @@ -2958,7 +3029,7 @@ def member_version(): ) declared = {} for binding, row in bindings.items(): - if draw(st.booleans()): + if draw(likely): # Derived columns alone are a complete declaration. declared[binding] = draw( st.sets(st.sampled_from(row["variables"] + row["derived"]), min_size=1) @@ -2975,6 +3046,7 @@ def member_version(): "version": version, "outputs": [], "reads": reads, + "rewrites": [], "dependencies": [*([] if version is None else [version]), *dependencies], "artifacts": artifacts, "holdout_selection": False, @@ -3034,7 +3106,8 @@ def _precalibration_reference_accepts(model): for consumer in consumers ): return False - for producer, column in node["reads"]: + # A rewrite receives its incumbent, so it is a read too. + for producer, column in node["reads"] + node["rewrites"]: source_binding = nodes[producer]["binding"] if ( source_binding in declared @@ -3078,6 +3151,9 @@ def version_id(version): for index, item in enumerate(nodes): name = "p1.calibrate" if item["kind"] == "calibrate" else f"p1.node_{index}" + inputs = {} + for producer, column in item["reads"]: + inputs.setdefault(nodes[producer]["entity"], []).append(column) if item["kind"] == "filter": base = version_id(item["base"]) graph_nodes.append( @@ -3086,6 +3162,10 @@ def version_id(version): "synthetic.filter@1", structural=StructuralDelta.FILTER, base=base, + inputs=tuple( + Slice(entity, tuple(columns)) + for entity, columns in inputs.items() + ), ) ) rows.append( @@ -3094,6 +3174,10 @@ def version_id(version): "kernel": "synthetic.filter@1", "structural": "filter", "base": base, + "inputs": [ + {"entity": entity, "columns": columns} + for entity, columns in inputs.items() + ], } ) continue @@ -3103,11 +3187,10 @@ def version_id(version): "person": RULES, "selector": "simulate.solve_zero@1", "holdout": "synthetic.holdout@1", + "plain": "synthetic.plain@1", "calibrate": "calibrate.synthetic@1", }[item["kind"]] - inputs = {} - for producer, column in item["reads"]: - inputs.setdefault(nodes[producer]["entity"], []).append(column) + rewritten = {column for _, column in item["rewrites"]} binding = item["binding"] params = {} prepared_params = {} @@ -3137,7 +3220,9 @@ def version_id(version): Slice(entity, tuple(columns)) for entity, columns in inputs.items() ), outputs=tuple( - Owned(item["entity"], column, "float64") + Owned( + item["entity"], column, "float64", rewrite=column in rewritten + ) for column in item["outputs"] ), params=prepared_params, @@ -3155,7 +3240,11 @@ def version_id(version): for entity, columns in inputs.items() ], "outputs": [ - {"entity": item["entity"], "column": column} + { + "entity": item["entity"], + "column": column, + **({"rewrite": True} if column in rewritten else {}), + } for column in item["outputs"] ], "params": params, From bfd5e85d0032bec334adf09a7d3296e15356304e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 01:21:18 -0400 Subject: [PATCH 11/20] Say which values the stage checks read as composition does The NZ hub's sign-off at 3cb81726d noted that the doc's "every field the checks read is validated first" overstates the code. Three values get no shape check, because the checks read them exactly as composition does: an output's rewrite flag, by truthiness like the compiler; a selection path, through mappings only, which composition also refuses otherwise; and a prepared value, which counts only as "engine_refs", the one prepared parameter composition admits. The doc now says so. No code changes. Co-Authored-By: Claude Opus 5.5 --- docs/transport-precalibration-stages.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index b03d1ca637..9eed432cc6 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -57,7 +57,8 @@ upstream of calibration. The composed graph is checked when there is one, otherwise the declaration. With stages listed, every field the checks read is validated before either -pass uses it, on every declared node and rules binding: +pass uses it, on every declared node and rules binding, except the three +values below that the checks read as composition does: - **Nodes.** `id` and `kernel` are text, and ids are distinct. `population`, `base` and `rules_binding` are text or null. `params` is an object. Each @@ -76,6 +77,14 @@ pass uses it, on every declared node and rules binding: extension instantiates has a text id that no skeleton row or other entitlement row uses, and its `params` follow the rules above. +Three values get no shape check of their own, because the checks read them +exactly as composition does. An output's `rewrite` flag is read by +truthiness, as the compiler reads it. A selection `path` is followed through +mappings only, as composition follows it: a path that leaves the resource's +mappings selects nothing for the checks, and composition refuses it. A +`prepared` value counts only when it is `"engine_refs"`, the one prepared +parameter composition admits. + A malformed field refuses, naming the node: for example "node 'mid' has malformed inputs", "node 'mid' is declared twice" or "node id None is not text". Read character by character, a string column, source or @@ -175,4 +184,5 @@ These are tested, as properties and examples, in admits: it checks the entitlement rows' shapes, then names the first offending node in graph order. - With stages listed, a malformed declaration never admits. Every field the - checks read is validated first, and a malformed one refuses naming its node. + checks read is validated first or read exactly as composition reads it, + and a malformed one refuses naming its node. From ebe6f3d1af466d3ee62f40e24e2bba0e23c21d89 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 03:22:35 -0400 Subject: [PATCH 12/20] Follow every writer the executor tracks; scan the inventory in linear passes Fixes the five findings of the round-3 review of 3cb81726d. - Rule 3 follows each read to every writer whose values it receives, as the executor's writer chain does (executor._writers_of). An owner that keeps earlier values passes on the writers below it: a rewrite, a masked owner (an output with a rows mask) and an EXPAND claim. An EXPAND writes the cells it installs, and a claim reads them without a Slice. A masked owner of a stage's private column no longer hides the values it leaves outside its mask, and a claim forwarding an EXPAND's private cell refuses. Both passes walk the chain; the composed pass reads EXPAND cells and claims from the compiled params as the executor does, and a malformed one refuses. The declaration reads literal expand_cells and materialized_expand_outputs and otherwise falls back: unknown cells write every column and produce none; non-literal claims claim every non-rewrite output. - A derived output may be produced by an EXPAND's cells. - With stages listed, a node's structural kind and its outputs' rows masks are shape-checked. - The prepared engine_refs inventory is a quoted "engine_refs" key and a colon, found after decoding JSON escapes layer by layer. Each layer is one linear pass, so a long harmless string costs one pass, not the quadratic suffix scan; the scan no longer mistakes plain text naming engine_refs for the map. Decoding stops after 32 layers, and a string still escaped then counts as carrying the map, so a crafted chain of escaped backslashes (a pass per layer, quadratic) refuses in at most 33 passes. JSON encoding doubles the backslashes per layer, so no encoded spelling comes near 32. - The non-text extension-row id refusal has a test. - I4 says which message is not fixed: with several dangling links, which one a refusal names follows set iteration, as without stages. The composed pass's writers are checked against the executor's own static writer lists (executor._input_writers): on every P1 graph, the writer and EXPAND examples, and the composed NZ and entitlement graphs. The P1 generator gains a "writers" domain: masked owners, EXPAND stage nodes whose cells a claim owns, and reads that name every writer. A focused mode keeps refusals to about one cause: in 3,000 draws, about 0.6% refuse only through a masked owner's later writer, and 5% only through a claim's read of its cell. Co-Authored-By: Claude Opus 5.5 --- .../transport-precalibration-stages.added.md | 2 +- docs/transport-precalibration-stages.md | 109 ++- .../src/microcosm/build/transport/compose.py | 327 +++++-- .../shared/test_transport_composed_graph.py | 850 ++++++++++++++++-- 4 files changed, 1085 insertions(+), 203 deletions(-) diff --git a/changelog.d/transport-precalibration-stages.added.md b/changelog.d/transport-precalibration-stages.added.md index b1bb8960ec..bbfa773ab9 100644 --- a/changelog.d/transport-precalibration-stages.added.md +++ b/changelog.d/transport-precalibration-stages.added.md @@ -1 +1 @@ -Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. A stage's undeclared outputs are restricted by producer: each read follows the reader's population-version lineage, so a column of the same name that another node owns in another version is not the stage's. With stages listed, a malformed node or binding field the checks read refuses, naming the node, instead of being read character by character; entitlement rows' selections are checked too, and the prepared engine-reference map is refused in any JSON spelling and in selected resource data. +Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. A stage's undeclared outputs are restricted by writer: each read follows the executor's writer chain through the reader's population-version lineage, where rewrites, masked owners and EXPAND claims pass earlier writers' values on and an EXPAND writes its cells, so a column of the same name that another node owns in another version is not the stage's. With stages listed, a malformed node or binding field the checks read refuses, naming the node, instead of being read character by character; entitlement rows' selections are checked too, and the prepared engine-reference map is refused in any JSON spelling, with escapes decoded layer by layer in one linear pass each, and in selected resource data. diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index 9eed432cc6..5b73a838eb 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -53,21 +53,26 @@ the same closed shape and purposes as an `outputs` row. It names any entity, may not repeat a binding variable on the binding's entity (that belongs in `outputs`), and may not repeat any other output of the stage or a hold-out measure. Each derived output must be produced by one of the stage's nodes -upstream of calibration. The composed graph is checked when there is one, -otherwise the declaration. +upstream of calibration: a node produces the columns it owns and, for an +EXPAND, the cells it installs. The composed graph is checked when there is +one, otherwise the declaration. With stages listed, every field the checks read is validated before either -pass uses it, on every declared node and rules binding, except the three -values below that the checks read as composition does: +pass uses it, on every declared node and rules binding, except three values +that the checks read as composition does and two EXPAND parameters they read +conservatively, both described below: - **Nodes.** `id` and `kernel` are text, and ids are distinct. `population`, - `base` and `rules_binding` are text or null. `params` is an object. Each - resource selection in it names a text resource, a list `path` and a known - encoding, as activation requires. Each binding selection (an object with - `binding` and `field`) names both as text. `inputs` is a list of objects - with a text `entity` and a list of text `columns`; `outputs` is a list of - objects with a text `entity` and a text `column`; `artifact_inputs` is a - list of objects with a text `producer`. `sources` is a list of text. + `base` and `rules_binding` are text or null, and `structural`, when + present, is one of `none`, `create`, `filter`, `expand` and `reweight`. + `params` is an object. Each resource selection in it names a text + resource, a list `path` and a known encoding, as activation requires. Each + binding selection (an object with `binding` and `field`) names both as + text. `inputs` is a list of objects with a text `entity` and a list of text + `columns`; `outputs` is a list of objects with a text `entity`, a text + `column` and, when present, a text `rows` mask (absent is every row); + `artifact_inputs` is a list of objects with a text `producer`. `sources` is + a list of text. - **Rules bindings.** Each names a distinct text `id`, a text `entity` and `rulespec_path`, and a list of text `variables`. - **The calibration endpoint**, when present, is text. An explicit null @@ -85,6 +90,17 @@ mappings selects nothing for the checks, and composition refuses it. A `prepared` value counts only when it is `"engine_refs"`, the one prepared parameter composition admits. +Two EXPAND parameters are read as the executor reads them when the +declaration spells them out, and otherwise conservatively. An EXPAND's +`expand_cells` names its cells when it is a list of `[entity, column, dtype]` +texts; any other value, such as a resource selection, leaves the cells to +composition, so the declaration pass counts that EXPAND as a writer of every +column read through its version, and as producing no cell. +A node's `materialized_expand_outputs` claims the cells it lists when it is a +list of `entity.column` texts; any other value claims every output that is not +a rewrite. The composed pass reads both from the compiled parameters exactly +as the executor does, and a malformed one refuses there. + A malformed field refuses, naming the node: for example "node 'mid' has malformed inputs", "node 'mid' is declared twice" or "node id None is not text". Read character by character, a string column, source or @@ -117,31 +133,49 @@ variant calibration reads is in that variant's ancestry, so it is checked. stage nodes. Any other benefit-unit semantic upstream refuses as "not a declared pre-calibration stage". - A listed person stage admits nothing new; it only adds the next rule. -- A node outside a listed stage that reads a column one of the stage's own - nodes produces upstream of calibration, other than its `outputs` and - `derived_outputs`, refuses as "undeclared pre-calibration stage output". +- A node outside a listed stage that reads a column whose values one of the + stage's own nodes wrote upstream of calibration, other than its `outputs` + and `derived_outputs`, refuses as "undeclared pre-calibration stage output". Reads among a stage's own nodes do not count. -- The rule follows each read to its producer, as the compiler resolves it: - the owner of the column in the reader's population version or, for an - inherited column, the owner in the nearest base version. A structural node - reads its base, and a rewrite receives the incumbent from below its version, - so both count as reads. Another binding that declares the same variable +- The rule follows each read to every writer whose values it receives, as the + executor tracks them. In the reader's population version that is the owner + of the column. An owner that keeps earlier values passes on the writers + below it too: a rewrite receives the incumbent, a masked owner (an output + with a `rows` mask) leaves the incumbent's values on the rows outside its + mask, and an EXPAND claim receives the EXPAND's values. While the version + owns the column nowhere, or its owner keeps earlier values, the version's + EXPAND counts when it installs the column as a cell, and the walk goes on to + the base version, down to CREATE. A full owner that is not a rewrite or + claim stops it. +- A structural node reads its inputs from its base. A rewrite reads the + column it rewrites, and a claim, a node that names an EXPAND cell in its + `materialized_expand_outputs`, reads that cell, both without a `Slice`. + Each counts as a read. Another binding that declares the same variable name, or another node that owns a column of the same name in another version, is therefore unaffected. -- The composed pass resolves reads exactly. Where a declaration leaves the - version unknown, the declaration pass counts every declared producer of the +- The composed pass resolves writers exactly. Where a declaration leaves the + version unknown, the declaration pass counts every declared writer of the column instead: a reader with no population, a version that is not declared - and a cycle of bases all fall back this way. A declared producer with no - population counts in every version. + and a cycle of bases all fall back this way. A declared owner without a + population version (a node with no population, such as CREATE, or a + structural node's own outputs) counts in every version. - Outputs are columns, so a stage's artifacts are never declared. A node outside the stage that reads an artifact one of the stage's nodes produces upstream of calibration refuses the same way. - A parameter that carries the prepared `engine_refs` inventory refuses as - "prepared engine_refs map", on stage nodes too. The inventory may be a - mapping, a JSON string in any spelling (braces written as escapes - included), nested in lists or mappings, JSON embedded in a longer string, - or the data a resource selection names. It is bookkeeping, not a stage - invocation. + "prepared engine_refs map", on stage nodes too. A mapping carries it when it + has an `engine_refs` key or any key or value carries it, and a list when + any item does. A string carries it when it holds the quoted key + `"engine_refs"` followed by a colon, as written or once JSON escapes are + decoded, layer by layer, up to 32 layers; a string still escaped after 32 + layers counts as carrying it. So the inventory may be a JSON string in any + spelling (braces or the key written as escapes included), nested in + strings (each layer of JSON encoding doubles the backslashes, so encoding + never comes near 32 layers), JSON embedded in a longer string, or the data + a resource selection names. Text that only names engine_refs, without the + quoted key and colon, is not the inventory. Each layer is one linear pass + over the string, so the check takes at most 33 passes. The inventory is + bookkeeping, not a stage invocation. - The hold-out, selection and digest checks still apply to every upstream node, stage nodes included, so each stage's ancestry is clean. They read each node's declared selections: its `transport_graph` row or, in the composed @@ -181,8 +215,17 @@ These are tested, as properties and examples, in naming its first malformed node even when an earlier node would refuse on substance. Otherwise a refusal names the first offending node in declaration order. The composed pass runs only after the declaration pass - admits: it checks the entitlement rows' shapes, then names the first - offending node in graph order. -- With stages listed, a malformed declaration never admits. Every field the - checks read is validated first or read exactly as composition reads it, - and a malformed one refuses naming its node. + admits: it checks the entitlement rows' shapes and reads the compiled + EXPAND cells and claims, then names the first offending node in graph + order. One message is not fixed: when several links name undeclared + nodes, which of them a refusal names follows set iteration order, as in + the guard without stages. +- With stages listed, the checks never read a malformed field so as to admit + more than composition would. Every field they read is validated first, + refusing naming its node, or read exactly as composition reads it. The two + EXPAND parameters are read conservatively in the declaration and exactly in + the composed pass, where a malformed one refuses. +- The composed pass gives every read the writers the executor's static + writer lists give it (`microcosm.graph.executor._input_writers`), on every + generated graph, the writer and EXPAND examples, and the composed NZ and + entitlement graphs. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 8d962acf12..12982d3516 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -56,6 +56,7 @@ import hashlib import json +import re from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass, replace from importlib.resources import files @@ -71,11 +72,13 @@ from microcosm.build.ledger_artifact import CHRONICLE_FEED_RESOURCE from microcosm.calibrate.ordered_kernels import OUTPUT_TYPES from microcosm.graph import ( + ROWS_ALL, ArtifactInput, ArtifactOutput, ArtifactType, CompiledGraph, Graph, + GraphError, Node, Owned, Ownership, @@ -86,6 +89,10 @@ compile_graph, ) from microcosm.graph.canonical import canonical_json +from microcosm.graph.expansion import ( + declared_expand_cells, + materialized_expand_coordinates, +) from .artifact_types import KERNEL_OUTPUTS from .column_kernels import validate_receipt_contract @@ -681,8 +688,34 @@ def _is_unit_entity(entity: object, unit_entities: tuple[object, ...]) -> bool: return any(entity == unit for unit in unit_entities) +_INVENTORY_KEY = re.compile(r'"engine_refs"\s*:') +_JSON_ESCAPE = re.compile(r'\\(?:u[0-9A-Fa-f]{4}|["\\/bfnrt])') +_JSON_ESCAPES = {"b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t"} +# Decoding stops after this many layers. Each layer of JSON string encoding +# doubles a string's backslashes, so no encoded spelling comes near; only a +# crafted chain of escaped backslashes goes deeper, and it counts as carrying +# the inventory. +_INVENTORY_LAYERS = 32 + + +def _json_unescape(match: re.Match[str]) -> str: + escape = match.group() + if escape[1] == "u": + return chr(int(escape[2:], 16)) + return _JSON_ESCAPES.get(escape[1], escape[1]) + + def _carries_engine_ref_inventory(value: object) -> bool: - """Whether a value carries a prepared ``engine_refs`` map, in any JSON spelling.""" + """Whether a value carries a prepared ``engine_refs`` map, in any JSON spelling. + + A string carries it when, with its JSON escapes decoded layer by layer, it + holds the quoted key and a colon. Decoding a JSON string literal gives the + same text as decoding its span of the enclosing string, and a key once + spelled out survives every later layer, so every spelling of the map, at + any depth of string encoding, is found. Each layer is one pass over the + string. A string still escaped after ``_INVENTORY_LAYERS`` layers counts + as carrying the map, so a string costs at most that many passes, plus one. + """ if isinstance(value, Mapping): return "engine_refs" in value or any( @@ -692,24 +725,15 @@ def _carries_engine_ref_inventory(value: object) -> bool: ) if isinstance(value, list | tuple): return any(_carries_engine_ref_inventory(item) for item in value) - # Every string is decoded: a JSON string can spell its braces as escapes. if not isinstance(value, str): return False - try: - decoded = json.loads(value) - except (ValueError, TypeError): - decoder = json.JSONDecoder() - for index, character in enumerate(value): - if character not in {"{", '"'}: - continue - try: - embedded, _ = decoder.raw_decode(value, index) - except ValueError: - continue - if _carries_engine_ref_inventory(embedded): - return True - return False - return decoded != value and _carries_engine_ref_inventory(decoded) + for _ in range(_INVENTORY_LAYERS + 1): + if _INVENTORY_KEY.search(value): + return True + value, decoded = _JSON_ESCAPE.subn(_json_unescape, value) + if not decoded: + return False + return True def _is_text(value: object) -> bool: @@ -828,14 +852,21 @@ def _precalibration_rows(document: Mapping) -> dict[str, Mapping]: field: row.get(field) is None or _is_text(row[field]) for field in ("population", "base", "rules_binding") }, + "structural": row.get("structural", StructuralDelta.NONE) + in tuple(StructuralDelta), "params": _precalibration_params(row.get("params", {})), "inputs": _is_rows( row.get("inputs", ()), entity=_is_text, columns=lambda columns: _is_list(columns, _is_text), ), + # An absent mask is every row. "outputs": _is_rows( row.get("outputs", ()), entity=_is_text, column=_is_text + ) + and all( + _is_text(output.get("rows", ROWS_ALL)) + for output in row.get("outputs", ()) ), "artifact_inputs": _is_rows( row.get("artifact_inputs", ()), producer=_is_text @@ -1033,110 +1064,219 @@ def _precalibration_extension_rows( return result +def _precalibration_claims(row: Mapping) -> set[tuple[str, str]]: + """The EXPAND cells a declared node claims, whose values it receives. + + A literal list of ``entity.column`` names is read as the executor reads + it. Any other value, such as a resource selection, claims every + non-rewrite output: only composition can tell. + """ + + params = row.get("params", {}) + if "materialized_expand_outputs" not in params: + return set() + claimed = params["materialized_expand_outputs"] + if _is_list(claimed, lambda name: isinstance(name, str) and "." in name): + return { + (entity, column) + for entity, column in (name.split(".", 1) for name in claimed) + } + return { + (output["entity"], output["column"]) + for output in row.get("outputs", ()) + if not output.get("rewrite") + } + + +def _precalibration_cells(row: Mapping) -> set[tuple[str, str]] | None: + """The cells a declared EXPAND may write; None when only composition can tell. + + A literal list of ``[entity, column, dtype]`` texts names them. Any other + value, such as a resource selection, is unknown. + """ + + if row.get("structural") != StructuralDelta.EXPAND: + return set() + cells = row.get("params", {}).get("expand_cells") + if _is_list(cells, lambda cell: _is_list(cell, _is_text) and len(cell) == 3): + return {(entity, column) for entity, column, _ in cells} + return None + + def _precalibration_declared_reads( rows: Mapping[str, Mapping], ) -> dict[str, set[tuple[str, str, str]]]: - """Each declared node's column reads, as (entity, column, producer). - - A read resolves in the reader's population version and then down its base - chain, as the compiler resolves it: a structural node reads its base, and - a rewrite receives the incumbent from below its version. Where the - declaration leaves the version unknown (no population, an undeclared - version or a cycle), every declared producer of the column counts, and a - producer without a population counts in every version. + """Each declared node's column reads, as (entity, column, writer). + + A read's writers are those the executor tracks, over declarations: the + owner in the reader's population version and, while that owner keeps + earlier values (a rewrite, a masked owner or an EXPAND claim) or the + version owns none, the version's EXPAND when it may write the cell, then + the writers in the base version. A structural node reads its base; a + rewrite and an EXPAND claim read their own cells. Where the declaration + leaves the version unknown (no population, an undeclared version or a + cycle), every declared writer of the column counts. A writer without a + population counts in every version, and an EXPAND whose cells only + composition can tell writes every column. """ owners: dict[tuple[object, str, str], set[str]] = {} - producers: dict[tuple[str, str], set[str]] = {} + writers: dict[tuple[str, str], set[str]] = {} + keeps: set[tuple[str, str, str]] = set() + cells = {name: _precalibration_cells(row) for name, row in rows.items()} + unknown = {name for name, written in cells.items() if written is None} for name, row in rows.items(): version = row.get("population") if row.get("base") is None else None + claimed = _precalibration_claims(row) for output in row.get("outputs", ()): key = (output["entity"], output["column"]) - producers.setdefault(key, set()).add(name) + writers.setdefault(key, set()).add(name) owners.setdefault((version, *key), set()).add(name) + # Rows outside a mask keep the incumbent's values, and a rewrite + # or claim receives them. + if ( + output.get("rewrite") + or output.get("rows", ROWS_ALL) != ROWS_ALL + or key in claimed + ): + keeps.add((name, *key)) + for key in cells[name] or (): + writers.setdefault(key, set()).add(name) def resolve(version: object, key: tuple[str, str]) -> set[str]: + everyone = writers.get(key, set()) | unknown if not isinstance(version, str): - return producers.get(key, set()) - floating = owners.get((None, *key), set()) + return everyone + found = set(owners.get((None, *key), set())) seen = set() while isinstance(version, str): if version in seen or version not in rows: - return producers.get(key, set()) + return everyone seen.add(version) - if (version, *key) in owners: - return owners[(version, *key)] | floating + owned = owners.get((version, *key), set()) + found |= owned + if owned and not any((owner, *key) in keeps for owner in owned): + return found + if cells[version] is None or key in cells[version]: + found.add(version) version = rows[version].get("base") - return floating + return found reads: dict[str, set[tuple[str, str, str]]] = {} for name, row in rows.items(): base = row.get("base") start = row.get("population") if base is None else base - holder = rows.get(row.get("population")) - # A rewrite's incumbent comes from below its version. - incumbent = None if holder is None else holder.get("base") - rewritten = { - (output["entity"], output["column"]) - for output in row.get("outputs", ()) - if output.get("rewrite") - } - keys = { - (item["entity"], column) - for item in row.get("inputs", ()) - for column in item["columns"] - } + keys = ( + { + (item["entity"], column) + for item in row.get("inputs", ()) + for column in item["columns"] + } + | { + (output["entity"], output["column"]) + for output in row.get("outputs", ()) + if output.get("rewrite") + } + | _precalibration_claims(row) + ) reads[name] = { - (*key, producer) - for key in keys | rewritten - for producer in resolve(incumbent if key in rewritten else start, key) - if producer != name + (*key, writer) + for key in keys + for writer in resolve(start, key) + if writer != name } return reads +def _precalibration_expansions( + graph: Graph, +) -> tuple[ + dict[str, frozenset[tuple[str, str]]], dict[str, frozenset[tuple[str, str]]] +]: + """Each compiled node's EXPAND claims, and the cells each EXPAND may write. + + Both are read as the executor reads them, so a malformed one refuses. + """ + + try: + claims = { + node.id: materialized_expand_coordinates(node) for node in graph.nodes + } + cells = { + node.id: frozenset( + (entity, column) for entity, column, _ in declared_expand_cells(node) + ) + if node.structural is StructuralDelta.EXPAND + else frozenset() + for node in graph.nodes + } + except GraphError as error: + raise ValueError(f"Transport calibration ancestry: {error}") from error + return claims, cells + + def _precalibration_compiled_reads( - graph: Graph, compiled: CompiledGraph + graph: Graph, + compiled: CompiledGraph, + claims: Mapping[str, frozenset[tuple[str, str]]], + cells: Mapping[str, frozenset[tuple[str, str]]], ) -> dict[str, set[tuple[str, str, str]]]: - """Each compiled node's column reads, as (entity, column, producer). - - The producer is the owner in the reader's version or, for an inherited - column, in the nearest base version that owns it. A structural node reads - its base; a rewrite receives the incumbent from below its version. + """Each compiled node's column reads, as (entity, column, writer). + + A read's writers are those the executor tracks: the owner in the reader's + version and, while that owner keeps earlier values (a rewrite, a masked + owner or an EXPAND claim) or the version owns none, the version's EXPAND + when it may write the cell, then the writers in the base version. A + structural node reads its base; a rewrite and an EXPAND claim read their + own cells. """ by_id = {node.id: node for node in graph.nodes} - def resolve(version: str | None, entity: str, column: str) -> str | None: - while version is not None: - owner = compiled.owners.get((version, entity, column)) + def resolve(version: str, coordinate: tuple[str, str]) -> set[str]: + found = set() + while True: + holder = by_id[version] + owner = compiled.owners.get((version, *coordinate)) if owner is not None: - return owner - version = by_id[version].base - return None + found.add(owner) + output = next( + item + for item in by_id[owner].outputs + if (item.entity, item.column) == coordinate + ) + if not ( + output.rewrite + or output.rows != ROWS_ALL + or coordinate in claims[owner] + ): + return found + if coordinate in cells[version]: + found.add(version) + if holder.structural is StructuralDelta.CREATE or holder.base is None: + return found + version = holder.base reads: dict[str, set[tuple[str, str, str]]] = {} for node in graph.nodes: - version = compiled.versions[node.id] - start = version if node.structural is StructuralDelta.NONE else node.base - rewritten = { - (item.entity, item.column) for item in node.outputs if item.rewrite - } - keys = { - (item.entity, column) for item in node.inputs for column in item.columns - } + if node.structural is StructuralDelta.CREATE: + reads[node.id] = set() + continue + start = ( + compiled.versions[node.id] + if node.structural is StructuralDelta.NONE + else node.base + ) + coordinates = ( + {(item.entity, column) for item in node.inputs for column in item.columns} + | {(item.entity, item.column) for item in node.outputs if item.rewrite} + | claims[node.id] + ) reads[node.id] = { - (entity, column, producer) - for entity, column in keys | rewritten - if ( - producer := resolve( - by_id[version].base if (entity, column) in rewritten else start, - entity, - column, - ) - ) - is not None - and producer != node.id + (*coordinate, writer) + for coordinate in coordinates + for writer in resolve(start, coordinate) + if writer != node.id } return reads @@ -1193,6 +1333,13 @@ def _validate_precalibration_ancestry( roots.add(endpoints["calibration"]) predecessors = {name: set() for name in rows} reads = _precalibration_declared_reads(rows) + writes = { + name: { + (output["entity"], output["column"]) for output in row.get("outputs", ()) + } + | (_precalibration_cells(row) or set()) + for name, row in rows.items() + } for name, row in rows.items(): base = row.get("base") if isinstance(base, str): @@ -1213,6 +1360,7 @@ def check( rows: Mapping[str, Mapping], predecessors: Mapping, reads: Mapping[str, set[tuple[str, str, str]]], + writes: Mapping[str, set[tuple[str, str]]], *, composed: bool, derived: bool, @@ -1244,12 +1392,13 @@ def check( # stage's ancestry passes the same checks; a stage binding's nodes # after calibration (extension checkpoints) are outputs, not inputs. ancestors = _ancestor_closure(predecessors, roots) + # A node produces what it owns and, for an EXPAND, its cells. produced = { binding: { - (item["entity"], item["column"]) - for name, row in rows.items() + column + for name in rows if name in ancestors and binding in node_bindings[name] - for item in row.get("outputs", ()) + for column in writes[name] } for binding in stages } @@ -1425,11 +1574,12 @@ def check( # A derived output is checked against the fullest view: the composed graph # when there is one, otherwise the declaration. - check(rows, predecessors, reads, composed=False, derived=graph is None) + check(rows, predecessors, reads, writes, composed=False, derived=graph is None) if graph is None: return declarations = {**rows, **_precalibration_extension_rows(spec, rows)} compiled = compile_graph(graph) + claims, cells = _precalibration_expansions(graph) roots.update( node.id for node in graph.nodes if node.kernel.startswith("calibrate.") ) @@ -1453,7 +1603,12 @@ def check( check( compiled_rows, compiled.predecessors, - _precalibration_compiled_reads(graph, compiled), + _precalibration_compiled_reads(graph, compiled, claims, cells), + { + node.id: {(item.entity, item.column) for item in node.outputs} + | cells[node.id] + for node in graph.nodes + }, composed=True, derived=True, ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 1ed6c45084..948541971a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -2805,38 +2805,57 @@ def unavailable(*args, **kwargs): "holdout_sources": False, "eager": True, }, + # Writer chains: masked owners keep the values below them, and an EXPAND + # writes a cell that a claim owns, so a read often has several writers. + "writers": { + "kinds": ("person", "filter", "plain", "expand"), + "holdout_sources": False, + "eager": True, + "masks": True, + "focused": True, + }, } +_PRECALIBRATION_STRUCTURAL_KINDS = {"filter", "expand"} -def _precalibration_edges(draw, nodes, version, visible): + +def _precalibration_edges(draw, nodes, version, writers, focused=False): """Dependencies, reads and artifact edges from earlier nodes. - A read names the producer the column resolves to in ``version``. + A read is chosen from the node its column resolves to first in + ``version``, and names every writer whose values it receives. When + ``focused``, only nodes outside every binding pass artifacts. """ dependencies, reads, artifacts = [], [], [] for predecessor, earlier in enumerate(nodes): - if earlier["kind"] == "filter": + if earlier["kind"] in _PRECALIBRATION_STRUCTURAL_KINDS: continue edge = draw(st.integers(min_value=0, max_value=3)) - if edge == 1: + if edge == 1 and not (focused and earlier["binding"] is not None): dependencies.append(predecessor) artifacts.append(predecessor) if edge > 1: columns = [ column for column in earlier["outputs"] - if visible(version, column) == predecessor + if writers(version, column)[:1] == [predecessor] ] if columns and edge == 2: columns = [draw(st.sampled_from(columns))] if columns: dependencies.append(predecessor) - reads.extend((predecessor, column) for column in columns) + reads.extend( + (writer, column) + for column in columns + for writer in writers(version, column) + ) return dependencies, reads, artifacts @st.composite -def _precalibration_ancestry_dags(draw, kinds, holdout_sources, eager=False): +def _precalibration_ancestry_dags( + draw, kinds, holdout_sources, eager=False, masks=False, focused=False +): """Small DAGs with independent binding declarations and column edges. A new binding may also declare another binding's variable name, which only @@ -2845,9 +2864,15 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources, eager=False): A filter node opens a population version on an earlier one and may read columns from its base. A binding or plain node may own a ``shared`` column that a node in another version also owns, and a person node in a filter - version may rewrite a person column it sees from below. Each read and - rewrite names the producer it resolves to: in the reader's version, in a - filter's base, or below a rewrite's version. + version may rewrite a person column it sees from below or, with + ``masks``, own it again on masked rows. An EXPAND opens a version too: its + own binding's stage node, it writes a derived cell that the next node + claims. Each read, rewrite and claim names every writer whose values it + receives, as the executor tracks them: the owner in the reader's version + (a filter's base, or below a rewrite's or claim's own version), and the + writers below it while it keeps earlier values, with an EXPAND's cell. + With ``focused``, a listed stage withholds at most one of its columns and + stage nodes pass no artifacts, so a refusal usually has one cause. """ kinds = draw( st.lists(st.sampled_from(kinds), min_size=1, max_size=9 if eager else 7) @@ -2855,21 +2880,32 @@ def _precalibration_ancestry_dags(draw, kinds, holdout_sources, eager=False): likely = st.sampled_from((True, True, True, False)) if eager else st.booleans() nodes = [] bindings = {} - # Version None is CREATE; a filter's index names its version. A version a - # filter branches from takes no new members, so structural edges point back. + # Version None is CREATE; a structural node's index names its version. A + # version a structural node branches from takes no new members, so + # structural edges point back. bases = {None: None} closed = set() owned = {} entities = {} + # Owners that keep earlier values: rewrites, masked owners and claims. + keeps = set() + # The cell each EXPAND writes, by its version. + expanded = {} # Versions a read resolved through: a later owner there would re-route it. frozen = set() - def visible(version, column): - while (version, column) not in owned: + def writers(version, column): + found = [] + while True: + if (version, column) in owned: + found.append(owned[version, column]) + if (version, column) not in keeps: + return found + if column in expanded.get(version, ()): + found.append(version) if version is None: - return None + return found version = bases[version] - return owned[version, column] def freeze(version, column): while (version, column) not in owned: @@ -2880,50 +2916,112 @@ def member_version(): return draw(st.sampled_from(sorted(set(bases) - closed, key=str))) def seen(version): - return sorted( - {column for _, column in owned if visible(version, column) is not None} + return sorted({column for _, column in owned if writers(version, column)}) + + def structural_node(kind, base): + # A structural node reads its inputs from its base; a focused one only + # columns that no binding's node writes. + columns = [ + column + for column in seen(base) + if not focused + or all(nodes[writer]["binding"] is None for writer in writers(base, column)) + ] + chosen = ( + draw( + st.lists( + st.sampled_from(columns), + unique=True, + min_size=1 if eager else 0, + max_size=2, + ) + ) + if columns + else [] ) + for column in chosen: + freeze(base, column) + return { + "kind": kind, + "binding": None, + "entity": "person", + "version": None, + "base": base, + "outputs": [], + "cells": [], + "reads": [ + (writer, column) + for column in chosen + for writer in writers(base, column) + ], + "rewrites": [], + "claims": [], + "masks": [], + # A version depends on its base and every member of it. + "dependencies": [ + *([] if base is None else [base]), + *( + member + for member, row in enumerate(nodes) + if row["kind"] not in _PRECALIBRATION_STRUCTURAL_KINDS + and row["version"] == base + ), + ], + "artifacts": [], + "holdout_selection": False, + "holdout_source": False, + } - for index, kind in enumerate(kinds): - if kind == "filter": + for kind in kinds: + index = len(nodes) + if kind in _PRECALIBRATION_STRUCTURAL_KINDS: base = draw(st.sampled_from(sorted(bases, key=str))) - # A structural node reads its inputs from its base. - columns = seen(base) - chosen = ( - draw( - st.lists( - st.sampled_from(columns), - unique=True, - min_size=1 if eager else 0, - max_size=2, - ) - ) - if columns - else [] - ) - for column in chosen: - freeze(base, column) + node = structural_node(kind, base) bases[index] = base closed.add(base) + nodes.append(node) + if kind == "filter": + continue + # The EXPAND is a person stage's node, of a new binding or one that + # earlier nodes use, and its cell is a derived output of that + # binding. The claim follows it. + cell, claim = f"cell_{index}", index + 1 + available = [ + name for name, row in bindings.items() if row["entity"] == "person" + ] + if available and draw(likely if focused else st.booleans()): + node["binding"] = draw(st.sampled_from(available)) + else: + node["binding"] = f"binding_{index}" + bindings[node["binding"]] = { + "entity": "person", + "variables": [], + "derived": [], + } + bindings[node["binding"]]["derived"].append(cell) + node["cells"] = [cell] + expanded[index] = {cell} + owned[index, cell] = claim + keeps.add((index, cell)) + entities[cell] = "person" nodes.append( { - "kind": kind, - "binding": None, + "kind": "claim", + # A claim may be one of the EXPAND's stage nodes. + "binding": node["binding"] if draw(st.booleans()) else None, "entity": "person", - "version": None, - "base": base, - "outputs": [], - "reads": [(visible(base, column), column) for column in chosen], + "version": index, + "outputs": [cell], + "cells": [], + "reads": [], "rewrites": [], - # A version depends on its base and every member of it. - "dependencies": [ - *([] if base is None else [base]), - *( - member - for member, row in enumerate(nodes) - if row["kind"] != "filter" and row["version"] == base - ), + "claims": [ + (writer, cell) + for writer in writers(index, cell) + if writer != claim ], + "masks": [], + "dependencies": [index], "artifacts": [], "holdout_selection": False, "holdout_source": False, @@ -2953,7 +3051,7 @@ def seen(version): if shared and draw(st.booleans()): bindings[binding]["variables"].append(draw(st.sampled_from(shared))) dependencies, reads, artifacts = _precalibration_edges( - draw, nodes, version, visible + draw, nodes, version, writers, focused ) for _, column in reads: freeze(version, column) @@ -2970,16 +3068,16 @@ def seen(version): and draw(likely) ): outputs.append(common) - rewrites = [] + rewrites, masked = [], [] if version is not None and entity == "person": # The incumbent comes from below the version; CREATE has none. A - # rewritten column joins the node's binding variables, which a - # derived output may never be. + # rewritten or masked column joins the node's binding variables, + # which a derived output or an EXPAND cell may never be. columns = [ column for column in seen(bases[version]) if entities[column] == "person" - and not column.startswith("derived_") + and not column.startswith(("derived_", "cell_")) and (version, column) not in taken and column not in outputs and column not in {read for _, read in reads} @@ -2993,7 +3091,16 @@ def seen(version): max_size=2 if eager else 1, ) ): - rewrites.append((visible(bases[version], column), column)) + # A masked owner leaves the incumbent's values outside its + # mask, and a rewrite receives them, so both keep them. + if masks and draw(likely): + masked.append(column) + else: + rewrites.extend( + (writer, column) + for writer in writers(bases[version], column) + ) + keeps.add((version, column)) outputs.append(column) if binding is not None: bindings[binding]["variables"].extend( @@ -3014,8 +3121,11 @@ def seen(version): "entity": entity, "version": version, "outputs": outputs, + "cells": [], "reads": reads, "rewrites": rewrites, + "claims": [], + "masks": masked, "dependencies": [ *([] if version is None else [version]), *dependencies, @@ -3027,16 +3137,9 @@ def seen(version): and draw(st.sampled_from((False, False, False, True))), } ) - declared = {} - for binding, row in bindings.items(): - if draw(likely): - # Derived columns alone are a complete declaration. - declared[binding] = draw( - st.sets(st.sampled_from(row["variables"] + row["derived"]), min_size=1) - ) version = member_version() dependencies, reads, artifacts = _precalibration_edges( - draw, nodes, version, visible + draw, nodes, version, writers, focused ) nodes.append( { @@ -3045,14 +3148,50 @@ def seen(version): "entity": "person", "version": version, "outputs": [], + "cells": [], "reads": reads, "rewrites": [], + "claims": [], + "masks": [], "dependencies": [*([] if version is None else [version]), *dependencies], "artifacts": artifacts, "holdout_selection": False, "holdout_source": False, } ) + relevant = _precalibration_reference_relevant(nodes) + # Columns whose values a later writer passes on: under a mask, or as a cell. + passed = {column for node in nodes for column in node["masks"] + node["cells"]} + declared = {} + for binding, row in bindings.items(): + if not draw(likely): + continue + # Derived columns alone are a complete declaration. + columns = row["variables"] + row["derived"] + if not focused: + declared[binding] = draw(st.sets(st.sampled_from(columns), min_size=1)) + continue + # A focused stage declares no derived output that its nodes upstream do + # not produce, and withholds at most one column, often one passed on. + produced = { + column + for index in relevant + if nodes[index]["binding"] == binding + for column in nodes[index]["outputs"] + nodes[index]["cells"] + } + columns = [ + column + for column in columns + if column in row["variables"] or column in produced + ] + if not columns: + continue + declared[binding] = set(columns) + if len(columns) > 1 and draw(likely): + candidates = [column for column in columns if column in passed] + if not candidates or not draw(likely): + candidates = columns + declared[binding].remove(draw(st.sampled_from(candidates))) return nodes, bindings, declared @@ -3069,11 +3208,12 @@ def _precalibration_reference_missing_derived(model, binding): """Declared derived columns that no node of the stage upstream produces.""" nodes, bindings, declared = model relevant = _precalibration_reference_relevant(nodes) + # A node produces what it owns and, for an EXPAND, its cells. produced = { column for index in relevant if nodes[index]["binding"] == binding - for column in nodes[index]["outputs"] + for column in nodes[index]["outputs"] + nodes[index]["cells"] } return declared[binding] & set(bindings[binding]["derived"]) - produced @@ -3106,8 +3246,9 @@ def _precalibration_reference_accepts(model): for consumer in consumers ): return False - # A rewrite receives its incumbent, so it is a read too. - for producer, column in node["reads"] + node["rewrites"]: + # A rewrite receives its incumbent and a claim its EXPAND's cell, so + # both are reads too. + for producer, column in node["reads"] + node["rewrites"] + node["claims"]: source_binding = nodes[producer]["binding"] if ( source_binding in declared @@ -3128,13 +3269,19 @@ def _precalibration_synthetic_graph(model): nodes, bindings, declared = model references = {name: f"synthetic-engine:{name}:end" for name in bindings} seed = Owned("person", "seed", "float64") + # CREATE also loads the boolean mask that masked owners read. + created = ( + (seed, Owned("person", "keep", "bool")) + if any(item["masks"] for item in nodes) + else (seed,) + ) trace = ArtifactType("synthetic.precalibration.trace", 1) graph_nodes = [ Node( "p1.create", "synthetic.create@1", structural=StructuralDelta.CREATE, - outputs=(seed,), + outputs=created, sources=("synthetic_donor",), ) ] @@ -3142,7 +3289,9 @@ def _precalibration_synthetic_graph(model): { "id": "p1.create", "kernel": "synthetic.create@1", - "outputs": [{"entity": "person", "column": "seed"}], + "outputs": [ + {"entity": output.entity, "column": output.column} for output in created + ], } ] @@ -3152,32 +3301,52 @@ def version_id(version): for index, item in enumerate(nodes): name = "p1.calibrate" if item["kind"] == "calibrate" else f"p1.node_{index}" inputs = {} + # A read names each of its writers; the node reads the column once. for producer, column in item["reads"]: - inputs.setdefault(nodes[producer]["entity"], []).append(column) - if item["kind"] == "filter": + columns = inputs.setdefault(nodes[producer]["entity"], []) + if column not in columns: + columns.append(column) + if item["masks"]: + inputs.setdefault("person", []).append("keep") + if item["kind"] in _PRECALIBRATION_STRUCTURAL_KINDS: base = version_id(item["base"]) + kernel = f"synthetic.{item['kind']}@1" + params = {} + prepared_params = {} + if item["cells"]: + cells = tuple(("person", cell, "float64") for cell in item["cells"]) + params = { + "engine_ref": {"binding": item["binding"], "field": "engine_ref"}, + "expand_cells": [list(cell) for cell in cells], + } + prepared_params = { + "engine_ref": references[item["binding"]], + "expand_cells": cells, + } graph_nodes.append( Node( name, - "synthetic.filter@1", - structural=StructuralDelta.FILTER, + kernel, + structural=StructuralDelta(item["kind"]), base=base, inputs=tuple( Slice(entity, tuple(columns)) for entity, columns in inputs.items() ), + params=prepared_params, ) ) rows.append( { "id": name, - "kernel": "synthetic.filter@1", - "structural": "filter", + "kernel": kernel, + "structural": item["kind"], "base": base, "inputs": [ {"entity": entity, "columns": columns} for entity, columns in inputs.items() ], + **({"params": params} if params else {}), } ) continue @@ -3188,6 +3357,7 @@ def version_id(version): "selector": "simulate.solve_zero@1", "holdout": "synthetic.holdout@1", "plain": "synthetic.plain@1", + "claim": "synthetic.claim@1", "calibrate": "calibrate.synthetic@1", }[item["kind"]] rewritten = {column for _, column in item["rewrites"]} @@ -3198,6 +3368,12 @@ def version_id(version): prepared_params["engine_ref"] = references[binding] if item["kind"] == "selector": params["engine_ref"] = {"binding": binding, "field": "engine_ref"} + if item["claims"]: + claimed = list( + dict.fromkeys(f"person.{cell}" for _, cell in item["claims"]) + ) + params["materialized_expand_outputs"] = claimed + prepared_params["materialized_expand_outputs"] = tuple(claimed) sources = ("synthetic_holdout_facts",) if item["holdout_source"] else () if item["kind"] == "holdout": if item["holdout_selection"]: @@ -3221,7 +3397,11 @@ def version_id(version): ), outputs=tuple( Owned( - item["entity"], column, "float64", rewrite=column in rewritten + item["entity"], + column, + "float64", + rewrite=column in rewritten, + **({"rows": "keep"} if column in item["masks"] else {}), ) for column in item["outputs"] ), @@ -3244,6 +3424,7 @@ def version_id(version): "entity": item["entity"], "column": column, **({"rewrite": True} if column in rewritten else {}), + **({"rows": "keep"} if column in item["masks"] else {}), } for column in item["outputs"] ], @@ -3324,14 +3505,16 @@ def version_id(version): ), tuple(graph_nodes), ) - config = TransportGraphConfig(engine_refs=references, create_outputs=(seed,)) + config = TransportGraphConfig(engine_refs=references, create_outputs=created) return spec, graph, config def _precalibration_property_outcomes(model): """Acceptance by the guard and by today's guard, in both passes.""" spec, graph, config = _precalibration_synthetic_graph(model) - compile_graph(graph) + compiled = compile_graph(graph) + # The composed pass gives each read the executor's writers. + assert _guard_writers(graph, compiled) == _executor_writers(graph, compiled) extension_spec = copy.deepcopy(spec) extension_spec["resources"]["transport_graph"]["nodes"] = [ spec["resources"]["transport_graph"]["nodes"][0], @@ -3806,6 +3989,9 @@ def _precalibration_inventory(engine_refs, encoding): """The prepared engine_refs map, spelled the ways a parameter can carry it.""" inventory = {"engine_refs": dict(sorted(engine_refs.items()))} canonical = canonical_json(inventory).decode() + deep = canonical + for _ in range(6): + deep = json.dumps(deep) return { "string": canonical, "tuple": (canonical,), @@ -3814,6 +4000,10 @@ def _precalibration_inventory(engine_refs, encoding): "embedded": f"inventory={json.dumps(inventory)};", # A JSON string spelling every brace as an escape: no literal brace. "escaped": json.dumps(canonical).replace("{", "\\u007b"), + # The key itself spelled with an escape. + "escaped_key": canonical.replace('"engine_refs"', '"\\u0065ngine_refs"'), + # Six layers of string encoding. + "deep": deep, }[encoding] @@ -3824,6 +4014,8 @@ def _precalibration_inventory(engine_refs, encoding): "nested", "embedded", "escaped", + "escaped_key", + "deep", ] @@ -3854,7 +4046,8 @@ def test_precalibration_ancestry_compiled_prepared_map_still_refuses( @pytest.mark.parametrize( - "encoding", ["string", "spaced", "nested", "embedded", "escaped"] + "encoding", + ["string", "spaced", "nested", "embedded", "escaped", "escaped_key", "deep"], ) def test_precalibration_ancestry_declared_prepared_map_refuses( calibration_ancestry_composed, encoding @@ -3931,9 +4124,9 @@ def _synthetic_stage(binding, outputs, derived=None): return stage -def _synthetic_outcomes(spec, nodes, engine_refs): - """The guard's outcome in the declaration pass and the composed pass.""" - graph = Graph( +def _synthetic_graph(nodes): + """A synthetic graph: CREATE, then the given nodes.""" + return Graph( "synthetic", ( SourceRef("donor", "synthetic@1"), @@ -3950,6 +4143,11 @@ def _synthetic_outcomes(spec, nodes, engine_refs): *nodes, ), ) + + +def _synthetic_outcomes(spec, nodes, engine_refs): + """The guard's outcome in the declaration pass and the composed pass.""" + graph = _synthetic_graph(nodes) config = TransportGraphConfig( engine_refs=engine_refs, create_outputs=(_SYNTHETIC_SEED,) ) @@ -4269,6 +4467,394 @@ def test_precalibration_ancestry_unresolved_version_counts_every_producer(): } +def _writer_case(shape): + """A's private ``shared`` reaches B's ``b`` in ``v2`` only through ``adjust``. + + A produces ``shared`` in CREATE and a FILTER makes ``v2``, where ``adjust``, + outside both stages, owns ``shared`` again. A full owner replaces A's + values (``shadowed``). A masked owner keeps them on the rows outside its + mask (``masked``, and two versions up in ``deep_masked``), and a rewrite + passes them on: ``rewrite_read_first`` declares ``b`` before ``adjust``, + so ``b`` is the first node that receives them. + """ + bindings = [ + _synthetic_binding(name, "person", ["shared", f"{name}_public"]) + for name in ("A", "B") + ] + stages = [_synthetic_stage(row, [f"{row['id']}_public"]) for row in bindings] + reader = "v3" if shape == "deep_masked" else "v2" + masked = shape.endswith("masked") + adjust_row = { + "id": "adjust", + "kernel": "synthetic.adjust@1", + "population": "v2", + "outputs": [{"entity": "person", "column": "shared"}], + } + adjust = Node( + "adjust", + "synthetic.adjust@1", + population="v2", + outputs=(Owned("person", "shared", "float64"),), + ) + if masked: + adjust_row["inputs"] = [{"entity": "person", "columns": ["keep"]}] + adjust_row["outputs"][0]["rows"] = "keep" + adjust = replace( + adjust, + inputs=(Slice("person", ("keep",)),), + outputs=(Owned("person", "shared", "float64", rows="keep"),), + ) + if shape == "rewrite_read_first": + adjust_row["outputs"][0]["rewrite"] = True + adjust = replace( + adjust, outputs=(Owned("person", "shared", "float64", rewrite=True),) + ) + members = [ + _version_member( + "a_source", + RULES, + "create", + binding="A", + outputs=(("shared", False), ("A_public", False)), + ), + ( + { + "id": "flags", + "kernel": "synthetic.flags@1", + "population": "create", + "outputs": [{"entity": "person", "column": "keep"}], + }, + Node( + "flags", + "synthetic.flags@1", + population="create", + outputs=(Owned("person", "keep", "bool"),), + ), + ), + _version_filter("v2", "create"), + (adjust_row, adjust), + ] + if shape == "deep_masked": + members.append(_version_filter("v3", "v2")) + members.extend( + [ + _version_member( + "b", + RULES, + reader, + binding="B", + inputs=("shared",), + outputs=(("B_public", False),), + ), + _version_member( + "cal", "calibrate.synthetic@1", reader, inputs=("B_public",) + ), + ] + ) + if shape == "rewrite_read_first": + members.insert(3, members.pop(4)) + rows, nodes = zip(*members, strict=True) + return _synthetic_stage_spec(bindings, stages, list(rows)), list(nodes) + + +@pytest.mark.parametrize("declared", [True, False], ids=["declared", "compiled_only"]) +@pytest.mark.parametrize( + ("shape", "refuses"), + [ + ("shadowed", None), + ("masked", "b"), + ("deep_masked", "b"), + ("rewrite_read_first", "b"), + ], +) +def test_precalibration_ancestry_reads_follow_every_writer(shape, refuses, declared): + """A read counts every writer whose values it receives, as the executor tracks them. + + The writers continue below an owner that keeps earlier values: a masked + owner leaves A's values outside its mask, and a rewrite receives them. A + full owner that is not a rewrite stops the chain. + """ + spec, nodes = _writer_case(shape) + if not declared: + spec["resources"]["transport_graph"]["nodes"][1:] = [ + {"id": "cal", "kernel": "calibrate.synthetic@1", "population": "create"} + ] + outcome = ( + None + if refuses is None + else ( + ValueError, + f"Transport calibration ancestry: node {refuses!r} carries undeclared " + "pre-calibration stage output.", + ) + ) + outcomes = _synthetic_outcomes(spec, nodes, {"A": "A-ref", "B": "B-ref"}) + assert outcomes == { + "declaration": outcome if declared else None, + "composed": outcome, + } + + +def _expand_case(shape): + """Stage A's EXPAND installs ``cell``; ``claim`` owns it and calibration reads it. + + ``claim`` names ``cell`` in ``materialized_expand_outputs``, so it receives + the EXPAND's values without a ``Slice``. It also owns ``other`` outright, + which replaces A's private ``other`` from CREATE. Calibration reads + ``cell``, ``other`` and CREATE's ``seed``. Stage A declares ``cell`` as a + derived output except in ``private`` and ``claim_in_stage``, where + ``claim`` is one of A's own nodes. + """ + binding = _synthetic_binding("A", "person", ["other", "A_public"]) + public = shape not in {"private", "claim_in_stage"} + stage = _synthetic_stage( + binding, ["A_public"], [("person", "cell")] if public else None + ) + cells = (("person", "cell", "float64"),) + claim_params = {"materialized_expand_outputs": ["person.cell"]} + if shape == "claim_in_stage": + claim_params["engine_ref"] = {"binding": "A", "field": "engine_ref"} + rows = [ + { + "id": "a_source", + "kernel": RULES, + "population": "create", + "rules_binding": "A", + "outputs": [ + {"entity": "person", "column": "other"}, + {"entity": "person", "column": "A_public"}, + ], + }, + { + "id": "a_expand", + "kernel": "synthetic.expand@1", + "structural": "expand", + "base": "create", + "params": { + "engine_ref": {"binding": "A", "field": "engine_ref"}, + "expand_cells": [list(cell) for cell in cells], + }, + }, + { + "id": "claim", + "kernel": "synthetic.claim@1", + "population": "a_expand", + "params": claim_params, + "outputs": [ + {"entity": "person", "column": "cell"}, + {"entity": "person", "column": "other"}, + ], + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": "a_expand", + "inputs": [{"entity": "person", "columns": ["cell", "other", "seed"]}], + }, + ] + nodes = [ + Node( + "a_source", + RULES, + population="create", + params={"engine_ref": "A-ref"}, + outputs=( + Owned("person", "other", "float64"), + Owned("person", "A_public", "float64"), + ), + ), + Node( + "a_expand", + "synthetic.expand@1", + structural=StructuralDelta.EXPAND, + base="create", + params={"engine_ref": "A-ref", "expand_cells": cells}, + ), + Node( + "claim", + "synthetic.claim@1", + population="a_expand", + params={ + "materialized_expand_outputs": ("person.cell",), + **({"engine_ref": "A-ref"} if shape == "claim_in_stage" else {}), + }, + outputs=( + Owned("person", "cell", "float64"), + Owned("person", "other", "float64"), + ), + ), + Node( + "cal", + "calibrate.synthetic@1", + population="a_expand", + inputs=(Slice("person", ("cell", "other", "seed")),), + ), + ] + spec = _synthetic_stage_spec([binding], [stage], rows) + # Values only composition resolves: the declaration cannot read them. + spec["resources"]["synthetic_expand"] = { + "cells": [list(cell) for cell in cells], + "claims": ["person.cell"], + } + if shape == "selected_cells": + rows[1]["params"]["expand_cells"] = { + "resource": "synthetic_expand", + "path": ["cells"], + } + if shape == "selected_claims": + claim_params["materialized_expand_outputs"] = { + "resource": "synthetic_expand", + "path": ["claims"], + } + return spec, nodes + + +def _stage_output_refusal(node): + return ( + ValueError, + f"Transport calibration ancestry: node {node!r} carries undeclared " + "pre-calibration stage output.", + ) + + +@pytest.mark.parametrize( + ("shape", "outcomes"), + [ + ("public", {"declaration": None, "composed": None}), + ( + "private", + { + "declaration": _stage_output_refusal("claim"), + "composed": _stage_output_refusal("claim"), + }, + ), + ( + "claim_in_stage", + { + "declaration": _stage_output_refusal("cal"), + "composed": _stage_output_refusal("cal"), + }, + ), + # Cells only composition can tell: the EXPAND writes every column the + # declaration reads through it, and produces none of them. + ( + "selected_cells", + { + "declaration": ( + ValueError, + "precalibration_stages derived output person.cell of " + "'stage.A' is not produced by a stage node upstream of " + "calibration.", + ), + "composed": _stage_output_refusal("cal"), + }, + ), + # Claims only composition can tell: every output is claimed, so the + # claim receives A's private ``other`` from below. + ( + "selected_claims", + { + "declaration": _stage_output_refusal("claim"), + "composed": _stage_output_refusal("claim"), + }, + ), + ], +) +def test_precalibration_ancestry_expand_claims_read_their_cells(shape, outcomes): + """An EXPAND writes its cells, and a claim receives them without a ``Slice``. + + An EXPAND produces its cells for its stage's derived outputs, and each + later reader of a cell counts the EXPAND among its writers. + """ + spec, nodes = _expand_case(shape) + assert _synthetic_outcomes(spec, nodes, {"A": "A-ref"}) == outcomes + + +@pytest.mark.parametrize( + ("node", "params", "message"), + [ + ("a_expand", {"engine_ref": "A-ref"}, "EXPAND node 'a_expand' needs tuple "), + ( + "claim", + {"materialized_expand_outputs": ("cell",)}, + "Node 'claim' params['materialized_expand_outputs'] must be a tuple ", + ), + ], + ids=["expand_cells", "materialized_expand_outputs"], +) +def test_precalibration_ancestry_malformed_compiled_expand_params_refuse( + node, params, message +): + """The composed pass reads EXPAND cells and claims as the executor does.""" + spec, nodes = _expand_case("public") + nodes = [ + replace(item, params=params) if item.id == node else item for item in nodes + ] + outcome = _synthetic_outcomes(spec, nodes, {"A": "A-ref"}) + assert outcome["declaration"] is None + assert outcome["composed"][0] is ValueError + assert outcome["composed"][1].startswith( + f"Transport calibration ancestry: {message}" + ) + + +def _guard_writers(graph, compiled): + """Each node's reads as the composed pass resolves them, as (entity, column, writer).""" + return compose_module._precalibration_compiled_reads( + graph, compiled, *compose_module._precalibration_expansions(graph) + ) + + +def _executor_writers(graph, compiled): + """Each node's reads from the executor's own static writer lists.""" + from microcosm.graph.executor import _input_writers + + return { + node.id: { + (*coordinate, writer) + for coordinate, writers in _input_writers(compiled, node.id).items() + for writer in writers + } + for node in graph.nodes + } + + +@pytest.mark.parametrize( + ("case", "shape"), + [ + *((_writer_case, shape) for shape in ("shadowed", "masked", "deep_masked")), + (_writer_case, "rewrite_read_first"), + *((_expand_case, shape) for shape in ("public", "private", "claim_in_stage")), + ], + ids=lambda value: getattr(value, "__name__", value), +) +def test_precalibration_compiled_writers_match_the_executor(case, shape): + """The composed pass and the executor give every read the same writers. + + The pass reimplements the executor's writer chain (``_writers_of``) + because the guard runs before execution; this differential holds them + together on the writer and EXPAND examples. + """ + _, nodes = case(shape) + graph = _synthetic_graph(nodes) + compiled = compile_graph(graph) + assert _guard_writers(graph, compiled) == _executor_writers(graph, compiled) + + +@pytest.mark.parametrize("source", ["skeleton", "extended", "entitlement"]) +def test_precalibration_compiled_writers_match_the_executor_on_composed_graphs( + request, source +): + """The same differential on the composed NZ graphs and the entitlement graph.""" + if source == "entitlement": + fixture = request.getfixturevalue("precalibration_entitlement") + graph = compose_transport_graph(fixture.spec, fixture.config) + else: + graph = request.getfixturevalue("composed").graph(extended=source == "extended") + compiled = compile_graph(graph) + assert _guard_writers(graph, compiled) == _executor_writers(graph, compiled) + + def _derived_case(*, derived, outputs=(), producer_upstream=True): """A person-binding gross-up solve whose output is not a binding variable. @@ -4516,6 +5102,18 @@ def _shape_outcome(guard, spec): "missing_columns": ("mid", "inputs", [{"entity": "family"}]), "string_inputs": ("mid", "inputs", "u_out"), "output_without_entity": ("mid", "outputs", [{"column": "mid_out"}]), + "list_output_rows": ( + "mid", + "outputs", + [{"entity": "person", "column": "mid_out", "rows": ["keep"]}], + ), + "null_output_rows": ( + "mid", + "outputs", + [{"entity": "person", "column": "mid_out", "rows": None}], + ), + "unknown_structural": ("mid", "structural", "explode"), + "list_structural": ("mid", "structural", ["expand"]), "edge_without_producer": ("cal", "artifact_inputs", [{"name": "trace"}]), "string_edges": ("cal", "artifact_inputs", "probe"), "missing_kernel": ("probe", "kernel", None), @@ -4690,6 +5288,91 @@ def test_precalibration_ancestry_malformed_calibration_endpoint_refuses(endpoint ) +@pytest.mark.parametrize( + "note", + [ + "the engine_refs map", + '{"note": "engine_refs"}', + '["engine_refs"]', + '"\\u0065ngine_refs"', + ], +) +def test_precalibration_ancestry_engine_refs_text_is_not_the_map(note): + """Only the map's key, followed by a colon, marks the inventory.""" + spec = _shape_case() + _shape_rows(spec)["mid"]["params"] = {"note": note} + assert ( + _shape_outcome(compose_module.validate_transport_calibration_ancestry, spec) + is None + ) + + +def test_precalibration_ancestry_inventory_scan_is_linear(): + """A harmless parameter full of escaped quotes costs one pass per layer. + + Decoding JSON from every quote made this quadratic: 8 KB took 0.6 s and + 32 KB took 24 s. Decoding layer by layer takes milliseconds. + """ + spec = _shape_case() + _shape_rows(spec)["mid"]["params"] = {"note": "prefix" + r"\"" * 16_000} + started = time.perf_counter() + assert ( + _shape_outcome(compose_module.validate_transport_calibration_ancestry, spec) + is None + ) + assert time.perf_counter() - started < 1 + + +def _escape_layers(text, layers): + """``text`` behind ``layers`` layers of JSON escapes, one decode each. + + Each quote becomes ``\\u0022``, then each backslash ``\\u005c`` once per + further layer, so the length grows by five characters a layer, not twofold + as with ``json.dumps``. + """ + text = text.replace('"', "\\u0022") + for _ in range(layers - 1): + text = text.replace("\\", "\\u005c") + return text + + +@pytest.mark.parametrize( + ("note", "outcome"), + [ + # The inventory at the deepest layer the scan decodes, and one deeper. + (_escape_layers('{"engine_refs":{}}', 32), "inventory"), + (_escape_layers('{"engine_refs":{}}', 33), "inventory"), + # Harmless text: decoded through 32 layers, or refused one deeper. + (_escape_layers('{"note":"x"}', 32), None), + (_escape_layers('{"note":"x"}', 33), "inventory"), + # A chain of escaped backslashes 40,000 layers deep, refused quickly. + ("\\u005c" + "u005c" * 40_000 + "u0022x", "inventory"), + ], + ids=["key_32", "key_33", "harmless_32", "harmless_33", "chain_40000"], +) +def test_precalibration_ancestry_inventory_scan_stops_at_32_layers(note, outcome): + """The scan decodes at most 32 layers; a string escaped deeper refuses. + + Each layer is a pass over the string, so a crafted chain one layer per + escaped backslash cost a pass per layer before the bound: quadratic. + """ + spec = _shape_case() + _shape_rows(spec)["mid"]["params"] = {"note": note} + started = time.perf_counter() + assert _shape_outcome( + compose_module.validate_transport_calibration_ancestry, spec + ) == ( + None + if outcome is None + else ( + ValueError, + "Transport calibration ancestry: node 'mid' carries prepared " + "engine_refs map.", + ) + ) + assert time.perf_counter() - started < 1 + + @pytest.mark.parametrize("node", ["probe", "mid"]) def test_precalibration_ancestry_selected_prepared_map_refuses(node): """Selected resource data carrying the inventory is the inventory.""" @@ -4812,8 +5495,9 @@ def test_precalibration_ancestry_extension_selection_without_stages_is_todays( "node 'nz.v.V1.targets' has malformed params.", ), ({"id": "nz.targets.compile"}, "node 'nz.targets.compile' is declared twice."), + ({"id": None}, "node id None is not text."), ], - ids=["malformed_params", "repeated_id"], + ids=["malformed_params", "repeated_id", "non_text_id"], ) def test_precalibration_ancestry_malformed_extension_row_refuses( precalibration_entitlement, change, message From 339445fbcc479206cbeac03095d01929857fcbd2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 04:24:34 -0400 Subject: [PATCH 13/20] Time the inventory scan at 128 KB, where only S5's code scans The round-4 mutant sweep at ebe6f3d1a killed 53 of 54 mutants. The survivor, I5, put the quadratic suffix scan back into _carries_engine_ref_inventory. At the test's 32 KB it took 0.47 s on an idle host, under the 1 s bound. Its cost quadruples per doubling (7.5 s at 128 KB), while the layered decoding takes 7 ms at 128 KB. The test also timed more than the inventory scan. Its spec listed a benefit-unit stage, so the guard's engine-reference scan (_embeds_engine_ref, on main before the stages) decoded JSON from every quote of the parameter too: 0.34 s at 32 KB and 5.2 s at 128 KB of the guard's time. That scan is filed as a separate task, because rewriting it changes the guard without stages. The test now runs at 128 KB in two cases: - the helper alone; - the declaration pass on a person-only stage spec, where the engine-reference scan has no reference to look for. Its four calls return at once and decode nothing. Both take 0.01 s at the head and about 5 s with I5. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_transport_composed_graph.py | 50 +++++++++++++++---- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 948541971a..4818b37f1d 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -5307,19 +5307,51 @@ def test_precalibration_ancestry_engine_refs_text_is_not_the_map(note): ) -def test_precalibration_ancestry_inventory_scan_is_linear(): +@pytest.mark.parametrize("scope", ["helper", "person_stage"]) +def test_precalibration_ancestry_inventory_scan_is_linear(scope): """A harmless parameter full of escaped quotes costs one pass per layer. - Decoding JSON from every quote made this quadratic: 8 KB took 0.6 s and - 32 KB took 24 s. Decoding layer by layer takes milliseconds. + Decoding JSON from every quote made the inventory scan quadratic, four + times the cost per doubling: on an idle host 32 KB took 0.5 s and 128 KB + 7.5 s. Decoding layer by layer takes milliseconds at 128 KB. The guard case + lists only a person stage, because the engine-reference scan that predates + the stages (``_embeds_engine_ref``) still decodes JSON from every quote of + a parameter when a benefit-unit engine reference is looked for. """ - spec = _shape_case() - _shape_rows(spec)["mid"]["params"] = {"note": "prefix" + r"\"" * 16_000} - started = time.perf_counter() - assert ( - _shape_outcome(compose_module.validate_transport_calibration_ancestry, spec) - is None + note = "prefix" + r"\"" * 64_000 + binding = {**_synthetic_binding("p", "person", ["p_out"]), "engine_ref": "p-ref"} + rows = [ + { + "id": "probe", + "kernel": "synthetic.probe@1", + "population": "create", + "params": {"engine_ref": {"binding": "p", "field": "engine_ref"}}, + "outputs": [{"entity": "person", "column": "p_out"}], + }, + { + "id": "mid", + "kernel": "synthetic.mid@1", + "population": "create", + "params": {"note": note}, + "inputs": [{"entity": "person", "columns": ["p_out"]}], + "outputs": [{"entity": "person", "column": "mid_out"}], + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": "create", + "inputs": [{"entity": "person", "columns": ["mid_out"]}], + }, + ] + spec = _synthetic_stage_spec( + [binding], [_synthetic_stage(binding, ["p_out"])], rows ) + guard = compose_module.validate_transport_calibration_ancestry + started = time.perf_counter() + if scope == "helper": + assert not compose_module._carries_engine_ref_inventory(note) + else: + assert _shape_outcome(guard, spec) is None assert time.perf_counter() - started < 1 From d42fbcb72fca8bdd6e9c651bd588d88fa1b2a328 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 05:30:58 -0400 Subject: [PATCH 14/20] Refuse a listed stage's structural node upstream of calibration The round-4 review (partial: the run stopped on a provider filter after about an hour) found that a stage could change the population. Kernels receive ids, memberships, weights, strata, the frame metadata and the mass log without a Slice, and the executor's writer lists, which the composed pass matches, hold no read for them. The reviewer's case: a stage EXPAND overlays person_household_id, calibration declares no Slice and receives [10, 20], and both passes accepted it. Only structural nodes write those values. The executor refuses a NONE node that owns an id or membership column or declares a weight transition, and a kernel's frame, keep, expand, weights and strata results are for CREATE, FILTER, EXPAND and REWEIGHT nodes. A stage declares only columns, so a node of a listed stage upstream of calibration may not be structural. Both passes now refuse such a node before their other checks, naming the node, its kind and its stage binding, as NZ asked. The compiled rows carry each node's kind for this. NZ checked that none of its stage nodes is structural (517 guard calls over 13 probes). A stage's EXPAND now always refuses, so its cells can no longer be a stage's production: the writes map is gone and a node produces the columns it owns. A stage claim that owns a cell still produces it. An EXPAND still counts as a writer of its cells in both passes, as the executor's lists do. Tests: - Each kind (CREATE, FILTER, EXPAND, REWEIGHT) refuses as a stage node, declared and compiled-only. Unbound structural nodes, and bound ones that are no ancestor of calibration, pass. - The reviewer's three-node graph, executed: calibration receives the stage's membership, the writer lists hold no read of it, and both passes refuse at the EXPAND. - _expand_case: the EXPAND is outside the stage and overlays A's cell. The claim chain is tested through it, including a read-first shape in which calibration is the first node that receives A's values. stage_expand refuses. - P1: EXPANDs are mostly binding-free. A structural node is a person binding's node one time in eight. A claim with a binding owns a new cell as that binding's derived output; otherwise the cell may overlay a person column the EXPAND sees. The reference refuses a listed structural stage node and counts owned columns only. Docs: the new rule and the refusal order. The writer-chain claims now say column reads. The 32-layer rationale is corrected: an encoder that writes \u005c grows linearly, and 33 layers of the key take 344 characters, so harmless text that deep refuses, as does prose that quotes the key and a colon. Binding refusals name the binding, post-calibration nodes still get shape checks, and surrogate pairs decode as two code points. Co-Authored-By: Claude Opus 5.5 --- .../transport-precalibration-stages.added.md | 2 +- docs/transport-precalibration-stages.md | 101 ++-- .../src/microcosm/build/transport/compose.py | 51 +- .../shared/test_transport_composed_graph.py | 509 +++++++++++++++--- 4 files changed, 523 insertions(+), 140 deletions(-) diff --git a/changelog.d/transport-precalibration-stages.added.md b/changelog.d/transport-precalibration-stages.added.md index bbfa773ab9..93dabfadab 100644 --- a/changelog.d/transport-precalibration-stages.added.md +++ b/changelog.d/transport-precalibration-stages.added.md @@ -1 +1 @@ -Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. A stage's undeclared outputs are restricted by writer: each read follows the executor's writer chain through the reader's population-version lineage, where rewrites, masked owners and EXPAND claims pass earlier writers' values on and an EXPAND writes its cells, so a column of the same name that another node owns in another version is not the stage's. With stages listed, a malformed node or binding field the checks read refuses, naming the node, instead of being read character by character; entitlement rows' selections are checked too, and the prepared engine-reference map is refused in any JSON spelling, with escapes decoded layer by layer in one linear pass each, and in selected resource data. +Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. A stage's undeclared outputs are restricted by writer: each read follows the executor's writer chain through the reader's population-version lineage, where rewrites, masked owners and EXPAND claims pass earlier writers' values on and an EXPAND counts as a writer of its cells, so a column of the same name that another node owns in another version is not the stage's. A stage's node upstream of calibration may not be structural (CREATE, FILTER, EXPAND or REWEIGHT), because the ids, memberships, weights and strata a structural node writes reach later kernels without a column read. With stages listed, a malformed node or binding field the checks read refuses, naming the node or binding, instead of being read character by character; entitlement rows' selections are checked too, and the prepared engine-reference map is refused in any JSON spelling, with escapes decoded layer by layer in one linear pass each, and in selected resource data. diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index 5b73a838eb..8b9d1e92b5 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -53,9 +53,8 @@ the same closed shape and purposes as an `outputs` row. It names any entity, may not repeat a binding variable on the binding's entity (that belongs in `outputs`), and may not repeat any other output of the stage or a hold-out measure. Each derived output must be produced by one of the stage's nodes -upstream of calibration: a node produces the columns it owns and, for an -EXPAND, the cells it installs. The composed graph is checked when there is -one, otherwise the declaration. +upstream of calibration, which produces the columns it owns. The composed +graph is checked when there is one, otherwise the declaration. With stages listed, every field the checks read is validated before either pass uses it, on every declared node and rules binding, except three values @@ -95,17 +94,19 @@ declaration spells them out, and otherwise conservatively. An EXPAND's `expand_cells` names its cells when it is a list of `[entity, column, dtype]` texts; any other value, such as a resource selection, leaves the cells to composition, so the declaration pass counts that EXPAND as a writer of every -column read through its version, and as producing no cell. +column read through its version. A node's `materialized_expand_outputs` claims the cells it lists when it is a list of `entity.column` texts; any other value claims every output that is not a rewrite. The composed pass reads both from the compiled parameters exactly as the executor does, and a malformed one refuses there. -A malformed field refuses, naming the node: for example "node 'mid' has +A malformed node field refuses, naming the node: for example "node 'mid' has malformed inputs", "node 'mid' is declared twice" or "node id None is not -text". Read character by character, a string column, source or -path list would otherwise drop a read, a hold-out source or a selection, and -a repeated id would hide a node. The guard without stages is unchanged. +text". A malformed rules binding refuses naming the binding, as in "needs +rules binding 'A' to name its entity, rulespec_path and variables". Read +character by character, a string column, source or path list would otherwise +drop a read, a hold-out source or a selection, and a repeated id would hide a +node. The guard without stages is unchanged. ## Which nodes belong to a stage @@ -123,8 +124,9 @@ Only nodes in the ancestry of a calibration node are checked. The calibration nodes are every `calibrate.*` node and the calibration endpoint, in the declaration and in the composed graph. A node outside the ancestry of every calibration node, such as a stage binding's extension checkpoint after the -fit, is an output and is unaffected. A node after the central fit that a -variant calibration reads is in that variant's ancestry, so it is checked. +fit, is an output, and these rules leave it alone; the shape checks above +still cover every declared node. A node after the central fit that a variant +calibration reads is in that variant's ancestry, so it is checked. - Absent, or `"stages": []`, is the guard without stages, outcomes and messages alike. @@ -132,21 +134,33 @@ variant calibration reads is in that variant's ancestry, so it is checked. outputs, and group-encoder outputs read upstream only by listed benefit-unit stage nodes. Any other benefit-unit semantic upstream refuses as "not a declared pre-calibration stage". -- A listed person stage admits nothing new; it only adds the next rule. +- A listed person stage admits nothing new; it only adds the rules below. +- A stage changes columns, not the population. A structural node (CREATE, + FILTER, EXPAND or REWEIGHT) that belongs to a listed stage refuses, + naming the node, its kind and its stage binding. A stage declares only + columns, but a structural node also writes what every later kernel of its + version receives without a `Slice`: ids, group memberships, weights, + strata, the frame metadata and the mass log. No other node writes them, so + with this rule they never come from a stage's node. A structural node + outside every stage is checked like any other: its inputs are reads, and + its parameters, sources and artifact inputs face the same checks. So an + EXPAND's cells are never a stage's output either; a stage claim that owns + a cell produces it. - A node outside a listed stage that reads a column whose values one of the stage's own nodes wrote upstream of calibration, other than its `outputs` and `derived_outputs`, refuses as "undeclared pre-calibration stage output". Reads among a stage's own nodes do not count. -- The rule follows each read to every writer whose values it receives, as the - executor tracks them. In the reader's population version that is the owner - of the column. An owner that keeps earlier values passes on the writers - below it too: a rewrite receives the incumbent, a masked owner (an output - with a `rows` mask) leaves the incumbent's values on the rows outside its - mask, and an EXPAND claim receives the EXPAND's values. While the version - owns the column nowhere, or its owner keeps earlier values, the version's - EXPAND counts when it installs the column as a cell, and the walk goes on to - the base version, down to CREATE. A full owner that is not a rewrite or - claim stops it. +- The rule follows each column read, explicit or through a rewrite or claim, + to every writer whose values it receives, as the executor tracks them. In + the reader's population version that is the owner of the column. An owner + that keeps earlier values passes on the writers below it too: a rewrite + receives the incumbent, a masked owner (an output with a `rows` mask) + leaves the incumbent's values on the rows outside its mask, and an EXPAND + claim receives the EXPAND's values. While the version owns the column + nowhere, or its owner keeps earlier values, the version's EXPAND counts + when it installs the column as a cell, and the walk goes on to the base + version, down to CREATE. A full owner that is not a rewrite or claim stops + it. - A structural node reads its inputs from its base. A rewrite reads the column it rewrites, and a claim, a node that names an EXPAND cell in its `materialized_expand_outputs`, reads that cell, both without a `Slice`. @@ -170,12 +184,17 @@ variant calibration reads is in that variant's ancestry, so it is checked. decoded, layer by layer, up to 32 layers; a string still escaped after 32 layers counts as carrying it. So the inventory may be a JSON string in any spelling (braces or the key written as escapes included), nested in - strings (each layer of JSON encoding doubles the backslashes, so encoding - never comes near 32 layers), JSON embedded in a longer string, or the data - a resource selection names. Text that only names engine_refs, without the - quoted key and colon, is not the inventory. Each layer is one linear pass - over the string, so the check takes at most 33 passes. The inventory is - bookkeeping, not a stage invocation. + strings, JSON embedded in a longer string, or the data a resource selection + names. An encoder that escapes a backslash as `\\` doubles them at each + layer, but one that writes `\u005c` grows only linearly: 33 layers of the + quoted key take 344 characters. So the bound is reachable, and text still + escaped after it refuses whatever it decodes to. Escapes decode one at a + time, so a surrogate pair stays two code points; the key is ASCII, so this + does not change what is found. Text that only names engine_refs, without + the quoted key and colon, is not the inventory; prose that quotes the key + and a colon refuses like the map itself. Each layer is one linear + pass over the string, so the check takes at most 33 passes. The inventory + is bookkeeping, not a stage invocation. - The hold-out, selection and digest checks still apply to every upstream node, stage nodes included, so each stage's ancestry is clean. They read each node's declared selections: its `transport_graph` row or, in the composed @@ -213,19 +232,25 @@ These are tested, as properties and examples, in - With stages listed, the outcome is deterministic. Shape checks run first, over every declared node and binding, so a malformed declaration refuses naming its first malformed node even when an earlier node would refuse on - substance. Otherwise a refusal names the first offending node in - declaration order. The composed pass runs only after the declaration pass - admits: it checks the entitlement rows' shapes and reads the compiled - EXPAND cells and claims, then names the first offending node in graph - order. One message is not fixed: when several links name undeclared - nodes, which of them a refusal names follows set iteration order, as in - the guard without stages. + substance. Then, in each pass, the first structural stage node upstream of + calibration refuses, then the first derived output no upstream stage node + produces, and otherwise the first offending node in declaration order. The + composed pass runs only after the declaration pass admits: it checks the + entitlement rows' shapes and reads the compiled EXPAND cells and claims, + then applies the same order to the graph's nodes. One message is not + fixed: when several links name undeclared nodes, which of them a refusal + names follows set iteration order, as in the guard without stages. - With stages listed, the checks never read a malformed field so as to admit more than composition would. Every field they read is validated first, refusing naming its node, or read exactly as composition reads it. The two EXPAND parameters are read conservatively in the declaration and exactly in the composed pass, where a malformed one refuses. -- The composed pass gives every read the writers the executor's static - writer lists give it (`microcosm.graph.executor._input_writers`), on every - generated graph, the writer and EXPAND examples, and the composed NZ and - entitlement graphs. +- With stages listed, no node of a listed stage upstream of calibration is + structural, in either pass, so the values kernels receive without a + `Slice` (ids, memberships, weights, strata, frame metadata and the mass + log) come upstream of calibration only from nodes outside every stage. +- The composed pass gives every column read the writers the executor's + static writer lists give it (`microcosm.graph.executor._input_writers`), + on every generated graph, the writer and EXPAND examples, and the composed + NZ and entitlement graphs. Those lists cover column reads only, which is + why the rule above covers the values kernels receive without one. diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 12982d3516..27730f7606 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -691,10 +691,11 @@ def _is_unit_entity(entity: object, unit_entities: tuple[object, ...]) -> bool: _INVENTORY_KEY = re.compile(r'"engine_refs"\s*:') _JSON_ESCAPE = re.compile(r'\\(?:u[0-9A-Fa-f]{4}|["\\/bfnrt])') _JSON_ESCAPES = {"b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t"} -# Decoding stops after this many layers. Each layer of JSON string encoding -# doubles a string's backslashes, so no encoded spelling comes near; only a -# crafted chain of escaped backslashes goes deeper, and it counts as carrying -# the inventory. +# Decoding stops after this many layers. A JSON encoder that escapes a +# backslash as ``\\`` doubles them at each layer, but one that writes +# ``\u005c`` grows only linearly: 33 layers of the quoted key take 344 +# characters. A string still escaped after the last layer therefore counts as +# carrying the inventory, whatever it decodes to, so the scan fails closed. _INVENTORY_LAYERS = 32 @@ -1333,13 +1334,6 @@ def _validate_precalibration_ancestry( roots.add(endpoints["calibration"]) predecessors = {name: set() for name in rows} reads = _precalibration_declared_reads(rows) - writes = { - name: { - (output["entity"], output["column"]) for output in row.get("outputs", ()) - } - | (_precalibration_cells(row) or set()) - for name, row in rows.items() - } for name, row in rows.items(): base = row.get("base") if isinstance(base, str): @@ -1360,7 +1354,6 @@ def check( rows: Mapping[str, Mapping], predecessors: Mapping, reads: Mapping[str, set[tuple[str, str, str]]], - writes: Mapping[str, set[tuple[str, str]]], *, composed: bool, derived: bool, @@ -1392,13 +1385,28 @@ def check( # stage's ancestry passes the same checks; a stage binding's nodes # after calibration (extension checkpoints) are outputs, not inputs. ancestors = _ancestor_closure(predecessors, roots) - # A node produces what it owns and, for an EXPAND, its cells. + # A stage declares only columns. A structural node also writes what + # every later kernel of its version receives without a Slice: ids, + # memberships, weights, strata, the frame metadata and the mass log. + # So a stage node upstream of calibration changes no population, and + # an EXPAND's cells are never a stage's. + for name, row in rows.items(): + own = sorted(node_bindings[name].intersection(stages)) + structural = row.get("structural", StructuralDelta.NONE) + if name in ancestors and own and structural != StructuralDelta.NONE: + raise ValueError( + f"Transport calibration ancestry: node {name!r} of " + f"pre-calibration stage binding{'s' * (len(own) > 1)} " + f"{', '.join(map(repr, own))} is structural ({structural}); a " + "stage changes columns, not the population." + ) + # A node produces what it owns. produced = { binding: { - column - for name in rows + (output["entity"], output["column"]) + for name, row in rows.items() if name in ancestors and binding in node_bindings[name] - for column in writes[name] + for output in row.get("outputs", ()) } for binding in stages } @@ -1574,7 +1582,7 @@ def check( # A derived output is checked against the fullest view: the composed graph # when there is one, otherwise the declaration. - check(rows, predecessors, reads, writes, composed=False, derived=graph is None) + check(rows, predecessors, reads, composed=False, derived=graph is None) if graph is None: return declarations = {**rows, **_precalibration_extension_rows(spec, rows)} @@ -1586,6 +1594,7 @@ def check( compiled_rows = { node.id: { "kernel": node.kernel, + "structural": node.structural.value, "inputs": [ {"entity": item.entity, "columns": item.columns} for item in node.inputs ], @@ -1604,11 +1613,6 @@ def check( compiled_rows, compiled.predecessors, _precalibration_compiled_reads(graph, compiled, claims, cells), - { - node.id: {(item.entity, item.column) for item in node.outputs} - | cells[node.id] - for node in graph.nodes - }, composed=True, derived=True, ) @@ -1631,7 +1635,8 @@ def validate_transport_calibration_ancestry( Benefit-unit attributes remain allowed; excluded columns are the unit rules bindings' outputs and the group encoder's declared outputs. Declared pre-calibration stages admit their own unit semantics, restrict - outputs flowing onward and recursively check each stage's ancestry. With + outputs flowing onward and recursively check each stage's ancestry; a + stage node upstream of calibration is never structural. With stages and a graph, each derived output must be produced in the composed graph; without a graph, in the declaration. Composition always runs the graph-less check first. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 4818b37f1d..4f1f2ef087 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -84,6 +84,7 @@ GraphError, KernelBase, KernelRegistry, + KernelResult, Node, NodeRejectedError, Owned, @@ -91,6 +92,7 @@ SourceRef, StoreMissError, StructuralDelta, + WeightTransition, compile_graph, graph_to_json, run_graph, @@ -2806,7 +2808,7 @@ def unavailable(*args, **kwargs): "eager": True, }, # Writer chains: masked owners keep the values below them, and an EXPAND - # writes a cell that a claim owns, so a read often has several writers. + # overlays a cell that a claim owns, so a read often has several writers. "writers": { "kinds": ("person", "filter", "plain", "expand"), "holdout_sources": False, @@ -2865,12 +2867,15 @@ def _precalibration_ancestry_dags( columns from its base. A binding or plain node may own a ``shared`` column that a node in another version also owns, and a person node in a filter version may rewrite a person column it sees from below or, with - ``masks``, own it again on masked rows. An EXPAND opens a version too: its - own binding's stage node, it writes a derived cell that the next node - claims. Each read, rewrite and claim names every writer whose values it - receives, as the executor tracks them: the owner in the reader's version - (a filter's base, or below a rewrite's or claim's own version), and the - writers below it while it keeps earlier values, with an EXPAND's cell. + ``masks``, own it again on masked rows. An EXPAND opens a version too and + overlays a cell that the next node claims: a new cell, the derived output + of the claim's binding when it has one, or else possibly a person column + the EXPAND sees. A structural node is now and then a person binding's + node, which makes it a stage node when that binding is listed. Each read, + rewrite and claim names every writer whose values it receives, as the + executor tracks them: the owner in the reader's version (a filter's base, + or below a rewrite's or claim's own version), and the writers below it + while it keeps earlier values, with an EXPAND's cell. With ``focused``, a listed stage withholds at most one of its columns and stage nodes pass no artifacts, so a refusal usually has one cause. """ @@ -2980,35 +2985,41 @@ def structural_node(kind, base): bases[index] = base closed.add(base) nodes.append(node) - if kind == "filter": - continue - # The EXPAND is a person stage's node, of a new binding or one that - # earlier nodes use, and its cell is a derived output of that - # binding. The claim follows it. - cell, claim = f"cell_{index}", index + 1 available = [ name for name, row in bindings.items() if row["entity"] == "person" ] - if available and draw(likely if focused else st.booleans()): + if available and draw(st.integers(min_value=0, max_value=7)) == 0: node["binding"] = draw(st.sampled_from(available)) + if kind == "filter": + continue + # The claim follows the EXPAND, often as a person binding's node. + claim, binding = index + 1, None + if draw(st.booleans()): + if available and draw(st.booleans()): + binding = draw(st.sampled_from(available)) + else: + binding = f"binding_{claim}" + bindings[binding] = { + "entity": "person", + "variables": [], + "derived": [], + } + overlaid = [column for column in seen(base) if entities[column] == "person"] + if binding is None and overlaid and draw(likely): + cell = draw(st.sampled_from(overlaid)) else: - node["binding"] = f"binding_{index}" - bindings[node["binding"]] = { - "entity": "person", - "variables": [], - "derived": [], - } - bindings[node["binding"]]["derived"].append(cell) + cell = f"cell_{index}" + entities[cell] = "person" + if binding is not None: + bindings[binding]["derived"].append(cell) node["cells"] = [cell] expanded[index] = {cell} owned[index, cell] = claim keeps.add((index, cell)) - entities[cell] = "person" nodes.append( { "kind": "claim", - # A claim may be one of the EXPAND's stage nodes. - "binding": node["binding"] if draw(st.booleans()) else None, + "binding": binding, "entity": "person", "version": index, "outputs": [cell], @@ -3177,7 +3188,7 @@ def structural_node(kind, base): column for index in relevant if nodes[index]["binding"] == binding - for column in nodes[index]["outputs"] + nodes[index]["cells"] + for column in nodes[index]["outputs"] } columns = [ column @@ -3208,12 +3219,12 @@ def _precalibration_reference_missing_derived(model, binding): """Declared derived columns that no node of the stage upstream produces.""" nodes, bindings, declared = model relevant = _precalibration_reference_relevant(nodes) - # A node produces what it owns and, for an EXPAND, its cells. + # A node produces what it owns. produced = { column for index in relevant if nodes[index]["binding"] == binding - for column in nodes[index]["outputs"] + nodes[index]["cells"] + for column in nodes[index]["outputs"] } return declared[binding] & set(bindings[binding]["derived"]) - produced @@ -3229,6 +3240,9 @@ def _precalibration_reference_accepts(model): binding = node["binding"] if node["kind"] == "holdout" or node["holdout_source"]: return False + # A stage changes columns, not the population. + if node["kind"] in _PRECALIBRATION_STRUCTURAL_KINDS and binding in declared: + return False if ( binding is not None and node["entity"] == "family" @@ -3246,7 +3260,7 @@ def _precalibration_reference_accepts(model): for consumer in consumers ): return False - # A rewrite receives its incumbent and a claim its EXPAND's cell, so + # A rewrite receives its incumbent and a claim its cell's writers, so # both are reads too. for producer, column in node["reads"] + node["rewrites"] + node["claims"]: source_binding = nodes[producer]["binding"] @@ -3313,16 +3327,16 @@ def version_id(version): kernel = f"synthetic.{item['kind']}@1" params = {} prepared_params = {} + if item["binding"] is not None: + params["engine_ref"] = { + "binding": item["binding"], + "field": "engine_ref", + } + prepared_params["engine_ref"] = references[item["binding"]] if item["cells"]: cells = tuple(("person", cell, "float64") for cell in item["cells"]) - params = { - "engine_ref": {"binding": item["binding"], "field": "engine_ref"}, - "expand_cells": [list(cell) for cell in cells], - } - prepared_params = { - "engine_ref": references[item["binding"]], - "expand_cells": cells, - } + params["expand_cells"] = [list(cell) for cell in cells] + prepared_params["expand_cells"] = cells graph_nodes.append( Node( name, @@ -4124,7 +4138,7 @@ def _synthetic_stage(binding, outputs, derived=None): return stage -def _synthetic_graph(nodes): +def _synthetic_graph(nodes, create_params=None): """A synthetic graph: CREATE, then the given nodes.""" return Graph( "synthetic", @@ -4139,15 +4153,16 @@ def _synthetic_graph(nodes): structural=StructuralDelta.CREATE, sources=("donor",), outputs=(_SYNTHETIC_SEED,), + params=create_params or {}, ), *nodes, ), ) -def _synthetic_outcomes(spec, nodes, engine_refs): +def _synthetic_outcomes(spec, nodes, engine_refs, create_params=None): """The guard's outcome in the declaration pass and the composed pass.""" - graph = _synthetic_graph(nodes) + graph = _synthetic_graph(nodes, create_params) config = TransportGraphConfig( engine_refs=engine_refs, create_outputs=(_SYNTHETIC_SEED,) ) @@ -4568,7 +4583,7 @@ def _writer_case(shape): ], ) def test_precalibration_ancestry_reads_follow_every_writer(shape, refuses, declared): - """A read counts every writer whose values it receives, as the executor tracks them. + """A column read counts every writer whose values it receives, as the executor does. The writers continue below an owner that keeps earlier values: a masked owner leaves A's values outside its mask, and a rewrite receives them. A @@ -4596,22 +4611,27 @@ def test_precalibration_ancestry_reads_follow_every_writer(shape, refuses, decla def _expand_case(shape): - """Stage A's EXPAND installs ``cell``; ``claim`` owns it and calibration reads it. - - ``claim`` names ``cell`` in ``materialized_expand_outputs``, so it receives - the EXPAND's values without a ``Slice``. It also owns ``other`` outright, - which replaces A's private ``other`` from CREATE. Calibration reads - ``cell``, ``other`` and CREATE's ``seed``. Stage A declares ``cell`` as a - derived output except in ``private`` and ``claim_in_stage``, where - ``claim`` is one of A's own nodes. + """A's ``cell`` reaches calibration through an EXPAND that overlays it. + + ``a_source`` owns A's ``cell`` and ``other`` in CREATE. The EXPAND, no + stage's node, overlays ``cell`` on its entrants, and ``claim`` names + ``cell`` in ``materialized_expand_outputs``, so it receives the values of + both without a ``Slice``. ``claim`` also owns ``other`` outright, which + replaces A's ``other``. Calibration reads ``cell``, ``other`` and CREATE's + ``seed``. Stage A lists ``cell`` except in the ``private`` shapes and + ``claim_in_stage``, where ``claim`` is one of A's own nodes. In + ``stage_expand`` the EXPAND is one of A's nodes too. + ``private_read_first`` declares calibration before ``claim``, so + calibration is the first node that receives A's values. """ - binding = _synthetic_binding("A", "person", ["other", "A_public"]) - public = shape not in {"private", "claim_in_stage"} - stage = _synthetic_stage( - binding, ["A_public"], [("person", "cell")] if public else None - ) + binding = _synthetic_binding("A", "person", ["cell", "other", "A_public"]) + public = not shape.startswith("private") and shape != "claim_in_stage" + stage = _synthetic_stage(binding, ["A_public", "cell"] if public else ["A_public"]) cells = (("person", "cell", "float64"),) + expand_params = {"expand_cells": [list(cell) for cell in cells]} claim_params = {"materialized_expand_outputs": ["person.cell"]} + if shape == "stage_expand": + expand_params["engine_ref"] = {"binding": "A", "field": "engine_ref"} if shape == "claim_in_stage": claim_params["engine_ref"] = {"binding": "A", "field": "engine_ref"} rows = [ @@ -4621,8 +4641,8 @@ def _expand_case(shape): "population": "create", "rules_binding": "A", "outputs": [ - {"entity": "person", "column": "other"}, - {"entity": "person", "column": "A_public"}, + {"entity": "person", "column": column} + for column in ("cell", "other", "A_public") ], }, { @@ -4630,10 +4650,7 @@ def _expand_case(shape): "kernel": "synthetic.expand@1", "structural": "expand", "base": "create", - "params": { - "engine_ref": {"binding": "A", "field": "engine_ref"}, - "expand_cells": [list(cell) for cell in cells], - }, + "params": expand_params, }, { "id": "claim", @@ -4658,9 +4675,9 @@ def _expand_case(shape): RULES, population="create", params={"engine_ref": "A-ref"}, - outputs=( - Owned("person", "other", "float64"), - Owned("person", "A_public", "float64"), + outputs=tuple( + Owned("person", column, "float64") + for column in ("cell", "other", "A_public") ), ), Node( @@ -4668,7 +4685,10 @@ def _expand_case(shape): "synthetic.expand@1", structural=StructuralDelta.EXPAND, base="create", - params={"engine_ref": "A-ref", "expand_cells": cells}, + params={ + "expand_cells": cells, + **({"engine_ref": "A-ref"} if shape == "stage_expand" else {}), + }, ), Node( "claim", @@ -4697,7 +4717,7 @@ def _expand_case(shape): "claims": ["person.cell"], } if shape == "selected_cells": - rows[1]["params"]["expand_cells"] = { + expand_params["expand_cells"] = { "resource": "synthetic_expand", "path": ["cells"], } @@ -4706,6 +4726,10 @@ def _expand_case(shape): "resource": "synthetic_expand", "path": ["claims"], } + if shape == "private_read_first": + declared = spec["resources"]["transport_graph"]["nodes"] + declared[3:] = declared[:2:-1] + nodes[2:] = nodes[:1:-1] return spec, nodes @@ -4717,6 +4741,15 @@ def _stage_output_refusal(node): ) +def _structural_stage_refusal(node, kind): + return ( + ValueError, + f"Transport calibration ancestry: node {node!r} of pre-calibration stage " + f"binding 'A' is structural ({kind}); a stage changes columns, not the " + "population.", + ) + + @pytest.mark.parametrize( ("shape", "outcomes"), [ @@ -4728,27 +4761,24 @@ def _stage_output_refusal(node): "composed": _stage_output_refusal("claim"), }, ), + # The claim keeps the values below it, so calibration receives them. ( - "claim_in_stage", + "private_read_first", { "declaration": _stage_output_refusal("cal"), "composed": _stage_output_refusal("cal"), }, ), - # Cells only composition can tell: the EXPAND writes every column the - # declaration reads through it, and produces none of them. ( - "selected_cells", + "claim_in_stage", { - "declaration": ( - ValueError, - "precalibration_stages derived output person.cell of " - "'stage.A' is not produced by a stage node upstream of " - "calibration.", - ), + "declaration": _stage_output_refusal("cal"), "composed": _stage_output_refusal("cal"), }, ), + # Cells only composition can tell: the declaration counts the EXPAND + # as a writer of every column read through it. + ("selected_cells", {"declaration": None, "composed": None}), # Claims only composition can tell: every output is claimed, so the # claim receives A's private ``other`` from below. ( @@ -4758,13 +4788,21 @@ def _stage_output_refusal(node): "composed": _stage_output_refusal("claim"), }, ), + ( + "stage_expand", + { + "declaration": _structural_stage_refusal("a_expand", "expand"), + "composed": _structural_stage_refusal("a_expand", "expand"), + }, + ), ], ) def test_precalibration_ancestry_expand_claims_read_their_cells(shape, outcomes): - """An EXPAND writes its cells, and a claim receives them without a ``Slice``. + """An EXPAND overlays its cells, and a claim receives them without a ``Slice``. - An EXPAND produces its cells for its stage's derived outputs, and each - later reader of a cell counts the EXPAND among its writers. + The overlay keeps the values below it, so a claim and each later reader of + a cell count the EXPAND and every writer below it. An EXPAND that is a + stage's own node refuses. """ spec, nodes = _expand_case(shape) assert _synthetic_outcomes(spec, nodes, {"A": "A-ref"}) == outcomes @@ -4773,7 +4811,7 @@ def test_precalibration_ancestry_expand_claims_read_their_cells(shape, outcomes) @pytest.mark.parametrize( ("node", "params", "message"), [ - ("a_expand", {"engine_ref": "A-ref"}, "EXPAND node 'a_expand' needs tuple "), + ("a_expand", {}, "EXPAND node 'a_expand' needs tuple "), ( "claim", {"materialized_expand_outputs": ("cell",)}, @@ -4798,6 +4836,321 @@ def test_precalibration_ancestry_malformed_compiled_expand_params_refuse( ) +_STRUCTURAL_KINDS = ("create", "filter", "expand", "reweight") + + +def _structural_stage_case(kind, *, bound=True, ancestor=True): + """Stage A publishes ``A_public``; a ``kind`` node may be one of A's nodes. + + ``change`` (CREATE itself for ``create``) carries A's engine reference + when ``bound``. As an ancestor it opens the version calibration reads; + otherwise calibration stays in CREATE. The EXPAND would overlay the unit + membership ``person_family_id``, which calibration receives without a + ``Slice``. Returns the spec, the nodes and CREATE's params. + """ + binding = _synthetic_binding("A", "person", ["A_public"]) + version = "change" if ancestor and kind != "create" else "create" + rows = [ + { + "id": "a_source", + "kernel": RULES, + "population": "create", + "rules_binding": "A", + "outputs": [{"entity": "person", "column": "A_public"}], + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": version, + "inputs": [{"entity": "person", "columns": ["A_public", "seed"]}], + }, + ] + nodes = [ + Node( + "a_source", + RULES, + population="create", + params={"engine_ref": "A-ref"}, + outputs=(Owned("person", "A_public", "float64"),), + ), + Node( + "cal", + "calibrate.synthetic@1", + population=version, + inputs=(Slice("person", ("A_public", "seed")),), + ), + ] + params, prepared, transition = {}, {}, {} + if bound: + params["engine_ref"] = {"binding": "A", "field": "engine_ref"} + prepared["engine_ref"] = "A-ref" + if kind != "create": + if kind == "expand": + cells = (("person", "person_family_id", "int64"),) + params["expand_cells"] = [list(cell) for cell in cells] + prepared["expand_cells"] = cells + if kind == "reweight": + transition = { + "weights": WeightTransition("person", "importance", mass="free"), + "mass": "free", + } + rows.insert( + 1, + { + "id": "change", + "kernel": f"synthetic.{kind}@1", + "structural": kind, + "base": "create", + **({"params": params} if params else {}), + }, + ) + nodes.insert( + 1, + Node( + "change", + f"synthetic.{kind}@1", + structural=StructuralDelta(kind), + base="create", + params=prepared, + **transition, + ), + ) + spec = _synthetic_stage_spec( + [binding], [_synthetic_stage(binding, ["A_public"])], rows + ) + if kind != "create": + return spec, nodes, None + create = spec["resources"]["transport_graph"]["nodes"][0] + create["structural"] = "create" + if bound: + create["rules_binding"] = "A" + return spec, nodes, prepared + + +@pytest.mark.parametrize("declared", [True, False], ids=["declared", "compiled_only"]) +@pytest.mark.parametrize("kind", _STRUCTURAL_KINDS) +def test_precalibration_ancestry_stage_nodes_change_no_population(kind, declared): + """A stage node upstream of calibration is never structural. + + A stage declares only columns, but a structural node also writes what + every later kernel of its version receives without a ``Slice``: ids, + memberships, weights, strata, the frame metadata and the mass log. + ``compiled_only`` declares just calibration, so the composed pass decides + alone. + """ + spec, nodes, create_params = _structural_stage_case(kind) + if not declared: + spec["resources"]["transport_graph"]["nodes"][:] = [ + {"id": "create", "kernel": "synthetic.create@1"}, + {"id": "cal", "kernel": "calibrate.synthetic@1", "population": "create"}, + ] + outcome = _structural_stage_refusal( + "create" if kind == "create" else "change", kind + ) + outcomes = _synthetic_outcomes(spec, nodes, {"A": "A-ref"}, create_params) + assert outcomes == { + "declaration": outcome if declared else None, + "composed": outcome, + } + + +@pytest.mark.parametrize( + ("kind", "bound", "ancestor"), + [ + *((kind, False, True) for kind in _STRUCTURAL_KINDS), + *((kind, True, False) for kind in _STRUCTURAL_KINDS[1:]), + ], + ids=lambda value: value if isinstance(value, str) else None, +) +def test_precalibration_ancestry_other_structural_nodes_pass(kind, bound, ancestor): + """A structural node outside every stage, or no ancestor of calibration, passes.""" + spec, nodes, create_params = _structural_stage_case( + kind, bound=bound, ancestor=ancestor + ) + outcomes = _synthetic_outcomes(spec, nodes, {"A": "A-ref"}, create_params) + assert outcomes == {"declaration": None, "composed": None} + + +class _MembershipCreate(KernelBase): + """Person 1 in household 10.""" + + ref = "synthetic.membership_create@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + import pandas as pd + + from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + person = { + "person_id": np.array([1], dtype="int64"), + "person_household_id": np.array([10], dtype="int64"), + "mask": np.array([True]), + } + return KernelResult( + frame=Frame( + { + "person": pd.DataFrame(person), + "household": pd.DataFrame( + {"household_id": np.array([10], dtype="int64")} + ), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.array([1.0]), WeightKind.DESIGN)}, + pd.Series(["s"], name="stratum"), + ) + ) + + +class _MembershipExpand(KernelBase): + """Clones person 1 into person 2 of a new household 20, halving the weights.""" + + ref = "synthetic.membership_expand@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND + ) + + def run(self, context): + import pandas as pd + + from microcosm.frame import WeightKind, Weights + + return KernelResult( + expand={ + "person": pd.Series( + [1], index=pd.Index([2], name="person_id"), dtype="int64" + ), + "household": pd.Series( + [10], index=pd.Index([20], name="household_id"), dtype="int64" + ), + }, + columns={ + ("person", "person_household_id"): pd.Series( + [10, 20], index=pd.Index([1, 2], name="person_id"), dtype="int64" + ) + }, + weights=Weights(np.array([0.5, 0.5]), WeightKind.DESIGN), + ) + + +class _MembershipCalibrate(KernelBase): + """Records the household membership it receives without a ``Slice``.""" + + ref = "calibrate.membership@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def __init__(self): + self.received = [] + + def run(self, context): + import pandas as pd + + person = context.tables["person"] + self.received.append(person["person_household_id"].tolist()) + return KernelResult( + columns={ + ("person", "result"): pd.Series( + person["person_household_id"].to_numpy(), + index=pd.Index(person["person_id"], name="person_id"), + dtype="int64", + ) + } + ) + + +def test_precalibration_ancestry_refuses_a_stage_membership_calibration_receives( + tmp_path, +): + """Round 4's case: a stage's EXPAND writes a membership calibration receives. + + Stage A's EXPAND clones person 1 into a new household and overlays + ``person_household_id``, and calibration declares no ``Slice``. Run, the + graph hands calibration the stage's membership while the executor's + writer lists hold no read of it, so only the structural rule can refuse + it. Both passes do, at the EXPAND. + """ + cells = (("person", "person_household_id", "int64"),) + weights = {"expand_weight_entity": "household", "expand_weight_kind": "design"} + nodes = ( + Node( + "create", + _MembershipCreate.ref, + structural=StructuralDelta.CREATE, + sources=("donor",), + outputs=(Owned("person", "mask", "bool"),), + ), + Node( + "stage_expand", + _MembershipExpand.ref, + structural=StructuralDelta.EXPAND, + base="create", + params={"engine_ref": "A-ref", "expand_cells": cells, **weights}, + ), + Node( + "cal", + _MembershipCalibrate.ref, + population="stage_expand", + outputs=(Owned("person", "result", "int64"),), + ), + ) + binding = _synthetic_binding("A", "person", ["public"]) + rows = [ + { + "id": "stage_expand", + "kernel": _MembershipExpand.ref, + "structural": "expand", + "base": "create", + "params": { + "engine_ref": {"binding": "A", "field": "engine_ref"}, + "expand_cells": [list(cell) for cell in cells], + **weights, + }, + }, + { + "id": "cal", + "kernel": _MembershipCalibrate.ref, + "population": "stage_expand", + "outputs": [{"entity": "person", "column": "result"}], + }, + ] + spec = _synthetic_stage_spec( + [binding], [_synthetic_stage(binding, ["public"])], rows + ) + spec["resources"]["benefit_unit_rule"] = {"entity": "household"} + graph = Graph("synthetic", (SourceRef("donor", "raw-bytes-v1"),), nodes) + compiled = compile_graph(graph) + config = TransportGraphConfig( + engine_refs={"A": "A-ref"}, create_outputs=nodes[0].outputs + ) + for candidate in (None, graph): + assert _precalibration_outcome( + compose_module.validate_transport_calibration_ancestry, + spec, + graph=candidate, + config=config, + ) == _structural_stage_refusal("stage_expand", "expand") + assert _executor_writers(graph, compiled) == { + "create": set(), + "stage_expand": set(), + "cal": set(), + } + calibrate = _MembershipCalibrate() + kernels = KernelRegistry() + for kernel in (_MembershipCreate(), _MembershipExpand(), calibrate): + kernels.register(kernel) + donor = tmp_path / "donor.txt" + donor.write_text("synthetic") + run_graph( + compiled, + sources={"donor": donor}, + store=ContentStore(tmp_path / "store"), + kernels=kernels, + ) + assert calibrate.received == [[10, 20]] + + def _guard_writers(graph, compiled): """Each node's reads as the composed pass resolves them, as (entity, column, writer).""" return compose_module._precalibration_compiled_reads( From 99593905ee0c8cfeaaaacda4f83f60770e1c4fc1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 05:45:06 -0400 Subject: [PATCH 15/20] Check a stage's calibration node as one of its nodes NZ noted (review of d42fbcb72) that the calibration ancestry includes its roots, so a calibrate.* node or the calibration endpoint that carries a stage binding refuses as structural, while the doc said only "upstream of calibration". The code is unchanged: the doc now says each calibration node is in its own ancestry, so "upstream of calibration" includes the calibration nodes, and the structural-stage rule names a REWEIGHT calibration that carries the stage's binding. A new test pins it. Calibration is a REWEIGHT reading A's public output; carrying A's engine reference, it refuses naming cal in both passes (the composed pass alone when only cal is declared); without it, it passes. Under a mutant that exempts the calibration nodes from the rule (ancestors - roots), the two bound cases fail. Run in memory, as the pytest file waits for the disk-hold lift. Co-Authored-By: Claude Opus 5.5 --- docs/transport-precalibration-stages.md | 33 ++++++++------- .../shared/test_transport_composed_graph.py | 42 +++++++++++++++++++ 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index 8b9d1e92b5..943359fea1 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -122,11 +122,14 @@ refuses upstream of calibration. Only nodes in the ancestry of a calibration node are checked. The calibration nodes are every `calibrate.*` node and the calibration endpoint, in the -declaration and in the composed graph. A node outside the ancestry of every -calibration node, such as a stage binding's extension checkpoint after the -fit, is an output, and these rules leave it alone; the shape checks above -still cover every declared node. A node after the central fit that a variant -calibration reads is in that variant's ancestry, so it is checked. +declaration and in the composed graph. Each is in its own ancestry, so +"upstream of calibration" here includes the calibration nodes: one that +belongs to a stage is checked as that stage's node. A node outside the +ancestry of every calibration node, such as a stage binding's extension +checkpoint after the fit, is an output, and these rules leave it alone; the +shape checks above still cover every declared node. A node after the central +fit that a variant calibration reads is in that variant's ancestry, so it is +checked. - Absent, or `"stages": []`, is the guard without stages, outcomes and messages alike. @@ -136,16 +139,16 @@ calibration reads is in that variant's ancestry, so it is checked. "not a declared pre-calibration stage". - A listed person stage admits nothing new; it only adds the rules below. - A stage changes columns, not the population. A structural node (CREATE, - FILTER, EXPAND or REWEIGHT) that belongs to a listed stage refuses, - naming the node, its kind and its stage binding. A stage declares only - columns, but a structural node also writes what every later kernel of its - version receives without a `Slice`: ids, group memberships, weights, - strata, the frame metadata and the mass log. No other node writes them, so - with this rule they never come from a stage's node. A structural node - outside every stage is checked like any other: its inputs are reads, and - its parameters, sources and artifact inputs face the same checks. So an - EXPAND's cells are never a stage's output either; a stage claim that owns - a cell produces it. + FILTER, EXPAND or REWEIGHT) that belongs to a listed stage refuses, a + REWEIGHT calibration that carries the stage's binding included, naming the + node, its kind and its stage binding. A stage declares only columns, but a + structural node also writes what every later kernel of its version receives + without a `Slice`: ids, group memberships, weights, strata, the frame + metadata and the mass log. No other node writes them, so with this rule they + never come from a stage's node. A structural node outside every stage is + checked like any other: its inputs are reads, and its parameters, sources + and artifact inputs face the same checks. So an EXPAND's cells are never a + stage's output either; a stage claim that owns a cell produces it. - A node outside a listed stage that reads a column whose values one of the stage's own nodes wrote upstream of calibration, other than its `outputs` and `derived_outputs`, refuses as "undeclared pre-calibration stage output". diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 4f1f2ef087..407ee55e3c 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -4971,6 +4971,48 @@ def test_precalibration_ancestry_other_structural_nodes_pass(kind, bound, ancest assert outcomes == {"declaration": None, "composed": None} +@pytest.mark.parametrize("declared", [True, False], ids=["declared", "compiled_only"]) +@pytest.mark.parametrize("bound", [True, False], ids=["bound", "unbound"]) +def test_precalibration_ancestry_checks_the_calibration_node_itself(bound, declared): + """Calibration is in its own ancestry, so a stage's structural calibration refuses. + + Calibration here is a REWEIGHT that carries A's engine reference itself + when ``bound``; unbound, it is the usual calibration and passes. + """ + spec, nodes, _ = _structural_stage_case("create", bound=False) + rows = spec["resources"]["transport_graph"]["nodes"] + if declared: + rows[-1] = { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "structural": "reweight", + "base": "create", + "inputs": [{"entity": "person", "columns": ["A_public", "seed"]}], + } + if bound: + rows[-1]["params"] = {"engine_ref": {"binding": "A", "field": "engine_ref"}} + else: + rows[:] = [ + {"id": "create", "kernel": "synthetic.create@1"}, + {"id": "cal", "kernel": "calibrate.synthetic@1", "population": "create"}, + ] + nodes[-1] = Node( + "cal", + "calibrate.synthetic@1", + structural=StructuralDelta.REWEIGHT, + base="create", + inputs=(Slice("person", ("A_public", "seed")),), + params={"engine_ref": "A-ref"} if bound else {}, + weights=WeightTransition("person", "importance", mass="free"), + mass="free", + ) + outcome = _structural_stage_refusal("cal", "reweight") if bound else None + assert _synthetic_outcomes(spec, nodes, {"A": "A-ref"}) == { + "declaration": outcome if declared else None, + "composed": outcome, + } + + class _MembershipCreate(KernelBase): """Person 1 in household 10.""" From 1d1801edb9591ec0fe0cb8eb2f03107ca49b7865 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 13:24:03 -0700 Subject: [PATCH 16/20] Close the mass-log channel, bound the stage scan, test a second claim Round 5 (Sol, at 99593905e) found three things. 1. A non-structural stage node can pass an undeclared value to calibration through the frame mass log. A receipt's frame_mass_log_append records reach every later structural node of the node's lineage without a Slice: population.patch appends them, and the executor gives a structural node its base version's cumulative log. No declaration names the log, so neither guard pass can follow it. The UK runtime appends from non-structural nodes, so the graph package cannot refuse appends in general. The transport runner (cli._run) now gives run_graph a population observer. It refuses any node whose admitted mass log differs from the one it received, whether the node ran or was restored from the store, before the node is persisted. No transport kernel appends, so no transport run changes. The doc's premise that only structural nodes write the mass log is corrected. 2. The composed pass read a node's stage bindings by calling _embeds_engine_ref once per binding. That call decodes JSON from every quote of every parameter, so a harmless parameter full of escaped quotes took quadratic time. _embedded_engine_refs now decodes escapes layer by layer, as the inventory scan does: one linear pass per layer, for all bindings at once. A parameter still escaped after 32 layers refuses. The shared decoder now decodes a surrogate pair to one character, as json.loads does. A Hypothesis property checks that the layered scan finds every reference _embeds_engine_ref finds. 3. No test had an EXPAND claim a second cell. The two_claims shape claims a public cell first and a private one second. It is added to the outcome test and to the static executor differential. The changelog's "unaffected" wording now says post-calibration nodes still face the shape checks. Co-Authored-By: Claude Opus 5.5 --- .../transport-precalibration-stages.added.md | 2 +- docs/transport-precalibration-stages.md | 53 ++- .../src/microcosm/build/transport/cli.py | 48 ++- .../src/microcosm/build/transport/compose.py | 88 +++- .../shared/test_transport_composed_graph.py | 398 +++++++++++++++++- 5 files changed, 555 insertions(+), 34 deletions(-) diff --git a/changelog.d/transport-precalibration-stages.added.md b/changelog.d/transport-precalibration-stages.added.md index 93dabfadab..1697845c70 100644 --- a/changelog.d/transport-precalibration-stages.added.md +++ b/changelog.d/transport-precalibration-stages.added.md @@ -1 +1 @@ -Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node are checked, so a stage binding's nodes after every calibration are unaffected. A stage's undeclared outputs are restricted by writer: each read follows the executor's writer chain through the reader's population-version lineage, where rewrites, masked owners and EXPAND claims pass earlier writers' values on and an EXPAND counts as a writer of its cells, so a column of the same name that another node owns in another version is not the stage's. A stage's node upstream of calibration may not be structural (CREATE, FILTER, EXPAND or REWEIGHT), because the ids, memberships, weights and strata a structural node writes reach later kernels without a column read. With stages listed, a malformed node or binding field the checks read refuses, naming the node or binding, instead of being read character by character; entitlement rows' selections are checked too, and the prepared engine-reference map is refused in any JSON spelling, with escapes decoded layer by layer in one linear pass each, and in selected resource data. +Allow transport specs to declare pre-calibration rules stages with explicit outputs, optional derived outputs that stage nodes compute, recursive ancestry checks and holdout isolation for person and unit bindings. Only nodes in the ancestry of a calibration node get the ancestry checks; a stage binding's nodes after every calibration face only the shape checks. A stage's undeclared outputs are restricted by writer: each read follows the executor's writer chain through the reader's population-version lineage, where rewrites, masked owners and EXPAND claims pass earlier writers' values on and an EXPAND counts as a writer of its cells, so a column of the same name that another node owns in another version is not the stage's. A stage's node upstream of calibration may not be structural (CREATE, FILTER, EXPAND or REWEIGHT), because the ids, memberships, weights and strata a structural node writes reach later kernels without a column read. For the same reason a transport run refuses any node that changes the frame mass log it received, whether it ran or was restored from the store; no transport kernel appends to it. With stages listed, a malformed node or binding field the checks read refuses, naming the node or binding, instead of being read character by character; entitlement rows' selections are checked too, and the prepared engine-reference map is refused in any JSON spelling, with escapes decoded layer by layer in one linear pass each, and in selected resource data. The composed pass reads each node's stage bindings from its engine references the same way. diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index 943359fea1..e75bacc6e2 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -113,10 +113,14 @@ node. The guard without stages is unchanged. In the declaration pass, a node belongs to a stage when its `rules_binding` is the stage's binding or any `{"binding": id}` selector in its parameters names it. In the composed pass, a node belongs to a stage when its parameters carry -the binding's prepared engine reference. A node that names a listed and an -unlisted benefit-unit binding refuses. A parameter carrying the whole prepared -`engine_refs` inventory is bookkeeping, not a stage invocation, and still -refuses upstream of calibration. +the binding's prepared engine reference, as written, JSON-escaped once, or +after JSON escapes are decoded, layer by layer, up to 32 layers, as for the +inventory below. Each layer is one linear pass over a string. A parameter +still escaped after 32 layers refuses, because its stage bindings cannot be +read. A node that names a listed and an unlisted benefit-unit binding +refuses. A parameter carrying the whole prepared `engine_refs` inventory is +bookkeeping, not a stage invocation, and still refuses upstream of +calibration. ## Rules @@ -143,12 +147,19 @@ checked. REWEIGHT calibration that carries the stage's binding included, naming the node, its kind and its stage binding. A stage declares only columns, but a structural node also writes what every later kernel of its version receives - without a `Slice`: ids, group memberships, weights, strata, the frame - metadata and the mass log. No other node writes them, so with this rule they - never come from a stage's node. A structural node outside every stage is - checked like any other: its inputs are reads, and its parameters, sources - and artifact inputs face the same checks. So an EXPAND's cells are never a - stage's output either; a stage claim that owns a cell produces it. + without a `Slice`: ids, group memberships, weights, strata and the frame + metadata. No other node writes them, so with this rule they never come from + a stage's node. The frame mass log is the exception. Any node's receipt can + append to it (`frame_mass_log_append`), and every later structural node of + its lineage, calibration included, receives the log without a `Slice`. No + declaration shows it, so the guard cannot follow it. Instead the transport + runner refuses any node whose admitted mass log differs from the one it + received, whether the node ran or was restored from the store. No transport + kernel appends, so a transport run's mass log is CREATE's. A structural + node outside every stage is checked like any other: its inputs are reads, + and its parameters, sources and artifact inputs face the same checks. So + an EXPAND's cells are never a stage's output either; a stage claim that + owns a cell produces it. - A node outside a listed stage that reads a column whose values one of the stage's own nodes wrote upstream of calibration, other than its `outputs` and `derived_outputs`, refuses as "undeclared pre-calibration stage output". @@ -192,8 +203,8 @@ checked. layer, but one that writes `\u005c` grows only linearly: 33 layers of the quoted key take 344 characters. So the bound is reachable, and text still escaped after it refuses whatever it decodes to. Escapes decode one at a - time, so a surrogate pair stays two code points; the key is ASCII, so this - does not change what is found. Text that only names engine_refs, without + time, except that a surrogate pair decodes to one character, as + `json.loads` decodes it. Text that only names engine_refs, without the quoted key and colon, is not the inventory; prose that quotes the key and a colon refuses like the map itself. Each layer is one linear pass over the string, so the check takes at most 33 passes. The inventory @@ -240,7 +251,8 @@ These are tested, as properties and examples, in produces, and otherwise the first offending node in declaration order. The composed pass runs only after the declaration pass admits: it checks the entitlement rows' shapes and reads the compiled EXPAND cells and claims, - then applies the same order to the graph's nodes. One message is not + refuses the first node with a parameter still escaped after 32 layers, then + applies the same order to the graph's nodes. One message is not fixed: when several links name undeclared nodes, which of them a refusal names follows set iteration order, as in the guard without stages. - With stages listed, the checks never read a malformed field so as to admit @@ -250,10 +262,19 @@ These are tested, as properties and examples, in the composed pass, where a malformed one refuses. - With stages listed, no node of a listed stage upstream of calibration is structural, in either pass, so the values kernels receive without a - `Slice` (ids, memberships, weights, strata, frame metadata and the mass - log) come upstream of calibration only from nodes outside every stage. + `Slice` (ids, memberships, weights, strata and frame metadata) come + upstream of calibration only from nodes outside every stage. +- A transport run refuses a node that changes the frame mass log it + received, a stage's node or not, calibration included, cold or restored + from a store a plain run wrote. The guard admits such a graph and a plain + `run_graph` delivers the appended record to calibration; the runner + refuses it before persisting the node. +- The composed pass's stage membership finds every engine reference that + decoding JSON from each quote or brace finds (`_embeds_engine_ref`), a + property tested over nested JSON, ASCII and embedded spellings. - The composed pass gives every column read the writers the executor's static writer lists give it (`microcosm.graph.executor._input_writers`), on every generated graph, the writer and EXPAND examples, and the composed NZ and entitlement graphs. Those lists cover column reads only, which is - why the rule above covers the values kernels receive without one. + why the structural rule and the runner's mass-log check cover the values + kernels receive without one. diff --git a/packages/microcosm-build/src/microcosm/build/transport/cli.py b/packages/microcosm-build/src/microcosm/build/transport/cli.py index 3d26dd15ff..8b5c2dea7a 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/cli.py +++ b/packages/microcosm-build/src/microcosm/build/transport/cli.py @@ -22,7 +22,7 @@ import json import os import sys -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, replace from pathlib import Path from tempfile import TemporaryDirectory @@ -38,14 +38,17 @@ from microcosm.frame.rules import RulesEngine from microcosm.frame.unit_construction import BenefitUnitRule from microcosm.graph import ( + CompiledGraph, ContentStore, Graph, KernelContext, Node, Owned, + Population, ResumePolicy, RunManifest, StoreMissError, + StructuralDelta, compile_graph, explain_html, graph_to_json, @@ -394,15 +397,56 @@ def _evidence(manifest: RunManifest, store: ContentStore, out: Path) -> dict: def _run(graph: Graph, registry, sources, store, resume) -> RunManifest: used = {name for node in graph.nodes for name in node.sources} + compiled = compile_graph(graph) return run_graph( - compile_graph(graph), + compiled, sources={name: path for name, path in sources.items() if name in used}, store=store, kernels=registry.kernels, resume=resume, + _population_observer=_mass_log_unchanged(compiled), + # The observer keeps only record reprs and changes nothing. + _population_observer_detach=False, ) +def _mass_log_unchanged(compiled: CompiledGraph) -> Callable[[str, Population], None]: + """Refuse a node that changes the frame mass log it received. + + A node's receipt can append records to the frame mass log + (``frame_mass_log_append``), and every later structural node of its + lineage, calibration included, receives that log without a ``Slice``. No + declaration names it, so the calibration ancestry guard cannot follow it: + a stage node could pass an undeclared value to calibration this way. No + transport kernel appends, so in a transport run the log is CREATE's. The + executor calls this for cold nodes and restored cache hits alike, before + it persists either, and the exception refuses the run. + """ + + logs: dict[str, tuple[str, ...]] = {} + + def observe(node_id: str, population: Population) -> None: + node = compiled.graph.node(node_id) + log = tuple(map(repr, population.frame.mass_log)) + if node.structural is StructuralDelta.NONE: + version = compiled.versions[node_id] + received = logs[version] + else: + version = node_id + received = ( + log if node.structural is StructuralDelta.CREATE else logs[node.base] + ) + if log != received: + raise ValueError( + f"Transport node {node_id!r} changed the frame mass log it " + "received; later kernels receive that log without a Slice, so " + "a transport run takes it from CREATE only." + ) + logs[version] = log + + return observe + + class _TimestampFreeHDFStore(pd.HDFStore): """Disable object timestamps and unused pandas search indexes.""" diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 27730f7606..0ba2daa27c 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -689,18 +689,26 @@ def _is_unit_entity(entity: object, unit_entities: tuple[object, ...]) -> bool: _INVENTORY_KEY = re.compile(r'"engine_refs"\s*:') -_JSON_ESCAPE = re.compile(r'\\(?:u[0-9A-Fa-f]{4}|["\\/bfnrt])') +_JSON_ESCAPE = re.compile( + r"\\u[dD][89abAB][0-9a-fA-F]{2}\\u[dD][c-fC-F][0-9a-fA-F]{2}" + r'|\\(?:u[0-9A-Fa-f]{4}|["\\/bfnrt])' +) _JSON_ESCAPES = {"b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t"} # Decoding stops after this many layers. A JSON encoder that escapes a # backslash as ``\\`` doubles them at each layer, but one that writes # ``\u005c`` grows only linearly: 33 layers of the quoted key take 344 # characters. A string still escaped after the last layer therefore counts as -# carrying the inventory, whatever it decodes to, so the scan fails closed. +# carrying the inventory, whatever it decodes to, and leaves a node's stage +# bindings unread, which refuses: both scans fail closed. _INVENTORY_LAYERS = 32 def _json_unescape(match: re.Match[str]) -> str: escape = match.group() + if len(escape) == 12: + # A surrogate pair is one character, as ``json.loads`` decodes it. + high, low = int(escape[2:6], 16), int(escape[8:], 16) + return chr(0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00)) if escape[1] == "u": return chr(int(escape[2:], 16)) return _JSON_ESCAPES.get(escape[1], escape[1]) @@ -737,6 +745,48 @@ def _carries_engine_ref_inventory(value: object) -> bool: return True +def _embedded_engine_refs(value: object, references: set[str]) -> set[str] | None: + """The engine references a value embeds, in any JSON spelling. + + A string embeds a reference when it, or what its JSON escapes decode to + layer by layer, holds the reference or the reference's JSON-escaped text. + Every layer is checked, and a JSON string literal decodes to the same text + as its span of the enclosing string, so this finds every reference + ``_embeds_engine_ref`` finds by decoding JSON from each quote or brace. + Each layer is one pass over the string, so a string costs at most + ``_INVENTORY_LAYERS`` passes, plus one. A string still escaped after them + gives ``None``, so the caller refuses rather than guess. + """ + + if not references: + return set() + escaped = { + reference: json.dumps(reference, ensure_ascii=False)[1:-1] + for reference in references + } + found: set[str] = set() + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, Mapping): + pending.extend(part for pair in item.items() for part in pair) + elif isinstance(item, list | tuple): + pending.extend(item) + elif isinstance(item, str): + for _ in range(_INVENTORY_LAYERS + 1): + found.update( + reference + for reference, text in escaped.items() + if reference in item or text in item + ) + item, decoded = _JSON_ESCAPE.subn(_json_unescape, item) + if not decoded: + break + else: + return None + return found + + def _is_text(value: object) -> bool: return isinstance(value, str) and bool(value) @@ -1320,6 +1370,16 @@ def _validate_precalibration_ancestry( } for name, row in bindings.items() } + # The composed pass reads a node's stage bindings from these references. + binding_engine_refs = ( + {} + if config is None + else { + binding: config.engine_refs[binding] + for binding in bindings + if binding in config.engine_refs + } + ) roots = { name for name, row in rows.items() if row["kernel"].startswith("calibrate.") } @@ -1361,14 +1421,19 @@ def check( node_bindings = {} for name, row in rows.items(): if composed: + found = _embedded_engine_refs( + row.get("params", {}), set(binding_engine_refs.values()) + ) + if found is None: + raise ValueError( + f"Transport calibration ancestry: node {name!r} has a " + f"parameter still JSON-escaped after {_INVENTORY_LAYERS} " + "layers, so its stage bindings cannot be read." + ) node_bindings[name] = { binding - for binding in bindings - if config is not None - and binding in config.engine_refs - and _embeds_engine_ref( - row.get("params", {}), {config.engine_refs[binding]} - ) + for binding, reference in binding_engine_refs.items() + if reference in found } else: node_bindings[name] = { @@ -1387,9 +1452,10 @@ def check( ancestors = _ancestor_closure(predecessors, roots) # A stage declares only columns. A structural node also writes what # every later kernel of its version receives without a Slice: ids, - # memberships, weights, strata, the frame metadata and the mass log. - # So a stage node upstream of calibration changes no population, and - # an EXPAND's cells are never a stage's. + # memberships, weights, strata and the frame metadata. So a stage node + # upstream of calibration changes no population, and an EXPAND's cells + # are never a stage's. Any node can append to the mass log, which no + # declaration shows; the transport runner refuses that (cli.py). for name, row in rows.items(): own = sorted(node_bindings[name].intersection(stages)) structural = row.get("structural", StructuralDelta.NONE) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 407ee55e3c..67e8f42edc 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -45,6 +45,7 @@ from collections.abc import Iterator, Mapping, Sequence from dataclasses import replace from pathlib import Path +from types import SimpleNamespace from unittest.mock import Mock import numpy as np @@ -4622,14 +4623,20 @@ def _expand_case(shape): ``claim_in_stage``, where ``claim`` is one of A's own nodes. In ``stage_expand`` the EXPAND is one of A's nodes too. ``private_read_first`` declares calibration before ``claim``, so - calibration is the first node that receives A's values. + calibration is the first node that receives A's values. In ``two_claims`` + the EXPAND overlays ``other`` too and ``claim`` claims it after ``cell``, + so the second claim receives A's private ``other``. """ binding = _synthetic_binding("A", "person", ["cell", "other", "A_public"]) public = not shape.startswith("private") and shape != "claim_in_stage" stage = _synthetic_stage(binding, ["A_public", "cell"] if public else ["A_public"]) cells = (("person", "cell", "float64"),) + claims = ("person.cell",) + if shape == "two_claims": + cells += (("person", "other", "float64"),) + claims += ("person.other",) expand_params = {"expand_cells": [list(cell) for cell in cells]} - claim_params = {"materialized_expand_outputs": ["person.cell"]} + claim_params = {"materialized_expand_outputs": list(claims)} if shape == "stage_expand": expand_params["engine_ref"] = {"binding": "A", "field": "engine_ref"} if shape == "claim_in_stage": @@ -4695,7 +4702,7 @@ def _expand_case(shape): "synthetic.claim@1", population="a_expand", params={ - "materialized_expand_outputs": ("person.cell",), + "materialized_expand_outputs": claims, **({"engine_ref": "A-ref"} if shape == "claim_in_stage" else {}), }, outputs=( @@ -4795,6 +4802,15 @@ def _structural_stage_refusal(node, kind): "composed": _structural_stage_refusal("a_expand", "expand"), }, ), + # Every claim counts, not only the first: the second receives A's + # private ``other``. + ( + "two_claims", + { + "declaration": _stage_output_refusal("claim"), + "composed": _stage_output_refusal("claim"), + }, + ), ], ) def test_precalibration_ancestry_expand_claims_read_their_cells(shape, outcomes): @@ -5193,6 +5209,233 @@ def test_precalibration_ancestry_refuses_a_stage_membership_calibration_receives assert calibrate.received == [[10, 20]] +_MASS_RECORD = { + "entity": "household", + "old_total": 1.0, + "new_total": 1.0, + "declared_factor": None, + "reason": "7", +} + + +def _mass_receipt(context): + """A conserving household mass record whose reason is 7, when asked for.""" + if not context.params.get("append"): + return {} + return {"frame_mass_log_append": [dict(_MASS_RECORD)]} + + +class _MassLogCreate(KernelBase): + """Person 1 in household 10, whose ``x`` is 1.""" + + ref = "synthetic.mass_log_create@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + import pandas as pd + + from microcosm.frame import EntitySchema, Frame, WeightKind, Weights + + return KernelResult( + frame=Frame( + { + "person": pd.DataFrame( + { + "person_id": np.array([1], dtype="int64"), + "person_household_id": np.array([10], dtype="int64"), + } + ), + "household": pd.DataFrame( + { + "household_id": np.array([10], dtype="int64"), + "x": np.array([1], dtype="int64"), + } + ), + }, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.array([1.0]), WeightKind.DESIGN)}, + pd.Series(["s"], name="stratum"), + ) + ) + + +class _MassLogColumn(KernelBase): + """Owns one person column, 7, and appends the mass record when asked.""" + + ref = "synthetic.mass_log_column@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + import pandas as pd + + (owned,) = context.node.outputs + return KernelResult( + columns={ + (owned.entity, owned.column): pd.Series( + [7], index=pd.Index([1], name="person_id"), dtype="int64" + ) + }, + receipt=_mass_receipt(context), + ) + + +class _MassLogCalibrate(KernelBase): + """Weights the household by the reason of the last mass record it receives.""" + + ref = "calibrate.mass_log@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def __init__(self): + self.received = [] + + def run(self, context): + from microcosm.frame import WeightKind, Weights + + log = context.frame_mass_log + self.received.append([record.reason for record in log]) + return KernelResult( + weights=Weights( + np.array([float(log[-1].reason) if log else 1.0]), + WeightKind.IMPORTANCE, + ), + receipt=_mass_receipt(context), + ) + + +@pytest.mark.parametrize("restored", [False, True], ids=["cold", "restored"]) +@pytest.mark.parametrize("appender", [None, "a", "b", "cal"]) +def test_transport_run_refuses_a_frame_mass_log_append(tmp_path, appender, restored): + """A transport run takes the frame mass log from CREATE only. + + A node's receipt can append to the mass log (``frame_mass_log_append``), + and every later structural node of its lineage receives the log without a + ``Slice``. Round 5's case: stage A's node ``a`` owns only an unpublished + column, yet the record it appends sets calibration's weight. No + declaration shows the log, so both guard passes admit the graph, and a + plain run delivers the value. The transport runner refuses any node that + appends, a stage's (``a``) or not (``b``), calibration included, whether + it ran or was restored from a store such a plain run wrote. + """ + + def params(name, **values): + return {**values, **({"append": True} if name == appender else {})} + + nodes = ( + Node( + "create", + _MassLogCreate.ref, + structural=StructuralDelta.CREATE, + sources=("donor",), + outputs=(Owned("household", "x", "int64"),), + ), + Node( + "a", + _MassLogColumn.ref, + population="create", + params=params("a", engine_ref="A-ref"), + outputs=(Owned("person", "private", "int64"),), + ), + Node( + "b", + _MassLogColumn.ref, + population="create", + params=params("b"), + outputs=(Owned("person", "b_out", "int64"),), + ), + Node( + "cal", + _MassLogCalibrate.ref, + structural=StructuralDelta.REWEIGHT, + base="create", + params=params("cal"), + inputs=(Slice("household", ("x",)),), + weights=WeightTransition("household", "importance", mass="free"), + mass="free", + ), + ) + binding = _synthetic_binding("A", "person", ["public"]) + rows = [ + { + "id": "a", + "kernel": _MassLogColumn.ref, + "population": "create", + "params": {"engine_ref": {"binding": "A", "field": "engine_ref"}}, + "outputs": [{"entity": "person", "column": "private"}], + }, + { + "id": "b", + "kernel": _MassLogColumn.ref, + "population": "create", + "outputs": [{"entity": "person", "column": "b_out"}], + }, + { + "id": "cal", + "kernel": _MassLogCalibrate.ref, + "structural": "reweight", + "base": "create", + "inputs": [{"entity": "household", "columns": ["x"]}], + }, + ] + spec = _synthetic_stage_spec( + [binding], [_synthetic_stage(binding, ["public"])], rows + ) + graph = Graph("synthetic", (SourceRef("donor", "raw-bytes-v1"),), nodes) + config = TransportGraphConfig( + engine_refs={"A": "A-ref"}, create_outputs=nodes[0].outputs + ) + for candidate in (None, graph): + assert ( + _precalibration_outcome( + compose_module.validate_transport_calibration_ancestry, + spec, + graph=candidate, + config=config, + ) + is None + ) + calibrate = _MassLogCalibrate() + kernels = KernelRegistry() + for kernel in (_MassLogCreate(), _MassLogColumn(), calibrate): + kernels.register(kernel) + donor = tmp_path / "donor.txt" + donor.write_text("synthetic") + store = ContentStore(tmp_path / "store") + if restored: + run_graph( + compile_graph(graph), sources={"donor": donor}, store=store, kernels=kernels + ) + assert calibrate.received == [["7"] if appender in {"a", "b"} else []] + calibrate.received.clear() + + def run(): + return cli_module._run( + graph, + SimpleNamespace(kernels=kernels), + {"donor": donor}, + store, + "require" if restored else "auto", + ) + + if appender is None: + manifest = run() + assert all(receipt.hit for receipt in manifest.nodes.values()) is restored + else: + with pytest.raises( + ValueError, + match=re.escape( + f"Transport node {appender!r} changed the frame mass log it received" + ), + ): + run() + # Calibration never runs on an appended log under the runner. + ran = not restored and appender in {None, "cal"} + assert calibrate.received == ([[]] if ran else []) + + def _guard_writers(graph, compiled): """Each node's reads as the composed pass resolves them, as (entity, column, writer).""" return compose_module._precalibration_compiled_reads( @@ -5219,7 +5462,10 @@ def _executor_writers(graph, compiled): [ *((_writer_case, shape) for shape in ("shadowed", "masked", "deep_masked")), (_writer_case, "rewrite_read_first"), - *((_expand_case, shape) for shape in ("public", "private", "claim_in_stage")), + *( + (_expand_case, shape) + for shape in ("public", "private", "claim_in_stage", "two_claims") + ), ], ids=lambda value: getattr(value, "__name__", value), ) @@ -5750,6 +5996,150 @@ def test_precalibration_ancestry_inventory_scan_is_linear(scope): assert time.perf_counter() - started < 1 +def test_precalibration_ancestry_composed_stage_scan_is_linear(monkeypatch): + """The composed pass reads stage bindings in one pass per escape layer. + + Round 5's case: a person stage, a node with a harmless parameter full of + escaped quotes, then calibration. Matching each binding's engine + reference by decoding JSON from every quote made this composed path + quadratic (1.8 s at 16 KB on a loaded host). It now decodes no JSON from + an offset of any parameter here, so the count is exact, not timed. + """ + + class Counted(json.JSONDecoder): + offsets = 0 + + def raw_decode(self, text, idx=0): + type(self).offsets += 1 + return super().raw_decode(text, idx) + + monkeypatch.setattr(json, "JSONDecoder", Counted) + note = "prefix" + r"\"" * 64_000 + binding = {**_synthetic_binding("p", "person", ["p_out"]), "engine_ref": "p-ref"} + rows = [ + { + "id": "probe", + "kernel": "synthetic.probe@1", + "population": "create", + "params": {"engine_ref": {"binding": "p", "field": "engine_ref"}}, + "outputs": [{"entity": "person", "column": "p_out"}], + }, + { + "id": "mid", + "kernel": "synthetic.mid@1", + "population": "create", + "params": {"note": note}, + "inputs": [{"entity": "person", "columns": ["p_out"]}], + "outputs": [{"entity": "person", "column": "mid_out"}], + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": "create", + "inputs": [{"entity": "person", "columns": ["mid_out"]}], + }, + ] + nodes = [ + Node( + "probe", + "synthetic.probe@1", + population="create", + params={"engine_ref": "p-ref"}, + outputs=(Owned("person", "p_out", "float64"),), + ), + Node( + "mid", + "synthetic.mid@1", + population="create", + params={"note": note}, + inputs=(Slice("person", ("p_out",)),), + outputs=(Owned("person", "mid_out", "float64"),), + ), + Node( + "cal", + "calibrate.synthetic@1", + population="create", + inputs=(Slice("person", ("mid_out",)),), + ), + ] + spec = _synthetic_stage_spec( + [binding], [_synthetic_stage(binding, ["p_out"])], rows + ) + started = time.perf_counter() + outcomes = _synthetic_outcomes(spec, nodes, {"p": "p-ref"}) + elapsed = time.perf_counter() - started + assert outcomes == {"declaration": None, "composed": None} + assert Counted.offsets == 0 + assert elapsed < 1 + + +def test_precalibration_ancestry_unreadable_composed_bindings_refuse(): + """A parameter still escaped after 32 layers refuses in the composed pass.""" + spec, nodes, _ = _structural_stage_case("create", bound=False) + deep = _escape_layers('"note"', 33) + nodes = [ + replace(node, params={**node.params, "note": deep}) + if node.id == "cal" + else node + for node in nodes + ] + assert _synthetic_outcomes(spec, nodes, {"A": "A-ref"}) == { + "declaration": None, + "composed": ( + ValueError, + "Transport calibration ancestry: node 'cal' has a parameter still " + "JSON-escaped after 32 layers, so its stage bindings cannot be read.", + ), + } + + +_SCANNED_REFS = ( + "p-ref", + '{"format":"synthetic","module":"a/b.yaml"}', + '{"module":"\U0001f600/\\\\ \\"q\\""}', +) + + +@st.composite +def _spelled_engine_ref(draw): + """An engine reference behind up to four random JSON spellings.""" + reference = draw(st.sampled_from(_SCANNED_REFS)) + value = reference + for _ in range(draw(st.integers(min_value=0, max_value=4))): + step = draw(st.sampled_from(("dumps", "ascii", "object", "list", "text"))) + if step == "dumps": + value = json.dumps(value, ensure_ascii=False) + elif step == "ascii": + value = json.dumps(value) + elif step == "object": + key = draw(st.text(max_size=3)) + value = json.dumps({key: value}, ensure_ascii=draw(st.booleans())) + elif step == "list": + value = json.dumps([draw(st.integers()), value]) + else: + value = draw(st.text(max_size=4)) + value + draw(st.text(max_size=4)) + return reference, value + + +@settings(max_examples=300, deadline=None) +@given(case=_spelled_engine_ref(), as_key=st.booleans()) +def test_precalibration_stage_scan_finds_what_json_decoding_finds(case, as_key): + """The layered scan finds every reference ``_embeds_engine_ref`` finds. + + ``_embeds_engine_ref`` decodes JSON from every quote and brace; the + composed pass's membership scan decodes escapes layer by layer instead. + Over nested JSON, ASCII-escaped (surrogate pairs included), object, list + and embedded-in-text spellings, the layered scan never misses a reference + the decoding scan finds. + """ + reference, value = case + params = {value: 1} if as_key else {"note": [value]} + found = compose_module._embedded_engine_refs(params, {reference}) + assert found is not None + if compose_module._embeds_engine_ref(params, {reference}): + assert reference in found + + def _escape_layers(text, layers): """``text`` behind ``layers`` layers of JSON escapes, one decode each. From 973c674c4cba44d28100dd0b0b195fc37d364529 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 16:29:31 -0700 Subject: [PATCH 17/20] Scope the escape bound to calibration's ancestry; pin escaped-text matches Fixes the round-6 Sol review of 1d1801edb (APPROVE, one minor and a nit). The composed pass refused a node with a parameter still JSON-escaped after 32 layers before it knew the ancestry, so a node after calibration, which faces only the shape checks, refused on a harmless note. The ancestry is now computed first, and the bound refuses only upstream of calibration, the calibration nodes included. No check reads a non-ancestor's bindings. The scan's escaped-text match had no test that needed it. A reference of a newline, a backslash and "n", written as its escaped text after a stray backslash, is lost by decoding, which pairs that backslash with the text's first escape; without the match the node left its stage and its derived output refused as unproduced. New tests: that guard case, a property that a reference's escaped text counts in any surrounding text (with the review's case and a non-ASCII one as explicit examples), and the decoding-scan differential over random references with quoteless and stray-backslash spellings. The bound's test now covers a stage node, the calibration node and a post-calibration node. Co-Authored-By: Claude Opus 5.5 --- docs/transport-precalibration-stages.md | 27 ++- .../src/microcosm/build/transport/compose.py | 30 ++- .../shared/test_transport_composed_graph.py | 216 ++++++++++++++++-- 3 files changed, 235 insertions(+), 38 deletions(-) diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index e75bacc6e2..73de006be5 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -115,12 +115,13 @@ the stage's binding or any `{"binding": id}` selector in its parameters names it. In the composed pass, a node belongs to a stage when its parameters carry the binding's prepared engine reference, as written, JSON-escaped once, or after JSON escapes are decoded, layer by layer, up to 32 layers, as for the -inventory below. Each layer is one linear pass over a string. A parameter -still escaped after 32 layers refuses, because its stage bindings cannot be -read. A node that names a listed and an unlisted benefit-unit binding -refuses. A parameter carrying the whole prepared `engine_refs` inventory is -bookkeeping, not a stage invocation, and still refuses upstream of -calibration. +inventory below. Each layer is one linear pass over a string. Upstream of +calibration, a parameter still escaped after 32 layers refuses, because its +node's stage bindings cannot be read; after calibration no binding is read, +so such a node faces only the shape checks. A node that names a listed and an +unlisted benefit-unit binding refuses. A parameter carrying the whole prepared +`engine_refs` inventory is bookkeeping, not a stage invocation, and still +refuses upstream of calibration. ## Rules @@ -251,10 +252,11 @@ These are tested, as properties and examples, in produces, and otherwise the first offending node in declaration order. The composed pass runs only after the declaration pass admits: it checks the entitlement rows' shapes and reads the compiled EXPAND cells and claims, - refuses the first node with a parameter still escaped after 32 layers, then - applies the same order to the graph's nodes. One message is not - fixed: when several links name undeclared nodes, which of them a refusal - names follows set iteration order, as in the guard without stages. + refuses the first node upstream of calibration with a parameter still + escaped after 32 layers, then applies the same order to the graph's nodes. + One message is not fixed: when several links name undeclared nodes, which of + them a refusal names follows set iteration order, as in the guard without + stages. - With stages listed, the checks never read a malformed field so as to admit more than composition would. Every field they read is validated first, refusing naming its node, or read exactly as composition reads it. The two @@ -271,7 +273,10 @@ These are tested, as properties and examples, in refuses it before persisting the node. - The composed pass's stage membership finds every engine reference that decoding JSON from each quote or brace finds (`_embeds_engine_ref`), a - property tested over nested JSON, ASCII and embedded spellings. + property tested over fixed and random references in nested JSON, ASCII, + quoteless and embedded spellings. A reference's JSON-escaped text counts + wherever it appears, even after a stray backslash that decoding pairs with + the text's first escape. - The composed pass gives every column read the writers the executor's static writer lists give it (`microcosm.graph.executor._input_writers`), on every generated graph, the writer and EXPAND examples, and the composed diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 0ba2daa27c..426264e7bc 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -699,7 +699,8 @@ def _is_unit_entity(entity: object, unit_entities: tuple[object, ...]) -> bool: # ``\u005c`` grows only linearly: 33 layers of the quoted key take 344 # characters. A string still escaped after the last layer therefore counts as # carrying the inventory, whatever it decodes to, and leaves a node's stage -# bindings unread, which refuses: both scans fail closed. +# bindings unread. Upstream of calibration, where both are read, either +# refuses: both scans fail closed. _INVENTORY_LAYERS = 32 @@ -755,7 +756,8 @@ def _embedded_engine_refs(value: object, references: set[str]) -> set[str] | Non ``_embeds_engine_ref`` finds by decoding JSON from each quote or brace. Each layer is one pass over the string, so a string costs at most ``_INVENTORY_LAYERS`` passes, plus one. A string still escaped after them - gives ``None``, so the caller refuses rather than guess. + gives ``None``, and upstream of calibration the caller refuses rather than + guess. """ if not references: @@ -1418,6 +1420,11 @@ def check( composed: bool, derived: bool, ) -> None: + # Only nodes upstream of calibration are pre-calibration. A stage node + # there brings its own ancestors into this closure, so each listed + # stage's ancestry passes the same checks; a stage binding's nodes + # after calibration (extension checkpoints) are outputs, not inputs. + ancestors = _ancestor_closure(predecessors, roots) node_bindings = {} for name, row in rows.items(): if composed: @@ -1425,11 +1432,15 @@ def check( row.get("params", {}), set(binding_engine_refs.values()) ) if found is None: - raise ValueError( - f"Transport calibration ancestry: node {name!r} has a " - f"parameter still JSON-escaped after {_INVENTORY_LAYERS} " - "layers, so its stage bindings cannot be read." - ) + if name in ancestors: + raise ValueError( + f"Transport calibration ancestry: node {name!r} has a " + "parameter still JSON-escaped after " + f"{_INVENTORY_LAYERS} layers, so its stage bindings " + "cannot be read." + ) + # The checks below read only an ancestor's bindings. + found = set() node_bindings[name] = { binding for binding, reference in binding_engine_refs.items() @@ -1445,11 +1456,6 @@ def check( } if row.get("rules_binding") in bindings: node_bindings[name].add(row["rules_binding"]) - # Only nodes upstream of calibration are pre-calibration. A stage node - # there brings its own ancestors into this closure, so each listed - # stage's ancestry passes the same checks; a stage binding's nodes - # after calibration (extension checkpoints) are outputs, not inputs. - ancestors = _ancestor_closure(predecessors, roots) # A stage declares only columns. A structural node also writes what # every later kernel of its version receives without a Slice: ids, # memberships, weights, strata and the frame metadata. So a stage node diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 67e8f42edc..4ddf9c0e41 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -50,7 +50,7 @@ import numpy as np import pytest -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st from microcosm.build.transport import cli as cli_module @@ -6073,23 +6073,164 @@ def raw_decode(self, text, idx=0): assert elapsed < 1 -def test_precalibration_ancestry_unreadable_composed_bindings_refuse(): - """A parameter still escaped after 32 layers refuses in the composed pass.""" - spec, nodes, _ = _structural_stage_case("create", bound=False) +@pytest.mark.parametrize( + ("carrier", "refused"), [("a_source", True), ("cal", True), ("post", False)] +) +def test_precalibration_ancestry_unreadable_composed_bindings_refuse_upstream( + carrier, refused +): + """A parameter still escaped after 32 layers refuses upstream of calibration. + + Its node's stage bindings cannot be read, so on stage A's node or on the + calibration node the composed pass refuses. ``post`` reads calibration's + output, so none of its bindings is read and it faces only the shape + checks; round 6 found it refused before the ancestry was known. + """ + binding = _synthetic_binding("A", "person", ["A_public"]) + rows = [ + { + "id": "a_source", + "kernel": RULES, + "population": "create", + "rules_binding": "A", + "outputs": [{"entity": "person", "column": "A_public"}], + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": "create", + "inputs": [{"entity": "person", "columns": ["A_public", "seed"]}], + "outputs": [{"entity": "person", "column": "cal_out"}], + }, + { + "id": "post", + "kernel": "synthetic.post@1", + "population": "create", + "inputs": [{"entity": "person", "columns": ["cal_out"]}], + }, + ] + nodes = [ + Node( + "a_source", + RULES, + population="create", + params={"engine_ref": "A-ref"}, + outputs=(Owned("person", "A_public", "float64"),), + ), + Node( + "cal", + "calibrate.synthetic@1", + population="create", + inputs=(Slice("person", ("A_public", "seed")),), + outputs=(Owned("person", "cal_out", "float64"),), + ), + Node( + "post", + "synthetic.post@1", + population="create", + inputs=(Slice("person", ("cal_out",)),), + ), + ] deep = _escape_layers('"note"', 33) nodes = [ replace(node, params={**node.params, "note": deep}) - if node.id == "cal" + if node.id == carrier else node for node in nodes ] + spec = _synthetic_stage_spec( + [binding], [_synthetic_stage(binding, ["A_public"])], rows + ) + refusal = ( + ValueError, + f"Transport calibration ancestry: node {carrier!r} has a parameter " + "still JSON-escaped after 32 layers, so its stage bindings cannot be " + "read.", + ) assert _synthetic_outcomes(spec, nodes, {"A": "A-ref"}) == { "declaration": None, - "composed": ( - ValueError, - "Transport calibration ancestry: node 'cal' has a parameter still " - "JSON-escaped after 32 layers, so its stage bindings cannot be read.", + "composed": refusal if refused else None, + } + + +def test_precalibration_ancestry_composed_stage_reads_escaped_reference_text(): + """A node carrying its reference's escaped text after a backslash is a member. + + Round 6: the reference is a newline, a backslash and ``n``, and ``mid``'s + note is a backslash, then the reference's JSON-escaped text. Decoding + pairs the note's backslash with the text's first, so no decoded layer + holds the reference. ``_embeds_engine_ref`` matches the escaped text as + written, and so must the composed pass, or ``mid`` leaves stage p and its + derived output refuses as unproduced. + """ + reference = "\n\\n" + note = "\\" + json.dumps(reference, ensure_ascii=False)[1:-1] + assert compose_module._embeds_engine_ref(note, {reference}) + assert compose_module._embedded_engine_refs(note, {reference}) == {reference} + binding = { + **_synthetic_binding("p", "person", ["p_out", "p_priv"]), + "engine_ref": reference, + } + rows = [ + { + "id": "probe", + "kernel": "synthetic.probe@1", + "population": "create", + "params": {"engine_ref": {"binding": "p", "field": "engine_ref"}}, + "outputs": [ + {"entity": "person", "column": "p_out"}, + {"entity": "person", "column": "p_priv"}, + ], + }, + { + "id": "mid", + "kernel": "synthetic.mid@1", + "population": "create", + "params": {"period": {"binding": "p", "field": "period"}, "note": note}, + "inputs": [{"entity": "person", "columns": ["p_priv"]}], + "outputs": [{"entity": "person", "column": "mid_out"}], + }, + { + "id": "cal", + "kernel": "calibrate.synthetic@1", + "population": "create", + "inputs": [{"entity": "person", "columns": ["mid_out"]}], + }, + ] + nodes = [ + Node( + "probe", + "synthetic.probe@1", + population="create", + params={"engine_ref": reference}, + outputs=( + Owned("person", "p_out", "float64"), + Owned("person", "p_priv", "float64"), + ), + ), + Node( + "mid", + "synthetic.mid@1", + population="create", + params={"period": "2026", "note": note}, + inputs=(Slice("person", ("p_priv",)),), + outputs=(Owned("person", "mid_out", "float64"),), ), + Node( + "cal", + "calibrate.synthetic@1", + population="create", + inputs=(Slice("person", ("mid_out",)),), + ), + ] + spec = _synthetic_stage_spec( + [binding], + [_synthetic_stage(binding, ["p_out"], [("person", "mid_out")])], + rows, + ) + assert _synthetic_outcomes(spec, nodes, {"p": reference}) == { + "declaration": None, + "composed": None, } @@ -6097,16 +6238,25 @@ def test_precalibration_ancestry_unreadable_composed_bindings_refuse(): "p-ref", '{"format":"synthetic","module":"a/b.yaml"}', '{"module":"\U0001f600/\\\\ \\"q\\""}', + # Round 6: decoding after a stray backslash loses it (see below). + "\n\\n", ) +# Characters whose escapes start with a backslash, and some that don't. +_ESCAPE_CHARACTERS = '\\"\nn{\u00e9' @st.composite def _spelled_engine_ref(draw): """An engine reference behind up to four random JSON spellings.""" - reference = draw(st.sampled_from(_SCANNED_REFS)) + reference = draw( + st.sampled_from(_SCANNED_REFS) + | st.text(alphabet=_ESCAPE_CHARACTERS, min_size=1, max_size=6) + ) value = reference for _ in range(draw(st.integers(min_value=0, max_value=4))): - step = draw(st.sampled_from(("dumps", "ascii", "object", "list", "text"))) + step = draw( + st.sampled_from(("dumps", "ascii", "object", "list", "content", "text")) + ) if step == "dumps": value = json.dumps(value, ensure_ascii=False) elif step == "ascii": @@ -6116,8 +6266,16 @@ def _spelled_engine_ref(draw): value = json.dumps({key: value}, ensure_ascii=draw(st.booleans())) elif step == "list": value = json.dumps([draw(st.integers()), value]) + elif step == "content": + # A string's content as JSON writes it, without its quotes. + value = json.dumps(value, ensure_ascii=draw(st.booleans()))[1:-1] else: - value = draw(st.text(max_size=4)) + value + draw(st.text(max_size=4)) + text = st.text(max_size=4) | st.text( + alphabet=_ESCAPE_CHARACTERS, max_size=4 + ) + # A stray backslash pairs with the value's first escape. + stray = draw(st.sampled_from(("", "\\"))) + value = draw(text) + stray + value + draw(text) return reference, value @@ -6128,9 +6286,10 @@ def test_precalibration_stage_scan_finds_what_json_decoding_finds(case, as_key): ``_embeds_engine_ref`` decodes JSON from every quote and brace; the composed pass's membership scan decodes escapes layer by layer instead. - Over nested JSON, ASCII-escaped (surrogate pairs included), object, list - and embedded-in-text spellings, the layered scan never misses a reference - the decoding scan finds. + Over fixed and random references, and nested JSON, ASCII-escaped + (surrogate pairs included), object, list, quoteless and embedded-in-text + spellings, the layered scan never misses a reference the decoding scan + finds. """ reference, value = case params = {value: 1} if as_key else {"note": [value]} @@ -6140,6 +6299,33 @@ def test_precalibration_stage_scan_finds_what_json_decoding_finds(case, as_key): assert reference in found +@settings(max_examples=300, deadline=None) +@given( + reference=st.text(alphabet=_ESCAPE_CHARACTERS, min_size=1, max_size=4), + prefix=st.text(alphabet=_ESCAPE_CHARACTERS, max_size=3), + suffix=st.text(alphabet=_ESCAPE_CHARACTERS, max_size=3), +) +@example(reference="\n\\n", prefix="\\", suffix="") +@example(reference="\n\\n\u00e9", prefix="\\", suffix="") +def test_precalibration_stage_scan_counts_escaped_text_anywhere( + reference, prefix, suffix +): + """A reference's JSON-escaped text counts in any surrounding text. + + Decoding alone can miss it: after a stray backslash the text's first + escape pairs with that backslash, so no decoded layer holds the reference + (round 6's example, a newline, a backslash and ``n``). ``_embeds_engine_ref`` + matches the text as written, and so does the layered scan, in the text + ``json.dumps`` writes without ASCII escapes, as ``_embeds_engine_ref`` + matches it. + """ + value = prefix + json.dumps(reference, ensure_ascii=False)[1:-1] + suffix + assert compose_module._embeds_engine_ref(value, {reference}) + found = compose_module._embedded_engine_refs({"note": value}, {reference}) + assert found is not None + assert reference in found + + def _escape_layers(text, layers): """``text`` behind ``layers`` layers of JSON escapes, one decode each. From 3b97bea9ab1a9bf0f9bf3c4d60e76c1872609b63 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 21:08:13 -0700 Subject: [PATCH 18/20] Word the round-7 review's two points exactly (comments and docs only) The round-7 Sol review of 973c674c4 (APPROVE, no findings) noted two wording points. The structural rule's loop computes a non-ancestor's bindings and discards them, so the checks below the escape bound use only an ancestor's bindings rather than reading only them. A reference's escaped text counts wherever it appears only within the 32-layer bound: a parameter with any string still escaped after it gives no membership, and upstream of calibration it refuses. The docs bullet and the property's docstring say so. No code changes: compose.py and cli.py parse to the same AST, and the test file differs only in one docstring. Co-Authored-By: Claude Opus 5.5 --- docs/transport-precalibration-stages.md | 4 +++- .../microcosm-build/src/microcosm/build/transport/compose.py | 2 +- .../tests/engine_free/shared/test_transport_composed_graph.py | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index 73de006be5..2af0aea7b8 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -276,7 +276,9 @@ These are tested, as properties and examples, in property tested over fixed and random references in nested JSON, ASCII, quoteless and embedded spellings. A reference's JSON-escaped text counts wherever it appears, even after a stray backslash that decoding pairs with - the text's first escape. + the text's first escape. That holds within the 32-layer bound: a parameter + with any string still escaped after it gives no membership, and upstream + of calibration it refuses. - The composed pass gives every column read the writers the executor's static writer lists give it (`microcosm.graph.executor._input_writers`), on every generated graph, the writer and EXPAND examples, and the composed diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 426264e7bc..7362e8ee37 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -1439,7 +1439,7 @@ def check( f"{_INVENTORY_LAYERS} layers, so its stage bindings " "cannot be read." ) - # The checks below read only an ancestor's bindings. + # The checks below use only an ancestor's bindings. found = set() node_bindings[name] = { binding diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 4ddf9c0e41..12ea19a968 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -6317,7 +6317,9 @@ def test_precalibration_stage_scan_counts_escaped_text_anywhere( (round 6's example, a newline, a backslash and ``n``). ``_embeds_engine_ref`` matches the text as written, and so does the layered scan, in the text ``json.dumps`` writes without ASCII escapes, as ``_embeds_engine_ref`` - matches it. + matches it. That holds within the 32-layer bound: a parameter with any + string still escaped after it gives ``None``, whatever was found. These + strings, at most 14 characters, cannot reach it. """ value = prefix + json.dumps(reference, ensure_ascii=False)[1:-1] + suffix assert compose_module._embeds_engine_ref(value, {reference}) From 91d46e1d1d62a901610177f52b97f9354fed1a41 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 25 Oct 2026 21:44:40 -0700 Subject: [PATCH 19/20] Say "used", not "read", for post-calibration bindings (docs only) The round-8 Sol review of 3b97bea9a (APPROVE) found the round-7 wording point in two more places: the docs' membership paragraph and the bound test's docstring said a post-calibration node's bindings are not read. The composed pass scans every node, and the structural rule computes every node's bindings, but only an ancestor's affect the checks. Both now say the bindings are not used. No code changes: the test file differs only in that docstring. Co-Authored-By: Claude Opus 5.5 --- docs/transport-precalibration-stages.md | 2 +- .../tests/engine_free/shared/test_transport_composed_graph.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/transport-precalibration-stages.md b/docs/transport-precalibration-stages.md index 2af0aea7b8..cf992b9a57 100644 --- a/docs/transport-precalibration-stages.md +++ b/docs/transport-precalibration-stages.md @@ -117,7 +117,7 @@ the binding's prepared engine reference, as written, JSON-escaped once, or after JSON escapes are decoded, layer by layer, up to 32 layers, as for the inventory below. Each layer is one linear pass over a string. Upstream of calibration, a parameter still escaped after 32 layers refuses, because its -node's stage bindings cannot be read; after calibration no binding is read, +node's stage bindings cannot be read; after calibration no binding is used, so such a node faces only the shape checks. A node that names a listed and an unlisted benefit-unit binding refuses. A parameter carrying the whole prepared `engine_refs` inventory is bookkeeping, not a stage invocation, and still diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 12ea19a968..4f612c8de9 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -6083,7 +6083,7 @@ def test_precalibration_ancestry_unreadable_composed_bindings_refuse_upstream( Its node's stage bindings cannot be read, so on stage A's node or on the calibration node the composed pass refuses. ``post`` reads calibration's - output, so none of its bindings is read and it faces only the shape + output, so none of its bindings is used and it faces only the shape checks; round 6 found it refused before the ancestry was known. """ binding = _synthetic_binding("A", "person", ["A_public"]) From 6197d743529f9d0fde2de365aa276cd17b412eb5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 07:43:59 -0700 Subject: [PATCH 20/20] Format compose.py with the repository's locked ruff uv.lock pins ruff 0.15.16, which wraps the multi-line lambda in _is_rows in parentheses. This is a layout-only change: the module's AST is identical before and after. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/build/transport/compose.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 7362e8ee37..080c6f70f7 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -806,8 +806,10 @@ def _is_rows(value: object, **fields: object) -> bool: """A list of objects whose named fields each pass their check.""" return _is_list( value, - lambda row: isinstance(row, Mapping) - and all(check(row.get(key)) for key, check in fields.items()), + lambda row: ( + isinstance(row, Mapping) + and all(check(row.get(key)) for key, check in fields.items()) + ), )