Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/nsc/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def _load_assets(profile_id: str, brand_id: str) -> tuple[dict, dict]:
def _make_ctx(brief: dict, out_dir: Path, router: Any = None) -> Any:
from nsc.passes import PassContext
from nsc.runtime.models import ModelRouter
from nsc.runtime.provenance import RunsStore, spec_fingerprint
from nsc.runtime.provenance import RunsStore, spec_domain_fingerprints, spec_fingerprint

profile, brand = _load_assets(brief.get("profile", ""), brief.get("brand", ""))
spec_files = list(Path("spec").rglob("*.py")) + list(Path("spec").rglob("*.yaml"))
Expand All @@ -34,6 +34,7 @@ def _make_ctx(brief: dict, out_dir: Path, router: Any = None) -> Any:
store=RunsStore(out_dir / "runs.db"),
ruleset_ver=spec_fingerprint(list(Path("spec/checks").rglob("*.yaml")))[:12],
spec_sha=spec_fingerprint(spec_files)[:12],
spec_shas=spec_domain_fingerprints(), # SW-02:缓存键分域;provenance 仍全量
promptset_ver=spec_fingerprint(prompts)[:12] if prompts else "seed",
out_dir=out_dir,
)
Expand Down
6 changes: 4 additions & 2 deletions src/nsc/eval/l1.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def _compile_brief(brief: dict[str, Any], retrieval_on: bool) -> dict[str, Any]:
from nsc.passes.pipeline import run_pipeline
from nsc.runtime.ir_io import build_view
from nsc.runtime.models import ModelRouter
from nsc.runtime.provenance import RunsStore, spec_fingerprint
from nsc.runtime.provenance import RunsStore, spec_domain_fingerprints, spec_fingerprint

profile = yaml.safe_load(Path(f"profiles/{brief.get('profile', '')}.yaml").read_text("utf-8"))
brand = yaml.safe_load(Path(f"brands/{brief.get('brand', '')}/brand.yaml").read_text("utf-8"))
Expand All @@ -122,6 +122,7 @@ def _compile_brief(brief: dict[str, Any], retrieval_on: bool) -> dict[str, Any]:
store=RunsStore(Path("out") / "eval_runs.db"),
ruleset_ver=spec_fingerprint(list(Path("spec/checks").rglob("*.yaml")))[:12],
spec_sha=spec_fingerprint(spec_files)[:12],
spec_shas=spec_domain_fingerprints(), # SW-02:缓存键分域
out_dir=Path("out") / "eval",
)
if retrieval_on:
Expand Down Expand Up @@ -294,7 +295,7 @@ def _compile_for_judge(brief: dict[str, Any]) -> dict[str, Any]:
from nsc.passes import PassContext, PassFailure
from nsc.passes.pipeline import run_pipeline
from nsc.runtime.models import ModelRouter
from nsc.runtime.provenance import RunsStore, spec_fingerprint
from nsc.runtime.provenance import RunsStore, spec_domain_fingerprints, spec_fingerprint

profile = yaml.safe_load(Path(f"profiles/{brief.get('profile', '')}.yaml").read_text("utf-8"))
brand = yaml.safe_load(Path(f"brands/{brief.get('brand', '')}/brand.yaml").read_text("utf-8"))
Expand All @@ -307,6 +308,7 @@ def _compile_for_judge(brief: dict[str, Any]) -> dict[str, Any]:
store=RunsStore(Path("out") / "eval_runs.db"),
ruleset_ver=spec_fingerprint(list(Path("spec/checks").rglob("*.yaml")))[:12],
spec_sha=spec_fingerprint(spec_files)[:12],
spec_shas=spec_domain_fingerprints(), # SW-02:缓存键分域
out_dir=Path("out") / "eval",
)
try:
Expand Down
23 changes: 22 additions & 1 deletion src/nsc/passes/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@ def new_id() -> str:
return str(ULID())


#: 进缓存键的 spec 域(SW-02):只含影响生成结构的域;checks 由 ruleset_ver 覆盖。
CACHE_SPEC_DOMAINS = ("ir", "passes")
Comment on lines +50 to +51

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已修正(fae8bae):采纳"按 pass 声明域"方案——新增 PASS_EXTRA_SPEC_DOMAINS = {"p5_dialogue": ("rules",)}scoped_spec_sha(pass_name) 取基础域 ∪ 该 pass 的额外域。现在 spec/rules/L3_canonical 的编辑会使 p5 缓存失效、不牵连 p3 等不读该域的 pass。回归测试 test_rules_domain_only_invalidates_p5(规则编辑 → p5 键变、p3 键不变)。关于建议中的 p5 cache miss 回归测试:键级断言(cache_versions 的 spec_sha 组成变化)与 cache miss 是同义变形(cached_pass 用 cache_versions 组键),已由 test_cache.py 的失效语义测试 + 本测试共同覆盖。

#: 个别 Pass 的额外缓存依赖域(review 修正):p5 的 self-check 经
#: nsc.revise.revision_brief 读 spec/rules/L3_canonical(VOICE RULES 五节),
#: 该域编辑必须使 p5 缓存失效(ruleset_ver 只覆盖 spec/checks,管不到这里)。
PASS_EXTRA_SPEC_DOMAINS: dict[str, tuple[str, ...]] = {"p5_dialogue": ("rules",)}


@dataclass
class PassContext:
"""一次编译的运行上下文。所有版本号集中在这里,缓存键由 cache_versions 给出。"""
Expand All @@ -62,6 +70,8 @@ class PassContext:
seed: int | None = 1
out_dir: Path = Path("out")
run_id: str = ""
#: SW-02 分域 spec 指纹(domain → sha12)。空 = 旧语义(缓存键用全量 spec_sha)。
spec_shas: dict[str, str] = field(default_factory=dict)
#: T-16 检索服务(None = 禁用检索;set 后 pipeline 会往 p1/p2/p3/p5 注入 retrieved_cases)
retrieval: Any = None

Expand All @@ -73,6 +83,17 @@ def _model_cfg(self, pass_name: str) -> dict[str, Any]:
return {}
return self.router.resolve(self.tier_of(pass_name))

def scoped_spec_sha(self, pass_name: str = "") -> str:
"""缓存键用 spec 指纹:分域只取相关域(含该 Pass 的额外依赖域)。

任一必需域缺失(半套指纹)时回退全量 spec_sha——宁可多失效,不可少失效
(review 修正:空域拼出的 "ir:|passes:" 会静默削弱缓存失效条件)。
"""
domains = CACHE_SPEC_DOMAINS + PASS_EXTRA_SPEC_DOMAINS.get(pass_name, ())
if not self.spec_shas or any(d not in self.spec_shas for d in domains):
return self.spec_sha
return "|".join(f"{d}:{self.spec_shas[d]}" for d in domains)

def cache_versions(self, pass_name: str) -> dict[str, Any]:
cfg = self._model_cfg(pass_name)
return {
Expand All @@ -83,7 +104,7 @@ def cache_versions(self, pass_name: str) -> dict[str, Any]:
"model_id": str(cfg.get("model", "none")),
"temperature": float(cfg.get("temperature", 0.0)),
"seed": self.seed,
"spec_sha": self.spec_sha,
"spec_sha": self.scoped_spec_sha(pass_name),
}

def record_run(
Expand Down
18 changes: 18 additions & 0 deletions src/nsc/runtime/provenance.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,24 @@ def spec_fingerprint(paths: list[Path]) -> str:
return h.hexdigest()


def spec_domain_fingerprints(root: Path = Path("spec")) -> dict[str, str]:
"""SW-02 分域指纹:按 spec 顶层子域分别取 sha256[:12]。

任何小编订只让所属域的指纹变化;PassContext 据此把缓存键里的 spec_sha
缩到影响生成结构的域(ir/passes),避免无关域(rubrics/feedback/...)编辑
使全量内容缓存失效。checks 域由既有 ruleset_ver 单独覆盖;
全量指纹仍走 spec_fingerprint(runs.spec_sha 不弱化)。
"""
domains: dict[str, list[Path]] = {}
for p in [*root.rglob("*.py"), *root.rglob("*.yaml")]:
rel = p.relative_to(root)
domain = rel.parts[0] if len(rel.parts) > 1 else "root"
if domain == "__pycache__":
continue
domains.setdefault(domain, []).append(p)
return {d: spec_fingerprint(ps)[:12] for d, ps in sorted(domains.items())}


@dataclass(slots=True)
class RunRecord:
"""对应 runs 表的一行(D20)。"""
Expand Down
118 changes: 118 additions & 0 deletions tests/test_spec_domains.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
"""SW-02 spec_sha 分域哈希:任何 spec 小编订不得使全量内容缓存失效。

- provenance.spec_domain_fingerprints:按 spec 顶层子域分别取指纹;
- PassContext.cache_versions 的 spec_sha 只取影响生成结构的域(ir+passes),
checks 域由既有的 ruleset_ver 单独覆盖,rubrics/feedback/rules 等不进缓存键;
- runs 表的 spec_sha 仍是全量指纹(provenance 不弱化)。
"""

from __future__ import annotations

from pathlib import Path

import yaml

from nsc.runtime.provenance import spec_domain_fingerprints


def _mk_spec(root: Path) -> None:
(root / "checks").mkdir(parents=True)
(root / "rubrics").mkdir()
(root / "passes").mkdir()
(root / "ir").mkdir()
(root / "checks" / "c1.yaml").write_text("a: 1\n", "utf-8")
(root / "rubrics" / "r1.yaml").write_text("b: 1\n", "utf-8")
(root / "passes" / "signatures.py").write_text("x = 1\n", "utf-8")
(root / "ir" / "nodes.py").write_text("y = 1\n", "utf-8")
(root / "BUDGETS.yaml").write_text("lines: {}\n", "utf-8")


def test_domain_fingerprints_isolate_edits(tmp_path):
root = tmp_path / "spec"
_mk_spec(root)
before = spec_domain_fingerprints(root)
assert set(before) == {"checks", "rubrics", "passes", "ir", "root"}

(root / "rubrics" / "r1.yaml").write_text("b: 2\n", "utf-8") # 只动 rubrics
after = spec_domain_fingerprints(root)
changed = {d for d in before if before[d] != after[d]}
assert changed == {"rubrics"}, "小编订必须只让所属域指纹变化"


def _ctx(tmp_path, spec_shas=None):
from nsc.passes import PassContext
from nsc.runtime.provenance import RunsStore

return PassContext(
profile={"version": "1", "model_tiers": {}},
brand={"version": "1"},
router=None,
store=RunsStore(tmp_path / "runs.db"),
ruleset_ver="r",
spec_sha="full123",
spec_shas=spec_shas or {},
)


def test_cache_versions_uses_scoped_domains(tmp_path):
full = {"ir": "ir1", "passes": "pa1", "rubrics": "ru1", "checks": "ck1", "rules": "rl1"}
ctx = _ctx(tmp_path, full)
assert ctx.cache_versions("p3_beatsheet")["spec_sha"] == "ir:ir1|passes:pa1"
# 与生成无关的域变化不进缓存键(同 Pass 前后比对)
ctx2 = _ctx(tmp_path, {**full, "rubrics": "ru2", "checks": "ck2"})
assert (
ctx2.cache_versions("p5_dialogue")["spec_sha"]
== ctx.cache_versions("p5_dialogue")["spec_sha"]
)
assert (
ctx2.cache_versions("p3_beatsheet")["spec_sha"]
== ctx.cache_versions("p3_beatsheet")["spec_sha"]
)
# 影响生成结构的域变化必须进缓存键
ctx3 = _ctx(tmp_path, {**full, "ir": "ir2"})
assert (
ctx3.cache_versions("p3_beatsheet")["spec_sha"]
!= ctx.cache_versions("p3_beatsheet")["spec_sha"]
)


def test_rules_domain_only_invalidates_p5(tmp_path):
"""review 修正:p5 的 self-check 读 spec/rules/L3_canonical(VOICE RULES),
rules 域编辑必须使 p5 缓存失效;不读该域的 pass(p3)不受牵连。"""
full = {"ir": "ir1", "passes": "pa1", "rubrics": "ru1", "checks": "ck1", "rules": "rl1"}
ctx = _ctx(tmp_path, full)
changed = _ctx(tmp_path, {**full, "rules": "rl2"})
assert (
changed.cache_versions("p5_dialogue")["spec_sha"]
!= ctx.cache_versions("p5_dialogue")["spec_sha"]
), "rules 域变化必须使 p5 缓存失效"
assert (
changed.cache_versions("p3_beatsheet")["spec_sha"]
== ctx.cache_versions("p3_beatsheet")["spec_sha"]
), "rules 域变化不得牵连不读该域的 pass"
assert ctx.cache_versions("p5_dialogue")["spec_sha"] == "ir:ir1|passes:pa1|rules:rl1"


def test_partial_domain_map_falls_back_to_full_sha(tmp_path):
"""review 修正:半套分域指纹(缺必需域)必须回退全量 spec_sha,
不得拼出 'ir:|passes:' 之类静默削弱缓存失效条件的键。"""
ctx = _ctx(tmp_path, {"ir": "ir1"}) # 缺 passes(且缺 p5 需要的 rules)
assert ctx.cache_versions("p3_beatsheet")["spec_sha"] == "full123"
assert ctx.cache_versions("p5_dialogue")["spec_sha"] == "full123"


def test_cache_versions_falls_back_to_full_sha(tmp_path):
"""未提供分域指纹(旧测试/旧调用方)时保持原语义:全量 spec_sha。"""
ctx = _ctx(tmp_path)
assert ctx.cache_versions("p1_bible")["spec_sha"] == "full123"


def test_make_ctx_wires_domain_fingerprints(tmp_path):
from nsc.cli import _make_ctx

brief = yaml.safe_load(Path("examples/demo_tea/brief.yaml").read_text("utf-8"))
ctx = _make_ctx(brief, tmp_path / "out")
assert {"ir", "passes", "checks"} <= set(ctx.spec_shas)
scoped = ctx.cache_versions("p3_beatsheet")["spec_sha"]
assert scoped != ctx.spec_sha, "分域指纹接线后,缓存键不得再混入全量 spec_sha"
assert scoped.startswith("ir:") and "passes:" in scoped
Loading