diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e44cc46..b965d50 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -20,7 +20,7 @@ jobs: cache-dependency-path: requirements-dev.txt - name: Install validation dependencies run: python -m pip install -r requirements-dev.txt - - name: Run review-hardening regression tests + - name: Run regression and generated-view tests run: python -m unittest discover -s tests -v - name: Validate schemas, records, and references run: python tools/validate.py @@ -46,3 +46,11 @@ jobs: run: python tools/build_year.py 2025 --format json > /tmp/modelmap-2025.json - name: Build 2026 year view run: python tools/build_year.py 2026 --format json > /tmp/modelmap-2026.json + - name: Build browse catalog + run: python tools/build_catalog.py > /tmp/modelmap-catalog.json + - name: Build core knowledge graph + run: python tools/build_graph.py --scope core > /tmp/modelmap-graph-core.json + - name: Build full knowledge graph + run: python tools/build_graph.py --scope all > /tmp/modelmap-graph-all.json + - name: Build representative model entity view + run: python tools/build_entity.py release:gpt-5.6 > /tmp/modelmap-gpt-5.6.json diff --git a/schemas/catalog-view.schema.json b/schemas/catalog-view.schema.json new file mode 100644 index 0000000..36374f0 --- /dev/null +++ b/schemas/catalog-view.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://modelmap.dev/schemas/catalog-view.schema.json", + "title": "ModelMap Generated Catalog View", + "type": "object", + "required": ["generated_from", "filter_type", "counts", "records"], + "properties": { + "generated_from": {"const": "canonical-records"}, + "filter_type": {"type": ["string", "null"]}, + "counts": {"type": "object", "additionalProperties": {"type": "integer", "minimum": 0}}, + "records": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "type", "record_path"], + "properties": { + "id": {"type": "string"}, + "type": {"type": "string"}, + "record_path": {"type": "string"} + }, + "additionalProperties": true + } + } + }, + "additionalProperties": false +} diff --git a/schemas/entity-view.schema.json b/schemas/entity-view.schema.json new file mode 100644 index 0000000..dffd5de --- /dev/null +++ b/schemas/entity-view.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://modelmap.dev/schemas/entity-view.schema.json", + "title": "ModelMap Generated Entity View", + "type": "object", + "required": ["generated_from", "entity", "outgoing_relationships", "incoming_relationships", "claims", "events", "variants", "related_products"], + "properties": { + "generated_from": {"const": "canonical-records"}, + "entity": {"type": "object", "required": ["id", "type", "record_path"], "additionalProperties": true}, + "outgoing_relationships": {"type": "array", "items": {"type": "object"}}, + "incoming_relationships": {"type": "array", "items": {"type": "object"}}, + "claims": {"type": "array", "items": {"type": "object"}}, + "events": {"type": "array", "items": {"type": "object"}}, + "variants": { + "type": "array", + "items": { + "type": "object", + "required": ["entity", "claims", "events"], + "properties": { + "entity": {"type": "object"}, + "claims": {"type": "array"}, + "events": {"type": "array"} + }, + "additionalProperties": false + } + }, + "related_products": {"type": "array", "items": {"type": "object"}} + }, + "additionalProperties": false +} diff --git a/schemas/graph-view.schema.json b/schemas/graph-view.schema.json new file mode 100644 index 0000000..03dc0af --- /dev/null +++ b/schemas/graph-view.schema.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://modelmap.dev/schemas/graph-view.schema.json", + "title": "ModelMap Generated Graph View", + "type": "object", + "required": ["generated_from", "scope", "counts", "nodes", "edges"], + "properties": { + "generated_from": {"const": "canonical-records-and-relationships"}, + "scope": {"enum": ["core", "all"]}, + "counts": { + "type": "object", + "required": ["nodes", "edges", "omitted_edges"], + "properties": { + "nodes": {"type": "integer", "minimum": 0}, + "edges": {"type": "integer", "minimum": 0}, + "omitted_edges": {"type": "integer", "minimum": 0} + }, + "additionalProperties": false + }, + "nodes": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "type", "label", "record_path"], + "properties": { + "id": {"type": "string"}, + "type": {"type": "string"}, + "label": {"type": "string"}, + "record_path": {"type": "string"} + }, + "additionalProperties": false + } + }, + "edges": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "subject", "predicate", "object", "claim_refs"], + "properties": { + "id": {"type": "string"}, + "subject": {"type": "string"}, + "predicate": {"type": "string"}, + "object": {"type": "string"}, + "claim_refs": {"type": "array", "items": {"type": "string"}} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/tests/test_generated_views.py b/tests/test_generated_views.py new file mode 100644 index 0000000..b3a2224 --- /dev/null +++ b/tests/test_generated_views.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +import sys +import unittest +from pathlib import Path + +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).resolve().parents[1] +TOOLS = ROOT / "tools" +if str(TOOLS) not in sys.path: + sys.path.insert(0, str(TOOLS)) + +from build_catalog import load_catalog # noqa: E402 +from build_entity import build_entity_view # noqa: E402 +from build_graph import build_graph # noqa: E402 + + +def load_schema(name: str) -> dict: + return json.loads((ROOT / "schemas" / name).read_text(encoding="utf-8")) + + +class GeneratedViewsTest(unittest.TestCase): + def test_catalog_is_schema_valid_and_deterministically_sorted(self) -> None: + payload = load_catalog() + Draft202012Validator(load_schema("catalog-view.schema.json")).validate(payload) + keys = [(record["type"], record["id"]) for record in payload["records"]] + self.assertEqual(keys, sorted(keys)) + self.assertEqual(sum(payload["counts"].values()), len(payload["records"])) + + def test_core_graph_is_schema_valid_and_edges_have_nodes(self) -> None: + payload = build_graph("core") + Draft202012Validator(load_schema("graph-view.schema.json")).validate(payload) + node_ids = {node["id"] for node in payload["nodes"]} + self.assertEqual(payload["counts"]["nodes"], len(payload["nodes"])) + self.assertEqual(payload["counts"]["edges"], len(payload["edges"])) + for edge in payload["edges"]: + self.assertIn(edge["subject"], node_ids) + self.assertIn(edge["object"], node_ids) + + def test_all_graph_keeps_every_relationship_edge(self) -> None: + payload = build_graph("all") + Draft202012Validator(load_schema("graph-view.schema.json")).validate(payload) + self.assertEqual(payload["counts"]["omitted_edges"], 0) + + def test_release_entity_view_materializes_variants_and_products(self) -> None: + payload = build_entity_view("release:gpt-5.6") + Draft202012Validator(load_schema("entity-view.schema.json")).validate(payload) + variant_ids = {item["entity"]["id"] for item in payload["variants"]} + self.assertTrue( + {"variant:gpt-5.6-sol", "variant:gpt-5.6-terra", "variant:gpt-5.6-luna"}.issubset(variant_ids) + ) + product_ids = {item["id"] for item in payload["related_products"]} + self.assertIn("product:gpt-5.6-ultra", product_ids) + + def test_unknown_entity_is_rejected(self) -> None: + with self.assertRaises(KeyError): + build_entity_view("release:not-a-real-model") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build_catalog.py b/tools/build_catalog.py new file mode 100755 index 0000000..8d36ce5 --- /dev/null +++ b/tools/build_catalog.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Build a deterministic browse catalog from canonical ModelMap records.""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from pathlib import Path +from typing import Any + +from records import iter_records + +ROOT = Path(__file__).resolve().parents[1] +DATA = ROOT / "data" + +CATALOG_TYPES = { + "architecture_spec", + "benchmark", + "benchmark_version", + "dataset", + "dataset_version", + "evaluation_protocol", + "evaluation_run", + "hardware_system", + "implementation", + "license", + "model_family", + "model_release", + "model_variant", + "organization", + "paper", + "person", + "product_system", + "technique", + "training_run", +} + + +def load_catalog(record_type: str | None = None) -> dict[str, Any]: + records: list[dict[str, Any]] = [] + for path, record in iter_records(DATA): + rtype = record.get("type") + if rtype not in CATALOG_TYPES: + continue + if record_type and rtype != record_type: + continue + item = dict(record) + item["record_path"] = path.relative_to(ROOT).as_posix() + records.append(item) + + records.sort(key=lambda item: (str(item.get("type", "")), str(item["id"]))) + counts = Counter(str(item["type"]) for item in records) + return { + "generated_from": "canonical-records", + "filter_type": record_type, + "counts": dict(sorted(counts.items())), + "records": records, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--type", dest="record_type", choices=sorted(CATALOG_TYPES)) + parser.add_argument("--compact", action="store_true") + args = parser.parse_args() + payload = load_catalog(args.record_type) + print(json.dumps(payload, indent=None if args.compact else 2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_entity.py b/tools/build_entity.py new file mode 100755 index 0000000..38d971a --- /dev/null +++ b/tools/build_entity.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Build a read-model payload for one canonical ModelMap entity.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from records import iter_records + +ROOT = Path(__file__).resolve().parents[1] +DATA = ROOT / "data" + + +def load_index() -> tuple[dict[str, dict[str, Any]], list[dict[str, Any]]]: + index: dict[str, dict[str, Any]] = {} + records: list[dict[str, Any]] = [] + for path, record in iter_records(DATA): + item = dict(record) + item["record_path"] = path.relative_to(ROOT).as_posix() + records.append(item) + if item.get("id"): + index[str(item["id"])] = item + return index, records + + +def build_entity_view(entity_id: str) -> dict[str, Any]: + index, records = load_index() + if entity_id not in index: + raise KeyError(f"Unknown canonical id: {entity_id}") + + outgoing = sorted( + [r for r in records if r.get("type") == "relationship" and r.get("subject") == entity_id], + key=lambda r: r["id"], + ) + incoming = sorted( + [r for r in records if r.get("type") == "relationship" and r.get("object") == entity_id], + key=lambda r: r["id"], + ) + claims = sorted( + [r for r in records if r.get("type") == "claim" and r.get("subject") == entity_id], + key=lambda r: r["id"], + ) + events = sorted( + [r for r in records if r.get("type") == "event" and r.get("subject") == entity_id], + key=lambda r: (str(r.get("occurred_at", "")), r["id"]), + ) + + variant_ids = sorted( + r["subject"] + for r in incoming + if r.get("predicate") == "variant_of" and str(r.get("subject", "")).startswith("variant:") + ) + variants: list[dict[str, Any]] = [] + for variant_id in variant_ids: + variant = index.get(variant_id) + if not variant: + continue + variant_claims = sorted( + [r for r in records if r.get("type") == "claim" and r.get("subject") == variant_id], + key=lambda r: r["id"], + ) + variant_events = sorted( + [r for r in records if r.get("type") == "event" and r.get("subject") == variant_id], + key=lambda r: (str(r.get("occurred_at", "")), r["id"]), + ) + variants.append({"entity": variant, "claims": variant_claims, "events": variant_events}) + + related_products = sorted( + [ + r + for r in records + if r.get("type") == "product_system" + and isinstance(r.get("metadata"), dict) + and r["metadata"].get("model_release") == entity_id + ], + key=lambda r: r["id"], + ) + + return { + "generated_from": "canonical-records", + "entity": index[entity_id], + "outgoing_relationships": outgoing, + "incoming_relationships": incoming, + "claims": claims, + "events": events, + "variants": variants, + "related_products": related_products, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("entity_id") + parser.add_argument("--compact", action="store_true") + args = parser.parse_args() + try: + payload = build_entity_view(args.entity_id) + except KeyError as exc: + parser.error(str(exc)) + print(json.dumps(payload, indent=None if args.compact else 2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_graph.py b/tools/build_graph.py new file mode 100755 index 0000000..6a9ccca --- /dev/null +++ b/tools/build_graph.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Build a deterministic knowledge-graph export from canonical ModelMap records.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from records import iter_records +from build_catalog import CATALOG_TYPES + +ROOT = Path(__file__).resolve().parents[1] +DATA = ROOT / "data" + + +def label_for(record: dict[str, Any]) -> str: + return str(record.get("name") or record.get("title") or record["id"]) + + +def build_graph(scope: str = "core") -> dict[str, Any]: + all_records: dict[str, tuple[Path, dict[str, Any]]] = {} + relationships: list[dict[str, Any]] = [] + for path, record in iter_records(DATA): + record_id = record.get("id") + if record_id: + all_records[str(record_id)] = (path, record) + if record.get("type") == "relationship": + relationships.append(record) + + if scope == "core": + node_ids = { + record_id + for record_id, (_, record) in all_records.items() + if record.get("type") in CATALOG_TYPES + } + else: + node_ids = { + record_id + for record_id, (_, record) in all_records.items() + if record.get("type") != "relationship" + } + + nodes: list[dict[str, Any]] = [] + for record_id in sorted(node_ids): + path, record = all_records[record_id] + nodes.append( + { + "id": record_id, + "type": record.get("type"), + "label": label_for(record), + "record_path": path.relative_to(ROOT).as_posix(), + } + ) + + edges: list[dict[str, Any]] = [] + omitted_edges = 0 + for relation in sorted(relationships, key=lambda item: item["id"]): + if relation["subject"] not in node_ids or relation["object"] not in node_ids: + omitted_edges += 1 + continue + edges.append( + { + "id": relation["id"], + "subject": relation["subject"], + "predicate": relation["predicate"], + "object": relation["object"], + "claim_refs": relation.get("claim_refs", []), + } + ) + + return { + "generated_from": "canonical-records-and-relationships", + "scope": scope, + "counts": { + "nodes": len(nodes), + "edges": len(edges), + "omitted_edges": omitted_edges, + }, + "nodes": nodes, + "edges": edges, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--scope", choices=["core", "all"], default="core") + parser.add_argument("--compact", action="store_true") + args = parser.parse_args() + payload = build_graph(args.scope) + print(json.dumps(payload, indent=None if args.compact else 2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())