diff --git a/.github/workflows/topic06-benchmark.yml b/.github/workflows/topic06-benchmark.yml new file mode 100644 index 0000000..6a9980a --- /dev/null +++ b/.github/workflows/topic06-benchmark.yml @@ -0,0 +1,47 @@ +name: Topic 06 Compiler Benchmark + +on: + push: + pull_request: + +jobs: + topic06-benchmark: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install ScratchV and benchmark dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[topic06]" + + - name: Verify Topic 06 integration contracts + run: python -m pytest tests/topic06/test_integration.py -q + + - name: Run supported correctness gates + run: | + python scripts/run_topic06_benchmarks.py --category activation --fail-on-test-failure + python scripts/run_topic06_benchmarks.py --category elementwise --fail-on-test-failure + python scripts/run_topic06_benchmarks.py --category loop --fail-on-test-failure + + - name: Generate full diagnostic benchmark report + run: python scripts/run_topic06_benchmarks.py --benchmark 3 + + - name: Upload Topic 06 reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: topic06-benchmark-reports + path: | + benchmark_reports/topic06/report.md + benchmark_reports/topic06/report.json + benchmark_reports/topic06/failures/ + benchmarks/topic06/baseline.json diff --git a/Makefile b/Makefile index 031f739..fc51a5a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # ScratchV developer makefile .POSIX: -.PHONY: quick-start install test bench bench-cnn clean lint +.PHONY: quick-start install test bench bench-topic06 bench-cnn clean lint # ── Beginner quick-start ───────────────────────────────────────────────────── @@ -48,6 +48,11 @@ bench: --output-json benchmark_reports/dsl_bench.json \ --output-html benchmark_reports/dsl_bench.html +# ── Topic 06 DSL correctness + TinyFive benchmark ───────────────────────── + +bench-topic06: + python3 scripts/run_topic06_benchmarks.py --benchmark 3 + # ── CNN RISC-V 编译 + 估算 ──────────────────────────────────────────────── bench-cnn: diff --git a/README.md b/README.md index a0f32ab..ec299b7 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ ```bash make quick-start # 打印新手引导 make test # 运行全部测试 +make bench-topic06 # 运行课题 06 TinyFive 性能测试套件 make bench-cnn # 编译 CNN 模型 + 性能估算 make bench-ci # 完整 CI 对比 (ScratchV vs LLVM) make bench-reports # 生成 Dashboard + 优化历史 @@ -62,9 +63,9 @@ ScratchV/ │ ├── 00~04-*.md ← 新手入门 5 篇 │ ├── topics/ ← 30 个模块详解 │ └── topics/html/ ← 🌐 交互式课程站点 -├── benchmarks/ ← 23 个 DSL 基准用例 -├── tests/ ← 348 个单元测试 -├── scripts/ ← 工具脚本 +├── benchmarks/ ← 通用基准与课题 06 性能基线 +├── tests/ ← 单元测试及课题 06 的 23 个 DSL 用例 +├── scripts/ ← 工具脚本及课题 06 测试驱动 └── models/ ← 测试用 ONNX 模型 ``` @@ -104,6 +105,7 @@ ONNX 模型 (.onnx) | [📖 文档导航](docs/INDEX.md) | 全部 Markdown 文档索引 | | [🏗️ 架构总览](docs/ARCHITECTURE.md) | ONNX→RISC-V 双路径详解 | | [📊 性能仪表盘](https://scratchv-compiler.github.io/ScratchV/dashboard.html) | LLVM vs ScratchV 对比 | +| [课题 06 测试套件](docs/topics/06-性能测试套件使用说明.md) | TinyFive 正确性验证、Benchmark 与回归报告 | | [📢 项目海报](https://scratchv-compiler.github.io/ScratchV/ScratchV.html) | 招募信息 + 3个月学习路线 + 课题精选 | --- diff --git a/benchmarks/topic06/baseline.json b/benchmarks/topic06/baseline.json new file mode 100644 index 0000000..7a7596b --- /dev/null +++ b/benchmarks/topic06/baseline.json @@ -0,0 +1,102 @@ +{ + "add_relu_relu": { + "category": "activation", + "avg_instr_count": 7.0, + "runs": 3 + }, + "relu_add": { + "category": "activation", + "avg_instr_count": 5.0, + "runs": 3 + }, + "relu_only": { + "category": "activation", + "avg_instr_count": 5.0, + "runs": 3 + }, + "relu_twice": { + "category": "activation", + "avg_instr_count": 6.0, + "runs": 3 + }, + "add_chain": { + "category": "elementwise", + "avg_instr_count": 4.0, + "runs": 3 + }, + "add_chain_3": { + "category": "elementwise", + "avg_instr_count": 5.0, + "runs": 3 + }, + "add_fan_in_4": { + "category": "elementwise", + "avg_instr_count": 5.0, + "runs": 3 + }, + "add_reuse": { + "category": "elementwise", + "avg_instr_count": 4.0, + "runs": 3 + }, + "vector_add": { + "category": "elementwise", + "avg_instr_count": 3.0, + "runs": 3 + }, + "loop_add_4": { + "category": "loop", + "avg_instr_count": 22.0, + "runs": 3 + }, + "loop_add_chain_4": { + "category": "loop", + "avg_instr_count": 26.0, + "runs": 3 + }, + "loop_relu_add_4": { + "category": "loop", + "avg_instr_count": 30.0, + "runs": 3 + }, + "dot_4": { + "category": "reduction", + "avg_instr_count": 3.0, + "runs": 3 + }, + "dot_8": { + "category": "reduction", + "avg_instr_count": 3.0, + "runs": 3 + }, + "dot_relu_4": { + "category": "reduction", + "avg_instr_count": 5.0, + "runs": 3 + }, + "dot_relu_8": { + "category": "reduction", + "avg_instr_count": 5.0, + "runs": 3 + }, + "matmul_2x2": { + "category": "tensor", + "avg_instr_count": 3.0, + "runs": 3 + }, + "matmul_4x4": { + "category": "tensor", + "avg_instr_count": 3.0, + "runs": 3 + }, + "matmul_add_2x2": { + "category": "tensor", + "avg_instr_count": 4.0, + "runs": 3 + }, + "matmul_relu_2x2": { + "category": "tensor", + "avg_instr_count": 5.0, + "runs": 3 + } +} \ No newline at end of file diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266.md" index 3637b6d..c18f5b8 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266.md" @@ -1,200 +1,24 @@ -# 课题6：编译器性能测试套件 +# 课题 06：编译器性能测试套件 -> **难度**：中 | **类型**：项目实战 | **源文件**：`benchmarks/bench_runner.py` | **行数**：~400 -> **状态**：✅ 已完成 +课题 06 测试套件已经集成到 ScratchV 仓库正式目录。它使用 23 个 DSL 用例验证 `DSL -> ScratchV -> RISC-V -> TinyFive` 真实执行路径，并支持指令数统计、耗时测量、Benchmark、性能基线、退化检测和 CI 报告。 ---- +- [使用说明](06-性能测试套件使用说明.md) +- [设计文档](06-性能测试套件设计文档.md) -## 概述 +主要入口： -设计一组基准测试程序（DSL用例），自动化执行编译、模拟运行、对比预期输出，生成性能报告。该套件用于持续验证编译器的正确性和性能变化。 - ---- - -## 理解背景 - -### 是什么？ - -性能基准套件（Benchmark Suite）自动化测试 ScratchV 编译器的**正确性**和**性能**。它收集了 23 个 DSL 测试用例，覆盖算术、神经网络算子、控制流等各类场景，每次运行自动生成 JSON/HTML/Markdown 报告。 - -### 为什么？ - -编译器每次改动都可能引入 bug 或性能衰退。手工测试既不全面也不可靠。自动化的基准测试套件让你： -- **回归检测**：改了代码后跑一遍，立即知道有没有破坏现有功能 -- **性能追踪**：记录每次运行的时间，发现性能衰退 -- **CI 集成**：每次 push 自动运行，阻止有问题的代码合入 - -### 核心概念 - -#### 测试用例格式 - -每个测试用例最多 2 个文件（放在 `benchmarks/cases/`）： -``` -001_simple_add.dsl ← DSL 源代码（必需） -001_simple_add.expected ← 期望输出（可选） -``` - -#### DSL 语法要点 - -数字可以直接内联，变量名自动解析： -```python -# 正确写法 -c = add(a, b) # a, b 作为输入变量 -t1 = add(2.0, 3.0) # 数字直接写 -return c - -# 错误写法（没有 const 操作符！） -a = const(3) # ❌ 不支持 -``` - -支持的算子：`add`, `sub`, `mul`, `div`, `neg`, `exp`, `relu`, `gelu`, `dot`, `matmul`, `softmax`, `maxpool` - -#### 23 个用例覆盖范围 - -| 类别 | 数量 | 示例 | -|------|------|------| -| 算术 | 5 | add, mul, sub, div, chained | -| 神经网络 | 6 | relu, gelu, softmax, matmul, dot, maxpool | -| 控制流 | 7 | for-loop, if/else, while, nested | -| 复杂场景 | 3 | NN pipeline, large chain | -| 常量 | 1 | constant propagation | - -#### 报告格式 - -| 格式 | 用途 | -|------|------| -| **终端输出** | 开发时快速查看 | -| **JSON** | CI dashboard 数据源 | -| **HTML** | 可视化展示 | -| **Markdown** | 文档、PR review | - ---- - -## 代码走读 - -### 运行全部测试 - -```bash -# 基础运行（指定用例目录） -python3 benchmarks/bench_runner.py benchmarks/cases - -# 生成 JSON + HTML 报告 -python3 benchmarks/bench_runner.py benchmarks/cases \ - --output-json /tmp/dsl_bench.json \ - --output-html /tmp/dsl_bench.html -``` - -实际输出（23/23 PASS）： -``` -Discovered 23 test case(s) in benchmarks/cases ------------------------------------------------------------- - [1/23] 001_simple_add ... PASS (0.008s parse, 2 inst) - [2/23] 002_simple_mul ... PASS (0.000s parse, 2 inst) - ... - [23/23] 023_large_chain ... PASS (0.000s parse, 7 inst) -``` - -### 用 Makefile 一键运行 - -```bash -make bench # 等价于上面的完整流程 +```text +scripts/run_topic06_benchmarks.py +tests/topic06/cases/ +benchmarks/topic06/baseline.json +.github/workflows/topic06-benchmark.yml ``` -### 添加新测试用例 +在仓库根目录运行： ```bash -cat > benchmarks/cases/024_my_test.dsl << 'EOF' -# 我的自定义测试: 10 + 20 - 5 -a = add(10, 20) -b = sub(a, 5) -return b -EOF - -echo "25" > benchmarks/cases/024_my_test.expected - -# 运行确认 -python3 benchmarks/bench_runner.py benchmarks/cases -``` - -> 💡 **关键**：数字直接内联使用（如 `10`, `20`），不需要 `const()` 包装。期望输出文件是纯文本，末尾不要有多余空行。 - -### Python API 用法 - -```python -from benchmarks.bench_runner import BenchmarkRunner - -# 创建 runner -runner = BenchmarkRunner( - test_dir="benchmarks/cases", - timeout=30.0, - verbose=True, -) - -# 发现测试用例 -cases = runner.discover_cases() -print(f"Found {len(cases)} test cases") - -# 运行全部（返回 BenchmarkReport） -report = runner.run_all() -report.print_summary() - -# 生成报告 -report.save_json("/tmp/results.json") -report.save_html("/tmp/results.html") -report.save_markdown("/tmp/results.md") +pip install -e ".[topic06]" +python scripts/run_topic06_benchmarks.py ``` -### CI 回归检测 - -```bash -# 生成基线 -python3 benchmarks/bench_runner.py benchmarks/cases --output-json /tmp/baseline.json - -# 修改代码后... -python3 benchmarks/bench_runner.py benchmarks/cases --output-json /tmp/current.json - -# 对比通过率 -python3 -c " -import json -baseline = json.load(open('/tmp/baseline.json')) -current = json.load(open('/tmp/current.json')) -if current['pass_rate'] < baseline['pass_rate']: - print('REGRESSION DETECTED!') - exit(1) -else: - print('OK: pass_rate stable') -" -``` - ---- - -## 动手练习 - -### 练习 1: 添加一个新测试用例 - -为你正在开发的编译器功能写一个测试用例，跑通并确认通过。 - -### 练习 2: 故意引入 bug - -修改代码引入一个 bug（比如把 `ADD` 映射成 `SUB`），跑 benchmark 看看哪些测试会失败。 - -### 练习 3: 分析性能趋势 - -连续跑 3 次 benchmark，对比 JSON 输出，看编译时间是否有波动。 - ---- - -## 常见坑 - -| 坑 | 说明 | -|----|------| -| **没有 `const` 操作符** | DSL 不支持 `a = const(3)`，直接用数字 `a = add(3, 5)` | -| **expected 文件格式** | .expected 文件应该是纯文本，末尾不要有多余换行 | -| **超时** | 某些测试可能死循环，BenchmarkRunner 有 timeout 保护 | - ---- - -## 进阶阅读 - -- [03-指标解读指南](../03-指标解读指南.md) — 如何解读 benchmark 数据 -- 相关课题: [课题19：Standalone RISC-V 编译器](19-Standalone-RISC-V编译器.md) | [课题30：CI Dashboard](30-CI-Dashboard.md) +当前真实 TinyFive 全量结果为 13 个通过、10 个失败。CI 将当前稳定的 `activation`、`elementwise` 和 `loop` 类别作为阻塞门禁，其余用例保留在全量诊断报告中。 diff --git "a/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 0000000..3b36fd2 --- /dev/null +++ "b/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,232 @@ +# ScratchV 课题 06 性能测试套件使用说明 + +本测试套件已经迁入 ScratchV 仓库正式目录，用于编译正确性与性能回归测试。它包含 23 个 DSL 测试用例、自动化测试脚本、性能基线、CI 工作流以及可重新生成的测试报告。 + +当前程序在原始自动编译、指令数统计、Benchmark、基线对比和报告功能上，进一步增加了真实 TinyFive 直通验证、元数据校验、编译与模拟超时、编译失败日志、统一 JSON schema、轻量/完整报告、可配置退化阈值和子集测试。 + +编译器通过 `--emit-register-map` 输出 JSON 格式的变量到物理寄存器映射，测试套件直接读取该文件注入标量输入，不再解析 `--dump-ir` 文本。 + +真实验证路径为： + +```text +DSL -> ScratchV 编译器 -> RISC-V 汇编 -> TinyFive -> a0/x10 返回值 -> 期望值对比 +``` + +最近一次全量运行结果为 13 个通过、10 个失败。3 个 `branch` 用例因 TinyFive 模拟超时失败；另外 7 个 `dot`/`matmul` 用例涉及尚未完成的数组内存输入输出约定和编译器后端降低逻辑。测试套件负责真实暴露、隔离和记录这些问题，不负责实现 ScratchV 的分支、数组或矩阵代码生成。 + +## 运行依赖 + +测试命令需要在 ScratchV 仓库根目录运行。`scripts/run_topic06_benchmarks.py` 会导入 `scratchv` 包，并通过 ScratchV 编译器把每个 DSL 用例编译为 RISC-V 汇编。 +调用真实 TinyFive 所需的配套接口位于 `scratchv/simulator/tinyfive.py`、 +`scratchv/backend/register_alloc.py`、`scratchv/compiler.py` 和 `scratchv/main.py`。 + +安装基础测试依赖： + +```powershell +pip install -e ".[topic06]" +``` + +基础模式需要以下依赖： + +- `tinyfive`：用于模拟执行生成的 RISC-V 汇编 +- `pytest`：用于测试支持 + +如需生成 HTML 和 PNG 完整报告，再安装可选依赖： + +```powershell +pip install -e ".[topic06-report]" +``` + +其中 `jinja2` 用于生成 HTML，`matplotlib` 用于生成性能图表。 + +## 目录结构 + +```text +scripts/run_topic06_benchmarks.py +tests/topic06/ + test_integration.py + cases/ + activation/ + branch/ + elementwise/ + loop/ + reduction/ + tensor/ +benchmarks/topic06/baseline.json +benchmark_reports/topic06/ # 运行时生成，不提交 +build/topic06/ # 运行时生成，不提交 +.github/workflows/topic06-benchmark.yml +``` + +`tests/topic06/cases/` 下共有 23 个 DSL 用例。每个用例由 `.dsl` 文件和对应的 +`.meta.json` 文件组成，`.meta.json` 中定义输入、期望返回值和用例说明。 + +## 运行测试 + +在仓库根目录执行： + +```powershell +python scripts/run_topic06_benchmarks.py +``` + +测试脚本会自动完成以下步骤： + +- 遍历 `tests/topic06/cases/` 下的所有 `.dsl` 文件 +- 调用 ScratchV 编译器生成 RISC-V 汇编 +- 读取编译器生成的 `build/topic06/*.registers.json`，按变量名注入真实物理寄存器 +- 编译单个用例超过 30 秒时终止该编译进程，并继续运行后续用例 +- 编译失败或超时时将命令、返回码、stdout 和 stderr 保存到 `benchmark_reports/topic06/failures/*.compile.log` +- 通过 ScratchV 的 TinyFive 适配层验证生成的汇编 +- 在独立子进程中执行 TinyFive；单次模拟超过 5 秒时终止该用例并继续后续测试 +- TinyFive 未安装或执行失败时明确判定为 FAIL，不回退到 stub +- 使用 TinyFive 返回寄存器 `a0/x10` 与 `.meta.json` 中的期望值进行结果对比 +- 使用 `time.perf_counter()` 记录编译、模拟和总耗时 +- 统计 PASS/FAIL 和指令数 +- 默认在 `benchmark_reports/topic06/` 下生成 Markdown 和固定 schema 的 JSON 报告 + +## Benchmark 模式 + +重复运行每个用例并统计平均指令数： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 +``` + +如果首次模拟已经 timeout，该用例不会再执行 Benchmark 重复；如果重复过程中发生 timeout 或模拟失败，脚本会立即停止剩余次数。失败运行不会作为 `0` 加入平均值，报告会记录实际完成次数和 `benchmark_stopped_reason`。 + +更新性能基线： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 --update-baseline +``` + +性能退化阈值默认是 5%，可以按场景调整： + +```powershell +# 超过 2% 即判定为退化 +python scripts/run_topic06_benchmarks.py --benchmark 3 --regression-threshold 2 + +# 超过 10% 才判定为退化 +python scripts/run_topic06_benchmarks.py --benchmark 3 --regression-threshold 10 +``` + +阈值必须大于或等于 0，并会写入 Markdown、HTML 和 JSON 报告。 + +## 子集测试 + +按类别运行： + +```powershell +python scripts/run_topic06_benchmarks.py --category activation +``` + +按用例名称进行大小写不敏感的包含匹配： + +```powershell +python scripts/run_topic06_benchmarks.py --filter matmul +``` + +组合筛选： + +```powershell +python scripts/run_topic06_benchmarks.py --category tensor --filter relu +``` + +没有匹配用例时脚本返回退出码 2，并保留已有报告。对子集使用 `--update-baseline` 时，只更新匹配用例的基线，不会删除其他用例的基线。 + +CI 或其他需要将失败传递给调用方的场景可以增加： + +```text +python scripts/run_topic06_benchmarks.py --category activation --fail-on-test-failure +``` + +默认模式仍会完成全部用例并生成报告；`--fail-on-test-failure` 会在存在 FAIL 时返回退出码 1。 + +性能基线文件位于： + +```text +benchmarks/topic06/baseline.json +``` + +## 测试报告 + +默认运行后生成轻量报告： + +```text +benchmark_reports/topic06/report.md +benchmark_reports/topic06/report.json +``` + +需要 HTML 和 PNG 时运行： + +```powershell +python scripts/run_topic06_benchmarks.py --full-report +``` + +完整模式额外生成： + +```text +benchmark_reports/topic06/report.html +benchmark_reports/topic06/course_report_instructions.png +``` + +`report.md` 适合提交或归档，`report.json` 供 CI 或其他程序稳定解析，`report.html` 适合在浏览器中查看。普通模式和 Benchmark 模式使用相同字段，Benchmark 专属字段在普通模式下为 `null`。JSON 顶层的 `selection` 会记录 `--category` 和 `--filter`，未筛选时两项均为 `null`。轻量模式不会加载 `jinja2` 或 `matplotlib`，也不会刷新已有的 HTML、PNG 文件。 + +编译失败时，报告中的 `compile_returncode`、`compile_timed_out`、`compile_error` 和 `compile_log` 会记录错误摘要及独立日志位置。成功用例的错误和日志字段为 `null`。 + +## 添加测试用例 + +新增用例时添加两个文件： + +```text +tests/topic06/cases/{category}/{name}.dsl +tests/topic06/cases/{category}/{name}.meta.json +``` + +DSL 示例： + +```text +result = add(a, b) +return result +``` + +元数据示例： + +```json +{ + "description": "Simple scalar add.", + "expected_output_type": "scalar", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} +``` + +矩阵输出需要声明类型、元素类型和形状： + +```json +{ + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "expected_return": [[19, 22], [43, 50]] +} +``` + +大型期望结果也可以使用 `expected_output_file` 引用同目录下的 JSON 文件。`expected_return` 与 `expected_output_file` 必须且只能填写一个。元数据中的类型、形状或元素值不合法时，仅当前用例标记为 FAIL，后续测试继续运行。 + +## CI 示例 + +GitHub Actions 示例位于： + +```text +.github/workflows/topic06-benchmark.yml +``` + +工作流位于仓库根目录。它会在 push 和 pull request 时运行 pytest 集成契约，对 activation、elementwise、loop 执行阻塞正确性检查，再运行全量诊断 Benchmark，并上传 Markdown、JSON、性能基线和失败日志。CI 默认使用轻量报告，不需要安装 `jinja2` 和 `matplotlib`。 + +## 详细设计 + +程序模块、报告字段、需求演进、Mentor Review 处理状态和当前失败原因见 [06-性能测试套件设计文档.md](06-性能测试套件设计文档.md)。 diff --git "a/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 0000000..fca8292 --- /dev/null +++ "b/docs/topics/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,692 @@ +# ScratchV 课题 06 性能测试套件设计文档 + +> 文档版本：v1.7 +> 编写日期：2026-07-22 +> 更新日期：2026-09-03 +> 集成位置：ScratchV 仓库正式目录 +> 核心文件：`scripts/run_topic06_benchmarks.py`、`tests/topic06/`、`benchmarks/topic06/` +> 功能范围：DSL 用例管理、真实 TinyFive 验证、元数据校验、编译与模拟超时保护、失败现场保存、指令数与耗时统计、Benchmark、性能基线对比、子集测试、统一 Markdown/JSON/HTML 报告、CI 示例 + +--- + +## 一、功能介绍 + +### 1.1 功能概述 + +本测试套件用于验证 ScratchV 编译器在多个 DSL 程序上的编译正确性、模拟结果和性能表现。测试脚本会自动遍历或按条件筛选 `tests/topic06/cases/` 下的 DSL 用例，调用 ScratchV 编译器生成 RISC-V 汇编，再通过真实 TinyFive 适配层执行汇编，并将返回值与用例元数据中的期望值比较。测试结束后会在 `benchmark_reports/topic06/` 生成适合人工查看和 CI 解析的报告。 + +当前套件包含 23 个 DSL 测试用例，覆盖以下类别： + +| 类别 | 说明 | +|---|---| +| `activation` | ReLU 等激活函数组合 | +| `elementwise` | 标量/向量加法、链式加法、复用输入 | +| `loop` | `for/endfor` 循环类用例 | +| `branch` | `if/else/endif` 分支类用例 | +| `reduction` | `dot` 等归约类用例 | +| `tensor` | `matmul` 等张量计算类用例 | + +测试输出不仅包含 PASS/FAIL，还包含编译状态、TinyFive 后端状态、期望值与实际值、指令数、编译耗时、模拟耗时、总耗时、Benchmark 平均值、95% 置信区间、性能基线对比和失败日志路径等信息。 + +### 1.2 设计目标 + +- **自动化**：一条命令完成所有用例的编译、模拟、结果比对和报告生成。 +- **真实性**：只使用 `DSL -> ScratchV -> RISC-V -> TinyFive` 路径判断结果，不使用 stub 或测试套件内置解释器替代真实后端。 +- **可扩展**：新增测试只需要添加 `.dsl` 和 `.meta.json` 文件，不需要修改主测试逻辑。 +- **可解释**：报告中展示每个用例的状态、预期输出、实际输出、模拟后端、指令数和耗时。 +- **可对比**：支持保存 Benchmark 基线，并在后续运行中检查性能变化率。 +- **可集成**：普通模式和 Benchmark 模式采用统一 JSON schema，便于 CI 稳定解析。 +- **可选择**：支持按类别和名称运行子集，缩短定位单个模块问题时的等待时间。 +- **避免卡死**：编译阶段设置 30 秒超时，TinyFive 模拟阶段设置 5 秒超时，单个异常用例不会阻塞整套测试。 +- **便于诊断**：编译失败时保存命令、返回码、stdout、stderr 和耗时等现场信息。 +- **轻量运行**：默认报告不依赖绘图库和模板库，需要可视化时再通过 `--full-report` 启用。 + +### 1.3 需求演进与当前范围 + +项目最初按照 W1-W12 实现自动编译、模拟、结果对比、性能统计、报告和 CI。经过实际接入 TinyFive 以及 Mentor Review 后，程序增加了可靠性和工程化能力。新增内容不是另一套测试目标，而是保证原有目标能够在真实编译链路中稳定执行。 + +| 来源 | 当前实现内容 | 解决的问题 | +|---|---|---| +| 原始 W1-W4 | DSL 用例格式、自动调用编译器和模拟器、PASS/FAIL 对比 | 完成基本自动化正确性测试 | +| 原始 W5-W10 | 指令数、耗时、Benchmark、置信区间、基线和退化检测 | 完成性能测量与回归比较 | +| 原始 W11-W12 | GitHub Actions 示例和使用文档 | 支持 CI 和交付使用 | +| S1、A2 | 真实 TinyFive 直通验证，禁用 stub 回退，模拟子进程超时 | 防止模拟结果被替代，并避免模拟卡死 | +| S2、T1 | 编译超时和独立失败日志 | 防止编译卡死并保留崩溃现场 | +| A1 | 标量/张量期望值、dtype/shape 和外部期望文件校验 | 提高测试元数据的表达能力和错误可读性 | +| S3 | Markdown、HTML、JSON 统一字段，未执行数据使用 `null` | 避免普通模式和 Benchmark 模式的报告解析冲突 | +| I1 | 默认轻量报告与 `--full-report` 可选完整报告 | 降低日常测试和 CI 的依赖成本 | +| I2、I3 | 可配置退化阈值、Benchmark 失败短路 | 适应不同检查标准并避免重复等待失败用例 | +| T2 | `--category`、`--filter` 和子集基线合并 | 支持快速定位和增量测试 | +| P1 集成整改 | 根目录 pytest 契约测试、正式 GitHub Actions workflow、结构化寄存器映射 | 消除独立目录与 IR 文本格式造成的集成风险 | +| 正式目录迁移 | 脚本、用例、基线和文档分别归入仓库现有目录 | 消除独立交付目录，形成可直接合入的 local 测试套件 | + +因此，当前程序的交付边界不仅是“运行 23 个用例并生成性能报告”，还包括真实后端验证、异常隔离、稳定报告接口、失败诊断和子集执行。编译器后端本身的分支、数组及矩阵指令生成不属于测试套件的实现范围；测试套件负责真实暴露并记录这些问题。 + +--- + +## 二、目录结构 + +迁移后的仓库结构如下： + +```text +ScratchV/ + .github/ + workflows/ + topic06-benchmark.yml + scripts/ + run_topic06_benchmarks.py + tests/ + topic06/ + test_integration.py + cases/ + activation/ + branch/ + elementwise/ + loop/ + reduction/ + tensor/ + benchmarks/ + topic06/ + baseline.json + benchmark_reports/ + topic06/ + report.md + report.json + report.html + course_report_instructions.png + failures/ + build/ + topic06/ +``` + +其中： + +- `scripts/run_topic06_benchmarks.py` 是测试主程序。 +- `tests/topic06/cases/` 保存 DSL 测试用例，`tests/topic06/test_integration.py` 检查用例清单、编译器寄存器映射和报告路径契约。 +- `benchmarks/topic06/baseline.json` 是纳入版本控制的性能基线。 +- `benchmark_reports/topic06/` 保存可重新生成的测试报告与失败日志，不纳入版本控制。 +- `build/topic06/` 保存可重新生成的汇编与寄存器映射，不纳入版本控制。 +- `pyproject.toml` 的 `topic06` 和 `topic06-report` 可选依赖组分别支持轻量测试和完整报告。 +- `.github/workflows/topic06-benchmark.yml` 是 GitHub 实际识别并执行的 CI 配置。 + +原来的 `ScratchV-topic06-deliverable/` 独立目录在迁移后删除，仓库根目录的 `LICENSE` 和 `pyproject.toml` 统一承担许可与打包配置。 + +--- + +## 三、测试用例格式 + +每个测试用例由两个文件组成： + +```text +tests/topic06/cases/{category}/{name}.dsl +tests/topic06/cases/{category}/{name}.meta.json +``` + +示例 DSL： + +```text +result = add(a, b) +return result +``` + +示例元数据： + +```json +{ + "description": "Simple scalar add.", + "expected_output_type": "scalar", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} +``` + +这里没有采用单独的 `.expected` 和 `.desc` 文件，而是使用 `.meta.json` 统一保存输入、描述和期望输出。这样结构更集中，也方便 Python 脚本直接解析。 + +`expected_output_type` 支持两种值： + +- `scalar`：标量结果，使用数字或布尔值；旧值 `return_value` 会兼容转换为 `scalar`。 +- `tensor`：向量或多维数组，必须同时声明 `output_dtype` 和 `output_shape`。 + +张量示例： + +```json +{ + "description": "2x2 matrix multiplication", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]] + }, + "expected_return": [[19, 22], [43, 50]] +} +``` + +支持的 `output_dtype` 为 `bool`、`int32`、`int64`、`float32` 和 `float64`。加载元数据时，测试脚本会递归检查数组是否规则、实际形状是否与 `output_shape` 一致，以及每个元素是否符合声明的类型。 + +期望结果较大时，可以不使用内联的 `expected_return`，改为引用当前用例目录下的 JSON 文件： + +```json +{ + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "expected_output_file": "matmul_2x2.expected.json" +} +``` + +`expected_return` 和 `expected_output_file` 必须且只能存在一个。外部 JSON 可以直接保存标量或数组，也可以保存包含 `expected_return` 字段的对象。格式、shape 或 dtype 校验失败时，该用例报告 `metadata error` 并判定为 FAIL，不影响后续用例。 + +这一协议解决的是期望值的表达和校验。张量用例要在真实 TinyFive 路径下通过，还需要编译器后端约定输出内存地址并生成完整的数组计算及写回指令；这属于后端输入输出约定，不由元数据格式代替。 + +--- + +## 四、核心实现设计 + +### 4.1 编译阶段 + +函数：`run_compile(dsl_file: Path, timeout: float = 30.0)` + +作用： + +1. 接收一个 `.dsl` 文件路径。 +2. 调用 ScratchV 编译器： + +```text +python -m scratchv.main -o --optimize all --emit-register-map +``` + +3. 将生成的 RISC-V 汇编写入 `build/topic06/{case_name}.s`。 +4. 将寄存器分配结果写入 `build/topic06/{case_name}.registers.json`。 +5. 返回编译进程结果、汇编路径和寄存器映射路径。 + +这一阶段使用 `subprocess.run()` 调用编译器，能够捕获 `stdout`、`stderr` 和返回码。调用设置了默认 30 秒的 `timeout`：如果编译器在某个 DSL 上卡死，脚本会终止该编译子进程，返回错误 `compile timeout after 30s`，将当前用例标记为 FAIL，并继续运行后续用例。编译超时使用返回码 `124` 表示，不会进入 TinyFive 模拟阶段。 + +编译返回码非 0 时，`write_compile_failure_log()` 会生成： + +```text +benchmark_reports/topic06/failures/{category}-{case}.compile.log +``` + +日志保存以下现场信息： + +- 失败时间和 DSL 用例路径。 +- 完整编译命令。 +- 返回码和是否触发 30 秒 timeout。 +- 编译耗时、目标汇编路径以及汇编文件是否存在。 +- 完整 stdout 和 stderr，包括 Python traceback。 + +报告中的每条结果固定包含 `compile_returncode`、`compile_timed_out`、`compile_error` 和 `compile_log`。`compile_error` 保存最后一条非空错误信息，方便 CI 快速显示；`compile_log` 指向完整日志。成功用例的错误摘要和日志路径为 `null`。 + +### 4.2 模拟验证阶段 + +函数：`run_simulation(asm_file: Path, timeout: float = 5.0)` + +作用： + +1. 读取编译生成的 `.s` 汇编文件。 +2. 在独立 Python 子进程中调用： + +```python +from scratchv.simulator.tinyfive import verify_assembly +``` + +3. 获取 TinyFive 适配层返回的模拟结果。 +4. 如果模拟超过 5 秒，则终止该子进程，并返回： + +```python +{ + "success": False, + "backend": "timeout", + "error": "simulation timeout after 5s" +} +``` + +TinyFive 执行采用两层保护： + +1. `verify_assembly()` 内部调用 `m.run(instructions=100_000_000)`，最多执行一亿条指令，防止生成的汇编无限运行。 +2. `run_simulation()` 在外部使用独立子进程和 5 秒 timeout。即使 TinyFive 的指令上限执行过慢，或者模拟器内部其他步骤没有及时返回，主测试进程仍能终止该子进程。 + +如果真实 TinyFive 没有安装，`verify_assembly()` 会立即返回： + +```python +{ + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "tinyfive", + "error": "tinyfive not installed" +} +``` + +测试套件不会在 TinyFive 不可用时自动回退到 stub，因此报告中的 `backend: tinyfive` 表示实际调用了 TinyFive。TinyFive 未安装、执行异常或超过 5 秒都会使当前用例判定为 FAIL，但后续用例仍会继续执行。 + +这个设计解决了 branch 分支用例在真实 TinyFive 路径下可能长时间不返回的问题。当前 3 个 branch 用例触发 timeout，说明保护机制生效；它们的汇编跳转问题仍属于编译器后端问题，不属于 A2 保护机制本身。 + +### 4.3 TinyFive 直通验证 + +当前测试的输出正确性已经改为由 TinyFive 直通路径判断。流程是： + +```text +DSL + -> ScratchV 编译器 + -> RISC-V 汇编 + -> TinyFive 模拟执行 + -> 读取 a0/x10 作为 return_value + -> 与 .meta.json 中的 expected_return 对比 +``` + +寄存器分配器会直接产生虚拟寄存器到物理寄存器的映射，`CompileResult.stats["register_map"]` 保存该结构化数据。编译器 CLI 的 `--emit-register-map` 将其写为 JSON，例如： + +```json +{ + "schema_version": 1, + "register_map": { + "input": "t0", + "bias": "t1" + } +} +``` + +`run_tests.py` 通过 `load_initial_registers()` 读取映射，并仅将 `.meta.json` 中的标量输入注入对应 TinyFive 寄存器。该流程不再解析 `--dump-ir` 的人类可读文本，因此 IR 的缩进、注释和打印格式变化不会影响测试输入。TinyFive 执行结束后，`verify_assembly()` 返回来自 `a0/x10` 的 `return_value`。 + +报告中的“TinyFive 输出”和“输出是否匹配”均来自真实编译产物的 TinyFive 执行结果。 + +### 4.4 PASS/FAIL 判断 + +一个用例通过需要同时满足： + +```python +ok = ( + result.returncode == 0 + and output_file.exists() + and sim_result["success"] + and matched + and not regression["regressed"] +) +``` + +也就是说，必须同时满足： + +- 编译成功。 +- 汇编文件生成成功。 +- TinyFive 模拟成功。 +- TinyFive 返回值与期望输出匹配。 +- Benchmark 模式下没有超过性能退化阈值。 +- Benchmark 重复没有因 timeout 或模拟失败提前停止。 + +如果 TinyFive 超时或执行失败，`sim_result["success"]` 为 `False`，该用例直接判定为 FAIL，不会使用其他执行结果替代。 + +### 4.5 指令数统计 + +模拟结果中包含 `instr_count` 字段。测试脚本会把该字段写入报告。 + +普通模式和 Benchmark 模式使用完全相同的报告字段。`instr_count` 始终记录首次 TinyFive 模拟的指令数；Benchmark 模式另外填写重复运行次数、平均值、最小值、最大值和 95% 置信区间。普通模式中的 Benchmark 专属字段统一写为 `null`，不再用 `0` 冒充未执行的统计结果。 + +相关函数： + +- `summarize_benchmark_runs(instr_counts)` +- `detect_regression(avg_instr_count, baseline_instr_count)` + +固定字段包括： + +| 字段 | 普通模式 | Benchmark 模式 | +|---|---|---| +| `mode` | `"normal"` | `"benchmark"` | +| `instr_count` | 单次模拟指令数 | 首次模拟指令数 | +| `compile_returncode` | 编译器返回码 | 编译器返回码 | +| `compile_timed_out` | 是否触发编译 timeout | 是否触发编译 timeout | +| `compile_error` / `compile_log` | 成功时为 `null`，失败时为摘要和日志路径 | 与普通模式相同 | +| `benchmark_runs` | `null` | 实际重复次数 | +| `benchmark_stopped_reason` | `null` | 正常完成时为 `null`，提前停止时记录原因 | +| `avg_instr_count` | `null` | 平均指令数 | +| `min_instr_count` / `max_instr_count` | `null` | 最小值/最大值 | +| `ci95_instr_count` | `null` | 95% 置信区间 | +| `baseline_instr_count` | `null` | 有基线时为基线值，否则为 `null` | +| `delta` / `delta_pct` | `null` | 有基线时为变化量和变化率，否则为 `null` | +| `threshold_pct` / `regressed` | `null` | 退化阈值和判断结果 | + +Markdown 和 HTML 报告固定展示相同列，不再根据模式增删列。CI 推荐读取 `benchmark_reports/topic06/report.json`，其顶层固定包含 `schema_version`、`mode`、`generated_at`、`summary` 和 `results`，当前 schema 版本为 `1`。 + +### 4.6 耗时统计 + +测试脚本使用 `time.perf_counter()` 记录每个用例的耗时。 + +当前记录的字段包括： + +| 字段 | 说明 | +|---|---| +| `compile_time_sec` | 编译 DSL 到 RISC-V 汇编的耗时 | +| `simulation_time_sec` | TinyFive 模拟验证耗时 | +| `total_time_sec` | 单个用例完整处理耗时 | + +这些字段会输出到 Markdown 和 HTML 报告中，便于观察哪些用例编译慢、模拟慢或因为 timeout 导致耗时较长。 + +### 4.7 Benchmark 与性能基线 + +运行： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 +``` + +表示每个用例重复模拟 3 次，统计平均指令数和 95% 置信区间。 + +运行： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 --update-baseline +``` + +表示将当前 Benchmark 结果保存为性能基线： + +```text +benchmarks/topic06/baseline.json +``` + +后续再次运行： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 +``` + +脚本会读取基线文件，并计算变化率： + +```text +变化率 = (当前平均指令数 - 基线指令数) / 基线指令数 * 100% +``` + +如果变化率超过配置阈值，则认为发生性能退化。默认阈值仍为 5%，可以使用命令行参数覆盖： + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 --regression-threshold 2 +``` + +上面的命令表示指令数相对基线增加超过 2% 就判定为退化。参数必须大于或等于 0；未指定时使用 `REGRESSION_THRESHOLD_PCT = 5.0`。实际阈值会传给 `detect_regression()`，并写入 Markdown、HTML 和 JSON 报告。这样日常开发、严格发布检查和宽松趋势监控可以使用不同标准。 + +Benchmark 模式会自动处理 timeout： + +1. 首次 TinyFive 模拟已经 timeout 时，直接跳过该用例的全部 Benchmark 重复。 +2. Benchmark 重复过程中一旦发生 timeout 或模拟失败，立即停止剩余次数。 +3. 只有成功运行的 `instr_count` 才进入平均值、最小值、最大值和置信区间计算，失败结果不会作为 `0` 混入统计。 +4. 报告中的 `benchmark_runs` 记录实际成功次数，`benchmark_stopped_reason` 记录跳过或提前停止原因。 +5. 没有成功 Benchmark 结果的用例不会写入性能基线，也不会进行变化率比较。 + +例如 branch 用例首次模拟等待 5 秒后 timeout，`--benchmark 3` 不会再额外等待 15 秒。 + +--- + +## 五、报告生成设计 + +### 5.1 Markdown 报告 + +输出文件： + +```text +benchmark_reports/topic06/report.md +``` + +内容包括： + +- 测试概览。 +- 用例总数、通过数量、失败数量、通过率。 +- 每个用例的状态、模拟后端、指令数、耗时、预期输出、实际输出。 +- Benchmark 模式下的平均指令数、置信区间、基线、变化率和是否退化。 +- Mermaid 指令数图表。 +- 每个用例的详细说明。 + +普通模式和 Benchmark 模式使用相同表头，未产生的 Benchmark 数据显示为 `null`。 + +Markdown 是默认轻量报告，不需要 `jinja2` 或 `matplotlib`。只有使用 `--full-report` 且图表生成成功时，Markdown 才会加入 PNG 图表链接，避免引用未生成或未刷新的图片。 + +### 5.2 JSON 报告 + +输出文件： + +```text +benchmark_reports/topic06/report.json +``` + +JSON 报告是提供给 CI 和其他程序的稳定解析接口。两种运行模式下字段集合保持一致，CI 只需要检查 `mode` 和字段值，不需要维护两套解析器。 + +顶层 `selection` 字段记录本次使用的 `category` 和 `filter`；未筛选时两个值均为 `null`，便于 CI 判断报告覆盖的是全量还是子集测试。 + +### 5.3 HTML 报告 + +输出文件： + +```text +benchmark_reports/topic06/report.html +``` + +HTML 报告主要用于浏览器查看，报告中会使用不同颜色标识 PASS 和 FAIL。它只在指定 `--full-report` 时生成，需要安装可选依赖 `jinja2`。 + +### 5.4 图表 + +输出文件： + +```text +benchmark_reports/topic06/course_report_instructions.png +``` + +图表由 `matplotlib` 生成，展示不同测试用例的指令数对比。它只在指定 `--full-report` 时生成；默认轻量模式不会导入 `matplotlib`，因此 CI 无需安装图表依赖。 + +### 5.5 报告模块重构 + +Commit Review 指出原 `run_tests.py` 曾保留多轮开发过程中产生的旧版中英文报告函数，存在多次重复定义。Python 实际只会使用最后一次定义，但这些不可达代码增加了文件体积和维护风险。 + +本次重构删除了全部未使用的历史报告生成器，只保留以下唯一实现： + +- `generate_unified_report_text_cn()`：生成 Markdown 报告。 +- `write_unified_html_report_cn()`：生成可选 HTML 报告。 +- `write_json_report()`：生成固定 schema 的 JSON 报告。 +- `write_report()`：统一协调 Markdown、JSON、HTML 和 PNG 输出。 + +`_markdown_cell()`、`_report_value()`、`_reported_instr_count()` 和 `write_chart()` 作为公共辅助函数继续保留。重构删除了约 850 行旧版代码；加入后续功能并迁移为 `scripts/run_topic06_benchmarks.py` 后仍约为 1000 行，且仅保留一套报告实现。报告字段和测试判定逻辑未因重构改变。 + +### 5.6 报告路径可移植性 + +测试运行时仍使用绝对路径调用编译器、读取用例和写入文件，避免工作目录变化影响执行。写入报告前，`_report_path()` 会将仓库内路径转换为相对于 ScratchV 仓库根目录的 POSIX 格式。 + +例如，原报告中的： + +```text +D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\activation\relu_only.dsl +``` + +现在写为： + +```text +tests/topic06/cases/activation/relu_only.dsl +``` + +该规则应用于测试用例路径、汇编文件、编译失败日志，以及 Markdown 概览中的测试目录、构建目录和基线文件。生成的 Markdown、JSON 和 HTML 报告不再依赖本机盘符与工作区位置，可以在 Windows、Linux、macOS 和 CI artifact 中查看。 + +--- + +## 六、运行方式 + +### 6.1 安装依赖 + +基础测试： + +```powershell +pip install -e ".[topic06]" +``` + +完整报告： + +```powershell +pip install -e ".[topic06-report]" +``` + +### 6.2 普通测试 + +```powershell +python scripts/run_topic06_benchmarks.py +``` + +默认只生成 `report.md` 和 `report.json`。生成 HTML 和 PNG 完整报告： + +```powershell +python scripts/run_topic06_benchmarks.py --full-report +``` + +### 6.3 子集测试 + +按测试目录类别筛选： + +```powershell +python scripts/run_topic06_benchmarks.py --category activation +``` + +按 DSL 文件名进行大小写不敏感的包含匹配： + +```powershell +python scripts/run_topic06_benchmarks.py --filter matmul +``` + +两个条件可以组合，组合时必须同时满足： + +```powershell +python scripts/run_topic06_benchmarks.py --category tensor --filter relu +``` + +筛选结果按路径排序，保证本地和 CI 的执行顺序稳定。没有匹配用例时返回退出码 2，不生成或覆盖报告。子集模式结合 `--update-baseline` 时会加载现有基线并只更新本次成功完成 Benchmark 的用例；全量模式仍会重建整份基线。 + +适合快速查看所有用例的 PASS/FAIL 和基本报告。 + +### 6.4 Benchmark 测试 + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 +``` + +适合统计平均指令数和置信区间。 + +### 6.5 更新性能基线 + +```powershell +python scripts/run_topic06_benchmarks.py --benchmark 3 --update-baseline +``` + +适合在修改编译器前或稳定版本上保存基线，供后续性能回归对比使用。 + +--- + +## 七、Mentor Review 对应状态 + +mentor review 中提出的问题分为架构、API、实现和测试覆盖四类。当前文档按真实代码状态整理如下： + +| 编号 | 问题 | 当前处理状态 | 说明 | +|---|---|---|---| +| P0 | 报告生成函数严重冗余 | 已处理 | 删除 5 组旧版/重复 Markdown 与 HTML 生成函数和约 850 行不可达代码；当前仅保留一套 Markdown、HTML、JSON 生成器和统一入口。 | +| P1a | 预生成报告包含 Windows 绝对路径 | 已处理 | `_report_path()` 在报告阶段统一输出相对仓库根目录的 POSIX 路径；生成报告迁入被忽略的 `benchmark_reports/topic06/`。 | +| P1b | 独立交付目录与项目测试/CI 脱节 | 已处理 | 核心脚本、用例、基线和文档已迁入仓库正式目录；`tests/topic06/test_integration.py` 和根目录 workflow 负责项目级集成。 | +| P1c | `infer_initial_registers()` 依赖 IR 文本格式 | 已处理 | 删除 IR 正则推断；寄存器分配器通过 `CompileResult.stats` 和 `--emit-register-map` 输出 JSON，测试套件按变量名读取真实物理寄存器。 | +| P2 | SPEC 中的 20 个通过、3 个失败与真实报告不一致 | 已处理 | 依据最新一次真实 TinyFive 全量运行结果，将文档统一更新为 13 个通过、10 个失败（通过率 56.5%），并在下文记录失败分类；不再沿用旧的 20/3 示例数据。 | +| S1 | 参考解释器与 ScratchV 编译器语义脱节 | 已处理 | 原方案由测试套件单独实现 DSL 求值逻辑，可能绕过真实编译路径，并形成两套需要同步维护的算子语义。现已删除参考解释器及其报告字段；测试结果只使用 `DSL -> ScratchV 编译器 -> RISC-V 汇编 -> TinyFive` 真实路径，并将 TinyFive 的 `return_value` 与 `expected_return` 比较。 | +| S2 | 编译阶段缺 timeout | 已处理 | `run_compile()` 默认设置 30 秒 timeout；超时后返回编译失败结果并继续执行后续用例。 | +| S3 | Benchmark/普通模式报告格式冲突 | 已处理 | Markdown、HTML 和 JSON 使用固定字段；普通模式的 Benchmark 专属字段为 `null`，CI 使用 `report.json` 统一解析。 | +| A1 | `.meta.json` 期望值表达能力有限 | 已处理 | 支持 `scalar`/`tensor`、嵌套数组、dtype/shape 校验和 `expected_output_file`；非法元数据只使当前用例失败。 | +| A2 | `verify_assembly` 调用风险 | 已处理 | TinyFive 未安装时明确失败且不回退 stub；内部设置一亿条指令上限，外层使用独立子进程和 5 秒 timeout，异常只影响当前用例。 | +| I1 | 报告依赖过重 | 已处理 | 默认只生成 Markdown 和 JSON，且不加载 `jinja2`/`matplotlib`；`--full-report` 才生成 HTML 和 PNG，可选依赖由 `pyproject.toml` 的 `topic06-report` 组管理。 | +| I2 | 5% 退化阈值过于刚性 | 已处理 | 新增 `--regression-threshold`，默认 5%，支持非负自定义值；实际阈值参与退化判断并写入 Markdown、HTML 和 JSON 报告。 | +| I3 | Benchmark 重复运行 timeout 用例 | 已处理 | 首次模拟 timeout 时跳过全部重复；重复过程中 timeout/失败则立即停止，失败结果不计入统计，报告记录实际次数和停止原因。 | +| T1 | 编译器 crash 无上下文保存 | 已处理 | 编译失败或超时时保存命令、返回码、耗时、stdout、stderr 和汇编状态到独立日志，并在 Markdown、HTML、JSON 和 CI artifact 中关联日志路径。 | +| T2 | 不支持增量/子集测试 | 已处理 | 新增大小写不敏感的 `--category` 和 `--filter`，支持组合筛选、稳定排序和无匹配退出码 2；子集更新基线采用合并策略。 | + +--- + +## 八、当前测试现状 + +最近一次使用真实 TinyFive 执行全部 23 个用例的报告结果如下（运行后可在 `benchmark_reports/topic06/` 查看）： + +```text +Total: 23 +Passed: 13 +Failed: 10 +Pass rate: 56.5% +``` + +早期 SPEC 中的 `Passed: 20, Failed: 3` 是过期的示例数据，与真实执行结果不一致，已不再作为验收结果。这里记录的 13/10 是当前编译器后端能力的真实状态；测试套件本身是否正确，由稳定类别的 CI 门禁和测试驱动集成测试另行验证。 + +失败用例分为两类。 + +第一类是 `branch/` 目录下 3 个 if 分支用例： + +```text +branch/if_else.dsl +branch/if_relu.dsl +branch/if_then.dsl +``` + +失败原因是 TinyFive 模拟阶段 5 秒超时： + +```text +simulation timeout after 5s +``` + +第二类是以下 7 个数组或矩阵用例： + +```text +reduction/dot_4.dsl +reduction/dot_8.dsl +reduction/dot_relu_8.dsl +tensor/matmul_2x2.dsl +tensor/matmul_4x4.dsl +tensor/matmul_add_2x2.dsl +tensor/matmul_relu_2x2.dsl +``` + +这些用例失败有两层原因： + +1. 测试套件目前根据编译器输出的结构化寄存器映射初始化标量输入，但仍会跳过 `.meta.json` 中的数组和矩阵输入，也没有把数组写入 TinyFive 内存、传递首地址或从结果内存读取矩阵。 +2. 更根本的原因是 RISC-V 指令选择器当前把 `dot` 和 `matmul` 都降低成单条 `mul`。生成汇编没有数组元素读取指令、循环累加或矩阵结果写回，也没有使用 IR 中保存的 `length`、`m`、`n`、`k` 属性。因此只完善测试套件的内存初始化仍不能得到正确结果，后续需要先明确数组调用约定并实现后端降低逻辑。 + +`reduction/dot_relu_4.dsl` 当前显示 PASS，但该用例期望值恰好为 0，而未初始化的数组输入也使 TinyFive 返回 0，属于偶然匹配，不能证明 `dot` 已正确实现。 + +当前报告中的另外 12 个通过用例可以完成编译、TinyFive 模拟和真实返回值对比。 + +--- + +## 九、CI 集成 + +仓库根目录包含正式 GitHub Actions workflow： + +```text +.github/workflows/topic06-benchmark.yml +``` + +CI 在 push 或 pull request 时执行三层检查： + +1. `tests/topic06/test_integration.py` 验证 23 组用例元数据、编译器结构化寄存器映射、Runner 映射读取和跨平台报告路径。 +2. `activation`、`elementwise`、`loop` 三个当前可靠类别使用 `--fail-on-test-failure` 作为阻塞门禁；任一用例失败时 workflow 失败。 +3. 全量 23 例运行 Benchmark 并生成诊断报告。已知未实现的 branch/reduction/tensor 后端能力不会隐藏，但在修复前不阻塞其他提交。 + +CI 使用 `pip install -e ".[topic06]"` 安装基础依赖，并通过 `if: always()` 上传 `benchmark_reports/topic06/` 中的报告与失败日志，同时保留 `benchmarks/topic06/baseline.json`。CI 不安装 `jinja2`、`matplotlib`，也不生成 HTML、PNG；完整可视化报告仍在本地通过 `--full-report` 生成。 + +--- + +## 十、已完成工作对照 + +| 阶段 | 完成情况 | +|---|---| +| W1 学习编译命令和 TinyFive 用法 | 已完成 | +| W2 使用 subprocess 自动调用编译器和模拟器 | 已完成 | +| W3 设计测试用例格式 | 已完成，采用 `.dsl + .meta.json` | +| W4 遍历用例并输出 PASS/FAIL | 已完成 | +| W5 提取指令数并加入报告 | 已完成 | +| W6 matplotlib 图表和 HTML 报告 | 已完成 | +| W7 扩充到 15 个以上用例并覆盖分支循环 | 已完成，共 23 个 | +| W8 使用 `time.perf_counter()` 统计耗时 | 已完成 | +| W9 回归测试模式和 5% 退化阈值 | 已完成 | +| W10 `--benchmark` 重复运行和置信区间 | 已完成 | +| W11 GitHub Actions CI 示例 | 已完成 | +| W12 完整文档 | 已完成 | + +--- diff --git a/docs/topics/INDEX.md b/docs/topics/INDEX.md index c8ebe44..a0eb344 100644 --- a/docs/topics/INDEX.md +++ b/docs/topics/INDEX.md @@ -39,7 +39,7 @@ | 编号 | 课题 | 类型 | 状态 | |------|------|------|------| -| [06](06-性能基准套件.md) | 编译器性能测试套件 | 项目实战 | ✅ | +| [06](06-性能测试套件使用说明.md) | 编译器性能测试套件 | 项目实战 | ✅ | | [12](12-指令计数统计器.md) | RISC-V 后端指令计数统计器 | 项目实战 | ✅ | | [23](23-Cache模型.md) | Cache 行为分析（组相联 + LRU） | 参考分析 | ✅ | | [24](24-Spike仿真.md) | Spike 仿真集成 | 参考分析 | ✅ | diff --git a/pyproject.toml b/pyproject.toml index ce12d92..9fbc77f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,8 @@ riscv = ["tinyfive"] llvm = ["llvmlite"] # optional: LLVM IR JIT execution verify = ["onnxruntime"] # optional: ONNX Runtime comparison all = ["tinyfive", "llvmlite", "onnxruntime"] +topic06 = ["tinyfive", "pytest"] +topic06-report = ["tinyfive", "pytest", "jinja2", "matplotlib"] [project.urls] Source = "https://github.com/kinsomwang/ScratchV" diff --git a/scratchv/backend/register_alloc.py b/scratchv/backend/register_alloc.py index 15d3e1b..134e214 100644 --- a/scratchv/backend/register_alloc.py +++ b/scratchv/backend/register_alloc.py @@ -64,6 +64,11 @@ def run(self) -> list[MachineInstr]: else: return self._allocate_greedy() + @property + def register_map(self) -> dict[str, str]: + """Return the current virtual-to-physical register mapping.""" + return dict(self._vreg_map) + def _allocate_naive(self) -> list[MachineInstr]: """Spill every virtual register to the stack.""" self._output = [] diff --git a/scratchv/compiler.py b/scratchv/compiler.py index bd5648a..c6ded79 100644 --- a/scratchv/compiler.py +++ b/scratchv/compiler.py @@ -218,6 +218,7 @@ class CompilerDriver: def __init__(self, config: CompilerConfig | None = None): self.config = config or CompilerConfig() + self._last_register_map: dict[str, str] = {} # ── Public API ────────────────────────────────────────────────────────── @@ -235,6 +236,7 @@ def compile(self, input_path: str, output_path: str | None = None, """ errors: list[str] = [] warnings: list[str] = [] + self._last_register_map = {} # Resolve output path if output_path is None: @@ -316,7 +318,11 @@ def compile(self, input_path: str, output_path: str | None = None, output_text=asm_text, output_path=output_path, ir_dump=ir_dump, - stats={"opt_message": opt_message, "cycle_report": cycle_report}, + stats={ + "opt_message": opt_message, + "cycle_report": cycle_report, + "register_map": dict(self._last_register_map), + }, warnings=warnings, ) @@ -410,10 +416,13 @@ def _generate_riscv_linear(self, program) -> str: ) ls_insts = block_from_machine_instrs(machine_instrs) lsa = LinearScanAllocator() - return lsa.emit(ls_insts) + asm_text = lsa.emit(ls_insts) + self._last_register_map = dict(lsa.alloc_map) + return asm_text alloc = RegisterAllocator(machine_instrs, mode=self.config.reg_alloc) allocated = alloc.run() + self._last_register_map = alloc.register_map emitter = AsmEmitter(allocated) return emitter.emit() @@ -434,6 +443,7 @@ def _generate_riscv_dag(self, program) -> str: alloc = RegisterAllocator(machine_instrs, mode=self.config.reg_alloc) allocated = alloc.run() + self._last_register_map = alloc.register_map emitter = AsmEmitter(allocated) return emitter.emit() diff --git a/scratchv/main.py b/scratchv/main.py index 25eaf33..71ec38f 100644 --- a/scratchv/main.py +++ b/scratchv/main.py @@ -12,7 +12,9 @@ from __future__ import annotations import argparse +import json import sys +from pathlib import Path from scratchv.compiler import CompilerConfig, CompilerDriver, CompileResult @@ -55,6 +57,11 @@ def build_arg_parser() -> argparse.ArgumentParser: "--dump-ir", action="store_true", help="Dump IR before and after optimization", ) + parser.add_argument( + "--emit-register-map", + metavar="PATH", + help="Write the virtual-to-physical register mapping as JSON", + ) # ── Verification ──────────────────────────────────────────────────── parser.add_argument( @@ -247,6 +254,24 @@ def main(argv: list[str] | None = None) -> int: print(result.ir_dump, file=sys.stderr) if result.success: + if args.emit_register_map: + register_map_path = Path(args.emit_register_map) + try: + register_map_path.parent.mkdir(parents=True, exist_ok=True) + register_map_path.write_text( + json.dumps( + { + "schema_version": 1, + "register_map": result.stats.get("register_map", {}), + }, + indent=2, + ), + encoding="utf-8", + ) + except OSError as exc: + print(f"Error: failed to write register map: {exc}", file=sys.stderr) + return 1 + print(f"OK {args.backend.upper()} output written to {result.output_path}", file=sys.stderr) for w in result.warnings: diff --git a/scratchv/simulator/tinyfive.py b/scratchv/simulator/tinyfive.py index 310d47e..2e7dc1a 100644 --- a/scratchv/simulator/tinyfive.py +++ b/scratchv/simulator/tinyfive.py @@ -173,6 +173,43 @@ def run( after = int(self._m.ops.get('total', 0)) self.instr_count = after - before + def run_until( + self, + end: int, + max_instructions: int = 100_000_000, + start: Optional[int] = None, + *, + strict: bool = False, + ): + """Execute until the program counter reaches *end* or the limit.""" + if not self._available: + return + start_pc = self.pc if start is None else start + before = int(self._m.ops.get('total', 0)) + self.last_error = None + try: + # TinyFive's exe() also normalizes pc to a scalar before decoding. + # Keeping its initial one-element NumPy array breaks branch updates. + self._m.pc = int(start_pc) + with np.errstate(over="ignore"): + for _ in range(max(0, max_instructions)): + if self.pc == end: + break + inst = self._m.read_i32(self.pc) + self._m.dec(np.binary_repr(self._m.u(inst), 32)) + else: + raise RuntimeError( + f"instruction limit exceeded: {max_instructions}" + ) + except Exception as exc: + self.last_error = f"{type(exc).__name__}: {exc}" + if strict: + raise RuntimeError(self.last_error) from exc + finally: + self._m.x[0] = 0 + after = int(self._m.ops.get('total', 0)) + self.instr_count = after - before + # ── Register access ───────────────────────────────────────────────── def get_reg(self, idx: int) -> int: @@ -251,7 +288,7 @@ def _write_u32(self, addr: int, value): def _set_pc(self, val: int): if not self._available: return - if hasattr(self._m.pc, '__getitem__'): + if isinstance(self._m.pc, np.ndarray): self._m.pc[0] = np.uint32(val) # type: ignore[index] else: self._m.pc = int(val) @@ -261,7 +298,7 @@ def pc(self) -> int: if not self._available: return 0 pc_val = self._m.pc - if hasattr(pc_val, '__getitem__'): + if isinstance(pc_val, np.ndarray): return int(pc_val[0]) # type: ignore[index] return int(pc_val) @@ -343,7 +380,47 @@ def pc(self, val: int): self._pc = val -def verify_assembly(asm_code: str, verbose: bool = False) -> dict: +_REG_NUMS: dict[str, int] = { + "zero": 0, "x0": 0, + "ra": 1, "x1": 1, + "sp": 2, "x2": 2, + "gp": 3, "x3": 3, + "tp": 4, "x4": 4, + "t0": 5, "x5": 5, + "t1": 6, "x6": 6, + "t2": 7, "x7": 7, + "s0": 8, "fp": 8, "x8": 8, + "s1": 9, "x9": 9, + "a0": 10, "x10": 10, + "a1": 11, "x11": 11, + "a2": 12, "x12": 12, + "a3": 13, "x13": 13, + "a4": 14, "x14": 14, + "a5": 15, "x15": 15, + "a6": 16, "x16": 16, + "a7": 17, "x17": 17, + "s2": 18, "x18": 18, + "s3": 19, "x19": 19, + "s4": 20, "x20": 20, + "s5": 21, "x21": 21, + "s6": 22, "x22": 22, + "s7": 23, "x23": 23, + "s8": 24, "x24": 24, + "s9": 25, "x25": 25, + "s10": 26, "x26": 26, + "s11": 27, "x27": 27, + "t3": 28, "x28": 28, + "t4": 29, "x29": 29, + "t5": 30, "x30": 30, + "t6": 31, "x31": 31, +} + + +def verify_assembly( + asm_code: str, + verbose: bool = False, + initial_registers: Optional[dict[str, int]] = None, +) -> dict: """Verify generated assembly by running it in TinyFive. Uses ``RISCVAEncoder`` to assemble text → binary, then executes each @@ -353,15 +430,18 @@ def verify_assembly(asm_code: str, verbose: bool = False) -> dict: Args: asm_code: RISC-V assembly text. verbose: Print performance info. + initial_registers: Optional ABI register values to set before execution. Returns: - dict with keys: success, instr_count, error + dict with keys: success, instr_count, return_value, error """ m = ProfiledMachine(mem_size=128 * 1024 * 1024) if not m.available: return { "success": False, "instr_count": 0, + "return_value": None, + "backend": "tinyfive", "error": "tinyfive not installed", } @@ -376,14 +456,35 @@ def verify_assembly(asm_code: str, verbose: bool = False) -> dict: for i in range(0, len(binary), 4) ] m.load_binary(words, origin=0) - m.run(instructions=len(words), start=0, strict=True) + + for reg_name, value in (initial_registers or {}).items(): + reg_num = _REG_NUMS.get(reg_name) + if reg_num is not None: + m.set_reg(reg_num, int(value)) + + return_address = len(binary) + m.set_reg(1, return_address) + m.run_until( + end=return_address, + max_instructions=100_000_000, + start=0, + strict=True, + ) except Exception as e: return { "success": False, "instr_count": m.instr_count, + "return_value": None, + "backend": "tinyfive", "error": str(e), } if verbose: m.print_perf() - return {"success": True, "instr_count": m.instr_count, "error": None} + return { + "success": True, + "instr_count": m.instr_count, + "return_value": m.get_reg(10), + "backend": "tinyfive", + "error": None, + } diff --git a/scripts/run_topic06_benchmarks.py b/scripts/run_topic06_benchmarks.py new file mode 100644 index 0000000..b81e488 --- /dev/null +++ b/scripts/run_topic06_benchmarks.py @@ -0,0 +1,1004 @@ +import argparse +import json +import math +import os +import re +import shlex +import subprocess +import sys +import tempfile +import time +from datetime import datetime +from pathlib import Path + +from scratchv.simulator.tinyfive import verify_assembly + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +TEST_DIR = PROJECT_ROOT / "tests" / "topic06" / "cases" +BUILD_DIR = PROJECT_ROOT / "build" / "topic06" +REPORT_DIR = PROJECT_ROOT / "benchmark_reports" / "topic06" +REPORT_FILE = REPORT_DIR / "report.md" +HTML_REPORT_FILE = REPORT_DIR / "report.html" +JSON_REPORT_FILE = REPORT_DIR / "report.json" +CHART_FILE = REPORT_DIR / "course_report_instructions.png" +BASELINE_FILE = PROJECT_ROOT / "benchmarks" / "topic06" / "baseline.json" +FAILURE_DIR = REPORT_DIR / "failures" +REGRESSION_THRESHOLD_PCT = 5.0 +COMPILE_TIMEOUT_SEC = 30.0 +SIMULATION_TIMEOUT_SEC = 5.0 + + +def run_compile(dsl_file: Path, timeout: float = COMPILE_TIMEOUT_SEC): + output_file = BUILD_DIR / (dsl_file.stem + ".s") + register_map_file = BUILD_DIR / (dsl_file.stem + ".registers.json") + register_map_file.unlink(missing_ok=True) + + cmd = [ + sys.executable, + "-m", + "scratchv.main", + str(dsl_file), + "-o", + str(output_file), + "--optimize", + "all", + "--emit-register-map", + str(register_map_file), + ] + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + encoding="utf-8", + errors="ignore", + cwd=PROJECT_ROOT, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout.decode("utf-8", errors="ignore") if isinstance(exc.stdout, bytes) else (exc.stdout or "") + stderr = exc.stderr.decode("utf-8", errors="ignore") if isinstance(exc.stderr, bytes) else (exc.stderr or "") + timeout_message = f"compile timeout after {timeout:.0f}s" + stderr = f"{stderr.rstrip()}\n{timeout_message}" if stderr else timeout_message + result = subprocess.CompletedProcess( + args=cmd, + returncode=124, + stdout=stdout, + stderr=stderr, + ) + + return result, output_file, register_map_file + + +def _last_nonempty_line(text): + lines = [line.strip() for line in (text or "").splitlines() if line.strip()] + return lines[-1] if lines else None + + +def write_compile_failure_log(dsl_file, result, output_file, compile_time_sec): + FAILURE_DIR.mkdir(parents=True, exist_ok=True) + safe_name = re.sub( + r"[^A-Za-z0-9_.-]+", + "_", + f"{dsl_file.parent.name}-{dsl_file.stem}", + ) + log_file = FAILURE_DIR / f"{safe_name}.compile.log" + args = result.args + if isinstance(args, (list, tuple)): + command = shlex.join(str(arg) for arg in args) + else: + command = str(args or "(compiler was not started)") + stdout = result.stdout or "" + stderr = result.stderr or "" + log_file.write_text( + "\n".join([ + f"Timestamp: {datetime.now().isoformat(timespec='seconds')}", + f"Case: {dsl_file}", + f"Command: {command}", + f"Return code: {result.returncode}", + f"Timed out: {result.returncode == 124}", + f"Compile time (s): {compile_time_sec:.4f}", + f"Assembly output: {output_file}", + f"Assembly exists: {output_file.exists()}", + "", + "--- stdout ---", + stdout, + "", + "--- stderr ---", + stderr, + "", + ]), + encoding="utf-8", + ) + return log_file + + +SUPPORTED_OUTPUT_DTYPES = {"bool", "int32", "int64", "float32", "float64"} + + +def _value_shape(value): + if not isinstance(value, list): + return () + if not value: + return (0,) + child_shapes = [_value_shape(item) for item in value] + if any(shape != child_shapes[0] for shape in child_shapes[1:]): + raise ValueError("expected output must be a rectangular tensor") + return (len(value),) + child_shapes[0] + + +def _validate_output_dtype(value, dtype): + values = value if isinstance(value, list) else [value] + for item in values: + if isinstance(item, list): + _validate_output_dtype(item, dtype) + elif dtype == "bool" and not isinstance(item, bool): + raise ValueError(f"expected output contains non-bool value: {item!r}") + elif dtype.startswith("int") and (isinstance(item, bool) or not isinstance(item, int)): + raise ValueError(f"expected output contains non-integer value: {item!r}") + elif dtype.startswith("float") and (isinstance(item, bool) or not isinstance(item, (int, float))): + raise ValueError(f"expected output contains non-numeric value: {item!r}") + + +def _load_expected_output(metadata, meta_file): + has_inline = "expected_return" in metadata + has_file = "expected_output_file" in metadata + if has_inline == has_file: + raise ValueError("define exactly one of expected_return or expected_output_file") + + if has_inline: + return metadata["expected_return"] + + expected_file = meta_file.parent / metadata["expected_output_file"] + payload = json.loads(expected_file.read_text(encoding="utf-8")) + if isinstance(payload, dict) and "expected_return" in payload: + return payload["expected_return"] + return payload + + +def _validate_metadata(metadata, meta_file): + expected = _load_expected_output(metadata, meta_file) + output_type = metadata.get("expected_output_type", "scalar") + if output_type == "return_value": + output_type = "scalar" + if output_type not in {"scalar", "tensor"}: + raise ValueError(f"unsupported expected_output_type: {output_type}") + + actual_shape = _value_shape(expected) + if output_type == "scalar" and actual_shape: + raise ValueError("scalar expected output cannot contain a list") + if output_type == "tensor" and not actual_shape: + raise ValueError("tensor expected output must contain a nested JSON array") + + declared_shape = metadata.get("output_shape") + if output_type == "tensor" and declared_shape is None: + raise ValueError("tensor expected output requires output_shape") + if declared_shape is not None: + if not isinstance(declared_shape, list) or any( + isinstance(size, bool) or not isinstance(size, int) or size < 0 + for size in declared_shape + ): + raise ValueError("output_shape must be a list of non-negative integers") + if tuple(declared_shape) != actual_shape: + raise ValueError( + f"output_shape {declared_shape} does not match expected output shape {list(actual_shape)}" + ) + + dtype = metadata.get("output_dtype") + if output_type == "tensor" and dtype is None: + raise ValueError("tensor expected output requires output_dtype") + if dtype is not None: + if dtype not in SUPPORTED_OUTPUT_DTYPES: + raise ValueError(f"unsupported output_dtype: {dtype}") + _validate_output_dtype(expected, dtype) + + metadata["expected_output_type"] = output_type + metadata["expected_return"] = expected + return metadata + + +def load_metadata(dsl_file: Path): + meta_file = dsl_file.with_suffix(".meta.json") + if not meta_file.exists(): + return { + "description": "", + "expected_output_type": "scalar", + "expected_return": None, + "_metadata_error": f"metadata file not found: {meta_file}", + } + try: + metadata = json.loads(meta_file.read_text(encoding="utf-8")) + return _validate_metadata(metadata, meta_file) + except (OSError, ValueError, TypeError, json.JSONDecodeError) as exc: + return { + "description": "", + "expected_output_type": "invalid", + "expected_return": None, + "_metadata_error": str(exc), + } + + +def load_initial_registers(register_map_file: Path, inputs: dict) -> dict[str, int]: + """Load compiler-emitted register assignments for scalar DSL inputs.""" + try: + payload = json.loads(register_map_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid compiler register map: {exc}") from exc + + register_map = payload.get("register_map") + if not isinstance(register_map, dict): + raise ValueError("invalid compiler register map: missing register_map object") + + result: dict[str, int] = {} + for name, value in inputs.items(): + register = register_map.get(name) + if not isinstance(register, str): + continue + if isinstance(value, bool): + result[register] = int(value) + elif isinstance(value, int): + result[register] = value + elif isinstance(value, float) and value.is_integer(): + result[register] = int(value) + return result + + +def run_simulation( + asm_file: Path, + initial_registers: dict[str, int] | None = None, + timeout: float = SIMULATION_TIMEOUT_SEC, +): + if not asm_file.exists(): + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": "assembly file not found", + } + + code = "\n".join([ + "import json, sys", + "from pathlib import Path", + "from scratchv.simulator.tinyfive import verify_assembly", + "asm = Path(sys.argv[1]).read_text(encoding='utf-8')", + "initial_registers = json.loads(sys.argv[2])", + "try:", + " result = verify_assembly(asm, initial_registers=initial_registers)", + "except Exception as exc:", + " result = {'success': False, 'instr_count': 0, " + "'return_value': None, 'backend': 'tinyfive', 'error': str(exc)}", + "print(json.dumps(result, ensure_ascii=False))", + ]) + + try: + completed = subprocess.run( + [ + sys.executable, + "-c", + code, + str(asm_file), + json.dumps(initial_registers or {}), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="ignore", + cwd=PROJECT_ROOT, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "timeout", + "error": f"simulation timeout after {timeout:.0f}s", + } + + if completed.returncode != 0: + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "tinyfive", + "error": (completed.stderr or completed.stdout or "simulation failed").strip(), + } + + try: + return json.loads(completed.stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError) as exc: + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "tinyfive", + "error": f"invalid simulation output: {exc}", + } + + +def values_equal(lhs, rhs): + if isinstance(lhs, list) and isinstance(rhs, list): + if len(lhs) != len(rhs): + return False + return all(values_equal(a, b) for a, b in zip(lhs, rhs)) + if isinstance(lhs, float) or isinstance(rhs, float): + return math.isclose(lhs, rhs, rel_tol=1e-7, abs_tol=1e-7) + return lhs == rhs + + +def summarize_benchmark_runs(instr_counts): + if not instr_counts: + return { + "runs": 0, + "avg_instr_count": None, + "min_instr_count": None, + "max_instr_count": None, + "ci95_instr_count": None, + } + avg = sum(instr_counts) / len(instr_counts) + if len(instr_counts) > 1: + variance = sum((value - avg) ** 2 for value in instr_counts) / (len(instr_counts) - 1) + ci95 = 1.96 * math.sqrt(variance) / math.sqrt(len(instr_counts)) + else: + ci95 = 0.0 + return { + "runs": len(instr_counts), + "avg_instr_count": avg, + "min_instr_count": min(instr_counts), + "max_instr_count": max(instr_counts), + "ci95_instr_count": ci95, + } + + +def detect_regression( + avg_instr_count, + baseline_instr_count, + threshold_pct=REGRESSION_THRESHOLD_PCT, +): + delta = avg_instr_count - baseline_instr_count + delta_pct = 0.0 if baseline_instr_count == 0 else (delta / baseline_instr_count) * 100.0 + return { + "baseline_instr_count": baseline_instr_count, + "delta": round(delta, 4), + "delta_pct": round(delta_pct, 4), + "threshold_pct": threshold_pct, + "regressed": delta_pct > threshold_pct, + } + + +def load_baseline(): + if not BASELINE_FILE.exists(): + return {} + return json.loads(BASELINE_FILE.read_text(encoding="utf-8")) + + +def save_baseline(results, preserve_existing=False): + BASELINE_FILE.parent.mkdir(parents=True, exist_ok=True) + payload = load_baseline() if preserve_existing else {} + for r in results: + if r["avg_instr_count"] is None: + continue + payload[r["name"]] = { + "category": r["category"], + "avg_instr_count": r["avg_instr_count"], + "runs": r["benchmark_runs"], + } + BASELINE_FILE.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + +def _markdown_cell(value): + return str(value).replace("\n", " ").replace("|", "\\|") + + +def _report_path(path): + path = Path(path) + try: + return path.resolve().relative_to(PROJECT_ROOT).as_posix() + except ValueError: + return path.as_posix() + + +def write_chart(results): + try: + mpl_config_dir = Path(tempfile.gettempdir()) / "scratchv-matplotlib" + mpl_config_dir.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("MPLCONFIGDIR", str(mpl_config_dir)) + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError: + return None + + names = [result["name"] for result in results] + values = [_reported_instr_count(result) for result in results] + width = max(10, len(names) * 0.45) + fig, ax = plt.subplots(figsize=(width, 5)) + ax.bar(range(len(names)), values, color="#2563eb") + ax.set_title("ScratchV Course Benchmark Instruction Counts") + ax.set_ylabel("Instructions") + ax.set_xticks(range(len(names))) + ax.set_xticklabels(names, rotation=60, ha="right", fontsize=8) + ax.grid(axis="y", linestyle="--", alpha=0.35) + fig.tight_layout() + fig.savefig(CHART_FILE, dpi=160) + plt.close(fig) + return CHART_FILE + + +def _reported_instr_count(result): + average = result.get("avg_instr_count") + return result["instr_count"] if average is None else average + + +def _report_value(value, precision=None, prefix=""): + if value is None: + return "null" + if precision is not None: + return f"{prefix}{value:.{precision}f}" + return f"{prefix}{value}" + + +def generate_unified_report_text_cn( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, + include_chart=False, +): + mode = results[0]["mode"] if results else "normal" + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + lines = [ + "# ScratchV DSL 编译器性能测试报告\n\n", + "## 测试概览\n\n", + f"- Schema 版本: 1\n", + f"- 运行模式: {mode}\n", + f"- 类别筛选: {_report_value(selection_category)}\n", + f"- 名称筛选: {_report_value(selection_filter)}\n", + f"- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + f"- 用例总数: {len(results)}\n", + f"- 通过数量: {passed}\n", + f"- 失败数量: {failed}\n", + f"- 通过率: {pass_rate:.1f}%\n", + f"- 测试目录: `{_report_path(TEST_DIR)}`\n", + f"- 汇编输出目录: `{_report_path(BUILD_DIR)}`\n", + f"- 性能基线文件: `{_report_path(BASELINE_FILE)}`\n", + f"- 性能退化阈值: {regression_threshold_pct:.2f}%\n", + f"- 单次编译超时: {COMPILE_TIMEOUT_SEC:.0f}s\n", + f"- 单次模拟超时: {SIMULATION_TIMEOUT_SEC:.0f}s\n\n", + "## 测试结果\n\n", + "| 用例 | 类别 | 状态 | 编译返回码 | 编译日志 | 模拟后端 | 指令数 | Benchmark 次数 | Benchmark 停止原因 | 平均指令数 | 95% 置信区间 | 最小 | 最大 | 编译耗时(s) | 模拟耗时(s) | 总耗时(s) | 基线 | 变化量 | 变化率(%) | 退化阈值(%) | 是否退化 | 预期输出 | TinyFive 输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---:|---|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|---|\n", + ] + + for result in results: + lines.append( + f"| {result['name']} | {result['category']} | {result['status']} | " + f"{result['compile_returncode']} | {_report_value(result['compile_log'])} | " + f"{result['backend']} | {result['instr_count']} | " + f"{_report_value(result['benchmark_runs'])} | " + f"{_report_value(result['benchmark_stopped_reason'])} | " + f"{_report_value(result['avg_instr_count'], 2)} | " + f"{_report_value(result['ci95_instr_count'], 2, '±')} | " + f"{_report_value(result['min_instr_count'])} | " + f"{_report_value(result['max_instr_count'])} | " + f"{result['compile_time_sec']:.4f} | {result['simulation_time_sec']:.4f} | " + f"{result['total_time_sec']:.4f} | " + f"{_report_value(result['baseline_instr_count'], 2)} | " + f"{_report_value(result['delta'], 2)} | " + f"{_report_value(result['delta_pct'], 2)} | " + f"{_report_value(result['threshold_pct'], 2)} | " + f"{_report_value(result['regressed'])} | {_markdown_cell(result['expected'])} | " + f"{_markdown_cell(result['actual'])} | {result['matched']} | {result['asm']} |\n" + ) + + if include_chart: + lines.extend([ + "\n## 性能图表\n\n", + f"![课程版指令数图表]({CHART_FILE.name})\n\n", + ]) + lines.append("\n## 用例详情\n\n") + for result in results: + lines.extend([ + f"### {result['name']}\n\n", + f"- 类别: {result['category']}\n", + f"- 描述: {result['description']}\n", + f"- 预期输出 ({result['expected_type']}): {result['expected']}\n", + f"- TinyFive 输出: {result['actual']}\n", + f"- 输出是否匹配: {result['matched']}\n", + f"- TinyFive 初始寄存器: {result['initial_registers']}\n", + f"- 模拟后端: {result['backend']}\n", + f"- 指令数: {result['instr_count']}\n", + f"- 编译返回码: {result['compile_returncode']}\n", + f"- 编译是否超时: {result['compile_timed_out']}\n", + f"- 编译错误摘要: {_report_value(result['compile_error'])}\n", + f"- 编译失败日志: {_report_value(result['compile_log'])}\n", + f"- Benchmark 重复次数: {_report_value(result['benchmark_runs'])}\n", + f"- Benchmark 停止原因: {_report_value(result['benchmark_stopped_reason'])}\n", + f"- 平均指令数: {_report_value(result['avg_instr_count'], 2)}\n", + f"- 95% 置信区间: {_report_value(result['ci95_instr_count'], 2, '±')}\n", + f"- 最小指令数: {_report_value(result['min_instr_count'])}\n", + f"- 最大指令数: {_report_value(result['max_instr_count'])}\n", + f"- 编译耗时(s): {result['compile_time_sec']:.4f}\n", + f"- 模拟耗时(s): {result['simulation_time_sec']:.4f}\n", + f"- 总耗时(s): {result['total_time_sec']:.4f}\n", + f"- 基线指令数: {_report_value(result['baseline_instr_count'], 2)}\n", + f"- 性能变化量: {_report_value(result['delta'], 2)}\n", + f"- 性能变化率(%): {_report_value(result['delta_pct'], 2)}\n", + f"- 性能退化阈值(%): {_report_value(result['threshold_pct'], 2)}\n", + f"- 是否性能退化: {_report_value(result['regressed'])}\n", + f"- 汇编文件: {result['asm']}\n\n", + ]) + return "".join(lines) + + +def write_unified_html_report_cn( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, +): + try: + from jinja2 import Template + except ImportError: + return None + + template = Template(""" + + + + ScratchV 测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

Schema 版本：1，运行模式：{{ mode }}

+

类别筛选：{{ fmt(selection_category) }}，名称筛选：{{ fmt(selection_filter) }}

+

用例总数：{{ total }}，通过：{{ passed }}，失败：{{ failed }}

+

性能退化阈值：{{ fmt(regression_threshold_pct, 2) }}%

+
+ 课程版指令数图表 +
+ + + + + + + + {% for r in results %} + + + + + + + + + + + {% endfor %} +
用例类别状态编译返回码编译日志模拟后端指令数Benchmark 次数Benchmark 停止原因平均指令数95% 置信区间最小最大编译耗时(s)模拟耗时(s)总耗时(s)基线变化量变化率(%)退化阈值(%)是否退化预期输出TinyFive 输出输出匹配汇编文件
{{ r.name }}{{ r.category }}{{ r.status }}{{ r.compile_returncode }}{{ fmt(r.compile_log) }}{{ r.backend }}{{ r.instr_count }}{{ fmt(r.benchmark_runs) }}{{ fmt(r.benchmark_stopped_reason) }}{{ fmt(r.avg_instr_count, 2) }}{{ fmt(r.ci95_instr_count, 2, '±') }}{{ fmt(r.min_instr_count) }}{{ fmt(r.max_instr_count) }}{{ fmt(r.compile_time_sec, 4) }}{{ fmt(r.simulation_time_sec, 4) }}{{ fmt(r.total_time_sec, 4) }}{{ fmt(r.baseline_instr_count, 2) }}{{ fmt(r.delta, 2) }}{{ fmt(r.delta_pct, 2) }}{{ fmt(r.threshold_pct, 2) }}{{ fmt(r.regressed) }}{{ r.expected }}{{ r.actual }}{{ r.matched }}{{ r.asm }}
+ + +""") + mode = results[0]["mode"] if results else "normal" + HTML_REPORT_FILE.write_text( + template.render( + mode=mode, + total=len(results), + passed=passed, + failed=failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + chart_name=CHART_FILE.name, + results=results, + fmt=_report_value, + ), + encoding="utf-8", + ) + return HTML_REPORT_FILE + + +def write_json_report( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, + full_report=False, +): + mode = results[0]["mode"] if results else "normal" + payload = { + "schema_version": 1, + "mode": mode, + "report_level": "full" if full_report else "light", + "regression_threshold_pct": regression_threshold_pct, + "selection": { + "category": selection_category, + "filter": selection_filter, + }, + "generated_at": datetime.now().isoformat(timespec="seconds"), + "summary": { + "total": len(results), + "passed": passed, + "failed": failed, + }, + "results": results, + } + JSON_REPORT_FILE.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + return JSON_REPORT_FILE + + +def write_report( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, + full_report=False, +): + REPORT_DIR.mkdir(parents=True, exist_ok=True) + chart_path = write_chart(results) if full_report else None + REPORT_FILE.write_text( + generate_unified_report_text_cn( + results, + passed, + failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + include_chart=chart_path is not None, + ), + encoding="utf-8", + ) + html_path = ( + write_unified_html_report_cn( + results, + passed, + failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + ) + if full_report else None + ) + json_path = write_json_report( + results, + passed, + failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + full_report=full_report, + ) + print(f"\nMarkdown report written to {REPORT_FILE}") + print(f"JSON report written to {json_path}") + if full_report: + if html_path: + print(f"HTML report written to {html_path}") + else: + print("HTML report skipped: jinja2 is not installed") + if chart_path: + print(f"Chart written to {chart_path}") + else: + print("Chart skipped: matplotlib is not installed") + + +def non_negative_float(value): + parsed = float(value) + if parsed < 0: + raise argparse.ArgumentTypeError("must be greater than or equal to 0") + return parsed + + +def select_dsl_files(test_dir, category=None, name_filter=None): + dsl_files = sorted(test_dir.rglob("*.dsl"), key=lambda path: str(path).lower()) + if category: + expected_category = category.lower() + dsl_files = [ + path for path in dsl_files + if path.parent.name.lower() == expected_category + ] + if name_filter: + expected_name = name_filter.lower() + dsl_files = [ + path for path in dsl_files + if expected_name in path.stem.lower() + ] + return dsl_files + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description="Run ScratchV DSL benchmark suite.") + parser.add_argument("--benchmark", type=int, default=0, metavar="N", + help="Run each case N times and report average instruction count.") + parser.add_argument("--update-baseline", action="store_true", + help="Write current benchmark averages to the baseline file.") + parser.add_argument("--full-report", action="store_true", + help="Also generate the optional HTML report and PNG chart.") + parser.add_argument( + "--regression-threshold", + type=non_negative_float, + default=REGRESSION_THRESHOLD_PCT, + metavar="PERCENT", + help="Mark instruction-count increases above this percentage as regressions (default: 5).", + ) + parser.add_argument( + "--category", + help="Only run cases in this test category (for example: activation or tensor).", + ) + parser.add_argument( + "--filter", + dest="name_filter", + help="Only run cases whose file name contains this text.", + ) + parser.add_argument( + "--fail-on-test-failure", + action="store_true", + help="Return exit code 1 when one or more selected cases fail.", + ) + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + BUILD_DIR.mkdir(parents=True, exist_ok=True) + baseline = load_baseline() if args.benchmark else {} + + all_dsl_files = list(TEST_DIR.rglob("*.dsl")) + + if not all_dsl_files: + print("No DSL test files found.") + return 2 + + dsl_files = select_dsl_files( + TEST_DIR, + category=args.category, + name_filter=args.name_filter, + ) + if not dsl_files: + print("No DSL test cases matched the selected filters.") + return 2 + + passed = 0 + failed = 0 + results = [] + + print("Running DSL compiler tests...") + print("=" * 50) + + for dsl_file in dsl_files: + print(f"\n[TEST] {dsl_file}") + + case_start = time.perf_counter() + meta = load_metadata(dsl_file) + compile_start = time.perf_counter() + metadata_error = meta.get("_metadata_error") + if metadata_error: + output_file = BUILD_DIR / (dsl_file.stem + ".s") + register_map_file = BUILD_DIR / (dsl_file.stem + ".registers.json") + result = subprocess.CompletedProcess( + args=[], + returncode=2, + stdout="", + stderr=f"metadata error: {metadata_error}", + ) + else: + result, output_file, register_map_file = run_compile(dsl_file) + compile_time_sec = time.perf_counter() - compile_start + compile_log = None + compile_error = None + if result.returncode != 0: + compile_log = write_compile_failure_log( + dsl_file, + result, + output_file, + compile_time_sec, + ) + compile_error = _last_nonempty_line(result.stderr or result.stdout) + register_map_error = None + initial_registers = {} + if result.returncode == 0: + try: + initial_registers = load_initial_registers( + register_map_file, + meta.get("inputs", {}), + ) + except ValueError as exc: + register_map_error = str(exc) + simulation_start = time.perf_counter() + if result.returncode != 0: + sim_result = { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": (result.stderr or result.stdout or "compile failed").strip(), + } + elif register_map_error: + sim_result = { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": register_map_error, + } + else: + sim_result = run_simulation(output_file, initial_registers) + simulation_time_sec = time.perf_counter() - simulation_start + expected_value = meta.get("expected_return") + actual_value = sim_result.get("return_value") + matched = bool(sim_result.get("success")) and values_equal(actual_value, expected_value) + benchmark_counts = [] + benchmark_summary = { + "runs": None, + "avg_instr_count": None, + "min_instr_count": None, + "max_instr_count": None, + "ci95_instr_count": None, + } + regression = { + "baseline_instr_count": None, + "delta": None, + "delta_pct": None, + "threshold_pct": None, + "regressed": None, + } + benchmark_stopped_reason = None + + if args.benchmark > 0: + regression["threshold_pct"] = args.regression_threshold + if result.returncode != 0 or not output_file.exists(): + benchmark_stopped_reason = "benchmark skipped: compile failed" + benchmark_summary = summarize_benchmark_runs([]) + elif sim_result.get("backend") == "timeout": + benchmark_stopped_reason = "benchmark skipped: initial simulation timeout" + benchmark_summary = summarize_benchmark_runs([]) + elif not sim_result.get("success"): + benchmark_stopped_reason = "benchmark skipped: initial simulation failed" + benchmark_summary = summarize_benchmark_runs([]) + else: + for run_index in range(1, args.benchmark + 1): + benchmark_result = run_simulation(output_file, initial_registers) + if benchmark_result.get("backend") == "timeout": + benchmark_stopped_reason = ( + f"benchmark stopped: timeout on run {run_index}" + ) + break + if not benchmark_result.get("success"): + benchmark_stopped_reason = ( + f"benchmark stopped: simulation failed on run {run_index}" + ) + break + benchmark_counts.append(benchmark_result.get("instr_count", 0)) + benchmark_summary = summarize_benchmark_runs(benchmark_counts) + + if benchmark_summary["avg_instr_count"] is not None and benchmark_stopped_reason is None: + regression["regressed"] = False + + baseline_entry = baseline.get(dsl_file.stem) + if ( + baseline_entry + and benchmark_summary["avg_instr_count"] is not None + and benchmark_stopped_reason is None + ): + regression = detect_regression( + avg_instr_count=benchmark_summary["avg_instr_count"], + baseline_instr_count=baseline_entry.get("avg_instr_count", 0.0), + threshold_pct=args.regression_threshold, + ) + total_time_sec = time.perf_counter() - case_start + + ok = ( + result.returncode == 0 + and output_file.exists() + and sim_result["success"] + and matched + and regression["regressed"] is not True + and benchmark_stopped_reason is None + ) + + if ok: + print("PASS") + passed += 1 + status = "PASS" + else: + print("FAIL") + failed += 1 + status = "FAIL" + if sim_result.get("error"): + print(sim_result["error"]) + else: + print( + "output mismatch: " + f"expected={expected_value}, " + f"tinyfive={actual_value}" + ) + + results.append({ + "mode": "benchmark" if args.benchmark > 0 else "normal", + "name": dsl_file.stem, + "category": dsl_file.parent.name, + "path": _report_path(dsl_file), + "status": status, + "description": meta.get("description", ""), + "expected_type": meta.get("expected_output_type", "scalar"), + "output_dtype": meta.get("output_dtype"), + "output_shape": meta.get("output_shape"), + "expected": expected_value, + "actual": actual_value, + "matched": matched, + "initial_registers": initial_registers, + "register_map": _report_path(register_map_file), + "backend": sim_result.get("backend", "none"), + "instr_count": sim_result.get("instr_count", 0), + "compile_returncode": result.returncode, + "compile_timed_out": result.returncode == 124, + "compile_error": compile_error, + "compile_log": _report_path(compile_log) if compile_log else None, + "compile_time_sec": compile_time_sec, + "simulation_time_sec": simulation_time_sec, + "total_time_sec": total_time_sec, + "benchmark_runs": benchmark_summary["runs"], + "benchmark_stopped_reason": benchmark_stopped_reason, + "avg_instr_count": benchmark_summary["avg_instr_count"], + "min_instr_count": benchmark_summary["min_instr_count"], + "max_instr_count": benchmark_summary["max_instr_count"], + "ci95_instr_count": benchmark_summary["ci95_instr_count"], + "baseline_instr_count": regression["baseline_instr_count"], + "delta": regression["delta"], + "delta_pct": regression["delta_pct"], + "threshold_pct": regression["threshold_pct"], + "regressed": regression["regressed"], + "asm": _report_path(output_file), + }) + + print("\n" + "=" * 50) + print(f"Total: {len(dsl_files)}") + print(f"Passed: {passed}") + print(f"Failed: {failed}") + + if args.benchmark and args.update_baseline: + save_baseline( + results, + preserve_existing=bool(args.category or args.name_filter), + ) + print(f"Baseline written to {BASELINE_FILE}") + + write_report( + results, + passed, + failed, + regression_threshold_pct=args.regression_threshold, + selection_category=args.category, + selection_filter=args.name_filter, + full_report=args.full_report, + ) + return 1 if args.fail_on_test_failure and failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_simulator.py b/tests/test_simulator.py index f32c708..3f0e5bf 100644 --- a/tests/test_simulator.py +++ b/tests/test_simulator.py @@ -122,6 +122,40 @@ def test_large_li_expands_and_simulates_equivalently(self): outputs.append(machine.get_reg(5)) assert outputs == [4098, 4098] + def test_run_until_executes_loop_and_stops_at_return_address(self): + binary = assemble_to_binary( + "li x5, 0\n" + "li x6, 4\n" + "loop:\n" + "addi x5, x5, 1\n" + "blt x5, x6, loop\n" + "jalr x0, x1, 0\n" + ) + words = [ + int.from_bytes(binary[i:i + 4], "little") + for i in range(0, len(binary), 4) + ] + machine = ProfiledMachine(mem_size=4096) + machine.load_binary(words, origin=0) + machine.set_reg(1, len(binary)) + + machine.run_until(end=len(binary), start=0, strict=True) + + assert machine.get_reg(5) == 4 + assert machine.pc == len(binary) + assert machine.instr_count == 11 + + def test_verify_assembly_injects_inputs_and_returns_a0(self): + result = verify_assembly( + "add x10, x5, x6\njalr x0, x1, 0\n", + initial_registers={"t0": -2, "t1": 7}, + ) + + assert result["success"] is True + assert result["backend"] == "tinyfive" + assert result["return_value"] == 5 + assert result["instr_count"] == 2 + @pytest.mark.parametrize("value", [ -(1 << 31), -4097, -2049, -2048, -1, 0, 2047, 2048, 4096, 4098, (1 << 31) - 1, diff --git a/tests/topic06/cases/activation/add_relu_relu.dsl b/tests/topic06/cases/activation/add_relu_relu.dsl new file mode 100644 index 0000000..52c1325 --- /dev/null +++ b/tests/topic06/cases/activation/add_relu_relu.dsl @@ -0,0 +1,5 @@ +# Add followed by two ReLU stages +x = add(input, bias) +y = relu(x) +result = relu(y) +return result diff --git a/tests/topic06/cases/activation/add_relu_relu.meta.json b/tests/topic06/cases/activation/add_relu_relu.meta.json new file mode 100644 index 0000000..e587d8a --- /dev/null +++ b/tests/topic06/cases/activation/add_relu_relu.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Add input and bias, then apply ReLU twice.", + "expected_output_type": "return_value", + "inputs": { + "input": -3, + "bias": 10 + }, + "expected_return": 7 +} diff --git a/tests/topic06/cases/activation/relu_add.dsl b/tests/topic06/cases/activation/relu_add.dsl new file mode 100644 index 0000000..7190772 --- /dev/null +++ b/tests/topic06/cases/activation/relu_add.dsl @@ -0,0 +1,4 @@ +# Add + ReLU activation +x = add(input, bias) +y = relu(x) +return y \ No newline at end of file diff --git a/tests/topic06/cases/activation/relu_add.meta.json b/tests/topic06/cases/activation/relu_add.meta.json new file mode 100644 index 0000000..f1b7f84 --- /dev/null +++ b/tests/topic06/cases/activation/relu_add.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Add input and bias, then apply one ReLU.", + "expected_output_type": "return_value", + "inputs": { + "input": -2, + "bias": 5 + }, + "expected_return": 3 +} diff --git a/tests/topic06/cases/activation/relu_only.dsl b/tests/topic06/cases/activation/relu_only.dsl new file mode 100644 index 0000000..404a911 --- /dev/null +++ b/tests/topic06/cases/activation/relu_only.dsl @@ -0,0 +1,3 @@ +# Single ReLU activation +result = relu(x) +return result diff --git a/tests/topic06/cases/activation/relu_only.meta.json b/tests/topic06/cases/activation/relu_only.meta.json new file mode 100644 index 0000000..b08af80 --- /dev/null +++ b/tests/topic06/cases/activation/relu_only.meta.json @@ -0,0 +1,8 @@ +{ + "description": "Apply ReLU directly to a single input value.", + "expected_output_type": "return_value", + "inputs": { + "x": -5 + }, + "expected_return": 0 +} diff --git a/tests/topic06/cases/activation/relu_twice.dsl b/tests/topic06/cases/activation/relu_twice.dsl new file mode 100644 index 0000000..7d2defe --- /dev/null +++ b/tests/topic06/cases/activation/relu_twice.dsl @@ -0,0 +1,4 @@ +# Two-stage ReLU activation +x = relu(input) +result = relu(x) +return result diff --git a/tests/topic06/cases/activation/relu_twice.meta.json b/tests/topic06/cases/activation/relu_twice.meta.json new file mode 100644 index 0000000..fabbb38 --- /dev/null +++ b/tests/topic06/cases/activation/relu_twice.meta.json @@ -0,0 +1,8 @@ +{ + "description": "Apply ReLU twice to the same activation path.", + "expected_output_type": "return_value", + "inputs": { + "input": 4 + }, + "expected_return": 4 +} diff --git a/tests/topic06/cases/branch/if_else.dsl b/tests/topic06/cases/branch/if_else.dsl new file mode 100644 index 0000000..6541768 --- /dev/null +++ b/tests/topic06/cases/branch/if_else.dsl @@ -0,0 +1,8 @@ +# Branch takes the else path when flag is zero. +if (flag != 0): +result = add(a, b) +return result +else +result = sub(a, b) +return result +endif diff --git a/tests/topic06/cases/branch/if_else.meta.json b/tests/topic06/cases/branch/if_else.meta.json new file mode 100644 index 0000000..fdacf6a --- /dev/null +++ b/tests/topic06/cases/branch/if_else.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch returns subtraction result when flag is zero.", + "expected_output_type": "return_value", + "inputs": { + "flag": 0, + "a": 9, + "b": 4 + }, + "expected_return": 5 +} diff --git a/tests/topic06/cases/branch/if_relu.dsl b/tests/topic06/cases/branch/if_relu.dsl new file mode 100644 index 0000000..1346060 --- /dev/null +++ b/tests/topic06/cases/branch/if_relu.dsl @@ -0,0 +1,8 @@ +# Branch selects whether to apply ReLU after an add. +sum = add(a, b) +if (use_relu != 0): +result = relu(sum) +return result +else +return sum +endif diff --git a/tests/topic06/cases/branch/if_relu.meta.json b/tests/topic06/cases/branch/if_relu.meta.json new file mode 100644 index 0000000..1925fb5 --- /dev/null +++ b/tests/topic06/cases/branch/if_relu.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch combined with add and relu.", + "expected_output_type": "return_value", + "inputs": { + "use_relu": 1, + "a": -8, + "b": 3 + }, + "expected_return": 0 +} diff --git a/tests/topic06/cases/branch/if_then.dsl b/tests/topic06/cases/branch/if_then.dsl new file mode 100644 index 0000000..c305c3d --- /dev/null +++ b/tests/topic06/cases/branch/if_then.dsl @@ -0,0 +1,8 @@ +# Branch takes the then path when flag is non-zero. +if (flag != 0): +result = add(a, b) +return result +else +result = sub(a, b) +return result +endif diff --git a/tests/topic06/cases/branch/if_then.meta.json b/tests/topic06/cases/branch/if_then.meta.json new file mode 100644 index 0000000..04748d1 --- /dev/null +++ b/tests/topic06/cases/branch/if_then.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch returns add result when flag is non-zero.", + "expected_output_type": "return_value", + "inputs": { + "flag": 1, + "a": 9, + "b": 4 + }, + "expected_return": 13 +} diff --git a/tests/topic06/cases/elementwise/add_chain.dsl b/tests/topic06/cases/elementwise/add_chain.dsl new file mode 100644 index 0000000..d3917c9 --- /dev/null +++ b/tests/topic06/cases/elementwise/add_chain.dsl @@ -0,0 +1,4 @@ +# Chain of two vector adds +x = add(a, b) +y = add(x, c) +return y \ No newline at end of file diff --git a/tests/topic06/cases/elementwise/add_chain.meta.json b/tests/topic06/cases/elementwise/add_chain.meta.json new file mode 100644 index 0000000..b07adaf --- /dev/null +++ b/tests/topic06/cases/elementwise/add_chain.meta.json @@ -0,0 +1,10 @@ +{ + "description": "Add a and b, then add c to the intermediate result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4 + }, + "expected_return": 9 +} diff --git a/tests/topic06/cases/elementwise/add_chain_3.dsl b/tests/topic06/cases/elementwise/add_chain_3.dsl new file mode 100644 index 0000000..aeca4df --- /dev/null +++ b/tests/topic06/cases/elementwise/add_chain_3.dsl @@ -0,0 +1,5 @@ +# Chain of three add operations +x = add(a, b) +y = add(x, c) +z = add(y, d) +return z diff --git a/tests/topic06/cases/elementwise/add_chain_3.meta.json b/tests/topic06/cases/elementwise/add_chain_3.meta.json new file mode 100644 index 0000000..368b3aa --- /dev/null +++ b/tests/topic06/cases/elementwise/add_chain_3.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Chain three add operations across four symbolic inputs.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4, + "d": 5 + }, + "expected_return": 14 +} diff --git a/tests/topic06/cases/elementwise/add_fan_in_4.dsl b/tests/topic06/cases/elementwise/add_fan_in_4.dsl new file mode 100644 index 0000000..0e4449e --- /dev/null +++ b/tests/topic06/cases/elementwise/add_fan_in_4.dsl @@ -0,0 +1,5 @@ +# Fan-in add over four inputs +x = add(a, b) +y = add(c, d) +result = add(x, y) +return result diff --git a/tests/topic06/cases/elementwise/add_fan_in_4.meta.json b/tests/topic06/cases/elementwise/add_fan_in_4.meta.json new file mode 100644 index 0000000..3eb1f4a --- /dev/null +++ b/tests/topic06/cases/elementwise/add_fan_in_4.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute two independent adds and then merge them with a final add.", + "expected_output_type": "return_value", + "inputs": { + "a": 1, + "b": 2, + "c": 3, + "d": 4 + }, + "expected_return": 10 +} diff --git a/tests/topic06/cases/elementwise/add_reuse.dsl b/tests/topic06/cases/elementwise/add_reuse.dsl new file mode 100644 index 0000000..4618948 --- /dev/null +++ b/tests/topic06/cases/elementwise/add_reuse.dsl @@ -0,0 +1,4 @@ +# Reuse intermediate add result +x = add(a, b) +result = add(x, x) +return result diff --git a/tests/topic06/cases/elementwise/add_reuse.meta.json b/tests/topic06/cases/elementwise/add_reuse.meta.json new file mode 100644 index 0000000..2d2c396 --- /dev/null +++ b/tests/topic06/cases/elementwise/add_reuse.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Reuse the same intermediate add result on both operands of a second add.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 10 +} diff --git a/tests/topic06/cases/elementwise/vector_add.dsl b/tests/topic06/cases/elementwise/vector_add.dsl new file mode 100644 index 0000000..36ac648 --- /dev/null +++ b/tests/topic06/cases/elementwise/vector_add.dsl @@ -0,0 +1,3 @@ +# Vector add +result = add(a, b) +return result \ No newline at end of file diff --git a/tests/topic06/cases/elementwise/vector_add.meta.json b/tests/topic06/cases/elementwise/vector_add.meta.json new file mode 100644 index 0000000..7347fff --- /dev/null +++ b/tests/topic06/cases/elementwise/vector_add.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Single add over two symbolic vector inputs.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} diff --git a/tests/topic06/cases/loop/loop_add_4.dsl b/tests/topic06/cases/loop/loop_add_4.dsl new file mode 100644 index 0000000..381f1c0 --- /dev/null +++ b/tests/topic06/cases/loop/loop_add_4.dsl @@ -0,0 +1,5 @@ +# Loop with repeated add body over 4 iterations +for i = 0, 4 +x = add(a, b) +endfor +return x diff --git a/tests/topic06/cases/loop/loop_add_4.meta.json b/tests/topic06/cases/loop/loop_add_4.meta.json new file mode 100644 index 0000000..fc31fae --- /dev/null +++ b/tests/topic06/cases/loop/loop_add_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} diff --git a/tests/topic06/cases/loop/loop_add_chain_4.dsl b/tests/topic06/cases/loop/loop_add_chain_4.dsl new file mode 100644 index 0000000..eca595b --- /dev/null +++ b/tests/topic06/cases/loop/loop_add_chain_4.dsl @@ -0,0 +1,6 @@ +# Loop with chained add inside the body +for i = 0, 4 +x = add(a, b) +y = add(x, c) +endfor +return y diff --git a/tests/topic06/cases/loop/loop_add_chain_4.meta.json b/tests/topic06/cases/loop/loop_add_chain_4.meta.json new file mode 100644 index 0000000..154d91d --- /dev/null +++ b/tests/topic06/cases/loop/loop_add_chain_4.meta.json @@ -0,0 +1,10 @@ +{ + "description": "Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4 + }, + "expected_return": 9 +} diff --git a/tests/topic06/cases/loop/loop_relu_add_4.dsl b/tests/topic06/cases/loop/loop_relu_add_4.dsl new file mode 100644 index 0000000..44f3080 --- /dev/null +++ b/tests/topic06/cases/loop/loop_relu_add_4.dsl @@ -0,0 +1,6 @@ +# Loop with add followed by ReLU in the body +for i = 0, 4 +x = add(input, bias) +y = relu(x) +endfor +return y diff --git a/tests/topic06/cases/loop/loop_relu_add_4.meta.json b/tests/topic06/cases/loop/loop_relu_add_4.meta.json new file mode 100644 index 0000000..8baca28 --- /dev/null +++ b/tests/topic06/cases/loop/loop_relu_add_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "input": -4, + "bias": 6 + }, + "expected_return": 2 +} diff --git a/tests/topic06/cases/reduction/dot_4.dsl b/tests/topic06/cases/reduction/dot_4.dsl new file mode 100644 index 0000000..2118c72 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_4.dsl @@ -0,0 +1,3 @@ +# Dot product of two 4-element vectors +result = dot(a, b, len:4) +return result \ No newline at end of file diff --git a/tests/topic06/cases/reduction/dot_4.meta.json b/tests/topic06/cases/reduction/dot_4.meta.json new file mode 100644 index 0000000..748f260 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute the dot product of two symbolic vectors of length 4.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 2, 3, 4], + "b": [5, 6, 7, 8] + }, + "expected_return": 70 +} diff --git a/tests/topic06/cases/reduction/dot_8.dsl b/tests/topic06/cases/reduction/dot_8.dsl new file mode 100644 index 0000000..e1b71fa --- /dev/null +++ b/tests/topic06/cases/reduction/dot_8.dsl @@ -0,0 +1,3 @@ +# Dot product of two 8-element vectors +result = dot(a, b, len:8) +return result diff --git a/tests/topic06/cases/reduction/dot_8.meta.json b/tests/topic06/cases/reduction/dot_8.meta.json new file mode 100644 index 0000000..d3172a2 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_8.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute the dot product of two symbolic vectors of length 8.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 2, 3, 4, 5, 6, 7, 8], + "b": [1, 1, 1, 1, 1, 1, 1, 1] + }, + "expected_return": 36 +} diff --git a/tests/topic06/cases/reduction/dot_relu_4.dsl b/tests/topic06/cases/reduction/dot_relu_4.dsl new file mode 100644 index 0000000..279fc19 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_relu_4.dsl @@ -0,0 +1,4 @@ +# Dot product followed by ReLU +x = dot(a, b, len:4) +result = relu(x) +return result diff --git a/tests/topic06/cases/reduction/dot_relu_4.meta.json b/tests/topic06/cases/reduction/dot_relu_4.meta.json new file mode 100644 index 0000000..24910bc --- /dev/null +++ b/tests/topic06/cases/reduction/dot_relu_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a length-4 dot product and pass it through ReLU.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, -2, 3, -4], + "b": [2, 3, 4, 5] + }, + "expected_return": 0 +} diff --git a/tests/topic06/cases/reduction/dot_relu_8.dsl b/tests/topic06/cases/reduction/dot_relu_8.dsl new file mode 100644 index 0000000..d2b906d --- /dev/null +++ b/tests/topic06/cases/reduction/dot_relu_8.dsl @@ -0,0 +1,4 @@ +# Dot product of length 8 followed by ReLU +x = dot(a, b, len:8) +result = relu(x) +return result diff --git a/tests/topic06/cases/reduction/dot_relu_8.meta.json b/tests/topic06/cases/reduction/dot_relu_8.meta.json new file mode 100644 index 0000000..776fab5 --- /dev/null +++ b/tests/topic06/cases/reduction/dot_relu_8.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a length-8 dot product and pass it through ReLU.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 0, 1, 0, 1, 0, 1, 0], + "b": [2, 2, 2, 2, 2, 2, 2, 2] + }, + "expected_return": 8 +} diff --git a/tests/topic06/cases/tensor/matmul_2x2.dsl b/tests/topic06/cases/tensor/matmul_2x2.dsl new file mode 100644 index 0000000..f0a80d9 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_2x2.dsl @@ -0,0 +1,3 @@ +# Matrix multiplication: 2x2 * 2x2 +result = matmul(A, B, m:2, n:2, k:2) +return result \ No newline at end of file diff --git a/tests/topic06/cases/tensor/matmul_2x2.meta.json b/tests/topic06/cases/tensor/matmul_2x2.meta.json new file mode 100644 index 0000000..31a0848 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_2x2.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute a symbolic 2x2 by 2x2 matrix multiplication.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]] + }, + "expected_return": [[19, 22], [43, 50]] +} diff --git a/tests/topic06/cases/tensor/matmul_4x4.dsl b/tests/topic06/cases/tensor/matmul_4x4.dsl new file mode 100644 index 0000000..ab5a865 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_4x4.dsl @@ -0,0 +1,3 @@ +# Matrix multiplication: 4x4 * 4x4 +result = matmul(A, B, m:4, n:4, k:4) +return result diff --git a/tests/topic06/cases/tensor/matmul_4x4.meta.json b/tests/topic06/cases/tensor/matmul_4x4.meta.json new file mode 100644 index 0000000..2adb159 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_4x4.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute a symbolic 4x4 by 4x4 matrix multiplication.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [4, 4], + "inputs": { + "A": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], + "B": [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] + }, + "expected_return": [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +} diff --git a/tests/topic06/cases/tensor/matmul_add_2x2.dsl b/tests/topic06/cases/tensor/matmul_add_2x2.dsl new file mode 100644 index 0000000..d41a2cb --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_add_2x2.dsl @@ -0,0 +1,4 @@ +# Matrix multiplication followed by add +x = matmul(A, B, m:2, n:2, k:2) +result = add(x, bias) +return result diff --git a/tests/topic06/cases/tensor/matmul_add_2x2.meta.json b/tests/topic06/cases/tensor/matmul_add_2x2.meta.json new file mode 100644 index 0000000..b336c80 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_add_2x2.meta.json @@ -0,0 +1,12 @@ +{ + "description": "Compute a 2x2 matmul and then add a symbolic bias term.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]], + "bias": [[1, 1], [1, 1]] + }, + "expected_return": [[20, 23], [44, 51]] +} diff --git a/tests/topic06/cases/tensor/matmul_relu_2x2.dsl b/tests/topic06/cases/tensor/matmul_relu_2x2.dsl new file mode 100644 index 0000000..28c7c61 --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_relu_2x2.dsl @@ -0,0 +1,4 @@ +# Matrix multiplication followed by ReLU +x = matmul(A, B, m:2, n:2, k:2) +result = relu(x) +return result diff --git a/tests/topic06/cases/tensor/matmul_relu_2x2.meta.json b/tests/topic06/cases/tensor/matmul_relu_2x2.meta.json new file mode 100644 index 0000000..e2efb5e --- /dev/null +++ b/tests/topic06/cases/tensor/matmul_relu_2x2.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute a 2x2 matmul and then apply ReLU to its result.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[-1, 2], [-3, 4]], + "B": [[1, 0], [0, 1]] + }, + "expected_return": [[0, 2], [0, 4]] +} diff --git a/tests/topic06/test_integration.py b/tests/topic06/test_integration.py new file mode 100644 index 0000000..417e64b --- /dev/null +++ b/tests/topic06/test_integration.py @@ -0,0 +1,83 @@ +"""Integration contracts for the Topic 06 benchmark suite.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +CASE_DIR = ROOT / "tests" / "topic06" / "cases" +RUNNER_PATH = ROOT / "scripts" / "run_topic06_benchmarks.py" + + +def _load_runner(): + spec = importlib.util.spec_from_file_location("topic06_runner", RUNNER_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_topic06_case_manifest_is_complete(): + dsl_files = sorted(CASE_DIR.rglob("*.dsl")) + meta_files = sorted(CASE_DIR.rglob("*.meta.json")) + + assert len(dsl_files) == 23 + assert len(meta_files) == 23 + assert {path.with_suffix("") for path in dsl_files} == { + path.with_suffix("").with_suffix("") for path in meta_files + } + + +def test_compiler_emits_register_map_for_topic06_case(tmp_path): + from scratchv.compiler import CompilerConfig, CompilerDriver + + driver = CompilerDriver(CompilerConfig( + backend="riscv", + optimize_level="all", + reg_alloc="greedy", + )) + source = CASE_DIR / "activation" / "relu_add.dsl" + result = driver.compile(str(source), str(tmp_path / "relu_add.s")) + + assert result.success + assert result.stats["register_map"]["input"] == "t0" + assert result.stats["register_map"]["bias"] == "t1" + + +def test_runner_loads_compiler_register_map(tmp_path): + runner = _load_runner() + register_map_file = tmp_path / "registers.json" + register_map_file.write_text( + json.dumps({ + "schema_version": 1, + "register_map": {"input": "t0", "bias": "t1"}, + }), + encoding="utf-8", + ) + + initial = runner.load_initial_registers( + register_map_file, + {"input": -3, "bias": 7, "tensor": [1, 2]}, + ) + + assert initial == {"t0": -3, "t1": 7} + + +def test_topic06_report_paths_are_portable(): + runner = _load_runner() + case = runner.TEST_DIR / "activation" / "relu_only.dsl" + + assert runner._report_path(case) == "tests/topic06/cases/activation/relu_only.dsl" + + +def test_runner_uses_project_layout(): + runner = _load_runner() + + assert runner.PROJECT_ROOT == ROOT + assert runner.TEST_DIR == ROOT / "tests" / "topic06" / "cases" + assert runner.BUILD_DIR == ROOT / "build" / "topic06" + assert runner.REPORT_DIR == ROOT / "benchmark_reports" / "topic06" + assert runner.BASELINE_FILE == ROOT / "benchmarks" / "topic06" / "baseline.json"