Merge pull request #182 from interscript/sadeedbench-hf-fetch

feat(sadeedbench): --data accepts the HF dataset id

Author: ronaldtse

pyproject.toml

@@ -41,6 +41,7 @@ sadeed = [
   "prettytable>=3.9",
   "pandas>=2.0",
   "pyarrow>=14.0",
+  "huggingface_hub>=0.23",
 ]
 dev = [
   "pytest>=8.0",

src/sadeedbench/cli.py

@@ -37,20 +37,33 @@ def _read_preds(path: Path, key: str) -> list[str]:
     return preds
 
 
-def _load_gold(path: Path) -> list[str]:
+def _load_gold(data: str) -> list[str]:
+    """Gold outputs from a local parquet or an HF dataset id.
+
+    A path (existing file) is read directly. Anything else is treated
+    as a Hugging Face dataset id (default split file train.parquet) and
+    resolved through huggingface_hub.snapshot_download with the local
+    cache."""
     import pandas as pd
 
-    table = pd.read_parquet(path)
-    return table["output"].tolist()
+    p = Path(data)
+    if p.exists():
+        return pd.read_parquet(p)["output"].tolist()
+    import huggingface_hub
+
+    snapshot = Path(
+        huggingface_hub.snapshot_download(data, repo_type="dataset")
+    )
+    return pd.read_parquet(snapshot / "train.parquet")["output"].tolist()
 
 
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(prog="interscript-sadeed-eval")
     sub = parser.add_subparsers(dest="cmd", required=True)
     score = sub.add_parser("score", help="score a predictions file")
     score.add_argument("--preds", type=Path, required=True)
-    score.add_argument("--data", type=Path, required=True,
-                       help="SadeedDiac-25 parquet (input/output columns)")
+    score.add_argument("--data", required=True,
+                       help="parquet path or HF dataset id (Misraj/SadeedDiac-25)")
     score.add_argument("--key", default="student",
                        help="JSONL row key carrying the prediction")
     score.add_argument("--vs", type=Path, help="reference predictions file")

tests/test_sadeedbench_cli.py

@@ -20,8 +20,8 @@
 GT = ["قَوْلُهُ فَحُكْمُهَا", "مُكْتَبَّةٌ جَمِيلَةٌ"]
 
 
-def _parquet(tmp_path: Path) -> Path:
-    p = tmp_path / "bench.parquet"
+def _parquet(tmp_path: Path, name: str = "bench.parquet") -> Path:
+    p = tmp_path / name
     pd.DataFrame({"input": ["قوله فحكمها", "مكتبة جميلة"], "output": GT}).to_parquet(p)
     return p
 
@@ -64,3 +64,24 @@ def test_bootstrap_vs_reference(tmp_path: Path, capsys) -> None:
     out = json.loads(capsys.readouterr().out)
     assert rc == 0
     assert out["vs"]["delta"] < 0  # candidate (perfect) better than stripped reference
+
+
+def test_data_accepts_hf_dataset_id(monkeypatch, tmp_path, capsys) -> None:
+    # --data may name the HF dataset instead of a local parquet; the
+    # loader resolves it through huggingface_hub with a local cache
+    import sadeedbench.cli as cli
+
+    cached = _parquet(tmp_path, name="train.parquet")
+    calls = {}
+
+    def fake_snapshot(repo_id, repo_type):
+        calls["repo_id"] = repo_id
+        return str(cached.parent)
+
+    monkeypatch.setattr("huggingface_hub.snapshot_download", fake_snapshot, raising=False)
+    rc = cli.main(["score", "--preds", str(_preds(tmp_path, "p.jsonl", [
+        {"idx": i, "student": g} for i, g in enumerate(GT)])), "--data", "Misraj/SadeedDiac-25"])
+    assert rc == 0
+    assert calls["repo_id"] == "Misraj/SadeedDiac-25"
+    out = json.loads(capsys.readouterr().out)
+    assert out["der_ce"] == 0.0