From afb472b25d66ab165b42075c18f7fb93af930883 Mon Sep 17 00:00:00 2001 From: Aivaras Navardauskas Date: Fri, 17 Jul 2026 19:04:48 +0100 Subject: [PATCH] Harden verify/decontaminate, add run/cli/publish tests, cut v0.2.0 Fix correctness bugs found while auditing the SDK-facing commands: - decontaminate: a benchmark prompt shorter than the n-gram size that appeared verbatim inside a longer training row was silently not flagged. Index short benchmark texts at their own length and match same-size windows so embedded contamination is caught. - verify --kind math: whole-number answers now require exact equality; the relative tolerance accepted off-by-one errors once the answer magnitude reached ~1e6. - verify --kind code: read the pass tally from a unique sentinel and the last match so a candidate can't spoof its own result; drop empty test entries instead of counting them as passes. - decontaminate: an n-gram size below 1 now fails instead of flagging and deleting every row. Security: verify --kind code runs dataset code in a subprocess, which is not a sandbox. Correct the misleading docstring and print a warning before running so it is not treated as safe on untrusted data. Tests: add coverage for run.py, cli.py, and publish.py via the client= injection hooks (no network or real SDK), plus a regression test for each fix above. Suite goes from 35 to 80 passing, ruff clean. Cut v0.2.0: move doctor/suggest out of "in progress" (both are shipped and tested), bump the package version, fix the [notebooks] -> [all] extra, refresh the MAP command table, and correct the quickstart Python version and pip name. Remove the no-op _tri() wrapper. --- CHANGELOG.md | 41 +++++- MAP.md | 9 +- README.md | 4 +- adaption_kit/__init__.py | 2 +- adaption_kit/cli.py | 25 ++-- adaption_kit/decontaminate.py | 34 ++++- adaption_kit/verify.py | 20 ++- guides/quickstart.md | 4 +- pyproject.toml | 2 +- tests/test_cli.py | 147 +++++++++++++++++++ tests/test_decontaminate.py | 24 ++++ tests/test_publish.py | 117 +++++++++++++++ tests/test_run.py | 260 ++++++++++++++++++++++++++++++++++ tests/test_verify.py | 24 ++++ 14 files changed, 677 insertions(+), 36 deletions(-) create mode 100644 tests/test_cli.py create mode 100644 tests/test_publish.py create mode 100644 tests/test_run.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 71db3dc..c547aa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ endorsed by Adaption Labs. ## [Unreleased] +## [0.2.0] - 2026-07-17 + ### Added - `verify` command (`adaption-kit verify`) to prove rows are correct before you @@ -26,9 +28,39 @@ endorsed by Adaption Labs. the cleaned rows. - Optional `verify` extra (`pip install -e ".[verify]"`) that adds sympy for the symbolic math equivalence check. -- `doctor` command to check your environment and configuration (in progress). -- `suggest` command to recommend recipes and brand controls for your domain - (in progress). +- `doctor` command (`adaption-kit doctor`) for an offline environment healthcheck + of Python, the SDK, your env vars, and the host. +- `suggest` command (`adaption-kit suggest`) that reads your file and recommends + the column mapping to use. + +### Fixed + +- `decontaminate`: a benchmark prompt shorter than the n-gram size that appeared + verbatim *inside* a longer training row was silently not flagged. Short + benchmark texts are now indexed at their own length and matched against + same-size windows, so embedded contamination is caught. +- `verify --kind math`: whole-number answers now require exact equality. The old + relative tolerance accepted off-by-one errors once the answer magnitude reached + ~1e6. +- `verify --kind code`: the pass/fail tally is read from a unique sentinel and the + last match, so a candidate that prints a `PASSED n` line can no longer spoof its + own result. Empty test entries no longer count as passes. +- `decontaminate`: an n-gram size below 1 now fails loudly instead of flagging and + deleting every row. + +### Security + +- `verify --kind code` executes dataset code in a subprocess, which is NOT a + security sandbox — it runs with your full user privileges. The misleading "never + your machine" wording was corrected, and the CLI now prints a warning before it + runs. Only verify code from datasets you trust. + +### Tests + +- Test coverage for the SDK-backed helpers (`run.py`), the CLI router (`cli.py`), + and the publish helper (`publish.py`), using the `client=` injection hooks so no + network or real SDK is required. +- Regression tests for each of the correctness fixes above. ## [0.1.0] - 2026-06-07 @@ -50,5 +82,6 @@ endorsed by Adaption Labs. - Templates for dataset schemas, dataset and model cards, a cover, and Kaggle metadata. -[Unreleased]: https://github.com/A1VARA5/adaption-devkit/compare/v0.1.0...HEAD +[Unreleased]: https://github.com/A1VARA5/adaption-devkit/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/A1VARA5/adaption-devkit/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/A1VARA5/adaption-devkit/releases/tag/v0.1.0 diff --git a/MAP.md b/MAP.md index f00b0f0..787ba94 100644 --- a/MAP.md +++ b/MAP.md @@ -24,7 +24,7 @@ tooling. | `cookbook/` | Runnable notebooks that walk the full lifecycle. | You learn best by running real code. | | `templates/` | Dataset schemas, dataset and model cards, a cover, Kaggle metadata. | You are preparing a release. | | `graphics/` | The diagrams embedded in the README, as Mermaid in Markdown. | You want the source of a diagram. | -| `pyproject.toml` | Package metadata and optional extras (`sdk`, `notebooks`). | You are installing or packaging. | +| `pyproject.toml` | Package metadata and optional extras (`sdk`, `verify`, `all`, ...). | You are installing or packaging. | | `LICENSE` | Apache-2.0. | You need the license text. | ### The CLI commands @@ -33,14 +33,17 @@ tooling. | Command | What it does | |---------|--------------| +| `doctor` | Offline healthcheck of Python, the SDK, your env vars, and the host. | +| `suggest` | Read your file and recommend the column mapping to use. | | `lint` | Preflight a dataset before a run. Catches duplicate prompts, encoding issues, and empty anchors before you spend credits. | +| `verify` | Prove math answers and code rows are correct before you adapt them. | +| `decontaminate` | Drop training rows that overlap a benchmark test set by an n-gram. | +| `convert` | Convert a dataset between CSV, JSONL, and Parquet (BOM-safe). | | `estimate` | Quote credits and time for a run without starting one. | | `run` | Start an adaptation run, estimate first, optionally wait and print `improvement_percent`. | | `publish` | Publish helper that packages a release for Hugging Face and Kaggle, because the platform publish endpoint returns 501. | | `card` | Generate a dataset card, a model card, or Kaggle metadata. | | `cover` | Render a cover image for your release. | -| `doctor` | Coming soon: check your environment and configuration. | -| `suggest` | Coming soon: suggest recipes and controls for your domain. | ### The guides diff --git a/README.md b/README.md index feae1d1..79774f0 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,8 @@ pip install -e ".[sdk]" # the symbolic math check used by 'adaption-kit verify --kind math' pip install -e ".[verify]" -# everything for the cookbook notebooks -pip install -e ".[notebooks]" +# everything (SDK, HF, Kaggle, Playwright cover, Parquet, sympy) for the cookbook +pip install -e ".[all]" ``` If an extra is not installed, the matching command tells you what to add. The diff --git a/adaption_kit/__init__.py b/adaption_kit/__init__.py index 43dbc3c..46fee6a 100644 --- a/adaption_kit/__init__.py +++ b/adaption_kit/__init__.py @@ -9,7 +9,7 @@ from __future__ import annotations -__version__ = "0.1.0" +__version__ = "0.2.0" __author__ = "Aivaras Navardauskas (MANIFESTA)" __license__ = "Apache-2.0" diff --git a/adaption_kit/cli.py b/adaption_kit/cli.py index 13fd35e..587d83e 100644 --- a/adaption_kit/cli.py +++ b/adaption_kit/cli.py @@ -70,6 +70,14 @@ def _cmd_lint(args: argparse.Namespace) -> int: def _cmd_verify(args: argparse.Namespace) -> int: from .verify import FAIL, verify_dataset + if args.kind == "code": + print( + "warning: 'verify --kind code' runs the dataset's code in a subprocess " + "with your full user privileges. It contains crashes and hangs but is " + "NOT a security sandbox. Only run it on data you trust.", + file=sys.stderr, + ) + report = verify_dataset( args.path, kind=args.kind, @@ -129,9 +137,9 @@ def _cmd_estimate(args: argparse.Namespace) -> int: completion=args.completion, context=_split_csv(args.context), chat=args.chat, - deduplication=_tri(args.deduplication), - prompt_rephrase=_tri(args.prompt_rephrase), - reasoning_traces=_tri(args.reasoning_traces), + deduplication=args.deduplication, + prompt_rephrase=args.prompt_rephrase, + reasoning_traces=args.reasoning_traces, ) except SdkNotInstalled as exc: print(str(exc), file=sys.stderr) @@ -157,9 +165,9 @@ def _cmd_run(args: argparse.Namespace) -> int: completion=args.completion, context=_split_csv(args.context), chat=args.chat, - deduplication=_tri(args.deduplication), - prompt_rephrase=_tri(args.prompt_rephrase), - reasoning_traces=_tri(args.reasoning_traces), + deduplication=args.deduplication, + prompt_rephrase=args.prompt_rephrase, + reasoning_traces=args.reasoning_traces, idempotency_key=args.idempotency_key, ) if args.pilot: @@ -290,11 +298,6 @@ def _cmd_cover(args: argparse.Namespace) -> int: # --------------------------------------------------------------------------- -def _tri(value: Optional[bool]) -> Optional[bool]: - """Pass through tri-state recipe flags (None = backend default).""" - return value - - def _write_or_print(text: str, out: Optional[Path], default_name: str) -> None: if out is None: sys.stdout.write(text) diff --git a/adaption_kit/decontaminate.py b/adaption_kit/decontaminate.py index 5427cdf..d0ed280 100644 --- a/adaption_kit/decontaminate.py +++ b/adaption_kit/decontaminate.py @@ -48,20 +48,41 @@ def ngrams(text: str, n: int = DEFAULT_N) -> set: class Decontaminator: - """Holds the benchmark n-gram set; flags any text that shares one.""" + """Holds the benchmark n-gram set; flags any text that shares one. + + Benchmark texts shorter than ``n`` tokens are indexed at their own length + (``min(n, len)``) rather than as one whole-string shingle, so a short + benchmark question embedded verbatim in a *longer* training row is still + caught. Matching compares training windows of the same sizes present in the + benchmark. + """ def __init__(self, n: int = DEFAULT_N) -> None: self.n = n self.bench: set = set() + self._sizes: set = set() def add_text(self, text: str) -> None: - self.bench |= ngrams(text, self.n) + toks = _normalize(text).split() + if not toks: + return + k = min(self.n, len(toks)) + self._sizes.add(k) + self.bench |= {" ".join(toks[i : i + k]) for i in range(len(toks) - k + 1)} def is_contaminated(self, text: str) -> bool: - g = ngrams(text, self.n) - if not g: + if not self.bench: return False - return not g.isdisjoint(self.bench) + toks = _normalize(text).split() + if not toks: + return False + for k in self._sizes: + if len(toks) < k: + continue + windows = {" ".join(toks[i : i + k]) for i in range(len(toks) - k + 1)} + if not windows.isdisjoint(self.bench): + return True + return False @dataclass @@ -155,6 +176,9 @@ def decontaminate( """ p = Path(path) report = DecontamReport(path=str(p), n=n) + if n < 1: + report.add(FAIL, "n-gram size must be >= 1 (got " + str(n) + ")") + return report if not p.exists(): report.add(FAIL, "file does not exist") return report diff --git a/adaption_kit/verify.py b/adaption_kit/verify.py index 3cd01d7..9badb53 100644 --- a/adaption_kit/verify.py +++ b/adaption_kit/verify.py @@ -6,8 +6,10 @@ - math: the final answer the worked solution reaches must be equivalent to a gold answer (normalized string, then numeric, then symbolic via sympy if installed). - code: the solution must run and pass the unit tests shipped with it. Each - candidate runs in a fresh, short-lived subprocess with a hard timeout, so a bad - row can only ever crash its own child, never your machine or the build. + candidate runs in a fresh, short-lived subprocess with a hard timeout, which + contains crashes and hangs -- but it is NOT a security sandbox. The row's code + runs with your full user privileges, so only verify code from datasets you + trust. The CLI prints a warning before it runs. Adapting unverified rows is the most common way beginners burn credits on multiple tries: you pay to polish data that was wrong to begin with. Filter first, then run. @@ -153,6 +155,10 @@ def answers_equivalent(gold: Optional[str], pred: Optional[str], timeout: float return True fg, fp = _try_float(ng), _try_float(npd) if fg is not None and fp is not None: + # Whole-number answers must match exactly; the relative tolerance below + # would otherwise accept off-by-one errors once |answer| >= ~1e6. + if float(fg).is_integer() and float(fp).is_integer(): + return fg == fp return abs(fg - fp) <= 1e-6 * max(1.0, abs(fg), abs(fp)) if not _HAVE_SYMPY: return False @@ -187,16 +193,16 @@ def _coerce_tests(tests: Any) -> List[str]: def code_passes(output: str, tests: Any, setup: str = "") -> bool: """True only if the solution runs and passes EVERY assert-style test.""" code = extract_code(output) - tests = _coerce_tests(tests) + tests = [t for t in _coerce_tests(tests) if t.strip()] if len(code) < 5 or not tests: return False harness = setup + "\n" + code + "\n\n_p = 0\n" for t in tests: harness += ( - "try:\n" + textwrap.indent(t.strip() or "pass", " ") + "try:\n" + textwrap.indent(t.strip(), " ") + "\n _p += 1\nexcept Exception:\n pass\n" ) - harness += "print('PASSED', _p)\n" + harness += "print('__ADK_PASSED__', _p)\n" with tempfile.TemporaryDirectory() as d: fp = os.path.join(d, "cand.py") with open(fp, "w", encoding="utf-8") as f: @@ -208,8 +214,8 @@ def code_passes(output: str, tests: Any, setup: str = "") -> bool: ) except Exception: return False - m = re.search(r"PASSED (\d+)", r.stdout or "") - return bool(m) and int(m.group(1)) == len(tests) + matches = re.findall(r"__ADK_PASSED__ (\d+)", r.stdout or "") + return bool(matches) and int(matches[-1]) == len(tests) # --------------------------------------------------------------------------- report diff --git a/guides/quickstart.md b/guides/quickstart.md index 98cc4fd..533e5c6 100644 --- a/guides/quickstart.md +++ b/guides/quickstart.md @@ -10,7 +10,7 @@ This guide takes you from nothing to your first Adaptive Data run, with a real ## 0. What you need -- Python 3.9 or newer. +- Python 3.10 or newer. - An Adaption API key. - A small data file to start with (`.csv`, `.json`, `.jsonl`, or `.parquet`). @@ -18,7 +18,7 @@ This guide takes you from nothing to your first Adaptive Data run, with a real ```bash pip install adaption # the official SDK -pip install adaption-devkit # this community toolkit (provides the adaption-kit CLI) +pip install adaption-kit # this community toolkit (provides the adaption-kit CLI) ``` If you are working from a checkout of this repo instead: diff --git a/pyproject.toml b/pyproject.toml index d4c03a5..40dca08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "adaption-kit" -version = "0.1.0" +version = "0.2.0" description = "Community, unofficial open source toolkit for starting fast with Adaption Adaptive Data and AutoScientist." readme = "adaption_kit/README.md" requires-python = ">=3.10" diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..2e7425c --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,147 @@ +"""Tests for adaption_kit.cli - the argparse router. + +The parser is exercised for every subcommand, and ``main`` is dispatched for the +offline commands (doctor, lint). The SDK-backed commands (estimate, run) are +forced down their SdkNotInstalled path by monkeypatching run._client, so no real +``adaption`` SDK and no network are needed even when the SDK happens to be +installed in the environment. +""" + +from __future__ import annotations + +import pytest + +from conftest import write_csv + +from adaption_kit import run as run_mod +from adaption_kit.cli import build_parser, main + + +# ------------------------------------------------------------------- build_parser +@pytest.mark.parametrize( + "argv, expected_command", + [ + (["doctor"], "doctor"), + (["suggest", "data.csv"], "suggest"), + (["lint", "data.csv"], "lint"), + (["verify", "data.jsonl", "--kind", "math"], "verify"), + (["decontaminate", "train.csv", "--against", "bench.csv"], "decontaminate"), + (["convert", "in.csv", "out.jsonl"], "convert"), + (["estimate", "ds_1"], "estimate"), + (["run", "ds_1"], "run"), + (["publish", "some_folder"], "publish"), + (["card", "dataset", "--title", "T"], "card"), + (["cover", "out.png"], "cover"), + ], +) +def test_build_parser_accepts_each_subcommand(argv, expected_command): + parser = build_parser() + args = parser.parse_args(argv) + assert args.command == expected_command + # Every subcommand wires a dispatch function. + assert callable(args.func) + + +def test_missing_subcommand_exits_2(): + """Subcommands are required; no command is an argparse error (exit 2).""" + with pytest.raises(SystemExit) as excinfo: + main([]) + assert excinfo.value.code == 2 + + +def test_missing_required_arg_exits_2(): + """verify without --kind is an argparse error (exit 2).""" + with pytest.raises(SystemExit) as excinfo: + main(["verify", "data.jsonl"]) + assert excinfo.value.code == 2 + + +# --------------------------------------------------------------------- dispatch +def test_main_doctor_returns_zero(capsys): + rc = main(["doctor"]) + assert rc == 0 + # The banner and the doctor summary are printed. + out = capsys.readouterr().out + assert "adaption-kit" in out + + +def test_main_lint_clean_csv_returns_zero(tmp_path, capsys): + src = write_csv( + tmp_path / "clean.csv", + ["completion"], + [ + {"completion": "Our SPF 50 serum absorbs in seconds."}, + {"completion": "This mascara lifts and separates every lash."}, + {"completion": "A matte lipstick that lasts through dinner."}, + ], + ) + + rc = main(["lint", str(src), "--completion", "completion"]) + + assert rc == 0 + out = capsys.readouterr().out + assert "PASS" in out + + +def test_main_lint_collapse_trap_returns_one(tmp_path): + """A constant prompt collapses under dedup -> preflight FAIL -> exit 1.""" + same = "Write a product description." + src = write_csv( + tmp_path / "templated.csv", + ["prompt", "product"], + [ + {"prompt": same, "product": "serum"}, + {"prompt": same, "product": "mascara"}, + {"prompt": same, "product": "lipstick"}, + ], + ) + + rc = main(["lint", str(src), "--prompt", "prompt", "--context", "product"]) + + assert rc == 1 + + +# ------------------------------------------------------ SDK-backed graceful fail +def test_estimate_without_sdk_exits_2(monkeypatch, capsys): + """When the SDK client cannot be built, estimate exits 2 (not a crash).""" + + def _boom(): + raise run_mod.SdkNotInstalled("the 'adaption' SDK is not installed") + + monkeypatch.setattr(run_mod, "_client", _boom) + + rc = main(["estimate", "ds_1", "--prompt", "instruction"]) + + assert rc == 2 + err = capsys.readouterr().err + assert "adaption" in err + + +def test_run_without_sdk_exits_2(monkeypatch): + def _boom(): + raise run_mod.SdkNotInstalled("the 'adaption' SDK is not installed") + + monkeypatch.setattr(run_mod, "_client", _boom) + + rc = main(["run", "ds_1", "--prompt", "instruction"]) + + assert rc == 2 + + +def test_estimate_bad_mapping_exits_2(monkeypatch): + """A mapping ValueError (no anchor) is caught and exits 2, with a fake client + so we never reach the real SDK.""" + from types import SimpleNamespace + + class _Datasets: + def run(self, *a, **k): # pragma: no cover - should never be called + raise AssertionError("run must not be called when mapping is invalid") + + # No prompt/completion/chat -> _build_column_mapping raises ValueError. + monkeypatch.setattr( + run_mod, "_client", lambda: SimpleNamespace(datasets=_Datasets()) + ) + + rc = main(["estimate", "ds_1"]) + + assert rc == 2 diff --git a/tests/test_decontaminate.py b/tests/test_decontaminate.py index 09e58f0..af56b65 100644 --- a/tests/test_decontaminate.py +++ b/tests/test_decontaminate.py @@ -140,3 +140,27 @@ def test_no_anchor_column_fails(tmp_path): def test_nonexistent_file_fails(tmp_path): report = decontaminate(tmp_path / "nope.jsonl", [tmp_path / "bench.jsonl"]) assert report.status == "FAIL" + + +# --------------------------------------------------------------- regressions +def test_short_benchmark_embedded_in_long_row_is_flagged(): + """A short benchmark question embedded verbatim in a LONGER training row + must be caught. Regression: short benchmark texts were stored as one + whole-string shingle and never intersected a long row's n-gram windows, so + embedded contamination slipped through silently.""" + decon = Decontaminator(n=13) + decon.add_text("what is the capital of france") # 6 tokens, shorter than n + assert decon.is_contaminated( + "trivia round two what is the capital of france answer paris for a point" + ) + assert not decon.is_contaminated( + "trivia round two what is the tallest mountain in the world answer everest" + ) + + +def test_zero_ngram_size_fails(tmp_path): + """--n 0 must fail loudly, not silently flag and delete every row.""" + train = write_jsonl(tmp_path / "train.jsonl", [{"prompt": "anything at all here"}]) + bench = write_jsonl(tmp_path / "bench.jsonl", [{"prompt": "something else entirely"}]) + report = decontaminate(train, [bench], n=0) + assert report.status == "FAIL" diff --git a/tests/test_publish.py b/tests/test_publish.py new file mode 100644 index 0000000..bc7d235 --- /dev/null +++ b/tests/test_publish.py @@ -0,0 +1,117 @@ +"""Tests for adaption_kit.publish - the manual 501 workaround. + +Only the input-validation and orchestration paths are exercised; nothing here +touches Hugging Face or Kaggle. The publish() orchestration test monkeypatches +the two upload helpers so we can assert the private/public wiring and the report +without any optional dependency or network. The direct-helper validation tests +are guarded with importorskip so they only run where the optional client is +importable (they still never reach the network - the directory check happens +before any auth or upload). +""" + +from __future__ import annotations + +import pytest + +from adaption_kit import publish as publish_mod +from adaption_kit.publish import PublishResult, publish + + +# ----------------------------------------------------------------- pure paths +def test_publish_requires_a_target(): + """Neither hf_repo nor kaggle_slug -> ValueError before any work.""" + with pytest.raises(ValueError): + publish("some_folder") + + +def test_publish_result_summary_reports_nothing_when_empty(): + result = PublishResult() + text = result.summary() + assert "nothing published" in text + + +def test_publish_result_summary_lists_targets_and_notes(): + result = PublishResult( + hf_url="https://huggingface.co/datasets/me/ds", + kaggle_ref="me/ds", + notes=["a note"], + ) + text = result.summary() + assert "https://huggingface.co/datasets/me/ds" in text + assert "me/ds" in text + assert "note: a note" in text + + +# ----------------------------------------------- orchestration (helpers faked) +def test_publish_defaults_private_and_records_both_targets(monkeypatch, tmp_path): + """publish() defaults to private: HF gets private=True, Kaggle public=False.""" + seen = {} + + def fake_hf(folder, hf_repo, private=True): + seen["hf"] = {"folder": str(folder), "hf_repo": hf_repo, "private": private} + return "https://huggingface.co/datasets/" + hf_repo + + def fake_kaggle(folder, kaggle_slug, public=False): + seen["kaggle"] = { + "folder": str(folder), + "kaggle_slug": kaggle_slug, + "public": public, + } + return kaggle_slug + + monkeypatch.setattr(publish_mod, "publish_to_hf", fake_hf) + monkeypatch.setattr(publish_mod, "publish_to_kaggle", fake_kaggle) + + result = publish( + tmp_path, + hf_repo="me/serum", + kaggle_slug="me/serum", + ) + + assert seen["hf"]["private"] is True + assert seen["kaggle"]["public"] is False + assert result.hf_url == "https://huggingface.co/datasets/me/serum" + assert result.kaggle_ref == "me/serum" + # The 501 workaround note and the Kaggle-visibility note are surfaced. + joined = " ".join(result.notes) + assert "501" in joined + assert any("private" in n for n in result.notes) + + +def test_publish_public_flag_flips_private_and_public(monkeypatch, tmp_path): + seen = {} + monkeypatch.setattr( + publish_mod, + "publish_to_hf", + lambda folder, hf_repo, private=True: seen.setdefault("private", private) + or "https://huggingface.co/datasets/" + hf_repo, + ) + monkeypatch.setattr( + publish_mod, + "publish_to_kaggle", + lambda folder, kaggle_slug, public=False: seen.setdefault("public", public) + or kaggle_slug, + ) + + publish(tmp_path, hf_repo="me/x", kaggle_slug="me/x", private=False) + + assert seen["private"] is False + assert seen["public"] is True + + +# ---------------------------------------- direct helpers: folder validation +def test_publish_to_hf_rejects_non_directory(tmp_path): + """A path that is not a directory is a ValueError before any auth/upload.""" + pytest.importorskip("huggingface_hub") + not_a_dir = tmp_path / "nope" + with pytest.raises(ValueError): + publish_mod.publish_to_hf(not_a_dir, "me/ds") + + +def test_publish_to_kaggle_requires_metadata(tmp_path): + """A real folder with no dataset-metadata.json is a RuntimeError before auth.""" + pytest.importorskip("kaggle") + folder = tmp_path / "ds" + folder.mkdir() + with pytest.raises(RuntimeError): + publish_mod.publish_to_kaggle(folder, "me/ds") diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 0000000..78efd1a --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,260 @@ +"""Tests for adaption_kit.run - the thin SDK helpers. + +Every entry point takes a ``client=`` parameter for dependency injection, so +these tests inject a fake SDK client and never import the real ``adaption`` SDK +and never touch the network. The fakes record the kwargs they are called with so +we can assert the exact wire shape (column_mapping, estimate flag, +job_specification), and return plain stub objects for status/evaluation. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from adaption_kit.run import ( + RunResult, + _build_column_mapping, + _build_recipe_spec, + estimate, + run_full, + wait_for_result, +) + + +# --------------------------------------------------------------- fake SDK client +class FakeRunDatasets: + """Records the last ``run`` call and returns a stub carrying a run_id.""" + + def __init__(self, run_return=None): + self.run_calls = [] + self._run_return = run_return or SimpleNamespace(run_id="run_123") + + def run(self, dataset_id, **kwargs): + self.run_calls.append((dataset_id, kwargs)) + return self._run_return + + +class FakeClient: + def __init__(self, datasets): + self.datasets = datasets + + +# ------------------------------------------------------- _build_column_mapping +def test_build_column_mapping_needs_an_anchor(): + """No prompt, completion, or chat -> ValueError before any credits.""" + with pytest.raises(ValueError): + _build_column_mapping() + + +def test_build_column_mapping_prompt_or_completion_anchor(): + assert _build_column_mapping(prompt="instruction") == {"prompt": "instruction"} + assert _build_column_mapping(completion="answer") == {"completion": "answer"} + + +def test_build_column_mapping_context_is_listified(): + """A context sequence is stored as a list alongside the anchor.""" + mapping = _build_column_mapping(prompt="p", context=("a", "b")) + assert mapping == {"prompt": "p", "context": ["a", "b"]} + assert isinstance(mapping["context"], list) + + +def test_build_column_mapping_chat_is_mutually_exclusive(): + """chat with any of prompt/completion/context is a ValueError.""" + assert _build_column_mapping(chat="messages") == {"chat": "messages"} + with pytest.raises(ValueError): + _build_column_mapping(prompt="p", chat="messages") + with pytest.raises(ValueError): + _build_column_mapping(completion="c", chat="messages") + with pytest.raises(ValueError): + _build_column_mapping(context=["ctx"], chat="messages") + + +# --------------------------------------------------------- _build_recipe_spec +def test_build_recipe_spec_all_none_returns_none(): + assert _build_recipe_spec() is None + assert _build_recipe_spec(None, None, None) is None + + +def test_build_recipe_spec_wraps_set_flags(): + """Only the flags that are not None appear, wrapped under 'recipes'.""" + assert _build_recipe_spec(deduplication=True) == {"recipes": {"deduplication": True}} + assert _build_recipe_spec( + deduplication=True, prompt_rephrase=False, reasoning_traces=True + ) == { + "recipes": { + "deduplication": True, + "prompt_rephrase": False, + "reasoning_traces": True, + } + } + + +# --------------------------------------------------------------------- estimate +def test_estimate_passes_estimate_true_and_mapping(): + datasets = FakeRunDatasets( + run_return=SimpleNamespace(estimated_credits_consumed=10, estimated_minutes=3) + ) + client = FakeClient(datasets) + + est = estimate("ds_1", prompt="prompt", context=["brand"], client=client) + + assert est.estimated_credits_consumed == 10 + dataset_id, kwargs = datasets.run_calls[-1] + assert dataset_id == "ds_1" + assert kwargs["estimate"] is True + assert kwargs["column_mapping"] == {"prompt": "prompt", "context": ["brand"]} + # No job_specification is ever sent on an estimate. + assert "job_specification" not in kwargs + + +def test_estimate_includes_recipe_spec_and_brand_controls_when_given(): + datasets = FakeRunDatasets() + client = FakeClient(datasets) + + estimate( + "ds_1", + completion="answer", + brand_controls={"length": "short"}, + deduplication=True, + client=client, + ) + + _, kwargs = datasets.run_calls[-1] + assert kwargs["brand_controls"] == {"length": "short"} + assert kwargs["recipe_specification"] == {"recipes": {"deduplication": True}} + + +# --------------------------------------------------------------------- run_full +def test_run_full_passes_estimate_false_and_job_spec(): + datasets = FakeRunDatasets() + client = FakeClient(datasets) + + run_obj = run_full( + "ds_2", + prompt="prompt", + max_rows=200, + idempotency_key="key-abc", + client=client, + ) + + assert run_obj.run_id == "run_123" + dataset_id, kwargs = datasets.run_calls[-1] + assert dataset_id == "ds_2" + assert kwargs["estimate"] is False + assert kwargs["column_mapping"] == {"prompt": "prompt"} + assert kwargs["job_specification"] == { + "max_rows": 200, + "idempotency_key": "key-abc", + } + + +def test_run_full_omits_job_spec_when_no_cap_or_key(): + """No max_rows and no idempotency_key -> no job_specification key at all.""" + datasets = FakeRunDatasets() + client = FakeClient(datasets) + + run_full("ds_3", completion="answer", client=client) + + _, kwargs = datasets.run_calls[-1] + assert "job_specification" not in kwargs + + +# ---------------------------------------------------------------- wait_for_result +class StatusEvalDatasets: + """Fake datasets whose status/evaluation come from scripted step lists. + + Each poll advances by one step (the last step repeats), so a run that is + already terminal while the evaluation is still running exercises the + 'both must be terminal' gate. + """ + + def __init__(self, statuses, evals): + self._statuses = statuses + self._evals = evals + self.status_calls = 0 + self.eval_calls = 0 + + def get_status(self, dataset_id): + i = min(self.status_calls, len(self._statuses) - 1) + self.status_calls += 1 + return self._statuses[i] + + def get_evaluation(self, dataset_id): + i = min(self.eval_calls, len(self._evals) - 1) + self.eval_calls += 1 + return self._evals[i] + + +def _eval(status, improvement=None, before=None, after=None): + quality = None + if improvement is not None or before is not None or after is not None: + quality = SimpleNamespace( + improvement_percent=improvement, + score_before=before, + score_after=after, + ) + return SimpleNamespace(status=status, quality=quality) + + +def test_wait_for_result_returns_improvement_when_both_terminal(): + status = SimpleNamespace(status="succeeded", run_id="run_9", error=None) + ev = _eval("succeeded", improvement=12.5, before=0.40, after=0.45) + client = FakeClient(StatusEvalDatasets([status], [ev])) + + result = wait_for_result("ds_1", timeout=30.0, poll_interval=0.0, client=client) + + assert isinstance(result, RunResult) + assert result.run_status == "succeeded" + assert result.evaluation_status == "succeeded" + assert result.run_id == "run_9" + assert result.improvement_percent == 12.5 + assert result.score_before == 0.40 + assert result.score_after == 0.45 + assert result.error is None + + +def test_wait_for_result_waits_for_eval_to_become_terminal(): + """Run is terminal immediately but evaluation only settles on the 2nd poll.""" + statuses = [SimpleNamespace(status="succeeded", run_id="run_7", error=None)] + evals = [_eval("running"), _eval("succeeded", improvement=8.0)] + datasets = StatusEvalDatasets(statuses, evals) + client = FakeClient(datasets) + + result = wait_for_result("ds_1", timeout=30.0, poll_interval=0.0, client=client) + + # It kept polling until the evaluation was terminal (>= 2 evaluation reads). + assert datasets.eval_calls >= 2 + assert result.evaluation_status == "succeeded" + assert result.improvement_percent == 8.0 + + +def test_wait_for_result_times_out_without_improvement(): + """Neither side terminal and a non-positive timeout -> a timeout result.""" + statuses = [SimpleNamespace(status="running", run_id="run_1", error=None)] + evals = [_eval("pending")] + client = FakeClient(StatusEvalDatasets(statuses, evals)) + + result = wait_for_result("ds_1", timeout=0.0, poll_interval=0.0, client=client) + + assert result.run_status == "running" + assert result.evaluation_status == "pending" + assert result.improvement_percent is None + assert result.error is not None + assert "timed out" in result.error + + +def test_wait_for_result_surfaces_run_error_message(): + err = SimpleNamespace(message="adaptation blew up") + status = SimpleNamespace(status="failed", run_id="run_2", error=err) + ev = _eval("failed") + client = FakeClient(StatusEvalDatasets([status], [ev])) + + result = wait_for_result("ds_1", timeout=30.0, poll_interval=0.0, client=client) + + assert result.run_status == "failed" + assert result.evaluation_status == "failed" + assert result.error == "adaptation blew up" + # No quality object -> improvement stays None. + assert result.improvement_percent is None diff --git a/tests/test_verify.py b/tests/test_verify.py index effaa05..89b5742 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -200,3 +200,27 @@ def test_bad_kind_fails(tmp_path): ) report = verify_dataset(src, kind="logic") assert report.status == "FAIL" + + +# --------------------------------------------------------------- regressions +def test_large_integer_off_by_one_not_equivalent(): + """Whole-number answers must match exactly. Regression: the relative + tolerance accepted off-by-one errors once |answer| >= ~1e6.""" + assert not answers_equivalent("1000000", "1000001") + assert answers_equivalent("1000000", "1000000") + + +def test_code_passes_ignores_candidate_stdout_spoof(): + """A candidate that prints a fake tally must not shadow the real result. + Regression: the first 'PASSED n' line in stdout won, so code could spoof it.""" + spoof = _fenced("def add(a, b):\n print('PASSED 1')\n return a - b") + assert not code_passes(spoof, ["assert add(1, 2) == 3"]) + + +def test_code_passes_empty_tests_are_not_counted(): + """Empty test entries must not count as passes. Regression: an empty string + was substituted with 'pass' and always incremented the pass tally.""" + code = _fenced("def add(a, b):\n return a + b") + assert not code_passes(code, ["", " "]) + # A real test alongside an empty one still verifies on the real one only. + assert code_passes(code, ["assert add(1, 2) == 3", ""])