Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/pypi.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ harbor run --task websitebench/medium@v1 --agent codex --model '<model>' \
--allow-environment-host registry.npmjs.org
```

Add `--source hf` to download from [Hugging Face](https://huggingface.co/datasets/TIGER-Lab/WebsiteBench) instead of [Harbor Hub](https://hub.harborframework.com/datasets/websitebench/WebsiteBench). Omit `--task` to download all 70 tasks. Use `--force` to replace existing task directories.
Add `--source hf` to download from [Hugging Face](https://huggingface.co/datasets/TIGER-Lab/WebsiteBench) instead of [Harbor Hub](https://hub.harborframework.com/datasets/websitebench/WebsiteBench). Omit `--task` to download all 70 tasks. Use `--force` to replace existing task directories, or `--skip-existing` to keep the ones that match the release and download the rest.

With [uv](https://docs.astral.sh/uv/), `uv tool install websitebench --with-executables-from harbor` puts both the `websitebench` and `harbor` commands on your PATH.

Expand Down
2 changes: 1 addition & 1 deletion docs/running.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ uv run python scripts/download_tasks.py --task medium
uv run python scripts/download_tasks.py --source hf --task medium --output local-tasks-hf # Hugging Face
```

Omit `--task` to download all 70 (9.7 GB), add `--force` to replace a task, and set `HF_TOKEN` for a higher Hugging Face rate limit.
Omit `--task` to download all 70 (9.7 GB), add `--force` to replace a task, and set `HF_TOKEN` for a higher Hugging Face rate limit. After an interrupted download, `--skip-existing` keeps the tasks that already match the release and fetches the rest.

Run a local copy with `--path`:

Expand Down
30 changes: 28 additions & 2 deletions src/websitebench/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,14 @@
from pathlib import Path

from . import __version__
from .download import fetch_hf, fetch_hub, find_command, install, load_index
from .download import (
fetch_hf,
fetch_hub,
find_command,
install,
load_index,
verify_task,
)

DESCRIPTION = """\
Download WebsiteBench tasks from Harbor Hub or Hugging Face. Every task is checked
Expand Down Expand Up @@ -60,9 +67,16 @@ def build_parser(index: dict, default_output: Path) -> argparse.ArgumentParser:
download.add_argument(
"--revision", help="Hugging Face revision; defaults to the pinned release"
)
download.add_argument(
existing = download.add_mutually_exclusive_group()
existing.add_argument(
"--force", action="store_true", help="replace existing task directories"
)
existing.add_argument(
"--skip-existing",
action="store_true",
help="keep existing task directories that match the release; "
"fail on ones that differ",
)
download.set_defaults(parser=download)

commands.add_parser(
Expand Down Expand Up @@ -101,6 +115,18 @@ def run_download(args: argparse.Namespace, index: dict, harbor: str | None) -> i
if unknown:
args.parser.error(f"not in the release: {', '.join(unknown)}")

if args.skip_existing:
present = [task for task in tasks if (args.output / task).exists()]
for task in present:
try:
verify_task(args.output / task, index["tasks"][task])
except (OSError, ValueError) as exc:
print(f"error: {exc}; pass --force to replace it", file=sys.stderr)
return 1
print(f"SKIP {task} (matches the release)")
tasks = [task for task in tasks if task not in present]
if not tasks:
return 0
staging_parent = args.output.parent if args.output.parent.exists() else None
with tempfile.TemporaryDirectory(
prefix=".websitebench-download-", dir=staging_parent
Expand Down
43 changes: 43 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,49 @@ def test_downloaded_task_is_checked_and_installed(self):
self.assertEqual((task / "task.toml").stat().st_mode & 0o777, 0o644)
self.assertEqual([p.name for p in self.root.iterdir()], ["local-tasks"])

def test_skip_existing_keeps_a_matching_task(self):
write_task(self.output / "sample")
with mock.patch.object(cli, "fetch_hub", side_effect=fake_hub) as fetch:
code, out, err = run_cli(
"download", "--skip-existing", "--output", str(self.output)
)
self.assertEqual(code, 0, err)
self.assertEqual(out, "SKIP sample (matches the release)\n")
fetch.assert_not_called()

def test_skip_existing_fails_on_a_task_that_differs(self):
write_task(self.output / "sample")
(self.output / "sample/instruction.md").write_text("Edited.\n")
with mock.patch.object(cli, "fetch_hub", side_effect=fake_hub) as fetch:
code, _, err = run_cli(
"download", "--skip-existing", "--output", str(self.output)
)
self.assertEqual(code, 1)
self.assertIn("sample: content hash", err)
self.assertIn("pass --force to replace it", err)
fetch.assert_not_called()

def test_skip_existing_downloads_the_missing_tasks(self):
entry = sample_index()["tasks"]["sample"]
index = {**INDEX, "tasks": {"alpha": entry, "beta": entry}}
write_task(self.output / "alpha")
with (
mock.patch.object(cli, "load_index", return_value=index),
mock.patch.object(cli, "fetch_hub", side_effect=fake_hub) as fetch,
):
code, out, err = run_cli(
"download", "--skip-existing", "--output", str(self.output)
)
self.assertEqual(code, 0, err)
self.assertEqual(fetch.call_args.args[1], ["beta"])
self.assertIn("SKIP alpha", out)
self.assertIn("OK beta", out)

def test_force_and_skip_existing_are_exclusive(self):
code, _, err = run_cli("download", "--force", "--skip-existing")
self.assertEqual(code, 2)
self.assertIn("not allowed with argument", err)

def test_failed_download_installs_nothing(self):
failure = subprocess.CalledProcessError(1, ["harbor", "download"])
with mock.patch.object(cli, "fetch_hub", side_effect=failure):
Expand Down
Loading