From 917eef1713f85767f1429ef5e29cce211818b6ff Mon Sep 17 00:00:00 2001 From: reacher-z <68239571+reacher-z@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:16:45 -0700 Subject: [PATCH] Add issue forms for task problems, bugs and feature requests A task problem report needs the task name, the kind of failure, the eval.json of the trial and the host's kernel and resources before anyone can reproduce it. The task form asks for those; the bug and feature forms cover the websitebench package, docs and CI. config.yml links the project page and Harbor's own tracker. --- .github/ISSUE_TEMPLATE/bug-report.yml | 25 +++++++++++ .github/ISSUE_TEMPLATE/config.yml | 8 ++++ .github/ISSUE_TEMPLATE/feature-request.yml | 21 +++++++++ .github/ISSUE_TEMPLATE/task-problem.yml | 50 ++++++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml create mode 100644 .github/ISSUE_TEMPLATE/task-problem.yml diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..e59be4ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,25 @@ +name: Bug report +description: The websitebench command, the docs or CI does something wrong +labels: [bug] +body: + - type: textarea + id: what + attributes: + label: What happened and what you expected + validations: + required: true + - type: textarea + id: output + attributes: + label: Command and output + render: shell + - type: input + id: version + attributes: + label: websitebench --version + placeholder: websitebench 0.0.1 (tasks v1, Harbor 0.25.0) + - type: input + id: platform + attributes: + label: Operating system and Python version + placeholder: macOS 15.5, Python 3.12.11 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..c7947ded --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Project page and leaderboard + url: https://website-bench.com/ + about: Published results and the paper + - name: Harbor framework + url: https://github.com/harbor-framework/harbor/issues + about: Problems with harbor run itself, agents or the Harbor Hub diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 00000000..a1ea4235 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,21 @@ +name: Feature request +description: Propose a change to the package, the docs or the benchmark +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: Problem + description: What you are trying to do and what gets in the way. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposal + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives diff --git a/.github/ISSUE_TEMPLATE/task-problem.yml b/.github/ISSUE_TEMPLATE/task-problem.yml new file mode 100644 index 00000000..2c2069dc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/task-problem.yml @@ -0,0 +1,50 @@ +name: Task or verifier problem +description: A task environment fails to build, the oracle scores below 1.0, or a case grades a candidate wrongly +labels: [tasks] +body: + - type: input + id: task + attributes: + label: Task + placeholder: medium + validations: + required: true + - type: dropdown + id: kind + attributes: + label: Problem + options: + - The environment or verifier image does not build + - The oracle reward is below 1.0 + - A case fails a candidate that matches the reference + - The instruction is unclear or contradicts the verifier + - The trial is not scored (no reward.txt) + - Something else + validations: + required: true + - type: textarea + id: details + attributes: + label: What happened + description: Include case IDs from verifier/case-results.json when a case is involved. + validations: + required: true + - type: textarea + id: eval + attributes: + label: verifier/eval.json + render: json + - type: input + id: run + attributes: + label: Command + placeholder: harbor run --task websitebench/medium@v1 --agent oracle + - type: input + id: host + attributes: + label: Host + description: Operating system, kernel version, vCPUs, memory and Docker version. + placeholder: Ubuntu 24.04, kernel 6.8, 8 vCPUs, 16 GiB, Docker 28.1 + - type: markdown + attributes: + value: A way for a candidate or agent to raise its reward without rebuilding the site, or to escape the sandbox, belongs in a private report as described in SECURITY.md.