diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..e59be4ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,25 @@ +name: Bug report +description: The websitebench command, the docs or CI does something wrong +labels: [bug] +body: + - type: textarea + id: what + attributes: + label: What happened and what you expected + validations: + required: true + - type: textarea + id: output + attributes: + label: Command and output + render: shell + - type: input + id: version + attributes: + label: websitebench --version + placeholder: websitebench 0.0.1 (tasks v1, Harbor 0.25.0) + - type: input + id: platform + attributes: + label: Operating system and Python version + placeholder: macOS 15.5, Python 3.12.11 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..c7947ded --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Project page and leaderboard + url: https://website-bench.com/ + about: Published results and the paper + - name: Harbor framework + url: https://github.com/harbor-framework/harbor/issues + about: Problems with harbor run itself, agents or the Harbor Hub diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 00000000..a1ea4235 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,21 @@ +name: Feature request +description: Propose a change to the package, the docs or the benchmark +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: Problem + description: What you are trying to do and what gets in the way. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposal + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives diff --git a/.github/ISSUE_TEMPLATE/task-problem.yml b/.github/ISSUE_TEMPLATE/task-problem.yml new file mode 100644 index 00000000..2c2069dc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/task-problem.yml @@ -0,0 +1,50 @@ +name: Task or verifier problem +description: A task environment fails to build, the oracle scores below 1.0, or a case grades a candidate wrongly +labels: [tasks] +body: + - type: input + id: task + attributes: + label: Task + placeholder: medium + validations: + required: true + - type: dropdown + id: kind + attributes: + label: Problem + options: + - The environment or verifier image does not build + - The oracle reward is below 1.0 + - A case fails a candidate that matches the reference + - The instruction is unclear or contradicts the verifier + - The trial is not scored (no reward.txt) + - Something else + validations: + required: true + - type: textarea + id: details + attributes: + label: What happened + description: Include case IDs from verifier/case-results.json when a case is involved. + validations: + required: true + - type: textarea + id: eval + attributes: + label: verifier/eval.json + render: json + - type: input + id: run + attributes: + label: Command + placeholder: harbor run --task websitebench/medium@v1 --agent oracle + - type: input + id: host + attributes: + label: Host + description: Operating system, kernel version, vCPUs, memory and Docker version. + placeholder: Ubuntu 24.04, kernel 6.8, 8 vCPUs, 16 GiB, Docker 28.1 + - type: markdown + attributes: + value: A way for a candidate or agent to raise its reward without rebuilding the site, or to escape the sandbox, belongs in a private report as described in SECURITY.md.