From 53a2441178e6c2684dd52bc7996235e5fa07b9e3 Mon Sep 17 00:00:00 2001 From: reacher-z <68239571+reacher-z@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:32:42 -0700 Subject: [PATCH] verifier: read cicd_check_id when budgeting a T3 case case_wall_clock_budget looked up case["check_id"], but manifest cases name their CI/CD check cicd_check_id (as _cicd reads it). The lookup never matched, so every T3 case got the 780 s budget of a case with no declared timeout instead of one derived from its check's timeout_sec (6,124 s for a 379 s check). Current checks declare 300 to 379 s and fit under 780 s; a check declaring more than about 760 s would have been abandoned as an infrastructure failure inside its own timeout. The same one-line change is applied to all 70 copies of formal.py, which stay byte-identical. It changes the tasks' content hashes, so it ships with the next task release. --- tasks/1password-web/tests/websitebench/harbor/formal.py | 4 ++-- tasks/33/tests/websitebench/harbor/formal.py | 4 ++-- tasks/ace-hardware/tests/websitebench/harbor/formal.py | 4 ++-- tasks/airtable/tests/websitebench/harbor/formal.py | 4 ++-- .../amazon-home-services/tests/websitebench/harbor/formal.py | 4 ++-- tasks/amazon/tests/websitebench/harbor/formal.py | 4 ++-- tasks/amc-theatres/tests/websitebench/harbor/formal.py | 4 ++-- tasks/angi/tests/websitebench/harbor/formal.py | 4 ++-- tasks/apple/tests/websitebench/harbor/formal.py | 4 ++-- tasks/article/tests/websitebench/harbor/formal.py | 4 ++-- tasks/asana/tests/websitebench/harbor/formal.py | 4 ++-- tasks/aspca-pet-insurance/tests/websitebench/harbor/formal.py | 4 ++-- tasks/autoslash/tests/websitebench/harbor/formal.py | 4 ++-- tasks/autotrader/tests/websitebench/harbor/formal.py | 4 ++-- tasks/backcountry/tests/websitebench/harbor/formal.py | 4 ++-- tasks/bean-box/tests/websitebench/harbor/formal.py | 4 ++-- tasks/beeradvocate/tests/websitebench/harbor/formal.py | 4 ++-- tasks/blinkist/tests/websitebench/harbor/formal.py | 4 ++-- tasks/blueapron/tests/websitebench/harbor/formal.py | 4 ++-- tasks/bluemercury/tests/websitebench/harbor/formal.py | 4 ++-- tasks/boardgamegeek/tests/websitebench/harbor/formal.py | 4 ++-- tasks/bookshop/tests/websitebench/harbor/formal.py | 4 ++-- tasks/breakoutgames-2/tests/websitebench/harbor/formal.py | 4 ++-- tasks/brooklinen/tests/websitebench/harbor/formal.py | 4 ++-- tasks/capterra/tests/websitebench/harbor/formal.py | 4 ++-- tasks/change/tests/websitebench/harbor/formal.py | 4 ++-- tasks/columbia/tests/websitebench/harbor/formal.py | 4 ++-- tasks/crumbl-cookies/tests/websitebench/harbor/formal.py | 4 ++-- tasks/crunchyroll/tests/websitebench/harbor/formal.py | 4 ++-- tasks/curology/tests/websitebench/harbor/formal.py | 4 ++-- tasks/dooney-bourke/tests/websitebench/harbor/formal.py | 4 ++-- tasks/eatthismuch/tests/websitebench/harbor/formal.py | 4 ++-- tasks/etsy/tests/websitebench/harbor/formal.py | 4 ++-- tasks/eventbrite/tests/websitebench/harbor/formal.py | 4 ++-- tasks/everydollar/tests/websitebench/harbor/formal.py | 4 ++-- tasks/everyplate/tests/websitebench/harbor/formal.py | 4 ++-- tasks/fandango/tests/websitebench/harbor/formal.py | 4 ++-- tasks/freshdirect/tests/websitebench/harbor/formal.py | 4 ++-- tasks/hipcamp/tests/websitebench/harbor/formal.py | 4 ++-- tasks/hmart/tests/websitebench/harbor/formal.py | 4 ++-- tasks/imdb/tests/websitebench/harbor/formal.py | 4 ++-- tasks/insureon/tests/websitebench/harbor/formal.py | 4 ++-- tasks/insurify/tests/websitebench/harbor/formal.py | 4 ++-- tasks/ioby/tests/websitebench/harbor/formal.py | 4 ++-- tasks/kanetix-ca/tests/websitebench/harbor/formal.py | 4 ++-- tasks/leetcode/tests/websitebench/harbor/formal.py | 4 ++-- tasks/legalnature/tests/websitebench/harbor/formal.py | 4 ++-- tasks/linkedin/tests/websitebench/harbor/formal.py | 4 ++-- tasks/llbean/tests/websitebench/harbor/formal.py | 4 ++-- tasks/lowes/tests/websitebench/harbor/formal.py | 4 ++-- tasks/medium/tests/websitebench/harbor/formal.py | 4 ++-- tasks/meetup/tests/websitebench/harbor/formal.py | 4 ++-- tasks/notion/tests/websitebench/harbor/formal.py | 4 ++-- tasks/ollie/tests/websitebench/harbor/formal.py | 4 ++-- tasks/ontario-parks/tests/websitebench/harbor/formal.py | 4 ++-- tasks/opentable/tests/websitebench/harbor/formal.py | 4 ++-- tasks/petco/tests/websitebench/harbor/formal.py | 4 ++-- tasks/petfinder/tests/websitebench/harbor/formal.py | 4 ++-- tasks/petsmart/tests/websitebench/harbor/formal.py | 4 ++-- tasks/ratemyprofessors/tests/websitebench/harbor/formal.py | 4 ++-- tasks/resy/tests/websitebench/harbor/formal.py | 4 ++-- tasks/simplify-jobs/tests/websitebench/harbor/formal.py | 4 ++-- tasks/target/tests/websitebench/harbor/formal.py | 4 ++-- tasks/taskrabbit/tests/websitebench/harbor/formal.py | 4 ++-- tasks/tripadvisor/tests/websitebench/harbor/formal.py | 4 ++-- tasks/tripit/tests/websitebench/harbor/formal.py | 4 ++-- tasks/ubereats/tests/websitebench/harbor/formal.py | 4 ++-- tasks/warbyparker/tests/websitebench/harbor/formal.py | 4 ++-- tasks/wayfair/tests/websitebench/harbor/formal.py | 4 ++-- tasks/zennioptical/tests/websitebench/harbor/formal.py | 4 ++-- 70 files changed, 140 insertions(+), 140 deletions(-) diff --git a/tasks/1password-web/tests/websitebench/harbor/formal.py b/tasks/1password-web/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/1password-web/tests/websitebench/harbor/formal.py +++ b/tasks/1password-web/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/33/tests/websitebench/harbor/formal.py b/tasks/33/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/33/tests/websitebench/harbor/formal.py +++ b/tasks/33/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/ace-hardware/tests/websitebench/harbor/formal.py b/tasks/ace-hardware/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/ace-hardware/tests/websitebench/harbor/formal.py +++ b/tasks/ace-hardware/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/airtable/tests/websitebench/harbor/formal.py b/tasks/airtable/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/airtable/tests/websitebench/harbor/formal.py +++ b/tasks/airtable/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/amazon-home-services/tests/websitebench/harbor/formal.py b/tasks/amazon-home-services/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/amazon-home-services/tests/websitebench/harbor/formal.py +++ b/tasks/amazon-home-services/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/amazon/tests/websitebench/harbor/formal.py b/tasks/amazon/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/amazon/tests/websitebench/harbor/formal.py +++ b/tasks/amazon/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/amc-theatres/tests/websitebench/harbor/formal.py b/tasks/amc-theatres/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/amc-theatres/tests/websitebench/harbor/formal.py +++ b/tasks/amc-theatres/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/angi/tests/websitebench/harbor/formal.py b/tasks/angi/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/angi/tests/websitebench/harbor/formal.py +++ b/tasks/angi/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/apple/tests/websitebench/harbor/formal.py b/tasks/apple/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/apple/tests/websitebench/harbor/formal.py +++ b/tasks/apple/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/article/tests/websitebench/harbor/formal.py b/tasks/article/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/article/tests/websitebench/harbor/formal.py +++ b/tasks/article/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/asana/tests/websitebench/harbor/formal.py b/tasks/asana/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/asana/tests/websitebench/harbor/formal.py +++ b/tasks/asana/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/aspca-pet-insurance/tests/websitebench/harbor/formal.py b/tasks/aspca-pet-insurance/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/aspca-pet-insurance/tests/websitebench/harbor/formal.py +++ b/tasks/aspca-pet-insurance/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/autoslash/tests/websitebench/harbor/formal.py b/tasks/autoslash/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/autoslash/tests/websitebench/harbor/formal.py +++ b/tasks/autoslash/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/autotrader/tests/websitebench/harbor/formal.py b/tasks/autotrader/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/autotrader/tests/websitebench/harbor/formal.py +++ b/tasks/autotrader/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/backcountry/tests/websitebench/harbor/formal.py b/tasks/backcountry/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/backcountry/tests/websitebench/harbor/formal.py +++ b/tasks/backcountry/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/bean-box/tests/websitebench/harbor/formal.py b/tasks/bean-box/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/bean-box/tests/websitebench/harbor/formal.py +++ b/tasks/bean-box/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/beeradvocate/tests/websitebench/harbor/formal.py b/tasks/beeradvocate/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/beeradvocate/tests/websitebench/harbor/formal.py +++ b/tasks/beeradvocate/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/blinkist/tests/websitebench/harbor/formal.py b/tasks/blinkist/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/blinkist/tests/websitebench/harbor/formal.py +++ b/tasks/blinkist/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/blueapron/tests/websitebench/harbor/formal.py b/tasks/blueapron/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/blueapron/tests/websitebench/harbor/formal.py +++ b/tasks/blueapron/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/bluemercury/tests/websitebench/harbor/formal.py b/tasks/bluemercury/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/bluemercury/tests/websitebench/harbor/formal.py +++ b/tasks/bluemercury/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/boardgamegeek/tests/websitebench/harbor/formal.py b/tasks/boardgamegeek/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/boardgamegeek/tests/websitebench/harbor/formal.py +++ b/tasks/boardgamegeek/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/bookshop/tests/websitebench/harbor/formal.py b/tasks/bookshop/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/bookshop/tests/websitebench/harbor/formal.py +++ b/tasks/bookshop/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/breakoutgames-2/tests/websitebench/harbor/formal.py b/tasks/breakoutgames-2/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/breakoutgames-2/tests/websitebench/harbor/formal.py +++ b/tasks/breakoutgames-2/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/brooklinen/tests/websitebench/harbor/formal.py b/tasks/brooklinen/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/brooklinen/tests/websitebench/harbor/formal.py +++ b/tasks/brooklinen/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/capterra/tests/websitebench/harbor/formal.py b/tasks/capterra/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/capterra/tests/websitebench/harbor/formal.py +++ b/tasks/capterra/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/change/tests/websitebench/harbor/formal.py b/tasks/change/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/change/tests/websitebench/harbor/formal.py +++ b/tasks/change/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/columbia/tests/websitebench/harbor/formal.py b/tasks/columbia/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/columbia/tests/websitebench/harbor/formal.py +++ b/tasks/columbia/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/crumbl-cookies/tests/websitebench/harbor/formal.py b/tasks/crumbl-cookies/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/crumbl-cookies/tests/websitebench/harbor/formal.py +++ b/tasks/crumbl-cookies/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/crunchyroll/tests/websitebench/harbor/formal.py b/tasks/crunchyroll/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/crunchyroll/tests/websitebench/harbor/formal.py +++ b/tasks/crunchyroll/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/curology/tests/websitebench/harbor/formal.py b/tasks/curology/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/curology/tests/websitebench/harbor/formal.py +++ b/tasks/curology/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/dooney-bourke/tests/websitebench/harbor/formal.py b/tasks/dooney-bourke/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/dooney-bourke/tests/websitebench/harbor/formal.py +++ b/tasks/dooney-bourke/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/eatthismuch/tests/websitebench/harbor/formal.py b/tasks/eatthismuch/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/eatthismuch/tests/websitebench/harbor/formal.py +++ b/tasks/eatthismuch/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/etsy/tests/websitebench/harbor/formal.py b/tasks/etsy/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/etsy/tests/websitebench/harbor/formal.py +++ b/tasks/etsy/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/eventbrite/tests/websitebench/harbor/formal.py b/tasks/eventbrite/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/eventbrite/tests/websitebench/harbor/formal.py +++ b/tasks/eventbrite/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/everydollar/tests/websitebench/harbor/formal.py b/tasks/everydollar/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/everydollar/tests/websitebench/harbor/formal.py +++ b/tasks/everydollar/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/everyplate/tests/websitebench/harbor/formal.py b/tasks/everyplate/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/everyplate/tests/websitebench/harbor/formal.py +++ b/tasks/everyplate/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/fandango/tests/websitebench/harbor/formal.py b/tasks/fandango/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/fandango/tests/websitebench/harbor/formal.py +++ b/tasks/fandango/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/freshdirect/tests/websitebench/harbor/formal.py b/tasks/freshdirect/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/freshdirect/tests/websitebench/harbor/formal.py +++ b/tasks/freshdirect/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/hipcamp/tests/websitebench/harbor/formal.py b/tasks/hipcamp/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/hipcamp/tests/websitebench/harbor/formal.py +++ b/tasks/hipcamp/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/hmart/tests/websitebench/harbor/formal.py b/tasks/hmart/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/hmart/tests/websitebench/harbor/formal.py +++ b/tasks/hmart/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/imdb/tests/websitebench/harbor/formal.py b/tasks/imdb/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/imdb/tests/websitebench/harbor/formal.py +++ b/tasks/imdb/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/insureon/tests/websitebench/harbor/formal.py b/tasks/insureon/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/insureon/tests/websitebench/harbor/formal.py +++ b/tasks/insureon/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/insurify/tests/websitebench/harbor/formal.py b/tasks/insurify/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/insurify/tests/websitebench/harbor/formal.py +++ b/tasks/insurify/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/ioby/tests/websitebench/harbor/formal.py b/tasks/ioby/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/ioby/tests/websitebench/harbor/formal.py +++ b/tasks/ioby/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/kanetix-ca/tests/websitebench/harbor/formal.py b/tasks/kanetix-ca/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/kanetix-ca/tests/websitebench/harbor/formal.py +++ b/tasks/kanetix-ca/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/leetcode/tests/websitebench/harbor/formal.py b/tasks/leetcode/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/leetcode/tests/websitebench/harbor/formal.py +++ b/tasks/leetcode/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/legalnature/tests/websitebench/harbor/formal.py b/tasks/legalnature/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/legalnature/tests/websitebench/harbor/formal.py +++ b/tasks/legalnature/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/linkedin/tests/websitebench/harbor/formal.py b/tasks/linkedin/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/linkedin/tests/websitebench/harbor/formal.py +++ b/tasks/linkedin/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/llbean/tests/websitebench/harbor/formal.py b/tasks/llbean/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/llbean/tests/websitebench/harbor/formal.py +++ b/tasks/llbean/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/lowes/tests/websitebench/harbor/formal.py b/tasks/lowes/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/lowes/tests/websitebench/harbor/formal.py +++ b/tasks/lowes/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/medium/tests/websitebench/harbor/formal.py b/tasks/medium/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/medium/tests/websitebench/harbor/formal.py +++ b/tasks/medium/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/meetup/tests/websitebench/harbor/formal.py b/tasks/meetup/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/meetup/tests/websitebench/harbor/formal.py +++ b/tasks/meetup/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/notion/tests/websitebench/harbor/formal.py b/tasks/notion/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/notion/tests/websitebench/harbor/formal.py +++ b/tasks/notion/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/ollie/tests/websitebench/harbor/formal.py b/tasks/ollie/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/ollie/tests/websitebench/harbor/formal.py +++ b/tasks/ollie/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/ontario-parks/tests/websitebench/harbor/formal.py b/tasks/ontario-parks/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/ontario-parks/tests/websitebench/harbor/formal.py +++ b/tasks/ontario-parks/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/opentable/tests/websitebench/harbor/formal.py b/tasks/opentable/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/opentable/tests/websitebench/harbor/formal.py +++ b/tasks/opentable/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/petco/tests/websitebench/harbor/formal.py b/tasks/petco/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/petco/tests/websitebench/harbor/formal.py +++ b/tasks/petco/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/petfinder/tests/websitebench/harbor/formal.py b/tasks/petfinder/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/petfinder/tests/websitebench/harbor/formal.py +++ b/tasks/petfinder/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/petsmart/tests/websitebench/harbor/formal.py b/tasks/petsmart/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/petsmart/tests/websitebench/harbor/formal.py +++ b/tasks/petsmart/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/ratemyprofessors/tests/websitebench/harbor/formal.py b/tasks/ratemyprofessors/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/ratemyprofessors/tests/websitebench/harbor/formal.py +++ b/tasks/ratemyprofessors/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/resy/tests/websitebench/harbor/formal.py b/tasks/resy/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/resy/tests/websitebench/harbor/formal.py +++ b/tasks/resy/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/simplify-jobs/tests/websitebench/harbor/formal.py b/tasks/simplify-jobs/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/simplify-jobs/tests/websitebench/harbor/formal.py +++ b/tasks/simplify-jobs/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/target/tests/websitebench/harbor/formal.py b/tasks/target/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/target/tests/websitebench/harbor/formal.py +++ b/tasks/target/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/taskrabbit/tests/websitebench/harbor/formal.py b/tasks/taskrabbit/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/taskrabbit/tests/websitebench/harbor/formal.py +++ b/tasks/taskrabbit/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/tripadvisor/tests/websitebench/harbor/formal.py b/tasks/tripadvisor/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/tripadvisor/tests/websitebench/harbor/formal.py +++ b/tasks/tripadvisor/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/tripit/tests/websitebench/harbor/formal.py b/tasks/tripit/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/tripit/tests/websitebench/harbor/formal.py +++ b/tasks/tripit/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/ubereats/tests/websitebench/harbor/formal.py b/tasks/ubereats/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/ubereats/tests/websitebench/harbor/formal.py +++ b/tasks/ubereats/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/warbyparker/tests/websitebench/harbor/formal.py b/tasks/warbyparker/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/warbyparker/tests/websitebench/harbor/formal.py +++ b/tasks/warbyparker/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/wayfair/tests/websitebench/harbor/formal.py b/tasks/wayfair/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/wayfair/tests/websitebench/harbor/formal.py +++ b/tasks/wayfair/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest) diff --git a/tasks/zennioptical/tests/websitebench/harbor/formal.py b/tasks/zennioptical/tests/websitebench/harbor/formal.py index 536ffb09..d42dd637 100644 --- a/tasks/zennioptical/tests/websitebench/harbor/formal.py +++ b/tasks/zennioptical/tests/websitebench/harbor/formal.py @@ -825,9 +825,9 @@ def case_wall_clock_budget(self, case: Mapping[str, Any]) -> float: if str(identifier) in self.visuals ) declared.extend( - float(self.checks[str(case["check_id"])]["timeout_sec"]) + float(self.checks[str(case["cicd_check_id"])]["timeout_sec"]) for _ in (0,) - if str(case.get("check_id")) in self.checks + if str(case.get("cicd_check_id")) in self.checks ) longest = max(declared) if declared else 0.0 startup = 3 * max(60.0, longest)