diff --git a/.claude/skills/scss-best-practices/SKILL.md b/.claude/skills/scss-best-practices/SKILL.md new file mode 100644 index 00000000000..0fe16578c54 --- /dev/null +++ b/.claude/skills/scss-best-practices/SKILL.md @@ -0,0 +1,588 @@ +--- +name: scss-best-practices +description: SCSS/Sassy CSS best practices and coding guidelines for maintainable, scalable stylesheets +--- + +# SCSS Best Practices + +You are an expert in SCSS (Sassy CSS), CSS architecture, and maintainable stylesheet development. + +## Key Principles + +- Write modular, reusable SCSS that scales with project complexity +- Follow the DRY (Don't Repeat Yourself) principle using variables, mixins, and functions +- Maintain clear separation between structure, skin, and state styles +- Prioritize readability and maintainability over clever abstractions + +## File Organization + +### Architecture Pattern (7-1 Pattern) +``` +scss/ +├── abstracts/ +│ ├── _variables.scss # Global variables +│ ├── _functions.scss # SCSS functions +│ ├── _mixins.scss # Reusable mixins +│ └── _placeholders.scss # Extendable placeholders +├── base/ +│ ├── _reset.scss # CSS reset/normalize +│ ├── _typography.scss # Typography rules +│ └── _base.scss # Base element styles +├── components/ +│ ├── _buttons.scss # Button components +│ ├── _cards.scss # Card components +│ └── _forms.scss # Form components +├── layout/ +│ ├── _header.scss # Header layout +│ ├── _footer.scss # Footer layout +│ ├── _grid.scss # Grid system +│ └── _navigation.scss # Navigation layout +├── pages/ +│ ├── _home.scss # Home page specific +│ └── _contact.scss # Contact page specific +├── themes/ +│ └── _default.scss # Default theme +├── vendors/ +│ └── _bootstrap.scss # Third-party overrides +└── main.scss # Main manifest file +``` + +### Import Order +```scss +// main.scss +@use 'abstracts/variables'; +@use 'abstracts/functions'; +@use 'abstracts/mixins'; +@use 'abstracts/placeholders'; + +@use 'vendors/normalize'; + +@use 'base/reset'; +@use 'base/typography'; +@use 'base/base'; + +@use 'layout/grid'; +@use 'layout/header'; +@use 'layout/navigation'; +@use 'layout/footer'; + +@use 'components/buttons'; +@use 'components/cards'; +@use 'components/forms'; + +@use 'pages/home'; + +@use 'themes/default'; +``` + +## Variables + +### Naming Convention +```scss +// Use semantic, descriptive names +// Format: $category-property-variant + +// Colors +$color-primary: #3498db; +$color-primary-light: lighten($color-primary, 15%); +$color-primary-dark: darken($color-primary, 15%); +$color-secondary: #2ecc71; +$color-text: #333333; +$color-text-muted: #666666; +$color-background: #ffffff; +$color-border: #e0e0e0; +$color-error: #e74c3c; +$color-success: #27ae60; +$color-warning: #f39c12; + +// Typography +$font-family-base: 'Helvetica Neue', Arial, sans-serif; +$font-family-heading: 'Georgia', serif; +$font-size-base: 1rem; +$font-size-small: 0.875rem; +$font-size-large: 1.25rem; +$font-weight-normal: 400; +$font-weight-bold: 700; +$line-height-base: 1.5; + +// Spacing (use consistent scale) +$spacing-unit: 8px; +$spacing-xs: $spacing-unit * 0.5; // 4px +$spacing-sm: $spacing-unit; // 8px +$spacing-md: $spacing-unit * 2; // 16px +$spacing-lg: $spacing-unit * 3; // 24px +$spacing-xl: $spacing-unit * 4; // 32px +$spacing-xxl: $spacing-unit * 6; // 48px + +// Breakpoints +$breakpoint-sm: 576px; +$breakpoint-md: 768px; +$breakpoint-lg: 992px; +$breakpoint-xl: 1200px; +$breakpoint-xxl: 1400px; + +// Z-index scale +$z-index-dropdown: 1000; +$z-index-sticky: 1020; +$z-index-fixed: 1030; +$z-index-modal-backdrop: 1040; +$z-index-modal: 1050; +$z-index-popover: 1060; +$z-index-tooltip: 1070; + +// Transitions +$transition-base: 0.3s ease; +$transition-fast: 0.15s ease; +$transition-slow: 0.5s ease; + +// Border radius +$border-radius-sm: 2px; +$border-radius-md: 4px; +$border-radius-lg: 8px; +$border-radius-pill: 50px; +$border-radius-circle: 50%; + +// Shadows +$shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.05); +$shadow-md: 0 4px 6px rgba(0, 0, 0, 0.1); +$shadow-lg: 0 10px 15px rgba(0, 0, 0, 0.1); +$shadow-xl: 0 20px 25px rgba(0, 0, 0, 0.15); +``` + +### Maps for Related Values +```scss +// Use maps for grouped values +$colors: ( + 'primary': #3498db, + 'secondary': #2ecc71, + 'danger': #e74c3c, + 'warning': #f39c12, + 'info': #17a2b8, + 'success': #27ae60 +); + +$breakpoints: ( + 'sm': 576px, + 'md': 768px, + 'lg': 992px, + 'xl': 1200px, + 'xxl': 1400px +); + +// Access with map-get +.element { + color: map-get($colors, 'primary'); +} +``` + +## Mixins + +### Responsive Breakpoints +```scss +@mixin respond-to($breakpoint) { + @if map-has-key($breakpoints, $breakpoint) { + @media (min-width: map-get($breakpoints, $breakpoint)) { + @content; + } + } @else { + @warn "Unknown breakpoint: #{$breakpoint}"; + } +} + +// Usage +.element { + width: 100%; + + @include respond-to('md') { + width: 50%; + } + + @include respond-to('lg') { + width: 33.333%; + } +} +``` + +### Flexbox Utilities +```scss +@mixin flex-center { + display: flex; + align-items: center; + justify-content: center; +} + +@mixin flex-between { + display: flex; + align-items: center; + justify-content: space-between; +} + +@mixin flex-column { + display: flex; + flex-direction: column; +} +``` + +### Typography +```scss +@mixin font-size($size, $line-height: null) { + font-size: $size; + @if $line-height { + line-height: $line-height; + } +} + +@mixin truncate($lines: 1) { + @if $lines == 1 { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } @else { + display: -webkit-box; + -webkit-line-clamp: $lines; + -webkit-box-orient: vertical; + overflow: hidden; + } +} +``` + +### Accessibility +```scss +@mixin visually-hidden { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + +@mixin focus-visible { + &:focus-visible { + outline: 2px solid $color-primary; + outline-offset: 2px; + } +} +``` + +## BEM Naming Convention + +### Structure +```scss +// Block: Standalone component +// Element: Part of block (block__element) +// Modifier: Variant (block--modifier or block__element--modifier) + +.card { + // Block styles + background: $color-background; + border-radius: $border-radius-md; + box-shadow: $shadow-md; + + // Element + &__header { + padding: $spacing-md; + border-bottom: 1px solid $color-border; + } + + &__title { + margin: 0; + font-size: $font-size-large; + font-weight: $font-weight-bold; + } + + &__body { + padding: $spacing-md; + } + + &__footer { + padding: $spacing-md; + border-top: 1px solid $color-border; + } + + // Modifier + &--featured { + border: 2px solid $color-primary; + } + + &--compact { + .card__header, + .card__body, + .card__footer { + padding: $spacing-sm; + } + } +} +``` + +## Nesting Rules + +### Maximum Nesting Depth +```scss +// BAD: Too deep nesting +.nav { + .nav-list { + .nav-item { + .nav-link { + .nav-icon { + // 5 levels deep - avoid this + } + } + } + } +} + +// GOOD: Keep nesting to 3 levels maximum +.nav { + // Level 1 +} + +.nav__list { + // Level 1 +} + +.nav__item { + // Level 1 +} + +.nav__link { + color: $color-text; + + &:hover, + &:focus { + // Level 2 - acceptable for states + color: $color-primary; + } + + &--active { + // Level 2 - acceptable for modifiers + color: $color-primary; + font-weight: $font-weight-bold; + } +} +``` + +### Acceptable Nesting +```scss +.component { + // Direct child pseudo-elements + &::before, + &::after { + content: ''; + } + + // State modifiers + &:hover, + &:focus, + &:active { + // State styles + } + + // BEM modifiers + &--variant { + // Modifier styles + } + + // Media queries + @include respond-to('md') { + // Responsive styles + } +} +``` + +## Functions + +### Color Functions +```scss +@function tint($color, $percentage) { + @return mix(white, $color, $percentage); +} + +@function shade($color, $percentage) { + @return mix(black, $color, $percentage); +} + +// Usage +.element { + background: tint($color-primary, 20%); + border-color: shade($color-primary, 10%); +} +``` + +### Unit Conversion +```scss +@function px-to-rem($px, $base: 16) { + @return ($px / $base) * 1rem; +} + +@function rem-to-px($rem, $base: 16) { + @return ($rem / 1rem) * $base * 1px; +} + +// Usage +.element { + font-size: px-to-rem(18); // 1.125rem + padding: px-to-rem(24); // 1.5rem +} +``` + +### Spacing Function +```scss +@function spacing($multiplier) { + @return $spacing-unit * $multiplier; +} + +// Usage +.element { + margin-bottom: spacing(2); // 16px + padding: spacing(3); // 24px +} +``` + +## Extend and Placeholders + +### Use Placeholders Over Classes +```scss +// Define placeholder +%button-base { + display: inline-flex; + align-items: center; + justify-content: center; + padding: $spacing-sm $spacing-md; + border: none; + border-radius: $border-radius-md; + font-family: inherit; + font-size: $font-size-base; + font-weight: $font-weight-bold; + text-decoration: none; + cursor: pointer; + transition: all $transition-base; + + &:disabled { + opacity: 0.5; + cursor: not-allowed; + } +} + +// Extend placeholder +.btn-primary { + @extend %button-base; + background: $color-primary; + color: white; + + &:hover:not(:disabled) { + background: darken($color-primary, 10%); + } +} + +.btn-secondary { + @extend %button-base; + background: transparent; + color: $color-primary; + border: 2px solid $color-primary; + + &:hover:not(:disabled) { + background: $color-primary; + color: white; + } +} +``` + +## Loops and Iteration + +### Generate Utility Classes +```scss +// Spacing utilities +$spacing-directions: ( + '': '', + 't': '-top', + 'r': '-right', + 'b': '-bottom', + 'l': '-left', + 'x': '-inline', + 'y': '-block' +); + +@each $abbr, $direction in $spacing-directions { + @for $i from 0 through 8 { + .m#{$abbr}-#{$i} { + margin#{$direction}: spacing($i); + } + .p#{$abbr}-#{$i} { + padding#{$direction}: spacing($i); + } + } +} + +// Color utilities +@each $name, $color in $colors { + .text-#{$name} { + color: $color; + } + .bg-#{$name} { + background-color: $color; + } + .border-#{$name} { + border-color: $color; + } +} +``` + +## Performance Best Practices + +- Avoid overly specific selectors; aim for specificity of 0-1-0 (single class) +- Never use `!important` except for utility classes +- Minimize use of `@extend` across files (can cause bloat) +- Use `@use` and `@forward` instead of `@import` (deprecated) +- Compile with source maps in development, without in production +- Use autoprefixer for vendor prefixes instead of manual prefixes + +## Modern SCSS Features + +### Module System +```scss +// _variables.scss +$primary: #3498db; + +// _mixins.scss +@use 'variables' as vars; + +@mixin themed-button { + background: vars.$primary; +} + +// main.scss +@use 'mixins'; + +.button { + @include mixins.themed-button; +} +``` + +### Built-in Modules +```scss +@use 'sass:math'; +@use 'sass:color'; +@use 'sass:list'; +@use 'sass:map'; +@use 'sass:string'; + +.element { + width: math.div(100%, 3); + background: color.adjust($color-primary, $lightness: 10%); +} +``` + +## Code Style + +- Use 2 spaces for indentation +- Use single quotes for strings +- Add a space after colons in declarations +- Add a space before opening braces +- Put closing braces on new lines +- Separate rule sets with blank lines +- Order properties logically (positioning, box model, typography, visual, misc) +- Comment complex calculations and non-obvious code diff --git a/flutterbench-www-implementation-spec.md b/flutterbench-www-implementation-spec.md new file mode 100644 index 00000000000..a0aa56227dd --- /dev/null +++ b/flutterbench-www-implementation-spec.md @@ -0,0 +1,323 @@ +# Implement FlutterBench results page on flutter.dev (`sites/www`) + +## Read this first + +Before writing any code, read the repository's own `AGENTS.md` at the repo +root and follow it. It documents the real, current directory layout, build +commands, and conventions for this repo — treat it as higher authority than +anything below if the two ever disagree. As of this writing it establishes: + +- The repo is a Dart pub workspace containing multiple sites under `sites/`. +- `sites/www/` is the implementation of **flutter.dev**, written in Dart + using **Jaspr** and **Jaspr Content**. + - `sites/www/content/` — Markdown-based marketing pages and structured + content. + - `sites/www/lib/` — Dart source code for the site (components, layouts, + data models, routing). + - `sites/www/firebase.json` — Firebase Hosting config for flutter.dev. +- Common commands (run from repo root): + ``` + dart pub get + dart run dash_site --site=www serve + dart run dash_site --site=www build + dart run dash_site --help + ``` + +**Before implementing anything, spend time exploring `sites/www/lib` and +`sites/www/content` for an existing page that already does something close +to what we need** — a filterable/sortable grid or table driven by structured +data (candidates to look for: a showcase/case-studies grid, a +release-notes or roadmap listing, a search/index page, anything under an +`/ai/` or `/community/` section). Mirror that page's patterns (routing, +component structure, styling, how it wires interactivity) rather than +inventing new ones. This repo already ships `.agents/` and `AGENTS.md` +tooling for agent contributors — check `.agents/` for any additional +skills or house rules before starting. + +Do not guess at exact Jaspr/Jaspr Content API signatures (component base +classes, the `@client` annotation, routing registration, CSS/Sass wiring). +Confirm the exact syntax against: +1. The installed package versions in `sites/www/pubspec.yaml` / + `pubspec.lock`. +2. Existing components elsewhere in `sites/www/lib`. +3. The upstream Jaspr and Jaspr Content docs if neither of the above + answers it. + +--- + +## What we're building + +A new page (plus supporting sub-pages) on flutter.dev that presents results +from **FlutterBench**, an internal AI-coding-agent benchmark for Dart and +Flutter tasks. This is the MVP slice only — see "Out of scope" below. + +The page must let a visitor answer, without reading raw JSON: + +1. Which model/agent configuration currently performs best on Dart/Flutter + tasks. +2. Whether Dart/Flutter-specific AI tooling (the Dart MCP server, bundled + skills) measurably helps. +3. Which kinds of tasks (CUJs) models handle well vs. poorly. + +### Source data shape + +FlutterBench produces one **job** per benchmark run. A job is a matrix of +`(agent config) × (task)` **trials**. You will be given (or need to write an +ingestion step for) data shaped like the following — treat this as the +canonical schema to model in Dart, not the final file layout: + +- **Job summary** (`result.json` at the job root): `id`, `started_at`, + `finished_at`, `n_total_trials`, and a `stats` block containing, per + `evals` key (an eval key looks like `"{agent}__{model}__{variant}"`): + - `n_trials`, `n_errors` + - `metrics`: `{ reward: {mean, median, min, max} }` + - `pass_at_k`: `{ "1": 0.67 }` + - `reward_stats`: reward value → list of trial names at that reward + - `exception_stats`: exception type → list of trial names + - plus job-level `n_input_tokens`, `n_cache_tokens`, `n_output_tokens`, + `cost_usd`. +- **Per-trial `result.json`**: `trial_name` (format + `{task-slug}__{shortid}`), `task_name`, `config.agent` (`name`, + `model_name`, `skills[]`, `mcp_servers[]`), `agent_result` + (`n_input_tokens`, `n_output_tokens`, `cost_usd`), `verifier_result.rewards.reward`, + `exception_info` (present only on errored trials — `exception_type`, + `exception_message`), and phase timestamps (`environment_setup`, + `agent_setup`, `agent_execution`, `verifier`). +- **Per-trial `verifier/reward-details.json`**: a nested scoring tree. + Top-level `reward` is a weighted aggregator over named criteria (typically + `outcome`, `quality`, `dx`), each of which is itself an aggregator over + further named criteria. Some criteria are `"kind": "llm"` and carry a + `reasoning` string per sub-criterion (LLM-judge output) in addition to a + numeric `value`. Sibling top-level blocks like `process` and `efficiency` + are `"diagnostic": true` — **do not** fold these into the reward number + anywhere in the UI; label them explicitly as diagnostic-only. +- **Per-trial `verifier/test-stdout.txt`**: human-readable verifier log, + useful as a raw-log fallback. +- **Per-trial `agent/trajectory.json`** (when present): ordered list of + `{ action, input, duration_ms }` steps. +- **Per-trial `agent/*.txt`**: raw agent tool-call transcript. +- **Per-trial `artifacts/manifest.json`**: list of `{ source, destination, + type, status }` describing files copied out of the sandboxed workspace + (e.g. `lib/bloc/counter_bloc.dart`), plus the files themselves under + `artifacts/workspace/...`. +- Trials that error out (`exception_info` non-null) have **no** + `verifier_result` and must never be silently averaged in as a `0` — + they're a distinct status, not a score. + +### Data ingestion decision to make explicitly + +FlutterBench data will not live in the website repo in raw form. Before +writing any Dart, decide and document (in the PR description) one of: + +- **(a) Build-time static import**: a script/tool step that pulls the + latest job's data (from wherever FlutterBench publishes it — GCS bucket, + internal API, etc.) and writes it into `sites/www/content/data/flutterbench/` + as JSON, checked into the content directory like other structured content, + regenerated periodically by CI. +- **(b) Client-time fetch**: the page fetches a published JSON endpoint at + runtime. + +**Default to (a)** unless told otherwise — flutter.dev pages are +statically generated, and every other data-driven page in this repo will be +doing the same via `sites/www/content/`. If no such ingestion pipeline +exists yet, build the MVP against a small set of **fixture JSON files** +matching the schema above (you can shape these directly off the FlutterBench +mock data referenced in this project's history) so the UI can be built and +reviewed while the real pipeline is worked out separately. Flag this +explicitly as a known gap in the PR description if you go this route. + +--- + +## Pages and routes + +Confirm the exact route prefix with whoever owns flutter.dev's top-level +nav before merging — recent site work has been organizing AI-related +content under an `/ai/` section (see `sites/docs` for the `/ai/` docs +tree), so `/ai/flutterbench` may be the right home; `/flutterbench` at the +top level is the fallback. Whichever you pick, wire it into the existing +top-nav data file used by the site (find it by searching for how the +current nav items are declared — likely a YAML/JSON/Dart data file +referenced by the layout, not hardcoded HTML). + +1. **`/ai/flutterbench` — Overview / Leaderboard** (the main deliverable) +2. **`/ai/flutterbench/tasks`** — Task (CUJ) explorer +3. **`/ai/flutterbench/tasks/`** — one task's cross-model results +4. **`/ai/flutterbench/trials/`** — single trial detail +5. **`/ai/flutterbench/methodology`** — static Markdown page (can be a + plain `.md` file under `sites/www/content/`, no custom component needed) + explaining the harness, scoring rubric, and reproduction steps. + +For (3) and (4): prefer generating one static page per task/trial at build +time (the same way this site already generates one page per blog post) over +a single client-side-routed page, so results are crawlable, linkable, and +don't require a JS data fetch to render. If the existing patterns in +`sites/www/lib` favor a different approach for list-detail content, follow +that instead. + +--- + +## Components to build + +Match these to whatever this site's actual component/layout terminology is +(Jaspr components, layouts, partials) — the names below are functional +descriptions, not literal class names to copy verbatim. + +### 1. `SummaryStatsBar` +Row of 3–4 stat cards above the fold: top model (by mean reward), overall +average reward for the latest job, trials run, trials errored. Pull straight +from the job-level `stats` block. Reuse this site's existing "stat card" or +metric-card styling if one exists anywhere on flutter.dev (check the +homepage or a stats/about page); don't invent new card chrome if a +convention already exists. + +### 2. `LeaderboardTable` +One row per eval key (`agent__model__variant`), columns: rank, agent name, +model, mean reward (show min–max as a small inline range, not just the +mean), pass@1, cost (`cost_usd`), token count, error count. Sort by any +numeric column, client-interactive (mark whichever component handles the +sort/filter state as a Jaspr `@client` component per this site's existing +convention for interactive islands — check an existing interactive widget +on the site, e.g. a search box or theme toggle, for the pattern). Default +sort: mean reward, descending. + +Row click routes to `/ai/flutterbench/tasks?agent=` (or opens an +inline expansion — match whatever disclosure pattern this site already uses +for "show more" content). + +### 3. `FilterBar` +Chips/segmented controls above the table: +- Provider/lab filter (derive from model name prefix — this is cosmetic + grouping, no new data field required for MVP). +- CUJ category filter, if tasks have been tagged with a category; otherwise + omit for MVP rather than inventing a taxonomy. +- **"With Dart tooling" / "Without" toggle.** This only makes sense once + the ingested trials actually contain matched pairs (same task, same base + model, `agent.skills` / `agent.mcp_servers` varied). If the current data + doesn't have such pairs yet, build the toggle as a no-op / disabled state + with a tooltip explaining why, rather than faking a comparison — do not + ship a misleading toggle. + +All filters operate client-side over data already present in the page +(no server round-trip needed for MVP-scale data). + +### 4. `TaskModelHeatmap` +Grid: rows = tasks, columns = eval keys, cell = reward, color-scaled +red→green using this site's existing danger/warning/success role colors +(don't introduce new raw hex values — reuse whatever Sass variables or CSS +custom properties already express status color on this site, matching +Material's semantic color conventions Flutter docs already use elsewhere). +Errored/timed-out cells get a visually distinct (hatched or dashed-border, +gray) treatment — never render an error as if it were a `0` score. +Clicking a cell routes to the corresponding +`/ai/flutterbench/trials/` page. + +Below the grid, render a short computed "best CUJs" / "worst CUJs" list per +selected model (top/bottom 3 by reward) as plain text — this is a +derived summary, not a new data source. + +### 5. `TrialDetailView` +Renders a single trial. Sub-sections (use this site's existing tab or +section-anchor pattern, whichever is idiomatic here): + +- **Summary**: reward, a status badge (pass / partial / fail / **error** — + error is its own visually distinct state, not a low score), task name, + agent/model, and a small timeline showing the four phase durations + (`environment_setup`, `agent_setup`, `agent_execution`, `verifier`). +- **Reward breakdown**: render `reward-details.json` as a nested, + expandable tree — top-level weighted criteria (`outcome`/`quality`/`dx`) + expanding into their sub-criteria, each showing its `value`, `weight`, + and `description`. For `"kind": "llm"` criteria, show the `reasoning` + text inline under that criterion. Render `process` and `efficiency` in a + clearly separated "diagnostic (not scored)" section. +- **Trajectory**: if `agent/trajectory.json` exists, render its steps as a + simple ordered list/timeline (action, duration). If it doesn't exist for + a given trial, omit the section entirely rather than showing an empty + state — not every trial will have this. +- **Artifacts**: list files from `artifacts/manifest.json`; render each + file's contents in a code block (this site already has Markdown/code + fence syntax highlighting via Jaspr Content — reuse it rather than adding + a new code viewer). +- **Raw logs**: collapsible section with `verifier/test-stdout.txt` and, if + present, `exception.txt`/`exception_info`. + +### 6. `ErrorStateBadge` +A small reusable component (not just a text label) for anywhere a trial's +`exception_info` is shown — used in the leaderboard, the heatmap, and the +trial detail page, so error styling is consistent everywhere it appears. + +--- + +## Styling + +- Do not introduce a new color palette. Use flutter.dev's existing brand + blue for navigation/accent affordances (active filters, links) and this + site's existing semantic danger/warning/success tokens for reward + severity — these must stay visually distinct from each other so users + never confuse "this is a link" with "this is a bad score." + Grep `sites/www` for the site's Sass/CSS variable names (or however + Jaspr styling is wired in this codebase — inline `Styles`, CSS files, or + Sass) before hardcoding any color, and reuse what's there. +- Confirm dark-mode behavior: flutter.dev supports light/dark; every color + used for reward severity and the heatmap must have a legible dark-mode + pairing (tinted background + higher-contrast text, not pure fills). +- This page is denser and more tabular than most flutter.dev marketing + pages — it's fine, and preferable, for it to look closer to a docs/API + reference page than a landing page. Check whether `sites/docs` has table + styling that's more appropriate to borrow than anything in `sites/www` + itself, since this content is closer in spirit to docs than marketing. + +--- + +## Build order (do these as separable PRs if possible) + +1. Data models + fixture JSON + ingestion decision documented (see above). +2. `/ai/flutterbench` leaderboard page: `SummaryStatsBar` + `FilterBar` + + `LeaderboardTable`. This alone should be reviewable and mergeable. +3. `/ai/flutterbench/trials/` detail page with the reward + breakdown tree. (Ship this before the heatmap — the heatmap is much + less trustworthy without a way to click through and see *why* a score + is what it is.) +4. `/ai/flutterbench/tasks` + task detail pages + `TaskModelHeatmap`. +5. The with/without-tooling filter state in `FilterBar` (real, once matched + pairs exist in the data — otherwise leave disabled per above). +6. `/ai/flutterbench/methodology` static page. + +## Out of scope for this MVP (do not build) + +- Trajectory action-category stacked bar charts. +- Reward distribution histograms. +- Cost/efficiency scatterplots. +- Any live/streaming updates — this is a static-rebuild-per-job site. +- A generalized job-history view (trends across multiple past jobs); MVP + shows only the latest job. + +## Acceptance checklist + +- [ ] `dart run dash_site --site=www serve` runs locally and + `/ai/flutterbench` renders the leaderboard from fixture data. +- [ ] Leaderboard sorts by reward, cost, and error count. +- [ ] Clicking a leaderboard row and a heatmap cell both reach a working + trial detail page. +- [ ] A trial with `exception_info` set renders as a distinct error state + everywhere it appears (leaderboard, heatmap, detail page) — never as + a `0` blended into an average. +- [ ] `process`/`efficiency` diagnostic scores are visually and textually + separated from the scored `reward` breakdown. +- [ ] Page passes this repo's existing checks: `dart run dash_site + format-dart --check`, `dart run dash_site analyze-dart`, and any + link-check command documented in `AGENTS.md`. +- [ ] Dark mode reviewed for the heatmap and all status colors. +- [ ] New route(s) added to the site's nav data file, not hardcoded into a + layout template. + +## Open questions to resolve with a human before/while implementing + +1. Final route prefix (`/ai/flutterbench` vs `/flutterbench`) and nav + placement. +2. Where FlutterBench's real job data will be published from, and who owns + the ingestion job that turns it into `sites/www/content/data/...`. +3. Whether task "CUJ category" tags exist anywhere yet, or need to be added + to FlutterBench's own output before the task filter can be meaningful. +4. Whether any matched with/without-tooling trial pairs currently exist in + the data (needed before item 5 in the build order can ship as + functional rather than disabled). diff --git a/sites/docs/lib/_sass/_site.scss b/sites/docs/lib/_sass/_site.scss index df8a1d9de68..29ef7144fb6 100644 --- a/sites/docs/lib/_sass/_site.scss +++ b/sites/docs/lib/_sass/_site.scss @@ -52,7 +52,6 @@ @use 'package:site_shared/_sass/components/tooltip'; // Styles for specific pages, alphabetically ordered. -@use 'pages/cuj-index'; @use 'pages/glossary'; @use 'pages/learning-resources-index'; @use 'pages/not-found'; diff --git a/sites/docs/lib/_sass/components/_filterable-index.scss b/sites/docs/lib/_sass/components/_filterable-index.scss index f21a0c155e4..47c560f6a18 100644 --- a/sites/docs/lib/_sass/components/_filterable-index.scss +++ b/sites/docs/lib/_sass/components/_filterable-index.scss @@ -129,7 +129,7 @@ $mobile-breakpoint: 839px; flex-direction: column; button.show-filters-button { - @media (min-width: $mobile-breakpoint + 1) { + @media (min-width: calc($mobile-breakpoint + 1px)) { display: none; } } diff --git a/sites/docs/lib/_sass/pages/_cuj-index.scss b/sites/docs/lib/_sass/pages/_cuj-index.scss deleted file mode 100644 index b4c1024e06b..00000000000 --- a/sites/docs/lib/_sass/pages/_cuj-index.scss +++ /dev/null @@ -1,136 +0,0 @@ -@use '../components/filterable-index'; - -$font-size-sm: 0.875rem; -$font-size-xl: 1.5rem; -$spacing-xs: 0.25rem; -$spacing-sm: 0.5rem; -$spacing-md: 1rem; -$spacing-lg: 1.5rem; -$transition-normal: 0.2s ease; - -// The critical user journey index reuses the two column layout, search field, -// and filter sidebar of the learning resources index, which are styled in -// `_filterable-index.scss`. -// -// The journeys themselves render as full-width expandable cards, following -// the glossary in `_glossary.scss`. Expanding and collapsing is wired up -// by the `_setUpExpandableCards` global script. - -// The feedback button is the sidebar footer, so it sits below the filter card -// and slides in with it when the sidebar becomes a drawer. -.cuj-feedback { - margin-block-start: $spacing-md; - - .outlined-button { - width: 100%; - justify-content: center; - } - - @media (max-width: filterable-index.$mobile-breakpoint) { - margin-block-start: 0; - padding: 0.75rem; - border-block-start: 1px solid var(--site-inset-borderColor); - } -} - -#all-cujs-list { - margin-block-start: $spacing-md; - - .cuj-card { - padding: 0.75rem $spacing-md; - gap: $spacing-xs; - - .card-header { - display: flex; - flex-direction: row; - justify-content: space-between; - align-items: flex-start; - gap: $spacing-sm; - } - - .cuj-card-heading { - display: flex; - flex-direction: column; - align-items: flex-start; - gap: 0.4rem; - // Allow long goals to wrap instead of widening the flex item. - min-width: 0; - } - - .card-title { - display: block; - margin: 0; - font-family: var(--site-ui-fontFamily); - font-size: 1.1rem; - font-weight: 500; - line-height: 1.35; - text-wrap: pretty; - } - - .card-header-buttons { - display: flex; - flex-direction: row; - align-items: center; - gap: $spacing-xs; - flex-shrink: 0; - - .icon-button { - border-radius: $spacing-lg; - - > span { - font-size: $font-size-xl; - } - } - } - - .cuj-task-count { - margin: 0; - font-size: $font-size-sm; - color: var(--site-base-fgColor-lighter); - } - - // The shared card styles lay `.card-content` out as a centered row. - // Journeys need a plain block so the task list stacks below the divider. - .card-content { - display: block; - border-block-start: 0.05rem solid var(--site-inset-borderColor); - margin-block-start: $spacing-sm; - padding-block-start: $spacing-sm; - } - - &.collapsed { - .card-content { - display: none; - } - - .expand-button { - transform: rotate(180deg); - } - } - - .expand-button { - transition: transform $transition-normal; - - @media (prefers-reduced-motion: reduce) { - transition: none; - } - } - - .cuj-task-list { - margin: 0; - padding-inline-start: 1.15rem; - - li { - padding-inline-start: 0; - margin-block-end: $spacing-sm; - font-size: $font-size-sm; - line-height: 1.45; - color: var(--site-base-fgColor-lighter); - - &:last-child { - margin-block-end: 0; - } - } - } - } -} diff --git a/sites/docs/lib/main.client.options.dart b/sites/docs/lib/main.client.options.dart index 54becaa8fdd..cc8c4540471 100644 --- a/sites/docs/lib/main.client.options.dart +++ b/sites/docs/lib/main.client.options.dart @@ -12,10 +12,6 @@ import 'package:docs_flutter_dev_site/src/components/common/client/os_selector.d deferred as _os_selector; import 'package:docs_flutter_dev_site/src/components/layout/client/pagenav.dart' deferred as _pagenav; -import 'package:docs_flutter_dev_site/src/components/pages/cuj/cuj_filters.dart' - deferred as _cuj_filters; -import 'package:docs_flutter_dev_site/src/components/pages/cuj/cuj_filters_sidebar.dart' - deferred as _cuj_filters_sidebar; import 'package:docs_flutter_dev_site/src/components/pages/archive_table.dart' deferred as _archive_table; import 'package:docs_flutter_dev_site/src/components/pages/glossary_search_section.dart' @@ -98,14 +94,6 @@ ClientOptions get defaultClientOptions => ClientOptions( ), loader: _archive_table.loadLibrary, ), - 'cuj_filters': ClientLoader( - (p) => _cuj_filters.CujFilters(), - loader: _cuj_filters.loadLibrary, - ), - 'cuj_filters_sidebar': ClientLoader( - (p) => _cuj_filters_sidebar.CujFiltersSidebar(), - loader: _cuj_filters_sidebar.loadLibrary, - ), 'glossary_search_section': ClientLoader( (p) => _glossary_search_section.GlossarySearchSection(), loader: _glossary_search_section.loadLibrary, diff --git a/sites/docs/lib/main.server.dart b/sites/docs/lib/main.server.dart index 196bed0f2b2..5068063b7a8 100644 --- a/sites/docs/lib/main.server.dart +++ b/sites/docs/lib/main.server.dart @@ -28,7 +28,6 @@ import 'src/components/common/code_preview.dart'; import 'src/components/common/dash_image.dart'; import 'src/components/pages/architecture_recommendations.dart'; import 'src/components/pages/archive_table.dart'; -import 'src/components/pages/cuj/cuj_index.dart'; import 'src/components/pages/devtools_release_notes_index.dart'; import 'src/components/pages/expansion_list.dart'; import 'src/components/pages/learning_resource_index.dart'; @@ -119,7 +118,6 @@ List get _embeddableComponents => [ defineComponent('OSSelector', const OsSelector()), defineComponentWithChild('Card', Card.fromAttributes), defineComponent('LearningResourceIndex', const LearningResourceIndex()), - defineComponent('CujIndex', const CujIndex()), defineComponentWithAttrs('ArchiveTable', ArchiveTable.fromAttributes), defineComponentWithAttrs( 'DownloadLatestButton', diff --git a/sites/docs/lib/main.server.options.dart b/sites/docs/lib/main.server.options.dart index f6ef0647a7a..18388f15036 100644 --- a/sites/docs/lib/main.server.options.dart +++ b/sites/docs/lib/main.server.options.dart @@ -11,10 +11,6 @@ import 'package:docs_flutter_dev_site/src/components/common/client/os_selector.d as _os_selector; import 'package:docs_flutter_dev_site/src/components/layout/client/pagenav.dart' as _pagenav; -import 'package:docs_flutter_dev_site/src/components/pages/cuj/cuj_filters.dart' - as _cuj_filters; -import 'package:docs_flutter_dev_site/src/components/pages/cuj/cuj_filters_sidebar.dart' - as _cuj_filters_sidebar; import 'package:docs_flutter_dev_site/src/components/pages/archive_table.dart' as _archive_table; import 'package:docs_flutter_dev_site/src/components/pages/glossary_search_section.dart' @@ -82,13 +78,6 @@ ServerOptions get defaultServerOptions => ServerOptions( 'archive_table', params: __archive_tableArchiveTable, ), - _cuj_filters.CujFilters: ClientTarget<_cuj_filters.CujFilters>( - 'cuj_filters', - ), - _cuj_filters_sidebar.CujFiltersSidebar: - ClientTarget<_cuj_filters_sidebar.CujFiltersSidebar>( - 'cuj_filters_sidebar', - ), _glossary_search_section.GlossarySearchSection: ClientTarget<_glossary_search_section.GlossarySearchSection>( 'glossary_search_section', diff --git a/sites/docs/lib/src/client/global_scripts.dart b/sites/docs/lib/src/client/global_scripts.dart index fc71855973a..02f9f7f26a4 100644 --- a/sites/docs/lib/src/client/global_scripts.dart +++ b/sites/docs/lib/src/client/global_scripts.dart @@ -21,6 +21,8 @@ void setUpSite() { _setUpToc(); _setUpSteppers(); _setUpIdeExplorers(); + _setUpGraderMatrix(); + _setUpInteractiveDetailCards(); } void _setUpSearchKeybindings() { @@ -600,3 +602,223 @@ void _setUpIdeExplorer(web.Element explorer) { explorer.addEventListener('click', handleClick.toJS); } + +/// Set up interactivity for the FlutterBench grader matrix component. +void _setUpGraderMatrix() { + final matrices = web.document.querySelectorAll('.grader-matrix'); + for (var i = 0; i < matrices.length; i++) { + _setUpSingleGraderMatrix(matrices.item(i) as web.Element); + } +} + +void _setUpSingleGraderMatrix(web.Element matrix) { + final buttons = matrix.querySelectorAll('.matrix-filters .filter-btn'); + final cards = matrix.querySelectorAll('.grader-cards-track .grader-card'); + final track = matrix.querySelector('.grader-cards-track') as web.HTMLElement?; + final prevBtn = + matrix.querySelector('.carousel-nav-btn.prev') as web.HTMLElement?; + final nextBtn = + matrix.querySelector('.carousel-nav-btn.next') as web.HTMLElement?; + + if (track == null) return; + + List getVisibleCards() { + final list = []; + for (var i = 0; i < cards.length; i++) { + final card = cards.item(i) as web.HTMLElement; + if (!card.classList.contains('hidden')) { + list.add(card); + } + } + return list; + } + + void scrollNext() { + final visibleCards = getVisibleCards(); + if (visibleCards.isEmpty) return; + + final maxScroll = track.scrollWidth - track.clientWidth; + final currentScroll = track.scrollLeft; + + // Find the first visible card that starts after currentScroll + 10px. + var scrolled = false; + for (final card in visibleCards) { + final cardOffset = card.offsetLeft - track.offsetLeft; + if (cardOffset > currentScroll + 10) { + track.scrollTo( + web.ScrollToOptions( + left: cardOffset.toDouble(), + behavior: 'smooth', + ), + ); + scrolled = true; + break; + } + } + + // If already at or near the end, rotate back to the start. + if (!scrolled || currentScroll >= maxScroll - 10) { + track.scrollTo( + web.ScrollToOptions( + left: 0, + behavior: 'smooth', + ), + ); + } + } + + void scrollPrev() { + final visibleCards = getVisibleCards(); + if (visibleCards.isEmpty) return; + + final maxScroll = track.scrollWidth - track.clientWidth; + final currentScroll = track.scrollLeft; + + // If at or near the beginning, rotate to the end. + if (currentScroll <= 10) { + track.scrollTo( + web.ScrollToOptions( + left: maxScroll.toDouble(), + behavior: 'smooth', + ), + ); + return; + } + + // Find the last visible card that starts before currentScroll - 10px. + for (var i = visibleCards.length - 1; i >= 0; i--) { + final card = visibleCards[i]; + final cardOffset = card.offsetLeft - track.offsetLeft; + if (cardOffset < currentScroll - 10) { + track.scrollTo( + web.ScrollToOptions( + left: cardOffset.toDouble(), + behavior: 'smooth', + ), + ); + return; + } + } + + track.scrollTo( + web.ScrollToOptions( + left: 0, + behavior: 'smooth', + ), + ); + } + + if (nextBtn != null) { + nextBtn.addEventListener( + 'click', + ((web.Event e) { + e.preventDefault(); + scrollNext(); + }).toJS, + ); + } + + if (prevBtn != null) { + prevBtn.addEventListener( + 'click', + ((web.Event e) { + e.preventDefault(); + scrollPrev(); + }).toJS, + ); + } + + for (var i = 0; i < buttons.length; i++) { + final btn = buttons.item(i) as web.HTMLElement; + final filter = btn.dataset['filter']; + + void handleClick(web.Event event) { + event.preventDefault(); + for (var j = 0; j < buttons.length; j++) { + (buttons.item(j) as web.Element).classList.remove('active'); + } + btn.classList.add('active'); + + for (var k = 0; k < cards.length; k++) { + final card = cards.item(k) as web.HTMLElement; + if (filter == 'all') { + card.classList.remove('hidden'); + } else if (filter == 'llm') { + card.classList.toggle('hidden', !card.classList.contains('cat-llm')); + } else if (filter == 'deterministic') { + card.classList.toggle( + 'hidden', + !card.classList.contains('cat-deterministic'), + ); + } else { + card.classList.toggle( + 'hidden', + !card.classList.contains('cat-$filter'), + ); + } + } + + // Reset scroll position to beginning on filter change. + track.scrollTo( + web.ScrollToOptions( + left: 0, + behavior: 'smooth', + ), + ); + } + + btn.addEventListener('click', handleClick.toJS); + } +} + +/// Set up interactivity for FlutterBench interactive detail cards +/// (e.g. ScoreTriage and EvaluationMatrix). +void _setUpInteractiveDetailCards() { + final cards = web.document.querySelectorAll( + '.interactive-detail-card, .score-triage', + ); + for (var i = 0; i < cards.length; i++) { + _setUpSingleInteractiveDetailCard(cards.item(i) as web.Element); + } +} + +void _setUpSingleInteractiveDetailCard(web.Element card) { + final buttons = card.querySelectorAll( + '.card-tabs-grid .card-tab-btn, .triage-tiers-grid .triage-tier-btn', + ); + final panels = card.querySelectorAll( + '.card-panels-container .card-panel, .triage-detail-card .triage-panel', + ); + final detailCard = card.querySelector( + '.card-panels-container, .triage-detail-card', + ) as web.HTMLElement?; + + for (var i = 0; i < buttons.length; i++) { + final btn = buttons.item(i) as web.HTMLElement; + final tabAttr = btn.dataset['tab']; + final tabId = tabAttr.isNotEmpty ? tabAttr : btn.dataset['tier']; + + void handleClick(web.Event event) { + event.preventDefault(); + for (var j = 0; j < buttons.length; j++) { + (buttons.item(j) as web.Element).classList.remove('active'); + } + btn.classList.add('active'); + + for (var k = 0; k < panels.length; k++) { + final panel = panels.item(k) as web.HTMLElement; + final panelTabAttr = panel.dataset['tab']; + final panelId = panelTabAttr.isNotEmpty + ? panelTabAttr + : panel.dataset['tier']; + panel.classList.toggle('active', panelId == tabId); + } + + if (detailCard != null) { + detailCard.scrollTop = 0; + } + } + + btn.addEventListener('click', handleClick.toJS); + } +} diff --git a/sites/docs/lib/src/components/pages/cuj/cuj_filters.dart b/sites/docs/lib/src/components/pages/cuj/cuj_filters.dart deleted file mode 100644 index 201a21571d7..00000000000 --- a/sites/docs/lib/src/components/pages/cuj/cuj_filters.dart +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2026 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:jaspr/dom.dart'; -import 'package:jaspr/jaspr.dart'; -import 'package:site_shared/components/common/button.dart'; -import 'package:universal_web/web.dart' as web; - -import '../../../models/cuj_model.dart'; -import '../filterable_index.dart'; -import 'cuj_filters_sidebar.dart'; - -/// The id of the search field, so its result count can label it. -const _searchId = 'cuj-search'; - -/// The search controls and result summary for the critical user journey index. -@client -class CujFilters extends StatefulComponent { - const CujFilters({super.key}); - - /// The ID of the checkbox that toggles the filter drawer on narrow screens. - static const String drawerToggleId = 'cuj-filter-toggle'; - - @override - State createState() => _CujFiltersState(); -} - -class _CujFiltersState extends State { - /// The filters selected in the critical user journey sidebar. - static CujFiltersNotifier get _filters => CujFiltersSidebar.filters; - - /// The journeys reconstructed from the rendered journey cards. - final List _cujs = []; - - /// The current search query. - String _searchQuery = ''; - - /// The number of journeys matching the active search and filters. - int _filteredCujCount = 0; - - @override - void initState() { - super.initState(); - - if (kIsWeb) { - _filters.addListener(_setFilters); - - final cujList = web.document.getElementById('all-cujs-list'); - if (cujList == null) { - return; - } - - _recreateCujs(cujList.querySelectorAll('.cuj-card')); - } - } - - /// Populates [_cujs] from [cujCards]. - void _recreateCujs(web.NodeList cujCards) { - for (var i = 0; i < cujCards.length; i++) { - final element = cujCards.item(i) as web.Element; - _cujs.add(Cuj.fromElement(element)); - } - _filteredCujCount = _cujs.length; - } - - /// Updates the filter state and re-evaluates which journeys to show. - /// - /// Use like the `setState` method by passing a callback that updates - /// the relevant state variables. - void _setFilters([void Function()? callback]) { - setState(callback ?? () {}); - - final cujsToShow = _filters.filterCujs(_cujs, _searchQuery); - _filteredCujCount = cujsToShow.length; - for (final cuj in _cujs) { - final element = - web.document.getElementById(cuj.elementId) as web.HTMLElement?; - if (element == null) { - continue; - } - - if (cujsToShow.contains(cuj)) { - element.classList.remove('hidden'); - } else { - element.classList.add('hidden'); - } - } - } - - @override - void dispose() { - if (kIsWeb) { - _filters.removeListener(_setFilters); - } - super.dispose(); - } - - @override - Component build(BuildContext context) { - return FilterSearchGroup( - drawerToggleId: CujFilters.drawerToggleId, - searchId: _searchId, - placeholder: 'Try "testing" or "architecture"...', - label: 'Search critical user journeys by goal, persona, and task', - value: _searchQuery, - onInput: (value) { - _setFilters(() { - _searchQuery = value; - }); - }, - children: [ - div(classes: 'label-row', [ - label( - attributes: {'for': _searchId, 'aria-live': 'polite'}, - [ - const .text('Showing '), - span([.text('$_filteredCujCount')]), - const .text(' / '), - span([.text('${_cujs.length}')]), - ], - ), - Button( - icon: 'close_small', - content: 'Clear filters', - size: ButtonSize.compact, - disabled: _searchQuery.isEmpty && !_filters.hasSelectedPersonas, - onClick: () { - // No setState needed, since resetting filters will trigger it. - _searchQuery = ''; - _filters.reset(); - }, - ), - ]), - ], - ); - } -} diff --git a/sites/docs/lib/src/components/pages/cuj/cuj_filters_sidebar.dart b/sites/docs/lib/src/components/pages/cuj/cuj_filters_sidebar.dart deleted file mode 100644 index 34aff7ae6ca..00000000000 --- a/sites/docs/lib/src/components/pages/cuj/cuj_filters_sidebar.dart +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2026 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:jaspr/dom.dart'; -import 'package:jaspr/jaspr.dart'; -import 'package:site_shared/components/common/button.dart'; - -import '../../../models/cuj_model.dart'; -import '../filterable_index.dart'; -import 'cuj_filters.dart'; - -// TODO(ewindmill): Replace with the real feedback destination once it exists. -const _feedbackUrl = 'https://github.com/flutter/evals/issues'; - -/// The persona filters for the critical user journey index. -@client -class CujFiltersSidebar extends StatelessComponent { - const CujFiltersSidebar({super.key}); - - /// The filter state for the critical user journey list. - /// - /// This is static so that [CujFilters] can access it, - /// since both client components don't share a common ancestor. - static final CujFiltersNotifier filters = CujFiltersNotifier(); - - @override - Component build(BuildContext context) { - return FiltersSidebar( - drawerToggleId: CujFilters.drawerToggleId, - footer: const [ - div(classes: 'cuj-feedback', [ - Button( - href: _feedbackUrl, - content: 'Provide feedback', - style: ButtonStyle.outlined, - title: 'Leave feedback or suggest new CUJs.', - attributes: { - 'target': '_blank', - 'rel': 'noopener', - }, - ), - ]), - ], - children: [ - ListenableBuilder( - listenable: filters, - builder: (context) { - return div(classes: 'table-content', [ - const h4([.text('Persona')]), - ul([ - for (final persona in CujPersona.values) - li([ - input( - type: InputType.checkbox, - attributes: {'name': 'cuj-filter-${persona.name}'}, - id: 'cuj-filter-${persona.name}', - checked: filters.isPersonaSelected(persona), - onChange: (checked) { - filters.setPersona( - persona, - isSelected: checked as bool? ?? false, - ); - }, - ), - label( - attributes: {'for': 'cuj-filter-${persona.name}'}, - [.text(persona.label)], - ), - ]), - ]), - ]); - }, - ), - ], - ); - } -} - -/// Stores the selected critical user journey filters and -/// notifies listeners when they change. -final class CujFiltersNotifier extends ChangeNotifier { - /// The currently selected personas. - final Set _selectedPersonas = {}; - - /// Whether any persona filters are selected. - bool get hasSelectedPersonas => _selectedPersonas.isNotEmpty; - - /// Whether [persona] is selected. - bool isPersonaSelected(CujPersona persona) => - _selectedPersonas.contains(persona); - - /// Updates whether [persona] is selected and notifies listeners. - void setPersona(CujPersona persona, {required bool isSelected}) { - if (isSelected) { - _selectedPersonas.add(persona); - } else { - _selectedPersonas.remove(persona); - } - notifyListeners(); - } - - /// Clears all selected personas. - void reset() { - _selectedPersonas.clear(); - notifyListeners(); - } - - /// Returns the journeys matching [searchQuery] and the selected filters. - Set filterCujs(List cujs, String searchQuery) { - searchQuery = searchQuery.trim().toLowerCase(); - - if (searchQuery.isEmpty && _selectedPersonas.isEmpty) { - // No filters applied, return all journeys. - return cujs.toSet(); - } - - final cujsToShow = {}; - - for (final cuj in cujs) { - final matchesPersona = - _selectedPersonas.isEmpty || isPersonaSelected(cuj.persona); - if (!matchesPersona) { - continue; - } - - final matchesSearchQuery = - searchQuery.isEmpty || - cuj.goal.toLowerCase().contains(searchQuery) || - cuj.persona.label.toLowerCase().contains(searchQuery) || - cuj.tasks.any( - (task) => task.task.toLowerCase().contains(searchQuery), - ); - if (!matchesSearchQuery) { - continue; - } - - cujsToShow.add(cuj); - } - - return cujsToShow; - } -} diff --git a/sites/docs/lib/src/components/pages/cuj/cuj_index.dart b/sites/docs/lib/src/components/pages/cuj/cuj_index.dart deleted file mode 100644 index 2e551ae5ecb..00000000000 --- a/sites/docs/lib/src/components/pages/cuj/cuj_index.dart +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2026 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:convert' show jsonEncode; - -import 'package:jaspr/dom.dart'; -import 'package:jaspr/jaspr.dart'; -import 'package:jaspr_content/jaspr_content.dart'; -import 'package:site_shared/components/common/button.dart'; -import 'package:site_shared/components/common/card.dart'; -import 'package:site_shared/components/common/tags.dart'; - -import '../../../models/cuj_model.dart'; -import 'cuj_filters.dart'; -import 'cuj_filters_sidebar.dart'; - -/// Renders the filterable critical user journey catalog. -final class CujIndex extends StatelessComponent { - const CujIndex({super.key}); - - @override - Component build(BuildContext context) { - final cujData = context.page.data['cujs'] as List; - - final cujs = [ - for (final cuj in cujData) Cuj.fromMap(cuj as Map), - ]; - - return div(classes: 'filterable-index', [ - div(classes: 'left-col', [ - const CujFilters(), - div(classes: 'card-list', id: 'all-cujs-list', [ - for (final cuj in cujs) _CujCard(cuj), - ]), - ]), - const CujFiltersSidebar(), - ]); - } -} - -/// An expandable card that summarizes a critical user journey and its tasks. -final class _CujCard extends StatelessComponent { - const _CujCard(this.cuj); - - /// The critical user journey displayed by this card. - final Cuj cuj; - - @override - Component build(BuildContext context) { - final cardId = cuj.elementId; - final taskCount = cuj.tasks.length; - - // Expanding and collapsing is handled for every `.expandable-card` - // by the `_setUpExpandableCards` global script. - return Card.expandable( - id: cardId, - outlined: true, - additionalClasses: 'cuj-card', - initiallyExpanded: false, - attributes: { - 'data-persona': cuj.persona.name, - 'data-goal': cuj.goal, - 'data-tasks': jsonEncode(cuj.tasks), - }, - header: [ - div(classes: 'cuj-card-heading', [ - Tag( - cuj.persona.label, - color: cuj.persona.tagColor, - size: TagSize.small, - ), - h2(classes: 'card-title', [.text(cuj.goal)]), - ]), - div(classes: 'card-header-buttons', [ - Button( - href: '#$cardId', - icon: 'tag', - classes: const ['share-button'], - title: 'Link to journey', - attributes: { - 'aria-label': 'Link to the "${cuj.goal}" journey', - }, - ), - Button( - icon: 'keyboard_arrow_up', - classes: const ['expand-button'], - title: 'Expand or collapse tasks', - attributes: { - 'aria-expanded': 'false', - 'aria-controls': '$cardId-content', - 'aria-label': 'Expand or collapse the tasks for "${cuj.goal}"', - }, - ), - ]), - ], - collapsedContent: [ - p(classes: 'cuj-task-count', [ - .text(taskCount == 1 ? '1 task' : '$taskCount tasks'), - ]), - ], - expandedContent: [ - ul(classes: 'cuj-task-list', [ - for (final task in cuj.tasks) li([.text(task.task)]), - ]), - ], - ); - } -} diff --git a/sites/docs/lib/src/layouts/doc_layout.dart b/sites/docs/lib/src/layouts/doc_layout.dart index ea78dcfbf01..6ea6cb5b4ef 100644 --- a/sites/docs/lib/src/layouts/doc_layout.dart +++ b/sites/docs/lib/src/layouts/doc_layout.dart @@ -42,6 +42,10 @@ class DocLayout extends FlutterDocsLayout { ); } + /// Builds an optional component to render at the top of the + /// `.after-leading-content` container, before the side menu and article. + Component? buildLeadingContent(Page page) => null; + @override Component buildBody(Page page, Component child) { final pageData = page.data.page; @@ -49,6 +53,9 @@ class DocLayout extends FlutterDocsLayout { final pageTitle = pageData['title'] as String; final pageDescription = (pageData['description'] as String?)?.trim(); final navigationData = page.navigationData; + + // Always show page header unless explicitly hidden. + final showPageHeading = (pageData['showPageHeader'] as bool?) ?? true; return super.buildBody( page, @@ -67,6 +74,7 @@ class DocLayout extends FlutterDocsLayout { PageNavBar(navigationData), ], ), + ?buildLeadingContent(page), ?buildBanner(page), div(classes: 'after-leading-content', [ if (navigationData case PageNavigationData( @@ -77,11 +85,12 @@ class DocLayout extends FlutterDocsLayout { DashTableOfContents(toc), ]), article([ - PageHeader( - title: pageTitle, - description: pageDescription, - showBreadcrumbs: showBreadcrumbsFor(page), - ), + if (showPageHeading) + PageHeader( + title: pageTitle, + description: pageDescription, + showBreadcrumbs: showBreadcrumbsFor(page), + ), child, PrevNext( diff --git a/sites/docs/lib/src/models/cuj_model.dart b/sites/docs/lib/src/models/cuj_model.dart deleted file mode 100644 index e0e58c95067..00000000000 --- a/sites/docs/lib/src/models/cuj_model.dart +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2026 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:convert'; - -import 'package:site_shared/components/common/tags.dart'; -import 'package:universal_web/web.dart' as web; - -/// Prefix used for the DOM ID of each journey's card element. -const _elementIdPrefix = 'cuj-'; - -/// A critical user journey and its associated tasks. -final class Cuj { - const Cuj._({ - required this.id, - required this.goal, - required this.persona, - required this.tasks, - }); - - /// Creates a journey from YAML-backed page data. - factory Cuj.fromMap(Map map) { - return Cuj._( - id: map['id'] as int, - goal: map['goal'] as String, - persona: CujPersona.fromDataValue(map['persona'] as String), - tasks: [ - for (final task in map['tasks'] as List) - CujTask.fromMap(task as Map), - ], - ); - } - - /// Creates a journey from data attributes on [element]. - factory Cuj.fromElement(web.Element element) { - final dataPersona = - element.getAttribute('data-persona') ?? - (throw StateError('CUJ card ${element.id} has no persona.')); - final dataGoal = - element.getAttribute('data-goal') ?? - (throw StateError('CUJ card ${element.id} has no goal.')); - final dataTasks = - element.getAttribute('data-tasks') ?? - (throw StateError('CUJ card ${element.id} has no tasks.')); - - return Cuj._( - id: int.parse(element.id.replaceFirst(_elementIdPrefix, '')), - goal: dataGoal, - persona: CujPersona.values.byName(dataPersona), - tasks: [ - for (final task in jsonDecode(dataTasks) as List) - CujTask.fromMap(task as Map), - ], - ); - } - - /// The stable numeric identifier for this journey. - final int id; - - /// The developer goal that this journey represents. - final String goal; - - /// The developer persona associated with this journey. - final CujPersona persona; - - /// The tasks that contribute to [goal]. - final List tasks; - - /// The identifier of the card element that renders this journey. - String get elementId => '$_elementIdPrefix$id'; -} - -/// A concrete task within a critical user journey. -final class CujTask { - const CujTask({ - required this.id, - required this.name, - required this.task, - }); - - /// Creates a task from YAML-backed page data or decoded JSON. - factory CujTask.fromMap(Map map) { - return CujTask( - id: map['id'] as int, - name: map['name'] as String, - task: map['task'] as String, - ); - } - - /// The stable numeric identifier for this task. - final int id; - - /// The stable machine-readable name of this task. - final String name; - - /// The reader-facing task description. - final String task; - - /// A JSON-compatible representation of this task. - Map toJson() => { - 'id': id, - 'name': name, - 'task': task, - }; -} - -/// The developer personas a critical user journey can belong to. -/// -/// [dataValue] must match the `persona` values used in `src/data/cujs.yaml`. -enum CujPersona { - appDeveloper('App developer', 'The App Developer', TagColor.blue), - techLead( - 'Tech lead / architect', - 'The Tech Lead / Architect', - TagColor.purple, - ), - pluginDeveloper('Plugin developer', 'The Plugin Developer', TagColor.teal), - fullStackDeveloper( - 'Full-stack developer', - 'The Full Stack Developer', - TagColor.magenta, - ), - hybridDeveloper( - 'Hybrid developer', - 'The Hybrid (Native + Flutter) Developer', - TagColor.amber, - ); - - const CujPersona(this.label, this.dataValue, this.tagColor); - - /// Returns the persona whose [dataValue] matches the YAML data. - /// - /// Throws an [ArgumentError] if the value is unknown. - static CujPersona fromDataValue(String dataValue) { - for (final persona in values) { - if (persona.dataValue == dataValue) { - return persona; - } - } - throw ArgumentError.value(dataValue, 'dataValue', 'Unknown CUJ persona'); - } - - /// The reader-facing name of this persona. - final String label; - - /// The persona value used in `cujs.yaml`. - final String dataValue; - - /// The color used for this persona's [Tag] badge. - final TagColor tagColor; -} diff --git a/sites/docs/lib/src/utils/inline_code.dart b/sites/docs/lib/src/utils/inline_code.dart new file mode 100644 index 00000000000..c1fe3ca416b --- /dev/null +++ b/sites/docs/lib/src/utils/inline_code.dart @@ -0,0 +1,23 @@ +// Copyright 2025 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +/// Renders text with optional inline code segments delimited by backticks. +Component renderDescriptionWithCode(String text) { + if (!text.contains('`')) { + return .text(text); + } + final parts = text.split('`'); + final children = []; + for (var i = 0; i < parts.length; i++) { + if (i.isOdd) { + children.add(code([.text(parts[i])])); + } else if (parts[i].isNotEmpty) { + children.add(.text(parts[i])); + } + } + return .fragment(children); +} diff --git a/sites/docs/src/content/ai/flutter-bench/cujs.md b/sites/docs/src/content/ai/flutter-bench/cujs.md deleted file mode 100644 index 3a0b693d33e..00000000000 --- a/sites/docs/src/content/ai/flutter-bench/cujs.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: Flutter critical user journeys -shortTitle: Flutter CUJs -description: >- - Browse the catalog of canonical Flutter and Dart critical user journeys that - the FlutterBench evaluations test. -bodyClass: wide-site-content -showToc: false -sitemap: false -noindex: true -dateModifiedSources: - - src/data/cujs.yaml ---- - -A _critical user journey_ (CUJ) is a goal that a developer sets out to -accomplish, such as "make an application accessible to all users" or -"diagnose and resolve layout overflow errors". -Each CUJ is broken down into the concrete tasks required to complete it. - -Product teams at Google treat CUJs as a source of truth: -they're how teams align on priorities, shape roadmaps, -measure product health, and more. -The Flutter team uses CUJs to derive evaluation tasks and -prompts for FlutterBench. - -The following catalogue lists the Flutter team's CUJs. It's a claim about -what matters in Flutter development. If the way you build Flutter apps -isn't represented here, the list is incomplete, and we -encourage [you to open an issue with your feedback][]. - - - -[you to open an issue with your feedback]: https://github.com/flutter/evals/issues diff --git a/sites/docs/src/content/ai/flutter-bench/index.md b/sites/docs/src/content/ai/flutter-bench/index.md deleted file mode 100644 index 2f3ec5dce8d..00000000000 --- a/sites/docs/src/content/ai/flutter-bench/index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: FlutterBench agent evaluations -shortTitle: FlutterBench -description: >- - How the Flutter team implements agent evals, and the results of those evals. ---- - -:::note Coming soon -Evaluation tooling and benchmarks are coming soon. -::: \ No newline at end of file diff --git a/sites/docs/src/data/cujs.yaml b/sites/docs/src/data/cujs.yaml deleted file mode 100644 index 278f62645c4..00000000000 --- a/sites/docs/src/data/cujs.yaml +++ /dev/null @@ -1,1051 +0,0 @@ -# List of all Flutter CUJs -# The source-of-truth is the tasks in the FlutterBench -# repository (which is currently private.) -# -# TODO(ewindmill): automate the generation of this file. -# -# CUJs should have: -# - id: The CUJs ID defined in the FlutterBench repository. -# - goal: The main CUJ text. -# - persona: The type of developer the CUJ speaks for from a list of ["The Tech Lead / Architect", "The App Developer", "The Plugin Developer", "Full-stack developer", "Hybrid (native + Flutter) developer"] -# - tasks: A list of tasks that a developer does to reach the CUJ goal. -- id: 0 - goal: Evaluate and select the technical stack, folder structure, state management, - and routing architecture for a project - persona: The Tech Lead / Architect - tasks: - - id: 1 - name: research-existing-options-available-architecture - task: Evaluate available architectural patterns, routing libraries, and state - management frameworks, documenting the rationale for the selected technology - stack. - - id: 2 - name: use-workspace-monorepo-repo-structure - task: Configure a multi-package Dart workspace or monorepo repository structure - to separate core domain logic from application UI features. -- id: 1 - goal: Enforce consistent code formatting, linting, and architectural standards - persona: The Tech Lead / Architect - tasks: - - id: 3 - name: compose-analysis-options-style-guide - task: Author comprehensive static analysis rules in "analysis_options.yaml" - and document architectural standards in a project style guide. - - id: 4 - name: ensure-codebase-uses-only-selected - task: Enforce that the codebase adheres strictly to documented architectural - decisions and state management patterns, avoiding unapproved approaches. -- id: 2 - goal: Manage dependency risks and audit third-party packages - persona: The Tech Lead / Architect - tasks: - - id: 5 - name: audit-third-party-pub-dev - task: Audit third-party pub.dev packages for license compliance, maintenance - activity, and security vulnerabilities before adoption. - - id: 6 - name: ensure-dependencies-are-installed-cli - task: Add project dependencies using official command-line package managers - rather than manually modifying configuration files. -- id: 3 - goal: Establish repository governance, branching conventions, code review standards, - and CI quality gates - persona: The Tech Lead / Architect - tasks: - - id: 7 - name: establish-repository-governance-standardize-branching - task: Establish repository governance policies to standardize branching models, - enforce peer code reviews, and automate CI quality gates. -- id: 4 - goal: Develop custom Dart CLI developer utilities and automation tools - persona: The App Developer - tasks: - - id: 8 - name: write-dart-cli-tool-generate - task: Develop a standalone Dart command-line utility that parses database schema - specifications and generates required boilerplate data access code. - - id: 9 - name: write-cli-tool-optimise-csv - task: Develop a Dart command-line utility to automate the parsing, validation, - and compression of CSV datasets and application resources. -- id: 5 - goal: Optimize application release builds for minimal bundle size - persona: The Tech Lead / Architect - tasks: - - id: 10 - name: analyze-size-analyze-size-devtools - task: Analyze application bundle composition and asset weight using command-line - size analysis tools and Flutter DevTools. - - id: 11 - name: enable-tree-shaking-obfuscation-split - task: Configure production build flags to enable code tree shaking, symbol obfuscation, - and split debug information. - - id: 12 - name: audit-compress-codebase-assets - task: Audit application resources to remove unused assets and compress images - and fonts for reduced download size. -- id: 6 - goal: Maintain accurate, up-to-date repository documentation and README guides - persona: The Tech Lead / Architect - tasks: - - id: 13 - name: review-update-readme-other-documentation - task: Audit and update repository documentation, including README guides and - architectural overviews, to align with recent codebase modifications. -- id: 7 - goal: Make an application accessible to all users - persona: The App Developer - tasks: - - id: 14 - name: evaluate-how-app-accessibility-is - task: Audit the application using Flutter DevTools and automated accessibility - inspection tools to identify compliance gaps. - - id: 15 - name: modify-app-add-semantic-labels - task: Refactor UI widgets to include descriptive Semantics properties and screen - reader labels for visually impaired users. - - id: 16 - name: modify-app-make-tappable-areas - task: Enforce minimum interactive touch target dimensions across all interactive - components to meet mobile accessibility standards. - - id: 17 - name: remove-fixed-text-scaler - task: Refactor text components to support dynamic system font scaling and remove - hardcoded text scale restrictions. - - id: 18 - name: add-high-contrast-color-themes - task: Implement high-contrast visual themes and color palettes to support users - with visual impairments. -- id: 8 - goal: Achieve comprehensive test coverage with unit, widget, and integration test - suites - persona: The App Developer - tasks: - - id: 19 - name: check-existing-test-coverage-percentage - task: Analyze current test coverage to identify untested code sections and - determine which parts of the application require additional test coverage. - - id: 20 - name: add-app-benchmarking-uses-binding - task: Implement automated performance benchmarking using binding.traceAction - to measure frame timing and verify that the 90th percentile execution duration - remains below defined latency thresholds. - - id: 21 - name: add-flutter-integration-tests-mobile - task: Develop end-to-end integration test suites using "package:integration_test" - to validate complete user journeys across mobile and web environments. -- id: 9 - goal: Diagnose and resolve layout overflow errors in UI component trees - persona: The App Developer - tasks: - - id: 22 - name: find-real-cause-ui-overflow - task: Diagnose and identify the root cause of layout overflow errors in the - UI component tree. - - id: 23 - name: fix-overflow-bug-with-proper-widgets - task: Refactor the layout using flexible scrolling or bounding widgets to resolve - the overflow error. - - id: 24 - name: write-widget-tests-edge-cases - task: Implement automated widget tests covering boundary conditions and large - data values to prevent regression of layout overflows. -- id: 10 - goal: Implement a structured routing and navigation system - persona: The App Developer - tasks: - - id: 25 - name: set-up-go-router-named - task: Configure declarative application routing using "package:go_router", implementing - named routes and dynamic URL path parameters. - - id: 26 - name: set-up-go-router-builder - task: Integrate "package:go_router_builder" and code generation to manage type-safe - route navigation and arguments. - - id: 27 - name: implement-deep-linking-trigger-deep - task: Configure platform-specific deep linking schemas and verify that external - links navigate correctly to target application screens. - - id: 28 - name: guard-routes-based-auth-state - task: Implement redirection guards within the routing configuration to restrict - access to authenticated user sessions. - - id: 29 - name: use-navigator-v1-route-does - task: Implement imperative navigation using standard Navigator 1.0 APIs for - simple internal modal dialogs and screen transitions. -- id: 11 - goal: Add a new UI screen to an existing application following established design - and architectural patterns - persona: The App Developer - tasks: - - id: 30 - name: add-new-screen-design-system - task: Add a new UI screen to the application that integrates with the existing - design system, routing architecture, and standard page structure. -- id: 12 - goal: Design responsive UI layouts that reflow cleanly across all window sizes and - device orientations - persona: The App Developer - tasks: - - id: 31 - name: define-central-breakpoints-m3-window - task: Define layout breakpoints based on Material Design 3 window size classes, - such as using compact layouts for widths under 600 logical pixels. - - id: 32 - name: use-mediaquery-sizeof-window-sizing - task: Refactor responsive sizing logic to use MediaQuery.sizeOf for global window - dimensions and LayoutBuilder for local widget constraint sizing, removing - hardcoded device-type checks. - - id: 33 - name: apply-safearea-notches-insets - task: Wrap visual layouts in SafeArea widgets to prevent content from obscuring - system status bars, display notches, and physical screen bezels. - - id: 34 - name: don-t-portrait-lock-support - task: Configure the application to support both portrait and landscape orientations, - verifying smooth UI reflow during device rotation. - - id: 35 - name: cap-content-width-large-windows - task: Constrain maximum content width on wide desktop or tablet displays using - BoxConstraints or by dynamically transitioning from ListView to GridView layouts. - - id: 36 - name: handle-foldable-letterboxing-support-all - task: Optimize layouts for foldable devices and letterboxed display modes across - various screen postures and orientations. -- id: 13 - goal: Optimize application rendering and memory performance - persona: The App Developer - tasks: - - id: 37 - name: use-devtools-profile-rendering-performance - task: Profile application frame rendering times and rasterization metrics using - Flutter DevTools. - - id: 38 - name: hunt-down-memory-leaks - task: Diagnose and resolve application memory leaks and retained object graphs - using memory profiling tools. - - id: 39 - name: add-renderrepaintboundary-s-widget-tree - task: Refactor the widget hierarchy by inserting RenderRepaintBoundary widgets - around frequently animating components to isolate repaint regions. -- id: 14 - goal: Implement state restoration to preserve user state across application restarts - persona: The App Developer - tasks: - - id: 40 - name: add-state-restoration-functionality-app - task: Implement Flutter state restoration APIs using RestorationManager and - RestorationBucket to preserve interface navigation and scroll states across - process terminations. - - id: 41 - name: add-hydrated-versions-state-management - task: Integrate persistent state management libraries, such as "package:hydrated_bloc", - to automatically serialize and restore application state across application - restarts. -- id: 15 - goal: Implement offline-first data caching and synchronization - persona: The App Developer - tasks: - - id: 42 - name: add-local-caching-solution-be - task: Implement an offline-first repository pattern that caches remote server - data locally and synchronizes pending mutations when network connectivity - is restored. -- id: 16 - goal: Build interactive widget preview catalogs and isolated design system showcases - persona: The App Developer - tasks: - - id: 43 - name: create-interactive-website-every-widget - task: Develop a standalone interactive web catalog showcasing every UI component - and visual state available within the component library. - - id: 44 - name: add-widget-previews-for-components - task: Implement isolated widget preview configurations using the official Flutter - widget previewer tool for all UI components in the application. -- id: 17 - goal: Implement a customizable, reusable UI design system - persona: The App Developer - tasks: - - id: 45 - name: create-totally-custom-design-system - task: Build an independent UI design system from scratch without relying on - standard Material or Cupertino widget libraries. - - id: 46 - name: customise-material-design-system-fit - task: Customize and extend Material Design widgets and styling tokens to implement - a proprietary visual design system. - - id: 47 - name: customise-cupertino-design-system-fit - task: Customize and extend Cupertino widgets to implement a proprietary iOS-styled - visual design system. -- id: 18 - goal: Implement a consistent visual design theme and styling across an application - persona: The App Developer - tasks: - - id: 48 - name: create-theme-data-from-design-document - task: Implement application ThemeData configurations derived from specifications - in a design document. - - id: 49 - name: add-dark-mode-support - task: Implement dark mode theming and color scheme switching. - - id: 50 - name: change-dropdown-popup-buttons-styling - task: Customize visual styling for dropdown menus, popup dialogs, and interactive - buttons by extending central ThemeData configurations. -- id: 19 - goal: Implement custom gesture detection and pointer interactions - persona: The App Developer - tasks: - - id: 51 - name: create-custom-widget-detects-hover - task: Implement a custom interactive component that combines MouseRegion for - hover detection with GestureDetector or InkWell for touch and pointer interactions. -- id: 20 - goal: Implement custom widget animation states and transitions - persona: The App Developer - tasks: - - id: 52 - name: create-widget-uses-animationcontrollers-animate - task: Develop an explicit animated component using AnimationController to manage - custom state transitions and tween animations. - - id: 53 - name: add-tests-confirm-animation-logic - task: Implement automated widget tests to verify that animation state machines - and value transitions execute correctly. - - id: 54 - name: replace-static-widget-animated-version - task: Refactor static UI components to use implicit animation widgets, such - as AnimatedContainer and AnimatedOpacity, for smooth state transitions. - - id: 55 - name: use-hero-transition-animations-between - task: Implement shared element routing transitions across navigation boundaries - using Hero animation widgets. -- id: 21 - goal: Integrate rich animated graphics and shaders into an application - persona: The App Developer - tasks: - - id: 56 - name: use-rive-lottie-or-some - task: Integrate animation libraries, such as "package:rive" or "package:lottie", - to render rich vector animations within the application. - - id: 57 - name: use-shaders-animate-things-app - task: Implement fragment shaders using GLSL shader programs to render custom - GPU-accelerated visual effects and animations. -- id: 22 - goal: Integrate interactive data visualization and charting libraries - persona: The App Developer - tasks: - - id: 58 - name: find-list-available-libraries-charts - task: Research available charting libraries on pub.dev and evaluate which packages - support the required chart types and features for the use case. - - id: 59 - name: install-chart-library-supports-bar - task: Install a third-party charting package that supports interactive bar charts - using official command-line tools ("flutter pub add") rather than manually - editing configuration files. - - id: 60 - name: implement-library-application-show-chart - task: Integrate the charting library into the application dashboard to render - interactive data visualizations, implementing automated widget tests to verify - chart rendering. -- id: 23 - goal: Configure package dependency overrides using Git repositories or local filesystem - paths - persona: The App Developer - tasks: - - id: 61 - name: add-dependency-override-git - task: Configure package dependency overrides in "pubspec.yaml" to target a specific - remote Git repository and subdirectory path, verifying that all automated - tests pass. - - id: 62 - name: add-dependency-override-local - task: Configure package dependency overrides in "pubspec.yaml" to link against - a local filesystem package path, verifying that all automated tests pass. -- id: 24 - goal: Build an application that renders Material UI on Android and Cupertino UI - on iOS - persona: The App Developer - tasks: - - id: 63 - name: create-new-app-android-ios - task: Create a new cross-platform Flutter application targeting both Android - and iOS. - - id: 64 - name: add-adaptive-material-cupertino-layouts - task: Implement navigation layouts that adaptively render Material Design components - on Android and Cupertino components on iOS. -- id: 25 - goal: Implement performant scrolling layouts for long-form content - persona: The App Developer - tasks: - - id: 65 - name: identify-overflow-refactor-layout-use - task: Diagnose vertical layout overflow errors and refactor the component hierarchy - to use SingleChildScrollView. - - id: 66 - name: migrate-customscrollview-slivers-more-complex - task: Refactor standard scroll views to use CustomScrollView and Sliver components - for advanced scrolling effects and header animations. - - id: 67 - name: have-long-list-items-variate - task: Implement programmatic scroll-to-index functionality for variable-height - item lists using scroll controllers or item alignment libraries. -- id: 26 - goal: Build intuitive, validated user forms with polished input UX - persona: The App Developer - tasks: - - id: 68 - name: auto-focus-first-invalid-field - task: Implement form validation logic that automatically transfers focus to - the first invalid input field when a user submits an incomplete form. - - id: 69 - name: add-floating-label-behavior-textfields - task: Configure text input fields with floating label behavior and error messaging - using InputDecoration properties. -- id: 27 - goal: Build an application that communicates with a REST API - persona: The App Developer - tasks: - - id: 70 - name: fetch-parse-json-http-or - task: Implement network calls to fetch and parse JSON payloads from a REST API - using "package:http" or "package:dio". - - id: 71 - name: handle-errors-timeouts-loading-states - task: Implement robust error handling, network request timeout management, and - UI loading state indicators. - - id: 72 - name: get-rid-ui-jank-due - task: Offload JSON serialization and deserialization to background worker isolates - to prevent main thread stutter and UI frame drops. - - id: 73 - name: if-api-has-spec-use - task: Generate type-safe API client code and data models automatically from - an OpenAPI specification using code generation tools. -- id: 28 - goal: Implement and evaluate state management architectures - persona: The App Developer - tasks: - - id: 74 - name: use-setstate-manage-state - task: Implement application state management using standard StatefulWidget and - setState mechanisms. - - id: 75 - name: use-provider-manage-state - task: Implement application state management using "package:provider" for dependency - injection and reactive updates. - - id: 76 - name: use-riverpod-manage-state-maybe - task: Implement application state management using "package:riverpod", optionally - incorporating "package:flutter_hooks" and code generation. - - id: 77 - name: use-bloc-cubit-manage-state - task: Implement application state management using the Business Logic Component - (BLoC) and Cubit patterns from "package:flutter_bloc". - - id: 78 - name: use-hooks-manage-state - task: Implement application state management using "package:flutter_hooks" to - manage widget lifecycle and local state with composable hook functions. - - id: 79 - name: use-rxdart-manage-state - task: Implement stream-based application state management using reactive programming - primitives from "package:rxdart". -- id: 29 - goal: Implement local data persistence in an application - persona: The App Developer - tasks: - - id: 80 - name: set-up-shared-preferences-package - task: Implement local persistence for simple key-value data and user preferences - using "package:shared_preferences". - - id: 81 - name: set-up-sqlite-sqflite-or - task: Implement a structured local relational database using "package:sqflite" - or "package:drift". - - id: 82 - name: set-up-secure-storage-store - task: Implement encrypted local storage for sensitive user data and authentication - tokens using "package:flutter_secure_storage". - - id: 83 - name: use-path-provider-locate-application - task: Integrate "package:path_provider" to locate platform-specific filesystem - directories for application documents and temporary files. -- id: 30 - goal: Offload CPU-intensive computation to background isolates to prevent UI freezing - persona: The App Developer - tasks: - - id: 84 - name: offload-cpu-intensive-work-isolate - task: Offload computationally expensive synchronous operations to background - worker isolates using Isolate.run. - - id: 85 - name: use-compute-one-shot-tasks - task: Execute one-shot background computations using the top-level compute function - to prevent UI thread blocking. - - id: 86 - name: set-up-long-lived-isolate - task: Implement a long-lived background isolate communicating via ReceivePort - and SendPort message passing to handle continuous asynchronous processing. -- id: 31 - goal: Adopt code generation tools to reduce boilerplate for models and immutable - data classes - persona: The App Developer - tasks: - - id: 87 - name: set-up-build-runner-json - task: Configure "package:build_runner" and "package:json_serializable" to generate - type-safe JSON serialization code for data models. - - id: 88 - name: use-freezed-immutable-data-classes - task: Integrate "package:freezed" to generate immutable data classes, union - types, and value equality boilerplate. - - id: 89 - name: use-build-verify-ci-cd - task: Configure automated CI/CD pipelines using "package:build_verify" to ensure - generated source code is synchronized with existing data models. -- id: 32 - goal: Diagnose and resolve native platform interop bugs and channel communication - errors - persona: The Plugin Developer - tasks: - - id: 90 - name: analyze-codebase-try-find-cause - task: Analyze native platform channel implementations and Dart bindings to diagnose - the root cause of platform communication failures. - - id: 91 - name: add-debug-logs-perform-test - task: Instrument platform interop channels with diagnostic logging and execute - test runs to isolate native execution errors. - - id: 92 - name: fix-issue - task: Refactor native host code and Dart channel handlers to resolve platform - interop exceptions and restore reliable communication. -- id: 33 - goal: Design a unified, well-documented cross-platform API surface for plugin consumers - persona: The Plugin Developer - tasks: - - id: 93 - name: design-intuitive-well-documented-unified - task: Design a unified, documented Dart API that abstracts native iOS and Android - implementation differences for plugin consumers. - - id: 94 - name: encapsulate-platform-interface-code - task: Structure the plugin package to ensure public APIs encapsulate and hide - internal platform interface implementations. - - id: 95 - name: generate-html-api-documentation - task: Generate HTML API documentation from inline dartdoc comments using command-line - tools to verify public API presentation. -- id: 34 - goal: Implement automated native platform test suites (XCTest, Espresso, JUnit) - to prevent OS upgrade regressions - persona: The Plugin Developer - tasks: - - id: 96 - name: write-automated-tests-validate-both - task: Implement automated test suites validating Dart logic and native implementations - using XCTest for iOS and JUnit or Espresso for Android. -- id: 35 - goal: Maintain high-quality published packages with rigorous semantic versioning, - detailed changelogs, and responsiveness to SDK updates - persona: The Plugin Developer - tasks: - - id: 97 - name: manage-versions-write-changelogs-get - task: Manage semantic versioning, maintain changelogs, achieve high pub.dev - quality scores, and publish package releases compatible with current Flutter - SDK versions. -- id: 36 - goal: Adopt optimal native interop mechanisms (FFI, Pigeon, JS Interop) based on - performance and platform requirements - persona: The Plugin Developer - tasks: - - id: 98 - name: migrate-pigeon-based-platform-channels - task: Migrate Pigeon-based platform channels to Foreign Function Interface (FFI) - bindings for performance-critical or synchronous native calls. - - id: 99 - name: replace-manual-platform-channel-boilerplate - task: Replace manual platform channel boilerplate with type-safe message passing - code generated by "package:pigeon". - - id: 100 - name: create-type-safe-bindings-between - task: Create type-safe bindings between Dart and JavaScript using "dart:js_interop" - and extension types to integrate with browser APIs and external JavaScript - libraries. - - id: 101 - name: check-all-resources-used-native - task: Ensure native memory allocations (malloc, calloc, FFI structs, OpenGL - handles, file descriptors) are properly released using NativeFinalizer and - the Finalizable interface. -- id: 37 - goal: Extend an existing federated plugin architecture to support a new target platform - persona: The Plugin Developer - tasks: - - id: 102 - name: add-implementation-new-platform-app - task: Implement native platform support for an additional operating system within - an existing plugin, verifying that all application-facing integration tests - pass. - - id: 103 - name: test-new-plugin-through-app - task: Verify the new platform implementation by running automated test suites - against the application-facing package. - - id: 104 - name: map-c-language-types-integers - task: Map C language data types (integers, structs, and pointers) to "dart:ffi" - types, utilizing AbiSpecificInteger for platform-dependent type sizing. - - id: 105 - name: refactor-plugin-split-it-into - task: Refactor a monolithic plugin into a federated architecture consisting - of separate application-facing, platform interface, and platform implementation - packages. -- id: 38 - goal: Implement automated cross-platform integration tests for plugins using modern - testing frameworks - persona: The Plugin Developer - tasks: - - id: 106 - name: use-patrol-package-be-able - task: Implement automated cross-platform integration tests using "package:patrol" - to verify plugin functionality across native environments. - - id: 107 - name: write-ci-cd-pipeline-test - task: Configure an automated CI/CD pipeline to execute plugin integration tests - on every pull request and prior to release publication. - - id: 108 - name: write-integration-tests-has-100% - task: Develop comprehensive integration test suites that achieve full test coverage - of native platform plugin functionality. -- id: 39 - goal: Implement user authentication flows and UI - persona: The Full Stack Developer - tasks: - - id: 109 - name: plan-auth-provider-design-system - task: Define the architectural requirements, user experience flows, and edge-case - handling for application authentication. - - id: 110 - name: code-auth-flow - task: Implement secure user authentication and registration workflows connecting - the frontend UI to the authentication service. - - id: 111 - name: test-all-auth-flows - task: Implement automated unit, widget, and integration test suites to verify - all authentication and session management workflows. -- id: 40 - goal: Integrate push notification services into an application - persona: The Full Stack Developer - tasks: - - id: 112 - name: add-firebase-cloud-messaging - task: Integrate "package:firebase_messaging" to enable push notifications across - mobile and web platforms. - - id: 113 - name: handle-foreground-background-terminated-states - task: Implement notification event listeners and handlers for foreground, background, - and terminated application lifecycle states. -- id: 41 - goal: Integrate third-party authentication providers into an application - persona: The Full Stack Developer - tasks: - - id: 114 - name: add-google-sign - task: Integrate "package:google_sign_in" to enable single sign-on authentication. - - id: 115 - name: handle-sign-sign-out-flows - task: Implement complete authentication state machines managing sign-in, sign-out, - session persistence, and OAuth token refresh workflows. -- id: 42 - goal: Integrate cloud file storage into an application - persona: The Full Stack Developer - tasks: - - id: 116 - name: add-firebase-storage - task: Integrate "package:firebase_storage" into the application to enable cloud - storage capabilities. - - id: 117 - name: implement-upload-download-delete - task: Implement user flows and repository methods to upload, download, and delete - cloud storage files. -- id: 43 - goal: Integrate crash reporting, error tracking, and production telemetry - persona: The Full Stack Developer - tasks: - - id: 118 - name: add-crashlytics - task: Integrate Firebase Crashlytics ("package:firebase_crashlytics") to capture - and monitor real-time fatal exception reports. - - id: 119 - name: add-custom-log-events-non - task: Implement custom error logging and non-fatal exception tracking to record - application telemetry and diagnostic metadata. -- id: 44 - goal: Integrate AWS Amplify authentication, cloud storage, and backend APIs into - an application - persona: The Full Stack Developer - tasks: - - id: 120 - name: integrate-amplify-auth-+-storage - task: Integrate AWS Amplify authentication, cloud storage, and API services - into the application architecture. -- id: 45 - goal: Integrate Supabase authentication, database services, and real-time subscriptions - into an application - persona: The Full Stack Developer - tasks: - - id: 121 - name: integrate-supabase-auth-database - task: Integrate Supabase authentication, relational database services, and real-time - data subscriptions into the application architecture. -- id: 46 - goal: Build a full-stack Dart web server backend with shared data models between - frontend and backend - persona: The Full Stack Developer - tasks: - - id: 122 - name: create-web-server-shelf - task: Develop a backend web server and HTTP API routing layer using "package:shelf". - - id: 123 - name: create-cloud-function-upload-firebase - task: Implement server-side logic or Google Cloud Functions to process file - uploads and store metadata in Firebase. - - id: 124 - name: decide-best-repo-structure-according - task: Design and implement a shared monorepo workspace structure to allow seamless - data model reuse between Dart frontend and backend services. -- id: 47 - goal: Implement API versioning and backward compatibility checks between frontend - applications and backend services - persona: The Full Stack Developer - tasks: - - id: 125 - name: create-ci-cd-pipeline-confirm - task: Configure automated CI/CD deployment pipelines to verify that the target - backend API version is active prior to releasing client applications. - - id: 126 - name: create-screen-app-app-version - task: Implement a dedicated application deprecation screen that informs users - when their client version is no longer supported by backend API services. -- id: 48 - goal: Develop Google Cloud Functions in Dart using the Genkit SDK - persona: The Full Stack Developer - tasks: - - id: 127 - name: write-google-cloud-function-dart - task: Develop and deploy serverless Google Cloud Functions written in Dart using - the Genkit framework ("package:genkit"). -- id: 49 - goal: Add internationalization (i18n) and localization (l10n) support to an application - persona: The App Developer - tasks: - - id: 128 - name: add-required-languages-locales-app - task: Configure supported languages and regional locales within the application - localization settings. - - id: 129 - name: confirm-default-flutter-ways-i18n - task: Verify that the codebase implements standard Flutter internationalization - practices using ARB files and generated localization delegates. -- id: 50 - goal: Embed interactive Flutter applications and widgets within existing HTML or - React web pages - persona: The Hybrid (Native + Flutter) Developer - tasks: - - id: 130 - name: create-website-jaspr - task: Build a server-rendered or static website in Dart using the Jaspr web - framework ("package:jaspr"). - - id: 131 - name: add-flutter-app-react-app - task: Embed a compiled Flutter web application as an interactive component within - an existing React web application. - - id: 132 - name: add-many-flutter-widgets-across - task: Embed multiple interactive Flutter widgets across a standard HTML web - page, utilizing Flutter multi-view mode to optimize rendering performance - and resource consumption. -- id: 51 - goal: Integrate Flutter modules into existing native Android and iOS applications - using Add-to-app - persona: The Hybrid (Native + Flutter) Developer - tasks: - - id: 133 - name: add-flutter-engine-view-android - task: Integrate a cached FlutterEngine and FlutterActivity into an existing - native Android application using Flutter Add-to-app workflows. - - id: 134 - name: add-flutter-engine-view-ios - task: Integrate a cached FlutterEngine and FlutterViewController into an existing - native iOS application using Flutter Add-to-app workflows. -- id: 52 - goal: Implement seamless cross-layer navigation and state synchronization between - native host apps and embedded Flutter modules - persona: The Hybrid (Native + Flutter) Developer - tasks: - - id: 135 - name: cache-pre-warm-flutter-engine - task: Configure native host applications to pre-warm and cache the FlutterEngine - during application startup to eliminate initialization latency. - - id: 136 - name: manage-complex-navigation-stacks-where - task: Implement bidirectional navigation stacks where users transition between - native Swift screens and embedded Flutter modules, ensuring native gesture - back-swipes behave naturally. - - id: 137 - name: securely-pass-active-user-session - task: Synchronize active session tokens, visual theme preferences, and user - state from the host native application into embedded Flutter modules to provide - a seamless user experience. -- id: 53 - goal: Build platform-specific home screen widgets (for iOS WidgetKit and Android) - that share data with the host application - persona: The Hybrid (Native + Flutter) Developer - tasks: - - id: 138 - name: set-up-home-screen-widget - task: Scaffold and configure native home screen widget extensions for iOS using - WidgetKit and for Android using AppWidgets. - - id: 139 - name: share-data-between-flutter-app - task: Implement shared local storage using App Groups on iOS and SharedPreferences - on Android to synchronize data between the Flutter application and native - widgets. - - id: 140 - name: update-widget-data-flutter - task: Trigger programmatic background updates and timeline reloads for native - home screen widgets directly from Dart application logic. -- id: 54 - goal: Identify and refactor architectural anti-patterns in the codebase - persona: The App Developer - tasks: - - id: 141 - name: analyze-codebase-anti-patterns - task: Analyze the codebase to identify and refactor architectural anti-patterns, - such as building complex widget trees inside helper methods rather than separate - widget classes. -- id: 55 - goal: Audit and migrate codebases away from deprecated frameworks, libraries, and - SDK APIs - persona: The Tech Lead / Architect - tasks: - - id: 142 - name: analyze-codebase-deprecated-api - task: Analyze the codebase to identify and migrate deprecated API usage, including - Material Design 2 components, direct window references in "dart:ui", and legacy - ThemeData styling properties. -- id: 56 - goal: Automate multi-flavor application build, configuration, and distribution pipelines - persona: The Tech Lead / Architect - tasks: - - id: 143 - name: config-flavors-app-different-naming - task: Configure multi-flavor build schemes across iOS and Android to support - distinct application names, bundle identifiers, and launcher icons for staging - and production environments. - - id: 144 - name: write-ci-cd-pipeline-deploy - task: Implement an automated CI/CD distribution pipeline to build and deploy - application binaries to internal testing tracks or distribution services. -- id: 57 - goal: Upgrade and migrate legacy Flutter applications to the latest SDK version - persona: The Tech Lead / Architect - tasks: - - id: 145 - name: audit-and-upgrade-flutter-sdk - task: Audit the local Flutter SDK installation and upgrade safely using version - management tools ("fvm") or system package managers. - - id: 146 - name: run-dart-fix-migration-tool - task: Execute "dart fix" to update deprecated syntax and resolve breaking API - changes across the codebase. -- id: 58 - goal: Extend an existing application to support an additional target platform - persona: The Tech Lead / Architect - tasks: - - id: 147 - name: analyze-repo-check-which-features - task: Audit existing codebase capabilities and third-party plugins to determine - feature compatibility with the target platform. - - id: 148 - name: verify-proper-command-is-used - task: Execute official platform scaffolding commands to generate target platform - projects and verify dependency compatibility. -- id: 59 - goal: Implement new application features utilizing modern language capabilities - and defensive coding practices - persona: The App Developer - tasks: - - id: 149 - name: verify-assert-calls-all-passed - task: Enforce defensive coding by adding runtime assert statements to validate - constructor and function parameter boundaries. - - id: 150 - name: use-record-structure-type-function - task: Refactor function signatures to return structured, type-safe multiple - values using modern Dart Record types. -- id: 60 - goal: Implement custom canvas drawing and custom painters for specialized UI components - persona: The App Developer - tasks: - - id: 151 - name: create-widget-displays-custom-pattern - task: Implement a custom component utilizing CustomPaint and Canvas primitives - to render specialized graphics above or below child widget layers. -- id: 61 - goal: Design and develop a new cross-platform plugin from scratch, selecting the - appropriate native interop mechanism - persona: The Plugin Developer - tasks: - - id: 152 - name: investigate-should-ffi-or-methodchannels - task: Evaluate whether Foreign Function Interface (FFI) bindings or asynchronous - MethodChannels provide the optimal architectural foundation for a new cross-platform - plugin. -- id: 62 - goal: Set up and configure a complete cross-platform Flutter development environment - persona: The App Developer - tasks: - - id: 153 - name: install-configure-xcode-command-line - task: Install and configure Xcode, command-line tools, and CocoaPods on a macOS - development environment to compile for all supported Flutter target platforms. - - id: 154 - name: sets-up-windows-environment-flutter - task: Set up and configure a Windows development environment for Flutter with - necessary dependencies to compile for all supported non-Apple target platforms. - - id: 155 - name: sets-up-linux-environment-flutter - task: Set up and configure a Linux development environment for Flutter with - necessary dependencies to compile for all supported non-Apple target platforms. -- id: 63 - goal: Build a responsive Flutter Web frontend for an Enterprise Resource Planning - (ERP) system - persona: The App Developer - tasks: - - id: 156 - name: create-flutter-web-app-has - task: Develop a responsive Flutter web frontend that dynamically adapts between - desktop browser layouts and mobile web layouts. - - id: 157 - name: use-proper-url-path-strategy - task: Configure the web URL routing strategy (hash-based or path-based) according - to target web hosting platform requirements. - - id: 158 - name: use-wasm-if-possible - task: Configure the web build pipeline to compile to WebAssembly (Wasm) for - high-performance browser execution. - - id: 159 - name: use-pwa-web-app-if - task: Configure Progressive Web App (PWA) manifest and service worker features - to enable offline support and desktop installation. - - id: 160 - name: use-package:web-dart:js-interop-interact - task: Implement browser API integrations and JavaScript interop using "package:web" - and modern Dart type-safe JS interop mechanisms. -- id: 64 - goal: Build adaptive UI layouts that dynamically adjust to platform conventions - and input methods (touch, mouse, keyboard, stylus) - persona: The App Developer - tasks: - - id: 161 - name: switch-nav-window-size:-bottom - task: Implement adaptive navigation that transitions between a bottom navigation - bar on compact screens and a side NavigationRail on expanded displays, sharing - routing destinations. - - id: 162 - name: target-android-tier-3-mouse - task: Optimize the interface for mouse and stylus input by utilizing Material - Design 3 components with built-in hover and focus states. - - id: 163 - name: scroll-wheel-custom-scrollables-listener - task: Refactor custom scrollable components using Listener widgets to support - mouse scroll wheel and trackpad navigation. - - id: 164 - name: tab-traversal-+-visible-focus - task: Implement keyboard tab navigation and visible focus highlights on custom - interactive components using FocusableActionDetector and FocusTraversalGroup. - - id: 165 - name: keyboard-shortcuts-shortcuts-actions-disable - task: Configure application-wide keyboard shortcuts using Shortcuts and Actions - widgets, ensuring shortcuts are disabled during text input. - - id: 166 - name: visualdensity-switched-input-mode-hit - task: Adjust widget VisualDensity dynamically based on active input mode to - optimize touch target sizes versus mouse precision sizing. -- id: 65 - goal: Migrate application architecture between state management solutions - persona: The App Developer - tasks: - - id: 167 - name: replace-setstate-riverpod - task: Refactor the codebase to use "package:riverpod" rather than StatefulWidgets. - - id: 168 - name: replace-setstate-provider - task: Refactor the codebase to use "package:provider" rather than StatefulWidgets. - - id: 169 - name: replace-inheritedwidget-provider - task: Refactor the codebase to use "package:provider" rather than custom InheritedWidgets. - - id: 170 - name: replace-provider-riverpod - task: Migrate existing state management from "package:provider" to "package:riverpod". - - id: 171 - name: replace-provider-bloc - task: Migrate existing state management from "package:provider" to the BLoC - ("package:flutter_bloc") architecture. - - id: 172 - name: replace-setstate-rxdart - task: Refactor the codebase to manage reactive state using "package:rxdart" - rather than StatefulWidgets. -- id: 66 - goal: Diagnose, debug, and resolve runtime exceptions and network defects - persona: The App Developer - tasks: - - id: 173 - name: reproduce-reported-defect-failing-test - task: Reproduce a reported defect in a failing test, then trace the root cause - using the Dart debugger and Flutter DevTools. - - id: 174 - name: fix-common-runtime-exceptions - task: Diagnose and resolve common runtime exceptions (null errors, late init - failures, RangeErrors, invalid setState calls). - - id: 175 - name: diagnose-fix-failed-network-request - task: Diagnose and resolve failed HTTP requests (non-200 status codes, timeouts, - JSON deserialization failures). -- id: 67 - goal: Refactor application code to improve modularity, component reusability, and - architectural maintainability - persona: The App Developer - tasks: - - id: 176 - name: extract-repeated-widget-code-into - task: Extract repeated widget trees into reusable components and consolidate - shared colors, spacing, and text styles into central theme constants. - - id: 177 - name: split-large-dart-class-into - task: Refactor large Dart classes into smaller units, separating business logic - from widget presentation. - - id: 178 - name: extract-shared-ui-logic-into-mixins - task: Extract shared UI behavior and state logic into reusable Dart mixins. -- id: 68 - goal: Call native platform APIs directly using MethodChannel and EventChannel implementations - on Android and iOS - persona: The App Developer - tasks: - - id: 179 - name: call-one-shot-native-method - task: Implement one-shot communication between Dart and native platforms via - MethodChannel (e.g., reading battery level or triggering haptic feedback), - writing handlers in Kotlin for Android and Swift for iOS. - - id: 180 - name: stream-continuous-native-events-into - task: Stream continuous native events into Dart via EventChannel (e.g., sensor - data or network connectivity state). diff --git a/sites/docs/src/data/sidenav/default.yml b/sites/docs/src/data/sidenav/default.yml index ad5decd077f..a8250456372 100644 --- a/sites/docs/src/data/sidenav/default.yml +++ b/sites/docs/src/data/sidenav/default.yml @@ -947,4 +947,3 @@ permalink: /reference/flutter-cli - title: API docs permalink: https://api.flutter.dev - diff --git a/sites/www/analysis_options.yaml b/sites/www/analysis_options.yaml index 6c0bdc59d96..a5c353a03c0 100644 --- a/sites/www/analysis_options.yaml +++ b/sites/www/analysis_options.yaml @@ -3,6 +3,7 @@ include: package:analysis_defaults/analysis.yaml analyzer: exclude: - build/** + - lib/src/data/raw_flutterbench_data/** language: strict-casts: true strict-inference: true diff --git a/sites/www/content/ai/flutterbench/cujs/index.md b/sites/www/content/ai/flutterbench/cujs/index.md new file mode 100644 index 00000000000..c98e61c672c --- /dev/null +++ b/sites/www/content/ai/flutterbench/cujs/index.md @@ -0,0 +1,8 @@ +--- +title: Flutter Critical User Journeys +bodyTags: interior flutterbench +description: Browse the catalog of canonical Flutter and Dart critical user journeys that the FlutterBench evaluations test. +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/index.md b/sites/www/content/ai/flutterbench/index.md new file mode 100644 index 00000000000..be59237af8e --- /dev/null +++ b/sites/www/content/ai/flutterbench/index.md @@ -0,0 +1,8 @@ +--- +title: FlutterBench Leaderboard +bodyTags: interior flutterbench +description: Benchmark results for AI coding agents on Dart and Flutter developer tasks. +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/methodology.md b/sites/www/content/ai/flutterbench/methodology.md new file mode 100644 index 00000000000..55744ca2afb --- /dev/null +++ b/sites/www/content/ai/flutterbench/methodology.md @@ -0,0 +1,8 @@ +--- +title: FlutterBench Methodology +bodyTags: interior flutterbench methodology +description: Detailed explanation of the FlutterBench harness, three-dimensional scoring rubric, and reproduction steps. +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/models/index.md b/sites/www/content/ai/flutterbench/models/index.md new file mode 100644 index 00000000000..33a03dba5cc --- /dev/null +++ b/sites/www/content/ai/flutterbench/models/index.md @@ -0,0 +1,8 @@ +--- +title: FlutterBench Models +bodyTags: interior flutterbench +description: Browse every model evaluated by FlutterBench, with per-model accuracy, cost, and latency across each Critical User Journey. +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/tasks/dart-build-cli-app.md b/sites/www/content/ai/flutterbench/tasks/dart-build-cli-app.md new file mode 100644 index 00000000000..dcaa255e2e6 --- /dev/null +++ b/sites/www/content/ai/flutterbench/tasks/dart-build-cli-app.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Task: Build Command-Line CLI App" +bodyTags: interior flutterbench +description: "Cross-model benchmark results and details for the Build Command-Line CLI App task." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/tasks/flutter-adaptive-material-cupertino.md b/sites/www/content/ai/flutterbench/tasks/flutter-adaptive-material-cupertino.md new file mode 100644 index 00000000000..0fe99a7aec0 --- /dev/null +++ b/sites/www/content/ai/flutterbench/tasks/flutter-adaptive-material-cupertino.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Task: Adaptive Material & Cupertino UI" +bodyTags: interior flutterbench +description: "Cross-model benchmark results and details for the Adaptive Material & Cupertino UI task." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/tasks/flutter-custom-render-object.md b/sites/www/content/ai/flutterbench/tasks/flutter-custom-render-object.md new file mode 100644 index 00000000000..847b2f98d9b --- /dev/null +++ b/sites/www/content/ai/flutterbench/tasks/flutter-custom-render-object.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Task: Custom RenderObject & Canvas" +bodyTags: interior flutterbench +description: "Cross-model benchmark results and details for the Custom RenderObject & Canvas task." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/tasks/flutter-manage-state-with-bloc.md b/sites/www/content/ai/flutterbench/tasks/flutter-manage-state-with-bloc.md new file mode 100644 index 00000000000..fe0d0d7e774 --- /dev/null +++ b/sites/www/content/ai/flutterbench/tasks/flutter-manage-state-with-bloc.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Task: Manage State with BLoC" +bodyTags: interior flutterbench +description: "Cross-model benchmark results and details for the Manage State with BLoC task." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/tasks/flutter-offline-sync-sqlite.md b/sites/www/content/ai/flutterbench/tasks/flutter-offline-sync-sqlite.md new file mode 100644 index 00000000000..2910379a29a --- /dev/null +++ b/sites/www/content/ai/flutterbench/tasks/flutter-offline-sync-sqlite.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Task: Offline SQLite Sync Repository" +bodyTags: interior flutterbench +description: "Cross-model benchmark results and details for the Offline SQLite Sync Repository task." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/tasks/index.md b/sites/www/content/ai/flutterbench/tasks/index.md new file mode 100644 index 00000000000..63ba3a387b0 --- /dev/null +++ b/sites/www/content/ai/flutterbench/tasks/index.md @@ -0,0 +1,8 @@ +--- +title: FlutterBench Tasks & CUJs +bodyTags: interior flutterbench +description: Explore Critical User Journeys (CUJs) and task performance across AI coding models in FlutterBench. +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t01-claude-3-7-sonnet.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t01-claude-3-7-sonnet.md new file mode 100644 index 00000000000..a04b3c8fb89 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t01-claude-3-7-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t01-claude-3-7-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t01-claude-3-7-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t06-claude-3-5-sonnet.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t06-claude-3-5-sonnet.md new file mode 100644 index 00000000000..7d46d6caa7c --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t06-claude-3-5-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t06-claude-3-5-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t06-claude-3-5-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t11-claude-3-5-haiku.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t11-claude-3-5-haiku.md new file mode 100644 index 00000000000..db39c3a06b2 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t11-claude-3-5-haiku.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t11-claude-3-5-haiku" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t11-claude-3-5-haiku." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t16-o3.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t16-o3.md new file mode 100644 index 00000000000..b13de38ca72 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t16-o3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t16-o3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t16-o3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t21-gpt-5.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t21-gpt-5.md new file mode 100644 index 00000000000..73a805f354d --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t21-gpt-5.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t21-gpt-5" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t21-gpt-5." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t26-gpt-4o.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t26-gpt-4o.md new file mode 100644 index 00000000000..cc8355c9b1a --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t26-gpt-4o.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t26-gpt-4o" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t26-gpt-4o." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t31-gpt-4o-mini.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t31-gpt-4o-mini.md new file mode 100644 index 00000000000..1d7ed1218c5 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t31-gpt-4o-mini.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t31-gpt-4o-mini" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t31-gpt-4o-mini." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t36-gemini-35-pro.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t36-gemini-35-pro.md new file mode 100644 index 00000000000..e2d7b4bd52c --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t36-gemini-35-pro.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t36-gemini-35-pro" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t36-gemini-35-pro." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t41-gemini-35-flash.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t41-gemini-35-flash.md new file mode 100644 index 00000000000..5878f4d8884 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t41-gemini-35-flash.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t41-gemini-35-flash" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t41-gemini-35-flash." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t46-gemini-31-flash-lite.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t46-gemini-31-flash-lite.md new file mode 100644 index 00000000000..88989ef7585 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t46-gemini-31-flash-lite.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t46-gemini-31-flash-lite" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t46-gemini-31-flash-lite." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t51-deepseek-r1.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t51-deepseek-r1.md new file mode 100644 index 00000000000..e53c4a633c8 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t51-deepseek-r1.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t51-deepseek-r1" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t51-deepseek-r1." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t56-deepseek-v3.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t56-deepseek-v3.md new file mode 100644 index 00000000000..39cddd1337b --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t56-deepseek-v3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t56-deepseek-v3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t56-deepseek-v3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t61-deepseek-coder-v2.md b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t61-deepseek-coder-v2.md new file mode 100644 index 00000000000..81fed073e3a --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/dart-build-cli-app__t61-deepseek-coder-v2.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: dart-build-cli-app__t61-deepseek-coder-v2" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Build Command-Line CLI App trial dart-build-cli-app__t61-deepseek-coder-v2." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet.md new file mode 100644 index 00000000000..50170564633 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet.md new file mode 100644 index 00000000000..673648102d5 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku.md new file mode 100644 index 00000000000..6bca3a8f078 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t14-claude-3-5-haiku" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t14-claude-3-5-haiku." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t19-o3.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t19-o3.md new file mode 100644 index 00000000000..48cd2eb9c3a --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t19-o3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t19-o3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t19-o3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t24-gpt-5.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t24-gpt-5.md new file mode 100644 index 00000000000..058a9c8bf6f --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t24-gpt-5.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t24-gpt-5" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t24-gpt-5." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t29-gpt-4o.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t29-gpt-4o.md new file mode 100644 index 00000000000..543b1e47e03 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t29-gpt-4o.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t29-gpt-4o" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t29-gpt-4o." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t34-gpt-4o-mini.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t34-gpt-4o-mini.md new file mode 100644 index 00000000000..fff300086ef --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t34-gpt-4o-mini.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t34-gpt-4o-mini" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t34-gpt-4o-mini." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t39-gemini-35-pro.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t39-gemini-35-pro.md new file mode 100644 index 00000000000..10af0d94ab2 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t39-gemini-35-pro.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t39-gemini-35-pro" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t39-gemini-35-pro." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t44-gemini-35-flash.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t44-gemini-35-flash.md new file mode 100644 index 00000000000..064d95d50db --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t44-gemini-35-flash.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t44-gemini-35-flash" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t44-gemini-35-flash." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite.md new file mode 100644 index 00000000000..c0be7a492df --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t54-deepseek-r1.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t54-deepseek-r1.md new file mode 100644 index 00000000000..19cd4a3f0d9 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t54-deepseek-r1.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t54-deepseek-r1" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t54-deepseek-r1." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t59-deepseek-v3.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t59-deepseek-v3.md new file mode 100644 index 00000000000..3e40644264f --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t59-deepseek-v3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t59-deepseek-v3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t59-deepseek-v3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2.md b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2.md new file mode 100644 index 00000000000..ae18b1f7f62 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-adaptive-material-cupertino__t64-deepseek-coder-v2" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Adaptive Material & Cupertino UI trial flutter-adaptive-material-cupertino__t64-deepseek-coder-v2." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t05-claude-3-7-sonnet.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t05-claude-3-7-sonnet.md new file mode 100644 index 00000000000..1a2342363ad --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t05-claude-3-7-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t05-claude-3-7-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t05-claude-3-7-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t10-claude-3-5-sonnet.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t10-claude-3-5-sonnet.md new file mode 100644 index 00000000000..1eefda000e3 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t10-claude-3-5-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t10-claude-3-5-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t10-claude-3-5-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t15-claude-3-5-haiku.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t15-claude-3-5-haiku.md new file mode 100644 index 00000000000..c64ad3b6db1 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t15-claude-3-5-haiku.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t15-claude-3-5-haiku" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t15-claude-3-5-haiku." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t20-o3.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t20-o3.md new file mode 100644 index 00000000000..83f8e347dfc --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t20-o3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t20-o3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t20-o3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t25-gpt-5.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t25-gpt-5.md new file mode 100644 index 00000000000..0844b641ae8 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t25-gpt-5.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t25-gpt-5" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t25-gpt-5." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t30-gpt-4o.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t30-gpt-4o.md new file mode 100644 index 00000000000..ce433bfb52f --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t30-gpt-4o.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t30-gpt-4o" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t30-gpt-4o." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t35-gpt-4o-mini.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t35-gpt-4o-mini.md new file mode 100644 index 00000000000..029ef52fcdd --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t35-gpt-4o-mini.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t35-gpt-4o-mini" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t35-gpt-4o-mini." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t40-gemini-35-pro.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t40-gemini-35-pro.md new file mode 100644 index 00000000000..1f9d478de2e --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t40-gemini-35-pro.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t40-gemini-35-pro" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t40-gemini-35-pro." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t45-gemini-35-flash.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t45-gemini-35-flash.md new file mode 100644 index 00000000000..892c4318b72 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t45-gemini-35-flash.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t45-gemini-35-flash" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t45-gemini-35-flash." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t50-gemini-31-flash-lite.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t50-gemini-31-flash-lite.md new file mode 100644 index 00000000000..2c6aa1d7720 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t50-gemini-31-flash-lite.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t50-gemini-31-flash-lite" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t50-gemini-31-flash-lite." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t55-deepseek-r1.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t55-deepseek-r1.md new file mode 100644 index 00000000000..89c9a257187 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t55-deepseek-r1.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t55-deepseek-r1" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t55-deepseek-r1." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t60-deepseek-v3.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t60-deepseek-v3.md new file mode 100644 index 00000000000..9c5782abb6f --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t60-deepseek-v3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t60-deepseek-v3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t60-deepseek-v3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t65-deepseek-coder-v2.md b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t65-deepseek-coder-v2.md new file mode 100644 index 00000000000..cb6bcc7e594 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-custom-render-object__t65-deepseek-coder-v2.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-custom-render-object__t65-deepseek-coder-v2" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Custom RenderObject & Canvas trial flutter-custom-render-object__t65-deepseek-coder-v2." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet.md new file mode 100644 index 00000000000..0156d7d0262 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t02-claude-3-7-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t02-claude-3-7-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet.md new file mode 100644 index 00000000000..0d06e46101c --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t07-claude-3-5-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t07-claude-3-5-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t12-claude-3-5-haiku.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t12-claude-3-5-haiku.md new file mode 100644 index 00000000000..ca93a556444 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t12-claude-3-5-haiku.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t12-claude-3-5-haiku" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t12-claude-3-5-haiku." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t17-o3.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t17-o3.md new file mode 100644 index 00000000000..c128b5d202f --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t17-o3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t17-o3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t17-o3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t22-gpt-5.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t22-gpt-5.md new file mode 100644 index 00000000000..3d8bc7fd951 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t22-gpt-5.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t22-gpt-5" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t22-gpt-5." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t27-gpt-4o.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t27-gpt-4o.md new file mode 100644 index 00000000000..8a3a0caaa62 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t27-gpt-4o.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t27-gpt-4o" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t27-gpt-4o." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t32-gpt-4o-mini.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t32-gpt-4o-mini.md new file mode 100644 index 00000000000..abb25cc092f --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t32-gpt-4o-mini.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t32-gpt-4o-mini" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t32-gpt-4o-mini." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t37-gemini-35-pro.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t37-gemini-35-pro.md new file mode 100644 index 00000000000..39bb25f7ba9 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t37-gemini-35-pro.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t37-gemini-35-pro" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t37-gemini-35-pro." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t42-gemini-35-flash.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t42-gemini-35-flash.md new file mode 100644 index 00000000000..0e989744b86 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t42-gemini-35-flash.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t42-gemini-35-flash" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t42-gemini-35-flash." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite.md new file mode 100644 index 00000000000..e94fcacdfe7 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t47-gemini-31-flash-lite" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t47-gemini-31-flash-lite." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t52-deepseek-r1.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t52-deepseek-r1.md new file mode 100644 index 00000000000..2cf6773f062 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t52-deepseek-r1.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t52-deepseek-r1" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t52-deepseek-r1." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t57-deepseek-v3.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t57-deepseek-v3.md new file mode 100644 index 00000000000..70f7aaa7b20 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t57-deepseek-v3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t57-deepseek-v3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t57-deepseek-v3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t62-deepseek-coder-v2.md b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t62-deepseek-coder-v2.md new file mode 100644 index 00000000000..2aff33af35b --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-manage-state-with-bloc__t62-deepseek-coder-v2.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-manage-state-with-bloc__t62-deepseek-coder-v2" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Manage State with BLoC trial flutter-manage-state-with-bloc__t62-deepseek-coder-v2." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet.md new file mode 100644 index 00000000000..f27ceb23b43 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t03-claude-3-7-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t03-claude-3-7-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet.md new file mode 100644 index 00000000000..9dad1edf00e --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t08-claude-3-5-sonnet" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t08-claude-3-5-sonnet." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t13-claude-3-5-haiku.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t13-claude-3-5-haiku.md new file mode 100644 index 00000000000..71c131ca519 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t13-claude-3-5-haiku.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t13-claude-3-5-haiku" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t13-claude-3-5-haiku." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t18-o3.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t18-o3.md new file mode 100644 index 00000000000..4725249b985 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t18-o3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t18-o3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t18-o3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t23-gpt-5.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t23-gpt-5.md new file mode 100644 index 00000000000..a90bd048f82 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t23-gpt-5.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t23-gpt-5" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t23-gpt-5." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t28-gpt-4o.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t28-gpt-4o.md new file mode 100644 index 00000000000..2470755d519 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t28-gpt-4o.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t28-gpt-4o" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t28-gpt-4o." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t33-gpt-4o-mini.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t33-gpt-4o-mini.md new file mode 100644 index 00000000000..5f2a0775ed6 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t33-gpt-4o-mini.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t33-gpt-4o-mini" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t33-gpt-4o-mini." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t38-gemini-35-pro.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t38-gemini-35-pro.md new file mode 100644 index 00000000000..6684e144f4c --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t38-gemini-35-pro.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t38-gemini-35-pro" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t38-gemini-35-pro." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t43-gemini-35-flash.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t43-gemini-35-flash.md new file mode 100644 index 00000000000..4f017958c2e --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t43-gemini-35-flash.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t43-gemini-35-flash" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t43-gemini-35-flash." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite.md new file mode 100644 index 00000000000..e913bf3c276 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t48-gemini-31-flash-lite" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t48-gemini-31-flash-lite." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t53-deepseek-r1.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t53-deepseek-r1.md new file mode 100644 index 00000000000..8a0fb61b1b9 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t53-deepseek-r1.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t53-deepseek-r1" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t53-deepseek-r1." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t58-deepseek-v3.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t58-deepseek-v3.md new file mode 100644 index 00000000000..0dffe1c41d9 --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t58-deepseek-v3.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t58-deepseek-v3" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t58-deepseek-v3." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t63-deepseek-coder-v2.md b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t63-deepseek-coder-v2.md new file mode 100644 index 00000000000..9c4b304978e --- /dev/null +++ b/sites/www/content/ai/flutterbench/trials/flutter-offline-sync-sqlite__t63-deepseek-coder-v2.md @@ -0,0 +1,8 @@ +--- +title: "FlutterBench Trial: flutter-offline-sync-sqlite__t63-deepseek-coder-v2" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for Offline SQLite Sync Repository trial flutter-offline-sync-sqlite__t63-deepseek-coder-v2." +publishDate: "2026-09-10" +--- + + diff --git a/sites/www/content/blog/flutter-bench-prologue/index.md b/sites/www/content/blog/flutter-bench-prologue/index.md new file mode 100644 index 00000000000..a697f658088 --- /dev/null +++ b/sites/www/content/blog/flutter-bench-prologue/index.md @@ -0,0 +1,11 @@ +--- +title: "FlutterBench" +description: >- + An introduction to the new FlutterBench project. +publishDate: 2026-08-20 +author: ericwindmill +category: deep-dive +layout: blog +--- + +Hello. \ No newline at end of file diff --git a/sites/www/content/data/flutterbench/cujs.json b/sites/www/content/data/flutterbench/cujs.json new file mode 100644 index 00000000000..4d82219c1ea --- /dev/null +++ b/sites/www/content/data/flutterbench/cujs.json @@ -0,0 +1,1387 @@ +{ + "cujs": [ + { + "id": 0, + "goal": "Evaluate and select the technical stack, folder structure, state management, and routing architecture for a project", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 1, + "name": "research-existing-options-available-architecture", + "task": "Evaluate available architectural patterns, routing libraries, and state management frameworks, documenting the rationale for the selected technology stack." + }, + { + "id": 2, + "name": "use-workspace-monorepo-repo-structure", + "task": "Configure a multi-package Dart workspace or monorepo repository structure to separate core domain logic from application UI features." + } + ] + }, + { + "id": 1, + "goal": "Enforce consistent code formatting, linting, and architectural standards", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 3, + "name": "compose-analysis-options-style-guide", + "task": "Author comprehensive static analysis rules in \"analysis_options.yaml\" and document architectural standards in a project style guide." + }, + { + "id": 4, + "name": "ensure-codebase-uses-only-selected", + "task": "Enforce that the codebase adheres strictly to documented architectural decisions and state management patterns, avoiding unapproved approaches." + } + ] + }, + { + "id": 2, + "goal": "Manage dependency risks and audit third-party packages", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 5, + "name": "audit-third-party-pub-dev", + "task": "Audit third-party pub.dev packages for license compliance, maintenance activity, and security vulnerabilities before adoption." + }, + { + "id": 6, + "name": "ensure-dependencies-are-installed-cli", + "task": "Add project dependencies using official command-line package managers rather than manually modifying configuration files." + } + ] + }, + { + "id": 3, + "goal": "Establish repository governance, branching conventions, code review standards, and CI quality gates", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 7, + "name": "establish-repository-governance-standardize-branching", + "task": "Establish repository governance policies to standardize branching models, enforce peer code reviews, and automate CI quality gates." + } + ] + }, + { + "id": 4, + "goal": "Develop custom Dart CLI developer utilities and automation tools", + "persona": "The App Developer", + "tasks": [ + { + "id": 8, + "name": "write-dart-cli-tool-generate", + "task": "Develop a standalone Dart command-line utility that parses database schema specifications and generates required boilerplate data access code." + }, + { + "id": 9, + "name": "write-cli-tool-optimise-csv", + "task": "Develop a Dart command-line utility to automate the parsing, validation, and compression of CSV datasets and application resources." + } + ] + }, + { + "id": 5, + "goal": "Optimize application release builds for minimal bundle size", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 10, + "name": "analyze-size-analyze-size-devtools", + "task": "Analyze application bundle composition and asset weight using command-line size analysis tools and Flutter DevTools." + }, + { + "id": 11, + "name": "enable-tree-shaking-obfuscation-split", + "task": "Configure production build flags to enable code tree shaking, symbol obfuscation, and split debug information." + }, + { + "id": 12, + "name": "audit-compress-codebase-assets", + "task": "Audit application resources to remove unused assets and compress images and fonts for reduced download size." + } + ] + }, + { + "id": 6, + "goal": "Maintain accurate, up-to-date repository documentation and README guides", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 13, + "name": "review-update-readme-other-documentation", + "task": "Audit and update repository documentation, including README guides and architectural overviews, to align with recent codebase modifications." + } + ] + }, + { + "id": 7, + "goal": "Make an application accessible to all users", + "persona": "The App Developer", + "tasks": [ + { + "id": 14, + "name": "evaluate-how-app-accessibility-is", + "task": "Audit the application using Flutter DevTools and automated accessibility inspection tools to identify compliance gaps." + }, + { + "id": 15, + "name": "modify-app-add-semantic-labels", + "task": "Refactor UI widgets to include descriptive Semantics properties and screen reader labels for visually impaired users." + }, + { + "id": 16, + "name": "modify-app-make-tappable-areas", + "task": "Enforce minimum interactive touch target dimensions across all interactive components to meet mobile accessibility standards." + }, + { + "id": 17, + "name": "remove-fixed-text-scaler", + "task": "Refactor text components to support dynamic system font scaling and remove hardcoded text scale restrictions." + }, + { + "id": 18, + "name": "add-high-contrast-color-themes", + "task": "Implement high-contrast visual themes and color palettes to support users with visual impairments." + } + ] + }, + { + "id": 8, + "goal": "Achieve comprehensive test coverage with unit, widget, and integration test suites", + "persona": "The App Developer", + "tasks": [ + { + "id": 19, + "name": "check-existing-test-coverage-percentage", + "task": "Analyze current test coverage to identify untested code sections and determine which parts of the application require additional test coverage." + }, + { + "id": 20, + "name": "add-app-benchmarking-uses-binding", + "task": "Implement automated performance benchmarking using binding.traceAction to measure frame timing and verify that the 90th percentile execution duration remains below defined latency thresholds." + }, + { + "id": 21, + "name": "add-flutter-integration-tests-mobile", + "task": "Develop end-to-end integration test suites using \"package:integration_test\" to validate complete user journeys across mobile and web environments." + } + ] + }, + { + "id": 9, + "goal": "Diagnose and resolve layout overflow errors in UI component trees", + "persona": "The App Developer", + "tasks": [ + { + "id": 22, + "name": "find-real-cause-ui-overflow", + "task": "Diagnose and identify the root cause of layout overflow errors in the UI component tree." + }, + { + "id": 23, + "name": "fix-overflow-bug-with-proper-widgets", + "task": "Refactor the layout using flexible scrolling or bounding widgets to resolve the overflow error." + }, + { + "id": 24, + "name": "write-widget-tests-edge-cases", + "task": "Implement automated widget tests covering boundary conditions and large data values to prevent regression of layout overflows." + } + ] + }, + { + "id": 10, + "goal": "Implement a structured routing and navigation system", + "persona": "The App Developer", + "tasks": [ + { + "id": 25, + "name": "set-up-go-router-named", + "task": "Configure declarative application routing using \"package:go_router\", implementing named routes and dynamic URL path parameters." + }, + { + "id": 26, + "name": "set-up-go-router-builder", + "task": "Integrate \"package:go_router_builder\" and code generation to manage type-safe route navigation and arguments." + }, + { + "id": 27, + "name": "implement-deep-linking-trigger-deep", + "task": "Configure platform-specific deep linking schemas and verify that external links navigate correctly to target application screens." + }, + { + "id": 28, + "name": "guard-routes-based-auth-state", + "task": "Implement redirection guards within the routing configuration to restrict access to authenticated user sessions." + }, + { + "id": 29, + "name": "use-navigator-v1-route-does", + "task": "Implement imperative navigation using standard Navigator 1.0 APIs for simple internal modal dialogs and screen transitions." + } + ] + }, + { + "id": 11, + "goal": "Add a new UI screen to an existing application following established design and architectural patterns", + "persona": "The App Developer", + "tasks": [ + { + "id": 30, + "name": "add-new-screen-design-system", + "task": "Add a new UI screen to the application that integrates with the existing design system, routing architecture, and standard page structure." + } + ] + }, + { + "id": 12, + "goal": "Design responsive UI layouts that reflow cleanly across all window sizes and device orientations", + "persona": "The App Developer", + "tasks": [ + { + "id": 31, + "name": "define-central-breakpoints-m3-window", + "task": "Define layout breakpoints based on Material Design 3 window size classes, such as using compact layouts for widths under 600 logical pixels." + }, + { + "id": 32, + "name": "use-mediaquery-sizeof-window-sizing", + "task": "Refactor responsive sizing logic to use MediaQuery.sizeOf for global window dimensions and LayoutBuilder for local widget constraint sizing, removing hardcoded device-type checks." + }, + { + "id": 33, + "name": "apply-safearea-notches-insets", + "task": "Wrap visual layouts in SafeArea widgets to prevent content from obscuring system status bars, display notches, and physical screen bezels." + }, + { + "id": 34, + "name": "don-t-portrait-lock-support", + "task": "Configure the application to support both portrait and landscape orientations, verifying smooth UI reflow during device rotation." + }, + { + "id": 35, + "name": "cap-content-width-large-windows", + "task": "Constrain maximum content width on wide desktop or tablet displays using BoxConstraints or by dynamically transitioning from ListView to GridView layouts." + }, + { + "id": 36, + "name": "handle-foldable-letterboxing-support-all", + "task": "Optimize layouts for foldable devices and letterboxed display modes across various screen postures and orientations." + } + ] + }, + { + "id": 13, + "goal": "Optimize application rendering and memory performance", + "persona": "The App Developer", + "tasks": [ + { + "id": 37, + "name": "use-devtools-profile-rendering-performance", + "task": "Profile application frame rendering times and rasterization metrics using Flutter DevTools." + }, + { + "id": 38, + "name": "hunt-down-memory-leaks", + "task": "Diagnose and resolve application memory leaks and retained object graphs using memory profiling tools." + }, + { + "id": 39, + "name": "add-renderrepaintboundary-s-widget-tree", + "task": "Refactor the widget hierarchy by inserting RenderRepaintBoundary widgets around frequently animating components to isolate repaint regions." + } + ] + }, + { + "id": 14, + "goal": "Implement state restoration to preserve user state across application restarts", + "persona": "The App Developer", + "tasks": [ + { + "id": 40, + "name": "add-state-restoration-functionality-app", + "task": "Implement Flutter state restoration APIs using RestorationManager and RestorationBucket to preserve interface navigation and scroll states across process terminations." + }, + { + "id": 41, + "name": "add-hydrated-versions-state-management", + "task": "Integrate persistent state management libraries, such as \"package:hydrated_bloc\", to automatically serialize and restore application state across application restarts." + } + ] + }, + { + "id": 15, + "goal": "Implement offline-first data caching and synchronization", + "persona": "The App Developer", + "tasks": [ + { + "id": 42, + "name": "add-local-caching-solution-be", + "task": "Implement an offline-first repository pattern that caches remote server data locally and synchronizes pending mutations when network connectivity is restored." + } + ] + }, + { + "id": 16, + "goal": "Build interactive widget preview catalogs and isolated design system showcases", + "persona": "The App Developer", + "tasks": [ + { + "id": 43, + "name": "create-interactive-website-every-widget", + "task": "Develop a standalone interactive web catalog showcasing every UI component and visual state available within the component library." + }, + { + "id": 44, + "name": "add-widget-previews-for-components", + "task": "Implement isolated widget preview configurations using the official Flutter widget previewer tool for all UI components in the application." + } + ] + }, + { + "id": 17, + "goal": "Implement a customizable, reusable UI design system", + "persona": "The App Developer", + "tasks": [ + { + "id": 45, + "name": "create-totally-custom-design-system", + "task": "Build an independent UI design system from scratch without relying on standard Material or Cupertino widget libraries." + }, + { + "id": 46, + "name": "customise-material-design-system-fit", + "task": "Customize and extend Material Design widgets and styling tokens to implement a proprietary visual design system." + }, + { + "id": 47, + "name": "customise-cupertino-design-system-fit", + "task": "Customize and extend Cupertino widgets to implement a proprietary iOS-styled visual design system." + } + ] + }, + { + "id": 18, + "goal": "Implement a consistent visual design theme and styling across an application", + "persona": "The App Developer", + "tasks": [ + { + "id": 48, + "name": "create-theme-data-from-design-document", + "task": "Implement application ThemeData configurations derived from specifications in a design document." + }, + { + "id": 49, + "name": "add-dark-mode-support", + "task": "Implement dark mode theming and color scheme switching." + }, + { + "id": 50, + "name": "change-dropdown-popup-buttons-styling", + "task": "Customize visual styling for dropdown menus, popup dialogs, and interactive buttons by extending central ThemeData configurations." + } + ] + }, + { + "id": 19, + "goal": "Implement custom gesture detection and pointer interactions", + "persona": "The App Developer", + "tasks": [ + { + "id": 51, + "name": "create-custom-widget-detects-hover", + "task": "Implement a custom interactive component that combines MouseRegion for hover detection with GestureDetector or InkWell for touch and pointer interactions." + } + ] + }, + { + "id": 20, + "goal": "Implement custom widget animation states and transitions", + "persona": "The App Developer", + "tasks": [ + { + "id": 52, + "name": "create-widget-uses-animationcontrollers-animate", + "task": "Develop an explicit animated component using AnimationController to manage custom state transitions and tween animations." + }, + { + "id": 53, + "name": "add-tests-confirm-animation-logic", + "task": "Implement automated widget tests to verify that animation state machines and value transitions execute correctly." + }, + { + "id": 54, + "name": "replace-static-widget-animated-version", + "task": "Refactor static UI components to use implicit animation widgets, such as AnimatedContainer and AnimatedOpacity, for smooth state transitions." + }, + { + "id": 55, + "name": "use-hero-transition-animations-between", + "task": "Implement shared element routing transitions across navigation boundaries using Hero animation widgets." + } + ] + }, + { + "id": 21, + "goal": "Integrate rich animated graphics and shaders into an application", + "persona": "The App Developer", + "tasks": [ + { + "id": 56, + "name": "use-rive-lottie-or-some", + "task": "Integrate animation libraries, such as \"package:rive\" or \"package:lottie\", to render rich vector animations within the application." + }, + { + "id": 57, + "name": "use-shaders-animate-things-app", + "task": "Implement fragment shaders using GLSL shader programs to render custom GPU-accelerated visual effects and animations." + } + ] + }, + { + "id": 22, + "goal": "Integrate interactive data visualization and charting libraries", + "persona": "The App Developer", + "tasks": [ + { + "id": 58, + "name": "find-list-available-libraries-charts", + "task": "Research available charting libraries on pub.dev and evaluate which packages support the required chart types and features for the use case." + }, + { + "id": 59, + "name": "install-chart-library-supports-bar", + "task": "Install a third-party charting package that supports interactive bar charts using official command-line tools (\"flutter pub add\") rather than manually editing configuration files." + }, + { + "id": 60, + "name": "implement-library-application-show-chart", + "task": "Integrate the charting library into the application dashboard to render interactive data visualizations, implementing automated widget tests to verify chart rendering." + } + ] + }, + { + "id": 23, + "goal": "Configure package dependency overrides using Git repositories or local filesystem paths", + "persona": "The App Developer", + "tasks": [ + { + "id": 61, + "name": "add-dependency-override-git", + "task": "Configure package dependency overrides in \"pubspec.yaml\" to target a specific remote Git repository and subdirectory path, verifying that all automated tests pass." + }, + { + "id": 62, + "name": "add-dependency-override-local", + "task": "Configure package dependency overrides in \"pubspec.yaml\" to link against a local filesystem package path, verifying that all automated tests pass." + } + ] + }, + { + "id": 24, + "goal": "Build an application that renders Material UI on Android and Cupertino UI on iOS", + "persona": "The App Developer", + "tasks": [ + { + "id": 63, + "name": "create-new-app-android-ios", + "task": "Create a new cross-platform Flutter application targeting both Android and iOS." + }, + { + "id": 64, + "name": "add-adaptive-material-cupertino-layouts", + "task": "Implement navigation layouts that adaptively render Material Design components on Android and Cupertino components on iOS." + } + ] + }, + { + "id": 25, + "goal": "Implement performant scrolling layouts for long-form content", + "persona": "The App Developer", + "tasks": [ + { + "id": 65, + "name": "identify-overflow-refactor-layout-use", + "task": "Diagnose vertical layout overflow errors and refactor the component hierarchy to use SingleChildScrollView." + }, + { + "id": 66, + "name": "migrate-customscrollview-slivers-more-complex", + "task": "Refactor standard scroll views to use CustomScrollView and Sliver components for advanced scrolling effects and header animations." + }, + { + "id": 67, + "name": "have-long-list-items-variate", + "task": "Implement programmatic scroll-to-index functionality for variable-height item lists using scroll controllers or item alignment libraries." + } + ] + }, + { + "id": 26, + "goal": "Build intuitive, validated user forms with polished input UX", + "persona": "The App Developer", + "tasks": [ + { + "id": 68, + "name": "auto-focus-first-invalid-field", + "task": "Implement form validation logic that automatically transfers focus to the first invalid input field when a user submits an incomplete form." + }, + { + "id": 69, + "name": "add-floating-label-behavior-textfields", + "task": "Configure text input fields with floating label behavior and error messaging using InputDecoration properties." + } + ] + }, + { + "id": 27, + "goal": "Build an application that communicates with a REST API", + "persona": "The App Developer", + "tasks": [ + { + "id": 70, + "name": "fetch-parse-json-http-or", + "task": "Implement network calls to fetch and parse JSON payloads from a REST API using \"package:http\" or \"package:dio\"." + }, + { + "id": 71, + "name": "handle-errors-timeouts-loading-states", + "task": "Implement robust error handling, network request timeout management, and UI loading state indicators." + }, + { + "id": 72, + "name": "get-rid-ui-jank-due", + "task": "Offload JSON serialization and deserialization to background worker isolates to prevent main thread stutter and UI frame drops." + }, + { + "id": 73, + "name": "if-api-has-spec-use", + "task": "Generate type-safe API client code and data models automatically from an OpenAPI specification using code generation tools." + } + ] + }, + { + "id": 28, + "goal": "Implement and evaluate state management architectures", + "persona": "The App Developer", + "tasks": [ + { + "id": 74, + "name": "use-setstate-manage-state", + "task": "Implement application state management using standard StatefulWidget and setState mechanisms." + }, + { + "id": 75, + "name": "use-provider-manage-state", + "task": "Implement application state management using \"package:provider\" for dependency injection and reactive updates." + }, + { + "id": 76, + "name": "use-riverpod-manage-state-maybe", + "task": "Implement application state management using \"package:riverpod\", optionally incorporating \"package:flutter_hooks\" and code generation." + }, + { + "id": 77, + "name": "use-bloc-cubit-manage-state", + "task": "Implement application state management using the Business Logic Component (BLoC) and Cubit patterns from \"package:flutter_bloc\"." + }, + { + "id": 78, + "name": "use-hooks-manage-state", + "task": "Implement application state management using \"package:flutter_hooks\" to manage widget lifecycle and local state with composable hook functions." + }, + { + "id": 79, + "name": "use-rxdart-manage-state", + "task": "Implement stream-based application state management using reactive programming primitives from \"package:rxdart\"." + } + ] + }, + { + "id": 29, + "goal": "Implement local data persistence in an application", + "persona": "The App Developer", + "tasks": [ + { + "id": 80, + "name": "set-up-shared-preferences-package", + "task": "Implement local persistence for simple key-value data and user preferences using \"package:shared_preferences\"." + }, + { + "id": 81, + "name": "set-up-sqlite-sqflite-or", + "task": "Implement a structured local relational database using \"package:sqflite\" or \"package:drift\"." + }, + { + "id": 82, + "name": "set-up-secure-storage-store", + "task": "Implement encrypted local storage for sensitive user data and authentication tokens using \"package:flutter_secure_storage\"." + }, + { + "id": 83, + "name": "use-path-provider-locate-application", + "task": "Integrate \"package:path_provider\" to locate platform-specific filesystem directories for application documents and temporary files." + } + ] + }, + { + "id": 30, + "goal": "Offload CPU-intensive computation to background isolates to prevent UI freezing", + "persona": "The App Developer", + "tasks": [ + { + "id": 84, + "name": "offload-cpu-intensive-work-isolate", + "task": "Offload computationally expensive synchronous operations to background worker isolates using Isolate.run." + }, + { + "id": 85, + "name": "use-compute-one-shot-tasks", + "task": "Execute one-shot background computations using the top-level compute function to prevent UI thread blocking." + }, + { + "id": 86, + "name": "set-up-long-lived-isolate", + "task": "Implement a long-lived background isolate communicating via ReceivePort and SendPort message passing to handle continuous asynchronous processing." + } + ] + }, + { + "id": 31, + "goal": "Adopt code generation tools to reduce boilerplate for models and immutable data classes", + "persona": "The App Developer", + "tasks": [ + { + "id": 87, + "name": "set-up-build-runner-json", + "task": "Configure \"package:build_runner\" and \"package:json_serializable\" to generate type-safe JSON serialization code for data models." + }, + { + "id": 88, + "name": "use-freezed-immutable-data-classes", + "task": "Integrate \"package:freezed\" to generate immutable data classes, union types, and value equality boilerplate." + }, + { + "id": 89, + "name": "use-build-verify-ci-cd", + "task": "Configure automated CI/CD pipelines using \"package:build_verify\" to ensure generated source code is synchronized with existing data models." + } + ] + }, + { + "id": 32, + "goal": "Diagnose and resolve native platform interop bugs and channel communication errors", + "persona": "The Plugin Developer", + "tasks": [ + { + "id": 90, + "name": "analyze-codebase-try-find-cause", + "task": "Analyze native platform channel implementations and Dart bindings to diagnose the root cause of platform communication failures." + }, + { + "id": 91, + "name": "add-debug-logs-perform-test", + "task": "Instrument platform interop channels with diagnostic logging and execute test runs to isolate native execution errors." + }, + { + "id": 92, + "name": "fix-issue", + "task": "Refactor native host code and Dart channel handlers to resolve platform interop exceptions and restore reliable communication." + } + ] + }, + { + "id": 33, + "goal": "Design a unified, well-documented cross-platform API surface for plugin consumers", + "persona": "The Plugin Developer", + "tasks": [ + { + "id": 93, + "name": "design-intuitive-well-documented-unified", + "task": "Design a unified, documented Dart API that abstracts native iOS and Android implementation differences for plugin consumers." + }, + { + "id": 94, + "name": "encapsulate-platform-interface-code", + "task": "Structure the plugin package to ensure public APIs encapsulate and hide internal platform interface implementations." + }, + { + "id": 95, + "name": "generate-html-api-documentation", + "task": "Generate HTML API documentation from inline dartdoc comments using command-line tools to verify public API presentation." + } + ] + }, + { + "id": 34, + "goal": "Implement automated native platform test suites (XCTest, Espresso, JUnit) to prevent OS upgrade regressions", + "persona": "The Plugin Developer", + "tasks": [ + { + "id": 96, + "name": "write-automated-tests-validate-both", + "task": "Implement automated test suites validating Dart logic and native implementations using XCTest for iOS and JUnit or Espresso for Android." + } + ] + }, + { + "id": 35, + "goal": "Maintain high-quality published packages with rigorous semantic versioning, detailed changelogs, and responsiveness to SDK updates", + "persona": "The Plugin Developer", + "tasks": [ + { + "id": 97, + "name": "manage-versions-write-changelogs-get", + "task": "Manage semantic versioning, maintain changelogs, achieve high pub.dev quality scores, and publish package releases compatible with current Flutter SDK versions." + } + ] + }, + { + "id": 36, + "goal": "Adopt optimal native interop mechanisms (FFI, Pigeon, JS Interop) based on performance and platform requirements", + "persona": "The Plugin Developer", + "tasks": [ + { + "id": 98, + "name": "migrate-pigeon-based-platform-channels", + "task": "Migrate Pigeon-based platform channels to Foreign Function Interface (FFI) bindings for performance-critical or synchronous native calls." + }, + { + "id": 99, + "name": "replace-manual-platform-channel-boilerplate", + "task": "Replace manual platform channel boilerplate with type-safe message passing code generated by \"package:pigeon\"." + }, + { + "id": 100, + "name": "create-type-safe-bindings-between", + "task": "Create type-safe bindings between Dart and JavaScript using \"dart:js_interop\" and extension types to integrate with browser APIs and external JavaScript libraries." + }, + { + "id": 101, + "name": "check-all-resources-used-native", + "task": "Ensure native memory allocations (malloc, calloc, FFI structs, OpenGL handles, file descriptors) are properly released using NativeFinalizer and the Finalizable interface." + } + ] + }, + { + "id": 37, + "goal": "Extend an existing federated plugin architecture to support a new target platform", + "persona": "The Plugin Developer", + "tasks": [ + { + "id": 102, + "name": "add-implementation-new-platform-app", + "task": "Implement native platform support for an additional operating system within an existing plugin, verifying that all application-facing integration tests pass." + }, + { + "id": 103, + "name": "test-new-plugin-through-app", + "task": "Verify the new platform implementation by running automated test suites against the application-facing package." + }, + { + "id": 104, + "name": "map-c-language-types-integers", + "task": "Map C language data types (integers, structs, and pointers) to \"dart:ffi\" types, utilizing AbiSpecificInteger for platform-dependent type sizing." + }, + { + "id": 105, + "name": "refactor-plugin-split-it-into", + "task": "Refactor a monolithic plugin into a federated architecture consisting of separate application-facing, platform interface, and platform implementation packages." + } + ] + }, + { + "id": 38, + "goal": "Implement automated cross-platform integration tests for plugins using modern testing frameworks", + "persona": "The Plugin Developer", + "tasks": [ + { + "id": 106, + "name": "use-patrol-package-be-able", + "task": "Implement automated cross-platform integration tests using \"package:patrol\" to verify plugin functionality across native environments." + }, + { + "id": 107, + "name": "write-ci-cd-pipeline-test", + "task": "Configure an automated CI/CD pipeline to execute plugin integration tests on every pull request and prior to release publication." + }, + { + "id": 108, + "name": "write-integration-tests-has-100%", + "task": "Develop comprehensive integration test suites that achieve full test coverage of native platform plugin functionality." + } + ] + }, + { + "id": 39, + "goal": "Implement user authentication flows and UI", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 109, + "name": "plan-auth-provider-design-system", + "task": "Define the architectural requirements, user experience flows, and edge-case handling for application authentication." + }, + { + "id": 110, + "name": "code-auth-flow", + "task": "Implement secure user authentication and registration workflows connecting the frontend UI to the authentication service." + }, + { + "id": 111, + "name": "test-all-auth-flows", + "task": "Implement automated unit, widget, and integration test suites to verify all authentication and session management workflows." + } + ] + }, + { + "id": 40, + "goal": "Integrate push notification services into an application", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 112, + "name": "add-firebase-cloud-messaging", + "task": "Integrate \"package:firebase_messaging\" to enable push notifications across mobile and web platforms." + }, + { + "id": 113, + "name": "handle-foreground-background-terminated-states", + "task": "Implement notification event listeners and handlers for foreground, background, and terminated application lifecycle states." + } + ] + }, + { + "id": 41, + "goal": "Integrate third-party authentication providers into an application", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 114, + "name": "add-google-sign", + "task": "Integrate \"package:google_sign_in\" to enable single sign-on authentication." + }, + { + "id": 115, + "name": "handle-sign-sign-out-flows", + "task": "Implement complete authentication state machines managing sign-in, sign-out, session persistence, and OAuth token refresh workflows." + } + ] + }, + { + "id": 42, + "goal": "Integrate cloud file storage into an application", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 116, + "name": "add-firebase-storage", + "task": "Integrate \"package:firebase_storage\" into the application to enable cloud storage capabilities." + }, + { + "id": 117, + "name": "implement-upload-download-delete", + "task": "Implement user flows and repository methods to upload, download, and delete cloud storage files." + } + ] + }, + { + "id": 43, + "goal": "Integrate crash reporting, error tracking, and production telemetry", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 118, + "name": "add-crashlytics", + "task": "Integrate Firebase Crashlytics (\"package:firebase_crashlytics\") to capture and monitor real-time fatal exception reports." + }, + { + "id": 119, + "name": "add-custom-log-events-non", + "task": "Implement custom error logging and non-fatal exception tracking to record application telemetry and diagnostic metadata." + } + ] + }, + { + "id": 44, + "goal": "Integrate AWS Amplify authentication, cloud storage, and backend APIs into an application", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 120, + "name": "integrate-amplify-auth-+-storage", + "task": "Integrate AWS Amplify authentication, cloud storage, and API services into the application architecture." + } + ] + }, + { + "id": 45, + "goal": "Integrate Supabase authentication, database services, and real-time subscriptions into an application", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 121, + "name": "integrate-supabase-auth-database", + "task": "Integrate Supabase authentication, relational database services, and real-time data subscriptions into the application architecture." + } + ] + }, + { + "id": 46, + "goal": "Build a full-stack Dart web server backend with shared data models between frontend and backend", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 122, + "name": "create-web-server-shelf", + "task": "Develop a backend web server and HTTP API routing layer using \"package:shelf\"." + }, + { + "id": 123, + "name": "create-cloud-function-upload-firebase", + "task": "Implement server-side logic or Google Cloud Functions to process file uploads and store metadata in Firebase." + }, + { + "id": 124, + "name": "decide-best-repo-structure-according", + "task": "Design and implement a shared monorepo workspace structure to allow seamless data model reuse between Dart frontend and backend services." + } + ] + }, + { + "id": 47, + "goal": "Implement API versioning and backward compatibility checks between frontend applications and backend services", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 125, + "name": "create-ci-cd-pipeline-confirm", + "task": "Configure automated CI/CD deployment pipelines to verify that the target backend API version is active prior to releasing client applications." + }, + { + "id": 126, + "name": "create-screen-app-app-version", + "task": "Implement a dedicated application deprecation screen that informs users when their client version is no longer supported by backend API services." + } + ] + }, + { + "id": 48, + "goal": "Develop Google Cloud Functions in Dart using the Genkit SDK", + "persona": "The Full Stack Developer", + "tasks": [ + { + "id": 127, + "name": "write-google-cloud-function-dart", + "task": "Develop and deploy serverless Google Cloud Functions written in Dart using the Genkit framework (\"package:genkit\")." + } + ] + }, + { + "id": 49, + "goal": "Add internationalization (i18n) and localization (l10n) support to an application", + "persona": "The App Developer", + "tasks": [ + { + "id": 128, + "name": "add-required-languages-locales-app", + "task": "Configure supported languages and regional locales within the application localization settings." + }, + { + "id": 129, + "name": "confirm-default-flutter-ways-i18n", + "task": "Verify that the codebase implements standard Flutter internationalization practices using ARB files and generated localization delegates." + } + ] + }, + { + "id": 50, + "goal": "Embed interactive Flutter applications and widgets within existing HTML or React web pages", + "persona": "The Hybrid (Native + Flutter) Developer", + "tasks": [ + { + "id": 130, + "name": "create-website-jaspr", + "task": "Build a server-rendered or static website in Dart using the Jaspr web framework (\"package:jaspr\")." + }, + { + "id": 131, + "name": "add-flutter-app-react-app", + "task": "Embed a compiled Flutter web application as an interactive component within an existing React web application." + }, + { + "id": 132, + "name": "add-many-flutter-widgets-across", + "task": "Embed multiple interactive Flutter widgets across a standard HTML web page, utilizing Flutter multi-view mode to optimize rendering performance and resource consumption." + } + ] + }, + { + "id": 51, + "goal": "Integrate Flutter modules into existing native Android and iOS applications using Add-to-app", + "persona": "The Hybrid (Native + Flutter) Developer", + "tasks": [ + { + "id": 133, + "name": "add-flutter-engine-view-android", + "task": "Integrate a cached FlutterEngine and FlutterActivity into an existing native Android application using Flutter Add-to-app workflows." + }, + { + "id": 134, + "name": "add-flutter-engine-view-ios", + "task": "Integrate a cached FlutterEngine and FlutterViewController into an existing native iOS application using Flutter Add-to-app workflows." + } + ] + }, + { + "id": 52, + "goal": "Implement seamless cross-layer navigation and state synchronization between native host apps and embedded Flutter modules", + "persona": "The Hybrid (Native + Flutter) Developer", + "tasks": [ + { + "id": 135, + "name": "cache-pre-warm-flutter-engine", + "task": "Configure native host applications to pre-warm and cache the FlutterEngine during application startup to eliminate initialization latency." + }, + { + "id": 136, + "name": "manage-complex-navigation-stacks-where", + "task": "Implement bidirectional navigation stacks where users transition between native Swift screens and embedded Flutter modules, ensuring native gesture back-swipes behave naturally." + }, + { + "id": 137, + "name": "securely-pass-active-user-session", + "task": "Synchronize active session tokens, visual theme preferences, and user state from the host native application into embedded Flutter modules to provide a seamless user experience." + } + ] + }, + { + "id": 53, + "goal": "Build platform-specific home screen widgets (for iOS WidgetKit and Android) that share data with the host application", + "persona": "The Hybrid (Native + Flutter) Developer", + "tasks": [ + { + "id": 138, + "name": "set-up-home-screen-widget", + "task": "Scaffold and configure native home screen widget extensions for iOS using WidgetKit and for Android using AppWidgets." + }, + { + "id": 139, + "name": "share-data-between-flutter-app", + "task": "Implement shared local storage using App Groups on iOS and SharedPreferences on Android to synchronize data between the Flutter application and native widgets." + }, + { + "id": 140, + "name": "update-widget-data-flutter", + "task": "Trigger programmatic background updates and timeline reloads for native home screen widgets directly from Dart application logic." + } + ] + }, + { + "id": 54, + "goal": "Identify and refactor architectural anti-patterns in the codebase", + "persona": "The App Developer", + "tasks": [ + { + "id": 141, + "name": "analyze-codebase-anti-patterns", + "task": "Analyze the codebase to identify and refactor architectural anti-patterns, such as building complex widget trees inside helper methods rather than separate widget classes." + } + ] + }, + { + "id": 55, + "goal": "Audit and migrate codebases away from deprecated frameworks, libraries, and SDK APIs", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 142, + "name": "analyze-codebase-deprecated-api", + "task": "Analyze the codebase to identify and migrate deprecated API usage, including Material Design 2 components, direct window references in \"dart:ui\", and legacy ThemeData styling properties." + } + ] + }, + { + "id": 56, + "goal": "Automate multi-flavor application build, configuration, and distribution pipelines", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 143, + "name": "config-flavors-app-different-naming", + "task": "Configure multi-flavor build schemes across iOS and Android to support distinct application names, bundle identifiers, and launcher icons for staging and production environments." + }, + { + "id": 144, + "name": "write-ci-cd-pipeline-deploy", + "task": "Implement an automated CI/CD distribution pipeline to build and deploy application binaries to internal testing tracks or distribution services." + } + ] + }, + { + "id": 57, + "goal": "Upgrade and migrate legacy Flutter applications to the latest SDK version", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 145, + "name": "audit-and-upgrade-flutter-sdk", + "task": "Audit the local Flutter SDK installation and upgrade safely using version management tools (\"fvm\") or system package managers." + }, + { + "id": 146, + "name": "run-dart-fix-migration-tool", + "task": "Execute \"dart fix\" to update deprecated syntax and resolve breaking API changes across the codebase." + } + ] + }, + { + "id": 58, + "goal": "Extend an existing application to support an additional target platform", + "persona": "The Tech Lead / Architect", + "tasks": [ + { + "id": 147, + "name": "analyze-repo-check-which-features", + "task": "Audit existing codebase capabilities and third-party plugins to determine feature compatibility with the target platform." + }, + { + "id": 148, + "name": "verify-proper-command-is-used", + "task": "Execute official platform scaffolding commands to generate target platform projects and verify dependency compatibility." + } + ] + }, + { + "id": 59, + "goal": "Implement new application features utilizing modern language capabilities and defensive coding practices", + "persona": "The App Developer", + "tasks": [ + { + "id": 149, + "name": "verify-assert-calls-all-passed", + "task": "Enforce defensive coding by adding runtime assert statements to validate constructor and function parameter boundaries." + }, + { + "id": 150, + "name": "use-record-structure-type-function", + "task": "Refactor function signatures to return structured, type-safe multiple values using modern Dart Record types." + } + ] + }, + { + "id": 60, + "goal": "Implement custom canvas drawing and custom painters for specialized UI components", + "persona": "The App Developer", + "tasks": [ + { + "id": 151, + "name": "create-widget-displays-custom-pattern", + "task": "Implement a custom component utilizing CustomPaint and Canvas primitives to render specialized graphics above or below child widget layers." + } + ] + }, + { + "id": 61, + "goal": "Design and develop a new cross-platform plugin from scratch, selecting the appropriate native interop mechanism", + "persona": "The Plugin Developer", + "tasks": [ + { + "id": 152, + "name": "investigate-should-ffi-or-methodchannels", + "task": "Evaluate whether Foreign Function Interface (FFI) bindings or asynchronous MethodChannels provide the optimal architectural foundation for a new cross-platform plugin." + } + ] + }, + { + "id": 62, + "goal": "Set up and configure a complete cross-platform Flutter development environment", + "persona": "The App Developer", + "tasks": [ + { + "id": 153, + "name": "install-configure-xcode-command-line", + "task": "Install and configure Xcode, command-line tools, and CocoaPods on a macOS development environment to compile for all supported Flutter target platforms." + }, + { + "id": 154, + "name": "sets-up-windows-environment-flutter", + "task": "Set up and configure a Windows development environment for Flutter with necessary dependencies to compile for all supported non-Apple target platforms." + }, + { + "id": 155, + "name": "sets-up-linux-environment-flutter", + "task": "Set up and configure a Linux development environment for Flutter with necessary dependencies to compile for all supported non-Apple target platforms." + } + ] + }, + { + "id": 63, + "goal": "Build a responsive Flutter Web frontend for an Enterprise Resource Planning (ERP) system", + "persona": "The App Developer", + "tasks": [ + { + "id": 156, + "name": "create-flutter-web-app-has", + "task": "Develop a responsive Flutter web frontend that dynamically adapts between desktop browser layouts and mobile web layouts." + }, + { + "id": 157, + "name": "use-proper-url-path-strategy", + "task": "Configure the web URL routing strategy (hash-based or path-based) according to target web hosting platform requirements." + }, + { + "id": 158, + "name": "use-wasm-if-possible", + "task": "Configure the web build pipeline to compile to WebAssembly (Wasm) for high-performance browser execution." + }, + { + "id": 159, + "name": "use-pwa-web-app-if", + "task": "Configure Progressive Web App (PWA) manifest and service worker features to enable offline support and desktop installation." + }, + { + "id": 160, + "name": "use-package:web-dart:js-interop-interact", + "task": "Implement browser API integrations and JavaScript interop using \"package:web\" and modern Dart type-safe JS interop mechanisms." + } + ] + }, + { + "id": 64, + "goal": "Build adaptive UI layouts that dynamically adjust to platform conventions and input methods (touch, mouse, keyboard, stylus)", + "persona": "The App Developer", + "tasks": [ + { + "id": 161, + "name": "switch-nav-window-size:-bottom", + "task": "Implement adaptive navigation that transitions between a bottom navigation bar on compact screens and a side NavigationRail on expanded displays, sharing routing destinations." + }, + { + "id": 162, + "name": "target-android-tier-3-mouse", + "task": "Optimize the interface for mouse and stylus input by utilizing Material Design 3 components with built-in hover and focus states." + }, + { + "id": 163, + "name": "scroll-wheel-custom-scrollables-listener", + "task": "Refactor custom scrollable components using Listener widgets to support mouse scroll wheel and trackpad navigation." + }, + { + "id": 164, + "name": "tab-traversal-+-visible-focus", + "task": "Implement keyboard tab navigation and visible focus highlights on custom interactive components using FocusableActionDetector and FocusTraversalGroup." + }, + { + "id": 165, + "name": "keyboard-shortcuts-shortcuts-actions-disable", + "task": "Configure application-wide keyboard shortcuts using Shortcuts and Actions widgets, ensuring shortcuts are disabled during text input." + }, + { + "id": 166, + "name": "visualdensity-switched-input-mode-hit", + "task": "Adjust widget VisualDensity dynamically based on active input mode to optimize touch target sizes versus mouse precision sizing." + } + ] + }, + { + "id": 65, + "goal": "Migrate application architecture between state management solutions", + "persona": "The App Developer", + "tasks": [ + { + "id": 167, + "name": "replace-setstate-riverpod", + "task": "Refactor the codebase to use \"package:riverpod\" rather than StatefulWidgets." + }, + { + "id": 168, + "name": "replace-setstate-provider", + "task": "Refactor the codebase to use \"package:provider\" rather than StatefulWidgets." + }, + { + "id": 169, + "name": "replace-inheritedwidget-provider", + "task": "Refactor the codebase to use \"package:provider\" rather than custom InheritedWidgets." + }, + { + "id": 170, + "name": "replace-provider-riverpod", + "task": "Migrate existing state management from \"package:provider\" to \"package:riverpod\"." + }, + { + "id": 171, + "name": "replace-provider-bloc", + "task": "Migrate existing state management from \"package:provider\" to the BLoC (\"package:flutter_bloc\") architecture." + }, + { + "id": 172, + "name": "replace-setstate-rxdart", + "task": "Refactor the codebase to manage reactive state using \"package:rxdart\" rather than StatefulWidgets." + } + ] + }, + { + "id": 66, + "goal": "Diagnose, debug, and resolve runtime exceptions and network defects", + "persona": "The App Developer", + "tasks": [ + { + "id": 173, + "name": "reproduce-reported-defect-failing-test", + "task": "Reproduce a reported defect in a failing test, then trace the root cause using the Dart debugger and Flutter DevTools." + }, + { + "id": 174, + "name": "fix-common-runtime-exceptions", + "task": "Diagnose and resolve common runtime exceptions (null errors, late init failures, RangeErrors, invalid setState calls)." + }, + { + "id": 175, + "name": "diagnose-fix-failed-network-request", + "task": "Diagnose and resolve failed HTTP requests (non-200 status codes, timeouts, JSON deserialization failures)." + } + ] + }, + { + "id": 67, + "goal": "Refactor application code to improve modularity, component reusability, and architectural maintainability", + "persona": "The App Developer", + "tasks": [ + { + "id": 176, + "name": "extract-repeated-widget-code-into", + "task": "Extract repeated widget trees into reusable components and consolidate shared colors, spacing, and text styles into central theme constants." + }, + { + "id": 177, + "name": "split-large-dart-class-into", + "task": "Refactor large Dart classes into smaller units, separating business logic from widget presentation." + }, + { + "id": 178, + "name": "extract-shared-ui-logic-into-mixins", + "task": "Extract shared UI behavior and state logic into reusable Dart mixins." + } + ] + }, + { + "id": 68, + "goal": "Call native platform APIs directly using MethodChannel and EventChannel implementations on Android and iOS", + "persona": "The App Developer", + "tasks": [ + { + "id": 179, + "name": "call-one-shot-native-method", + "task": "Implement one-shot communication between Dart and native platforms via MethodChannel (e.g., reading battery level or triggering haptic feedback), writing handlers in Kotlin for Android and Swift for iOS." + }, + { + "id": 180, + "name": "stream-continuous-native-events-into", + "task": "Stream continuous native events into Dart via EventChannel (e.g., sensor data or network connectivity state)." + } + ] + } + ] +} diff --git a/sites/www/content/data/flutterbench/job.json b/sites/www/content/data/flutterbench/job.json new file mode 100644 index 00000000000..f9e331da703 --- /dev/null +++ b/sites/www/content/data/flutterbench/job.json @@ -0,0 +1,610 @@ +{ + "id": "9b1deb4d-3b7d-4bad-9bdd-2b0d7b3dcb6d", + "started_at": "2026-09-09T19:00:00.000000", + "finished_at": "2026-09-09T22:30:00.000000", + "n_total_trials": 65, + "n_completed_trials": 62, + "n_errored_trials": 3, + "cost_usd": 10.6607, + "n_input_tokens": 5284692, + "n_cache_tokens": 3823561, + "n_output_tokens": 229186, + "top_model_name": "claude-3-7-sonnet", + "top_model_reward": 0.89, + "overall_average_reward": 0.6999999999999998, + "evals": [ + { + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "provider": "Anthropic", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.89, + "outcome_score": 0.89, + "quality_score": 0.88, + "dx_score": 0.92, + "min_reward": 0.71, + "max_reward": 0.98, + "median_reward": 0.93, + "pass_at_1": 0.8, + "cost_usd": 1.7532999999999999, + "input_tokens": 469581, + "output_tokens": 22969, + "has_dart_tooling": true, + "best_cujs": [ + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.98, + "status": "pass" + }, + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.97, + "status": "pass" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.93, + "status": "pass" + } + ], + "worst_cujs": [] + }, + { + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "provider": "Google", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.89, + "outcome_score": 0.89, + "quality_score": 0.87, + "dx_score": 0.95, + "min_reward": 0.69, + "max_reward": 0.99, + "median_reward": 0.91, + "pass_at_1": 0.8, + "cost_usd": 0.6865999999999999, + "input_tokens": 466788, + "output_tokens": 20636, + "has_dart_tooling": true, + "best_cujs": [ + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.99, + "status": "pass" + }, + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.98, + "status": "pass" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.91, + "status": "pass" + } + ], + "worst_cujs": [] + }, + { + "eval_key": "codex-agent__o3__adhoc", + "agent_name": "codex-agent", + "model_name": "o3", + "model_short_name": "o3", + "provider": "OpenAI", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.88, + "outcome_score": 0.88, + "quality_score": 0.88, + "dx_score": 0.95, + "min_reward": 0.73, + "max_reward": 0.97, + "median_reward": 0.9, + "pass_at_1": 0.8, + "cost_usd": 3.4046000000000003, + "input_tokens": 567432, + "output_tokens": 28371, + "has_dart_tooling": true, + "best_cujs": [ + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.97, + "status": "pass" + }, + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.97, + "status": "pass" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.9, + "status": "pass" + } + ], + "worst_cujs": [] + }, + { + "eval_key": "codex-agent__gpt-5__adhoc", + "agent_name": "codex-agent", + "model_name": "gpt-5", + "model_short_name": "gpt-5", + "provider": "OpenAI", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.87, + "outcome_score": 0.87, + "quality_score": 0.86, + "dx_score": 0.96, + "min_reward": 0.72, + "max_reward": 0.98, + "median_reward": 0.86, + "pass_at_1": 0.8, + "cost_usd": 1.3166, + "input_tokens": 443902, + "output_tokens": 20682, + "has_dart_tooling": true, + "best_cujs": [ + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.98, + "status": "pass" + }, + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.97, + "status": "pass" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.86, + "status": "pass" + } + ], + "worst_cujs": [] + }, + { + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek-r1", + "model_short_name": "deepseek-r1", + "provider": "DeepSeek", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.86, + "outcome_score": 0.85, + "quality_score": 0.84, + "dx_score": 0.96, + "min_reward": 0.72, + "max_reward": 0.99, + "median_reward": 0.87, + "pass_at_1": 0.6, + "cost_usd": 0.3379, + "input_tokens": 519066, + "output_tokens": 23956, + "has_dart_tooling": true, + "best_cujs": [ + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.99, + "status": "pass" + }, + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.94, + "status": "pass" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.87, + "status": "pass" + } + ], + "worst_cujs": [] + }, + { + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "provider": "Google", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.81, + "outcome_score": 0.8, + "quality_score": 0.78, + "dx_score": 0.94, + "min_reward": 0.65, + "max_reward": 0.93, + "median_reward": 0.8, + "pass_at_1": 0.6, + "cost_usd": 0.0335, + "input_tokens": 389376, + "output_tokens": 14228, + "has_dart_tooling": true, + "best_cujs": [ + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.93, + "status": "pass" + }, + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.88, + "status": "pass" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.8, + "status": "pass" + } + ], + "worst_cujs": [] + }, + { + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "provider": "Anthropic", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.68, + "outcome_score": 0.72, + "quality_score": 0.62, + "dx_score": 0.6, + "min_reward": 0.52, + "max_reward": 0.79, + "median_reward": 0.71, + "pass_at_1": 0.0, + "cost_usd": 1.4893999999999998, + "input_tokens": 403047, + "output_tokens": 18679, + "has_dart_tooling": false, + "best_cujs": [ + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.79, + "status": "partial" + }, + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.75, + "status": "partial" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.71, + "status": "partial" + } + ], + "worst_cujs": [] + }, + { + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent_name": "codex-agent", + "model_name": "gpt-4o", + "model_short_name": "gpt-4o", + "provider": "OpenAI", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.62, + "outcome_score": 0.66, + "quality_score": 0.56, + "dx_score": 0.6, + "min_reward": 0.47, + "max_reward": 0.71, + "median_reward": 0.65, + "pass_at_1": 0.0, + "cost_usd": 1.1506, + "input_tokens": 393890, + "output_tokens": 16585, + "has_dart_tooling": false, + "best_cujs": [ + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.71, + "status": "partial" + }, + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.69, + "status": "partial" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.65, + "status": "partial" + } + ], + "worst_cujs": [ + { + "task_slug": "flutter-custom-render-object", + "task_name": "Custom RenderObject & Canvas", + "reward": 0.47, + "status": "partial" + } + ] + }, + { + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek-v3", + "model_short_name": "deepseek-v3", + "provider": "DeepSeek", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.62, + "outcome_score": 0.64, + "quality_score": 0.57, + "dx_score": 0.59, + "min_reward": 0.49, + "max_reward": 0.71, + "median_reward": 0.64, + "pass_at_1": 0.0, + "cost_usd": 0.0568, + "input_tokens": 374478, + "output_tokens": 15843, + "has_dart_tooling": false, + "best_cujs": [ + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.71, + "status": "partial" + }, + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.67, + "status": "partial" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.64, + "status": "partial" + } + ], + "worst_cujs": [ + { + "task_slug": "flutter-custom-render-object", + "task_name": "Custom RenderObject & Canvas", + "reward": 0.49, + "status": "partial" + } + ] + }, + { + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "provider": "DeepSeek", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.53, + "outcome_score": 0.55, + "quality_score": 0.47, + "dx_score": 0.61, + "min_reward": 0.42, + "max_reward": 0.61, + "median_reward": 0.55, + "pass_at_1": 0.0, + "cost_usd": 0.0548, + "input_tokens": 361781, + "output_tokens": 14778, + "has_dart_tooling": false, + "best_cujs": [ + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.61, + "status": "partial" + }, + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.59, + "status": "partial" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.55, + "status": "partial" + } + ], + "worst_cujs": [ + { + "task_slug": "flutter-custom-render-object", + "task_name": "Custom RenderObject & Canvas", + "reward": 0.42, + "status": "partial" + } + ] + }, + { + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent_name": "claude-code", + "model_name": "claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "provider": "Anthropic", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 0, + "mean_reward": 0.48, + "outcome_score": 0.49, + "quality_score": 0.42, + "dx_score": 0.62, + "min_reward": 0.36, + "max_reward": 0.56, + "median_reward": 0.48, + "pass_at_1": 0.0, + "cost_usd": 0.3188, + "input_tokens": 334612, + "output_tokens": 12794, + "has_dart_tooling": false, + "best_cujs": [ + { + "task_slug": "dart-build-cli-app", + "task_name": "Build Command-Line CLI App", + "reward": 0.56, + "status": "partial" + }, + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": 0.55, + "status": "partial" + } + ], + "worst_cujs": [ + { + "task_slug": "flutter-custom-render-object", + "task_name": "Custom RenderObject & Canvas", + "reward": 0.36, + "status": "partial" + }, + { + "task_slug": "flutter-adaptive-material-cupertino", + "task_name": "Adaptive Material & Cupertino UI", + "reward": 0.45, + "status": "partial" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.48, + "status": "partial" + } + ] + }, + { + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent_name": "codex-agent", + "model_name": "gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "provider": "OpenAI", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 1, + "mean_reward": 0.43, + "outcome_score": 0.43, + "quality_score": 0.38, + "dx_score": 0.61, + "min_reward": 0.39, + "max_reward": 0.48, + "median_reward": 0.46, + "pass_at_1": 0.0, + "cost_usd": 0.039900000000000005, + "input_tokens": 233248, + "output_tokens": 8276, + "has_dart_tooling": false, + "best_cujs": [], + "worst_cujs": [ + { + "task_slug": "flutter-custom-render-object", + "task_name": "Custom RenderObject & Canvas", + "reward": null, + "status": "error" + }, + { + "task_slug": "flutter-adaptive-material-cupertino", + "task_name": "Adaptive Material & Cupertino UI", + "reward": 0.39, + "status": "partial" + }, + { + "task_slug": "flutter-offline-sync-sqlite", + "task_name": "Offline SQLite Sync Repository", + "reward": 0.39, + "status": "partial" + } + ] + }, + { + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent_name": "gemini-cli", + "model_name": "gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "provider": "Google", + "variant": "adhoc", + "n_trials": 5, + "n_errors": 2, + "mean_reward": 0.33, + "outcome_score": 0.32, + "quality_score": 0.28, + "dx_score": 0.57, + "min_reward": 0.28, + "max_reward": 0.39, + "median_reward": 0.33, + "pass_at_1": 0.0, + "cost_usd": 0.0045000000000000005, + "input_tokens": 160224, + "output_tokens": 5547, + "has_dart_tooling": false, + "best_cujs": [], + "worst_cujs": [ + { + "task_slug": "flutter-manage-state-with-bloc", + "task_name": "Manage State with BLoC", + "reward": null, + "status": "error" + }, + { + "task_slug": "flutter-custom-render-object", + "task_name": "Custom RenderObject & Canvas", + "reward": null, + "status": "error" + }, + { + "task_slug": "flutter-adaptive-material-cupertino", + "task_name": "Adaptive Material & Cupertino UI", + "reward": 0.28, + "status": "partial" + } + ] + } + ] +} \ No newline at end of file diff --git a/sites/www/content/data/flutterbench/methodology.json b/sites/www/content/data/flutterbench/methodology.json new file mode 100644 index 00000000000..8cd2db26856 --- /dev/null +++ b/sites/www/content/data/flutterbench/methodology.json @@ -0,0 +1,827 @@ +{ + "overview": { + "lead_text": "FlutterBench is our evaluation framework designed to measure how AI coding agents perform within the Dart and Flutter ecosystem. The evaluation system consists of four core components:", + "rows": [ + { + "label": "Dataset", + "description": "Real-world development tasks derived from critical user journeys (CUJs).", + "anchor": "dataset-tasks" + }, + { + "label": "Test matrix", + "description": "Multidimensional testing framework across models, agents, tooling configurations, and SDK versions.", + "anchor": "evaluation-test-matrix" + }, + { + "label": "Scoring system", + "description": "Unified grading approach evaluating functional outcomes, code quality, and developer experience, paired with diagnostic telemetry.", + "anchor": "scoring-architecture" + }, + { + "label": "Harness", + "description": "Containerized automation infrastructure that executes evaluations at scale.", + "anchor": "evaluation-harness" + } + ] + }, + "task_anatomy": { + "intro_text": "Each task contains an instruction, a target codebase and environment, verification criteria, and metadata. Using the CUJ example above, a corresponding Harbor task looks like this:", + "root_id": "task", + "root_label": "Task", + "tree": [ + { + "type": "folder", + "id": "environment", + "label": "environment/", + "subtitle": "Containerized Flutter workspace pre-seeded for the agent", + "starts_closed": false, + "body": "The target codebase is an isolated, containerized Flutter workspace.\n\nWhen a benchmark run starts, the evaluation harness boots an ephemeral Docker container pre-seeded with this project. The agent is given access to tools (such as reading files, editing code, and running terminal commands) to investigate and resolve the issue.\n\nThe agent only sees the files inside this directory. Grading scripts and reference solutions remain strictly isolated outside the container until the agent completes its run.", + "children": [ + { + "type": "folder", + "id": "lib", + "label": "lib/", + "subtitle": "Application source code", + "body": "Contains the Flutter application source code.\n\nIn this evaluation task, the agent inspects `lib/main.dart` to locate the source of the `RenderFlex` layout errors and applies appropriate widget modifications.", + "children": [ + { + "type": "file", + "id": "main-dart", + "label": "main.dart", + "subtitle": "Flutter app containing horizontal and vertical overflow bugs", + "body": "The starter code for the Flutter application. Alternatively, the docker file may point to a remote codebase." + } + ] + }, + { + "type": "folder", + "id": "test", + "label": "test/", + "subtitle": "Target directory for agent-authored widget tests", + "body": "The test directory for the target project.\n\nInitially, this directory is empty. The task instruction directs the agent to create `test/main_test.dart` to verify its bug fix with automated widget tests.", + "children": [ + { + "type": "file", + "id": "main-test-dart", + "label": "main_test.dart", + "subtitle": "Widget tests the agent must write to verify its fix", + "body": "The widget test file that the agent is expected to author." + } + ] + }, + { + "type": "file", + "id": "pubspec", + "label": "pubspec.yaml", + "subtitle": "Project manifest declaring Flutter and lint dependencies", + "body": "Declares dependencies and environment constraints for the Flutter project." + }, + { + "type": "file", + "id": "analysis-options", + "label": "analysis_options.yaml", + "subtitle": "Linter configuration enforcing const and style rules", + "body": "Defines the static analysis rules enforced across the project.\n\nDuring grading, `StaticAnalysisGrader` runs `flutter analyze` against the modified codebase. Solutions that introduce analyzer errors, warnings, or lint violations (such as missing `const` constructors) lose quality points.", + "code": { + "lang": "yaml", + "text": "include: package:flutter_lints/flutter.yaml\n\nlinter:\n rules:\n # Strict lints for this task" + } + }, + { + "type": "file", + "id": "dockerfile", + "label": "Dockerfile", + "subtitle": "Container setup, SDK caching, and baseline commit", + "body": "Defines the container image for the task environment.\n\nWorkflow:\n1. Inherits from the pre-warmed `flutter-linux:latest` base image, which contains the Flutter and Dart SDKs.\n2. Copies project manifests and initial source files into `/workspace`.\n3. Runs `flutter pub get` so dependencies are pre-fetched.\n4. Executes `init-baseline` to snapshot the clean repository state. This allows the evaluation harness to measure the agent's work as a precise Git diff against the starting baseline." + } + ] + }, + { + "type": "folder", + "id": "tests", + "label": "tests/", + "subtitle": "Multi-dimensional evaluation harness hidden from the agent", + "badge": "hidden from agent", + "badge_color": "warning", + "body": "The automated verification harness used to grade the agent's performance.\n\nThis entire directory is hidden from the agent during task execution. It is mounted into the evaluation container only when the grading phase starts.\n\nThe harness evaluates the agent's output across three weighted dimensions:\n\nOutcome: Functional correctness, test pass rates, and layout fixes.\nQuality: Static analysis, formatting, and code craftsmanship.\nDX: Developer experience and tool interaction efficiency.", + "children": [ + { + "type": "file", + "id": "graders-dart", + "label": "graders.dart", + "subtitle": "Outcome, quality, and craftsmanship scoring rules via eval_scoring", + "body": "Defines the multi-dimensional scoring pipeline using a separate scoring package.", + "code": { + "lang": "dart", + "text": "// Heavily edited\nAggregateGrader grader(context) =>\n AggregateGrader.result(\n outcome: _outcomeGraders,\n quality: _qualityGraders,\n dx: _dxGraders,\n );" + } + }, + { + "type": "file", + "id": "test-sh", + "label": "test.sh", + "subtitle": "Verification entry point running the eval_scoring suite", + "body": "The test harness entrypoint script executed inside the grading container.", + "code": { + "lang": "bash", + "text": "#!/bin/bash\neval_scoring run [tasks]" + } + } + ] + }, + { + "type": "folder", + "id": "solution", + "label": "solution/", + "subtitle": "Oracle reference solution used to validate the eval task", + "body": "Contains the ground truth reference implementation maintained by the benchmark authors.\n\nBefore a task is added to the benchmark suite, the reference solution is executed through the grading harness to confirm that it achieves a full `1.0` reward. This ensures the task is solvable, unambiguous, and calibrated correctly.", + "children": [ + { + "type": "file", + "id": "solve-sh", + "label": "solve.sh", + "subtitle": "Reference script fixing layout bugs and adding layout tests", + "body": "The reference shell script that applies the canonical fix and generates comprehensive tests. Not safe to share publicly." + } + ] + }, + { + "type": "file", + "id": "instruction", + "label": "instruction.md", + "subtitle": "The prompt the agent receives", + "badge": "input", + "badge_color": "tip", + "is_default_page": true, + "body": "The instruction is the task prompt provided to the agent. It mimics real-world workflows from developers, and is written in a way that real developers interact with agents.\n\nDesign principles:\n\nPrompts state symptoms and expected outcomes without naming exact remedy widgets.\nPrompts require the agent to write regression tests, measuring both implementation skills and testing rigor.", + "code": { + "lang": "markdown", + "text": "We currently keep receiving `RenderFlex overflowed` errors both horizontally and vertically. Implement the correct widgets to resolve these layout issues.\n\nWrite widget tests in\n`test/main_test.dart` that verifies the changes." + } + }, + { + "type": "file", + "id": "task-toml", + "label": "task.toml", + "subtitle": "Task definition, difficulty notes, and timeouts", + "body": "The task configuration file specifies execution bounds, metadata, and target artifacts evaluated during grading.\n\nKey sections:\n\n`artifacts`: Lists files that must be present in the workspace after the run.\n`task` definition: Task-related data, like name, author and tags.\ninfra and agent configurations, such as timeout limits and retries." + } + ] + }, + "grader_tiers": { + "rows": [ + { + "label": "Code-based", + "detail": "Compilers, test runners, `dart analyze`, DCM, structural checkers", + "description": "Deterministic, unambiguous source of truth for syntax, compilation, and functional test assertions." + }, + { + "label": "LLM judge (BINEVAL)", + "detail": "Frontier model rubric evaluation", + "description": "Evaluates qualitative dimensions (visual UI, idiomatic review, trajectory, recovery) using binary yes/no questions." + }, + { + "label": "Human audit", + "detail": "Flutter engineer manual review", + "description": "Ground truth calibration, failure root-cause analysis, and conflict resolution." + } + ] + }, + "diagnostic_telemetry": { + "rows": [ + { + "label": "Token usage", + "description": "Tracks total input, cache, and output tokens consumed to measure efficiency deltas and verify token reductions from skill optimizations." + }, + { + "label": "Expected tool calls", + "description": "Compares actual tool invocations against expected tools. If an agent succeeds without using an expected tool, it is not penalized; this telemetry helps evaluate whether the tool is necessary for that user journey." + } + ] + }, + "root_cause_audits": { + "items": [ + { + "label": "Reasoning trace review", + "detail": "Inspect the agent's internal thoughts to identify where misunderstandings of Dart/Flutter APIs occurred." + }, + { + "label": "Plan adherence audit", + "detail": "Check whether the agent derailed due to ambiguous task prompts or missing context." + }, + { + "label": "Harness diagnostics", + "detail": "Audit error messages returned to the agent during failed compile/test steps to see why recovery failed." + } + ] + }, + "transparency": { + "harbor_example": { + "task": "dataset/flutter/manage-state-with-bloc", + "agent": "antigravity-sdk", + "model": "google/gemini-3.5-flash", + "mcp": "dart" + } + }, + "cuj_example": [ + { + "label": "User", + "icon": "person", + "variant": "grey", + "items": [ + "As an application developer" + ] + }, + { + "label": "Goal", + "icon": "star", + "variant": "blue", + "items": [ + "I want to resolve layout overflow errors in UI component trees" + ] + }, + { + "label": "Tasks", + "icon": "apps", + "variant": "purple", + "items": [ + "I use the error messaging to identify widgets causing the layout overflow in the widget tree.", + "I refactor the layout using flexible scrolling or bounding widgets to resolve the overflow error.", + "Implement widget tests to prevent regressions." + ] + } + ], + "task_specifications": [ + { + "id": "structure", + "title": "Task Structure", + "icon": "schema", + "expanded": true, + "description": "Directory layout, instructions, environments, and verification criteria.", + "lead_text": "Each task lives in its own directory and contains four elements:", + "tables": [ + { + "headers": [ + "Element", + "Description" + ], + "rows": [ + { + "label": "Instruction", + "description": "A realistic prompt written the way developers talk to agents (typically 1\u20132 sentences, behavior-focused rather than prescriptive)." + }, + { + "label": "Target codebase & environment", + "description": "A containerized Docker environment preseeded with a Dart or Flutter codebase, testing greenfield generation or existing codebases with bugs or debt." + }, + { + "label": "Verification criteria", + "description": "Verification scripts (`tests/graders.dart` and `test.sh`) executing `package:eval_scoring`." + }, + { + "label": "Configuration & metadata", + "description": "The `task.toml` configuration defining associated CUJs, priority tiers, expected skills, and MCP tools." + } + ] + } + ] + }, + { + "id": "categories", + "title": "Task Categories", + "icon": "category", + "expanded": false, + "description": "Primary agent capabilities and workflows evaluated across benchmarks.", + "lead_text": "Tasks are categorized by the primary agent capability being evaluated:", + "tables": [ + { + "headers": [ + "Category", + "Description" + ], + "rows": [ + { + "label": "Greenfield generation", + "description": "Creating new features or applications from scratch." + }, + { + "label": "Hill climbing", + "description": "Iterative debugging, test repair, and multi-turn problem-solving." + }, + { + "label": "Refactoring", + "description": "Restructuring existing code while maintaining functionality." + }, + { + "label": "Migration", + "description": "Upgrading deprecated APIs or transitioning between architectural patterns." + }, + { + "label": "Integration", + "description": "Adding platform-specific features, native plugins, or third-party packages." + } + ] + } + ] + }, + { + "id": "tiers", + "title": "Task Tiers & Prioritization", + "icon": "layers", + "expanded": false, + "description": "Execution cadences and priority levels balancing coverage and speed.", + "lead_text": "To balance comprehensive coverage with evaluation speed, tasks are organized into execution tiers and priorities:", + "tables": [ + { + "title": "Execution tiers", + "headers": [ + "Tier", + "Description" + ], + "rows": [ + { + "label": "Tier 1", + "description": "Core benchmark tasks executed monthly across the complete evaluation matrix." + }, + { + "label": "Tier 2", + "description": "Maturing tasks slated to graduate into Tier 1 once calibrated." + }, + { + "label": "Tier 3", + "description": "Experimental tasks used for ad-hoc investigations and targeted questions." + } + ] + }, + { + "title": "Task priority", + "headers": [ + "Priority", + "Description" + ], + "rows": [ + { + "label": "P0", + "description": "Critical production workflows and high-frequency productivity tasks" + }, + { + "label": "P1", + "description": "Core functionality and API consistency verification" + }, + { + "label": "P2", + "description": "Standard features and application maturity tasks" + }, + { + "label": "P3", + "description": "Edge cases and cosmetic polish" + } + ] + } + ] + }, + { + "id": "qa", + "title": "Quality Assurance", + "icon": "verified", + "expanded": false, + "description": "Engineering audits and human reviewer calibration for dataset integrity.", + "lead_text": "Before a task graduates into the core benchmark suite, it undergoes an engineering audit. Human reviewers inspect initial trial runs and label results as:", + "tables": [ + { + "headers": [ + "Audit label", + "Definition" + ], + "rows": [ + { + "label": "True positive", + "description": "Agent correctly passed the task." + }, + { + "label": "True negative", + "description": "Agent correctly failed the task." + }, + { + "label": "False positive", + "description": "Agent passed incorrectly due to overly lenient checks." + }, + { + "label": "False negative", + "description": "Agent failed incorrectly due to brittle or flaky tests." + } + ] + } + ], + "footer_text": "This process ensures that the dataset produces reliable, actionable signals rather than noise." + } + ], + "evaluation_matrix": { + "title": "4-Axis Evaluation Matrix", + "description": "Select an evaluation axis to explore its configurations, supported targets, and methodology.", + "axes": [ + { + "id": "configurations", + "tab_label": "1. Configurations", + "tab_sublabel": "Tooling Modes", + "heading": "Axis 1: Tooling Configurations", + "badge": "3 Setups", + "variant": "functional", + "overview": "FlutterBench tests three configurations to isolate the impact of specialized Dart and Flutter AI tools (skills, MCP tools, compiler diagnostics, and sandboxes).", + "items_label": "Configurations Tested:", + "items": [ + { + "label": "Baseline", + "detail": "No specialized Dart or Flutter tools loaded \u2014 tests raw model capability and baseline reasoning." + }, + { + "label": "Enhanced", + "detail": "Full suite of Dart and Flutter skills, MCP server tools, and compiler diagnostics loaded." + }, + { + "label": "Enhanced Minus N", + "detail": "Full suite with a specific tool or skill group ablated to measure its isolated delta." + } + ], + "footer_text": "This A/B testing approach isolates whether adding specific tools (such as widget inspection or analyzer fixes) measurably improves task success rates." + }, + { + "id": "models", + "tab_label": "2. Frontier Models", + "tab_sublabel": "Model Families", + "heading": "Axis 2: Frontier Model Families", + "badge": "3 Families", + "variant": "quality", + "overview": "Evaluations span major frontier model families across high-capability reasoning tiers and low-latency production tiers.", + "items_label": "Model Families Evaluated:", + "items": [ + { + "label": "Gemini", + "detail": "Evaluated across Gemini Pro (deep reasoning and complex architecture) and Gemini Flash (low-latency generation) tiers." + }, + { + "label": "Claude", + "detail": "Evaluated across Claude Opus (heavy multi-file refactoring) and Claude Sonnet (daily developer workflows)." + }, + { + "label": "ChatGPT", + "detail": "Evaluated across the GPT-4o series and specialized reasoning checkpoints." + } + ], + "footer_text": "Additional providers and local open-weights models are integrated continuously based on community feedback and developer adoption." + }, + { + "id": "harnesses", + "tab_label": "3. Agent Harnesses", + "tab_sublabel": "CLI Runtimes", + "heading": "Axis 3: Agent Runtime Harnesses", + "badge": "2 Harnesses", + "variant": "dx", + "overview": "Different agents employ distinct system prompts, context management techniques, and tool-loading strategies. We evaluate the CLI agents Flutter developers use most.", + "items_label": "Agent Harnesses Tested:", + "items": [ + { + "label": "Antigravity CLI", + "detail": "Google's developer agent harness featuring optimized tool discovery, compact context budgets, and deep IDE integration." + }, + { + "label": "Claude Code", + "detail": "Anthropic's terminal-based agent harness with autonomous multi-turn problem-solving capabilities." + } + ], + "footer_text": "Testing across multiple harnesses isolates model intelligence from agent runtime orchestration, measuring how execution environments influence success." + }, + { + "id": "sdks", + "tab_label": "4. SDK Branches", + "tab_sublabel": "Release Channels", + "heading": "Axis 4: SDK Release Channels", + "badge": "2 Channels", + "variant": "perfect", + "overview": "Evaluations run against both Flutter release channels to verify developer-ready reliability and detect framework regressions early.", + "items_label": "Release Channels Tracked:", + "items": [ + { + "label": "Stable", + "detail": "The current production SDK release \u2014 measures developer-ready reliability on the version used by most teams." + }, + { + "label": "Beta", + "detail": "Monthly beta releases \u2014 catches deprecation migrations, API evolutions, and analyzer changes before stable rollout." + } + ], + "footer_text": "Ensures AI tooling and skills maintain precision as the Flutter framework and Dart language evolve." + } + ] + }, + "dimensions": [ + { + "title": "1. Outcome", + "icon": "check_circle", + "category": "outcome", + "description": "Evaluates whether the generated code compiles, runs, and fulfills all functional UI and task requirements correctly.", + "footer_items": "Build, Tests, Visual, Heuristics", + "badge": "Functional" + }, + { + "title": "2. Code Quality", + "icon": "code", + "category": "quality", + "description": "Measures code maintainability, static analysis lints, architectural health, and idiomatic Dart 3 standards.", + "footer_items": "Lints, DCM, Structure, Patterns", + "badge": "Maintainable" + }, + { + "title": "3. Developer Experience", + "icon": "bolt", + "category": "dx", + "description": "Tracks agent execution friction, tool call accuracy, error recovery loops, and step efficiency.", + "footer_items": "Tooling, Trajectory, Recovery", + "badge": "Efficiency" + } + ], + "grader_matrix": { + "title": "Grader Matrix", + "description": "Explore deterministic, LLM-as-a-judge, and heuristic evaluation graders.", + "filters": [ + { + "id": "all", + "label": "All Graders" + }, + { + "id": "outcome", + "label": "Outcome" + }, + { + "id": "quality", + "label": "Quality" + }, + { + "id": "dx", + "label": "DX" + } + ], + "graders": [ + { + "name": "Build & Run", + "category": "outcome", + "category_label": "Outcome", + "type": "deterministic", + "type_label": "Deterministic", + "description": "Binary pass/fail checking if code compiles and launches cleanly." + }, + { + "name": "Unit & Widget Testing", + "category": "outcome", + "category_label": "Outcome", + "type": "deterministic", + "type_label": "Deterministic", + "description": "Executes test suites and scores passing test ratio across affected code." + }, + { + "name": "Visual Validation", + "category": "outcome", + "category_label": "Outcome", + "type": "llm", + "type_label": "LLM-as-Judge", + "description": "Compares DevTools screenshot captures against visual UI expectations." + }, + { + "name": "Task Heuristics", + "category": "outcome", + "category_label": "Outcome", + "type": "heuristic", + "type_label": "Heuristic", + "description": "Task-specific checks verifying dependency additions, required config, and files." + }, + { + "name": "Static Analysis", + "category": "quality", + "category_label": "Quality", + "type": "deterministic", + "type_label": "Deterministic", + "description": "Runs `dart analyze` enforcing strict project lint rules." + }, + { + "name": "Dart Code Metrics (DCM)", + "category": "quality", + "category_label": "Quality", + "type": "deterministic", + "type_label": "Deterministic", + "description": "Detects dead code, widget complexity, undisposed controllers, and memory leaks." + }, + { + "name": "Idiomatic Dart Review", + "category": "quality", + "category_label": "Quality", + "type": "llm", + "type_label": "LLM-as-Judge", + "description": "Evaluates modern Dart 3 pattern compliance and adherence to Effective Dart." + }, + { + "name": "Structural Validation", + "category": "quality", + "category_label": "Quality", + "type": "deterministic", + "type_label": "Deterministic", + "description": "Verifies directory structure, naming conventions, and required file placement." + }, + { + "name": "Tool Usage Analysis", + "category": "dx", + "category_label": "DX", + "type": "deterministic", + "type_label": "Deterministic", + "description": "Checks whether appropriate MCP tools and CLI utilities were called effectively." + }, + { + "name": "Step & Token Economy", + "category": "dx", + "category_label": "DX", + "type": "deterministic", + "type_label": "Deterministic", + "description": "Measures trajectory length, token burn, and redundant execution steps." + }, + { + "name": "Error Recovery", + "category": "dx", + "category_label": "DX", + "type": "llm", + "type_label": "LLM-as-Judge", + "description": "Rates how fluidly the agent adapts to compiler, analysis, and test errors." + } + ] + }, + "reliability": { + "cards": [ + { + "tag": "Ceiling Metric", + "math_pill": "pass@k", + "title": "Capability", + "is_north_star": false, + "description_parts": [ + { + "text": "The probability that an agent succeeds " + }, + { + "em": "at least once" + }, + { + "text": " across " + }, + { + "strong": "k" + }, + { + "text": " attempts. Shows what the model can achieve under ideal sample runs." + } + ] + }, + { + "tag": "Our North Star", + "tag_icon": "star", + "math_pill": "pass^k", + "title": "Consistency", + "is_north_star": true, + "description_parts": [ + { + "text": "The probability that an agent succeeds " + }, + { + "em": "every single time" + }, + { + "text": " across all " + }, + { + "strong": "k" + }, + { + "text": " attempts. Solves unreliability friction for real developer workflows." + } + ] + } + ] + }, + "score_triage": { + "title": "Score Triage & Action Matrix", + "description": "Select a score range to view its criteria and immediate engineering triage actions.", + "tiers": [ + { + "id": "perfect", + "score": "1.00", + "name": "Perfect Success", + "heading": "Score 1.00 \u2014 Perfect Success", + "badge": "Criteria & Triage", + "criteria": "Code compiles, runs, and passes all unit and widget tests. Zero analyzer or DCM warnings. Idiomatic Dart 3 and flawless DX.", + "actions_label": "Engineering Triage Actions:", + "actions": [ + { + "label": "Telemetry check", + "detail": "Validate whether all expected tools were utilized." + }, + { + "label": "Skip optimization", + "detail": "If expected tools were skipped successfully, re-evaluate if the tool is redundant." + }, + { + "label": "Token burn review", + "detail": "If token consumption was high, optimize prompt context and create fast-path shortcut tools." + } + ] + }, + { + "id": "functional", + "score": "0.75 \u2013 0.99", + "name": "Minor Flaws", + "heading": "Score 0.75 \u2013 0.99 \u2014 Minor Flaws", + "badge": "Criteria & Triage", + "criteria": "Code works and functional tests pass, but exhibits minor lint warnings, slightly unidiomatic patterns, or minor DX friction.", + "actions_label": "Engineering Triage Actions:", + "actions": [ + { + "label": "Prompt tuning", + "detail": "Tune skill prompt instructions for better Dart 3 style enforcement." + }, + { + "label": "Schema refinement", + "detail": "Refine tool parameter schemas and validation logic to prevent minor parameter retry hiccups." + }, + { + "label": "Model collaboration", + "detail": "Collaborate with model team to polish code generation formatting." + } + ] + }, + { + "id": "partial", + "score": "0.50 \u2013 0.74", + "name": "Partial Success", + "heading": "Score 0.50 \u2013 0.74 \u2014 Partial Success", + "badge": "Criteria & Triage", + "criteria": "Core requirements work but some widget or unit tests fail, lint warnings are significant, or the agent entered noticeable recovery loops.", + "actions_label": "Engineering Triage Actions:", + "actions": [ + { + "label": "Assertion triage", + "detail": "Investigate specific test assertion failures." + }, + { + "label": "Context gap analysis", + "detail": "Analyze whether the agent lacked key framework or package context." + }, + { + "label": "Diagnostic feedback", + "detail": "Refine compiler diagnostic feedback to help the agent self-correct faster." + } + ] + }, + { + "id": "poor", + "score": "0.25 \u2013 0.49", + "name": "Poor Implementation", + "heading": "Score 0.25 \u2013 0.49 \u2014 Poor Implementation", + "badge": "Criteria & Triage", + "criteria": "Fails to compile, ignores constraints, or encountered severe developer experience breakdown during execution.", + "actions_label": "Engineering Triage Actions:", + "actions": [ + { + "label": "Tool discoverability", + "detail": "If expected tools were ignored, fix tool discoverability, frontmatter, and prompt instructions." + }, + { + "label": "Tool output clarity", + "detail": "If tools were used but failed, improve helper tool output clarity and error messaging." + }, + { + "label": "Loop prevention", + "detail": "If token burn was high due to endless fix loops, improve compiler errors for single-step healing." + } + ] + }, + { + "id": "failure", + "score": "0.00", + "name": "Total Failure", + "heading": "Score 0.00 \u2014 Total Failure", + "badge": "Criteria & Triage", + "criteria": "No working code produced, severe runtime crash, or agent caught in an infinite loop.", + "actions_label": "Engineering Triage Actions:", + "actions": [ + { + "label": "Human root-cause audit", + "detail": "Engineers inspect reasoning traces, plan adherence, and test harness logs." + }, + { + "label": "Behavior categorization", + "detail": "Identify premature surrender versus infinite loop traps." + }, + { + "label": "Constraint refinement", + "detail": "Update benchmark task prompt constraints and bounding parameters." + } + ] + } + ] + } +} \ No newline at end of file diff --git a/sites/www/content/data/flutterbench/tasks.json b/sites/www/content/data/flutterbench/tasks.json new file mode 100644 index 00000000000..324d2d19f9f --- /dev/null +++ b/sites/www/content/data/flutterbench/tasks.json @@ -0,0 +1,1099 @@ +{ + "tasks": [ + { + "slug": "flutter-custom-render-object", + "task_name": "google/flutter-custom-render-object", + "display_name": "Custom RenderObject & Canvas", + "category": "Advanced Rendering", + "description": "Implement a custom RenderBox with layout constraints, custom painting, intrinsic dimensions, and pointer hit-testing.", + "trials": [ + { + "trial_name": "flutter-custom-render-object__t15-claude-3-5-haiku", + "status": "partial", + "reward": 0.36, + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t35-gpt-4o-mini", + "status": "error", + "reward": null, + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": "AgentTimeoutError" + }, + { + "trial_name": "flutter-custom-render-object__t40-gemini-35-pro", + "status": "partial", + "reward": 0.69, + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t50-gemini-31-flash-lite", + "status": "error", + "reward": null, + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "agent_name": "gemini-cli", + "has_dart_tooling": false, + "exception_type": "AgentTimeoutError" + }, + { + "trial_name": "flutter-custom-render-object__t25-gpt-5", + "status": "partial", + "reward": 0.72, + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t10-claude-3-5-sonnet", + "status": "partial", + "reward": 0.52, + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t65-deepseek-coder-v2", + "status": "partial", + "reward": 0.42, + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t60-deepseek-v3", + "status": "partial", + "reward": 0.49, + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t55-deepseek-r1", + "status": "partial", + "reward": 0.72, + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "agent_name": "deepseek-cli", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t05-claude-3-7-sonnet", + "status": "partial", + "reward": 0.71, + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t20-o3", + "status": "partial", + "reward": 0.73, + "model_name": "openai/o3", + "model_short_name": "o3", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t30-gpt-4o", + "status": "partial", + "reward": 0.47, + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-custom-render-object__t45-gemini-35-flash", + "status": "partial", + "reward": 0.65, + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + } + ], + "scores_by_eval": { + "claude-code__claude-3-5-haiku__adhoc": { + "trial_name": "flutter-custom-render-object__t15-claude-3-5-haiku", + "status": "partial", + "reward": 0.36, + "exception_type": null + }, + "codex-agent__gpt-4o-mini__adhoc": { + "trial_name": "flutter-custom-render-object__t35-gpt-4o-mini", + "status": "error", + "reward": null, + "exception_type": "AgentTimeoutError" + }, + "antigravity-sdk__gemini-3.5-pro__adhoc": { + "trial_name": "flutter-custom-render-object__t40-gemini-35-pro", + "status": "partial", + "reward": 0.69, + "exception_type": null + }, + "gemini-cli__gemini-3.1-flash-lite__adhoc": { + "trial_name": "flutter-custom-render-object__t50-gemini-31-flash-lite", + "status": "error", + "reward": null, + "exception_type": "AgentTimeoutError" + }, + "codex-agent__gpt-5__adhoc": { + "trial_name": "flutter-custom-render-object__t25-gpt-5", + "status": "partial", + "reward": 0.72, + "exception_type": null + }, + "claude-code__claude-3-5-sonnet__adhoc": { + "trial_name": "flutter-custom-render-object__t10-claude-3-5-sonnet", + "status": "partial", + "reward": 0.52, + "exception_type": null + }, + "deepseek-cli__deepseek-coder-v2__adhoc": { + "trial_name": "flutter-custom-render-object__t65-deepseek-coder-v2", + "status": "partial", + "reward": 0.42, + "exception_type": null + }, + "deepseek-cli__deepseek-v3__adhoc": { + "trial_name": "flutter-custom-render-object__t60-deepseek-v3", + "status": "partial", + "reward": 0.49, + "exception_type": null + }, + "deepseek-cli__deepseek-r1__adhoc": { + "trial_name": "flutter-custom-render-object__t55-deepseek-r1", + "status": "partial", + "reward": 0.72, + "exception_type": null + }, + "claude-code__claude-3-7-sonnet__adhoc": { + "trial_name": "flutter-custom-render-object__t05-claude-3-7-sonnet", + "status": "partial", + "reward": 0.71, + "exception_type": null + }, + "codex-agent__o3__adhoc": { + "trial_name": "flutter-custom-render-object__t20-o3", + "status": "partial", + "reward": 0.73, + "exception_type": null + }, + "codex-agent__gpt-4o__adhoc": { + "trial_name": "flutter-custom-render-object__t30-gpt-4o", + "status": "partial", + "reward": 0.47, + "exception_type": null + }, + "antigravity-sdk__gemini-3.5-flash__adhoc": { + "trial_name": "flutter-custom-render-object__t45-gemini-35-flash", + "status": "partial", + "reward": 0.65, + "exception_type": null + } + } + }, + { + "slug": "dart-build-cli-app", + "task_name": "google/dart-build-cli-app", + "display_name": "Build Command-Line CLI App", + "category": "Dart Utilities", + "description": "Create a robust Dart command-line interface application with argument parsing, formatted output, and exit code handling.", + "trials": [ + { + "trial_name": "dart-build-cli-app__t01-claude-3-7-sonnet", + "status": "pass", + "reward": 0.97, + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t21-gpt-5", + "status": "pass", + "reward": 0.98, + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t16-o3", + "status": "pass", + "reward": 0.97, + "model_name": "openai/o3", + "model_short_name": "o3", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t11-claude-3-5-haiku", + "status": "partial", + "reward": 0.56, + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t46-gemini-31-flash-lite", + "status": "partial", + "reward": 0.39, + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "agent_name": "gemini-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t06-claude-3-5-sonnet", + "status": "partial", + "reward": 0.79, + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t26-gpt-4o", + "status": "partial", + "reward": 0.71, + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t56-deepseek-v3", + "status": "partial", + "reward": 0.71, + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t36-gemini-35-pro", + "status": "pass", + "reward": 0.98, + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t51-deepseek-r1", + "status": "pass", + "reward": 0.99, + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "agent_name": "deepseek-cli", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t41-gemini-35-flash", + "status": "pass", + "reward": 0.93, + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t31-gpt-4o-mini", + "status": "partial", + "reward": 0.48, + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "dart-build-cli-app__t61-deepseek-coder-v2", + "status": "partial", + "reward": 0.61, + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + } + ], + "scores_by_eval": { + "claude-code__claude-3-7-sonnet__adhoc": { + "trial_name": "dart-build-cli-app__t01-claude-3-7-sonnet", + "status": "pass", + "reward": 0.97, + "exception_type": null + }, + "codex-agent__gpt-5__adhoc": { + "trial_name": "dart-build-cli-app__t21-gpt-5", + "status": "pass", + "reward": 0.98, + "exception_type": null + }, + "codex-agent__o3__adhoc": { + "trial_name": "dart-build-cli-app__t16-o3", + "status": "pass", + "reward": 0.97, + "exception_type": null + }, + "claude-code__claude-3-5-haiku__adhoc": { + "trial_name": "dart-build-cli-app__t11-claude-3-5-haiku", + "status": "partial", + "reward": 0.56, + "exception_type": null + }, + "gemini-cli__gemini-3.1-flash-lite__adhoc": { + "trial_name": "dart-build-cli-app__t46-gemini-31-flash-lite", + "status": "partial", + "reward": 0.39, + "exception_type": null + }, + "claude-code__claude-3-5-sonnet__adhoc": { + "trial_name": "dart-build-cli-app__t06-claude-3-5-sonnet", + "status": "partial", + "reward": 0.79, + "exception_type": null + }, + "codex-agent__gpt-4o__adhoc": { + "trial_name": "dart-build-cli-app__t26-gpt-4o", + "status": "partial", + "reward": 0.71, + "exception_type": null + }, + "deepseek-cli__deepseek-v3__adhoc": { + "trial_name": "dart-build-cli-app__t56-deepseek-v3", + "status": "partial", + "reward": 0.71, + "exception_type": null + }, + "antigravity-sdk__gemini-3.5-pro__adhoc": { + "trial_name": "dart-build-cli-app__t36-gemini-35-pro", + "status": "pass", + "reward": 0.98, + "exception_type": null + }, + "deepseek-cli__deepseek-r1__adhoc": { + "trial_name": "dart-build-cli-app__t51-deepseek-r1", + "status": "pass", + "reward": 0.99, + "exception_type": null + }, + "antigravity-sdk__gemini-3.5-flash__adhoc": { + "trial_name": "dart-build-cli-app__t41-gemini-35-flash", + "status": "pass", + "reward": 0.93, + "exception_type": null + }, + "codex-agent__gpt-4o-mini__adhoc": { + "trial_name": "dart-build-cli-app__t31-gpt-4o-mini", + "status": "partial", + "reward": 0.48, + "exception_type": null + }, + "deepseek-cli__deepseek-coder-v2__adhoc": { + "trial_name": "dart-build-cli-app__t61-deepseek-coder-v2", + "status": "partial", + "reward": 0.61, + "exception_type": null + } + } + }, + { + "slug": "flutter-offline-sync-sqlite", + "task_name": "google/flutter-offline-sync-sqlite", + "display_name": "Offline SQLite Sync Repository", + "category": "Data & Storage", + "description": "Build an offline-first repository using SQLite with background synchronization, retry queues, and conflict resolution.", + "trials": [ + { + "trial_name": "flutter-offline-sync-sqlite__t13-claude-3-5-haiku", + "status": "partial", + "reward": 0.48, + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t38-gemini-35-pro", + "status": "pass", + "reward": 0.91, + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t33-gpt-4o-mini", + "status": "partial", + "reward": 0.39, + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t03-claude-3-7-sonnet", + "status": "pass", + "reward": 0.93, + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t53-deepseek-r1", + "status": "pass", + "reward": 0.87, + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "agent_name": "deepseek-cli", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t48-gemini-31-flash-lite", + "status": "partial", + "reward": 0.33, + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "agent_name": "gemini-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t28-gpt-4o", + "status": "partial", + "reward": 0.65, + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t58-deepseek-v3", + "status": "partial", + "reward": 0.64, + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t43-gemini-35-flash", + "status": "pass", + "reward": 0.8, + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t08-claude-3-5-sonnet", + "status": "partial", + "reward": 0.71, + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t23-gpt-5", + "status": "pass", + "reward": 0.86, + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t63-deepseek-coder-v2", + "status": "partial", + "reward": 0.55, + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t18-o3", + "status": "pass", + "reward": 0.9, + "model_name": "openai/o3", + "model_short_name": "o3", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + } + ], + "scores_by_eval": { + "claude-code__claude-3-5-haiku__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t13-claude-3-5-haiku", + "status": "partial", + "reward": 0.48, + "exception_type": null + }, + "antigravity-sdk__gemini-3.5-pro__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t38-gemini-35-pro", + "status": "pass", + "reward": 0.91, + "exception_type": null + }, + "codex-agent__gpt-4o-mini__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t33-gpt-4o-mini", + "status": "partial", + "reward": 0.39, + "exception_type": null + }, + "claude-code__claude-3-7-sonnet__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t03-claude-3-7-sonnet", + "status": "pass", + "reward": 0.93, + "exception_type": null + }, + "deepseek-cli__deepseek-r1__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t53-deepseek-r1", + "status": "pass", + "reward": 0.87, + "exception_type": null + }, + "gemini-cli__gemini-3.1-flash-lite__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t48-gemini-31-flash-lite", + "status": "partial", + "reward": 0.33, + "exception_type": null + }, + "codex-agent__gpt-4o__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t28-gpt-4o", + "status": "partial", + "reward": 0.65, + "exception_type": null + }, + "deepseek-cli__deepseek-v3__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t58-deepseek-v3", + "status": "partial", + "reward": 0.64, + "exception_type": null + }, + "antigravity-sdk__gemini-3.5-flash__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t43-gemini-35-flash", + "status": "pass", + "reward": 0.8, + "exception_type": null + }, + "claude-code__claude-3-5-sonnet__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t08-claude-3-5-sonnet", + "status": "partial", + "reward": 0.71, + "exception_type": null + }, + "codex-agent__gpt-5__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t23-gpt-5", + "status": "pass", + "reward": 0.86, + "exception_type": null + }, + "deepseek-cli__deepseek-coder-v2__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t63-deepseek-coder-v2", + "status": "partial", + "reward": 0.55, + "exception_type": null + }, + "codex-agent__o3__adhoc": { + "trial_name": "flutter-offline-sync-sqlite__t18-o3", + "status": "pass", + "reward": 0.9, + "exception_type": null + } + } + }, + { + "slug": "flutter-manage-state-with-bloc", + "task_name": "google/flutter-manage-state-with-bloc", + "display_name": "Manage State with BLoC", + "category": "State Management", + "description": "Implement an immutable state management layer using package:flutter_bloc, connecting UI events to business logic with unit tests.", + "trials": [ + { + "trial_name": "flutter-manage-state-with-bloc__t62-deepseek-coder-v2", + "status": "partial", + "reward": 0.59, + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t52-deepseek-r1", + "status": "pass", + "reward": 0.94, + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "agent_name": "deepseek-cli", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t32-gpt-4o-mini", + "status": "partial", + "reward": 0.46, + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t17-o3", + "status": "pass", + "reward": 0.97, + "model_name": "openai/o3", + "model_short_name": "o3", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t37-gemini-35-pro", + "status": "pass", + "reward": 0.99, + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t27-gpt-4o", + "status": "partial", + "reward": 0.69, + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t12-claude-3-5-haiku", + "status": "partial", + "reward": 0.55, + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t07-claude-3-5-sonnet", + "status": "partial", + "reward": 0.75, + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t22-gpt-5", + "status": "pass", + "reward": 0.97, + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t42-gemini-35-flash", + "status": "pass", + "reward": 0.88, + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t02-claude-3-7-sonnet", + "status": "pass", + "reward": 0.98, + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t47-gemini-31-flash-lite", + "status": "error", + "reward": null, + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "agent_name": "gemini-cli", + "has_dart_tooling": false, + "exception_type": "AgentTimeoutError" + }, + { + "trial_name": "flutter-manage-state-with-bloc__t57-deepseek-v3", + "status": "partial", + "reward": 0.67, + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + } + ], + "scores_by_eval": { + "deepseek-cli__deepseek-coder-v2__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t62-deepseek-coder-v2", + "status": "partial", + "reward": 0.59, + "exception_type": null + }, + "deepseek-cli__deepseek-r1__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t52-deepseek-r1", + "status": "pass", + "reward": 0.94, + "exception_type": null + }, + "codex-agent__gpt-4o-mini__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t32-gpt-4o-mini", + "status": "partial", + "reward": 0.46, + "exception_type": null + }, + "codex-agent__o3__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t17-o3", + "status": "pass", + "reward": 0.97, + "exception_type": null + }, + "antigravity-sdk__gemini-3.5-pro__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t37-gemini-35-pro", + "status": "pass", + "reward": 0.99, + "exception_type": null + }, + "codex-agent__gpt-4o__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t27-gpt-4o", + "status": "partial", + "reward": 0.69, + "exception_type": null + }, + "claude-code__claude-3-5-haiku__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t12-claude-3-5-haiku", + "status": "partial", + "reward": 0.55, + "exception_type": null + }, + "claude-code__claude-3-5-sonnet__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t07-claude-3-5-sonnet", + "status": "partial", + "reward": 0.75, + "exception_type": null + }, + "codex-agent__gpt-5__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t22-gpt-5", + "status": "pass", + "reward": 0.97, + "exception_type": null + }, + "antigravity-sdk__gemini-3.5-flash__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t42-gemini-35-flash", + "status": "pass", + "reward": 0.88, + "exception_type": null + }, + "claude-code__claude-3-7-sonnet__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t02-claude-3-7-sonnet", + "status": "pass", + "reward": 0.98, + "exception_type": null + }, + "gemini-cli__gemini-3.1-flash-lite__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t47-gemini-31-flash-lite", + "status": "error", + "reward": null, + "exception_type": "AgentTimeoutError" + }, + "deepseek-cli__deepseek-v3__adhoc": { + "trial_name": "flutter-manage-state-with-bloc__t57-deepseek-v3", + "status": "partial", + "reward": 0.67, + "exception_type": null + } + } + }, + { + "slug": "flutter-adaptive-material-cupertino", + "task_name": "google/flutter-adaptive-material-cupertino", + "display_name": "Adaptive Material & Cupertino UI", + "category": "Multi-Platform UI", + "description": "Build adaptive Flutter widgets that render Material 3 on Android/Web and native Cupertino patterns on iOS/macOS.", + "trials": [ + { + "trial_name": "flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet", + "status": "partial", + "reward": 0.61, + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t64-deepseek-coder-v2", + "status": "partial", + "reward": 0.5, + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t29-gpt-4o", + "status": "partial", + "reward": 0.6, + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t44-gemini-35-flash", + "status": "partial", + "reward": 0.78, + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t14-claude-3-5-haiku", + "status": "partial", + "reward": 0.45, + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "agent_name": "claude-code", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite", + "status": "partial", + "reward": 0.28, + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "agent_name": "gemini-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t34-gpt-4o-mini", + "status": "partial", + "reward": 0.39, + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "agent_name": "codex-agent", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t59-deepseek-v3", + "status": "partial", + "reward": 0.59, + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "agent_name": "deepseek-cli", + "has_dart_tooling": false, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t19-o3", + "status": "pass", + "reward": 0.85, + "model_name": "openai/o3", + "model_short_name": "o3", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t39-gemini-35-pro", + "status": "pass", + "reward": 0.87, + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "agent_name": "antigravity-sdk", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t54-deepseek-r1", + "status": "partial", + "reward": 0.78, + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "agent_name": "deepseek-cli", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t24-gpt-5", + "status": "pass", + "reward": 0.84, + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "agent_name": "codex-agent", + "has_dart_tooling": true, + "exception_type": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet", + "status": "pass", + "reward": 0.85, + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "agent_name": "claude-code", + "has_dart_tooling": true, + "exception_type": null + } + ], + "scores_by_eval": { + "claude-code__claude-3-5-sonnet__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet", + "status": "partial", + "reward": 0.61, + "exception_type": null + }, + "deepseek-cli__deepseek-coder-v2__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t64-deepseek-coder-v2", + "status": "partial", + "reward": 0.5, + "exception_type": null + }, + "codex-agent__gpt-4o__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t29-gpt-4o", + "status": "partial", + "reward": 0.6, + "exception_type": null + }, + "antigravity-sdk__gemini-3.5-flash__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t44-gemini-35-flash", + "status": "partial", + "reward": 0.78, + "exception_type": null + }, + "claude-code__claude-3-5-haiku__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t14-claude-3-5-haiku", + "status": "partial", + "reward": 0.45, + "exception_type": null + }, + "gemini-cli__gemini-3.1-flash-lite__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite", + "status": "partial", + "reward": 0.28, + "exception_type": null + }, + "codex-agent__gpt-4o-mini__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t34-gpt-4o-mini", + "status": "partial", + "reward": 0.39, + "exception_type": null + }, + "deepseek-cli__deepseek-v3__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t59-deepseek-v3", + "status": "partial", + "reward": 0.59, + "exception_type": null + }, + "codex-agent__o3__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t19-o3", + "status": "pass", + "reward": 0.85, + "exception_type": null + }, + "antigravity-sdk__gemini-3.5-pro__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t39-gemini-35-pro", + "status": "pass", + "reward": 0.87, + "exception_type": null + }, + "deepseek-cli__deepseek-r1__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t54-deepseek-r1", + "status": "partial", + "reward": 0.78, + "exception_type": null + }, + "codex-agent__gpt-5__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t24-gpt-5", + "status": "pass", + "reward": 0.84, + "exception_type": null + }, + "claude-code__claude-3-7-sonnet__adhoc": { + "trial_name": "flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet", + "status": "pass", + "reward": 0.85, + "exception_type": null + } + } + } + ] +} \ No newline at end of file diff --git a/sites/www/content/data/flutterbench/trials.json b/sites/www/content/data/flutterbench/trials.json new file mode 100644 index 00000000000..0d6434a61b5 --- /dev/null +++ b/sites/www/content/data/flutterbench/trials.json @@ -0,0 +1,13593 @@ +{ + "trials": [ + { + "trial_name": "flutter-custom-render-object__t15-claude-3-5-haiku", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.36, + "outcome_score": 0.35, + "quality_score": 0.29, + "dx_score": 0.64, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 78.0, + "verifier": 33.0 + }, + "input_tokens": 68757, + "cache_tokens": 47276, + "output_tokens": 2629, + "cost_usd": 0.0655, + "reward_tree": { + "reward": { + "score": 0.36, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.35, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (35%)." + }, + { + "name": "quality", + "value": 0.29, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (29%)." + }, + { + "name": "dx", + "value": 0.64, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (64%)." + } + ] + }, + "outcome": { + "score": 0.35, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.27999999999999997, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.35, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.29, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.29, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.29, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44799999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.64, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.64, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: claude-3-5-haiku\n// Verification score: 0.36 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: claude-3-5-haiku\n// Verification score: 0.36 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: claude-3-5-haiku\n// Verification score: 0.36 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.36)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t01-claude-3-7-sonnet", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "provider": "Anthropic", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.97, + "outcome_score": 1.0, + "quality_score": 0.94, + "dx_score": 0.93, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 67.0, + "verifier": 23.0 + }, + "input_tokens": 93916, + "cache_tokens": 70367, + "output_tokens": 4594, + "cost_usd": 0.3507, + "reward_tree": { + "reward": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 1.0, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (100%)." + }, + { + "name": "quality", + "value": 0.94, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (94%)." + }, + { + "name": "dx", + "value": 0.93, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (93%)." + } + ] + }, + "outcome": { + "score": 1.0, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 1.0, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.94, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.94, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.93, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-7-sonnet\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-7-sonnet\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-7-sonnet\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-7-sonnet\n// Verification score: 0.97 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Build Command-Line CLI App unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.97)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t13-claude-3-5-haiku", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.48, + "outcome_score": 0.49, + "quality_score": 0.41, + "dx_score": 0.6, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 126.0, + "verifier": 32.0 + }, + "input_tokens": 69726, + "cache_tokens": 54790, + "output_tokens": 2666, + "cost_usd": 0.0664, + "reward_tree": { + "reward": { + "score": 0.48, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.49, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (49%)." + }, + { + "name": "quality", + "value": 0.41, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (41%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.49, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.49, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.41, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.41, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.41, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-5-haiku\n// Verification score: 0.48 (partial)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-5-haiku\n// Verification score: 0.48 (partial)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-5-haiku\n// Verification score: 0.48 (partial)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-5-haiku\n// Verification score: 0.48 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Offline SQLite Sync Repository tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.48)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t62-deepseek-coder-v2", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.59, + "outcome_score": 0.62, + "quality_score": 0.53, + "dx_score": 0.64, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 97.0, + "verifier": 30.0 + }, + "input_tokens": 74695, + "cache_tokens": 49868, + "output_tokens": 3051, + "cost_usd": 0.0113, + "reward_tree": { + "reward": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.62, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (62%)." + }, + { + "name": "quality", + "value": 0.53, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (53%)." + }, + { + "name": "dx", + "value": 0.64, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (64%)." + } + ] + }, + "outcome": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.62, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.53, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.53, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.53, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44799999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.64, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.64, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-coder-v2\n// Verification score: 0.59 (partial)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-coder-v2\n// Verification score: 0.59 (partial)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-coder-v2\n// Verification score: 0.59 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-coder-v2\n// Verification score: 0.59 (partial)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-coder-v2\n// Verification score: 0.59 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Manage State with BLoC tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.59)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t38-gemini-35-pro", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.91, + "outcome_score": 0.91, + "quality_score": 0.88, + "dx_score": 0.95, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 84.0, + "verifier": 28.0 + }, + "input_tokens": 94065, + "cache_tokens": 65762, + "output_tokens": 4159, + "cost_usd": 0.1384, + "reward_tree": { + "reward": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.91, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (91%)." + }, + { + "name": "quality", + "value": 0.88, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (88%)." + }, + { + "name": "dx", + "value": 0.95, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (95%)." + } + ] + }, + "outcome": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.88, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.88, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.88, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.95, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.95, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.95, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.5-pro\n// Verification score: 0.91 (pass)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.5-pro\n// Verification score: 0.91 (pass)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.5-pro\n// Verification score: 0.91 (pass)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.5-pro\n// Verification score: 0.91 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Offline SQLite Sync Repository unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.91)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.61, + "outcome_score": 0.64, + "quality_score": 0.56, + "dx_score": 0.6, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 127.0, + "verifier": 27.0 + }, + "input_tokens": 80954, + "cache_tokens": 55986, + "output_tokens": 3752, + "cost_usd": 0.2991, + "reward_tree": { + "reward": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.64, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (64%)." + }, + { + "name": "quality", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (56%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.64, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.56, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.56, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-5-sonnet\n// Verification score: 0.61 (partial)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-5-sonnet\n// Verification score: 0.61 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-5-sonnet\n// Verification score: 0.61 (partial)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-5-sonnet\n// Verification score: 0.61 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.61)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t64-deepseek-coder-v2", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.5, + "outcome_score": 0.51, + "quality_score": 0.44, + "dx_score": 0.58, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 69.0, + "verifier": 26.0 + }, + "input_tokens": 70846, + "cache_tokens": 55111, + "output_tokens": 2894, + "cost_usd": 0.0107, + "reward_tree": { + "reward": { + "score": 0.5, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.51, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (51%)." + }, + { + "name": "quality", + "value": 0.44, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (44%)." + }, + { + "name": "dx", + "value": 0.58, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (58%)." + } + ] + }, + "outcome": { + "score": 0.51, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.51, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 0.51, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.44, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.44, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.44, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.40599999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-coder-v2\n// Verification score: 0.5 (partial)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-coder-v2\n// Verification score: 0.5 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-coder-v2\n// Verification score: 0.5 (partial)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-coder-v2\n// Verification score: 0.5 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.5)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t35-gpt-4o-mini", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "error", + "reward": null, + "outcome_score": null, + "quality_score": null, + "dx_score": null, + "exception_type": "AgentTimeoutError", + "exception_message": "Agent exceeded maximum timeout of 300.0 seconds during execution.", + "exception_traceback": "Traceback (most recent call last):\n File \"harbor/trial/trial.py\", line 1455, in _execute_agent\n raise TimeoutError(\"Agent exceeded timeout of 300.0s\")\nTimeoutError: Agent exceeded timeout of 300.0s\n", + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 305.0 + }, + "input_tokens": 0, + "cache_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "reward_tree": null, + "diagnostic_tree": {}, + "trajectory": null, + "artifacts": [], + "test_stdout": null, + "exception_log": "Exception: AgentTimeoutError\nAgent exceeded maximum timeout of 300.0 seconds during execution.\n\nTraceback (most recent call last):\n File \"harbor/trial/trial.py\", line 1455, in _execute_agent\n raise TimeoutError(\"Agent exceeded timeout of 300.0s\")\nTimeoutError: Agent exceeded timeout of 300.0s\n" + }, + { + "trial_name": "dart-build-cli-app__t21-gpt-5", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.98, + "outcome_score": 0.99, + "quality_score": 0.98, + "dx_score": 0.96, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 132.0, + "verifier": 32.0 + }, + "input_tokens": 94854, + "cache_tokens": 68917, + "output_tokens": 4419, + "cost_usd": 0.2813, + "reward_tree": { + "reward": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.99, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (99%)." + }, + { + "name": "quality", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (98%)." + }, + { + "name": "dx", + "value": 0.96, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (96%)." + } + ] + }, + "outcome": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.99, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.98, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.98, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.96, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-5\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-5\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-5\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-5\n// Verification score: 0.98 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Build Command-Line CLI App unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.98)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t52-deepseek-r1", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "provider": "DeepSeek", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.94, + "outcome_score": 0.93, + "quality_score": 0.96, + "dx_score": 0.97, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 130.0, + "verifier": 28.0 + }, + "input_tokens": 99704, + "cache_tokens": 66373, + "output_tokens": 4602, + "cost_usd": 0.0649, + "reward_tree": { + "reward": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.93, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (93%)." + }, + { + "name": "quality", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (96%)." + }, + { + "name": "dx", + "value": 0.97, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (97%)." + } + ] + }, + "outcome": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.96, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.96, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.97, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-r1\n// Verification score: 0.94 (pass)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-r1\n// Verification score: 0.94 (pass)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-r1\n// Verification score: 0.94 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-r1\n// Verification score: 0.94 (pass)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-r1\n// Verification score: 0.94 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Manage State with BLoC unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.94)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t29-gpt-4o", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.6, + "outcome_score": 0.63, + "quality_score": 0.54, + "dx_score": 0.57, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 72.0, + "verifier": 24.0 + }, + "input_tokens": 80107, + "cache_tokens": 56940, + "output_tokens": 3373, + "cost_usd": 0.234, + "reward_tree": { + "reward": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.63, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (63%)." + }, + { + "name": "quality", + "value": 0.54, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (54%)." + }, + { + "name": "dx", + "value": 0.57, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (57%)." + } + ] + }, + "outcome": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.63, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.54, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.54, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.54, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.39899999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-4o\n// Verification score: 0.6 (partial)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-4o\n// Verification score: 0.6 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-4o\n// Verification score: 0.6 (partial)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-4o\n// Verification score: 0.6 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.6)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t33-gpt-4o-mini", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.39, + "outcome_score": 0.38, + "quality_score": 0.33, + "dx_score": 0.63, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 65.0, + "verifier": 25.0 + }, + "input_tokens": 61681, + "cache_tokens": 47277, + "output_tokens": 2189, + "cost_usd": 0.0106, + "reward_tree": { + "reward": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.38, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (38%)." + }, + { + "name": "quality", + "value": 0.33, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (33%)." + }, + { + "name": "dx", + "value": 0.63, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (63%)." + } + ] + }, + "outcome": { + "score": 0.38, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.38, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.38, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.33, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.33, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.33, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44099999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-4o-mini\n// Verification score: 0.39 (partial)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-4o-mini\n// Verification score: 0.39 (partial)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-4o-mini\n// Verification score: 0.39 (partial)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-4o-mini\n// Verification score: 0.39 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Offline SQLite Sync Repository tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.39)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t03-claude-3-7-sonnet", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "provider": "Anthropic", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.93, + "outcome_score": 0.93, + "quality_score": 0.93, + "dx_score": 0.96, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 98.0, + "verifier": 26.0 + }, + "input_tokens": 99235, + "cache_tokens": 72794, + "output_tokens": 4854, + "cost_usd": 0.3705, + "reward_tree": { + "reward": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.93, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (93%)." + }, + { + "name": "quality", + "value": 0.93, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (93%)." + }, + { + "name": "dx", + "value": 0.96, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (96%)." + } + ] + }, + "outcome": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.93, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.93, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.96, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-7-sonnet\n// Verification score: 0.93 (pass)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-7-sonnet\n// Verification score: 0.93 (pass)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-7-sonnet\n// Verification score: 0.93 (pass)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-7-sonnet\n// Verification score: 0.93 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Offline SQLite Sync Repository unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.93)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t16-o3", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "codex-agent__o3__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/o3", + "model_short_name": "o3", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.97, + "outcome_score": 0.97, + "quality_score": 0.96, + "dx_score": 0.96, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 91.0, + "verifier": 32.0 + }, + "input_tokens": 121468, + "cache_tokens": 82862, + "output_tokens": 6073, + "cost_usd": 0.7288, + "reward_tree": { + "reward": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.97, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (97%)." + }, + { + "name": "quality", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (96%)." + }, + { + "name": "dx", + "value": 0.96, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (96%)." + } + ] + }, + "outcome": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.97, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.96, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.96, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.96, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: o3\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: o3\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: o3\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: o3\n// Verification score: 0.97 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Build Command-Line CLI App unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.97)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t40-gemini-35-pro", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "partial", + "reward": 0.69, + "outcome_score": 0.68, + "quality_score": 0.63, + "dx_score": 0.91, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 82.0, + "verifier": 30.0 + }, + "input_tokens": 95191, + "cache_tokens": 66257, + "output_tokens": 4208, + "cost_usd": 0.14, + "reward_tree": { + "reward": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.68, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (68%)." + }, + { + "name": "quality", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (63%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.68, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.68, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.63, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.63, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gemini-3.5-pro\n// Verification score: 0.69 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gemini-3.5-pro\n// Verification score: 0.69 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gemini-3.5-pro\n// Verification score: 0.69 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.69)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t32-gpt-4o-mini", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.46, + "outcome_score": 0.47, + "quality_score": 0.41, + "dx_score": 0.58, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 67.0, + "verifier": 20.0 + }, + "input_tokens": 57785, + "cache_tokens": 38850, + "output_tokens": 2050, + "cost_usd": 0.0099, + "reward_tree": { + "reward": { + "score": 0.46, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.47, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (47%)." + }, + { + "name": "quality", + "value": 0.41, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (41%)." + }, + { + "name": "dx", + "value": 0.58, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (58%)." + } + ] + }, + "outcome": { + "score": 0.47, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.47, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 0.5, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.41, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.41, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.41, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.40599999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o-mini\n// Verification score: 0.46 (partial)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o-mini\n// Verification score: 0.46 (partial)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o-mini\n// Verification score: 0.46 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o-mini\n// Verification score: 0.46 (partial)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o-mini\n// Verification score: 0.46 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Manage State with BLoC tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.46)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t53-deepseek-r1", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "provider": "DeepSeek", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.87, + "outcome_score": 0.86, + "quality_score": 0.87, + "dx_score": 0.97, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 96.0, + "verifier": 32.0 + }, + "input_tokens": 103946, + "cache_tokens": 78215, + "output_tokens": 4797, + "cost_usd": 0.0677, + "reward_tree": { + "reward": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.86, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (86%)." + }, + { + "name": "quality", + "value": 0.87, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (87%)." + }, + { + "name": "dx", + "value": 0.97, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (97%)." + } + ] + }, + "outcome": { + "score": 0.86, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.86, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.87, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.87, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.97, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-r1\n// Verification score: 0.87 (pass)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-r1\n// Verification score: 0.87 (pass)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-r1\n// Verification score: 0.87 (pass)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-r1\n// Verification score: 0.87 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Offline SQLite Sync Repository unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.87)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t11-claude-3-5-haiku", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.56, + "outcome_score": 0.58, + "quality_score": 0.49, + "dx_score": 0.63, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 120.0, + "verifier": 25.0 + }, + "input_tokens": 61382, + "cache_tokens": 46705, + "output_tokens": 2347, + "cost_usd": 0.0585, + "reward_tree": { + "reward": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.58, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (58%)." + }, + { + "name": "quality", + "value": 0.49, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (49%)." + }, + { + "name": "dx", + "value": 0.63, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (63%)." + } + ] + }, + "outcome": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.58, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.49, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.49, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44099999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-5-haiku\n// Verification score: 0.56 (partial)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-5-haiku\n// Verification score: 0.56 (partial)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-5-haiku\n// Verification score: 0.56 (partial)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-5-haiku\n// Verification score: 0.56 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Build Command-Line CLI App tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.56)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t44-gemini-35-flash", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "partial", + "reward": 0.78, + "outcome_score": 0.76, + "quality_score": 0.74, + "dx_score": 0.98, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 95.0, + "verifier": 24.0 + }, + "input_tokens": 79062, + "cache_tokens": 51452, + "output_tokens": 2889, + "cost_usd": 0.0068, + "reward_tree": { + "reward": { + "score": 0.78, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.76, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (76%)." + }, + { + "name": "quality", + "value": 0.74, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (74%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.76, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.76, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.74, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.74, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.74, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.5-flash\n// Verification score: 0.78 (partial)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.5-flash\n// Verification score: 0.78 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.5-flash\n// Verification score: 0.78 (partial)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.5-flash\n// Verification score: 0.78 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.78)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t50-gemini-31-flash-lite", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent_name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "provider": "Google", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "error", + "reward": null, + "outcome_score": null, + "quality_score": null, + "dx_score": null, + "exception_type": "AgentTimeoutError", + "exception_message": "Agent exceeded maximum timeout of 300.0 seconds during execution.", + "exception_traceback": "Traceback (most recent call last):\n File \"harbor/trial/trial.py\", line 1455, in _execute_agent\n raise TimeoutError(\"Agent exceeded timeout of 300.0s\")\nTimeoutError: Agent exceeded timeout of 300.0s\n", + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 305.0 + }, + "input_tokens": 0, + "cache_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "reward_tree": null, + "diagnostic_tree": {}, + "trajectory": null, + "artifacts": [], + "test_stdout": null, + "exception_log": "Exception: AgentTimeoutError\nAgent exceeded maximum timeout of 300.0 seconds during execution.\n\nTraceback (most recent call last):\n File \"harbor/trial/trial.py\", line 1455, in _execute_agent\n raise TimeoutError(\"Agent exceeded timeout of 300.0s\")\nTimeoutError: Agent exceeded timeout of 300.0s\n" + }, + { + "trial_name": "flutter-custom-render-object__t25-gpt-5", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "partial", + "reward": 0.72, + "outcome_score": 0.69, + "quality_score": 0.69, + "dx_score": 0.97, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 62.0, + "verifier": 29.0 + }, + "input_tokens": 80847, + "cache_tokens": 61330, + "output_tokens": 3767, + "cost_usd": 0.2398, + "reward_tree": { + "reward": { + "score": 0.72, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.69, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (69%)." + }, + { + "name": "quality", + "value": 0.69, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (69%)." + }, + { + "name": "dx", + "value": 0.97, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (97%)." + } + ] + }, + "outcome": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.69, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.69, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.69, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.97, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gpt-5\n// Verification score: 0.72 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gpt-5\n// Verification score: 0.72 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gpt-5\n// Verification score: 0.72 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.72)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t17-o3", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "codex-agent__o3__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/o3", + "model_short_name": "o3", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.97, + "outcome_score": 0.97, + "quality_score": 0.99, + "dx_score": 0.91, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 145.0, + "verifier": 30.0 + }, + "input_tokens": 115035, + "cache_tokens": 85951, + "output_tokens": 5752, + "cost_usd": 0.6902, + "reward_tree": { + "reward": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.97, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (97%)." + }, + { + "name": "quality", + "value": 0.99, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (99%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.99, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.99, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: o3\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: o3\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: o3\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: o3\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: o3\n// Verification score: 0.97 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Manage State with BLoC unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.97)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t10-claude-3-5-sonnet", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.52, + "outcome_score": 0.54, + "quality_score": 0.47, + "dx_score": 0.59, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 93.0, + "verifier": 21.0 + }, + "input_tokens": 83422, + "cache_tokens": 61675, + "output_tokens": 3866, + "cost_usd": 0.3083, + "reward_tree": { + "reward": { + "score": 0.52, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.54, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (54%)." + }, + { + "name": "quality", + "value": 0.47, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (47%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.54, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.43200000000000005, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.54, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.47, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.47, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.47, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: claude-3-5-sonnet\n// Verification score: 0.52 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: claude-3-5-sonnet\n// Verification score: 0.52 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: claude-3-5-sonnet\n// Verification score: 0.52 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.52)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t46-gemini-31-flash-lite", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent_name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "provider": "Google", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.39, + "outcome_score": 0.39, + "quality_score": 0.34, + "dx_score": 0.57, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 80.0, + "verifier": 27.0 + }, + "input_tokens": 53142, + "cache_tokens": 41521, + "output_tokens": 1840, + "cost_usd": 0.0015, + "reward_tree": { + "reward": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.39, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (39%)." + }, + { + "name": "quality", + "value": 0.34, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (34%)." + }, + { + "name": "dx", + "value": 0.57, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (57%)." + } + ] + }, + "outcome": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.39, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.34, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.34, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.34, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.39899999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.39 (partial)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.39 (partial)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.39 (partial)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.39 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Build Command-Line CLI App tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.39)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t65-deepseek-coder-v2", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.42, + "outcome_score": 0.43, + "quality_score": 0.36, + "dx_score": 0.59, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 132.0, + "verifier": 27.0 + }, + "input_tokens": 72593, + "cache_tokens": 55605, + "output_tokens": 2965, + "cost_usd": 0.011, + "reward_tree": { + "reward": { + "score": 0.42, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.43, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (43%)." + }, + { + "name": "quality", + "value": 0.36, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (36%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.43, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.34400000000000003, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.43, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.36, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.36, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.36, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: deepseek-coder-v2\n// Verification score: 0.42 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: deepseek-coder-v2\n// Verification score: 0.42 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: deepseek-coder-v2\n// Verification score: 0.42 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.42)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t06-claude-3-5-sonnet", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.79, + "outcome_score": 0.85, + "quality_score": 0.73, + "dx_score": 0.6, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 86.0, + "verifier": 24.0 + }, + "input_tokens": 76557, + "cache_tokens": 57509, + "output_tokens": 3548, + "cost_usd": 0.2829, + "reward_tree": { + "reward": { + "score": 0.79, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.85, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (85%)." + }, + { + "name": "quality", + "value": 0.73, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (73%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.85, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.73, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.73, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.73, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-5-sonnet\n// Verification score: 0.79 (partial)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-5-sonnet\n// Verification score: 0.79 (partial)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-5-sonnet\n// Verification score: 0.79 (partial)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: claude-3-5-sonnet\n// Verification score: 0.79 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Build Command-Line CLI App tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.79)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t60-deepseek-v3", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.49, + "outcome_score": 0.49, + "quality_score": 0.45, + "dx_score": 0.57, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 82.0, + "verifier": 23.0 + }, + "input_tokens": 71806, + "cache_tokens": 47805, + "output_tokens": 3038, + "cost_usd": 0.0109, + "reward_tree": { + "reward": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.49, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (49%)." + }, + { + "name": "quality", + "value": 0.45, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (45%)." + }, + { + "name": "dx", + "value": 0.57, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (57%)." + } + ] + }, + "outcome": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.392, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.49, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.45, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.45, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.45, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.39899999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: deepseek-v3\n// Verification score: 0.49 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: deepseek-v3\n// Verification score: 0.49 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: deepseek-v3\n// Verification score: 0.49 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.49)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t48-gemini-31-flash-lite", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent_name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "provider": "Google", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.33, + "outcome_score": 0.32, + "quality_score": 0.27, + "dx_score": 0.59, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 136.0, + "verifier": 23.0 + }, + "input_tokens": 54133, + "cache_tokens": 36431, + "output_tokens": 1874, + "cost_usd": 0.0015, + "reward_tree": { + "reward": { + "score": 0.33, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.32, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (32%)." + }, + { + "name": "quality", + "value": 0.27, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (27%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.32, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.32, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.32, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.27, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.27, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.27, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.33 (partial)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.33 (partial)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.33 (partial)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.33 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Offline SQLite Sync Repository tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.33)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t28-gpt-4o", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.65, + "outcome_score": 0.68, + "quality_score": 0.6, + "dx_score": 0.59, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 104.0, + "verifier": 24.0 + }, + "input_tokens": 79623, + "cache_tokens": 58576, + "output_tokens": 3353, + "cost_usd": 0.2326, + "reward_tree": { + "reward": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.68, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (68%)." + }, + { + "name": "quality", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (60%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.68, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.68, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.68, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.6, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.6, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-4o\n// Verification score: 0.65 (partial)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-4o\n// Verification score: 0.65 (partial)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-4o\n// Verification score: 0.65 (partial)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-4o\n// Verification score: 0.65 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Offline SQLite Sync Repository tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.65)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t55-deepseek-r1", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "provider": "DeepSeek", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "partial", + "reward": 0.72, + "outcome_score": 0.71, + "quality_score": 0.65, + "dx_score": 0.98, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 73.0, + "verifier": 20.0 + }, + "input_tokens": 96985, + "cache_tokens": 77104, + "output_tokens": 4476, + "cost_usd": 0.0631, + "reward_tree": { + "reward": { + "score": 0.72, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.71, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (71%)." + }, + { + "name": "quality", + "value": 0.65, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (65%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.71, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.65, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.65, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: deepseek-r1\n// Verification score: 0.72 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: deepseek-r1\n// Verification score: 0.72 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: deepseek-r1\n// Verification score: 0.72 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.72)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t14-claude-3-5-haiku", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.45, + "outcome_score": 0.45, + "quality_score": 0.4, + "dx_score": 0.6, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 114.0, + "verifier": 32.0 + }, + "input_tokens": 66988, + "cache_tokens": 45287, + "output_tokens": 2561, + "cost_usd": 0.0638, + "reward_tree": { + "reward": { + "score": 0.45, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.45, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (45%)." + }, + { + "name": "quality", + "value": 0.4, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (40%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.45, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.45, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 0.45, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.4, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.4, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.4, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-5-haiku\n// Verification score: 0.45 (partial)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-5-haiku\n// Verification score: 0.45 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-5-haiku\n// Verification score: 0.45 (partial)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-5-haiku\n// Verification score: 0.45 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.45)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t37-gemini-35-pro", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.99, + "outcome_score": 0.99, + "quality_score": 0.99, + "dx_score": 0.96, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 142.0, + "verifier": 24.0 + }, + "input_tokens": 98941, + "cache_tokens": 72520, + "output_tokens": 4374, + "cost_usd": 0.1455, + "reward_tree": { + "reward": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.99, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (99%)." + }, + { + "name": "quality", + "value": 0.99, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (99%)." + }, + { + "name": "dx", + "value": 0.96, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (96%)." + } + ] + }, + "outcome": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.99, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.99, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.99, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.96, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-pro\n// Verification score: 0.99 (pass)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-pro\n// Verification score: 0.99 (pass)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-pro\n// Verification score: 0.99 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-pro\n// Verification score: 0.99 (pass)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-pro\n// Verification score: 0.99 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Manage State with BLoC unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.99)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t58-deepseek-v3", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.64, + "outcome_score": 0.67, + "quality_score": 0.58, + "dx_score": 0.6, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 78.0, + "verifier": 23.0 + }, + "input_tokens": 79232, + "cache_tokens": 62070, + "output_tokens": 3352, + "cost_usd": 0.012, + "reward_tree": { + "reward": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.67, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (67%)." + }, + { + "name": "quality", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (58%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.67, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.67, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.67, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.58, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.58, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-v3\n// Verification score: 0.64 (partial)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-v3\n// Verification score: 0.64 (partial)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-v3\n// Verification score: 0.64 (partial)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-v3\n// Verification score: 0.64 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Offline SQLite Sync Repository tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.64)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent_name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "provider": "Google", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.28, + "outcome_score": 0.26, + "quality_score": 0.23, + "dx_score": 0.56, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 124.0, + "verifier": 25.0 + }, + "input_tokens": 52949, + "cache_tokens": 41880, + "output_tokens": 1833, + "cost_usd": 0.0015, + "reward_tree": { + "reward": { + "score": 0.28, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.26, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (26%)." + }, + { + "name": "quality", + "value": 0.23, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (23%)." + }, + { + "name": "dx", + "value": 0.56, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (56%)." + } + ] + }, + "outcome": { + "score": 0.26, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.26, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 0.26, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.23, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.23, + "raw": true, + "weight": 0.5, + "description": "8 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.23, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.392, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.28 (partial)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.28 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.28 (partial)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.1-flash-lite\n// Verification score: 0.28 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.28)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t34-gpt-4o-mini", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.39, + "outcome_score": 0.39, + "quality_score": 0.34, + "dx_score": 0.59, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 139.0, + "verifier": 24.0 + }, + "input_tokens": 56871, + "cache_tokens": 43229, + "output_tokens": 2018, + "cost_usd": 0.0097, + "reward_tree": { + "reward": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.39, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (39%)." + }, + { + "name": "quality", + "value": 0.34, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (34%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.39, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 0.39, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.34, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.34, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.34, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-4o-mini\n// Verification score: 0.39 (partial)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-4o-mini\n// Verification score: 0.39 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-4o-mini\n// Verification score: 0.39 (partial)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-4o-mini\n// Verification score: 0.39 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.39)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t26-gpt-4o", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.71, + "outcome_score": 0.76, + "quality_score": 0.65, + "dx_score": 0.62, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 70.0, + "verifier": 27.0 + }, + "input_tokens": 80011, + "cache_tokens": 62123, + "output_tokens": 3369, + "cost_usd": 0.2337, + "reward_tree": { + "reward": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.76, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (76%)." + }, + { + "name": "quality", + "value": 0.65, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (65%)." + }, + { + "name": "dx", + "value": 0.62, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (62%)." + } + ] + }, + "outcome": { + "score": 0.76, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.76, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.65, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.65, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.434, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-4o\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-4o\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-4o\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-4o\n// Verification score: 0.71 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Build Command-Line CLI App tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.71)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t05-claude-3-7-sonnet", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "provider": "Anthropic", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "partial", + "reward": 0.71, + "outcome_score": 0.69, + "quality_score": 0.7, + "dx_score": 0.91, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 101.0, + "verifier": 28.0 + }, + "input_tokens": 94141, + "cache_tokens": 69860, + "output_tokens": 4605, + "cost_usd": 0.3515, + "reward_tree": { + "reward": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.69, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (69%)." + }, + { + "name": "quality", + "value": 0.7, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (70%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.69, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.7, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.7, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.7, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: claude-3-7-sonnet\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: claude-3-7-sonnet\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: claude-3-7-sonnet\n// Verification score: 0.71 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.71)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t43-gemini-35-flash", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.8, + "outcome_score": 0.8, + "quality_score": 0.77, + "dx_score": 0.9, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 123.0, + "verifier": 23.0 + }, + "input_tokens": 83545, + "cache_tokens": 65913, + "output_tokens": 3053, + "cost_usd": 0.0072, + "reward_tree": { + "reward": { + "score": 0.8, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.8, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (80%)." + }, + { + "name": "quality", + "value": 0.77, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (77%)." + }, + { + "name": "dx", + "value": 0.9, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (90%)." + } + ] + }, + "outcome": { + "score": 0.8, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.8, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.77, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.77, + "raw": true, + "weight": 0.5, + "description": "2 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.77, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.9, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.9, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.5-flash\n// Verification score: 0.8 (pass)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.5-flash\n// Verification score: 0.8 (pass)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.5-flash\n// Verification score: 0.8 (pass)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gemini-3.5-flash\n// Verification score: 0.8 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Offline SQLite Sync Repository unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.8)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t59-deepseek-v3", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.59, + "outcome_score": 0.61, + "quality_score": 0.54, + "dx_score": 0.58, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 123.0, + "verifier": 32.0 + }, + "input_tokens": 75026, + "cache_tokens": 56404, + "output_tokens": 3174, + "cost_usd": 0.0114, + "reward_tree": { + "reward": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.61, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (61%)." + }, + { + "name": "quality", + "value": 0.54, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (54%)." + }, + { + "name": "dx", + "value": 0.58, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (58%)." + } + ] + }, + "outcome": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.61, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.54, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.54, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.54, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.40599999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-v3\n// Verification score: 0.59 (partial)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-v3\n// Verification score: 0.59 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-v3\n// Verification score: 0.59 (partial)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-v3\n// Verification score: 0.59 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.59)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t19-o3", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "codex-agent__o3__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/o3", + "model_short_name": "o3", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.85, + "outcome_score": 0.85, + "quality_score": 0.82, + "dx_score": 0.98, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 66.0, + "verifier": 32.0 + }, + "input_tokens": 116947, + "cache_tokens": 84269, + "output_tokens": 5847, + "cost_usd": 0.7017, + "reward_tree": { + "reward": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.85, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (85%)." + }, + { + "name": "quality", + "value": 0.82, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (82%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.85, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.82, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.82, + "raw": true, + "weight": 0.5, + "description": "2 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.82, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: o3\n// Verification score: 0.85 (pass)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: o3\n// Verification score: 0.85 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: o3\n// Verification score: 0.85 (pass)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: o3\n// Verification score: 0.85 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Adaptive Material & Cupertino UI unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.85)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t56-deepseek-v3", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.71, + "outcome_score": 0.75, + "quality_score": 0.68, + "dx_score": 0.56, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 71.0, + "verifier": 20.0 + }, + "input_tokens": 71073, + "cache_tokens": 47079, + "output_tokens": 3007, + "cost_usd": 0.0108, + "reward_tree": { + "reward": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.75, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (75%)." + }, + { + "name": "quality", + "value": 0.68, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (68%)." + }, + { + "name": "dx", + "value": 0.56, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (56%)." + } + ] + }, + "outcome": { + "score": 0.75, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.75, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.68, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.68, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.68, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.392, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-v3\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-v3\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-v3\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-v3\n// Verification score: 0.71 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Build Command-Line CLI App tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.71)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t08-claude-3-5-sonnet", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.71, + "outcome_score": 0.76, + "quality_score": 0.65, + "dx_score": 0.61, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 109.0, + "verifier": 22.0 + }, + "input_tokens": 85453, + "cache_tokens": 55797, + "output_tokens": 3960, + "cost_usd": 0.3158, + "reward_tree": { + "reward": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.76, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (76%)." + }, + { + "name": "quality", + "value": 0.65, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (65%)." + }, + { + "name": "dx", + "value": 0.61, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (61%)." + } + ] + }, + "outcome": { + "score": 0.76, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.76, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.65, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.65, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.427, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-5-sonnet\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-5-sonnet\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-5-sonnet\n// Verification score: 0.71 (partial)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: claude-3-5-sonnet\n// Verification score: 0.71 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Offline SQLite Sync Repository tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.71)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t23-gpt-5", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.86, + "outcome_score": 0.86, + "quality_score": 0.85, + "dx_score": 0.94, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 71.0, + "verifier": 20.0 + }, + "input_tokens": 88953, + "cache_tokens": 62264, + "output_tokens": 4144, + "cost_usd": 0.2638, + "reward_tree": { + "reward": { + "score": 0.86, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.86, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (86%)." + }, + { + "name": "quality", + "value": 0.85, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (85%)." + }, + { + "name": "dx", + "value": 0.94, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (94%)." + } + ] + }, + "outcome": { + "score": 0.86, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.86, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.85, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.85, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.94, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.94, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-5\n// Verification score: 0.86 (pass)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-5\n// Verification score: 0.86 (pass)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-5\n// Verification score: 0.86 (pass)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: gpt-5\n// Verification score: 0.86 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Offline SQLite Sync Repository unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.86)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t27-gpt-4o", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.69, + "outcome_score": 0.74, + "quality_score": 0.62, + "dx_score": 0.6, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 141.0, + "verifier": 24.0 + }, + "input_tokens": 75480, + "cache_tokens": 58588, + "output_tokens": 3178, + "cost_usd": 0.2205, + "reward_tree": { + "reward": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.74, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (74%)." + }, + { + "name": "quality", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (62%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.74, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.74, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.62, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.62, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o\n// Verification score: 0.69 (partial)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o\n// Verification score: 0.69 (partial)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o\n// Verification score: 0.69 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o\n// Verification score: 0.69 (partial)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-4o\n// Verification score: 0.69 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Manage State with BLoC tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.69)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t36-gemini-35-pro", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.98, + "outcome_score": 0.98, + "quality_score": 0.99, + "dx_score": 0.98, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 104.0, + "verifier": 33.0 + }, + "input_tokens": 89215, + "cache_tokens": 66083, + "output_tokens": 3944, + "cost_usd": 0.1312, + "reward_tree": { + "reward": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.98, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (98%)." + }, + { + "name": "quality", + "value": 0.99, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (99%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.98, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.99, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.99, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.5-pro\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.5-pro\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.5-pro\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.5-pro\n// Verification score: 0.98 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Build Command-Line CLI App unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.98)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t51-deepseek-r1", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "provider": "DeepSeek", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.99, + "outcome_score": 1.0, + "quality_score": 0.98, + "dx_score": 0.92, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 130.0, + "verifier": 23.0 + }, + "input_tokens": 109101, + "cache_tokens": 83327, + "output_tokens": 5035, + "cost_usd": 0.071, + "reward_tree": { + "reward": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 1.0, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (100%)." + }, + { + "name": "quality", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (98%)." + }, + { + "name": "dx", + "value": 0.92, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (92%)." + } + ] + }, + "outcome": { + "score": 1.0, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 1.0, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.98, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.98, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.92, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.92, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.92, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-r1\n// Verification score: 0.99 (pass)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-r1\n// Verification score: 0.99 (pass)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-r1\n// Verification score: 0.99 (pass)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-r1\n// Verification score: 0.99 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Build Command-Line CLI App unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.99)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t20-o3", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "codex-agent__o3__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/o3", + "model_short_name": "o3", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "partial", + "reward": 0.73, + "outcome_score": 0.71, + "quality_score": 0.71, + "dx_score": 0.95, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 118.0, + "verifier": 26.0 + }, + "input_tokens": 103938, + "cache_tokens": 76507, + "output_tokens": 5197, + "cost_usd": 0.6236, + "reward_tree": { + "reward": { + "score": 0.73, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.71, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (71%)." + }, + { + "name": "quality", + "value": 0.71, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (71%)." + }, + { + "name": "dx", + "value": 0.95, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (95%)." + } + ] + }, + "outcome": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.71, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.71, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.71, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.95, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.95, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.95, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: o3\n// Verification score: 0.73 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: o3\n// Verification score: 0.73 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: o3\n// Verification score: 0.73 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.73)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t41-gemini-35-flash", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.93, + "outcome_score": 0.94, + "quality_score": 0.92, + "dx_score": 0.9, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 143.0, + "verifier": 22.0 + }, + "input_tokens": 76509, + "cache_tokens": 56237, + "output_tokens": 2796, + "cost_usd": 0.0066, + "reward_tree": { + "reward": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.94, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (94%)." + }, + { + "name": "quality", + "value": 0.92, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (92%)." + }, + { + "name": "dx", + "value": 0.9, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (90%)." + } + ] + }, + "outcome": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.94, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.92, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.92, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.92, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.9, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.9, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.5-flash\n// Verification score: 0.93 (pass)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.5-flash\n// Verification score: 0.93 (pass)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.5-flash\n// Verification score: 0.93 (pass)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gemini-3.5-flash\n// Verification score: 0.93 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Build Command-Line CLI App unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.93)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t12-claude-3-5-haiku", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "model_short_name": "claude-3-5-haiku", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.55, + "outcome_score": 0.57, + "quality_score": 0.5, + "dx_score": 0.62, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 138.0, + "verifier": 34.0 + }, + "input_tokens": 67759, + "cache_tokens": 48563, + "output_tokens": 2591, + "cost_usd": 0.0646, + "reward_tree": { + "reward": { + "score": 0.55, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.57, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (57%)." + }, + { + "name": "quality", + "value": 0.5, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (50%)." + }, + { + "name": "dx", + "value": 0.62, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (62%)." + } + ] + }, + "outcome": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.57, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 0.5, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.5, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.5, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.5, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.434, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-haiku\n// Verification score: 0.55 (partial)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-haiku\n// Verification score: 0.55 (partial)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-haiku\n// Verification score: 0.55 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-haiku\n// Verification score: 0.55 (partial)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-haiku\n// Verification score: 0.55 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Manage State with BLoC tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.55)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t30-gpt-4o", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o", + "model_short_name": "gpt-4o", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.47, + "outcome_score": 0.49, + "quality_score": 0.4, + "dx_score": 0.61, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 77.0, + "verifier": 30.0 + }, + "input_tokens": 78669, + "cache_tokens": 54055, + "output_tokens": 3312, + "cost_usd": 0.2298, + "reward_tree": { + "reward": { + "score": 0.47, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.49, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (49%)." + }, + { + "name": "quality", + "value": 0.4, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (40%)." + }, + { + "name": "dx", + "value": 0.61, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (61%)." + } + ] + }, + "outcome": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.392, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.49, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.4, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.4, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.4, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.427, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gpt-4o\n// Verification score: 0.47 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gpt-4o\n// Verification score: 0.47 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gpt-4o\n// Verification score: 0.47 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.47)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t07-claude-3-5-sonnet", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "model_short_name": "claude-3-5-sonnet", + "provider": "Anthropic", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.75, + "outcome_score": 0.79, + "quality_score": 0.71, + "dx_score": 0.62, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 101.0, + "verifier": 31.0 + }, + "input_tokens": 76661, + "cache_tokens": 55217, + "output_tokens": 3553, + "cost_usd": 0.2833, + "reward_tree": { + "reward": { + "score": 0.75, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.79, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (79%)." + }, + { + "name": "quality", + "value": 0.71, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (71%)." + }, + { + "name": "dx", + "value": 0.62, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (62%)." + } + ] + }, + "outcome": { + "score": 0.79, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.79, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.71, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.71, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.434, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-sonnet\n// Verification score: 0.75 (partial)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-sonnet\n// Verification score: 0.75 (partial)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-sonnet\n// Verification score: 0.75 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-sonnet\n// Verification score: 0.75 (partial)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-5-sonnet\n// Verification score: 0.75 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Manage State with BLoC tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.75)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t22-gpt-5", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.97, + "outcome_score": 0.99, + "quality_score": 0.94, + "dx_score": 0.95, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 127.0, + "verifier": 28.0 + }, + "input_tokens": 84280, + "cache_tokens": 62974, + "output_tokens": 3927, + "cost_usd": 0.25, + "reward_tree": { + "reward": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.99, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (99%)." + }, + { + "name": "quality", + "value": 0.94, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (94%)." + }, + { + "name": "dx", + "value": 0.95, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (95%)." + } + ] + }, + "outcome": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.99, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.94, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.94, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.95, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.95, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.95, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-5\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-5\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-5\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-5\n// Verification score: 0.97 (pass)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gpt-5\n// Verification score: 0.97 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Manage State with BLoC unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.97)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t31-gpt-4o-mini", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "model_short_name": "gpt-4o-mini", + "provider": "OpenAI", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.48, + "outcome_score": 0.48, + "quality_score": 0.42, + "dx_score": 0.63, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 132.0, + "verifier": 23.0 + }, + "input_tokens": 56911, + "cache_tokens": 37416, + "output_tokens": 2019, + "cost_usd": 0.0097, + "reward_tree": { + "reward": { + "score": 0.48, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.48, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (48%)." + }, + { + "name": "quality", + "value": 0.42, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (42%)." + }, + { + "name": "dx", + "value": 0.63, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (63%)." + } + ] + }, + "outcome": { + "score": 0.48, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.48, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.42, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.42, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.42, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44099999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-4o-mini\n// Verification score: 0.48 (partial)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-4o-mini\n// Verification score: 0.48 (partial)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-4o-mini\n// Verification score: 0.48 (partial)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: gpt-4o-mini\n// Verification score: 0.48 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Build Command-Line CLI App tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.48)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t39-gemini-35-pro", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "model_short_name": "gemini-3.5-pro", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.87, + "outcome_score": 0.87, + "quality_score": 0.85, + "dx_score": 0.95, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 120.0, + "verifier": 29.0 + }, + "input_tokens": 89376, + "cache_tokens": 63398, + "output_tokens": 3951, + "cost_usd": 0.1315, + "reward_tree": { + "reward": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.87, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (87%)." + }, + { + "name": "quality", + "value": 0.85, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (85%)." + }, + { + "name": "dx", + "value": 0.95, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (95%)." + } + ] + }, + "outcome": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.87, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.85, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.85, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.95, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.95, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.95, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.5-pro\n// Verification score: 0.87 (pass)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.5-pro\n// Verification score: 0.87 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.5-pro\n// Verification score: 0.87 (pass)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gemini-3.5-pro\n// Verification score: 0.87 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Adaptive Material & Cupertino UI unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.87)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t63-deepseek-coder-v2", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.55, + "outcome_score": 0.57, + "quality_score": 0.48, + "dx_score": 0.61, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 107.0, + "verifier": 32.0 + }, + "input_tokens": 71650, + "cache_tokens": 46814, + "output_tokens": 2927, + "cost_usd": 0.0109, + "reward_tree": { + "reward": { + "score": 0.55, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.57, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (57%)." + }, + { + "name": "quality", + "value": 0.48, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (48%)." + }, + { + "name": "dx", + "value": 0.61, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (61%)." + } + ] + }, + "outcome": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.57, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.48, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.48, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.48, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.427, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-coder-v2\n// Verification score: 0.55 (partial)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-coder-v2\n// Verification score: 0.55 (partial)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-coder-v2\n// Verification score: 0.55 (partial)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: deepseek-coder-v2\n// Verification score: 0.55 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Offline SQLite Sync Repository tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.55)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t54-deepseek-r1", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "model_short_name": "deepseek-r1", + "provider": "DeepSeek", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "partial", + "reward": 0.78, + "outcome_score": 0.76, + "quality_score": 0.74, + "dx_score": 0.98, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 99.0, + "verifier": 25.0 + }, + "input_tokens": 109330, + "cache_tokens": 79069, + "output_tokens": 5046, + "cost_usd": 0.0712, + "reward_tree": { + "reward": { + "score": 0.78, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.76, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (76%)." + }, + { + "name": "quality", + "value": 0.74, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (74%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.76, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.76, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.74, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.74, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.74, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-r1\n// Verification score: 0.78 (partial)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-r1\n// Verification score: 0.78 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-r1\n// Verification score: 0.78 (partial)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: deepseek-r1\n// Verification score: 0.78 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.78)\n", + "exception_log": null + }, + { + "trial_name": "flutter-custom-render-object__t45-gemini-35-flash", + "task_name": "google/flutter-custom-render-object", + "task_slug": "flutter-custom-render-object", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "partial", + "reward": 0.65, + "outcome_score": 0.61, + "quality_score": 0.61, + "dx_score": 0.97, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 147.0, + "verifier": 32.0 + }, + "input_tokens": 71164, + "cache_tokens": 48785, + "output_tokens": 2600, + "cost_usd": 0.0061, + "reward_tree": { + "reward": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.61, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (61%)." + }, + { + "name": "quality", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (61%)." + }, + { + "name": "dx", + "value": 0.97, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (97%)." + } + ] + }, + "outcome": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.61, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.61, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.61, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.97, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gemini-3.5-flash\n// Verification score: 0.65 (partial)\n" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gemini-3.5-flash\n// Verification score: 0.65 (partial)\n" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-custom-render-object\n// Model: gemini-3.5-flash\n// Verification score: 0.65 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Custom RenderObject & Canvas tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.65)\n", + "exception_log": null + }, + { + "trial_name": "dart-build-cli-app__t61-deepseek-coder-v2", + "task_name": "google/dart-build-cli-app", + "task_slug": "dart-build-cli-app", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "model_short_name": "deepseek-coder-v2", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.61, + "outcome_score": 0.64, + "quality_score": 0.56, + "dx_score": 0.62, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 132.0, + "verifier": 28.0 + }, + "input_tokens": 71997, + "cache_tokens": 56271, + "output_tokens": 2941, + "cost_usd": 0.0109, + "reward_tree": { + "reward": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.64, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (64%)." + }, + { + "name": "quality", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (56%)." + }, + { + "name": "dx", + "value": 0.62, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (62%)." + } + ] + }, + "outcome": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.64, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.56, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.56, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.434, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-coder-v2\n// Verification score: 0.61 (partial)\n" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-coder-v2\n// Verification score: 0.61 (partial)\n" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-coder-v2\n// Verification score: 0.61 (partial)\n" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok", + "content": "// Generated implementation for dart-build-cli-app\n// Model: deepseek-coder-v2\n// Verification score: 0.61 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Build Command-Line CLI App tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.61)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t42-gemini-35-flash", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent_name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "model_short_name": "gemini-3.5-flash", + "provider": "Google", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.88, + "outcome_score": 0.87, + "quality_score": 0.88, + "dx_score": 0.93, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 84.0, + "verifier": 34.0 + }, + "input_tokens": 79096, + "cache_tokens": 54691, + "output_tokens": 2890, + "cost_usd": 0.0068, + "reward_tree": { + "reward": { + "score": 0.88, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.87, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (87%)." + }, + { + "name": "quality", + "value": 0.88, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (88%)." + }, + { + "name": "dx", + "value": 0.93, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (93%)." + } + ] + }, + "outcome": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.87, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.88, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.88, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.88, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.93, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-flash\n// Verification score: 0.88 (pass)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-flash\n// Verification score: 0.88 (pass)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-flash\n// Verification score: 0.88 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-flash\n// Verification score: 0.88 (pass)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: gemini-3.5-flash\n// Verification score: 0.88 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Manage State with BLoC unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.88)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t02-claude-3-7-sonnet", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "provider": "Anthropic", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.98, + "outcome_score": 0.98, + "quality_score": 1.0, + "dx_score": 0.91, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 113.0, + "verifier": 31.0 + }, + "input_tokens": 86588, + "cache_tokens": 63739, + "output_tokens": 4235, + "cost_usd": 0.3233, + "reward_tree": { + "reward": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.98, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (98%)." + }, + { + "name": "quality", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (100%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 1.0, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-7-sonnet\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-7-sonnet\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-7-sonnet\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-7-sonnet\n// Verification score: 0.98 (pass)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: claude-3-7-sonnet\n// Verification score: 0.98 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Manage State with BLoC unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.98)\n", + "exception_log": null + }, + { + "trial_name": "flutter-offline-sync-sqlite__t18-o3", + "task_name": "google/flutter-offline-sync-sqlite", + "task_slug": "flutter-offline-sync-sqlite", + "eval_key": "codex-agent__o3__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/o3", + "model_short_name": "o3", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.9, + "outcome_score": 0.9, + "quality_score": 0.9, + "dx_score": 0.93, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 118.0, + "verifier": 29.0 + }, + "input_tokens": 110044, + "cache_tokens": 77484, + "output_tokens": 5502, + "cost_usd": 0.6603, + "reward_tree": { + "reward": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.9, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (90%)." + }, + { + "name": "quality", + "value": 0.9, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (90%)." + }, + { + "name": "dx", + "value": 0.93, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (93%)." + } + ] + }, + "outcome": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.9, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.9, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.9, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.93, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: o3\n// Verification score: 0.9 (pass)\n" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: o3\n// Verification score: 0.9 (pass)\n" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: o3\n// Verification score: 0.9 (pass)\n" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-offline-sync-sqlite\n// Model: o3\n// Verification score: 0.9 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Offline SQLite Sync Repository unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.9)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t47-gemini-31-flash-lite", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent_name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "model_short_name": "gemini-3.1-flash-lite", + "provider": "Google", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "error", + "reward": null, + "outcome_score": null, + "quality_score": null, + "dx_score": null, + "exception_type": "AgentTimeoutError", + "exception_message": "Agent exceeded maximum timeout of 300.0 seconds during execution.", + "exception_traceback": "Traceback (most recent call last):\n File \"harbor/trial/trial.py\", line 1455, in _execute_agent\n raise TimeoutError(\"Agent exceeded timeout of 300.0s\")\nTimeoutError: Agent exceeded timeout of 300.0s\n", + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 305.0 + }, + "input_tokens": 0, + "cache_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "reward_tree": null, + "diagnostic_tree": {}, + "trajectory": null, + "artifacts": [], + "test_stdout": null, + "exception_log": "Exception: AgentTimeoutError\nAgent exceeded maximum timeout of 300.0 seconds during execution.\n\nTraceback (most recent call last):\n File \"harbor/trial/trial.py\", line 1455, in _execute_agent\n raise TimeoutError(\"Agent exceeded timeout of 300.0s\")\nTimeoutError: Agent exceeded timeout of 300.0s\n" + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t24-gpt-5", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent_name": "codex-agent", + "model_name": "openai/gpt-5", + "model_short_name": "gpt-5", + "provider": "OpenAI", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.84, + "outcome_score": 0.82, + "quality_score": 0.83, + "dx_score": 0.98, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 119.0, + "verifier": 33.0 + }, + "input_tokens": 94968, + "cache_tokens": 64976, + "output_tokens": 4425, + "cost_usd": 0.2817, + "reward_tree": { + "reward": { + "score": 0.84, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.82, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (82%)." + }, + { + "name": "quality", + "value": 0.83, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (83%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.82, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.82, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.83, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.83, + "raw": true, + "weight": 0.5, + "description": "2 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.83, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-5\n// Verification score: 0.84 (pass)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-5\n// Verification score: 0.84 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-5\n// Verification score: 0.84 (pass)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: gpt-5\n// Verification score: 0.84 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Adaptive Material & Cupertino UI unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.84)\n", + "exception_log": null + }, + { + "trial_name": "flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet", + "task_name": "google/flutter-adaptive-material-cupertino", + "task_slug": "flutter-adaptive-material-cupertino", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent_name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "model_short_name": "claude-3-7-sonnet", + "provider": "Anthropic", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + "dart" + ], + "has_dart_tooling": true, + "status": "pass", + "reward": 0.85, + "outcome_score": 0.85, + "quality_score": 0.83, + "dx_score": 0.91, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 122.0, + "verifier": 32.0 + }, + "input_tokens": 95701, + "cache_tokens": 67346, + "output_tokens": 4681, + "cost_usd": 0.3573, + "reward_tree": { + "reward": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.85, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (85%)." + }, + { + "name": "quality", + "value": 0.83, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (83%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.85, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.83, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.83, + "raw": true, + "weight": 0.5, + "description": "2 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.83, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ], + "artifacts": [ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-7-sonnet\n// Verification score: 0.85 (pass)\n" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-7-sonnet\n// Verification score: 0.85 (pass)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-7-sonnet\n// Verification score: 0.85 (pass)\n" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-adaptive-material-cupertino\n// Model: claude-3-7-sonnet\n// Verification score: 0.85 (pass)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: All Adaptive Material & Cupertino UI unit tests passed.\n00:04 +4: Static analysis: 0 warnings, 0 errors.\n00:05 +5: Rubric grader verification completed.\nOverall result: PASS (Score: 0.85)\n", + "exception_log": null + }, + { + "trial_name": "flutter-manage-state-with-bloc__t57-deepseek-v3", + "task_name": "google/flutter-manage-state-with-bloc", + "task_slug": "flutter-manage-state-with-bloc", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent_name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "model_short_name": "deepseek-v3", + "provider": "DeepSeek", + "skills": [], + "mcp_servers": [], + "has_dart_tooling": false, + "status": "partial", + "reward": 0.67, + "outcome_score": 0.7, + "quality_score": 0.61, + "dx_score": 0.63, + "exception_type": null, + "exception_message": null, + "exception_traceback": null, + "durations": { + "environment_setup": 10.0, + "agent_setup": 10.0, + "agent_execution": 115.0, + "verifier": 34.0 + }, + "input_tokens": 77341, + "cache_tokens": 57438, + "output_tokens": 3272, + "cost_usd": 0.0117, + "reward_tree": { + "reward": { + "score": 0.67, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.7, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (70%)." + }, + { + "name": "quality", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (61%)." + }, + { + "name": "dx", + "value": 0.63, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (63%)." + } + ] + }, + "outcome": { + "score": 0.7, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.7, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.61, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.61, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44099999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } + }, + "diagnostic_tree": {}, + "trajectory": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ], + "artifacts": [ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-v3\n// Verification score: 0.67 (partial)\n" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-v3\n// Verification score: 0.67 (partial)\n" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-v3\n// Verification score: 0.67 (partial)\n" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-v3\n// Verification score: 0.67 (partial)\n" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok", + "content": "// Generated implementation for flutter-manage-state-with-bloc\n// Model: deepseek-v3\n// Verification score: 0.67 (partial)\n" + } + ], + "test_stdout": "00:00 +0: loading tests/graders.dart\n00:01 +1: Environment setup verification passed.\n00:02 +2: Task codebase compilation check.\n00:03 +3: 3/5 Manage State with BLoC tests passed.\n00:04 +3 -1: 2 edge-case assertions failed.\n00:05 +4: Static analysis completed with minor hints.\nOverall result: PARTIAL (Score: 0.67)\n", + "exception_log": null + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/main.client.options.dart b/sites/www/lib/main.client.options.dart index 6fdfce6fc98..db185a72962 100644 --- a/sites/www/lib/main.client.options.dart +++ b/sites/www/lib/main.client.options.dart @@ -12,6 +12,18 @@ import 'package:flutter_website/src/components/common/newsletter_form.dart' deferred as _newsletter_form; import 'package:flutter_website/src/components/common/tabs.dart' deferred as _tabs; +import 'package:flutter_website/src/components/flutterbench/cuj_catalog.dart' + deferred as _cuj_catalog; +import 'package:flutter_website/src/components/flutterbench/grader_matrix.dart' + deferred as _grader_matrix; +import 'package:flutter_website/src/components/flutterbench/interactive_detail_card.dart' + deferred as _interactive_detail_card; +import 'package:flutter_website/src/components/flutterbench/leaderboard_table.dart' + deferred as _leaderboard_table; +import 'package:flutter_website/src/components/flutterbench/models_explorer.dart' + deferred as _models_explorer; +import 'package:flutter_website/src/components/flutterbench/task_specifications.dart' + deferred as _task_specifications; import 'package:flutter_website/src/components/layout/header.dart' deferred as _header; import 'package:flutter_website/src/components/pages/consultants_cookie_snack.dart' @@ -98,6 +110,68 @@ ClientOptions get defaultClientOptions => ClientOptions( ), loader: _tabs.loadLibrary, ), + 'cuj_catalog': ClientLoader( + (p) => _cuj_catalog.CujCatalog( + cujs: (p['cujs'] as List) + .map((i) => (i as Map)) + .toList(), + ), + loader: _cuj_catalog.loadLibrary, + ), + 'grader_matrix': ClientLoader( + (p) => _grader_matrix.GraderMatrix( + title: p['title'] as String, + description: p['description'] as String, + filters: (p['filters'] as List) + .map((i) => (i as Map)) + .toList(), + graders: (p['graders'] as List) + .map((i) => (i as Map)) + .toList(), + ), + loader: _grader_matrix.loadLibrary, + ), + 'interactive_detail_card': ClientLoader( + (p) => _interactive_detail_card.InteractiveDetailCard( + title: p['title'] as String, + description: p['description'] as String, + tabs: (p['tabs'] as List) + .map((i) => (i as Map)) + .toList(), + classes: p['classes'] as String, + ), + loader: _interactive_detail_card.loadLibrary, + ), + 'leaderboard_table': ClientLoader( + (p) => _leaderboard_table.LeaderboardTable( + evals: (p['evals'] as List) + .map((i) => (i as Map)) + .toList(), + benchmarks: (p['benchmarks'] as List) + .map((i) => (i as Map)) + .toList(), + ), + loader: _leaderboard_table.loadLibrary, + ), + 'models_explorer': ClientLoader( + (p) => _models_explorer.ModelsExplorer( + evals: (p['evals'] as List) + .map((i) => (i as Map)) + .toList(), + benchmarks: (p['benchmarks'] as List) + .map((i) => (i as Map)) + .toList(), + ), + loader: _models_explorer.loadLibrary, + ), + 'task_specifications': ClientLoader( + (p) => _task_specifications.TaskSpecifications( + specs: (p['specs'] as List) + .map((i) => (i as Map)) + .toList(), + ), + loader: _task_specifications.loadLibrary, + ), 'header': ClientLoader( (p) => _header.Header( contrastLogoSrc: p['contrastLogoSrc'] as String, diff --git a/sites/www/lib/main.server.dart b/sites/www/lib/main.server.dart index ad3623362a4..0372f916010 100644 --- a/sites/www/lib/main.server.dart +++ b/sites/www/lib/main.server.dart @@ -34,6 +34,13 @@ import 'src/pages/ecosystem_page.dart'; import 'src/pages/embedded_page.dart'; import 'src/pages/events_page.dart'; import 'src/pages/flip_page.dart'; +import 'src/pages/flutter_bench/flutterbench_cujs_page.dart'; +import 'src/pages/flutter_bench/flutterbench_leaderboard_page.dart'; +import 'src/pages/flutter_bench/flutterbench_methodology_page.dart'; +import 'src/pages/flutter_bench/flutterbench_models_page.dart'; +import 'src/pages/flutter_bench/flutterbench_task_detail_page.dart'; +import 'src/pages/flutter_bench/flutterbench_tasks_page.dart'; +import 'src/pages/flutter_bench/flutterbench_trial_detail_page.dart'; import 'src/pages/games_page.dart'; import 'src/pages/google_integrations_page.dart'; import 'src/pages/home_page.dart'; @@ -128,6 +135,34 @@ void main() { defineComponent('FlipPage', const FlipPage()), defineComponent('NewsPage', const NewsPage()), defineComponent('WhyFlutterPage', const WhyFlutterPage()), + defineComponent( + 'FlutterBenchLeaderboardPage', + const FlutterBenchLeaderboardPage(), + ), + defineComponent( + 'FlutterBenchModelsPage', + const FlutterBenchModelsPage(), + ), + defineComponent( + 'FlutterBenchTasksPage', + const FlutterBenchTasksPage(), + ), + defineComponent( + 'FlutterBenchMethodologyPage', + const FlutterBenchMethodologyPage(), + ), + defineComponent( + 'FlutterBenchCujsPage', + const FlutterBenchCujsPage(), + ), + defineComponentWithAttrs( + 'FlutterBenchTaskDetailPage', + FlutterBenchTaskDetailPage.fromAttrs, + ), + defineComponentWithAttrs( + 'FlutterBenchTrialDetailPage', + FlutterBenchTrialDetailPage.fromAttrs, + ), defineComponentWithAttrs('Image', Image.fromAttrs), CustomComponent( diff --git a/sites/www/lib/main.server.options.dart b/sites/www/lib/main.server.options.dart index c68f3a7351a..15d5230e241 100644 --- a/sites/www/lib/main.server.options.dart +++ b/sites/www/lib/main.server.options.dart @@ -10,6 +10,18 @@ import 'package:flutter_website/src/components/common/carousel.dart' import 'package:flutter_website/src/components/common/newsletter_form.dart' as _newsletter_form; import 'package:flutter_website/src/components/common/tabs.dart' as _tabs; +import 'package:flutter_website/src/components/flutterbench/cuj_catalog.dart' + as _cuj_catalog; +import 'package:flutter_website/src/components/flutterbench/grader_matrix.dart' + as _grader_matrix; +import 'package:flutter_website/src/components/flutterbench/interactive_detail_card.dart' + as _interactive_detail_card; +import 'package:flutter_website/src/components/flutterbench/leaderboard_table.dart' + as _leaderboard_table; +import 'package:flutter_website/src/components/flutterbench/models_explorer.dart' + as _models_explorer; +import 'package:flutter_website/src/components/flutterbench/task_specifications.dart' + as _task_specifications; import 'package:flutter_website/src/components/layout/header.dart' as _header; import 'package:flutter_website/src/components/pages/consultants_cookie_snack.dart' as _consultants_cookie_snack; @@ -69,6 +81,34 @@ ServerOptions get defaultServerOptions => ServerOptions( _newsletter_form.NewsletterForm: ClientTarget<_newsletter_form.NewsletterForm>('newsletter_form'), _tabs.Tabs: ClientTarget<_tabs.Tabs>('tabs', params: __tabsTabs), + _cuj_catalog.CujCatalog: ClientTarget<_cuj_catalog.CujCatalog>( + 'cuj_catalog', + params: __cuj_catalogCujCatalog, + ), + _grader_matrix.GraderMatrix: ClientTarget<_grader_matrix.GraderMatrix>( + 'grader_matrix', + params: __grader_matrixGraderMatrix, + ), + _interactive_detail_card.InteractiveDetailCard: + ClientTarget<_interactive_detail_card.InteractiveDetailCard>( + 'interactive_detail_card', + params: __interactive_detail_cardInteractiveDetailCard, + ), + _leaderboard_table.LeaderboardTable: + ClientTarget<_leaderboard_table.LeaderboardTable>( + 'leaderboard_table', + params: __leaderboard_tableLeaderboardTable, + ), + _models_explorer.ModelsExplorer: + ClientTarget<_models_explorer.ModelsExplorer>( + 'models_explorer', + params: __models_explorerModelsExplorer, + ), + _task_specifications.TaskSpecifications: + ClientTarget<_task_specifications.TaskSpecifications>( + 'task_specifications', + params: __task_specificationsTaskSpecifications, + ), _header.Header: ClientTarget<_header.Header>( 'header', params: __headerHeader, @@ -159,6 +199,34 @@ Map __tabsTabs(_tabs.Tabs c) => { 'tabs': c.tabs.map((i) => i.toMap()).toList(), 'noSpy': c.noSpy, }; +Map __cuj_catalogCujCatalog(_cuj_catalog.CujCatalog c) => { + 'cujs': c.cujs, +}; +Map __grader_matrixGraderMatrix( + _grader_matrix.GraderMatrix c, +) => { + 'title': c.title, + 'description': c.description, + 'filters': c.filters, + 'graders': c.graders, +}; +Map __interactive_detail_cardInteractiveDetailCard( + _interactive_detail_card.InteractiveDetailCard c, +) => { + 'title': c.title, + 'description': c.description, + 'tabs': c.tabs, + 'classes': c.classes, +}; +Map __leaderboard_tableLeaderboardTable( + _leaderboard_table.LeaderboardTable c, +) => {'evals': c.evals, 'benchmarks': c.benchmarks}; +Map __models_explorerModelsExplorer( + _models_explorer.ModelsExplorer c, +) => {'evals': c.evals, 'benchmarks': c.benchmarks}; +Map __task_specificationsTaskSpecifications( + _task_specifications.TaskSpecifications c, +) => {'specs': c.specs}; Map __headerHeader(_header.Header c) => { 'contrastLogoSrc': c.contrastLogoSrc, 'defaultLogoSrc': c.defaultLogoSrc, diff --git a/sites/www/lib/src/components/common/drawer.dart b/sites/www/lib/src/components/common/drawer.dart new file mode 100644 index 00000000000..8c21663ea66 --- /dev/null +++ b/sites/www/lib/src/components/common/drawer.dart @@ -0,0 +1,268 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'dart:async'; + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:universal_web/web.dart' as web; + +import 'icon.dart'; + +/// The viewport edge a [Drawer] slides in from. +enum DrawerSide { + /// The leading edge, which is the left edge in left-to-right locales. + start, + + /// The trailing edge, which is the right edge in left-to-right locales. + end, +} + +/// A panel that slides in from an edge of the viewport, +/// covering the page behind a dimmed scrim. +/// +/// The drawer stays mounted while closed so both the enter +/// and exit transitions can run, and so the content remains +/// visible while the drawer animates away. +/// Control its visibility with [isOpen] and +/// react to dismissals — the scrim, the close button, +/// or the Escape key — with [onClose]. +class Drawer extends StatefulComponent { + const Drawer( + this.children, { + required this.id, + required this.isOpen, + required this.onClose, + required this.title, + this.subtitle, + this.titleVisible = true, + this.side = DrawerSide.end, + this.classes, + super.key, + }); + + /// The contents of the scrollable region below the header. + final List children; + + /// A page-unique identifier used to derive the drawer's element IDs. + /// + /// Supplied by the caller rather than generated, so the IDs + /// rendered on the server match the ones the client hydrates with. + final String id; + + /// Whether the drawer is currently shown. + final bool isOpen; + + /// Called when the user dismisses the drawer. + /// + /// The drawer doesn't close itself, so the owner + /// must set [isOpen] to `false` in response. + final void Function() onClose; + + /// The heading shown in the drawer header, + /// which also labels the dialog for screen readers. + final String title; + + /// Optional supporting text shown beneath the [title]. + final String? subtitle; + + /// Whether to render [title] and [subtitle] in the header. + /// + /// Set this to `false` when [children] already provide a heading. The + /// [title] still names the dialog for screen readers. + final bool titleVisible; + + /// The viewport edge the drawer slides in from. + final DrawerSide side; + + /// Extra classes to apply to the drawer's root element. + final String? classes; + + @override + State createState() => _DrawerState(); +} + +class _DrawerState extends State { + String get _titleId => '${component.id}-title'; + String get _panelId => '${component.id}-panel'; + String get _closeButtonId => '${component.id}-close'; + + StreamSubscription? _keyDownSubscription; + + /// The element focused before the drawer opened, restored on close. + web.Element? _previouslyFocused; + + @override + void initState() { + super.initState(); + if (!kIsWeb) return; + + _keyDownSubscription = web.EventStreamProviders.keyDownEvent + .forTarget(web.document) + .listen(_onKeyDown); + + if (component.isOpen) _handleOpened(); + } + + @override + void didUpdateComponent(covariant Drawer oldComponent) { + super.didUpdateComponent(oldComponent); + if (!kIsWeb || oldComponent.isOpen == component.isOpen) return; + + if (component.isOpen) { + _handleOpened(); + } else { + _handleClosed(); + } + } + + @override + void dispose() { + if (_keyDownSubscription case final subscription?) { + unawaited(subscription.cancel()); + } + if (kIsWeb && component.isOpen) { + _unlockPageScroll(); + } + super.dispose(); + } + + void _onKeyDown(web.KeyboardEvent event) { + if (!component.isOpen) return; + + switch (event.key) { + case 'Escape': + event.preventDefault(); + component.onClose(); + case 'Tab': + _trapFocus(event); + } + } + + /// Keeps tabbing inside the dialog, as required for a modal. + void _trapFocus(web.KeyboardEvent event) { + final focusable = _focusableElements(); + if (focusable.isEmpty) return; + + final first = focusable.first; + final last = focusable.last; + final active = web.document.activeElement; + + if (event.shiftKey && (active == null || active == first)) { + event.preventDefault(); + last.focus(); + } else if (!event.shiftKey && active == last) { + event.preventDefault(); + first.focus(); + } + } + + List _focusableElements() { + final panel = web.document.getElementById(_panelId); + if (panel == null) return const []; + + final nodes = panel.querySelectorAll( + 'a[href], button:not([disabled]), input:not([disabled]), ' + 'select:not([disabled]), textarea:not([disabled]), [tabindex="0"]', + ); + + return [ + for (var i = 0; i < nodes.length; i++) + if (nodes.item(i) case final web.HTMLElement element) element, + ]; + } + + void _handleOpened() { + _previouslyFocused = web.document.activeElement; + _lockPageScroll(); + // The close button is only focusable once the drawer + // is no longer inert, which happens after the DOM updates. + Timer.run(() { + if (!component.isOpen) return; + (web.document.getElementById(_closeButtonId) as web.HTMLElement?) + ?.focus(); + }); + } + + void _handleClosed() { + _unlockPageScroll(); + (_previouslyFocused as web.HTMLElement?)?.focus(); + _previouslyFocused = null; + } + + void _lockPageScroll() { + web.document.body?.style.overflow = 'hidden'; + } + + void _unlockPageScroll() { + web.document.body?.style.removeProperty('overflow'); + } + + @override + Component build(BuildContext context) { + final isOpen = component.isOpen; + + return div( + classes: [ + 'drawer', + 'drawer--${component.side.name}', + if (isOpen) 'drawer--open', + ...?component.classes?.split(' '), + ].join(' '), + [ + div( + classes: 'drawer__scrim', + attributes: const {'aria-hidden': 'true'}, + events: {'click': (_) => component.onClose()}, + const [], + ), + aside( + id: _panelId, + classes: 'drawer__panel', + attributes: { + 'role': 'dialog', + 'aria-modal': 'true', + if (component.titleVisible) + 'aria-labelledby': _titleId + else + 'aria-label': component.title, + 'tabindex': '-1', + // Keeps the offscreen drawer out of the tab order + // and hidden from assistive technology. + if (!isOpen) 'inert': '', + }, + [ + div( + classes: [ + 'drawer__header', + if (!component.titleVisible) 'drawer__header--bare', + ].join(' '), + [ + if (component.titleVisible) + div(classes: 'drawer__heading-group', [ + h2(id: _titleId, classes: 'drawer__title', [ + .text(component.title), + ]), + if (component.subtitle case final subtitle?) + p(classes: 'drawer__subtitle', [.text(subtitle)]), + ]), + button( + id: _closeButtonId, + classes: 'drawer__close', + attributes: const { + 'type': 'button', + 'aria-label': 'Close panel', + }, + onClick: component.onClose, + const [Icon(symbol: 'close')], + ), + ], + ), + div(classes: 'drawer__body', component.children), + ], + ), + ], + ); + } +} diff --git a/sites/www/lib/src/components/common/filters.dart b/sites/www/lib/src/components/common/filters.dart index d9325176922..d91ce7a38c5 100644 --- a/sites/www/lib/src/components/common/filters.dart +++ b/sites/www/lib/src/components/common/filters.dart @@ -9,6 +9,8 @@ import 'package:universal_web/web.dart' as web; import 'filters_dropdown.dart'; +export 'filters_dropdown.dart'; + /// A single, independently selectable dimension items can be filtered by, /// such as their location or their host. class FilterType { diff --git a/sites/www/lib/src/components/flutterbench/bench_formatters.dart b/sites/www/lib/src/components/flutterbench/bench_formatters.dart new file mode 100644 index 00000000000..7b399e8204a --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/bench_formatters.dart @@ -0,0 +1,36 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +/// Shared number formatting for FlutterBench result surfaces. +library; + +/// Formats a token count as a compact, human-readable string. +String formatTokens(int count) { + if (count <= 0) return '—'; + if (count >= 1000000) return '${(count / 1000000).toStringAsFixed(1)}M'; + if (count >= 1000) return '${(count / 1000).toStringAsFixed(0)}k'; + return count.toString(); +} + +/// Formats a US dollar amount, keeping enough precision for sub-cent costs. +String formatCost(double? costUsd) { + if (costUsd == null) return '—'; + if (costUsd == 0) return r'$0'; + if (costUsd < 0.01) return '\$${costUsd.toStringAsFixed(4)}'; + return '\$${costUsd.toStringAsFixed(3)}'; +} + +/// Formats a duration in seconds as minutes and seconds. +String formatDuration(double? seconds) { + if (seconds == null) return '—'; + if (seconds < 60) return '${seconds.round()}s'; + + final minutes = seconds ~/ 60; + final remainder = (seconds - minutes * 60).round(); + return '${minutes}m ${remainder}s'; +} + +/// Formats a 0–1 reward score to two decimal places. +String formatScore(double? score) => + score == null ? '—' : score.toStringAsFixed(2); diff --git a/sites/www/lib/src/components/flutterbench/benchmark_scores.dart b/sites/www/lib/src/components/flutterbench/benchmark_scores.dart new file mode 100644 index 00000000000..2abece018e3 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/benchmark_scores.dart @@ -0,0 +1,189 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import '../../models/content/flutterbench_content.dart'; + +/// How a benchmark metric is ordered when ranking models against each other. +enum BenchmarkMetric { + /// Reward on a 0–1 scale, where a higher score is better. + accuracy(label: 'Accuracy', lowerIsBetter: false), + + /// US dollars spent on a single trial, where a lower cost is better. + cost(label: 'Cost', lowerIsBetter: true), + + /// Wall-clock seconds for a single trial, where a lower time is better. + latency(label: 'Latency', lowerIsBetter: true); + + const BenchmarkMetric({required this.label, required this.lowerIsBetter}); + + /// The tab label shown for this metric. + final String label; + + /// Whether smaller values rank ahead of larger ones. + final bool lowerIsBetter; +} + +/// One model configuration's result on a single benchmark task. +class BenchmarkScore { + const BenchmarkScore({ + required this.status, + this.reward, + this.costUsd, + this.latencySeconds, + }); + + /// Reads a score from the map produced by [toMap]. + factory BenchmarkScore.fromMap(Map map) => BenchmarkScore( + status: map['status'] as String? ?? 'unknown', + reward: (map['reward'] as num?)?.toDouble(), + costUsd: (map['cost'] as num?)?.toDouble(), + latencySeconds: (map['latency'] as num?)?.toDouble(), + ); + + /// The trial outcome: `pass`, `partial`, `fail`, or `error`. + final String status; + + final double? reward; + final double? costUsd; + final double? latencySeconds; + + /// Whether the trial failed before producing a score. + bool get isErrored => status == 'error'; + + /// The value for [metric], or `null` when it wasn't recorded. + double? valueFor(BenchmarkMetric metric) => switch (metric) { + BenchmarkMetric.accuracy => reward, + BenchmarkMetric.cost => costUsd, + BenchmarkMetric.latency => latencySeconds, + }; + + /// Serializes the score for transport across a `@client` boundary. + Map toMap() => { + 'status': status, + 'reward': ?reward, + 'cost': ?costUsd, + 'latency': ?latencySeconds, + }; +} + +/// A benchmark task alongside every model configuration's result on it. +class BenchmarkRow { + const BenchmarkRow({ + required this.slug, + required this.name, + required this.category, + required this.scores, + }); + + /// Reads a row from the map produced by [toMap]. + factory BenchmarkRow.fromMap(Map map) => BenchmarkRow( + slug: map['slug'] as String, + name: map['name'] as String, + category: map['category'] as String? ?? '', + scores: { + for (final entry + in (map['scores'] as Map? ?? {}).entries) + entry.key as String: BenchmarkScore.fromMap( + (entry.value as Map).cast(), + ), + }, + ); + + final String slug; + final String name; + final String category; + + /// Results keyed by eval key. + final Map scores; + + /// The recorded values of [metric] across every model, unsorted. + Iterable valuesFor(BenchmarkMetric metric) => + scores.values.map((score) => score.valueFor(metric)).nonNulls; + + /// The 1-based rank of [evalKey] for [metric], with the number of models + /// that have a recorded value. + /// + /// Returns `null` when this model has no value to rank. + ({int rank, int total})? rankOf(String evalKey, BenchmarkMetric metric) { + final value = scores[evalKey]?.valueFor(metric); + if (value == null) return null; + + final values = valuesFor(metric).toList()..sort(); + if (metric.lowerIsBetter) { + return (rank: values.indexOf(value) + 1, total: values.length); + } + return ( + rank: values.length - values.lastIndexOf(value), + total: values.length, + ); + } + + /// Serializes the row for transport across a `@client` boundary. + Map toMap() => { + 'slug': slug, + 'name': name, + 'category': category, + 'scores': { + for (final entry in scores.entries) entry.key: entry.value.toMap(), + }, + }; +} + +/// Joins task scores with their trials to build one [BenchmarkRow] per task. +/// +/// Tasks record which trial produced each model's score, and the trials carry +/// the cost and duration measurements, so the two have to be stitched together +/// before the model detail view can rank models on all three metrics. +List buildBenchmarkRows({ + required FlutterBenchTasksData tasks, + required FlutterBenchTrialsData trials, +}) { + final trialsByName = { + for (final trial in trials.trials) trial.trialName: trial, + }; + + return [ + for (final task in tasks.tasks) + BenchmarkRow( + slug: task.slug, + name: task.displayName, + category: task.category, + scores: { + for (final entry in task.scoresByEval.entries) + if (entry.value case final Map score) + entry.key: _buildScore( + score.cast(), + trialsByName, + ), + }, + ), + ]; +} + +BenchmarkScore _buildScore( + Map score, + Map trialsByName, +) { + final trial = trialsByName[score['trial_name'] as String?]; + final durations = trial?.durations.values; + + return BenchmarkScore( + status: score['status'] as String? ?? 'unknown', + reward: (score['reward'] as num?)?.toDouble(), + costUsd: trial?.costUsd, + latencySeconds: durations == null || durations.isEmpty + ? null + : durations.reduce((a, b) => a + b), + ); +} + +/// Serializes [rows] for transport across a `@client` boundary. +List> benchmarkRowsToMaps(List rows) => [ + for (final row in rows) row.toMap(), +]; + +/// Reads the rows produced by [benchmarkRowsToMaps]. +List benchmarkRowsFromMaps(List> maps) => [ + for (final map in maps) BenchmarkRow.fromMap(map), +]; diff --git a/sites/www/lib/src/components/flutterbench/cuj_catalog.dart b/sites/www/lib/src/components/flutterbench/cuj_catalog.dart new file mode 100644 index 00000000000..a102d93741a --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/cuj_catalog.dart @@ -0,0 +1,194 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:site_shared/components/common/material_icon.dart'; +import 'package:site_shared/util.dart'; + +import '../common/filters.dart'; + +/// The five FlutterBench CUJ personas, in catalog display order. +const List cujPersonas = [ + 'The App Developer', + 'The Tech Lead / Architect', + 'The Plugin Developer', + 'The Full Stack Developer', + 'The Hybrid (Native + Flutter) Developer', +]; + +/// A short, reader-facing label for a raw CUJ persona value. +String personaLabel(String persona) => switch (persona) { + 'The App Developer' => 'App developer', + 'The Tech Lead / Architect' => 'Tech lead / architect', + 'The Plugin Developer' => 'Plugin developer', + 'The Full Stack Developer' => 'Full-stack developer', + 'The Hybrid (Native + Flutter) Developer' => 'Hybrid developer', + _ => persona, +}; + +/// A CSS color-modifier slug for a raw CUJ persona value. +String _personaColorClass(String persona) => switch (persona) { + 'The App Developer' => 'blue', + 'The Tech Lead / Architect' => 'purple', + 'The Plugin Developer' => 'teal', + 'The Full Stack Developer' => 'magenta', + 'The Hybrid (Native + Flutter) Developer' => 'amber', + _ => 'grey', +}; + +/// Filterable, searchable catalog of FlutterBench critical user journeys. +/// +/// Hydrated on the client via Jaspr's `@client` boundary. +@client +class CujCatalog extends StatefulComponent { + const CujCatalog({required this.cujs, super.key}); + + final List> cujs; + + @override + State createState() => _CujCatalogState(); +} + +class _CujCatalogState extends State { + Map> _activeFilters = {}; + String _searchQuery = ''; + final Set _expandedIds = {}; + + static const String _personaFilterId = 'personas'; + + List _buildFilters() { + return [ + FilterType( + _personaFilterId, + 'Persona', + [for (final persona in cujPersonas) personaLabel(persona)], + ), + ]; + } + + void _applyFilters(Map> filters) { + setState(() => _activeFilters = filters); + } + + void _toggleExpanded(int id) { + setState(() { + if (_expandedIds.contains(id)) { + _expandedIds.remove(id); + } else { + _expandedIds.add(id); + } + }); + } + + @override + Component build(BuildContext context) { + final filterType = _buildFilters().first; + final activePersonaLabels = _activeFilters[filterType] ?? {}; + + final filtered = component.cujs.where((cuj) { + final persona = cuj['persona'] as String? ?? ''; + if (activePersonaLabels.isNotEmpty && + !activePersonaLabels.contains(personaLabel(persona))) { + return false; + } + + if (_searchQuery.trim().isNotEmpty) { + final query = _searchQuery.toLowerCase(); + final goal = (cuj['goal'] as String? ?? '').toLowerCase(); + final tasks = (cuj['tasks'] as List? ?? const []) + .whereType>(); + final matchesTask = tasks.any( + (t) => (t['task'] as String? ?? '').toLowerCase().contains(query), + ); + if (!goal.contains(query) && + !personaLabel(persona).toLowerCase().contains(query) && + !matchesTask) { + return false; + } + } + + return true; + }).toList(); + + return div(classes: 'cuj-catalog', [ + div(classes: 'bench-filter-bar', [ + FiltersDropdown( + filters: [filterType], + activeFilters: _activeFilters, + applyFilters: _applyFilters, + ), + div(classes: 'filter-group search-input-group', [ + input( + type: InputType.search, + classes: 'bench-search-input', + value: _searchQuery, + attributes: const { + 'placeholder': 'Try "testing" or "architecture"...', + }, + onInput: (value) { + setState(() => _searchQuery = value?.toString() ?? ''); + }, + ), + ]), + span(classes: 'cuj-result-count', [ + .text('${filtered.length} / ${component.cujs.length} journeys'), + ]), + ]), + if (filtered.isEmpty) + const div(classes: 'empty-table-message', [ + .text('No journeys match the selected filters.'), + ]) + else + div(classes: 'cuj-card-list', [ + for (final cuj in filtered) _buildCard(cuj), + ]), + ]); + } + + Component _buildCard(Map cuj) { + final id = (cuj['id'] as num).toInt(); + final goal = cuj['goal'] as String? ?? ''; + final persona = cuj['persona'] as String? ?? ''; + final tasks = (cuj['tasks'] as List? ?? const []) + .whereType>() + .toList(); + final isExpanded = _expandedIds.contains(id); + + return div( + classes: ['cuj-card', if (isExpanded) 'expanded'].toClasses, + id: 'cuj-$id', + [ + button( + classes: 'cuj-card-header', + type: ButtonType.button, + attributes: {'aria-expanded': '$isExpanded'}, + events: {'click': (_) => _toggleExpanded(id)}, + [ + span( + classes: [ + 'persona-tag', + 'color-${_personaColorClass(persona)}', + ].toClasses, + [.text(personaLabel(persona))], + ), + h3(classes: 'cuj-goal', [.text(goal)]), + span(classes: 'cuj-task-count', [ + .text(tasks.length == 1 ? '1 task' : '${tasks.length} tasks'), + ]), + MaterialIcon( + isExpanded ? 'expand_less' : 'expand_more', + label: 'Expand or collapse tasks', + ), + ], + ), + if (isExpanded) + ul(classes: 'cuj-task-list', [ + for (final task in tasks) + li([.text(task['task'] as String? ?? '')]), + ]), + ], + ); + } +} diff --git a/sites/www/lib/src/components/flutterbench/data_coming_soon.dart b/sites/www/lib/src/components/flutterbench/data_coming_soon.dart new file mode 100644 index 00000000000..376f8aba30a --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/data_coming_soon.dart @@ -0,0 +1,28 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +/// A placeholder for a panel whose data FlutterBench doesn't collect yet. +/// +/// Keeps the layout of a section intact so the page doesn't reflow once the +/// real data lands. +class DataComingSoon extends StatelessComponent { + const DataComingSoon({this.note, super.key}); + + /// Optional detail about what will eventually appear here. + final String? note; + + @override + Component build(BuildContext context) { + return div(classes: 'bench-coming-soon', [ + const span(classes: 'bench-coming-soon__label', [ + .text('Data coming soon'), + ]), + if (note case final note?) + span(classes: 'bench-coming-soon__note', [.text(note)]), + ]); + } +} diff --git a/sites/www/lib/src/components/flutterbench/error_state_badge.dart b/sites/www/lib/src/components/flutterbench/error_state_badge.dart new file mode 100644 index 00000000000..95fe6bff09e --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/error_state_badge.dart @@ -0,0 +1,46 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +/// A reusable component representing an errored trial state. +/// +/// Ensures error styling is visually distinct (hatched/amber/red borders, +/// clear warning icon) everywhere it appears (leaderboard, heatmap, detail page), +/// so errors are never misread as zero scores. +class ErrorStateBadge extends StatelessComponent { + const ErrorStateBadge({ + this.exceptionType, + this.message, + this.compact = false, + super.key, + }); + + /// The exception type name, e.g. "AgentTimeoutError". + final String? exceptionType; + + /// Optional detail message. + final String? message; + + /// Whether to render in a compact pill format for dense table cells. + final bool compact; + + @override + Component build(BuildContext context) { + final label = exceptionType ?? 'Error'; + + return span( + classes: ['bench-error-badge', if (compact) 'compact'].join(' '), + attributes: { + 'title': message ?? label, + 'role': 'status', + }, + [ + const span(classes: 'bench-error-icon', [.text('⚠')]), + span(classes: 'bench-error-label', [.text(label)]), + ], + ); + } +} diff --git a/sites/www/lib/src/components/flutterbench/grader_matrix.dart b/sites/www/lib/src/components/flutterbench/grader_matrix.dart new file mode 100644 index 00000000000..f3a785cf5d6 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/grader_matrix.dart @@ -0,0 +1,100 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:site_shared/util.dart'; + +import 'methodology_components.dart'; + +/// Interactive filterable Grader Matrix showing evaluation graders in a carousel. +@client +class GraderMatrix extends StatefulComponent { + const GraderMatrix({ + required this.title, + required this.description, + required this.filters, + required this.graders, + super.key, + }); + + final String title; + final String description; + final List> filters; + final List> graders; + + @override + State createState() => _GraderMatrixState(); +} + +class _GraderMatrixState extends State { + String _activeFilter = 'all'; + + @override + Component build(BuildContext context) { + final filteredGraders = component.graders.where((g) { + if (_activeFilter == 'all') return true; + final cat = g['category'] as String? ?? ''; + return cat.toLowerCase() == _activeFilter.toLowerCase(); + }).toList(); + + return div(classes: 'grader-matrix', [ + div(classes: 'matrix-header', [ + div(classes: 'matrix-title-area', [ + h3([.text(component.title)]), + p([renderDescriptionWithCode(component.description)]), + ]), + if (component.filters.isNotEmpty) + div(classes: 'matrix-filters', [ + for (final filter in component.filters) + button( + classes: filter['id'] == _activeFilter + ? 'filter-btn active' + : 'filter-btn', + events: { + 'click': (_) { + setState(() { + _activeFilter = filter['id'] as String? ?? 'all'; + }); + }, + }, + [.text(filter['label'] as String? ?? '')], + ), + ]), + ]), + div(classes: 'grader-carousel-wrapper', [ + div(classes: 'grader-cards-track', [ + for (final grader in filteredGraders) + div( + classes: [ + 'grader-card', + 'cat-${grader['category'] as String? ?? ''}', + 'cat-${grader['type'] as String? ?? ''}', + ].toClasses, + [ + div(classes: 'grader-header', [ + span( + classes: + 'grader-cat ${grader['category'] as String? ?? ''}', + [.text(grader['category_label'] as String? ?? '')], + ), + span( + classes: + 'badge grader-badge badge-${grader['type'] as String? ?? ''}', + [.text(grader['type_label'] as String? ?? '')], + ), + ]), + h4([.text(grader['name'] as String? ?? '')]), + p([ + renderDescriptionWithCode( + grader['description'] as String? ?? '', + ), + ]), + ], + ), + ]), + ]), + ]); + } +} diff --git a/sites/www/lib/src/components/flutterbench/interactive_detail_card.dart b/sites/www/lib/src/components/flutterbench/interactive_detail_card.dart new file mode 100644 index 00000000000..edde1b7d1b7 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/interactive_detail_card.dart @@ -0,0 +1,121 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:site_shared/util.dart'; + +import 'methodology_components.dart'; + +/// Interactive tabbed card used for Score Triage and Evaluation Matrix. +@client +class InteractiveDetailCard extends StatefulComponent { + const InteractiveDetailCard({ + required this.title, + required this.description, + required this.tabs, + this.classes = 'interactive-detail-card', + super.key, + }); + + final String title; + final String description; + final List> tabs; + final String classes; + + @override + State createState() => _InteractiveDetailCardState(); +} + +class _InteractiveDetailCardState extends State { + late String _activeTab = (component.tabs.firstOrNull?['id'] as String?) ?? ''; + + @override + Component build(BuildContext context) { + return div(classes: component.classes, [ + div(classes: 'card-header-area triage-header', [ + h3([.text(component.title)]), + p([.text(component.description)]), + ]), + div(classes: 'card-tabs-grid triage-tiers-grid', [ + for (final tab in component.tabs) + button( + classes: [ + 'card-tab-btn', + 'triage-tier-btn', + 'variant-${tab['variant'] as String? ?? 'blue'}', + 'tier-${tab['variant'] as String? ?? 'blue'}', + if (tab['id'] == _activeTab) 'active', + ].toClasses, + events: { + 'click': (_) { + setState(() { + _activeTab = tab['id'] as String? ?? ''; + }); + }, + }, + [ + div( + classes: 'tab-primary-label tier-score', + [.text(tab['primary_label'] as String? ?? '')], + ), + div( + classes: 'tab-secondary-label tier-name', + [.text(tab['secondary_label'] as String? ?? '')], + ), + ], + ), + ]), + div(classes: 'card-panels-container triage-detail-card', [ + for (final tab in component.tabs) + div( + classes: [ + 'card-panel', + 'triage-panel', + if (tab['id'] == _activeTab) 'active', + ].toClasses, + [ + div(classes: 'panel-heading', [ + h4([.text(tab['heading'] as String? ?? '')]), + if (tab['badge'] case final String badge) + span( + classes: + 'panel-badge badge-${tab['variant'] as String? ?? 'blue'}', + [.text(badge)], + ), + ]), + p( + classes: 'panel-overview criteria-text', + [renderDescriptionWithCode(tab['overview'] as String? ?? '')], + ), + if (tab['items'] case final List itemsList) + if (itemsList.isNotEmpty) + div(classes: 'panel-items-section actions-section', [ + if (tab['items_label'] case final String label) + div(classes: 'items-label actions-label', [.text(label)]), + ul([ + for (final item + in itemsList.whereType>()) + li([ + strong([ + .text('${item['label'] as String? ?? ''}: '), + ]), + renderDescriptionWithCode( + item['detail'] as String? ?? '', + ), + ]), + ]), + ]), + if (tab['footer_text'] case final String footer) + if (footer.isNotEmpty) + p( + classes: 'panel-footer-text', + [renderDescriptionWithCode(footer)], + ), + ], + ), + ]), + ]); + } +} diff --git a/sites/www/lib/src/components/flutterbench/leaderboard_table.dart b/sites/www/lib/src/components/flutterbench/leaderboard_table.dart new file mode 100644 index 00000000000..3dcbd4645d5 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/leaderboard_table.dart @@ -0,0 +1,505 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:site_shared/util.dart'; +import 'package:universal_web/web.dart' as web; + +import '../common/drawer.dart'; +import '../common/filters.dart'; +import 'bench_formatters.dart'; +import 'benchmark_scores.dart'; +import 'error_state_badge.dart'; +import 'model_detail_view.dart'; +import 'model_name_formatter.dart'; + +enum LeaderboardSortColumn { + model, + outcomeScore, + qualityScore, + dxScore, + tokens, + cost, + overallScore, +} + +/// Interactive leaderboard table and filter controls. +/// +/// Hydrated on the client via Jaspr's `@client` boundary. +@client +class LeaderboardTable extends StatefulComponent { + const LeaderboardTable({ + required this.evals, + this.benchmarks = const [], + super.key, + }); + + final List> evals; + + /// Per-task results for every eval, shown in the details drawer. + final List> benchmarks; + + @override + State createState() => _LeaderboardTableState(); +} + +class _LeaderboardTableState extends State { + LeaderboardSortColumn _sortColumn = LeaderboardSortColumn.overallScore; + bool _sortAscending = false; + Map> _activeFilters = {}; + String _searchQuery = ''; + + /// The eval shown in the details drawer. + /// + /// This outlives [_isDrawerOpen] so the drawer keeps + /// rendering its contents while it slides closed. + Map? _drawerEval; + bool _isDrawerOpen = false; + + late final List _benchmarks = benchmarkRowsFromMaps( + component.benchmarks, + ); + + static const String _providerFilterId = 'providers'; + static const String _toolingFilterId = 'toolings'; + + List _buildFilters() { + final providers = {}; + for (final e in component.evals) { + if (e['provider'] case final String p) { + providers.add(p); + } else { + providers.add('Community'); + } + } + final sortedProviders = providers.toList()..sort(); + + return [ + FilterType(_providerFilterId, 'Provider', sortedProviders), + const FilterType(_toolingFilterId, 'Tooling', [ + 'With tooling', + 'Without tooling', + ]), + ]; + } + + @override + void initState() { + super.initState(); + if (kIsWeb) { + _loadFiltersFromUrl(); + } + } + + String _paramName(FilterType type) => type.id; + + void _loadFiltersFromUrl() { + final url = Uri.parse(web.window.location.href); + final filters = _buildFilters(); + final loaded = >{}; + + for (final type in filters) { + final param = url.queryParameters[_paramName(type)]; + if (param != null && param.isNotEmpty) { + loaded[type] = param.split(',').toSet(); + } + } + if (loaded.isNotEmpty) { + _activeFilters = loaded; + } + } + + void _applyFilters(Map> filters) { + if (kIsWeb) { + final url = Uri.parse(web.window.location.href); + final newQueryParameters = {...url.queryParameters}; + + for (final type in _buildFilters()) { + final paramName = _paramName(type); + newQueryParameters.remove(paramName); + if (filters[type] case final options? when options.isNotEmpty) { + newQueryParameters[paramName] = options.join(','); + } + } + + web.window.history.replaceState( + web.window.history.state, + '', + url.withQueryParameters(newQueryParameters).toString(), + ); + } + + setState(() { + _activeFilters = filters; + }); + } + + void _onSort(LeaderboardSortColumn column) { + setState(() { + if (_sortColumn == column) { + _sortAscending = !_sortAscending; + } else { + _sortColumn = column; + _sortAscending = false; + } + }); + } + + void _openDrawer(Map item) { + setState(() { + _drawerEval = item; + _isDrawerOpen = true; + }); + } + + void _closeDrawer() { + setState(() { + _isDrawerOpen = false; + }); + } + + @override + Component build(BuildContext context) { + final filters = _buildFilters(); + final providerType = filters[0]; + final toolingType = filters[1]; + + final activeProviders = _activeFilters[providerType] ?? {}; + final activeTooling = _activeFilters[toolingType] ?? {}; + + // 1. Filter + final filtered = component.evals.where((item) { + if (activeProviders.isNotEmpty) { + final provider = (item['provider'] as String?) ?? 'Community'; + if (!activeProviders.contains(provider)) { + return false; + } + } + + if (activeTooling.isNotEmpty) { + final hasTooling = item['has_dart_tooling'] == true; + final wantsWith = activeTooling.contains('With tooling'); + final wantsWithout = activeTooling.contains('Without tooling'); + if (wantsWith && !wantsWithout && !hasTooling) { + return false; + } + if (wantsWithout && !wantsWith && hasTooling) { + return false; + } + } + + if (_searchQuery.trim().isNotEmpty) { + final query = _searchQuery.toLowerCase(); + final model = ((item['model_name'] as String?) ?? '').toLowerCase(); + final agent = ((item['agent_name'] as String?) ?? '').toLowerCase(); + if (!model.contains(query) && !agent.contains(query)) { + return false; + } + } + + return true; + }).toList(); + + // 2. Sort + filtered.sort((itemA, itemB) { + int cmp; + switch (_sortColumn) { + case LeaderboardSortColumn.model: + final ma = (itemA['model_short_name'] as String? ?? '').toLowerCase(); + final mb = (itemB['model_short_name'] as String? ?? '').toLowerCase(); + cmp = ma.compareTo(mb); + case LeaderboardSortColumn.outcomeScore: + final oa = (itemA['outcome_score'] as num?)?.toDouble() ?? 0.0; + final ob = (itemB['outcome_score'] as num?)?.toDouble() ?? 0.0; + cmp = oa.compareTo(ob); + case LeaderboardSortColumn.qualityScore: + final qa = (itemA['quality_score'] as num?)?.toDouble() ?? 0.0; + final qb = (itemB['quality_score'] as num?)?.toDouble() ?? 0.0; + cmp = qa.compareTo(qb); + case LeaderboardSortColumn.dxScore: + final da = (itemA['dx_score'] as num?)?.toDouble() ?? 0.0; + final db = (itemB['dx_score'] as num?)?.toDouble() ?? 0.0; + cmp = da.compareTo(db); + case LeaderboardSortColumn.tokens: + final ta = + ((itemA['input_tokens'] as num?)?.toInt() ?? 0) + + ((itemA['output_tokens'] as num?)?.toInt() ?? 0); + final tb = + ((itemB['input_tokens'] as num?)?.toInt() ?? 0) + + ((itemB['output_tokens'] as num?)?.toInt() ?? 0); + cmp = ta.compareTo(tb); + case LeaderboardSortColumn.cost: + final ca = (itemA['cost_usd'] as num?)?.toDouble() ?? 0.0; + final cb = (itemB['cost_usd'] as num?)?.toDouble() ?? 0.0; + cmp = ca.compareTo(cb); + case LeaderboardSortColumn.overallScore: + final ra = (itemA['mean_reward'] as num?)?.toDouble() ?? 0.0; + final rb = (itemB['mean_reward'] as num?)?.toDouble() ?? 0.0; + cmp = ra.compareTo(rb); + } + return _sortAscending ? cmp : -cmp; + }); + + return div(classes: 'bench-leaderboard-section', [ + // FilterBar + div(classes: 'bench-filter-bar', [ + FiltersDropdown( + filters: filters, + activeFilters: _activeFilters, + applyFilters: _applyFilters, + ), + + div(classes: 'filter-group search-input-group', [ + input( + type: InputType.search, + classes: 'bench-search-input', + value: _searchQuery, + attributes: const {'placeholder': 'Search models...'}, + onInput: (value) { + setState(() => _searchQuery = value?.toString() ?? ''); + }, + ), + ]), + ]), + + // Table + div(classes: 'bench-table-wrapper', [ + table(classes: 'bench-table', [ + thead([ + tr([ + th( + classes: 'col-model sortable', + events: {'click': (_) => _onSort(LeaderboardSortColumn.model)}, + [ + const .text('Model'), + _buildSortIndicator(LeaderboardSortColumn.model), + ], + ), + th( + classes: 'col-outcome sortable', + events: { + 'click': (_) => _onSort(LeaderboardSortColumn.outcomeScore), + }, + [ + const .text('Outcome Score'), + _buildSortIndicator(LeaderboardSortColumn.outcomeScore), + ], + ), + th( + classes: 'col-quality sortable', + events: { + 'click': (_) => _onSort(LeaderboardSortColumn.qualityScore), + }, + [ + const .text('Quality Score'), + _buildSortIndicator(LeaderboardSortColumn.qualityScore), + ], + ), + th( + classes: 'col-dx sortable', + events: { + 'click': (_) => _onSort(LeaderboardSortColumn.dxScore), + }, + [ + const .text('DX Score'), + _buildSortIndicator(LeaderboardSortColumn.dxScore), + ], + ), + th( + classes: 'col-tokens sortable', + events: {'click': (_) => _onSort(LeaderboardSortColumn.tokens)}, + [ + const .text('Token'), + _buildSortIndicator(LeaderboardSortColumn.tokens), + ], + ), + th( + classes: 'col-cost sortable', + events: {'click': (_) => _onSort(LeaderboardSortColumn.cost)}, + [ + const .text('Cost'), + _buildSortIndicator(LeaderboardSortColumn.cost), + ], + ), + th( + classes: 'col-overall sortable', + events: { + 'click': (_) => _onSort(LeaderboardSortColumn.overallScore), + }, + [ + const .text('Overall Score'), + _buildSortIndicator(LeaderboardSortColumn.overallScore), + ], + ), + ]), + ]), + tbody([ + if (filtered.isEmpty) + const tr([ + td( + attributes: {'colspan': '7'}, + classes: 'empty-table-message', + [.text('No models match the selected filters.')], + ), + ]) + else + for (var i = 0; i < filtered.length; i++) + _buildTableRow(filtered[i], rank: i + 1), + ]), + ]), + ]), + + _buildDetailsDrawer(), + ]); + } + + Component _buildDetailsDrawer() { + final item = _drawerEval; + + return Drawer( + id: 'bench-leaderboard-details', + isOpen: _isDrawerOpen, + onClose: _closeDrawer, + classes: 'bench-details-drawer', + titleVisible: false, + title: item == null + ? 'Model details' + : '${formatModelName(item['model_short_name'] as String)} details', + [ + if (item != null) + ModelDetailView( + eval: item, + evals: component.evals, + benchmarks: _benchmarks, + modelsPageLink: '/ai/flutterbench/models?model=${item['eval_key']}', + ), + ], + ); + } + + Component _buildSortIndicator(LeaderboardSortColumn column) { + if (_sortColumn != column) { + return const span(classes: 'sort-indicator inactive', [.text(' ↕')]); + } + return span( + classes: 'sort-indicator active', + [.text(_sortAscending ? ' ↑' : ' ↓')], + ); + } + + Component _buildTableRow(Map item, {required int rank}) { + final evalKey = item['eval_key'] as String; + final modelShort = item['model_short_name'] as String; + final agentName = item['agent_name'] as String; + final provider = item['provider'] as String? ?? 'Community'; + + final outcomeScore = (item['outcome_score'] as num?)?.toDouble(); + final qualityScore = (item['quality_score'] as num?)?.toDouble(); + final dxScore = (item['dx_score'] as num?)?.toDouble(); + final mean = (item['mean_reward'] as num?)?.toDouble() ?? 0.0; + final min = (item['min_reward'] as num?)?.toDouble() ?? 0.0; + final max = (item['max_reward'] as num?)?.toDouble() ?? 0.0; + final cost = (item['cost_usd'] as num?)?.toDouble() ?? 0.0; + final inTok = (item['input_tokens'] as num?)?.toInt() ?? 0; + final outTok = (item['output_tokens'] as num?)?.toInt() ?? 0; + final totalTok = inTok + outTok; + final nErrors = (item['n_errors'] as num?)?.toInt() ?? 0; + final nTrials = (item['n_trials'] as num?)?.toInt() ?? 0; + final isErrored = nErrors > 0 && nTrials == 0; + + final isSelected = _isDrawerOpen && _drawerEval?['eval_key'] == evalKey; + + return tr( + classes: [ + 'bench-row', + if (isSelected) 'selected', + ].join(' '), + attributes: { + 'tabindex': '0', + 'aria-haspopup': 'dialog', + 'aria-expanded': '$isSelected', + }, + events: { + 'click': (_) => _openDrawer(item), + 'keydown': (event) { + if (event case web.KeyboardEvent(:final key) + when key == 'Enter' || key == ' ') { + event.preventDefault(); + _openDrawer(item); + } + }, + }, + [ + td(classes: 'col-model', [ + div(classes: 'model-info-cell', [ + div(classes: 'model-title-row', [ + span(classes: 'rank-pill', [.text('#$rank')]), + span( + classes: 'model-name-text', + [.text(formatModelName(modelShort))], + ), + ]), + span(classes: 'agent-subtext', [.text('$agentName · $provider')]), + ]), + ]), + td(classes: 'col-outcome', [ + _buildScoreBadge(outcomeScore, isErrored), + ]), + td(classes: 'col-quality', [ + _buildScoreBadge(qualityScore, isErrored), + ]), + td(classes: 'col-dx', [ + _buildScoreBadge(dxScore, isErrored), + ]), + td(classes: 'col-tokens', [ + .text(formatTokens(totalTok)), + ]), + td(classes: 'col-cost', [ + .text(cost > 0 ? '\$${cost.toStringAsFixed(3)}' : '—'), + ]), + td(classes: 'col-overall', [ + if (isErrored) + const ErrorStateBadge(exceptionType: 'Errored', compact: true) + else + div(classes: 'reward-cell', [ + span( + classes: [ + 'reward-main-score', + if (mean >= 0.8) + 'score-high' + else if (mean >= 0.5) + 'score-mid' + else + 'score-low', + ].join(' '), + [.text(mean.toStringAsFixed(2))], + ), + if (min != max) + span(classes: 'reward-range', [ + .text( + ' (${min.toStringAsFixed(2)}–${max.toStringAsFixed(2)})', + ), + ]), + ]), + ]), + ], + ); + } + + Component _buildScoreBadge(double? score, bool isErrored) { + if (isErrored || score == null) { + return const span(classes: 'text-muted', [.text('—')]); + } + final scoreClass = score >= 0.8 + ? 'score-high' + : score >= 0.5 + ? 'score-mid' + : 'score-low'; + return span( + classes: 'score-pill $scoreClass', + [.text(score.toStringAsFixed(2))], + ); + } +} diff --git a/sites/www/lib/src/components/flutterbench/methodology_components.dart b/sites/www/lib/src/components/flutterbench/methodology_components.dart new file mode 100644 index 00000000000..583a0f9618d --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/methodology_components.dart @@ -0,0 +1,162 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:site_shared/components/common/material_icon.dart'; +import 'package:site_shared/util.dart'; + +export 'grader_matrix.dart'; +export 'interactive_detail_card.dart'; +export 'task_specifications.dart'; + +/// Renders text with optional inline code segments delimited by backticks. +Component renderDescriptionWithCode(String text) { + if (!text.contains('`')) { + return .text(text); + } + final parts = text.split('`'); + final children = []; + for (var i = 0; i < parts.length; i++) { + if (i.isOdd) { + children.add(code([.text(parts[i])])); + } else if (parts[i].isNotEmpty) { + children.add(.text(parts[i])); + } + } + return .fragment(children); +} + +// ----------------------------------------------------------------------------- +// 1. Three Dimensions Cards +// ----------------------------------------------------------------------------- + +/// Cards displaying the Three Core Dimensions of FlutterBench evaluation. +class ThreeDimensionsCards extends StatelessComponent { + const ThreeDimensionsCards({required this.dimensions, super.key}); + + final List> dimensions; + + @override + Component build(BuildContext context) { + return div(classes: 'dimension-cards-grid', [ + for (final dim in dimensions) + div(classes: 'dimension-card', [ + div(classes: 'card-header-row', [ + div( + classes: [ + 'card-icon-wrap', + dim['category'] as String? ?? '', + ].toClasses, + [MaterialIcon(dim['icon'] as String? ?? 'info')], + ), + h4([.text(dim['title'] as String? ?? '')]), + ]), + p([renderDescriptionWithCode(dim['description'] as String? ?? '')]), + if (dim['footer_items'] != null || dim['badge'] != null) + div(classes: 'card-footer-info', [ + if (dim['footer_items'] case final String footer) + span([.text(footer)]), + if (dim['badge'] case final String badge) + span( + classes: [ + 'badge', + dim['category'] as String? ?? '', + ].toClasses, + [.text(badge)], + ), + ]), + ]), + ]); + } +} + +// ----------------------------------------------------------------------------- +// 2. Reliability Comparison Cards +// ----------------------------------------------------------------------------- + +/// Comparison cards for Capability (pass@k) vs Consistency (pass^k). +class ReliabilityCards extends StatelessComponent { + const ReliabilityCards({required this.cards, super.key}); + + final List> cards; + + @override + Component build(BuildContext context) { + return div(classes: 'reliability-comparison-grid', [ + for (final card in cards) + div( + classes: [ + 'reliability-card', + if (card['is_north_star'] == true) 'north-star', + ].toClasses, + [ + div(classes: 'reliability-header', [ + span(classes: 'tag', [ + if (card['tag_icon'] case final String icon) ...[ + MaterialIcon(icon), + const .text(' '), + ], + .text(card['tag'] as String? ?? ''), + ]), + span(classes: 'math-pill', [ + .text(card['math_pill'] as String? ?? ''), + ]), + ]), + h4([.text(card['title'] as String? ?? '')]), + if (card['description_parts'] case final List parts) + p([ + for (final part in parts.whereType>()) ...[ + if (part['text'] case final String text) .text(text), + if (part['em'] case final String emText) em([.text(emText)]), + if (part['strong'] case final String strongText) + strong([.text(strongText)]), + ], + ]) + else if (card['description'] case final String desc) + p([renderDescriptionWithCode(desc)]), + ], + ), + ]); + } +} + +// ----------------------------------------------------------------------------- +// 3. CUJ Journey Diagram +// ----------------------------------------------------------------------------- + +/// Diagram visualizing developer persona -> goal -> task steps. +class CujDiagram extends StatelessComponent { + const CujDiagram({required this.sections, super.key}); + + final List> sections; + + @override + Component build(BuildContext context) { + return div(classes: 'cuj-diagram-card', [ + for (final section in sections) + div(classes: 'cuj-diagram-section', [ + div(classes: 'section-sidebar', [ + div( + classes: + 'section-icon variant-${section['variant'] as String? ?? 'blue'}', + [MaterialIcon(section['icon'] as String? ?? 'info')], + ), + span( + classes: 'section-label', + [.text(section['label'] as String? ?? '')], + ), + ]), + div(classes: 'section-items', [ + for (final item in (section['items'] as List? ?? const [])) + div( + classes: + 'cuj-pill pill-${section['variant'] as String? ?? 'blue'}', + [.text(item.toString())], + ), + ]), + ]), + ]); + } +} diff --git a/sites/www/lib/src/components/flutterbench/model_detail_view.dart b/sites/www/lib/src/components/flutterbench/model_detail_view.dart new file mode 100644 index 00000000000..199379ee173 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/model_detail_view.dart @@ -0,0 +1,406 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'dart:math' as math; + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../common/icon.dart'; +import 'bench_formatters.dart'; +import 'benchmark_scores.dart'; +import 'data_coming_soon.dart'; +import 'error_state_badge.dart'; +import 'model_name_formatter.dart'; + +/// The full result profile for one model configuration. +/// +/// Rendered both in the leaderboard's details drawer and as the right-hand +/// pane of the models explorer, so the two surfaces stay in sync. +class ModelDetailView extends StatefulComponent { + const ModelDetailView({ + required this.eval, + required this.evals, + required this.benchmarks, + this.modelsPageLink, + super.key, + }); + + /// The model configuration being described. + final Map eval; + + /// Every configuration on the leaderboard, used to place [eval] within the + /// distribution of results. + final List> evals; + + /// Per-task results for every configuration. + final List benchmarks; + + /// Where the "View in models page" action points, or `null` to hide it. + final String? modelsPageLink; + + @override + State createState() => _ModelDetailViewState(); +} + +class _ModelDetailViewState extends State { + BenchmarkMetric _metric = BenchmarkMetric.accuracy; + + String get _evalKey => component.eval['eval_key'] as String; + + @override + Component build(BuildContext context) { + return div(classes: 'model-detail', [ + if (component.modelsPageLink case final link?) + div(classes: 'model-detail__actions', [ + a(href: link, classes: 'bench-btn-sm', const [ + .text('View in models page →'), + ]), + ]), + _buildHeader(), + _buildSpecGrid(), + _buildHeadlineStats(), + _buildHyperparameters(), + _buildBenchmarks(), + ]); + } + + Component _buildHeader() { + final eval = component.eval; + final provider = eval['provider'] as String? ?? 'Community'; + + return header(classes: 'model-detail__header', [ + p(classes: 'model-detail__eyebrow', [.text(provider.toUpperCase())]), + h2(classes: 'model-detail__title', [ + .text(formatModelName(eval['model_short_name'] as String)), + ]), + p(classes: 'model-detail__release', [ + .text('EVAL KEY: ${eval['eval_key']}'), + ]), + ]); + } + + Component _buildSpecGrid() { + final eval = component.eval; + final nTrials = (eval['n_trials'] as num?)?.toInt() ?? 0; + final nErrors = (eval['n_errors'] as num?)?.toInt() ?? 0; + final inputTokens = (eval['input_tokens'] as num?)?.toInt() ?? 0; + final outputTokens = (eval['output_tokens'] as num?)?.toInt() ?? 0; + final hasTooling = eval['has_dart_tooling'] == true; + + return div(classes: 'model-spec-block', [ + dl(classes: 'model-spec-grid', [ + ..._buildSpecRow( + 'Developer', + eval['provider'] as String? ?? 'Community', + ), + ..._buildSpecRow('Agent', eval['agent_name'] as String), + ..._buildSpecRow('Model ID', eval['model_name'] as String), + ..._buildSpecRow('Variant', eval['variant'] as String? ?? '—'), + ..._buildSpecRow( + 'Dart tooling', + hasTooling ? 'Enabled' : 'Not enabled', + ), + ..._buildSpecRow( + 'Trials', + nErrors > 0 ? '$nTrials ($nErrors errored)' : '$nTrials', + ), + ..._buildSpecRow( + 'Tokens (in/out)', + '${formatTokens(inputTokens)} / ${formatTokens(outputTokens)}', + ), + ..._buildSpecRow( + 'Pass@1', + formatScore((eval['pass_at_1'] as num?)?.toDouble()), + ), + ]), + const DataComingSoon( + note: + 'Release date, context window, max output tokens, published token ' + 'pricing, weight availability, and input modalities.', + ), + ]); + } + + List _buildSpecRow(String label, String value) => [ + dt(classes: 'model-spec-grid__label', [.text(label.toUpperCase())]), + dd(classes: 'model-spec-grid__value', [.text(value)]), + ]; + + Component _buildHeadlineStats() { + final isErrored = _isErrored(component.eval); + + return div(classes: 'model-stat-cards', [ + _buildStatCard( + metric: BenchmarkMetric.accuracy, + label: 'Overall score', + value: isErrored ? '—' : formatScore(_accuracyOf(component.eval)), + detail: _rewardRange(), + ), + _buildStatCard( + metric: BenchmarkMetric.cost, + label: 'Cost / trial', + value: formatCost(_costOf(component.eval)), + detail: + 'Total ${formatCost((component.eval['cost_usd'] as num?)?.toDouble())}', + ), + _buildStatCard( + metric: BenchmarkMetric.latency, + label: 'Latency / trial', + value: formatDuration(_latencyOf(component.eval)), + detail: 'Mean wall clock', + ), + ]); + } + + Component _buildStatCard({ + required BenchmarkMetric metric, + required String label, + required String value, + required String detail, + }) { + final values = component.evals.map(_valueOf(metric)).nonNulls; + + return div(classes: 'model-stat-card model-stat-card--${metric.name}', [ + span(classes: 'model-stat-card__label', [.text(label.toUpperCase())]), + span(classes: 'model-stat-card__value', [.text(value)]), + span(classes: 'model-stat-card__detail', [.text(detail)]), + _buildDistribution( + values: values, + current: _valueOf(metric)(component.eval), + logScale: metric.lowerIsBetter, + ), + ]); + } + + Component _buildHyperparameters() { + return const details(classes: 'model-hyperparams', [ + summary(classes: 'model-hyperparams__summary', [ + Icon(symbol: 'chevron_right', size: .sm), + .text('View hyperparameter settings'), + ]), + div(classes: 'model-hyperparams__body', [ + DataComingSoon( + note: + 'Temperature, top-p, reasoning effort, and tool configuration ' + 'for each run.', + ), + ]), + ]); + } + + Component _buildBenchmarks() { + return div(classes: 'model-benchmarks', [ + div(classes: 'bench-segmented-control', [ + for (final metric in BenchmarkMetric.values) + button( + classes: ['segment-btn', if (metric == _metric) 'active'].join(' '), + attributes: { + 'type': 'button', + 'aria-pressed': '${metric == _metric}', + }, + onClick: () => setState(() => _metric = metric), + [.text(metric.label)], + ), + ]), + p(classes: 'model-benchmarks__hint', [ + .text(switch (_metric) { + BenchmarkMetric.accuracy => + 'Reward on a 0–1 scale. Higher is better. ' + 'Rankings compare this configuration against every other one ' + 'scored on the same task.', + BenchmarkMetric.cost => + 'Cost of a single trial. Lower is better. ' + 'Each tick is a scored configuration on a log scale, ' + 'with this one marked.', + BenchmarkMetric.latency => + 'Wall-clock time for a single trial. Lower is better. ' + 'Each tick is a scored configuration on a log scale, ' + 'with this one marked.', + }), + ]), + _buildBenchmarkTable(), + ]); + } + + Component _buildBenchmarkTable() { + if (component.benchmarks.isEmpty) { + return const DataComingSoon(note: 'Per-task results for this model.'); + } + + return div(classes: 'bench-table-wrapper', [ + table(classes: 'bench-table model-benchmarks__table', [ + thead([ + tr([ + const th(classes: 'col-benchmark', [.text('Benchmark')]), + th(classes: 'col-distribution', [ + .text(switch (_metric) { + BenchmarkMetric.accuracy => 'Score', + BenchmarkMetric.cost => 'Cost distribution', + BenchmarkMetric.latency => 'Latency distribution', + }), + ]), + th(classes: 'col-value', [.text(_metric.label)]), + const th(classes: 'col-ranking', [.text('Ranking')]), + ]), + ]), + tbody([ + for (final row in component.benchmarks) _buildBenchmarkRow(row), + ]), + ]), + ]); + } + + Component _buildBenchmarkRow(BenchmarkRow row) { + final score = row.scores[_evalKey]; + final value = score?.valueFor(_metric); + final ranking = row.rankOf(_evalKey, _metric); + + return tr(classes: 'benchmark-row', [ + td(classes: 'col-benchmark', [ + a( + href: '/ai/flutterbench/tasks/${row.slug}', + classes: 'benchmark-row__link', + [.text(row.name)], + ), + ]), + td(classes: 'col-distribution', [ + if (score?.isErrored ?? false) + const ErrorStateBadge(exceptionType: 'Errored', compact: true) + else if (value == null) + const span(classes: 'text-muted', [.text('No data for this task.')]) + else if (_metric == BenchmarkMetric.accuracy) + _buildScoreBar(value) + else + _buildDistribution( + values: row.valuesFor(_metric), + current: value, + logScale: true, + ), + ]), + td(classes: 'col-value', [ + .text(switch (_metric) { + BenchmarkMetric.accuracy => formatScore(value), + BenchmarkMetric.cost => formatCost(value), + BenchmarkMetric.latency => formatDuration(value), + }), + ]), + td(classes: 'col-ranking', [ + if (ranking case final ranking?) + span(classes: 'benchmark-row__rank', [ + span(classes: 'benchmark-row__rank-value', [ + .text('${ranking.rank}'), + ]), + .text(' / ${ranking.total}'), + ]) + else + const span(classes: 'text-muted', [.text('—')]), + ]), + ]); + } + + Component _buildScoreBar(double reward) { + final level = reward >= 0.8 + ? 'score-high' + : reward >= 0.5 + ? 'score-mid' + : 'score-low'; + + return div( + classes: 'score-bar', + attributes: { + 'role': 'img', + 'aria-label': '${(reward * 100).round()} out of 100', + }, + [ + div( + classes: 'score-bar__fill $level', + styles: Styles(raw: {'width': '${(reward * 100).clamp(0, 100)}%'}), + const [], + ), + ], + ); + } + + /// A strip of ticks, one per scored configuration, with [current] marked. + Component _buildDistribution({ + required Iterable values, + required double? current, + required bool logScale, + }) { + final sorted = values.toList()..sort(); + if (sorted.isEmpty) { + return const span(classes: 'text-muted', [.text('—')]); + } + + final min = sorted.first; + final max = sorted.last; + // A log scale needs strictly positive bounds, and a flat distribution has + // no spread to map onto, so both fall back to centering every tick. + final useLog = logScale && min > 0 && max > min; + double position(double value) { + if (max <= min) return 50; + final fraction = useLog + ? (math.log(value) - math.log(min)) / (math.log(max) - math.log(min)) + : (value - min) / (max - min); + return (fraction * 100).clamp(0, 100); + } + + return div(classes: 'distribution', [ + div(classes: 'distribution__track', [ + for (final value in sorted) + span( + classes: 'distribution__tick', + styles: Styles(raw: {'left': '${position(value)}%'}), + const [], + ), + if (current != null) + span( + classes: 'distribution__marker', + styles: Styles(raw: {'left': '${position(current)}%'}), + const [], + ), + ]), + ]); + } + + String _rewardRange() { + final min = (component.eval['min_reward'] as num?)?.toDouble(); + final max = (component.eval['max_reward'] as num?)?.toDouble(); + if (min == null || max == null || min == max) return 'Mean reward'; + return 'Range ${formatScore(min)}–${formatScore(max)}'; + } + + double? Function(Map) _valueOf(BenchmarkMetric metric) => + switch (metric) { + BenchmarkMetric.accuracy => _accuracyOf, + BenchmarkMetric.cost => _costOf, + BenchmarkMetric.latency => _latencyOf, + }; + + double? _accuracyOf(Map eval) => + _isErrored(eval) ? null : (eval['mean_reward'] as num?)?.toDouble(); + + double? _costOf(Map eval) { + final cost = (eval['cost_usd'] as num?)?.toDouble(); + final trials = (eval['n_trials'] as num?)?.toInt() ?? 0; + if (cost == null || trials <= 0) return null; + return cost / trials; + } + + double? _latencyOf(Map eval) { + final key = eval['eval_key'] as String; + final latencies = [ + for (final row in component.benchmarks) ?row.scores[key]?.latencySeconds, + ]; + if (latencies.isEmpty) return null; + return latencies.reduce((sum, next) => sum + next) / latencies.length; + } + + bool _isErrored(Map eval) { + final nErrors = (eval['n_errors'] as num?)?.toInt() ?? 0; + final nTrials = (eval['n_trials'] as num?)?.toInt() ?? 0; + return nErrors > 0 && nTrials == 0; + } +} diff --git a/sites/www/lib/src/components/flutterbench/model_name_formatter.dart b/sites/www/lib/src/components/flutterbench/model_name_formatter.dart new file mode 100644 index 00000000000..0bda75d07a9 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/model_name_formatter.dart @@ -0,0 +1,54 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +final RegExp _allDigits = RegExp(r'^\d+$'); + +/// Word-level overrides for tokens whose readable casing isn't a simple +/// capitalize-first-letter (brand names, version codes, OpenAI's lowercase +/// "o" model line). +const Map _wordOverrides = { + 'gpt': 'GPT', + 'deepseek': 'DeepSeek', + 'v2': 'V2', + 'v3': 'V3', + 'r1': 'R1', + 'o1': 'o1', + 'o3': 'o3', + '4o': '4o', +}; + +/// Formats a raw FlutterBench model identifier (e.g. `gpt-5`, +/// `claude-3-5-haiku`, `gemini-3.5-pro`) into a human-readable display name +/// (e.g. `GPT 5`, `Claude 3.5 Haiku`, `Gemini 3.5 Pro`). +String formatModelName(String rawModelName) { + final tokens = rawModelName.split('-').where((t) => t.isNotEmpty).toList(); + final words = []; + + var i = 0; + while (i < tokens.length) { + final token = tokens[i]; + if (_allDigits.hasMatch(token)) { + final versionParts = [token]; + var j = i + 1; + while (j < tokens.length && _allDigits.hasMatch(tokens[j])) { + versionParts.add(tokens[j]); + j++; + } + words.add(versionParts.join('.')); + i = j; + continue; + } + words.add(_formatWord(token)); + i++; + } + + return words.join(' '); +} + +String _formatWord(String word) { + final lower = word.toLowerCase(); + final override = _wordOverrides[lower]; + if (override != null) return override; + return lower[0].toUpperCase() + lower.substring(1); +} diff --git a/sites/www/lib/src/components/flutterbench/models_explorer.dart b/sites/www/lib/src/components/flutterbench/models_explorer.dart new file mode 100644 index 00000000000..e0b7ac952b2 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/models_explorer.dart @@ -0,0 +1,230 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:site_shared/util.dart'; +import 'package:universal_web/web.dart' as web; + +import 'bench_formatters.dart'; +import 'benchmark_scores.dart'; +import 'model_detail_view.dart'; +import 'model_name_formatter.dart'; + +/// Two-pane browser for every evaluated model configuration. +/// +/// The left rail lists and filters configurations, and the right pane shows +/// the same [ModelDetailView] the leaderboard drawer uses. +@client +class ModelsExplorer extends StatefulComponent { + const ModelsExplorer({ + required this.evals, + this.benchmarks = const [], + super.key, + }); + + final List> evals; + + /// Per-task results for every eval. + final List> benchmarks; + + @override + State createState() => _ModelsExplorerState(); +} + +class _ModelsExplorerState extends State { + static const String _modelQueryParameter = 'model'; + + String _searchQuery = ''; + String _providerFilter = _allProviders; + String? _selectedEvalKey; + + static const String _allProviders = 'All companies'; + + late final List _benchmarks = benchmarkRowsFromMaps( + component.benchmarks, + ); + + @override + void initState() { + super.initState(); + if (kIsWeb) { + final url = Uri.parse(web.window.location.href); + _selectedEvalKey = url.queryParameters[_modelQueryParameter]; + } + // Falls back to the top-ranked configuration so the detail pane is never + // empty on first paint. + if (!_evalKeys.contains(_selectedEvalKey)) { + _selectedEvalKey = _sortedEvals.firstOrNull?['eval_key'] as String?; + } + } + + Set get _evalKeys => { + for (final eval in component.evals) eval['eval_key'] as String, + }; + + List> get _sortedEvals { + return [...component.evals]..sort((evalA, evalB) { + final rewardA = (evalA['mean_reward'] as num?)?.toDouble() ?? 0.0; + final rewardB = (evalB['mean_reward'] as num?)?.toDouble() ?? 0.0; + return rewardB.compareTo(rewardA); + }); + } + + List get _providers { + final providers = { + for (final eval in component.evals) + eval['provider'] as String? ?? 'Community', + }.toList()..sort(); + return [_allProviders, ...providers]; + } + + List> get _visibleEvals { + final query = _searchQuery.trim().toLowerCase(); + + return _sortedEvals.where((eval) { + if (_providerFilter != _allProviders && + (eval['provider'] as String? ?? 'Community') != _providerFilter) { + return false; + } + if (query.isEmpty) return true; + + final model = (eval['model_short_name'] as String? ?? '').toLowerCase(); + final agent = (eval['agent_name'] as String? ?? '').toLowerCase(); + return model.contains(query) || agent.contains(query); + }).toList(); + } + + void _select(String evalKey) { + setState(() => _selectedEvalKey = evalKey); + + if (!kIsWeb) return; + // Keeps the selection shareable and survivable across reloads. + final url = Uri.parse(web.window.location.href); + web.window.history.replaceState( + web.window.history.state, + '', + url.withQueryParameters({ + ...url.queryParameters, + _modelQueryParameter: evalKey, + }).toString(), + ); + } + + @override + Component build(BuildContext context) { + final visible = _visibleEvals; + final selected = component.evals.firstWhere( + (eval) => eval['eval_key'] == _selectedEvalKey, + orElse: () => const {}, + ); + + return div(classes: 'models-explorer', [ + _buildSidebar(visible), + div(classes: 'models-explorer__detail', [ + if (selected.isEmpty) + const p(classes: 'text-muted', [ + .text('Select a model configuration to see its full results.'), + ]) + else + ModelDetailView( + eval: selected, + evals: component.evals, + benchmarks: _benchmarks, + ), + ]), + ]); + } + + Component _buildSidebar(List> visible) { + return aside( + classes: 'models-explorer__sidebar', + attributes: const {'aria-label': 'Model configurations'}, + [ + div(classes: 'models-explorer__controls', [ + label(classes: 'models-explorer__field', [ + const span(classes: 'models-explorer__field-label', [ + .text('Company'), + ]), + select( + classes: 'models-explorer__select', + value: _providerFilter, + onChange: (values) { + setState( + () => _providerFilter = values.firstOrNull ?? _allProviders, + ); + }, + [ + for (final provider in _providers) + option(value: provider, [.text(provider)]), + ], + ), + ]), + label(classes: 'models-explorer__field', [ + const span(classes: 'models-explorer__field-label', [ + .text('Search'), + ]), + input( + type: InputType.search, + classes: 'bench-search-input', + value: _searchQuery, + attributes: const {'placeholder': 'Search models...'}, + onInput: (value) { + setState(() => _searchQuery = value?.toString() ?? ''); + }, + ), + ]), + ]), + const div(classes: 'models-explorer__list-header', [ + span([.text('Model')]), + span([.text('Score')]), + ]), + if (visible.isEmpty) + const p(classes: 'text-muted models-explorer__empty', [ + .text('No models match the current filters.'), + ]) + else + ul(classes: 'models-explorer__list', [ + for (final eval in visible) _buildListItem(eval), + ]), + ], + ); + } + + Component _buildListItem(Map eval) { + final evalKey = eval['eval_key'] as String; + final isSelected = evalKey == _selectedEvalKey; + final reward = (eval['mean_reward'] as num?)?.toDouble(); + + return li([ + button( + classes: [ + 'models-explorer__item', + if (isSelected) 'selected', + ].join(' '), + attributes: { + 'type': 'button', + 'aria-current': '$isSelected', + }, + onClick: () => _select(evalKey), + [ + span(classes: 'models-explorer__item-main', [ + span(classes: 'models-explorer__item-name', [ + .text(formatModelName(eval['model_short_name'] as String)), + ]), + span(classes: 'models-explorer__item-meta', [ + .text( + '${eval['agent_name']} · ' + '${eval['provider'] as String? ?? 'Community'}', + ), + ]), + ]), + span(classes: 'models-explorer__item-score', [ + .text(formatScore(reward)), + ]), + ], + ), + ]); + } +} diff --git a/sites/www/lib/src/components/flutterbench/story_chapter.dart b/sites/www/lib/src/components/flutterbench/story_chapter.dart new file mode 100644 index 00000000000..55b4134c149 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/story_chapter.dart @@ -0,0 +1,76 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +/// Renders one numbered "story chapter" section for the FlutterBench +/// methodology page, matching the chapter treatment used on the original +/// docs version of this page (numbered sticky headers with per-chapter +/// accent colors). +/// +/// When [children] is empty, no `.chapter-content` wrapper is rendered, +/// producing a heading-only chapter. +class StoryChapter extends StatelessComponent { + const StoryChapter({ + required this.number, + required this.title, + required this.anchorId, + this.children = const [], + super.key, + }); + + /// The two-digit chapter number, e.g. `'01'`. + final String number; + final String title; + final String anchorId; + final List children; + + @override + Component build(BuildContext context) { + return section( + classes: 'story-chapter', + attributes: {'data-chapter': number}, + [ + div(classes: 'chapter-header-group', [ + div(classes: 'chapter-kicker', [ + span(classes: 'chapter-number', [.text(number)]), + const span(classes: 'chapter-rule', []), + ]), + div(classes: 'header-wrapper', [ + h2(id: anchorId, [.text(title)]), + a(href: '#$anchorId', classes: 'heading-link', const [.text('#')]), + ]), + ]), + if (children.isNotEmpty) div(classes: 'chapter-content', children), + ], + ); + } +} + +/// Renders an `h3` with the same anchor-link treatment [StoryChapter] gives +/// its chapter headings, for sub-headings inside a chapter's content. +Component storyH3(String text, {String? id}) { + final headingId = id ?? _slugify(text); + return div(classes: 'header-wrapper', [ + h3(id: headingId, [.text(text)]), + a(href: '#$headingId', classes: 'heading-link', const [.text('#')]), + ]); +} + +/// Renders an `h4` with the same anchor-link treatment [StoryChapter] gives +/// its chapter headings, for sub-sub-headings inside a chapter's content. +Component storyH4(String text, {String? id}) { + final headingId = id ?? _slugify(text); + return div(classes: 'header-wrapper', [ + h4(id: headingId, [.text(text)]), + a(href: '#$headingId', classes: 'heading-link', const [.text('#')]), + ]); +} + +String _slugify(String text) => text + .toLowerCase() + .replaceAll(RegExp(r'[^a-z0-9\s-]'), '') + .trim() + .replaceAll(RegExp(r'\s+'), '-'); diff --git a/sites/www/lib/src/components/flutterbench/summary_stats_bar.dart b/sites/www/lib/src/components/flutterbench/summary_stats_bar.dart new file mode 100644 index 00000000000..db8a371525b --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/summary_stats_bar.dart @@ -0,0 +1,102 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../../models/content/flutterbench_content.dart'; +import 'error_state_badge.dart'; +import 'model_name_formatter.dart'; + +/// Row of summary metric cards displayed above the fold on the leaderboard. +class SummaryStatsBar extends StatelessComponent { + const SummaryStatsBar({required this.job, super.key}); + + final FlutterBenchJobData job; + + @override + Component build(BuildContext context) { + final topModelRewardFormatted = (job.topModelReward * 100).toStringAsFixed( + 0, + ); + final overallAvgFormatted = (job.overallAverageReward * 100) + .toStringAsFixed(0); + + return div(classes: 'bench-stats-bar-container', [ + div(classes: 'bench-stats-grid', [ + // Card 1: Top Model + div(classes: 'bench-stat-card card-primary', [ + const div(classes: 'stat-header', [ + span(classes: 'stat-label', [.text('Top Model')]), + span(classes: 'stat-badge badge-blue', [.text('Leader')]), + ]), + div(classes: 'stat-value', [ + .text(formatModelName(job.topModelName)), + ]), + div(classes: 'stat-meta', [ + span(classes: 'meta-highlight', [ + .text('$topModelRewardFormatted%'), + ]), + const span(classes: 'meta-description', [ + .text(' mean reward across benchmark'), + ]), + ]), + ]), + + // Card 2: Overall Average Reward + div(classes: 'bench-stat-card', [ + const div(classes: 'stat-header', [ + span(classes: 'stat-label', [.text('Benchmark Average')]), + span(classes: 'stat-badge badge-neutral', [.text('All Models')]), + ]), + div(classes: 'stat-value', [ + .text('$overallAvgFormatted%'), + ]), + const div(classes: 'stat-meta', [ + span(classes: 'meta-description', [ + .text('Mean composite reward across completed trials'), + ]), + ]), + ]), + + // Card 3: Trials Completed + div(classes: 'bench-stat-card', [ + const div(classes: 'stat-header', [ + span(classes: 'stat-label', [.text('Trials Run')]), + span(classes: 'stat-badge badge-green', [.text('Active Matrix')]), + ]), + div(classes: 'stat-value', [ + .text('${job.nTotalTrials}'), + ]), + div(classes: 'stat-meta', [ + span(classes: 'meta-description', [ + .text('${job.nCompletedTrials} evaluated in latest run'), + ]), + ]), + ]), + + // Card 4: Trials Errored + div(classes: 'bench-stat-card card-warning', [ + const div(classes: 'stat-header', [ + span(classes: 'stat-label', [.text('Execution Errors')]), + ]), + div(classes: 'stat-value error-value', [ + if (job.nErroredTrials > 0) + ErrorStateBadge( + exceptionType: '${job.nErroredTrials} Errored', + message: 'Errors are tracked separately and excluded from mean reward calculations', + ) + else + const span(classes: 'text-success', [.text('0')]), + ]), + const div(classes: 'stat-meta', [ + span(classes: 'meta-description', [ + .text('Excluded from average scores (not treated as 0)'), + ]), + ]), + ]), + ]), + ]); + } +} diff --git a/sites/www/lib/src/components/flutterbench/task_anatomy.dart b/sites/www/lib/src/components/flutterbench/task_anatomy.dart new file mode 100644 index 00000000000..a20ceb627f3 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/task_anatomy.dart @@ -0,0 +1,86 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:site_shared/components/common/ide_explorer/ide_explorer.dart'; +import 'package:site_shared/components/common/ide_explorer/models.dart'; + +import '../../models/content/flutterbench_content.dart'; +import 'methodology_components.dart'; + +/// Maps [FlutterBenchTaskTreeNode] data into the [IdeTreeNode] tree consumed +/// by [IdeExplorer]'s sidebar. +List buildIdeTreeNodes(List nodes) { + return [for (final node in nodes) _buildIdeTreeNode(node)]; +} + +IdeTreeNode _buildIdeTreeNode(FlutterBenchTaskTreeNode node) { + final badgeColor = node.badgeColor == null + ? null + : IdeBadgeColor.fromString(node.badgeColor!); + + if (node.type == 'folder') { + return IdeFolderNode( + id: node.id, + label: node.label, + subtitle: node.subtitle, + badge: node.badge, + badgeColor: badgeColor, + isDefaultPage: node.isDefaultPage, + startsClosed: node.startsClosed, + children: buildIdeTreeNodes(node.children), + ); + } + + return IdeFileNode( + id: node.id, + label: node.label, + subtitle: node.subtitle, + badge: node.badge, + badgeColor: badgeColor, + isDefaultPage: node.isDefaultPage, + ); +} + +/// Flattens [FlutterBenchTaskTreeNode] data (including nested children) into +/// a map of node id to the detail-pane [Component] [IdeExplorer] should show +/// for it, built from each node's prose `body` and optional `code` sample. +Map buildIdeCustomContents( + List nodes, +) { + final contents = {}; + for (final node in nodes) { + contents[node.id] = _buildNodeDetail(node); + contents.addAll(buildIdeCustomContents(node.children)); + } + return contents; +} + +Component _buildNodeDetail(FlutterBenchTaskTreeNode node) { + final children = []; + + final body = node.body; + if (body != null && body.isNotEmpty) { + for (final paragraph in body.split('\n\n')) { + if (paragraph.trim().isEmpty) continue; + children.add(p([renderDescriptionWithCode(paragraph)])); + } + } + + final codeSample = node.code; + if (codeSample != null) { + children.add( + div(classes: 'code-block-wrapper', [ + pre([ + code(classes: 'language-${codeSample.lang}', [ + .text(codeSample.text), + ]), + ]), + ]), + ); + } + + return .fragment(children); +} diff --git a/sites/www/lib/src/components/flutterbench/task_model_heatmap.dart b/sites/www/lib/src/components/flutterbench/task_model_heatmap.dart new file mode 100644 index 00000000000..576b0ec6e06 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/task_model_heatmap.dart @@ -0,0 +1,204 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../../models/content/flutterbench_content.dart'; +import 'error_state_badge.dart'; +import 'model_name_formatter.dart'; + +/// Heatmap matrix component mapping Tasks (rows) vs Models/Configurations (columns). +/// +/// Cells are color-coded based on normalized reward (0.0 to 1.0). +/// Errored trials are displayed with a distinct hatched/gray treatment and never as 0. +class TaskModelHeatmap extends StatelessComponent { + const TaskModelHeatmap({ + required this.tasks, + required this.evals, + super.key, + }); + + final List tasks; + final List evals; + + @override + Component build(BuildContext context) { + return div(classes: 'bench-heatmap-container', [ + div(classes: 'bench-table-wrapper heatmap-scroll', [ + table(classes: 'bench-heatmap-table', [ + thead([ + tr([ + const th(classes: 'col-task-header', [ + .text('Critical User Journey (CUJ)'), + ]), + const th(classes: 'col-category-header', [.text('Category')]), + for (final eval in evals) + th(classes: 'col-model-header', [ + div(classes: 'model-header-content', [ + span(classes: 'model-name-title', [ + .text(formatModelName(eval.modelShortName)), + ]), + span(classes: 'agent-tag', [.text(eval.agentName)]), + if (eval.hasDartTooling) + const span(classes: 'tooling-icon-badge', [ + span(classes: 'tooling-dot', []), + .text('Tools'), + ]), + ]), + ]), + ]), + ]), + tbody([ + for (final task in tasks) + tr(classes: 'heatmap-task-row', [ + td(classes: 'col-task-name', [ + a( + href: '/ai/flutterbench/tasks/${task.slug}', + classes: 'task-title-link', + [.text(task.displayName)], + ), + ]), + td(classes: 'col-task-category', [ + span(classes: 'category-pill', [.text(task.category)]), + ]), + for (final eval in evals) _buildCell(task, eval), + ]), + ]), + ]), + ]), + + // Below the grid: Best & Worst CUJs summary per model + div(classes: 'bench-cuj-model-summaries', [ + const h3(classes: 'summaries-title', [ + .text('Model Strengths & Weaknesses Across CUJs'), + ]), + div(classes: 'summaries-grid', [ + for (final eval in evals) + div(classes: 'model-summary-card', [ + div(classes: 'card-header', [ + h4( + classes: 'model-title', + [.text(formatModelName(eval.modelShortName))], + ), + span(classes: 'agent-subtitle', [.text(eval.agentName)]), + ]), + div(classes: 'card-body', [ + div(classes: 'cuj-list-group', [ + const span(classes: 'group-label text-success', [ + .text('Strongest CUJs:'), + ]), + if (eval.bestCujs.isEmpty) + const p(classes: 'text-muted sm', [ + .text('No high-scoring tasks.'), + ]) + else + ul(classes: 'cuj-bullet-list', [ + for (final cuj in eval.bestCujs) + li([ + a(href: '/ai/flutterbench/tasks/${cuj.taskSlug}', [ + .text(cuj.taskName), + ]), + span(classes: 'cuj-score score-high', [ + .text(' (${(cuj.reward ?? 0).toStringAsFixed(2)})'), + ]), + ]), + ]), + ]), + div(classes: 'cuj-list-group', [ + const span(classes: 'group-label text-danger', [ + .text('Growth Areas / Weaknesses:'), + ]), + if (eval.worstCujs.isEmpty) + const p(classes: 'text-muted sm', [.text('None recorded.')]) + else + ul(classes: 'cuj-bullet-list', [ + for (final cuj in eval.worstCujs) + li([ + a(href: '/ai/flutterbench/tasks/${cuj.taskSlug}', [ + .text(cuj.taskName), + ]), + if (cuj.status == 'error') + const span(classes: 'cuj-score score-error', [ + .text(' (Error)'), + ]) + else + span(classes: 'cuj-score score-low', [ + .text( + ' (${(cuj.reward ?? 0).toStringAsFixed(2)})', + ), + ]), + ]), + ]), + ]), + ]), + ]), + ]), + ]), + ]); + } + + Component _buildCell(FlutterBenchTaskItem task, FlutterBenchEvalItem eval) { + final scoreEntry = task.scoresByEval[eval.evalKey] as Map?; + + if (scoreEntry == null) { + return const td(classes: 'heatmap-cell cell-empty', [ + span(classes: 'text-muted', [.text('—')]), + ]); + } + + final trialName = scoreEntry['trial_name'] as String? ?? ''; + final status = scoreEntry['status'] as String? ?? ''; + final reward = (scoreEntry['reward'] as num?)?.toDouble(); + final exceptionType = scoreEntry['exception_type'] as String?; + + if (status == 'error' || exceptionType != null) { + return td( + classes: 'heatmap-cell cell-error', + attributes: { + 'title': + 'Error during trial: ${exceptionType ?? 'Exception'} (Excluded from averages)', + }, + [ + a( + href: '/ai/flutterbench/trials/$trialName', + classes: 'cell-link error-link', + [ + ErrorStateBadge( + exceptionType: exceptionType ?? 'Error', + compact: true, + ), + ], + ), + ], + ); + } + + final score = reward ?? 0.0; + final colorClass = score >= 0.80 + ? 'score-high' + : score >= 0.50 + ? 'score-mid' + : 'score-low'; + + return td( + classes: 'heatmap-cell cell-scored $colorClass', + attributes: { + 'title': + 'Reward: ${score.toStringAsFixed(2)} for ${task.displayName} (${formatModelName(eval.modelShortName)})', + }, + [ + a( + href: '/ai/flutterbench/trials/$trialName', + classes: 'cell-link', + [ + span(classes: 'cell-score-value', [ + .text(score.toStringAsFixed(2)), + ]), + ], + ), + ], + ); + } +} diff --git a/sites/www/lib/src/components/flutterbench/task_specifications.dart b/sites/www/lib/src/components/flutterbench/task_specifications.dart new file mode 100644 index 00000000000..82ccb0e3927 --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/task_specifications.dart @@ -0,0 +1,143 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:site_shared/components/common/material_icon.dart'; +import 'package:site_shared/util.dart'; + +import 'methodology_components.dart'; + +/// Collapsible specification panels with tables. +@client +class TaskSpecifications extends StatefulComponent { + const TaskSpecifications({required this.specs, super.key}); + + final List> specs; + + @override + State createState() => _TaskSpecificationsState(); +} + +class _TaskSpecificationsState extends State { + late final Set _expandedPanels = { + for (final s in component.specs) + if (s['expanded'] == true) s['id'] as String? ?? '', + }; + + void _toggle(String id) { + setState(() { + if (_expandedPanels.contains(id)) { + _expandedPanels.remove(id); + } else { + _expandedPanels.add(id); + } + }); + } + + @override + Component build(BuildContext context) { + return div(classes: 'task-specs-list', [ + for (final spec in component.specs) ...[ + () { + final id = spec['id'] as String? ?? ''; + final isExpanded = _expandedPanels.contains(id); + final badgeVariant = spec['badge_variant'] as String? ?? 'blue'; + final tables = (spec['tables'] as List? ?? const []) + .whereType>() + .toList(); + + return div(classes: 'task-spec-panel', [ + a( + classes: [ + 'collapsible', + if (!isExpanded) 'collapsed', + ].toClasses, + href: '#task-spec-$id', + events: { + 'click': (e) { + e.preventDefault(); + _toggle(id); + }, + }, + [ + div(classes: 'panel-header-left', [ + div( + classes: [ + 'panel-icon-wrap', + 'variant-$badgeVariant', + ].toClasses, + [MaterialIcon(spec['icon'] as String? ?? 'info')], + ), + div(classes: 'panel-header-content', [ + div(classes: 'panel-title-row', [ + h4([.text(spec['title'] as String? ?? '')]), + if (spec['badge'] case final String badge) + if (badge.isNotEmpty) + span( + classes: 'badge badge-$badgeVariant', + [.text(badge)], + ), + ]), + if (spec['description'] case final String desc) + p(classes: 'panel-description', [ + renderDescriptionWithCode(desc), + ]), + ]), + ]), + ], + ), + div( + classes: [ + 'task-spec-body', + if (isExpanded) 'show', + ].toClasses, + [ + if (spec['lead_text'] case final String lead) + p(classes: 'lead-text', [renderDescriptionWithCode(lead)]), + for (final tbl in tables) ...[ + if (tbl['title'] case final String tTitle) + if (tTitle.isNotEmpty) + div(classes: 'table-subheading', [.text(tTitle)]), + div(classes: 'table-wrapper', [ + table(classes: 'spec-table', [ + if (tbl['headers'] case final List headers) + if (headers.isNotEmpty) + thead([ + tr([ + for (final h in headers) + th([.text(h.toString())]), + ]), + ]), + tbody([ + for (final row + in (tbl['rows'] as List? ?? const []) + .whereType>()) + tr([ + td([ + strong([.text(row['label'] as String? ?? '')]), + ]), + td([ + renderDescriptionWithCode( + row['description'] as String? ?? '', + ), + ]), + ]), + ]), + ]), + ]), + ], + if (spec['footer_text'] case final String footer) + if (footer.isNotEmpty) + p(classes: 'footer-text', [ + renderDescriptionWithCode(footer), + ]), + ], + ), + ]); + }(), + ], + ]); + } +} diff --git a/sites/www/lib/src/components/flutterbench/trial_detail_view.dart b/sites/www/lib/src/components/flutterbench/trial_detail_view.dart new file mode 100644 index 00000000000..1e6d59a605f --- /dev/null +++ b/sites/www/lib/src/components/flutterbench/trial_detail_view.dart @@ -0,0 +1,531 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../../models/content/flutterbench_content.dart'; +import 'error_state_badge.dart'; +import 'model_name_formatter.dart'; + +/// Comprehensive detail view for a single FlutterBench trial. +class TrialDetailView extends StatelessComponent { + const TrialDetailView({required this.trial, super.key}); + + final FlutterBenchTrialDetail trial; + + @override + Component build(BuildContext context) { + return div(classes: 'bench-trial-detail-container', [ + // Navigation breadcrumbs + div(classes: 'bench-breadcrumbs', [ + const a(href: '/ai', [.text('AI')]), + const span(classes: 'breadcrumb-sep', [.text(' / ')]), + const a(href: '/ai/flutterbench', [.text('FlutterBench')]), + const span(classes: 'breadcrumb-sep', [.text(' / ')]), + const a(href: '/ai/flutterbench/tasks', [.text('Tasks')]), + const span(classes: 'breadcrumb-sep', [.text(' / ')]), + a(href: '/ai/flutterbench/tasks/${trial.taskSlug}', [ + .text(trial.taskSlug), + ]), + const span(classes: 'breadcrumb-sep', [.text(' / ')]), + span(classes: 'current-page', [.text(trial.trialName)]), + ]), + + // 1. Summary Hero Section + _buildSummarySection(), + + // 2. Reward Breakdown Tree + _buildRewardBreakdownSection(), + + // 3. Trajectory Section (if present) + if (trial.trajectory != null && trial.trajectory!.isNotEmpty) + _buildTrajectorySection(), + + // 4. Artifacts Code Viewer (if present) + if (trial.artifacts.isNotEmpty) _buildArtifactsSection(), + + // 5. Raw Logs Section + _buildLogsSection(), + ]); + } + + Component _buildSummarySection() { + final isError = trial.status == 'error'; + final reward = trial.reward; + + return section(classes: 'bench-card trial-summary-card', [ + div(classes: 'summary-top-row', [ + div(classes: 'task-title-group', [ + const span(classes: 'sub-tag', [.text('Trial Evaluation')]), + h1(classes: 'trial-title', [.text(trial.trialName)]), + div(classes: 'task-link-row', [ + const span(classes: 'task-label', [.text('Task: ')]), + a( + href: '/ai/flutterbench/tasks/${trial.taskSlug}', + classes: 'task-anchor', + [.text(trial.taskName)], + ), + ]), + ]), + div(classes: 'status-badge-container', [ + if (isError) + ErrorStateBadge( + exceptionType: trial.exceptionType ?? 'Trial Error', + message: trial.exceptionMessage, + ) + else ...[ + span( + classes: [ + 'trial-status-badge', + if (trial.status == 'pass') + 'status-pass' + else if (trial.status == 'partial') + 'status-partial' + else + 'status-fail', + ].join(' '), + [.text(trial.status.toUpperCase())], + ), + if (reward != null) + div(classes: 'hero-score-badge', [ + span(classes: 'score-num', [ + .text((reward * 100).toStringAsFixed(0)), + ]), + const span(classes: 'score-pct', [.text('%')]), + const span(classes: 'score-caption', [ + .text('Composite Reward'), + ]), + ]), + ], + ]), + ]), + + const div(classes: 'meta-divider', []), + + // Metadata grid + div(classes: 'trial-meta-grid', [ + div(classes: 'meta-col', [ + const span(classes: 'meta-label', [.text('Model')]), + span( + classes: 'meta-val bold', + [.text(formatModelName(trial.modelShortName))], + ), + span(classes: 'meta-sub', [.text(trial.provider)]), + ]), + div(classes: 'meta-col', [ + const span(classes: 'meta-label', [.text('Agent Harness')]), + span(classes: 'meta-val', [.text(trial.agentName)]), + if (trial.hasDartTooling) + const span(classes: 'tooling-pill', [.text('Dart MCP + Skills')]) + else + const span(classes: 'tooling-pill uninstrumented', [ + .text('Standard Baseline'), + ]), + ]), + div(classes: 'meta-col', [ + const span(classes: 'meta-label', [.text('Tokens')]), + span(classes: 'meta-val', [ + .text(_formatTokens(trial.inputTokens + trial.outputTokens)), + ]), + span(classes: 'meta-sub', [ + .text( + '${_formatTokens(trial.inputTokens)} in / ${_formatTokens(trial.outputTokens)} out', + ), + ]), + ]), + div(classes: 'meta-col', [ + const span(classes: 'meta-label', [.text('Cost')]), + span(classes: 'meta-val', [ + .text( + trial.costUsd > 0 ? '\$${trial.costUsd.toStringAsFixed(4)}' : '—', + ), + ]), + const span(classes: 'meta-sub', [.text('USD estimated')]), + ]), + ]), + + // Phase durations timeline + if (trial.durations.isNotEmpty) ...[ + const div(classes: 'meta-divider', []), + div(classes: 'phase-durations-section', [ + const span(classes: 'phase-title', [ + .text('Execution Phase Durations'), + ]), + div(classes: 'phase-bars-row', [ + _buildPhaseBar( + 'Environment Setup', + trial.durations['environment_setup'], + ), + _buildPhaseBar('Agent Setup', trial.durations['agent_setup']), + _buildPhaseBar( + 'Agent Execution', + trial.durations['agent_execution'], + ), + _buildPhaseBar('Verifier', trial.durations['verifier']), + ]), + ]), + ], + ]); + } + + Component _buildPhaseBar(String label, double? seconds) { + if (seconds == null) return const div([]); + final formattedSec = seconds >= 60 + ? '${(seconds / 60).toStringAsFixed(1)}m' + : '${seconds.toStringAsFixed(1)}s'; + + return div(classes: 'phase-pill', [ + span(classes: 'phase-label', [.text(label)]), + span(classes: 'phase-time', [.text(formattedSec)]), + ]); + } + + Component _buildRewardBreakdownSection() { + final rewardTree = trial.rewardTree; + if (rewardTree == null) { + if (trial.status == 'error') { + return section(classes: 'bench-card error-card-section', [ + const h2(classes: 'section-heading', [.text('Execution Failure')]), + div(classes: 'error-callout', [ + const span(classes: 'callout-icon', [.text('⚠')]), + div(classes: 'callout-body', [ + h3([.text(trial.exceptionType ?? 'Trial Error')]), + p([ + .text( + trial.exceptionMessage ?? + 'The agent execution failed before verifier completion.', + ), + ]), + if (trial.exceptionTraceback != null) + pre(classes: 'traceback-pre', [ + code([.text(trial.exceptionTraceback!)]), + ]), + ]), + ]), + ]); + } + return const div([]); + } + + // Top-level reward aggregator + final rewardNode = rewardTree['reward'] as Map?; + final criteria = + (rewardNode?['criteria'] as List?) + ?.whereType>() + .toList() ?? + []; + + return section(classes: 'bench-card reward-breakdown-section', [ + const div(classes: 'section-header-row', [ + h2(classes: 'section-heading', [.text('Scoring Rubric Breakdown')]), + span(classes: 'rubric-formula-badge', [ + .text('Composite Reward = 0.60×Outcome + 0.30×Quality + 0.10×DX'), + ]), + ]), + const p(classes: 'section-intro-text', [ + .text( + 'Detailed criteria evaluation produced by automated test graders and LLM rubrics:', + ), + ]), + + // Criteria Trees + div(classes: 'criteria-tree', [ + for (final criterion in criteria) + _buildScoredCriterionCard(criterion, rewardTree), + ]), + + // Diagnostic Section (Process & Efficiency) + if (trial.diagnosticTree.isNotEmpty) + div(classes: 'diagnostic-breakdown-panel', [ + const div(classes: 'diagnostic-banner', [ + span(classes: 'diag-icon', [.text('ℹ')]), + div(classes: 'diag-banner-text', [ + h3([ + .text('Diagnostic Telemetry (Not Scored in Primary Reward)'), + ]), + p([ + .text( + 'These metrics measure execution velocity, plan adherence, and token efficiency. ' + 'They provide operational observability and are strictly separated from composite reward scores.', + ), + ]), + ]), + ]), + div(classes: 'diagnostic-grids', [ + for (final entry in trial.diagnosticTree.entries) + _buildDiagnosticCard( + entry.key, + entry.value as Map, + ), + ]), + ]), + ]); + } + + Component _buildScoredCriterionCard( + Map criterion, + Map fullTree, + ) { + final name = criterion['name'] as String? ?? 'criterion'; + final value = (criterion['value'] as num?)?.toDouble() ?? 0.0; + final weight = (criterion['weight'] as num?)?.toDouble() ?? 0.0; + final description = criterion['description'] as String? ?? ''; + + // Check for expanded node in fullTree + final subNode = fullTree[name] as Map?; + final subCriteria = + (subNode?['criteria'] as List?) + ?.whereType>() + .toList() ?? + []; + final kind = subNode?['kind'] as String?; + + return details( + classes: 'criterion-accordion', + attributes: const {'open': 'true'}, + [ + summary(classes: 'criterion-summary', [ + div(classes: 'summary-left', [ + span(classes: 'criterion-name', [.text(name.toUpperCase())]), + span(classes: 'weight-pill', [ + .text('Weight ${(weight * 100).toStringAsFixed(0)}%'), + ]), + ]), + div(classes: 'summary-right', [ + span( + classes: [ + 'score-pill', + if (value >= 0.8) + 'score-high' + else if (value >= 0.5) + 'score-mid' + else + 'score-low', + ].join(' '), + [.text('${(value * 100).toStringAsFixed(0)}%')], + ), + ]), + ]), + div(classes: 'criterion-body', [ + if (description.isNotEmpty && subCriteria.isEmpty) + pre(classes: 'description-box', [.text(description)]), + if (subCriteria.isNotEmpty) + div(classes: 'sub-criteria-list', [ + for (final sub in subCriteria) + _buildSubCriterionItem(sub, isLlmJudge: kind == 'llm'), + ]), + ]), + ], + ); + } + + Component _buildSubCriterionItem( + Map item, { + required bool isLlmJudge, + }) { + final name = item['name'] as String? ?? ''; + final value = (item['value'] as num?)?.toDouble() ?? 0.0; + final weight = (item['weight'] as num?)?.toDouble(); + final description = item['description'] as String? ?? ''; + final reasoning = item['reasoning'] as String?; + + return div(classes: 'sub-criterion-row', [ + div(classes: 'sub-header-line', [ + span(classes: 'sub-name', [.text(name)]), + if (weight != null) + span(classes: 'sub-weight', [ + .text('wt: ${(weight * 100).toStringAsFixed(0)}%'), + ]), + span( + classes: [ + 'sub-score-badge', + if (value >= 0.8) + 'score-high' + else if (value >= 0.5) + 'score-mid' + else + 'score-low', + ].join(' '), + [.text(value.toStringAsFixed(2))], + ), + ]), + if (description.isNotEmpty) + p(classes: 'sub-description', [.text(description)]), + if (reasoning != null && reasoning.isNotEmpty) + div(classes: 'llm-reasoning-card', [ + const div(classes: 'reasoning-header', [ + span(classes: 'reasoning-badge', [.text('LLM Judge Evaluation')]), + ]), + p(classes: 'reasoning-text', [.text(reasoning)]), + ]), + ]); + } + + Component _buildDiagnosticCard(String title, Map node) { + final score = (node['score'] as num?)?.toDouble() ?? 0.0; + final criteria = + (node['criteria'] as List?) + ?.whereType>() + .toList() ?? + []; + + return div(classes: 'diagnostic-card', [ + div(classes: 'diag-card-header', [ + h4(classes: 'diag-title', [.text(title.toUpperCase())]), + span(classes: 'diag-score', [ + .text('${(score * 100).toStringAsFixed(0)}%'), + ]), + ]), + ul(classes: 'diag-criteria-list', [ + for (final item in criteria) + li([ + div(classes: 'diag-item-row', [ + span(classes: 'diag-item-name', [ + .text(item['name'] as String? ?? ''), + ]), + span(classes: 'diag-item-val', [ + .text( + ((item['value'] as num?)?.toDouble() ?? 0.0).toStringAsFixed( + 2, + ), + ), + ]), + ]), + if (item['description'] != null) + p(classes: 'diag-item-desc', [ + .text(item['description'] as String), + ]), + ]), + ]), + ]); + } + + Component _buildTrajectorySection() { + final steps = trial.trajectory!; + + return section(classes: 'bench-card trajectory-section', [ + const h2(classes: 'section-heading', [ + .text('Agent Execution Trajectory'), + ]), + const p(classes: 'section-intro-text', [ + .text( + 'Sequential step timeline captured during autonomous agent execution:', + ), + ]), + ol(classes: 'trajectory-timeline', [ + for (var i = 0; i < steps.length; i++) + _buildTrajectoryStep(steps[i], stepIndex: i + 1), + ]), + ]); + } + + Component _buildTrajectoryStep( + Map step, { + required int stepIndex, + }) { + final action = step['action'] as String? ?? 'step'; + final input = step['input'] as String? ?? ''; + final durationMs = (step['duration_ms'] as num?)?.toInt() ?? 0; + + return li(classes: 'trajectory-step-item', [ + div(classes: 'step-marker', [.text('$stepIndex')]), + div(classes: 'step-content', [ + div(classes: 'step-top-line', [ + span(classes: 'action-badge action-$action', [.text(action)]), + span(classes: 'duration-pill', [.text('${durationMs}ms')]), + ]), + if (input.isNotEmpty) code(classes: 'step-input-code', [.text(input)]), + ]), + ]); + } + + Component _buildArtifactsSection() { + return section(classes: 'bench-card artifacts-section', [ + const h2(classes: 'section-heading', [.text('Generated Code Artifacts')]), + const p(classes: 'section-intro-text', [ + .text('Files modified or generated by the agent during this trial:'), + ]), + div(classes: 'artifacts-list', [ + for (final artifact in trial.artifacts) _buildArtifactCard(artifact), + ]), + ]); + } + + Component _buildArtifactCard(Map artifact) { + final dest = + artifact['destination'] as String? ?? + artifact['source'] as String? ?? + 'file'; + final content = artifact['content'] as String?; + + return details( + classes: 'artifact-card', + attributes: const {'open': 'true'}, + [ + summary(classes: 'artifact-header', [ + const span(classes: 'file-path-icon', [.text('📄')]), + span(classes: 'artifact-path', [.text(dest)]), + const span(classes: 'artifact-status-pill', [.text('Generated')]), + ]), + div(classes: 'artifact-content', [ + if (content != null && content.isNotEmpty) + pre(classes: 'code-viewer', [ + code([.text(content)]), + ]) + else + const p(classes: 'text-muted p-3', [ + .text('(Empty or binary file)'), + ]), + ]), + ], + ); + } + + Component _buildLogsSection() { + final stdout = trial.testStdout; + final excLog = trial.exceptionLog; + + if (stdout == null && excLog == null) { + return const div([]); + } + + return section(classes: 'bench-card logs-section', [ + const h2(classes: 'section-heading', [ + .text('Execution & Verifier Logs'), + ]), + if (excLog != null && excLog.isNotEmpty) + details( + classes: 'log-details error-log', + attributes: const {'open': 'true'}, + [ + const summary(classes: 'log-summary', [ + span(classes: 'text-danger bold', [ + .text('Exception Traceback (exception.txt)'), + ]), + ]), + pre(classes: 'raw-log-pre', [ + code([.text(excLog)]), + ]), + ], + ), + if (stdout != null && stdout.isNotEmpty) + details(classes: 'log-details', [ + const summary(classes: 'log-summary', [ + .text('Verifier Standard Output (test-stdout.txt)'), + ]), + pre(classes: 'raw-log-pre', [ + code([.text(stdout)]), + ]), + ]), + ]); + } + + String _formatTokens(int count) { + if (count <= 0) return '0'; + if (count >= 1000000) return '${(count / 1000000).toStringAsFixed(1)}M'; + if (count >= 1000) return '${(count / 1000).toStringAsFixed(1)}k'; + return count.toString(); + } +} diff --git a/sites/www/lib/src/data/nav_items.dart b/sites/www/lib/src/data/nav_items.dart index 4c737b6b1d2..ec7a08563b3 100644 --- a/sites/www/lib/src/data/nav_items.dart +++ b/sites/www/lib/src/data/nav_items.dart @@ -18,6 +18,7 @@ final List headerNavItems = [ ], secondColumn: [ NavLink(label: 'AI', href: '/ai'), + NavLink(label: 'FlutterBench', href: '/ai/flutterbench'), NavLink(label: 'Google integrations', href: '/google-integrations'), NavLink(label: 'Game development', href: '/games'), NavLink(label: 'Monetization', href: '/monetization'), diff --git a/sites/www/lib/src/data/raw_flutterbench_data/config.json b/sites/www/lib/src/data/raw_flutterbench_data/config.json new file mode 100644 index 00000000000..b522ba510c6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/config.json @@ -0,0 +1,146 @@ +{ + "job_name": "2026-09-09__12-00-00_mock", + "jobs_dir": "jobs", + "n_attempts": 1, + "install_only": false, + "timeout_multiplier": 1.0, + "n_concurrent_trials": 4, + "quiet": false, + "environment": { + "type": "docker", + "delete": true + }, + "agents": [ + { + "name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + }, + { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "skills": [], + "mcp_servers": [] + }, + { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "skills": [], + "mcp_servers": [] + }, + { + "name": "codex-agent", + "model_name": "openai/o3", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + }, + { + "name": "codex-agent", + "model_name": "openai/gpt-5", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + }, + { + "name": "codex-agent", + "model_name": "openai/gpt-4o", + "skills": [], + "mcp_servers": [] + }, + { + "name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "skills": [], + "mcp_servers": [] + }, + { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + }, + { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + }, + { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "skills": [], + "mcp_servers": [] + }, + { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + }, + { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "skills": [], + "mcp_servers": [] + }, + { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "skills": [], + "mcp_servers": [] + } + ], + "tasks": [ + { + "path": "dataset/dart-build-cli-app" + }, + { + "path": "dataset/flutter-manage-state-with-bloc" + }, + { + "path": "dataset/flutter-offline-sync-sqlite" + }, + { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + { + "path": "dataset/flutter-custom-render-object" + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..d24232c7e40 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..b366fff5bee --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-7-sonnet +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..b366fff5bee --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-7-sonnet +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..b366fff5bee --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-7-sonnet +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..b366fff5bee --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-7-sonnet +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/result.json new file mode 100644 index 00000000000..902a12be012 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000001", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t01-claude-3-7-sonnet", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t01-claude-3-7-sonnet", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t01-claude-3-7-sonnet", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-7-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 93916, + "n_cache_tokens": 70367, + "n_output_tokens": 4594, + "cost_usd": 0.3507 + }, + "verifier_result": { + "rewards": { + "reward": 0.97 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:01:25.000000Z", + "finished_at": "2026-09-09T19:03:15.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:01:25.000000Z", + "finished_at": "2026-09-09T19:01:35.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:01:35.000000Z", + "finished_at": "2026-09-09T19:01:45.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:01:45.000000Z", + "finished_at": "2026-09-09T19:02:52.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:02:52.000000Z", + "finished_at": "2026-09-09T19:03:15.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..da04114132d --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 1.0, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (100%)." + }, + { + "name": "quality", + "value": 0.94, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (94%)." + }, + { + "name": "dx", + "value": 0.93, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (93%)." + } + ] + }, + "outcome": { + "score": 1.0, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 1.0, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.94, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.94, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.93, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..f09e7245908 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t01-claude-3-7-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Build Command-Line CLI App unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.97) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..7d28b0bf345 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..ce6c71fe251 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-5-sonnet +// Verification score: 0.79 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..ce6c71fe251 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-5-sonnet +// Verification score: 0.79 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..ce6c71fe251 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-5-sonnet +// Verification score: 0.79 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..ce6c71fe251 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-5-sonnet +// Verification score: 0.79 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/result.json new file mode 100644 index 00000000000..d42f4ae3a60 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000006", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t06-claude-3-5-sonnet", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t06-claude-3-5-sonnet", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t06-claude-3-5-sonnet", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 76557, + "n_cache_tokens": 57509, + "n_output_tokens": 3548, + "cost_usd": 0.2829 + }, + "verifier_result": { + "rewards": { + "reward": 0.79 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:08:30.000000Z", + "finished_at": "2026-09-09T19:10:40.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:08:30.000000Z", + "finished_at": "2026-09-09T19:08:40.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:08:40.000000Z", + "finished_at": "2026-09-09T19:08:50.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:08:50.000000Z", + "finished_at": "2026-09-09T19:10:16.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:10:16.000000Z", + "finished_at": "2026-09-09T19:10:40.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..13a2832b83f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.79, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.85, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (85%)." + }, + { + "name": "quality", + "value": 0.73, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (73%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.85, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.73, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.73, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.73, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..7d388675093 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t06-claude-3-5-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Build Command-Line CLI App tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.79) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/agent/trajectory.json new file mode 100644 index 00000000000..7d28b0bf345 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..f1078812ac3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-5-haiku +// Verification score: 0.56 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..f1078812ac3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-5-haiku +// Verification score: 0.56 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..f1078812ac3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-5-haiku +// Verification score: 0.56 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..f1078812ac3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: claude-3-5-haiku +// Verification score: 0.56 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/result.json new file mode 100644 index 00000000000..24ff27b4d36 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000000b", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t11-claude-3-5-haiku", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t11-claude-3-5-haiku", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t11-claude-3-5-haiku", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-haiku", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 61382, + "n_cache_tokens": 46705, + "n_output_tokens": 2347, + "cost_usd": 0.0585 + }, + "verifier_result": { + "rewards": { + "reward": 0.56 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:15:35.000000Z", + "finished_at": "2026-09-09T19:18:20.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:15:35.000000Z", + "finished_at": "2026-09-09T19:15:45.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:15:45.000000Z", + "finished_at": "2026-09-09T19:15:55.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:15:55.000000Z", + "finished_at": "2026-09-09T19:17:55.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:17:55.000000Z", + "finished_at": "2026-09-09T19:18:20.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/verifier/reward-details.json new file mode 100644 index 00000000000..196b09ed712 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.58, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (58%)." + }, + { + "name": "quality", + "value": 0.49, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (49%)." + }, + { + "name": "dx", + "value": 0.63, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (63%)." + } + ] + }, + "outcome": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.58, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.49, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.49, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44099999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/verifier/test-stdout.txt new file mode 100644 index 00000000000..7bb87ec2e79 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t11-claude-3-5-haiku/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Build Command-Line CLI App tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.56) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/agent/trajectory.json new file mode 100644 index 00000000000..d24232c7e40 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..2562c298c57 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: o3 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..2562c298c57 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: o3 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..2562c298c57 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: o3 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..2562c298c57 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: o3 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/result.json new file mode 100644 index 00000000000..fa636a81b66 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000010", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t16-o3", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t16-o3", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t16-o3", + "eval_key": "codex-agent__o3__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/o3", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "o3", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 121468, + "n_cache_tokens": 82862, + "n_output_tokens": 6073, + "cost_usd": 0.7288 + }, + "verifier_result": { + "rewards": { + "reward": 0.97 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:22:40.000000Z", + "finished_at": "2026-09-09T19:25:03.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:22:40.000000Z", + "finished_at": "2026-09-09T19:22:50.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:22:50.000000Z", + "finished_at": "2026-09-09T19:23:00.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:23:00.000000Z", + "finished_at": "2026-09-09T19:24:31.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:24:31.000000Z", + "finished_at": "2026-09-09T19:25:03.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/verifier/reward-details.json new file mode 100644 index 00000000000..a868718365f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.97, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (97%)." + }, + { + "name": "quality", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (96%)." + }, + { + "name": "dx", + "value": 0.96, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (96%)." + } + ] + }, + "outcome": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.97, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.96, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.96, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.96, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/verifier/test-stdout.txt new file mode 100644 index 00000000000..f09e7245908 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t16-o3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Build Command-Line CLI App unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.97) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/agent/trajectory.json new file mode 100644 index 00000000000..d24232c7e40 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..90a0f3664cf --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-5 +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..90a0f3664cf --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-5 +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..90a0f3664cf --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-5 +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..90a0f3664cf --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-5 +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/result.json new file mode 100644 index 00000000000..2b90c496a21 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000015", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t21-gpt-5", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t21-gpt-5", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t21-gpt-5", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-5", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-5", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 94854, + "n_cache_tokens": 68917, + "n_output_tokens": 4419, + "cost_usd": 0.2813 + }, + "verifier_result": { + "rewards": { + "reward": 0.98 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:29:45.000000Z", + "finished_at": "2026-09-09T19:32:49.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:29:45.000000Z", + "finished_at": "2026-09-09T19:29:55.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:29:55.000000Z", + "finished_at": "2026-09-09T19:30:05.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:30:05.000000Z", + "finished_at": "2026-09-09T19:32:17.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:32:17.000000Z", + "finished_at": "2026-09-09T19:32:49.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/verifier/reward-details.json new file mode 100644 index 00000000000..dbd1e27b7d7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.99, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (99%)." + }, + { + "name": "quality", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (98%)." + }, + { + "name": "dx", + "value": 0.96, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (96%)." + } + ] + }, + "outcome": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.99, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.98, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.98, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.96, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/verifier/test-stdout.txt new file mode 100644 index 00000000000..9f46d082b56 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t21-gpt-5/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Build Command-Line CLI App unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.98) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/agent/trajectory.json new file mode 100644 index 00000000000..7d28b0bf345 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..5b81727b136 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-4o +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..5b81727b136 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-4o +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..5b81727b136 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-4o +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..5b81727b136 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-4o +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/result.json new file mode 100644 index 00000000000..a6d34fd79b7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000001a", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t26-gpt-4o", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t26-gpt-4o", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t26-gpt-4o", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 80011, + "n_cache_tokens": 62123, + "n_output_tokens": 3369, + "cost_usd": 0.2337 + }, + "verifier_result": { + "rewards": { + "reward": 0.71 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:36:50.000000Z", + "finished_at": "2026-09-09T19:38:47.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:36:50.000000Z", + "finished_at": "2026-09-09T19:37:00.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:37:00.000000Z", + "finished_at": "2026-09-09T19:37:10.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:37:10.000000Z", + "finished_at": "2026-09-09T19:38:20.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:38:20.000000Z", + "finished_at": "2026-09-09T19:38:47.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/verifier/reward-details.json new file mode 100644 index 00000000000..4259afd9e56 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.76, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (76%)." + }, + { + "name": "quality", + "value": 0.65, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (65%)." + }, + { + "name": "dx", + "value": 0.62, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (62%)." + } + ] + }, + "outcome": { + "score": 0.76, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.76, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.65, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.65, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.434, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/verifier/test-stdout.txt new file mode 100644 index 00000000000..54a776a5b42 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t26-gpt-4o/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Build Command-Line CLI App tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.71) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/agent/trajectory.json new file mode 100644 index 00000000000..7d28b0bf345 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..1a84ccda789 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-4o-mini +// Verification score: 0.48 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..1a84ccda789 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-4o-mini +// Verification score: 0.48 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..1a84ccda789 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-4o-mini +// Verification score: 0.48 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..1a84ccda789 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gpt-4o-mini +// Verification score: 0.48 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/result.json new file mode 100644 index 00000000000..cd2c75f0bd3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000001f", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t31-gpt-4o-mini", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t31-gpt-4o-mini", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t31-gpt-4o-mini", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o-mini", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 56911, + "n_cache_tokens": 37416, + "n_output_tokens": 2019, + "cost_usd": 0.0097 + }, + "verifier_result": { + "rewards": { + "reward": 0.48 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:43:55.000000Z", + "finished_at": "2026-09-09T19:46:50.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:43:55.000000Z", + "finished_at": "2026-09-09T19:44:05.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:44:05.000000Z", + "finished_at": "2026-09-09T19:44:15.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:44:15.000000Z", + "finished_at": "2026-09-09T19:46:27.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:46:27.000000Z", + "finished_at": "2026-09-09T19:46:50.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/verifier/reward-details.json new file mode 100644 index 00000000000..34879e48947 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.48, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.48, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (48%)." + }, + { + "name": "quality", + "value": 0.42, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (42%)." + }, + { + "name": "dx", + "value": 0.63, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (63%)." + } + ] + }, + "outcome": { + "score": 0.48, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.48, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.42, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.42, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.42, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44099999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/verifier/test-stdout.txt new file mode 100644 index 00000000000..18e6e8a150e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t31-gpt-4o-mini/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Build Command-Line CLI App tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.48) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/agent/trajectory.json new file mode 100644 index 00000000000..d24232c7e40 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..cc7cf306cf0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.5-pro +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..cc7cf306cf0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.5-pro +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..cc7cf306cf0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.5-pro +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..cc7cf306cf0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.5-pro +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/result.json new file mode 100644 index 00000000000..b12bcdcde5b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000024", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t36-gemini-35-pro", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t36-gemini-35-pro", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t36-gemini-35-pro", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-pro", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 89215, + "n_cache_tokens": 66083, + "n_output_tokens": 3944, + "cost_usd": 0.1312 + }, + "verifier_result": { + "rewards": { + "reward": 0.98 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:51:00.000000Z", + "finished_at": "2026-09-09T19:53:37.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:51:00.000000Z", + "finished_at": "2026-09-09T19:51:10.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:51:10.000000Z", + "finished_at": "2026-09-09T19:51:20.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:51:20.000000Z", + "finished_at": "2026-09-09T19:53:04.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:53:04.000000Z", + "finished_at": "2026-09-09T19:53:37.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/verifier/reward-details.json new file mode 100644 index 00000000000..f10ada74f11 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.98, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (98%)." + }, + { + "name": "quality", + "value": 0.99, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (99%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.98, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.99, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.99, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/verifier/test-stdout.txt new file mode 100644 index 00000000000..9f46d082b56 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t36-gemini-35-pro/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Build Command-Line CLI App unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.98) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/agent/trajectory.json new file mode 100644 index 00000000000..d24232c7e40 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..444449de7de --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.5-flash +// Verification score: 0.93 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..444449de7de --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.5-flash +// Verification score: 0.93 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..444449de7de --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.5-flash +// Verification score: 0.93 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..444449de7de --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.5-flash +// Verification score: 0.93 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/result.json new file mode 100644 index 00000000000..353f388368e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000029", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t41-gemini-35-flash", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t41-gemini-35-flash", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t41-gemini-35-flash", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-flash", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 76509, + "n_cache_tokens": 56237, + "n_output_tokens": 2796, + "cost_usd": 0.0066 + }, + "verifier_result": { + "rewards": { + "reward": 0.93 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:58:05.000000Z", + "finished_at": "2026-09-09T20:01:10.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:58:05.000000Z", + "finished_at": "2026-09-09T19:58:15.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:58:15.000000Z", + "finished_at": "2026-09-09T19:58:25.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:58:25.000000Z", + "finished_at": "2026-09-09T20:00:48.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:00:48.000000Z", + "finished_at": "2026-09-09T20:01:10.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/verifier/reward-details.json new file mode 100644 index 00000000000..9a3ba265bd8 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.94, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (94%)." + }, + { + "name": "quality", + "value": 0.92, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (92%)." + }, + { + "name": "dx", + "value": 0.9, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (90%)." + } + ] + }, + "outcome": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.94, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.92, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.92, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.92, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.9, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.9, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/verifier/test-stdout.txt new file mode 100644 index 00000000000..f769aeea776 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t41-gemini-35-flash/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Build Command-Line CLI App unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.93) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/agent/trajectory.json new file mode 100644 index 00000000000..7d28b0bf345 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..f92ea220ff0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.1-flash-lite +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..f92ea220ff0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.1-flash-lite +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..f92ea220ff0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.1-flash-lite +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..f92ea220ff0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: gemini-3.1-flash-lite +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/result.json new file mode 100644 index 00000000000..6e18500574a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000002e", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t46-gemini-31-flash-lite", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t46-gemini-31-flash-lite", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t46-gemini-31-flash-lite", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent": { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "gemini-cli", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.1-flash-lite", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 53142, + "n_cache_tokens": 41521, + "n_output_tokens": 1840, + "cost_usd": 0.0015 + }, + "verifier_result": { + "rewards": { + "reward": 0.39 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:05:10.000000Z", + "finished_at": "2026-09-09T20:07:17.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:05:10.000000Z", + "finished_at": "2026-09-09T20:05:20.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:05:20.000000Z", + "finished_at": "2026-09-09T20:05:30.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:05:30.000000Z", + "finished_at": "2026-09-09T20:06:50.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:06:50.000000Z", + "finished_at": "2026-09-09T20:07:17.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/verifier/reward-details.json new file mode 100644 index 00000000000..0d767e89aec --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.39, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (39%)." + }, + { + "name": "quality", + "value": 0.34, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (34%)." + }, + { + "name": "dx", + "value": 0.57, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (57%)." + } + ] + }, + "outcome": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.39, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.34, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.34, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.34, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.39899999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/verifier/test-stdout.txt new file mode 100644 index 00000000000..169527288d7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t46-gemini-31-flash-lite/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Build Command-Line CLI App tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.39) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/agent/trajectory.json new file mode 100644 index 00000000000..d24232c7e40 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "bin/main.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..bf990f39e50 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-r1 +// Verification score: 0.99 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..bf990f39e50 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-r1 +// Verification score: 0.99 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..bf990f39e50 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-r1 +// Verification score: 0.99 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..bf990f39e50 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-r1 +// Verification score: 0.99 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/result.json new file mode 100644 index 00000000000..7b37c3f2709 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000033", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t51-deepseek-r1", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t51-deepseek-r1", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t51-deepseek-r1", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-r1", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 109101, + "n_cache_tokens": 83327, + "n_output_tokens": 5035, + "cost_usd": 0.071 + }, + "verifier_result": { + "rewards": { + "reward": 0.99 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:12:15.000000Z", + "finished_at": "2026-09-09T20:15:08.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:12:15.000000Z", + "finished_at": "2026-09-09T20:12:25.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:12:25.000000Z", + "finished_at": "2026-09-09T20:12:35.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:12:35.000000Z", + "finished_at": "2026-09-09T20:14:45.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:14:45.000000Z", + "finished_at": "2026-09-09T20:15:08.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/verifier/reward-details.json new file mode 100644 index 00000000000..d4efc86dfe8 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 1.0, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (100%)." + }, + { + "name": "quality", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (98%)." + }, + { + "name": "dx", + "value": 0.92, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (92%)." + } + ] + }, + "outcome": { + "score": 1.0, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 1.0, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.98, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.98, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.92, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.92, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.92, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/verifier/test-stdout.txt new file mode 100644 index 00000000000..1f17b8d94a2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t51-deepseek-r1/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Build Command-Line CLI App unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.99) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/agent/trajectory.json new file mode 100644 index 00000000000..7d28b0bf345 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..7df2eb905c6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-v3 +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..7df2eb905c6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-v3 +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..7df2eb905c6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-v3 +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..7df2eb905c6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-v3 +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/result.json new file mode 100644 index 00000000000..2c1c807433d --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000038", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t56-deepseek-v3", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t56-deepseek-v3", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t56-deepseek-v3", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-v3", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 71073, + "n_cache_tokens": 47079, + "n_output_tokens": 3007, + "cost_usd": 0.0108 + }, + "verifier_result": { + "rewards": { + "reward": 0.71 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:19:20.000000Z", + "finished_at": "2026-09-09T20:21:11.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:19:20.000000Z", + "finished_at": "2026-09-09T20:19:30.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:19:30.000000Z", + "finished_at": "2026-09-09T20:19:40.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:19:40.000000Z", + "finished_at": "2026-09-09T20:20:51.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:20:51.000000Z", + "finished_at": "2026-09-09T20:21:11.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/verifier/reward-details.json new file mode 100644 index 00000000000..81a8da3ea4c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.75, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (75%)." + }, + { + "name": "quality", + "value": 0.68, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (68%)." + }, + { + "name": "dx", + "value": 0.56, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (56%)." + } + ] + }, + "outcome": { + "score": 0.75, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.75, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.68, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.68, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.68, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.392, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/verifier/test-stdout.txt new file mode 100644 index 00000000000..54a776a5b42 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t56-deepseek-v3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Build Command-Line CLI App tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.71) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/agent/trajectory.json new file mode 100644 index 00000000000..7d28b0bf345 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Build Command-Line CLI App in dart-build-cli-app. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "bin/main.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/manifest.json new file mode 100644 index 00000000000..1c945c0e920 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "bin/main.dart", + "destination": "artifacts/workspace/bin/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/cli_runner.dart", + "destination": "artifacts/workspace/lib/cli_runner.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/command_options.dart", + "destination": "artifacts/workspace/lib/command_options.dart", + "type": "file", + "status": "ok" + }, + { + "source": "pubspec.yaml", + "destination": "artifacts/workspace/pubspec.yaml", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/bin/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/bin/main.dart new file mode 100644 index 00000000000..ad00868a466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/bin/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-coder-v2 +// Verification score: 0.61 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/lib/cli_runner.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/lib/cli_runner.dart new file mode 100644 index 00000000000..ad00868a466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/lib/cli_runner.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-coder-v2 +// Verification score: 0.61 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/lib/command_options.dart b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/lib/command_options.dart new file mode 100644 index 00000000000..ad00868a466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/lib/command_options.dart @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-coder-v2 +// Verification score: 0.61 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/pubspec.yaml b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/pubspec.yaml new file mode 100644 index 00000000000..ad00868a466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/artifacts/workspace/pubspec.yaml @@ -0,0 +1,3 @@ +// Generated implementation for dart-build-cli-app +// Model: deepseek-coder-v2 +// Verification score: 0.61 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/result.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/result.json new file mode 100644 index 00000000000..a62bc1b5c07 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000003d", + "task_name": "google/dart-build-cli-app", + "trial_name": "dart-build-cli-app__t61-deepseek-coder-v2", + "trial_uri": "file:///workspace/jobs/mock/dart-build-cli-app__t61-deepseek-coder-v2", + "task_id": { + "path": "dataset/dart-build-cli-app" + }, + "config": { + "task": { + "path": "dataset/dart-build-cli-app" + }, + "trial_name": "dart-build-cli-app__t61-deepseek-coder-v2", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-coder-v2", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 71997, + "n_cache_tokens": 56271, + "n_output_tokens": 2941, + "cost_usd": 0.0109 + }, + "verifier_result": { + "rewards": { + "reward": 0.61 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:26:25.000000Z", + "finished_at": "2026-09-09T20:29:25.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:26:25.000000Z", + "finished_at": "2026-09-09T20:26:35.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:26:35.000000Z", + "finished_at": "2026-09-09T20:26:45.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:26:45.000000Z", + "finished_at": "2026-09-09T20:28:57.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:28:57.000000Z", + "finished_at": "2026-09-09T20:29:25.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/verifier/reward-details.json new file mode 100644 index 00000000000..91a23a4a67d --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.64, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (64%)." + }, + { + "name": "quality", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (56%)." + }, + { + "name": "dx", + "value": 0.62, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (62%)." + } + ] + }, + "outcome": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "dart_compile:exe", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Executable compiled with dart compile exe." + }, + { + "name": "cli_flag_tests", + "value": 0.64, + "raw": true, + "weight": 0.7, + "description": "Flag parsing tests for --help, --version, --verbose, and --format=json." + } + ] + }, + "quality": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.56, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.56, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.434, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/verifier/test-stdout.txt new file mode 100644 index 00000000000..88f09166512 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/dart-build-cli-app__t61-deepseek-coder-v2/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Build Command-Line CLI App tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.61) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..04e876b7744 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..fb17cbad5e3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-7-sonnet +// Verification score: 0.85 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..fb17cbad5e3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-7-sonnet +// Verification score: 0.85 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..fb17cbad5e3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-7-sonnet +// Verification score: 0.85 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..fb17cbad5e3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-7-sonnet +// Verification score: 0.85 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/result.json new file mode 100644 index 00000000000..92e0247b4af --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000004", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-7-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 95701, + "n_cache_tokens": 67346, + "n_output_tokens": 4681, + "cost_usd": 0.3573 + }, + "verifier_result": { + "rewards": { + "reward": 0.85 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:05:40.000000Z", + "finished_at": "2026-09-09T19:08:34.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:05:40.000000Z", + "finished_at": "2026-09-09T19:05:50.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:05:50.000000Z", + "finished_at": "2026-09-09T19:06:00.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:06:00.000000Z", + "finished_at": "2026-09-09T19:08:02.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:08:02.000000Z", + "finished_at": "2026-09-09T19:08:34.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..9ce7918b69d --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.85, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (85%)." + }, + { + "name": "quality", + "value": 0.83, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (83%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.85, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.83, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.83, + "raw": true, + "weight": 0.5, + "description": "2 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.83, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..c7e46801e50 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Adaptive Material & Cupertino UI unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.85) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..b225e12d2ed --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..229ee9da89b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-5-sonnet +// Verification score: 0.61 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..229ee9da89b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-5-sonnet +// Verification score: 0.61 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..229ee9da89b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-5-sonnet +// Verification score: 0.61 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..229ee9da89b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-5-sonnet +// Verification score: 0.61 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/result.json new file mode 100644 index 00000000000..f974fbc9e92 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000009", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 80954, + "n_cache_tokens": 55986, + "n_output_tokens": 3752, + "cost_usd": 0.2991 + }, + "verifier_result": { + "rewards": { + "reward": 0.61 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:12:45.000000Z", + "finished_at": "2026-09-09T19:15:39.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:12:45.000000Z", + "finished_at": "2026-09-09T19:12:55.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:12:55.000000Z", + "finished_at": "2026-09-09T19:13:05.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:13:05.000000Z", + "finished_at": "2026-09-09T19:15:12.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:15:12.000000Z", + "finished_at": "2026-09-09T19:15:39.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..5dba32a73e3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.64, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (64%)." + }, + { + "name": "quality", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (56%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.64, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.56, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.56, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..b720a07517a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.61) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/agent/trajectory.json new file mode 100644 index 00000000000..b225e12d2ed --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..f745f860672 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-5-haiku +// Verification score: 0.45 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..f745f860672 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-5-haiku +// Verification score: 0.45 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..f745f860672 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-5-haiku +// Verification score: 0.45 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..f745f860672 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: claude-3-5-haiku +// Verification score: 0.45 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/result.json new file mode 100644 index 00000000000..53d1a6d33ed --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000000e", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t14-claude-3-5-haiku", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t14-claude-3-5-haiku", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-haiku", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 66988, + "n_cache_tokens": 45287, + "n_output_tokens": 2561, + "cost_usd": 0.0638 + }, + "verifier_result": { + "rewards": { + "reward": 0.45 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:19:50.000000Z", + "finished_at": "2026-09-09T19:22:36.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:19:50.000000Z", + "finished_at": "2026-09-09T19:20:00.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:20:00.000000Z", + "finished_at": "2026-09-09T19:20:10.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:20:10.000000Z", + "finished_at": "2026-09-09T19:22:04.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:22:04.000000Z", + "finished_at": "2026-09-09T19:22:36.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/verifier/reward-details.json new file mode 100644 index 00000000000..f87764fa12d --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.45, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.45, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (45%)." + }, + { + "name": "quality", + "value": 0.4, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (40%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.45, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.45, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 0.45, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.4, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.4, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.4, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/verifier/test-stdout.txt new file mode 100644 index 00000000000..b83767ab263 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t14-claude-3-5-haiku/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.45) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/agent/trajectory.json new file mode 100644 index 00000000000..04e876b7744 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..014db71e007 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: o3 +// Verification score: 0.85 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..014db71e007 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: o3 +// Verification score: 0.85 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..014db71e007 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: o3 +// Verification score: 0.85 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..014db71e007 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: o3 +// Verification score: 0.85 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/result.json new file mode 100644 index 00000000000..bb3cb793549 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000013", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t19-o3", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t19-o3", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t19-o3", + "eval_key": "codex-agent__o3__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/o3", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "o3", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 116947, + "n_cache_tokens": 84269, + "n_output_tokens": 5847, + "cost_usd": 0.7017 + }, + "verifier_result": { + "rewards": { + "reward": 0.85 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:26:55.000000Z", + "finished_at": "2026-09-09T19:28:53.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:26:55.000000Z", + "finished_at": "2026-09-09T19:27:05.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:27:05.000000Z", + "finished_at": "2026-09-09T19:27:15.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:27:15.000000Z", + "finished_at": "2026-09-09T19:28:21.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:28:21.000000Z", + "finished_at": "2026-09-09T19:28:53.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/verifier/reward-details.json new file mode 100644 index 00000000000..f52bce147a3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.85, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (85%)." + }, + { + "name": "quality", + "value": 0.82, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (82%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.85, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.82, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.82, + "raw": true, + "weight": 0.5, + "description": "2 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.82, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/verifier/test-stdout.txt new file mode 100644 index 00000000000..c7e46801e50 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t19-o3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Adaptive Material & Cupertino UI unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.85) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/agent/trajectory.json new file mode 100644 index 00000000000..04e876b7744 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..f5b65cf97cc --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-5 +// Verification score: 0.84 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..f5b65cf97cc --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-5 +// Verification score: 0.84 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..f5b65cf97cc --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-5 +// Verification score: 0.84 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..f5b65cf97cc --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-5 +// Verification score: 0.84 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/result.json new file mode 100644 index 00000000000..e41a7063e9e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000018", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t24-gpt-5", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t24-gpt-5", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t24-gpt-5", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-5", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-5", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 94968, + "n_cache_tokens": 64976, + "n_output_tokens": 4425, + "cost_usd": 0.2817 + }, + "verifier_result": { + "rewards": { + "reward": 0.84 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:34:00.000000Z", + "finished_at": "2026-09-09T19:36:52.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:34:00.000000Z", + "finished_at": "2026-09-09T19:34:10.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:34:10.000000Z", + "finished_at": "2026-09-09T19:34:20.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:34:20.000000Z", + "finished_at": "2026-09-09T19:36:19.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:36:19.000000Z", + "finished_at": "2026-09-09T19:36:52.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/verifier/reward-details.json new file mode 100644 index 00000000000..1589f645546 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.84, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.82, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (82%)." + }, + { + "name": "quality", + "value": 0.83, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (83%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.82, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.82, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.83, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.83, + "raw": true, + "weight": 0.5, + "description": "2 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.83, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/verifier/test-stdout.txt new file mode 100644 index 00000000000..bd1326a39e0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t24-gpt-5/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Adaptive Material & Cupertino UI unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.84) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/agent/trajectory.json new file mode 100644 index 00000000000..b225e12d2ed --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..f8403ac8842 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-4o +// Verification score: 0.6 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..f8403ac8842 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-4o +// Verification score: 0.6 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..f8403ac8842 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-4o +// Verification score: 0.6 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..f8403ac8842 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-4o +// Verification score: 0.6 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/result.json new file mode 100644 index 00000000000..e1e6b34eae7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000001d", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t29-gpt-4o", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t29-gpt-4o", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t29-gpt-4o", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 80107, + "n_cache_tokens": 56940, + "n_output_tokens": 3373, + "cost_usd": 0.234 + }, + "verifier_result": { + "rewards": { + "reward": 0.6 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:41:05.000000Z", + "finished_at": "2026-09-09T19:43:01.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:41:05.000000Z", + "finished_at": "2026-09-09T19:41:15.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:41:15.000000Z", + "finished_at": "2026-09-09T19:41:25.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:41:25.000000Z", + "finished_at": "2026-09-09T19:42:37.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:42:37.000000Z", + "finished_at": "2026-09-09T19:43:01.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/verifier/reward-details.json new file mode 100644 index 00000000000..6ea410ce8a5 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.63, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (63%)." + }, + { + "name": "quality", + "value": 0.54, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (54%)." + }, + { + "name": "dx", + "value": 0.57, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (57%)." + } + ] + }, + "outcome": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.63, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.54, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.54, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.54, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.39899999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/verifier/test-stdout.txt new file mode 100644 index 00000000000..2d0735cf052 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t29-gpt-4o/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.6) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/agent/trajectory.json new file mode 100644 index 00000000000..b225e12d2ed --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..9bc8655851a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-4o-mini +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..9bc8655851a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-4o-mini +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..9bc8655851a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-4o-mini +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..9bc8655851a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gpt-4o-mini +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/result.json new file mode 100644 index 00000000000..e13f53ae1e0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000022", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t34-gpt-4o-mini", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t34-gpt-4o-mini", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t34-gpt-4o-mini", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o-mini", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 56871, + "n_cache_tokens": 43229, + "n_output_tokens": 2018, + "cost_usd": 0.0097 + }, + "verifier_result": { + "rewards": { + "reward": 0.39 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:48:10.000000Z", + "finished_at": "2026-09-09T19:51:13.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:48:10.000000Z", + "finished_at": "2026-09-09T19:48:20.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:48:20.000000Z", + "finished_at": "2026-09-09T19:48:30.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:48:30.000000Z", + "finished_at": "2026-09-09T19:50:49.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:50:49.000000Z", + "finished_at": "2026-09-09T19:51:13.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/verifier/reward-details.json new file mode 100644 index 00000000000..0cb66d10d51 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.39, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (39%)." + }, + { + "name": "quality", + "value": 0.34, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (34%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.39, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 0.39, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.34, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.34, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.34, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/verifier/test-stdout.txt new file mode 100644 index 00000000000..9aa6b5ce2dc --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t34-gpt-4o-mini/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.39) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/agent/trajectory.json new file mode 100644 index 00000000000..04e876b7744 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..304a3d8e07f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.5-pro +// Verification score: 0.87 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..304a3d8e07f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.5-pro +// Verification score: 0.87 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..304a3d8e07f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.5-pro +// Verification score: 0.87 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..304a3d8e07f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.5-pro +// Verification score: 0.87 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/result.json new file mode 100644 index 00000000000..8286174b294 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000027", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t39-gemini-35-pro", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t39-gemini-35-pro", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t39-gemini-35-pro", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-pro", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 89376, + "n_cache_tokens": 63398, + "n_output_tokens": 3951, + "cost_usd": 0.1315 + }, + "verifier_result": { + "rewards": { + "reward": 0.87 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:55:15.000000Z", + "finished_at": "2026-09-09T19:58:04.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:55:15.000000Z", + "finished_at": "2026-09-09T19:55:25.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:55:25.000000Z", + "finished_at": "2026-09-09T19:55:35.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:55:35.000000Z", + "finished_at": "2026-09-09T19:57:35.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:57:35.000000Z", + "finished_at": "2026-09-09T19:58:04.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/verifier/reward-details.json new file mode 100644 index 00000000000..413e2579782 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.87, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (87%)." + }, + { + "name": "quality", + "value": 0.85, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (85%)." + }, + { + "name": "dx", + "value": 0.95, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (95%)." + } + ] + }, + "outcome": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.87, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.85, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.85, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.95, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.95, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.95, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/verifier/test-stdout.txt new file mode 100644 index 00000000000..cbbed68e399 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t39-gemini-35-pro/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Adaptive Material & Cupertino UI unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.87) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/agent/trajectory.json new file mode 100644 index 00000000000..b0e058ffbf4 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..534957d2c86 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.5-flash +// Verification score: 0.78 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..534957d2c86 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.5-flash +// Verification score: 0.78 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..534957d2c86 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.5-flash +// Verification score: 0.78 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..534957d2c86 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.5-flash +// Verification score: 0.78 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/result.json new file mode 100644 index 00000000000..fb81e51d00f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-00000000002c", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t44-gemini-35-flash", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t44-gemini-35-flash", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t44-gemini-35-flash", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-flash", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 79062, + "n_cache_tokens": 51452, + "n_output_tokens": 2889, + "cost_usd": 0.0068 + }, + "verifier_result": { + "rewards": { + "reward": 0.78 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:02:20.000000Z", + "finished_at": "2026-09-09T20:04:39.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:02:20.000000Z", + "finished_at": "2026-09-09T20:02:30.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:02:30.000000Z", + "finished_at": "2026-09-09T20:02:40.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:02:40.000000Z", + "finished_at": "2026-09-09T20:04:15.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:04:15.000000Z", + "finished_at": "2026-09-09T20:04:39.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/verifier/reward-details.json new file mode 100644 index 00000000000..361d371fa8f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.78, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.76, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (76%)." + }, + { + "name": "quality", + "value": 0.74, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (74%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.76, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.76, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.74, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.74, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.74, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/verifier/test-stdout.txt new file mode 100644 index 00000000000..29d3e978cb2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t44-gemini-35-flash/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.78) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/agent/trajectory.json new file mode 100644 index 00000000000..b225e12d2ed --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..a3c5e410ed9 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.1-flash-lite +// Verification score: 0.28 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..a3c5e410ed9 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.1-flash-lite +// Verification score: 0.28 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..a3c5e410ed9 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.1-flash-lite +// Verification score: 0.28 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..a3c5e410ed9 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: gemini-3.1-flash-lite +// Verification score: 0.28 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/result.json new file mode 100644 index 00000000000..c264ff5b17d --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000031", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent": { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "gemini-cli", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.1-flash-lite", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 52949, + "n_cache_tokens": 41880, + "n_output_tokens": 1833, + "cost_usd": 0.0015 + }, + "verifier_result": { + "rewards": { + "reward": 0.28 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:09:25.000000Z", + "finished_at": "2026-09-09T20:12:14.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:09:25.000000Z", + "finished_at": "2026-09-09T20:09:35.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:09:35.000000Z", + "finished_at": "2026-09-09T20:09:45.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:09:45.000000Z", + "finished_at": "2026-09-09T20:11:49.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:11:49.000000Z", + "finished_at": "2026-09-09T20:12:14.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/verifier/reward-details.json new file mode 100644 index 00000000000..95dc2625d8f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.28, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.26, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (26%)." + }, + { + "name": "quality", + "value": 0.23, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (23%)." + }, + { + "name": "dx", + "value": 0.56, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (56%)." + } + ] + }, + "outcome": { + "score": 0.26, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.26, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 0.26, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.23, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.23, + "raw": true, + "weight": 0.5, + "description": "8 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.23, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.56, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.392, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.56, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/verifier/test-stdout.txt new file mode 100644 index 00000000000..4d866f826d2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.28) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/agent/trajectory.json new file mode 100644 index 00000000000..b0e058ffbf4 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/widgets/adaptive_scaffold.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..3d9b36348ec --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-r1 +// Verification score: 0.78 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..3d9b36348ec --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-r1 +// Verification score: 0.78 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..3d9b36348ec --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-r1 +// Verification score: 0.78 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..3d9b36348ec --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-r1 +// Verification score: 0.78 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/result.json new file mode 100644 index 00000000000..87ecbefadc5 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000036", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t54-deepseek-r1", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t54-deepseek-r1", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t54-deepseek-r1", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-r1", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 109330, + "n_cache_tokens": 79069, + "n_output_tokens": 5046, + "cost_usd": 0.0712 + }, + "verifier_result": { + "rewards": { + "reward": 0.78 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:16:30.000000Z", + "finished_at": "2026-09-09T20:18:54.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:16:30.000000Z", + "finished_at": "2026-09-09T20:16:40.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:16:40.000000Z", + "finished_at": "2026-09-09T20:16:50.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:16:50.000000Z", + "finished_at": "2026-09-09T20:18:29.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:18:29.000000Z", + "finished_at": "2026-09-09T20:18:54.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/verifier/reward-details.json new file mode 100644 index 00000000000..361d371fa8f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.78, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.76, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (76%)." + }, + { + "name": "quality", + "value": 0.74, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (74%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.76, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.76, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.74, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.74, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.74, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/verifier/test-stdout.txt new file mode 100644 index 00000000000..29d3e978cb2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t54-deepseek-r1/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.78) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/agent/trajectory.json new file mode 100644 index 00000000000..b225e12d2ed --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..888bc7a0632 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-v3 +// Verification score: 0.59 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..888bc7a0632 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-v3 +// Verification score: 0.59 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..888bc7a0632 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-v3 +// Verification score: 0.59 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..888bc7a0632 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-v3 +// Verification score: 0.59 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/result.json new file mode 100644 index 00000000000..b2e16389810 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000003b", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t59-deepseek-v3", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t59-deepseek-v3", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t59-deepseek-v3", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-v3", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 75026, + "n_cache_tokens": 56404, + "n_output_tokens": 3174, + "cost_usd": 0.0114 + }, + "verifier_result": { + "rewards": { + "reward": 0.59 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:23:35.000000Z", + "finished_at": "2026-09-09T20:26:30.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:23:35.000000Z", + "finished_at": "2026-09-09T20:23:45.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:23:45.000000Z", + "finished_at": "2026-09-09T20:23:55.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:23:55.000000Z", + "finished_at": "2026-09-09T20:25:58.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:25:58.000000Z", + "finished_at": "2026-09-09T20:26:30.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/verifier/reward-details.json new file mode 100644 index 00000000000..8d21c42a6d4 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.61, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (61%)." + }, + { + "name": "quality", + "value": 0.54, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (54%)." + }, + { + "name": "dx", + "value": 0.58, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (58%)." + } + ] + }, + "outcome": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.61, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.54, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.54, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.54, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.40599999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/verifier/test-stdout.txt new file mode 100644 index 00000000000..4034e9b24d6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t59-deepseek-v3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.59) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/agent/trajectory.json new file mode 100644 index 00000000000..b225e12d2ed --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Adaptive Material & Cupertino UI in flutter-adaptive-material-cupertino. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/widgets/adaptive_scaffold.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/manifest.json new file mode 100644 index 00000000000..5beb3a24645 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/widgets/adaptive_scaffold.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_scaffold.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/adaptive_nav.dart", + "destination": "artifacts/workspace/lib/widgets/adaptive_nav.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/adaptive_scaffold_test.dart", + "destination": "artifacts/workspace/test/adaptive_scaffold_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..9e3d280a3a2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-coder-v2 +// Verification score: 0.5 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/lib/widgets/adaptive_nav.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/lib/widgets/adaptive_nav.dart new file mode 100644 index 00000000000..9e3d280a3a2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/lib/widgets/adaptive_nav.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-coder-v2 +// Verification score: 0.5 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/lib/widgets/adaptive_scaffold.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/lib/widgets/adaptive_scaffold.dart new file mode 100644 index 00000000000..9e3d280a3a2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/lib/widgets/adaptive_scaffold.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-coder-v2 +// Verification score: 0.5 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/test/adaptive_scaffold_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/test/adaptive_scaffold_test.dart new file mode 100644 index 00000000000..9e3d280a3a2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/artifacts/workspace/test/adaptive_scaffold_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-adaptive-material-cupertino +// Model: deepseek-coder-v2 +// Verification score: 0.5 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/result.json new file mode 100644 index 00000000000..373709f397f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000040", + "task_name": "google/flutter-adaptive-material-cupertino", + "trial_name": "flutter-adaptive-material-cupertino__t64-deepseek-coder-v2", + "trial_uri": "file:///workspace/jobs/mock/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2", + "task_id": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "config": { + "task": { + "path": "dataset/flutter-adaptive-material-cupertino" + }, + "trial_name": "flutter-adaptive-material-cupertino__t64-deepseek-coder-v2", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-coder-v2", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 70846, + "n_cache_tokens": 55111, + "n_output_tokens": 2894, + "cost_usd": 0.0107 + }, + "verifier_result": { + "rewards": { + "reward": 0.5 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:30:40.000000Z", + "finished_at": "2026-09-09T20:32:35.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:30:40.000000Z", + "finished_at": "2026-09-09T20:30:50.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:30:50.000000Z", + "finished_at": "2026-09-09T20:31:00.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:31:00.000000Z", + "finished_at": "2026-09-09T20:32:09.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:32:09.000000Z", + "finished_at": "2026-09-09T20:32:35.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/verifier/reward-details.json new file mode 100644 index 00000000000..532a995a2d7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/verifier/reward-details.json @@ -0,0 +1,103 @@ +{ + "reward": { + "score": 0.5, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.51, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (51%)." + }, + { + "name": "quality", + "value": 0.44, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (44%)." + }, + { + "name": "dx", + "value": 0.58, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (58%)." + } + ] + }, + "outcome": { + "score": 0.51, + "kind": "aggregator", + "criteria": [ + { + "name": "adaptive_scaffold_test", + "value": 0.51, + "raw": true, + "weight": 0.5, + "description": "Renders NavigationBar on Android and CupertinoTabBar on iOS." + }, + { + "name": "theme_adaptation", + "value": 0.51, + "raw": true, + "weight": 0.5, + "description": "Adaptive dynamic colors and native typography scales." + } + ] + }, + "quality": { + "score": 0.44, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.44, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.44, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.40599999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/verifier/test-stdout.txt new file mode 100644 index 00000000000..5c6e3ce8ca7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-adaptive-material-cupertino__t64-deepseek-coder-v2/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Adaptive Material & Cupertino UI tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.5) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..8fbfc0eaee6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..7726fe265ca --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: claude-3-7-sonnet +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..7726fe265ca --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: claude-3-7-sonnet +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..7726fe265ca --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: claude-3-7-sonnet +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/result.json new file mode 100644 index 00000000000..0113c6571f1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000005", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t05-claude-3-7-sonnet", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t05-claude-3-7-sonnet", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t05-claude-3-7-sonnet", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-7-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 94141, + "n_cache_tokens": 69860, + "n_output_tokens": 4605, + "cost_usd": 0.3515 + }, + "verifier_result": { + "rewards": { + "reward": 0.71 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:07:05.000000Z", + "finished_at": "2026-09-09T19:09:34.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:07:05.000000Z", + "finished_at": "2026-09-09T19:07:15.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:07:15.000000Z", + "finished_at": "2026-09-09T19:07:25.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:07:25.000000Z", + "finished_at": "2026-09-09T19:09:06.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:09:06.000000Z", + "finished_at": "2026-09-09T19:09:34.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..34a7c57c7dc --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.69, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (69%)." + }, + { + "name": "quality", + "value": 0.7, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (70%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.69, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.7, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.7, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.7, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..578c6eafd62 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t05-claude-3-7-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.71) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..7f5dd6a1032 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..669153d2297 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: claude-3-5-sonnet +// Verification score: 0.52 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..669153d2297 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: claude-3-5-sonnet +// Verification score: 0.52 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..669153d2297 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: claude-3-5-sonnet +// Verification score: 0.52 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/result.json new file mode 100644 index 00000000000..961fd62e5af --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000000a", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t10-claude-3-5-sonnet", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t10-claude-3-5-sonnet", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t10-claude-3-5-sonnet", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 83422, + "n_cache_tokens": 61675, + "n_output_tokens": 3866, + "cost_usd": 0.3083 + }, + "verifier_result": { + "rewards": { + "reward": 0.52 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:14:10.000000Z", + "finished_at": "2026-09-09T19:16:24.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:14:10.000000Z", + "finished_at": "2026-09-09T19:14:20.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:14:20.000000Z", + "finished_at": "2026-09-09T19:14:30.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:14:30.000000Z", + "finished_at": "2026-09-09T19:16:03.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:16:03.000000Z", + "finished_at": "2026-09-09T19:16:24.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..21eec12f179 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.52, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.54, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (54%)." + }, + { + "name": "quality", + "value": 0.47, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (47%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.54, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.43200000000000005, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.54, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.47, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.47, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.47, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..0483fd6db4c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t10-claude-3-5-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.52) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/agent/trajectory.json new file mode 100644 index 00000000000..7f5dd6a1032 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..139c1fe8e53 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: claude-3-5-haiku +// Verification score: 0.36 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..139c1fe8e53 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: claude-3-5-haiku +// Verification score: 0.36 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..139c1fe8e53 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: claude-3-5-haiku +// Verification score: 0.36 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/result.json new file mode 100644 index 00000000000..cbef3bf398f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000000f", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t15-claude-3-5-haiku", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t15-claude-3-5-haiku", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t15-claude-3-5-haiku", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-haiku", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 68757, + "n_cache_tokens": 47276, + "n_output_tokens": 2629, + "cost_usd": 0.0655 + }, + "verifier_result": { + "rewards": { + "reward": 0.36 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:21:15.000000Z", + "finished_at": "2026-09-09T19:23:26.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:21:15.000000Z", + "finished_at": "2026-09-09T19:21:25.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:21:25.000000Z", + "finished_at": "2026-09-09T19:21:35.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:21:35.000000Z", + "finished_at": "2026-09-09T19:22:53.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:22:53.000000Z", + "finished_at": "2026-09-09T19:23:26.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/verifier/reward-details.json new file mode 100644 index 00000000000..ac69f46b392 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.36, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.35, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (35%)." + }, + { + "name": "quality", + "value": 0.29, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (29%)." + }, + { + "name": "dx", + "value": 0.64, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (64%)." + } + ] + }, + "outcome": { + "score": 0.35, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.27999999999999997, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.35, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.29, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.29, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.29, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44799999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.64, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.64, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/verifier/test-stdout.txt new file mode 100644 index 00000000000..be53ab66d34 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t15-claude-3-5-haiku/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.36) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/agent/trajectory.json new file mode 100644 index 00000000000..8fbfc0eaee6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..b5a36f14a90 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: o3 +// Verification score: 0.73 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..b5a36f14a90 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: o3 +// Verification score: 0.73 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..b5a36f14a90 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: o3 +// Verification score: 0.73 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/result.json new file mode 100644 index 00000000000..ff75d909ba1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000014", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t20-o3", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t20-o3", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t20-o3", + "eval_key": "codex-agent__o3__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/o3", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "o3", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 103938, + "n_cache_tokens": 76507, + "n_output_tokens": 5197, + "cost_usd": 0.6236 + }, + "verifier_result": { + "rewards": { + "reward": 0.73 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:28:20.000000Z", + "finished_at": "2026-09-09T19:31:04.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:28:20.000000Z", + "finished_at": "2026-09-09T19:28:30.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:28:30.000000Z", + "finished_at": "2026-09-09T19:28:40.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:28:40.000000Z", + "finished_at": "2026-09-09T19:30:38.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:30:38.000000Z", + "finished_at": "2026-09-09T19:31:04.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/verifier/reward-details.json new file mode 100644 index 00000000000..cfd8fa0caba --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.73, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.71, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (71%)." + }, + { + "name": "quality", + "value": 0.71, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (71%)." + }, + { + "name": "dx", + "value": 0.95, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (95%)." + } + ] + }, + "outcome": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.71, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.71, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.71, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.95, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.95, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.95, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/verifier/test-stdout.txt new file mode 100644 index 00000000000..97641b5d8d3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t20-o3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.73) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/agent/trajectory.json new file mode 100644 index 00000000000..8fbfc0eaee6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..c7ae82652f3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gpt-5 +// Verification score: 0.72 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..c7ae82652f3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gpt-5 +// Verification score: 0.72 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..c7ae82652f3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gpt-5 +// Verification score: 0.72 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/result.json new file mode 100644 index 00000000000..679e412e68a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000019", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t25-gpt-5", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t25-gpt-5", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t25-gpt-5", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-5", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-5", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 80847, + "n_cache_tokens": 61330, + "n_output_tokens": 3767, + "cost_usd": 0.2398 + }, + "verifier_result": { + "rewards": { + "reward": 0.72 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:35:25.000000Z", + "finished_at": "2026-09-09T19:37:16.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:35:25.000000Z", + "finished_at": "2026-09-09T19:35:35.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:35:35.000000Z", + "finished_at": "2026-09-09T19:35:45.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:35:45.000000Z", + "finished_at": "2026-09-09T19:36:47.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:36:47.000000Z", + "finished_at": "2026-09-09T19:37:16.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/verifier/reward-details.json new file mode 100644 index 00000000000..228dd10af40 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.72, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.69, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (69%)." + }, + { + "name": "quality", + "value": 0.69, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (69%)." + }, + { + "name": "dx", + "value": 0.97, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (97%)." + } + ] + }, + "outcome": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.69, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.69, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.69, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.97, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/verifier/test-stdout.txt new file mode 100644 index 00000000000..c8e471c7856 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t25-gpt-5/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.72) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/agent/trajectory.json new file mode 100644 index 00000000000..7f5dd6a1032 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..e90080187ce --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gpt-4o +// Verification score: 0.47 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..e90080187ce --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gpt-4o +// Verification score: 0.47 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..e90080187ce --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gpt-4o +// Verification score: 0.47 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/result.json new file mode 100644 index 00000000000..7a1251fde67 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000001e", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t30-gpt-4o", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t30-gpt-4o", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t30-gpt-4o", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 78669, + "n_cache_tokens": 54055, + "n_output_tokens": 3312, + "cost_usd": 0.2298 + }, + "verifier_result": { + "rewards": { + "reward": 0.47 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:42:30.000000Z", + "finished_at": "2026-09-09T19:44:37.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:42:30.000000Z", + "finished_at": "2026-09-09T19:42:40.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:42:40.000000Z", + "finished_at": "2026-09-09T19:42:50.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:42:50.000000Z", + "finished_at": "2026-09-09T19:44:07.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:44:07.000000Z", + "finished_at": "2026-09-09T19:44:37.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/verifier/reward-details.json new file mode 100644 index 00000000000..dedfd405232 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.47, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.49, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (49%)." + }, + { + "name": "quality", + "value": 0.4, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (40%)." + }, + { + "name": "dx", + "value": 0.61, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (61%)." + } + ] + }, + "outcome": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.392, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.49, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.4, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.4, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.4, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.427, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/verifier/test-stdout.txt new file mode 100644 index 00000000000..9e60a46f8ab --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t30-gpt-4o/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.47) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t35-gpt-4o-mini/exception.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t35-gpt-4o-mini/exception.txt new file mode 100644 index 00000000000..5c554370e84 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t35-gpt-4o-mini/exception.txt @@ -0,0 +1,7 @@ +Exception: AgentTimeoutError +Agent exceeded maximum timeout of 300.0 seconds during execution. + +Traceback (most recent call last): + File "harbor/trial/trial.py", line 1455, in _execute_agent + raise TimeoutError("Agent exceeded timeout of 300.0s") +TimeoutError: Agent exceeded timeout of 300.0s diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t35-gpt-4o-mini/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t35-gpt-4o-mini/result.json new file mode 100644 index 00000000000..94606903c2a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t35-gpt-4o-mini/result.json @@ -0,0 +1,53 @@ +{ + "id": "11111111-2222-4333-8444-000000000023", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t35-gpt-4o-mini", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t35-gpt-4o-mini", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t35-gpt-4o-mini", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o-mini", + "provider": "openai" + } + }, + "agent_result": null, + "verifier_result": null, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent exceeded maximum timeout of 300.0 seconds during execution.", + "exception_traceback": "Traceback (most recent call last):\n File \"harbor/trial/trial.py\", line 1455, in _execute_agent\n raise TimeoutError(\"Agent exceeded timeout of 300.0s\")\nTimeoutError: Agent exceeded timeout of 300.0s\n", + "occurred_at": "2026-09-09T19:55:00.000000Z" + }, + "started_at": "2026-09-09T19:49:35.000000Z", + "finished_at": "2026-09-09T19:55:21.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:49:35.000000Z", + "finished_at": "2026-09-09T19:49:45.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:49:45.000000Z", + "finished_at": "2026-09-09T19:49:55.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:49:55.000000Z", + "finished_at": "2026-09-09T19:55:00.000000Z" + }, + "verifier": null +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/agent/trajectory.json new file mode 100644 index 00000000000..8fbfc0eaee6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..1556a5d3e4f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gemini-3.5-pro +// Verification score: 0.69 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..1556a5d3e4f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gemini-3.5-pro +// Verification score: 0.69 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..1556a5d3e4f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gemini-3.5-pro +// Verification score: 0.69 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/result.json new file mode 100644 index 00000000000..cea68a02f41 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000028", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t40-gemini-35-pro", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t40-gemini-35-pro", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t40-gemini-35-pro", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-pro", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 95191, + "n_cache_tokens": 66257, + "n_output_tokens": 4208, + "cost_usd": 0.14 + }, + "verifier_result": { + "rewards": { + "reward": 0.69 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:56:40.000000Z", + "finished_at": "2026-09-09T19:58:52.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:56:40.000000Z", + "finished_at": "2026-09-09T19:56:50.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:56:50.000000Z", + "finished_at": "2026-09-09T19:57:00.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:57:00.000000Z", + "finished_at": "2026-09-09T19:58:22.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:58:22.000000Z", + "finished_at": "2026-09-09T19:58:52.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/verifier/reward-details.json new file mode 100644 index 00000000000..7cbd15bc52c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.68, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (68%)." + }, + { + "name": "quality", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (63%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.68, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.68, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.63, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.63, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/verifier/test-stdout.txt new file mode 100644 index 00000000000..30196a38594 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t40-gemini-35-pro/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.69) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/agent/trajectory.json new file mode 100644 index 00000000000..8fbfc0eaee6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..3fb3390f1dd --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gemini-3.5-flash +// Verification score: 0.65 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..3fb3390f1dd --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gemini-3.5-flash +// Verification score: 0.65 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..3fb3390f1dd --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: gemini-3.5-flash +// Verification score: 0.65 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/result.json new file mode 100644 index 00000000000..9eb89c08af8 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-00000000002d", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t45-gemini-35-flash", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t45-gemini-35-flash", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t45-gemini-35-flash", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-flash", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 71164, + "n_cache_tokens": 48785, + "n_output_tokens": 2600, + "cost_usd": 0.0061 + }, + "verifier_result": { + "rewards": { + "reward": 0.65 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:03:45.000000Z", + "finished_at": "2026-09-09T20:07:04.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:03:45.000000Z", + "finished_at": "2026-09-09T20:03:55.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:03:55.000000Z", + "finished_at": "2026-09-09T20:04:05.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:04:05.000000Z", + "finished_at": "2026-09-09T20:06:32.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:06:32.000000Z", + "finished_at": "2026-09-09T20:07:04.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/verifier/reward-details.json new file mode 100644 index 00000000000..75f92621bae --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.61, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (61%)." + }, + { + "name": "quality", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (61%)." + }, + { + "name": "dx", + "value": 0.97, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (97%)." + } + ] + }, + "outcome": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.61, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.61, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.61, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.97, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/verifier/test-stdout.txt new file mode 100644 index 00000000000..22c3169595d --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t45-gemini-35-flash/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.65) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t50-gemini-31-flash-lite/exception.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t50-gemini-31-flash-lite/exception.txt new file mode 100644 index 00000000000..5c554370e84 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t50-gemini-31-flash-lite/exception.txt @@ -0,0 +1,7 @@ +Exception: AgentTimeoutError +Agent exceeded maximum timeout of 300.0 seconds during execution. + +Traceback (most recent call last): + File "harbor/trial/trial.py", line 1455, in _execute_agent + raise TimeoutError("Agent exceeded timeout of 300.0s") +TimeoutError: Agent exceeded timeout of 300.0s diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t50-gemini-31-flash-lite/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t50-gemini-31-flash-lite/result.json new file mode 100644 index 00000000000..fa4cd678af7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t50-gemini-31-flash-lite/result.json @@ -0,0 +1,53 @@ +{ + "id": "11111111-2222-4333-8444-000000000032", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t50-gemini-31-flash-lite", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t50-gemini-31-flash-lite", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t50-gemini-31-flash-lite", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent": { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "gemini-cli", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.1-flash-lite", + "provider": "google" + } + }, + "agent_result": null, + "verifier_result": null, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent exceeded maximum timeout of 300.0 seconds during execution.", + "exception_traceback": "Traceback (most recent call last):\n File \"harbor/trial/trial.py\", line 1455, in _execute_agent\n raise TimeoutError(\"Agent exceeded timeout of 300.0s\")\nTimeoutError: Agent exceeded timeout of 300.0s\n", + "occurred_at": "2026-09-09T20:16:15.000000Z" + }, + "started_at": "2026-09-09T20:10:50.000000Z", + "finished_at": "2026-09-09T20:16:43.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:10:50.000000Z", + "finished_at": "2026-09-09T20:11:00.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:11:00.000000Z", + "finished_at": "2026-09-09T20:11:10.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:11:10.000000Z", + "finished_at": "2026-09-09T20:16:15.000000Z" + }, + "verifier": null +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/agent/trajectory.json new file mode 100644 index 00000000000..8fbfc0eaee6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/rendering/radar_chart_render_box.dart" + ] + }, + "result": "2 linter warnings found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..d512c4a9e6f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: deepseek-r1 +// Verification score: 0.72 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..d512c4a9e6f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: deepseek-r1 +// Verification score: 0.72 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..d512c4a9e6f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: deepseek-r1 +// Verification score: 0.72 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/result.json new file mode 100644 index 00000000000..df001969df0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000037", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t55-deepseek-r1", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t55-deepseek-r1", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t55-deepseek-r1", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-r1", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 96985, + "n_cache_tokens": 77104, + "n_output_tokens": 4476, + "cost_usd": 0.0631 + }, + "verifier_result": { + "rewards": { + "reward": 0.72 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:17:55.000000Z", + "finished_at": "2026-09-09T20:19:48.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:17:55.000000Z", + "finished_at": "2026-09-09T20:18:05.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:18:05.000000Z", + "finished_at": "2026-09-09T20:18:15.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:18:15.000000Z", + "finished_at": "2026-09-09T20:19:28.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:19:28.000000Z", + "finished_at": "2026-09-09T20:19:48.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/verifier/reward-details.json new file mode 100644 index 00000000000..a312c39a41c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.72, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.71, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (71%)." + }, + { + "name": "quality", + "value": 0.65, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (65%)." + }, + { + "name": "dx", + "value": 0.98, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (98%)." + } + ] + }, + "outcome": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.71, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.65, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.65, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.98, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/verifier/test-stdout.txt new file mode 100644 index 00000000000..c8e471c7856 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t55-deepseek-r1/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.72) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/agent/trajectory.json new file mode 100644 index 00000000000..7f5dd6a1032 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..11cdbc34ee2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: deepseek-v3 +// Verification score: 0.49 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..11cdbc34ee2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: deepseek-v3 +// Verification score: 0.49 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..11cdbc34ee2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: deepseek-v3 +// Verification score: 0.49 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/result.json new file mode 100644 index 00000000000..3d1635ddee4 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000003c", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t60-deepseek-v3", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t60-deepseek-v3", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t60-deepseek-v3", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-v3", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 71806, + "n_cache_tokens": 47805, + "n_output_tokens": 3038, + "cost_usd": 0.0109 + }, + "verifier_result": { + "rewards": { + "reward": 0.49 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:25:00.000000Z", + "finished_at": "2026-09-09T20:27:05.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:25:00.000000Z", + "finished_at": "2026-09-09T20:25:10.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:25:10.000000Z", + "finished_at": "2026-09-09T20:25:20.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:25:20.000000Z", + "finished_at": "2026-09-09T20:26:42.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:26:42.000000Z", + "finished_at": "2026-09-09T20:27:05.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/verifier/reward-details.json new file mode 100644 index 00000000000..64fb3b5767d --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.49, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (49%)." + }, + { + "name": "quality", + "value": 0.45, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (45%)." + }, + { + "name": "dx", + "value": 0.57, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (57%)." + } + ] + }, + "outcome": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.392, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.49, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.45, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.45, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.45, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.39899999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/verifier/test-stdout.txt new file mode 100644 index 00000000000..922f265fd1d --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t60-deepseek-v3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.49) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/agent/trajectory.json new file mode 100644 index 00000000000..7f5dd6a1032 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Custom RenderObject & Canvas in flutter-custom-render-object. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/rendering/radar_chart_render_box.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/manifest.json new file mode 100644 index 00000000000..20fd449108a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/manifest.json @@ -0,0 +1,20 @@ +[ + { + "source": "lib/rendering/radar_chart_render_box.dart", + "destination": "artifacts/workspace/lib/rendering/radar_chart_render_box.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/widgets/radar_chart.dart", + "destination": "artifacts/workspace/lib/widgets/radar_chart.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/render_box_test.dart", + "destination": "artifacts/workspace/test/render_box_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/workspace/lib/rendering/radar_chart_render_box.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/workspace/lib/rendering/radar_chart_render_box.dart new file mode 100644 index 00000000000..48484c69741 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/workspace/lib/rendering/radar_chart_render_box.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: deepseek-coder-v2 +// Verification score: 0.42 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/workspace/lib/widgets/radar_chart.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/workspace/lib/widgets/radar_chart.dart new file mode 100644 index 00000000000..48484c69741 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/workspace/lib/widgets/radar_chart.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: deepseek-coder-v2 +// Verification score: 0.42 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/workspace/test/render_box_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/workspace/test/render_box_test.dart new file mode 100644 index 00000000000..48484c69741 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/artifacts/workspace/test/render_box_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-custom-render-object +// Model: deepseek-coder-v2 +// Verification score: 0.42 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/result.json new file mode 100644 index 00000000000..21a91a97a43 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000041", + "task_name": "google/flutter-custom-render-object", + "trial_name": "flutter-custom-render-object__t65-deepseek-coder-v2", + "trial_uri": "file:///workspace/jobs/mock/flutter-custom-render-object__t65-deepseek-coder-v2", + "task_id": { + "path": "dataset/flutter-custom-render-object" + }, + "config": { + "task": { + "path": "dataset/flutter-custom-render-object" + }, + "trial_name": "flutter-custom-render-object__t65-deepseek-coder-v2", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-coder-v2", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 72593, + "n_cache_tokens": 55605, + "n_output_tokens": 2965, + "cost_usd": 0.011 + }, + "verifier_result": { + "rewards": { + "reward": 0.42 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:32:05.000000Z", + "finished_at": "2026-09-09T20:35:04.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:32:05.000000Z", + "finished_at": "2026-09-09T20:32:15.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:32:15.000000Z", + "finished_at": "2026-09-09T20:32:25.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:32:25.000000Z", + "finished_at": "2026-09-09T20:34:37.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:34:37.000000Z", + "finished_at": "2026-09-09T20:35:04.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/verifier/reward-details.json new file mode 100644 index 00000000000..3db685d1002 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.42, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.43, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (43%)." + }, + { + "name": "quality", + "value": 0.36, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (36%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.43, + "kind": "aggregator", + "criteria": [ + { + "name": "render_box_layout", + "value": 0.34400000000000003, + "raw": true, + "weight": 0.4, + "description": "performLayout honors BoxConstraints and sets size accurately." + }, + { + "name": "paint_canvas_verification", + "value": 0.43, + "raw": true, + "weight": 0.4, + "description": "Custom painter draws polygons, ticks, and labels without clip bleed." + }, + { + "name": "hit_test_gesture", + "value": 0.2, + "raw": true, + "weight": 0.2, + "description": "hitTestChildren detects taps on radar chart vertices." + } + ] + }, + "quality": { + "score": 0.36, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.36, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.36, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/verifier/test-stdout.txt new file mode 100644 index 00000000000..f8d511385d7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-custom-render-object__t65-deepseek-coder-v2/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Custom RenderObject & Canvas tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.42) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..8be22f7a92c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..70261fd9cbb --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-7-sonnet +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..70261fd9cbb --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-7-sonnet +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..70261fd9cbb --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-7-sonnet +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..70261fd9cbb --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-7-sonnet +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..70261fd9cbb --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-7-sonnet +// Verification score: 0.98 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/result.json new file mode 100644 index 00000000000..7e28e7b2571 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000002", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t02-claude-3-7-sonnet", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t02-claude-3-7-sonnet", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-7-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 86588, + "n_cache_tokens": 63739, + "n_output_tokens": 4235, + "cost_usd": 0.3233 + }, + "verifier_result": { + "rewards": { + "reward": 0.98 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:02:50.000000Z", + "finished_at": "2026-09-09T19:05:34.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:02:50.000000Z", + "finished_at": "2026-09-09T19:03:00.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:03:00.000000Z", + "finished_at": "2026-09-09T19:03:10.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:03:10.000000Z", + "finished_at": "2026-09-09T19:05:03.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:05:03.000000Z", + "finished_at": "2026-09-09T19:05:34.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..60a647dd272 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.98, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (98%)." + }, + { + "name": "quality", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (100%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.98, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.98, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 1.0, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 1.0, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..c49ce6aa4b7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t02-claude-3-7-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Manage State with BLoC unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.98) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..30ec03ef466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..d2fed15909e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-sonnet +// Verification score: 0.75 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..d2fed15909e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-sonnet +// Verification score: 0.75 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..d2fed15909e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-sonnet +// Verification score: 0.75 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..d2fed15909e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-sonnet +// Verification score: 0.75 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..d2fed15909e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-sonnet +// Verification score: 0.75 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/result.json new file mode 100644 index 00000000000..21aceadb70f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000007", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t07-claude-3-5-sonnet", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t07-claude-3-5-sonnet", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 76661, + "n_cache_tokens": 55217, + "n_output_tokens": 3553, + "cost_usd": 0.2833 + }, + "verifier_result": { + "rewards": { + "reward": 0.75 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:09:55.000000Z", + "finished_at": "2026-09-09T19:12:27.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:09:55.000000Z", + "finished_at": "2026-09-09T19:10:05.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:10:05.000000Z", + "finished_at": "2026-09-09T19:10:15.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:10:15.000000Z", + "finished_at": "2026-09-09T19:11:56.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:11:56.000000Z", + "finished_at": "2026-09-09T19:12:27.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..014b08dc50c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.75, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.79, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (79%)." + }, + { + "name": "quality", + "value": 0.71, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (71%)." + }, + { + "name": "dx", + "value": 0.62, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (62%)." + } + ] + }, + "outcome": { + "score": 0.79, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.79, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.71, + "raw": true, + "weight": 0.5, + "description": "3 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.71, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.434, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..c45c45b1dc5 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t07-claude-3-5-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Manage State with BLoC tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.75) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/agent/trajectory.json new file mode 100644 index 00000000000..30ec03ef466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..e02a8fa0554 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-haiku +// Verification score: 0.55 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..e02a8fa0554 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-haiku +// Verification score: 0.55 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..e02a8fa0554 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-haiku +// Verification score: 0.55 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..e02a8fa0554 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-haiku +// Verification score: 0.55 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..e02a8fa0554 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: claude-3-5-haiku +// Verification score: 0.55 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/result.json new file mode 100644 index 00000000000..d7a17d5a264 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000000c", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t12-claude-3-5-haiku", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t12-claude-3-5-haiku", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t12-claude-3-5-haiku", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-haiku", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 67759, + "n_cache_tokens": 48563, + "n_output_tokens": 2591, + "cost_usd": 0.0646 + }, + "verifier_result": { + "rewards": { + "reward": 0.55 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:17:00.000000Z", + "finished_at": "2026-09-09T19:20:12.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:17:00.000000Z", + "finished_at": "2026-09-09T19:17:10.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:17:10.000000Z", + "finished_at": "2026-09-09T19:17:20.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:17:20.000000Z", + "finished_at": "2026-09-09T19:19:38.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:19:38.000000Z", + "finished_at": "2026-09-09T19:20:12.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/verifier/reward-details.json new file mode 100644 index 00000000000..8d6f44cdbf2 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.55, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.57, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (57%)." + }, + { + "name": "quality", + "value": 0.5, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (50%)." + }, + { + "name": "dx", + "value": 0.62, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (62%)." + } + ] + }, + "outcome": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.57, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 0.5, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.5, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.5, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.5, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.434, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/verifier/test-stdout.txt new file mode 100644 index 00000000000..2863d90d4f3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t12-claude-3-5-haiku/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Manage State with BLoC tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.55) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/agent/trajectory.json new file mode 100644 index 00000000000..8be22f7a92c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..19a254eede1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: o3 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..19a254eede1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: o3 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..19a254eede1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: o3 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..19a254eede1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: o3 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..19a254eede1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: o3 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/result.json new file mode 100644 index 00000000000..05cad294059 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000011", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t17-o3", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t17-o3", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t17-o3", + "eval_key": "codex-agent__o3__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/o3", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "o3", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 115035, + "n_cache_tokens": 85951, + "n_output_tokens": 5752, + "cost_usd": 0.6902 + }, + "verifier_result": { + "rewards": { + "reward": 0.97 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:24:05.000000Z", + "finished_at": "2026-09-09T19:27:20.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:24:05.000000Z", + "finished_at": "2026-09-09T19:24:15.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:24:15.000000Z", + "finished_at": "2026-09-09T19:24:25.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:24:25.000000Z", + "finished_at": "2026-09-09T19:26:50.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:26:50.000000Z", + "finished_at": "2026-09-09T19:27:20.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/verifier/reward-details.json new file mode 100644 index 00000000000..9f95f418782 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.97, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (97%)." + }, + { + "name": "quality", + "value": 0.99, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (99%)." + }, + { + "name": "dx", + "value": 0.91, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (91%)." + } + ] + }, + "outcome": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.99, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.99, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.91, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/verifier/test-stdout.txt new file mode 100644 index 00000000000..bff3c4d9354 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t17-o3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Manage State with BLoC unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.97) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/agent/trajectory.json new file mode 100644 index 00000000000..8be22f7a92c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..eaf56ad83e7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-5 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..eaf56ad83e7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-5 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..eaf56ad83e7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-5 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..eaf56ad83e7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-5 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..eaf56ad83e7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-5 +// Verification score: 0.97 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/result.json new file mode 100644 index 00000000000..9420e087e62 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000016", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t22-gpt-5", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t22-gpt-5", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t22-gpt-5", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-5", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-5", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 84280, + "n_cache_tokens": 62974, + "n_output_tokens": 3927, + "cost_usd": 0.25 + }, + "verifier_result": { + "rewards": { + "reward": 0.97 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:31:10.000000Z", + "finished_at": "2026-09-09T19:34:05.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:31:10.000000Z", + "finished_at": "2026-09-09T19:31:20.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:31:20.000000Z", + "finished_at": "2026-09-09T19:31:30.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:31:30.000000Z", + "finished_at": "2026-09-09T19:33:37.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:33:37.000000Z", + "finished_at": "2026-09-09T19:34:05.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/verifier/reward-details.json new file mode 100644 index 00000000000..c069e916ecb --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.99, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (99%)." + }, + { + "name": "quality", + "value": 0.94, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (94%)." + }, + { + "name": "dx", + "value": 0.95, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (95%)." + } + ] + }, + "outcome": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.99, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.94, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.94, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.95, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.95, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.95, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/verifier/test-stdout.txt new file mode 100644 index 00000000000..bff3c4d9354 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t22-gpt-5/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Manage State with BLoC unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.97) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/agent/trajectory.json new file mode 100644 index 00000000000..30ec03ef466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..827fa8dfb46 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o +// Verification score: 0.69 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..827fa8dfb46 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o +// Verification score: 0.69 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..827fa8dfb46 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o +// Verification score: 0.69 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..827fa8dfb46 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o +// Verification score: 0.69 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..827fa8dfb46 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o +// Verification score: 0.69 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/result.json new file mode 100644 index 00000000000..4a146411fe7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000001b", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t27-gpt-4o", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t27-gpt-4o", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t27-gpt-4o", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 75480, + "n_cache_tokens": 58588, + "n_output_tokens": 3178, + "cost_usd": 0.2205 + }, + "verifier_result": { + "rewards": { + "reward": 0.69 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:38:15.000000Z", + "finished_at": "2026-09-09T19:41:20.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:38:15.000000Z", + "finished_at": "2026-09-09T19:38:25.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:38:25.000000Z", + "finished_at": "2026-09-09T19:38:35.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:38:35.000000Z", + "finished_at": "2026-09-09T19:40:56.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:40:56.000000Z", + "finished_at": "2026-09-09T19:41:20.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/verifier/reward-details.json new file mode 100644 index 00000000000..d8919707264 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.69, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.74, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (74%)." + }, + { + "name": "quality", + "value": 0.62, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (62%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.74, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.74, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.62, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.62, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/verifier/test-stdout.txt new file mode 100644 index 00000000000..e0418c41c7f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t27-gpt-4o/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Manage State with BLoC tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.69) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/agent/trajectory.json new file mode 100644 index 00000000000..30ec03ef466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..a13b90fd287 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o-mini +// Verification score: 0.46 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..a13b90fd287 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o-mini +// Verification score: 0.46 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..a13b90fd287 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o-mini +// Verification score: 0.46 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..a13b90fd287 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o-mini +// Verification score: 0.46 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..a13b90fd287 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gpt-4o-mini +// Verification score: 0.46 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/result.json new file mode 100644 index 00000000000..11749ccf097 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000020", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t32-gpt-4o-mini", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t32-gpt-4o-mini", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t32-gpt-4o-mini", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o-mini", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 57785, + "n_cache_tokens": 38850, + "n_output_tokens": 2050, + "cost_usd": 0.0099 + }, + "verifier_result": { + "rewards": { + "reward": 0.46 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:45:20.000000Z", + "finished_at": "2026-09-09T19:47:07.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:45:20.000000Z", + "finished_at": "2026-09-09T19:45:30.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:45:30.000000Z", + "finished_at": "2026-09-09T19:45:40.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:45:40.000000Z", + "finished_at": "2026-09-09T19:46:47.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:46:47.000000Z", + "finished_at": "2026-09-09T19:47:07.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/verifier/reward-details.json new file mode 100644 index 00000000000..55dd3df780b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.46, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.47, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (47%)." + }, + { + "name": "quality", + "value": 0.41, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (41%)." + }, + { + "name": "dx", + "value": 0.58, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (58%)." + } + ] + }, + "outcome": { + "score": 0.47, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.47, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 0.5, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.41, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.41, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.41, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.40599999999999997, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/verifier/test-stdout.txt new file mode 100644 index 00000000000..558f8a681b8 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t32-gpt-4o-mini/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Manage State with BLoC tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.46) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/agent/trajectory.json new file mode 100644 index 00000000000..8be22f7a92c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..30093d5d3ca --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-pro +// Verification score: 0.99 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..30093d5d3ca --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-pro +// Verification score: 0.99 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..30093d5d3ca --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-pro +// Verification score: 0.99 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..30093d5d3ca --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-pro +// Verification score: 0.99 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..30093d5d3ca --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-pro +// Verification score: 0.99 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/result.json new file mode 100644 index 00000000000..85492e48504 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000025", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t37-gemini-35-pro", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t37-gemini-35-pro", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t37-gemini-35-pro", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-pro", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 98941, + "n_cache_tokens": 72520, + "n_output_tokens": 4374, + "cost_usd": 0.1455 + }, + "verifier_result": { + "rewards": { + "reward": 0.99 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:52:25.000000Z", + "finished_at": "2026-09-09T19:55:31.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:52:25.000000Z", + "finished_at": "2026-09-09T19:52:35.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:52:35.000000Z", + "finished_at": "2026-09-09T19:52:45.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:52:45.000000Z", + "finished_at": "2026-09-09T19:55:07.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:55:07.000000Z", + "finished_at": "2026-09-09T19:55:31.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/verifier/reward-details.json new file mode 100644 index 00000000000..03ac93f9523 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.99, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (99%)." + }, + { + "name": "quality", + "value": 0.99, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (99%)." + }, + { + "name": "dx", + "value": 0.96, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (96%)." + } + ] + }, + "outcome": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.99, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.99, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.99, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.99, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.96, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/verifier/test-stdout.txt new file mode 100644 index 00000000000..812389f13f3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t37-gemini-35-pro/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Manage State with BLoC unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.99) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/agent/trajectory.json new file mode 100644 index 00000000000..8be22f7a92c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..3823adbe053 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-flash +// Verification score: 0.88 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..3823adbe053 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-flash +// Verification score: 0.88 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..3823adbe053 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-flash +// Verification score: 0.88 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..3823adbe053 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-flash +// Verification score: 0.88 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..3823adbe053 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: gemini-3.5-flash +// Verification score: 0.88 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/result.json new file mode 100644 index 00000000000..ae9ccf05e58 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-00000000002a", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t42-gemini-35-flash", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t42-gemini-35-flash", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t42-gemini-35-flash", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-flash", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 79096, + "n_cache_tokens": 54691, + "n_output_tokens": 2890, + "cost_usd": 0.0068 + }, + "verifier_result": { + "rewards": { + "reward": 0.88 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:59:30.000000Z", + "finished_at": "2026-09-09T20:01:48.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:59:30.000000Z", + "finished_at": "2026-09-09T19:59:40.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:59:40.000000Z", + "finished_at": "2026-09-09T19:59:50.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:59:50.000000Z", + "finished_at": "2026-09-09T20:01:14.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:01:14.000000Z", + "finished_at": "2026-09-09T20:01:48.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/verifier/reward-details.json new file mode 100644 index 00000000000..08f7f84449b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.88, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.87, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (87%)." + }, + { + "name": "quality", + "value": 0.88, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (88%)." + }, + { + "name": "dx", + "value": 0.93, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (93%)." + } + ] + }, + "outcome": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.87, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.88, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.88, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.88, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.93, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/verifier/test-stdout.txt new file mode 100644 index 00000000000..3b662f5f6aa --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t42-gemini-35-flash/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Manage State with BLoC unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.88) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite/exception.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite/exception.txt new file mode 100644 index 00000000000..5c554370e84 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite/exception.txt @@ -0,0 +1,7 @@ +Exception: AgentTimeoutError +Agent exceeded maximum timeout of 300.0 seconds during execution. + +Traceback (most recent call last): + File "harbor/trial/trial.py", line 1455, in _execute_agent + raise TimeoutError("Agent exceeded timeout of 300.0s") +TimeoutError: Agent exceeded timeout of 300.0s diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite/result.json new file mode 100644 index 00000000000..d04d1056eaa --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite/result.json @@ -0,0 +1,53 @@ +{ + "id": "11111111-2222-4333-8444-00000000002f", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t47-gemini-31-flash-lite", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t47-gemini-31-flash-lite", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t47-gemini-31-flash-lite", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent": { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "gemini-cli", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.1-flash-lite", + "provider": "google" + } + }, + "agent_result": null, + "verifier_result": null, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent exceeded maximum timeout of 300.0 seconds during execution.", + "exception_traceback": "Traceback (most recent call last):\n File \"harbor/trial/trial.py\", line 1455, in _execute_agent\n raise TimeoutError(\"Agent exceeded timeout of 300.0s\")\nTimeoutError: Agent exceeded timeout of 300.0s\n", + "occurred_at": "2026-09-09T20:12:00.000000Z" + }, + "started_at": "2026-09-09T20:06:35.000000Z", + "finished_at": "2026-09-09T20:12:27.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:06:35.000000Z", + "finished_at": "2026-09-09T20:06:45.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:06:45.000000Z", + "finished_at": "2026-09-09T20:06:55.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:06:55.000000Z", + "finished_at": "2026-09-09T20:12:00.000000Z" + }, + "verifier": null +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/agent/trajectory.json new file mode 100644 index 00000000000..8be22f7a92c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/bloc/counter_bloc.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..81191779358 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-r1 +// Verification score: 0.94 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..81191779358 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-r1 +// Verification score: 0.94 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..81191779358 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-r1 +// Verification score: 0.94 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..81191779358 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-r1 +// Verification score: 0.94 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..81191779358 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-r1 +// Verification score: 0.94 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/result.json new file mode 100644 index 00000000000..0a41651f3a7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000034", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t52-deepseek-r1", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t52-deepseek-r1", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t52-deepseek-r1", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-r1", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 99704, + "n_cache_tokens": 66373, + "n_output_tokens": 4602, + "cost_usd": 0.0649 + }, + "verifier_result": { + "rewards": { + "reward": 0.94 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:13:40.000000Z", + "finished_at": "2026-09-09T20:16:38.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:13:40.000000Z", + "finished_at": "2026-09-09T20:13:50.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:13:50.000000Z", + "finished_at": "2026-09-09T20:14:00.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:14:00.000000Z", + "finished_at": "2026-09-09T20:16:10.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:16:10.000000Z", + "finished_at": "2026-09-09T20:16:38.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/verifier/reward-details.json new file mode 100644 index 00000000000..08eb60a09d0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.93, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (93%)." + }, + { + "name": "quality", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (96%)." + }, + { + "name": "dx", + "value": 0.97, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (97%)." + } + ] + }, + "outcome": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.96, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.96, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.97, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/verifier/test-stdout.txt new file mode 100644 index 00000000000..c7104a04a96 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t52-deepseek-r1/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Manage State with BLoC unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.94) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/agent/trajectory.json new file mode 100644 index 00000000000..30ec03ef466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..3fc015db00c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-v3 +// Verification score: 0.67 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..3fc015db00c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-v3 +// Verification score: 0.67 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..3fc015db00c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-v3 +// Verification score: 0.67 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..3fc015db00c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-v3 +// Verification score: 0.67 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..3fc015db00c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-v3 +// Verification score: 0.67 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/result.json new file mode 100644 index 00000000000..ee628fdc97c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000039", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t57-deepseek-v3", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t57-deepseek-v3", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t57-deepseek-v3", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-v3", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 77341, + "n_cache_tokens": 57438, + "n_output_tokens": 3272, + "cost_usd": 0.0117 + }, + "verifier_result": { + "rewards": { + "reward": 0.67 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:20:45.000000Z", + "finished_at": "2026-09-09T20:23:34.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:20:45.000000Z", + "finished_at": "2026-09-09T20:20:55.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:20:55.000000Z", + "finished_at": "2026-09-09T20:21:05.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:21:05.000000Z", + "finished_at": "2026-09-09T20:23:00.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:23:00.000000Z", + "finished_at": "2026-09-09T20:23:34.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/verifier/reward-details.json new file mode 100644 index 00000000000..0a19f3a5b03 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.67, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.7, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (70%)." + }, + { + "name": "quality", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (61%)." + }, + { + "name": "dx", + "value": 0.63, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (63%)." + } + ] + }, + "outcome": { + "score": 0.7, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.7, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.61, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.61, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44099999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/verifier/test-stdout.txt new file mode 100644 index 00000000000..6e8531e5bf7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t57-deepseek-v3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Manage State with BLoC tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.67) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/agent/trajectory.json new file mode 100644 index 00000000000..30ec03ef466 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Manage State with BLoC in flutter-manage-state-with-bloc. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/bloc/counter_bloc.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/manifest.json new file mode 100644 index 00000000000..fd7bd663e35 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/manifest.json @@ -0,0 +1,32 @@ +[ + { + "source": "lib/bloc/counter_bloc.dart", + "destination": "artifacts/workspace/lib/bloc/counter_bloc.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_event.dart", + "destination": "artifacts/workspace/lib/bloc/counter_event.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/bloc/counter_state.dart", + "destination": "artifacts/workspace/lib/bloc/counter_state.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/main.dart", + "destination": "artifacts/workspace/lib/main.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/counter_bloc_test.dart", + "destination": "artifacts/workspace/test/counter_bloc_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/bloc/counter_bloc.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/bloc/counter_bloc.dart new file mode 100644 index 00000000000..6b7c1e6732e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/bloc/counter_bloc.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-coder-v2 +// Verification score: 0.59 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/bloc/counter_event.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/bloc/counter_event.dart new file mode 100644 index 00000000000..6b7c1e6732e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/bloc/counter_event.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-coder-v2 +// Verification score: 0.59 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/bloc/counter_state.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/bloc/counter_state.dart new file mode 100644 index 00000000000..6b7c1e6732e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/bloc/counter_state.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-coder-v2 +// Verification score: 0.59 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/main.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/main.dart new file mode 100644 index 00000000000..6b7c1e6732e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/lib/main.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-coder-v2 +// Verification score: 0.59 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/test/counter_bloc_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/test/counter_bloc_test.dart new file mode 100644 index 00000000000..6b7c1e6732e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/artifacts/workspace/test/counter_bloc_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-manage-state-with-bloc +// Model: deepseek-coder-v2 +// Verification score: 0.59 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/result.json new file mode 100644 index 00000000000..48ed3e56f6e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000003e", + "task_name": "google/flutter-manage-state-with-bloc", + "trial_name": "flutter-manage-state-with-bloc__t62-deepseek-coder-v2", + "trial_uri": "file:///workspace/jobs/mock/flutter-manage-state-with-bloc__t62-deepseek-coder-v2", + "task_id": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "config": { + "task": { + "path": "dataset/flutter-manage-state-with-bloc" + }, + "trial_name": "flutter-manage-state-with-bloc__t62-deepseek-coder-v2", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-coder-v2", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 74695, + "n_cache_tokens": 49868, + "n_output_tokens": 3051, + "cost_usd": 0.0113 + }, + "verifier_result": { + "rewards": { + "reward": 0.59 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:27:50.000000Z", + "finished_at": "2026-09-09T20:30:17.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:27:50.000000Z", + "finished_at": "2026-09-09T20:28:00.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:28:00.000000Z", + "finished_at": "2026-09-09T20:28:10.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:28:10.000000Z", + "finished_at": "2026-09-09T20:29:47.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:29:47.000000Z", + "finished_at": "2026-09-09T20:30:17.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/verifier/reward-details.json new file mode 100644 index 00000000000..ea8d89f23f5 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.62, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (62%)." + }, + { + "name": "quality", + "value": 0.53, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (53%)." + }, + { + "name": "dx", + "value": 0.64, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (64%)." + } + ] + }, + "outcome": { + "score": 0.62, + "kind": "aggregator", + "criteria": [ + { + "name": "flutter_build:bundle", + "value": 1.0, + "raw": true, + "weight": 0.2, + "description": "flutter build bundle succeeded." + }, + { + "name": "bloc_unit_tests", + "value": 0.62, + "raw": true, + "weight": 0.4, + "description": "Bloc unit tests for state transition validation." + }, + { + "name": "outcome_heuristics/bloc_provider_present", + "value": 1.0, + "raw": true, + "weight": 0.4, + "description": "BlocProvider wrapping widget tree in main.dart." + } + ] + }, + "quality": { + "score": 0.53, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.53, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.53, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44799999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.64, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.64, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/verifier/test-stdout.txt new file mode 100644 index 00000000000..595f1cac9e7 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-manage-state-with-bloc__t62-deepseek-coder-v2/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Manage State with BLoC tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.59) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..df46937f27b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..456d8d243d1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-7-sonnet +// Verification score: 0.93 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..456d8d243d1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-7-sonnet +// Verification score: 0.93 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..456d8d243d1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-7-sonnet +// Verification score: 0.93 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..456d8d243d1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-7-sonnet +// Verification score: 0.93 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/result.json new file mode 100644 index 00000000000..06b08510821 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000003", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t03-claude-3-7-sonnet", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t03-claude-3-7-sonnet", + "eval_key": "claude-code__claude-3-7-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-7-sonnet", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-7-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 99235, + "n_cache_tokens": 72794, + "n_output_tokens": 4854, + "cost_usd": 0.3705 + }, + "verifier_result": { + "rewards": { + "reward": 0.93 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:04:15.000000Z", + "finished_at": "2026-09-09T19:06:39.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:04:15.000000Z", + "finished_at": "2026-09-09T19:04:25.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:04:25.000000Z", + "finished_at": "2026-09-09T19:04:35.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:04:35.000000Z", + "finished_at": "2026-09-09T19:06:13.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:06:13.000000Z", + "finished_at": "2026-09-09T19:06:39.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..00db63cecb9 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.93, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (93%)." + }, + { + "name": "quality", + "value": 0.93, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (93%)." + }, + { + "name": "dx", + "value": 0.96, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (96%)." + } + ] + }, + "outcome": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.93, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.93, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.96, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.96, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.96, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..d385d3fcff0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t03-claude-3-7-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Offline SQLite Sync Repository unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.93) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/agent/trajectory.json new file mode 100644 index 00000000000..5cb802c452a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..28867d9bb17 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-5-sonnet +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..28867d9bb17 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-5-sonnet +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..28867d9bb17 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-5-sonnet +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..28867d9bb17 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-5-sonnet +// Verification score: 0.71 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/result.json new file mode 100644 index 00000000000..2a7c6d68b44 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000008", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t08-claude-3-5-sonnet", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t08-claude-3-5-sonnet", + "eval_key": "claude-code__claude-3-5-sonnet__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-sonnet", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-sonnet", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 85453, + "n_cache_tokens": 55797, + "n_output_tokens": 3960, + "cost_usd": 0.3158 + }, + "verifier_result": { + "rewards": { + "reward": 0.71 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:11:20.000000Z", + "finished_at": "2026-09-09T19:13:51.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:11:20.000000Z", + "finished_at": "2026-09-09T19:11:30.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:11:30.000000Z", + "finished_at": "2026-09-09T19:11:40.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:11:40.000000Z", + "finished_at": "2026-09-09T19:13:29.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:13:29.000000Z", + "finished_at": "2026-09-09T19:13:51.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/verifier/reward-details.json new file mode 100644 index 00000000000..9164cfd5664 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.71, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.76, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (76%)." + }, + { + "name": "quality", + "value": 0.65, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (65%)." + }, + { + "name": "dx", + "value": 0.61, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (61%)." + } + ] + }, + "outcome": { + "score": 0.76, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.76, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.65, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.65, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.427, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/verifier/test-stdout.txt new file mode 100644 index 00000000000..f01e220bf1c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t08-claude-3-5-sonnet/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Offline SQLite Sync Repository tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.71) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/agent/trajectory.json new file mode 100644 index 00000000000..5cb802c452a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..0884ea89954 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-5-haiku +// Verification score: 0.48 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..0884ea89954 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-5-haiku +// Verification score: 0.48 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..0884ea89954 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-5-haiku +// Verification score: 0.48 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..0884ea89954 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: claude-3-5-haiku +// Verification score: 0.48 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/result.json new file mode 100644 index 00000000000..2b6fe11dc71 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000000d", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t13-claude-3-5-haiku", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t13-claude-3-5-haiku", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t13-claude-3-5-haiku", + "eval_key": "claude-code__claude-3-5-haiku__adhoc", + "agent": { + "name": "claude-code", + "model_name": "anthropic/claude-3-5-haiku", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "claude-code", + "version": "0.3.0", + "model_info": { + "name": "claude-3-5-haiku", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 69726, + "n_cache_tokens": 54790, + "n_output_tokens": 2666, + "cost_usd": 0.0664 + }, + "verifier_result": { + "rewards": { + "reward": 0.48 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:18:25.000000Z", + "finished_at": "2026-09-09T19:21:23.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:18:25.000000Z", + "finished_at": "2026-09-09T19:18:35.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:18:35.000000Z", + "finished_at": "2026-09-09T19:18:45.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:18:45.000000Z", + "finished_at": "2026-09-09T19:20:51.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:20:51.000000Z", + "finished_at": "2026-09-09T19:21:23.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/verifier/reward-details.json new file mode 100644 index 00000000000..ff1e69bb732 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.48, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.49, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (49%)." + }, + { + "name": "quality", + "value": 0.41, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (41%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.49, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.49, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.49, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.41, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.41, + "raw": true, + "weight": 0.5, + "description": "6 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.41, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/verifier/test-stdout.txt new file mode 100644 index 00000000000..0d9fe03721f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t13-claude-3-5-haiku/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Offline SQLite Sync Repository tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.48) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/agent/trajectory.json new file mode 100644 index 00000000000..df46937f27b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..e2837bbc33a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: o3 +// Verification score: 0.9 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..e2837bbc33a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: o3 +// Verification score: 0.9 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..e2837bbc33a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: o3 +// Verification score: 0.9 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..e2837bbc33a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: o3 +// Verification score: 0.9 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/result.json new file mode 100644 index 00000000000..abbe843ff9b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000012", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t18-o3", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t18-o3", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t18-o3", + "eval_key": "codex-agent__o3__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/o3", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "o3", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 110044, + "n_cache_tokens": 77484, + "n_output_tokens": 5502, + "cost_usd": 0.6603 + }, + "verifier_result": { + "rewards": { + "reward": 0.9 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:25:30.000000Z", + "finished_at": "2026-09-09T19:28:17.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:25:30.000000Z", + "finished_at": "2026-09-09T19:25:40.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:25:40.000000Z", + "finished_at": "2026-09-09T19:25:50.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:25:50.000000Z", + "finished_at": "2026-09-09T19:27:48.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:27:48.000000Z", + "finished_at": "2026-09-09T19:28:17.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/verifier/reward-details.json new file mode 100644 index 00000000000..51715877f44 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.9, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (90%)." + }, + { + "name": "quality", + "value": 0.9, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (90%)." + }, + { + "name": "dx", + "value": 0.93, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (93%)." + } + ] + }, + "outcome": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.9, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.9, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.9, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.93, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.93, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.93, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/verifier/test-stdout.txt new file mode 100644 index 00000000000..31a255a50d6 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t18-o3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Offline SQLite Sync Repository unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.9) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/agent/trajectory.json new file mode 100644 index 00000000000..df46937f27b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..fb51b90c3ef --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-5 +// Verification score: 0.86 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..fb51b90c3ef --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-5 +// Verification score: 0.86 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..fb51b90c3ef --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-5 +// Verification score: 0.86 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..fb51b90c3ef --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-5 +// Verification score: 0.86 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/result.json new file mode 100644 index 00000000000..561b7872f99 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000017", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t23-gpt-5", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t23-gpt-5", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t23-gpt-5", + "eval_key": "codex-agent__gpt-5__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-5", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-5", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 88953, + "n_cache_tokens": 62264, + "n_output_tokens": 4144, + "cost_usd": 0.2638 + }, + "verifier_result": { + "rewards": { + "reward": 0.86 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:32:35.000000Z", + "finished_at": "2026-09-09T19:34:26.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:32:35.000000Z", + "finished_at": "2026-09-09T19:32:45.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:32:45.000000Z", + "finished_at": "2026-09-09T19:32:55.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:32:55.000000Z", + "finished_at": "2026-09-09T19:34:06.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:34:06.000000Z", + "finished_at": "2026-09-09T19:34:26.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/verifier/reward-details.json new file mode 100644 index 00000000000..a07b3b36408 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.86, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.86, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (86%)." + }, + { + "name": "quality", + "value": 0.85, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (85%)." + }, + { + "name": "dx", + "value": 0.94, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (94%)." + } + ] + }, + "outcome": { + "score": 0.86, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.86, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.85, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.85, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.85, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.94, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.94, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.94, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/verifier/test-stdout.txt new file mode 100644 index 00000000000..5f7eb79c331 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t23-gpt-5/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Offline SQLite Sync Repository unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.86) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/agent/trajectory.json new file mode 100644 index 00000000000..5cb802c452a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..66378e8ae20 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-4o +// Verification score: 0.65 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..66378e8ae20 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-4o +// Verification score: 0.65 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..66378e8ae20 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-4o +// Verification score: 0.65 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..66378e8ae20 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-4o +// Verification score: 0.65 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/result.json new file mode 100644 index 00000000000..cd75f06704f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000001c", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t28-gpt-4o", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t28-gpt-4o", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t28-gpt-4o", + "eval_key": "codex-agent__gpt-4o__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 79623, + "n_cache_tokens": 58576, + "n_output_tokens": 3353, + "cost_usd": 0.2326 + }, + "verifier_result": { + "rewards": { + "reward": 0.65 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:39:40.000000Z", + "finished_at": "2026-09-09T19:42:08.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:39:40.000000Z", + "finished_at": "2026-09-09T19:39:50.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:39:50.000000Z", + "finished_at": "2026-09-09T19:40:00.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:40:00.000000Z", + "finished_at": "2026-09-09T19:41:44.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:41:44.000000Z", + "finished_at": "2026-09-09T19:42:08.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/verifier/reward-details.json new file mode 100644 index 00000000000..3c053d6ea6a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.65, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.68, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (68%)." + }, + { + "name": "quality", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (60%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.68, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.68, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.68, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.6, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.6, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/verifier/test-stdout.txt new file mode 100644 index 00000000000..e6a4bd724c8 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t28-gpt-4o/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Offline SQLite Sync Repository tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.65) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/agent/trajectory.json new file mode 100644 index 00000000000..5cb802c452a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..752ed9d9301 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-4o-mini +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..752ed9d9301 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-4o-mini +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..752ed9d9301 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-4o-mini +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..752ed9d9301 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gpt-4o-mini +// Verification score: 0.39 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/result.json new file mode 100644 index 00000000000..34a33fe5294 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000021", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t33-gpt-4o-mini", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t33-gpt-4o-mini", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t33-gpt-4o-mini", + "eval_key": "codex-agent__gpt-4o-mini__adhoc", + "agent": { + "name": "codex-agent", + "model_name": "openai/gpt-4o-mini", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "codex-agent", + "version": "0.3.0", + "model_info": { + "name": "gpt-4o-mini", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 61681, + "n_cache_tokens": 47277, + "n_output_tokens": 2189, + "cost_usd": 0.0106 + }, + "verifier_result": { + "rewards": { + "reward": 0.39 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:46:45.000000Z", + "finished_at": "2026-09-09T19:48:35.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:46:45.000000Z", + "finished_at": "2026-09-09T19:46:55.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:46:55.000000Z", + "finished_at": "2026-09-09T19:47:05.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:47:05.000000Z", + "finished_at": "2026-09-09T19:48:10.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:48:10.000000Z", + "finished_at": "2026-09-09T19:48:35.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/verifier/reward-details.json new file mode 100644 index 00000000000..8829cdea1d0 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.39, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.38, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (38%)." + }, + { + "name": "quality", + "value": 0.33, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (33%)." + }, + { + "name": "dx", + "value": 0.63, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (63%)." + } + ] + }, + "outcome": { + "score": 0.38, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.38, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.38, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.33, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.33, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.33, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.63, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.44099999999999995, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.63, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/verifier/test-stdout.txt new file mode 100644 index 00000000000..655d97633da --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t33-gpt-4o-mini/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Offline SQLite Sync Repository tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.39) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/agent/trajectory.json new file mode 100644 index 00000000000..df46937f27b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..731864715f1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.5-pro +// Verification score: 0.91 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..731864715f1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.5-pro +// Verification score: 0.91 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..731864715f1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.5-pro +// Verification score: 0.91 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..731864715f1 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.5-pro +// Verification score: 0.91 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/result.json new file mode 100644 index 00000000000..dc13dbbe885 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000026", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t38-gemini-35-pro", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t38-gemini-35-pro", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t38-gemini-35-pro", + "eval_key": "antigravity-sdk__gemini-3.5-pro__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-pro", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-pro", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 94065, + "n_cache_tokens": 65762, + "n_output_tokens": 4159, + "cost_usd": 0.1384 + }, + "verifier_result": { + "rewards": { + "reward": 0.91 + } + }, + "exception_info": null, + "started_at": "2026-09-09T19:53:50.000000Z", + "finished_at": "2026-09-09T19:56:02.000000Z", + "environment_setup": { + "started_at": "2026-09-09T19:53:50.000000Z", + "finished_at": "2026-09-09T19:54:00.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T19:54:00.000000Z", + "finished_at": "2026-09-09T19:54:10.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T19:54:10.000000Z", + "finished_at": "2026-09-09T19:55:34.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T19:55:34.000000Z", + "finished_at": "2026-09-09T19:56:02.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/verifier/reward-details.json new file mode 100644 index 00000000000..04d7fc62e27 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.91, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (91%)." + }, + { + "name": "quality", + "value": 0.88, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (88%)." + }, + { + "name": "dx", + "value": 0.95, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (95%)." + } + ] + }, + "outcome": { + "score": 0.91, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.91, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.88, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.88, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.88, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.95, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.95, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.95, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/verifier/test-stdout.txt new file mode 100644 index 00000000000..6ccded534ae --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t38-gemini-35-pro/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Offline SQLite Sync Repository unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.91) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/agent/trajectory.json new file mode 100644 index 00000000000..df46937f27b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..9cd6f63b2ac --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.5-flash +// Verification score: 0.8 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..9cd6f63b2ac --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.5-flash +// Verification score: 0.8 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..9cd6f63b2ac --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.5-flash +// Verification score: 0.8 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..9cd6f63b2ac --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.5-flash +// Verification score: 0.8 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/result.json new file mode 100644 index 00000000000..da860b98fcb --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-00000000002b", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t43-gemini-35-flash", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t43-gemini-35-flash", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t43-gemini-35-flash", + "eval_key": "antigravity-sdk__gemini-3.5-flash__adhoc", + "agent": { + "name": "antigravity-sdk", + "model_name": "google/gemini-3.5-flash", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "antigravity-sdk", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.5-flash", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 83545, + "n_cache_tokens": 65913, + "n_output_tokens": 3053, + "cost_usd": 0.0072 + }, + "verifier_result": { + "rewards": { + "reward": 0.8 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:00:55.000000Z", + "finished_at": "2026-09-09T20:03:41.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:00:55.000000Z", + "finished_at": "2026-09-09T20:01:05.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:01:05.000000Z", + "finished_at": "2026-09-09T20:01:15.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:01:15.000000Z", + "finished_at": "2026-09-09T20:03:18.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:03:18.000000Z", + "finished_at": "2026-09-09T20:03:41.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/verifier/reward-details.json new file mode 100644 index 00000000000..b064ab8b759 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.8, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.8, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (80%)." + }, + { + "name": "quality", + "value": 0.77, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (77%)." + }, + { + "name": "dx", + "value": 0.9, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (90%)." + } + ] + }, + "outcome": { + "score": 0.8, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.8, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.77, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.77, + "raw": true, + "weight": 0.5, + "description": "2 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.77, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.9, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.9, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.9, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/verifier/test-stdout.txt new file mode 100644 index 00000000000..c42a930cda9 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t43-gemini-35-flash/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Offline SQLite Sync Repository unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.8) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/agent/trajectory.json new file mode 100644 index 00000000000..5cb802c452a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..9853b4c371e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.1-flash-lite +// Verification score: 0.33 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..9853b4c371e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.1-flash-lite +// Verification score: 0.33 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..9853b4c371e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.1-flash-lite +// Verification score: 0.33 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..9853b4c371e --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: gemini-3.1-flash-lite +// Verification score: 0.33 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/result.json new file mode 100644 index 00000000000..afc20a40176 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-000000000030", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t48-gemini-31-flash-lite", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t48-gemini-31-flash-lite", + "eval_key": "gemini-cli__gemini-3.1-flash-lite__adhoc", + "agent": { + "name": "gemini-cli", + "model_name": "google/gemini-3.1-flash-lite", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "gemini-cli", + "version": "0.3.0", + "model_info": { + "name": "gemini-3.1-flash-lite", + "provider": "google" + } + }, + "agent_result": { + "n_input_tokens": 54133, + "n_cache_tokens": 36431, + "n_output_tokens": 1874, + "cost_usd": 0.0015 + }, + "verifier_result": { + "rewards": { + "reward": 0.33 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:08:00.000000Z", + "finished_at": "2026-09-09T20:10:59.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:08:00.000000Z", + "finished_at": "2026-09-09T20:08:10.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:08:10.000000Z", + "finished_at": "2026-09-09T20:08:20.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:08:20.000000Z", + "finished_at": "2026-09-09T20:10:36.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:10:36.000000Z", + "finished_at": "2026-09-09T20:10:59.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/verifier/reward-details.json new file mode 100644 index 00000000000..f840666d44c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.33, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.32, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (32%)." + }, + { + "name": "quality", + "value": 0.27, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (27%)." + }, + { + "name": "dx", + "value": 0.59, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (59%)." + } + ] + }, + "outcome": { + "score": 0.32, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 0.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.32, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.32, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.27, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.27, + "raw": true, + "weight": 0.5, + "description": "7 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.27, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.59, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.413, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.59, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/verifier/test-stdout.txt new file mode 100644 index 00000000000..3a73e23899b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t48-gemini-31-flash-lite/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Offline SQLite Sync Repository tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.33) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/agent/trajectory.json new file mode 100644 index 00000000000..df46937f27b --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/agent/trajectory.json @@ -0,0 +1,52 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "read_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "replace_file_content", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "dart_analyze_files", + "parameters": { + "paths": [ + "lib/data/local_database.dart" + ] + }, + "result": "0 diagnostics found." + }, + { + "step": 6, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "All tests passed." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..61a5c7d1c2a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-r1 +// Verification score: 0.87 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..61a5c7d1c2a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-r1 +// Verification score: 0.87 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..61a5c7d1c2a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-r1 +// Verification score: 0.87 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..61a5c7d1c2a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-r1 +// Verification score: 0.87 (pass) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/result.json new file mode 100644 index 00000000000..3657aa77eec --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/result.json @@ -0,0 +1,66 @@ +{ + "id": "11111111-2222-4333-8444-000000000035", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t53-deepseek-r1", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t53-deepseek-r1", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t53-deepseek-r1", + "eval_key": "deepseek-cli__deepseek-r1__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-r1", + "skills": [ + "dart-flutter-skills" + ], + "mcp_servers": [ + { + "name": "dart" + } + ] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-r1", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 103946, + "n_cache_tokens": 78215, + "n_output_tokens": 4797, + "cost_usd": 0.0677 + }, + "verifier_result": { + "rewards": { + "reward": 0.87 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:15:05.000000Z", + "finished_at": "2026-09-09T20:17:33.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:15:05.000000Z", + "finished_at": "2026-09-09T20:15:15.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:15:15.000000Z", + "finished_at": "2026-09-09T20:15:25.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:15:25.000000Z", + "finished_at": "2026-09-09T20:17:01.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:17:01.000000Z", + "finished_at": "2026-09-09T20:17:33.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/verifier/reward-details.json new file mode 100644 index 00000000000..8f17fd3fd99 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.86, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (86%)." + }, + { + "name": "quality", + "value": 0.87, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (87%)." + }, + { + "name": "dx", + "value": 0.97, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (97%)." + } + ] + }, + "outcome": { + "score": 0.86, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.86, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.87, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.87, + "raw": true, + "weight": 0.5, + "description": "Zero static analysis issues found." + }, + { + "name": "structural_validation", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.87, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.97, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.97, + "raw": true, + "weight": 0.4, + "description": "Dart MCP tools (analyze_files, dtd, hot_reload) utilized efficiently." + }, + { + "name": "trajectory", + "value": 0.97, + "raw": true, + "weight": 0.3, + "description": "Linear problem-solving without excessive tool backtracking." + }, + { + "name": "error_recovery", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/verifier/test-stdout.txt new file mode 100644 index 00000000000..c71a3dd2be3 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t53-deepseek-r1/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: All Offline SQLite Sync Repository unit tests passed. +00:04 +4: Static analysis: 0 warnings, 0 errors. +00:05 +5: Rubric grader verification completed. +Overall result: PASS (Score: 0.87) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/agent/trajectory.json new file mode 100644 index 00000000000..5cb802c452a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..02af6c792ef --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-v3 +// Verification score: 0.64 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..02af6c792ef --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-v3 +// Verification score: 0.64 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..02af6c792ef --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-v3 +// Verification score: 0.64 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..02af6c792ef --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-v3 +// Verification score: 0.64 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/result.json new file mode 100644 index 00000000000..0909be1a19f --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000003a", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t58-deepseek-v3", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t58-deepseek-v3", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t58-deepseek-v3", + "eval_key": "deepseek-cli__deepseek-v3__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-v3", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-v3", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 79232, + "n_cache_tokens": 62070, + "n_output_tokens": 3352, + "cost_usd": 0.012 + }, + "verifier_result": { + "rewards": { + "reward": 0.64 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:22:10.000000Z", + "finished_at": "2026-09-09T20:24:11.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:22:10.000000Z", + "finished_at": "2026-09-09T20:22:20.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:22:20.000000Z", + "finished_at": "2026-09-09T20:22:30.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:22:30.000000Z", + "finished_at": "2026-09-09T20:23:48.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:23:48.000000Z", + "finished_at": "2026-09-09T20:24:11.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/verifier/reward-details.json new file mode 100644 index 00000000000..873592dac04 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.64, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.67, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (67%)." + }, + { + "name": "quality", + "value": 0.58, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (58%)." + }, + { + "name": "dx", + "value": 0.6, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (60%)." + } + ] + }, + "outcome": { + "score": 0.67, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.67, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.67, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.58, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.58, + "raw": true, + "weight": 0.5, + "description": "4 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.58, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.6, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.42, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/verifier/test-stdout.txt new file mode 100644 index 00000000000..665de822d79 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t58-deepseek-v3/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Offline SQLite Sync Repository tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.64) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/agent/trajectory.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/agent/trajectory.json new file mode 100644 index 00000000000..5cb802c452a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/agent/trajectory.json @@ -0,0 +1,41 @@ +{ + "steps": [ + { + "step": 1, + "type": "thought", + "content": "Analyzing task requirements for Offline SQLite Sync Repository in flutter-offline-sync-sqlite. Examining project root and pubspec.yaml." + }, + { + "step": 2, + "type": "tool_call", + "tool": "view_file", + "parameters": { + "path": "pubspec.yaml" + }, + "result": "Successfully read pubspec.yaml with dependencies." + }, + { + "step": 3, + "type": "thought", + "content": "Identifying target source files. Formulating implementation strategy adhering to Effective Dart and task constraints." + }, + { + "step": 4, + "type": "tool_call", + "tool": "write_to_file", + "parameters": { + "path": "lib/data/local_database.dart" + }, + "result": "File written successfully." + }, + { + "step": 5, + "type": "tool_call", + "tool": "run_command", + "parameters": { + "command": "dart test" + }, + "result": "Test run completed with warnings." + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/manifest.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/manifest.json new file mode 100644 index 00000000000..a02375f9a59 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/manifest.json @@ -0,0 +1,26 @@ +[ + { + "source": "lib/data/local_database.dart", + "destination": "artifacts/workspace/lib/data/local_database.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/repositories/sync_repository.dart", + "destination": "artifacts/workspace/lib/repositories/sync_repository.dart", + "type": "file", + "status": "ok" + }, + { + "source": "lib/models/sync_item.dart", + "destination": "artifacts/workspace/lib/models/sync_item.dart", + "type": "file", + "status": "ok" + }, + { + "source": "test/sync_repository_test.dart", + "destination": "artifacts/workspace/test/sync_repository_test.dart", + "type": "file", + "status": "ok" + } +] \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/lib/data/local_database.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/lib/data/local_database.dart new file mode 100644 index 00000000000..d76a5274960 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/lib/data/local_database.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-coder-v2 +// Verification score: 0.55 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/lib/models/sync_item.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/lib/models/sync_item.dart new file mode 100644 index 00000000000..d76a5274960 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/lib/models/sync_item.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-coder-v2 +// Verification score: 0.55 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/lib/repositories/sync_repository.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/lib/repositories/sync_repository.dart new file mode 100644 index 00000000000..d76a5274960 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/lib/repositories/sync_repository.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-coder-v2 +// Verification score: 0.55 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/test/sync_repository_test.dart b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/test/sync_repository_test.dart new file mode 100644 index 00000000000..d76a5274960 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/artifacts/workspace/test/sync_repository_test.dart @@ -0,0 +1,3 @@ +// Generated implementation for flutter-offline-sync-sqlite +// Model: deepseek-coder-v2 +// Verification score: 0.55 (partial) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/result.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/result.json new file mode 100644 index 00000000000..2f35c029f7a --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/result.json @@ -0,0 +1,60 @@ +{ + "id": "11111111-2222-4333-8444-00000000003f", + "task_name": "google/flutter-offline-sync-sqlite", + "trial_name": "flutter-offline-sync-sqlite__t63-deepseek-coder-v2", + "trial_uri": "file:///workspace/jobs/mock/flutter-offline-sync-sqlite__t63-deepseek-coder-v2", + "task_id": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "config": { + "task": { + "path": "dataset/flutter-offline-sync-sqlite" + }, + "trial_name": "flutter-offline-sync-sqlite__t63-deepseek-coder-v2", + "eval_key": "deepseek-cli__deepseek-coder-v2__adhoc", + "agent": { + "name": "deepseek-cli", + "model_name": "deepseek/deepseek-coder-v2", + "skills": [], + "mcp_servers": [] + } + }, + "agent_info": { + "name": "deepseek-cli", + "version": "0.3.0", + "model_info": { + "name": "deepseek-coder-v2", + "provider": "deepseek" + } + }, + "agent_result": { + "n_input_tokens": 71650, + "n_cache_tokens": 46814, + "n_output_tokens": 2927, + "cost_usd": 0.0109 + }, + "verifier_result": { + "rewards": { + "reward": 0.55 + } + }, + "exception_info": null, + "started_at": "2026-09-09T20:29:15.000000Z", + "finished_at": "2026-09-09T20:31:54.000000Z", + "environment_setup": { + "started_at": "2026-09-09T20:29:15.000000Z", + "finished_at": "2026-09-09T20:29:25.000000Z" + }, + "agent_setup": { + "started_at": "2026-09-09T20:29:25.000000Z", + "finished_at": "2026-09-09T20:29:35.000000Z" + }, + "agent_execution": { + "started_at": "2026-09-09T20:29:35.000000Z", + "finished_at": "2026-09-09T20:31:22.000000Z" + }, + "verifier": { + "started_at": "2026-09-09T20:31:22.000000Z", + "finished_at": "2026-09-09T20:31:54.000000Z" + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/verifier/reward-details.json b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/verifier/reward-details.json new file mode 100644 index 00000000000..7b871e8c076 --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/verifier/reward-details.json @@ -0,0 +1,110 @@ +{ + "reward": { + "score": 0.55, + "kind": "aggregator", + "criteria": [ + { + "name": "outcome", + "value": 0.57, + "raw": true, + "weight": 0.6, + "description": "Functional correctness and test completion (57%)." + }, + { + "name": "quality", + "value": 0.48, + "raw": true, + "weight": 0.3, + "description": "Static analysis, clean architecture, and idioms (48%)." + }, + { + "name": "dx", + "value": 0.61, + "raw": true, + "weight": 0.1, + "description": "Tool adherence and trajectory velocity (61%)." + } + ] + }, + "outcome": { + "score": 0.57, + "kind": "aggregator", + "criteria": [ + { + "name": "sqlite_migrations", + "value": 1.0, + "raw": true, + "weight": 0.3, + "description": "Schema migration and SQLite table initialization." + }, + { + "name": "sync_queue_tests", + "value": 0.57, + "raw": true, + "weight": 0.4, + "description": "Background sync queue and mutation synchronization." + }, + { + "name": "conflict_resolution", + "value": 0.57, + "raw": true, + "weight": 0.3, + "description": "Conflict resolution strategy using timestamp heuristics." + } + ] + }, + "quality": { + "score": 0.48, + "kind": "aggregator", + "criteria": [ + { + "name": "StaticAnalysisGrader", + "value": 0.48, + "raw": true, + "weight": 0.5, + "description": "5 linter warnings detected." + }, + { + "name": "structural_validation", + "value": 0.6, + "raw": true, + "weight": 0.3, + "description": "Clean separation of concerns and architectural layers." + }, + { + "name": "idiomatic_review", + "value": 0.48, + "raw": true, + "weight": 0.2, + "description": "Dart 3 idioms (pattern matching, records, sealed classes) adherence." + } + ] + }, + "dx": { + "score": 0.61, + "kind": "aggregator", + "criteria": [ + { + "name": "tool_usage", + "value": 0.427, + "raw": true, + "weight": 0.4, + "description": "Standard command-line execution without language server telemetry." + }, + { + "name": "trajectory", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Multiple retry cycles before settling on solution." + }, + { + "name": "error_recovery", + "value": 0.61, + "raw": true, + "weight": 0.3, + "description": "Fast turnaround from initial compile warnings to clean build." + } + ] + } +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/verifier/test-stdout.txt b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/verifier/test-stdout.txt new file mode 100644 index 00000000000..0afa572e17c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/flutter-offline-sync-sqlite__t63-deepseek-coder-v2/verifier/test-stdout.txt @@ -0,0 +1,7 @@ +00:00 +0: loading tests/graders.dart +00:01 +1: Environment setup verification passed. +00:02 +2: Task codebase compilation check. +00:03 +3: 3/5 Offline SQLite Sync Repository tests passed. +00:04 +3 -1: 2 edge-case assertions failed. +00:05 +4: Static analysis completed with minor hints. +Overall result: PARTIAL (Score: 0.55) diff --git a/sites/www/lib/src/data/raw_flutterbench_data/lock.json b/sites/www/lib/src/data/raw_flutterbench_data/lock.json new file mode 100644 index 00000000000..6499770dfda --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/lock.json @@ -0,0 +1,661 @@ +{ + "schema_version": 3, + "created_at": "2026-09-09T19:00:00.000000Z", + "harbor": { + "version": "0.22.0", + "is_editable": false + }, + "n_concurrent_trials": 4, + "trials": [ + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task1" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task2" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task3" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task4" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task5" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task6" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task7" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task8" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task9" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task10" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task11" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task12" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task13" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task14" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task15" + }, + "agent": { + "name": "claude-code" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task16" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task17" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task18" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task19" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task20" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task21" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task22" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task23" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task24" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task25" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task26" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task27" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task28" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task29" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task30" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task31" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task32" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task33" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task34" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task35" + }, + "agent": { + "name": "codex-agent" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task36" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task37" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task38" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task39" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task40" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task41" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task42" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task43" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task44" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task45" + }, + "agent": { + "name": "antigravity-sdk" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task46" + }, + "agent": { + "name": "gemini-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task47" + }, + "agent": { + "name": "gemini-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task48" + }, + "agent": { + "name": "gemini-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task49" + }, + "agent": { + "name": "gemini-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task50" + }, + "agent": { + "name": "gemini-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task51" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task52" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task53" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task54" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task55" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task56" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task57" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task58" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task59" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task60" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "dart-build-cli-app", + "digest": "sha256:task61" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-manage-state-with-bloc", + "digest": "sha256:task62" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-offline-sync-sqlite", + "digest": "sha256:task63" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-adaptive-material-cupertino", + "digest": "sha256:task64" + }, + "agent": { + "name": "deepseek-cli" + } + }, + { + "schema_version": 2, + "task": { + "name": "flutter-custom-render-object", + "digest": "sha256:task65" + }, + "agent": { + "name": "deepseek-cli" + } + } + ] +} \ No newline at end of file diff --git a/sites/www/lib/src/data/raw_flutterbench_data/result.json b/sites/www/lib/src/data/raw_flutterbench_data/result.json new file mode 100644 index 00000000000..6174fcd339c --- /dev/null +++ b/sites/www/lib/src/data/raw_flutterbench_data/result.json @@ -0,0 +1,485 @@ +{ + "id": "9b1deb4d-3b7d-4bad-9bdd-2b0d7b3dcb6d", + "started_at": "2026-09-09T19:00:00.000000", + "updated_at": "2026-09-09T22:30:00.000000", + "finished_at": "2026-09-09T22:30:00.000000", + "n_total_trials": 65, + "stats": { + "n_completed_trials": 62, + "n_errored_trials": 3, + "n_running_trials": 0, + "n_pending_trials": 0, + "n_cancelled_trials": 0, + "n_retries": 0, + "evals": { + "claude-code__claude-3-7-sonnet__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.89, + "mean": 0.89, + "median": 0.93, + "min": 0.71, + "max": 0.98 + } + ], + "pass_at_k": { + "1": 0.8 + }, + "reward_stats": { + "reward": { + "0.97": [ + "dart-build-cli-app__t01-claude-3-7-sonnet" + ], + "0.98": [ + "flutter-manage-state-with-bloc__t02-claude-3-7-sonnet" + ], + "0.93": [ + "flutter-offline-sync-sqlite__t03-claude-3-7-sonnet" + ], + "0.85": [ + "flutter-adaptive-material-cupertino__t04-claude-3-7-sonnet" + ], + "0.71": [ + "flutter-custom-render-object__t05-claude-3-7-sonnet" + ] + } + }, + "exception_stats": {} + }, + "claude-code__claude-3-5-sonnet__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.68, + "mean": 0.68, + "median": 0.71, + "min": 0.52, + "max": 0.79 + } + ], + "pass_at_k": { + "1": 0.0 + }, + "reward_stats": { + "reward": { + "0.79": [ + "dart-build-cli-app__t06-claude-3-5-sonnet" + ], + "0.75": [ + "flutter-manage-state-with-bloc__t07-claude-3-5-sonnet" + ], + "0.71": [ + "flutter-offline-sync-sqlite__t08-claude-3-5-sonnet" + ], + "0.61": [ + "flutter-adaptive-material-cupertino__t09-claude-3-5-sonnet" + ], + "0.52": [ + "flutter-custom-render-object__t10-claude-3-5-sonnet" + ] + } + }, + "exception_stats": {} + }, + "claude-code__claude-3-5-haiku__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.48, + "mean": 0.48, + "median": 0.48, + "min": 0.36, + "max": 0.56 + } + ], + "pass_at_k": { + "1": 0.0 + }, + "reward_stats": { + "reward": { + "0.56": [ + "dart-build-cli-app__t11-claude-3-5-haiku" + ], + "0.55": [ + "flutter-manage-state-with-bloc__t12-claude-3-5-haiku" + ], + "0.48": [ + "flutter-offline-sync-sqlite__t13-claude-3-5-haiku" + ], + "0.45": [ + "flutter-adaptive-material-cupertino__t14-claude-3-5-haiku" + ], + "0.36": [ + "flutter-custom-render-object__t15-claude-3-5-haiku" + ] + } + }, + "exception_stats": {} + }, + "codex-agent__o3__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.88, + "mean": 0.88, + "median": 0.9, + "min": 0.73, + "max": 0.97 + } + ], + "pass_at_k": { + "1": 0.8 + }, + "reward_stats": { + "reward": { + "0.97": [ + "dart-build-cli-app__t16-o3", + "flutter-manage-state-with-bloc__t17-o3" + ], + "0.90": [ + "flutter-offline-sync-sqlite__t18-o3" + ], + "0.85": [ + "flutter-adaptive-material-cupertino__t19-o3" + ], + "0.73": [ + "flutter-custom-render-object__t20-o3" + ] + } + }, + "exception_stats": {} + }, + "codex-agent__gpt-5__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.87, + "mean": 0.87, + "median": 0.86, + "min": 0.72, + "max": 0.98 + } + ], + "pass_at_k": { + "1": 0.8 + }, + "reward_stats": { + "reward": { + "0.98": [ + "dart-build-cli-app__t21-gpt-5" + ], + "0.97": [ + "flutter-manage-state-with-bloc__t22-gpt-5" + ], + "0.86": [ + "flutter-offline-sync-sqlite__t23-gpt-5" + ], + "0.84": [ + "flutter-adaptive-material-cupertino__t24-gpt-5" + ], + "0.72": [ + "flutter-custom-render-object__t25-gpt-5" + ] + } + }, + "exception_stats": {} + }, + "codex-agent__gpt-4o__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.62, + "mean": 0.62, + "median": 0.65, + "min": 0.47, + "max": 0.71 + } + ], + "pass_at_k": { + "1": 0.0 + }, + "reward_stats": { + "reward": { + "0.71": [ + "dart-build-cli-app__t26-gpt-4o" + ], + "0.69": [ + "flutter-manage-state-with-bloc__t27-gpt-4o" + ], + "0.65": [ + "flutter-offline-sync-sqlite__t28-gpt-4o" + ], + "0.60": [ + "flutter-adaptive-material-cupertino__t29-gpt-4o" + ], + "0.47": [ + "flutter-custom-render-object__t30-gpt-4o" + ] + } + }, + "exception_stats": {} + }, + "codex-agent__gpt-4o-mini__adhoc": { + "n_trials": 5, + "n_errors": 1, + "metrics": [ + { + "reward": 0.43, + "mean": 0.43, + "median": 0.46, + "min": 0.39, + "max": 0.48 + } + ], + "pass_at_k": { + "1": 0.0 + }, + "reward_stats": { + "reward": { + "0.48": [ + "dart-build-cli-app__t31-gpt-4o-mini" + ], + "0.46": [ + "flutter-manage-state-with-bloc__t32-gpt-4o-mini" + ], + "0.39": [ + "flutter-offline-sync-sqlite__t33-gpt-4o-mini", + "flutter-adaptive-material-cupertino__t34-gpt-4o-mini" + ] + } + }, + "exception_stats": { + "AgentTimeoutError": [ + "flutter-custom-render-object__t35-gpt-4o-mini" + ] + } + }, + "antigravity-sdk__gemini-3.5-pro__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.89, + "mean": 0.89, + "median": 0.91, + "min": 0.69, + "max": 0.99 + } + ], + "pass_at_k": { + "1": 0.8 + }, + "reward_stats": { + "reward": { + "0.98": [ + "dart-build-cli-app__t36-gemini-35-pro" + ], + "0.99": [ + "flutter-manage-state-with-bloc__t37-gemini-35-pro" + ], + "0.91": [ + "flutter-offline-sync-sqlite__t38-gemini-35-pro" + ], + "0.87": [ + "flutter-adaptive-material-cupertino__t39-gemini-35-pro" + ], + "0.69": [ + "flutter-custom-render-object__t40-gemini-35-pro" + ] + } + }, + "exception_stats": {} + }, + "antigravity-sdk__gemini-3.5-flash__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.81, + "mean": 0.81, + "median": 0.8, + "min": 0.65, + "max": 0.93 + } + ], + "pass_at_k": { + "1": 0.6 + }, + "reward_stats": { + "reward": { + "0.93": [ + "dart-build-cli-app__t41-gemini-35-flash" + ], + "0.88": [ + "flutter-manage-state-with-bloc__t42-gemini-35-flash" + ], + "0.80": [ + "flutter-offline-sync-sqlite__t43-gemini-35-flash" + ], + "0.78": [ + "flutter-adaptive-material-cupertino__t44-gemini-35-flash" + ], + "0.65": [ + "flutter-custom-render-object__t45-gemini-35-flash" + ] + } + }, + "exception_stats": {} + }, + "gemini-cli__gemini-3.1-flash-lite__adhoc": { + "n_trials": 5, + "n_errors": 2, + "metrics": [ + { + "reward": 0.33, + "mean": 0.33, + "median": 0.33, + "min": 0.28, + "max": 0.39 + } + ], + "pass_at_k": { + "1": 0.0 + }, + "reward_stats": { + "reward": { + "0.39": [ + "dart-build-cli-app__t46-gemini-31-flash-lite" + ], + "0.33": [ + "flutter-offline-sync-sqlite__t48-gemini-31-flash-lite" + ], + "0.28": [ + "flutter-adaptive-material-cupertino__t49-gemini-31-flash-lite" + ] + } + }, + "exception_stats": { + "AgentTimeoutError": [ + "flutter-manage-state-with-bloc__t47-gemini-31-flash-lite", + "flutter-custom-render-object__t50-gemini-31-flash-lite" + ] + } + }, + "deepseek-cli__deepseek-r1__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.86, + "mean": 0.86, + "median": 0.87, + "min": 0.72, + "max": 0.99 + } + ], + "pass_at_k": { + "1": 0.6 + }, + "reward_stats": { + "reward": { + "0.99": [ + "dart-build-cli-app__t51-deepseek-r1" + ], + "0.94": [ + "flutter-manage-state-with-bloc__t52-deepseek-r1" + ], + "0.87": [ + "flutter-offline-sync-sqlite__t53-deepseek-r1" + ], + "0.78": [ + "flutter-adaptive-material-cupertino__t54-deepseek-r1" + ], + "0.72": [ + "flutter-custom-render-object__t55-deepseek-r1" + ] + } + }, + "exception_stats": {} + }, + "deepseek-cli__deepseek-v3__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.62, + "mean": 0.62, + "median": 0.64, + "min": 0.49, + "max": 0.71 + } + ], + "pass_at_k": { + "1": 0.0 + }, + "reward_stats": { + "reward": { + "0.71": [ + "dart-build-cli-app__t56-deepseek-v3" + ], + "0.67": [ + "flutter-manage-state-with-bloc__t57-deepseek-v3" + ], + "0.64": [ + "flutter-offline-sync-sqlite__t58-deepseek-v3" + ], + "0.59": [ + "flutter-adaptive-material-cupertino__t59-deepseek-v3" + ], + "0.49": [ + "flutter-custom-render-object__t60-deepseek-v3" + ] + } + }, + "exception_stats": {} + }, + "deepseek-cli__deepseek-coder-v2__adhoc": { + "n_trials": 5, + "n_errors": 0, + "metrics": [ + { + "reward": 0.53, + "mean": 0.53, + "median": 0.55, + "min": 0.42, + "max": 0.61 + } + ], + "pass_at_k": { + "1": 0.0 + }, + "reward_stats": { + "reward": { + "0.61": [ + "dart-build-cli-app__t61-deepseek-coder-v2" + ], + "0.59": [ + "flutter-manage-state-with-bloc__t62-deepseek-coder-v2" + ], + "0.55": [ + "flutter-offline-sync-sqlite__t63-deepseek-coder-v2" + ], + "0.50": [ + "flutter-adaptive-material-cupertino__t64-deepseek-coder-v2" + ], + "0.42": [ + "flutter-custom-render-object__t65-deepseek-coder-v2" + ] + } + }, + "exception_stats": {} + } + }, + "n_input_tokens": 5284692, + "n_cache_tokens": 3823561, + "n_output_tokens": 229186, + "cost_usd": 10.6607 + } +} \ No newline at end of file diff --git a/sites/www/lib/src/models/content/flutterbench_content.dart b/sites/www/lib/src/models/content/flutterbench_content.dart new file mode 100644 index 00000000000..361c644e815 --- /dev/null +++ b/sites/www/lib/src/models/content/flutterbench_content.dart @@ -0,0 +1,531 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:dart_mappable/dart_mappable.dart'; + +part 'flutterbench_content.mapper.dart'; + +/// Top-level FlutterBench job summary data loaded from `data.flutterbench.job`. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchJobData with FlutterBenchJobDataMappable { + const FlutterBenchJobData({ + required this.id, + required this.startedAt, + required this.finishedAt, + required this.nTotalTrials, + required this.nCompletedTrials, + required this.nErroredTrials, + required this.costUsd, + required this.nInputTokens, + required this.nCacheTokens, + required this.nOutputTokens, + required this.topModelName, + required this.topModelReward, + required this.overallAverageReward, + required this.evals, + }); + + final String id; + final String startedAt; + final String finishedAt; + final int nTotalTrials; + final int nCompletedTrials; + final int nErroredTrials; + final double costUsd; + final int nInputTokens; + final int nCacheTokens; + final int nOutputTokens; + final String topModelName; + final double topModelReward; + final double overallAverageReward; + final List evals; + + /// Parses FlutterBench job data from JSON/YAML map. + static FlutterBenchJobData fromJson(Map json) => + FlutterBenchJobDataMapper.fromMap(json); +} + +/// A single evaluation configuration row on the leaderboard. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchEvalItem with FlutterBenchEvalItemMappable { + const FlutterBenchEvalItem({ + required this.evalKey, + required this.agentName, + required this.modelName, + required this.modelShortName, + required this.provider, + required this.variant, + required this.nTrials, + required this.nErrors, + required this.meanReward, + required this.minReward, + required this.maxReward, + required this.medianReward, + required this.passAt1, + required this.costUsd, + required this.inputTokens, + required this.outputTokens, + required this.hasDartTooling, + this.outcomeScore, + this.qualityScore, + this.dxScore, + this.bestCujs = const [], + this.worstCujs = const [], + }); + + final String evalKey; + final String agentName; + final String modelName; + final String modelShortName; + final String provider; + final String variant; + final int nTrials; + final int nErrors; + final double meanReward; + final double minReward; + final double maxReward; + final double medianReward; + @MappableField(key: 'pass_at_1') + final double passAt1; + final double costUsd; + final int inputTokens; + final int outputTokens; + final bool hasDartTooling; + final double? outcomeScore; + final double? qualityScore; + final double? dxScore; + final List bestCujs; + final List worstCujs; + + /// Parses an evaluation item from JSON/YAML map. + static FlutterBenchEvalItem fromJson(Map json) => + FlutterBenchEvalItemMapper.fromMap(json); +} + +/// Summary of a CUJ performance for a model. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchCujSummary with FlutterBenchCujSummaryMappable { + const FlutterBenchCujSummary({ + required this.taskSlug, + required this.taskName, + this.reward, + required this.status, + }); + + final String taskSlug; + final String taskName; + final double? reward; + final String status; + + /// Parses a CUJ summary item from JSON/YAML map. + static FlutterBenchCujSummary fromJson(Map json) => + FlutterBenchCujSummaryMapper.fromMap(json); +} + +/// Tasks collection data loaded from `data.flutterbench.tasks`. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTasksData with FlutterBenchTasksDataMappable { + const FlutterBenchTasksData({required this.tasks}); + + final List tasks; + + /// Parses tasks data from JSON/YAML map. + static FlutterBenchTasksData fromJson(Map json) => + FlutterBenchTasksDataMapper.fromMap(json); +} + +/// An individual task / Critical User Journey (CUJ). +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTaskItem with FlutterBenchTaskItemMappable { + const FlutterBenchTaskItem({ + required this.slug, + required this.taskName, + required this.displayName, + required this.category, + required this.description, + this.trials = const [], + this.scoresByEval = const {}, + }); + + final String slug; + final String taskName; + final String displayName; + final String category; + final String description; + final List trials; + final Map scoresByEval; + + /// Parses a task item from JSON/YAML map. + static FlutterBenchTaskItem fromJson(Map json) => + FlutterBenchTaskItemMapper.fromMap(json); +} + +/// Summary of a trial inside a task. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTrialSummary with FlutterBenchTrialSummaryMappable { + const FlutterBenchTrialSummary({ + required this.trialName, + required this.status, + this.reward, + required this.modelName, + required this.modelShortName, + required this.agentName, + required this.hasDartTooling, + this.exceptionType, + }); + + final String trialName; + final String status; + final double? reward; + final String modelName; + final String modelShortName; + final String agentName; + final bool hasDartTooling; + final String? exceptionType; + + /// Parses a trial summary item from JSON/YAML map. + static FlutterBenchTrialSummary fromJson(Map json) => + FlutterBenchTrialSummaryMapper.fromMap(json); +} + +/// Trials collection loaded from `data.flutterbench.trials`. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTrialsData with FlutterBenchTrialsDataMappable { + const FlutterBenchTrialsData({required this.trials}); + + final List trials; + + /// Parses trials collection from JSON/YAML map. + static FlutterBenchTrialsData fromJson(Map json) => + FlutterBenchTrialsDataMapper.fromMap(json); +} + +/// Detailed single trial information. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTrialDetail with FlutterBenchTrialDetailMappable { + const FlutterBenchTrialDetail({ + required this.trialName, + required this.taskName, + required this.taskSlug, + required this.agentName, + required this.modelName, + required this.modelShortName, + required this.provider, + this.skills = const [], + this.mcpServers = const [], + required this.hasDartTooling, + required this.status, + this.reward, + this.exceptionType, + this.exceptionMessage, + this.exceptionTraceback, + this.durations = const {}, + required this.inputTokens, + required this.cacheTokens, + required this.outputTokens, + required this.costUsd, + this.rewardTree, + this.diagnosticTree = const {}, + this.trajectory, + this.artifacts = const [], + this.testStdout, + this.exceptionLog, + }); + + final String trialName; + final String taskName; + final String taskSlug; + final String agentName; + final String modelName; + final String modelShortName; + final String provider; + final List skills; + final List mcpServers; + final bool hasDartTooling; + final String status; // 'pass', 'partial', 'fail', 'error' + final double? reward; + final String? exceptionType; + final String? exceptionMessage; + final String? exceptionTraceback; + final Map durations; + final int inputTokens; + final int cacheTokens; + final int outputTokens; + final double costUsd; + final Map? rewardTree; + final Map diagnosticTree; + final List>? trajectory; + final List> artifacts; + final String? testStdout; + final String? exceptionLog; + + /// Parses a trial detail from JSON/YAML map. + static FlutterBenchTrialDetail fromJson(Map json) => + FlutterBenchTrialDetailMapper.fromMap(json); +} + +/// Methodology page content loaded from `data.flutterbench.methodology`. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchMethodologyData with FlutterBenchMethodologyDataMappable { + const FlutterBenchMethodologyData({ + required this.overview, + this.cujExample = const [], + this.taskSpecifications = const [], + required this.taskAnatomy, + this.evaluationMatrix = const {}, + this.dimensions = const [], + this.graderMatrix = const {}, + required this.graderTiers, + required this.diagnosticTelemetry, + this.reliability = const {}, + this.scoreTriage = const {}, + required this.rootCauseAudits, + required this.transparency, + }); + + final FlutterBenchMethodologyOverview overview; + final List> cujExample; + final List> taskSpecifications; + final FlutterBenchTaskAnatomy taskAnatomy; + final Map evaluationMatrix; + final List> dimensions; + final Map graderMatrix; + final FlutterBenchTableSection graderTiers; + final FlutterBenchTableSection diagnosticTelemetry; + final Map reliability; + final Map scoreTriage; + final FlutterBenchItemList rootCauseAudits; + final FlutterBenchTransparency transparency; + + /// Parses methodology page data from JSON/YAML map. + static FlutterBenchMethodologyData fromJson(Map json) => + FlutterBenchMethodologyDataMapper.fromMap(json); +} + +/// The "Overview" chapter's lead paragraph and 4-row component table. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchMethodologyOverview + with FlutterBenchMethodologyOverviewMappable { + const FlutterBenchMethodologyOverview({ + required this.leadText, + required this.rows, + }); + + final String leadText; + final List rows; + + /// Parses overview data from JSON/YAML map. + static FlutterBenchMethodologyOverview fromJson( + Map json, + ) => FlutterBenchMethodologyOverviewMapper.fromMap(json); +} + +/// A generic labeled table row used by the Overview, Grader Tiers, and +/// Diagnostic Telemetry tables. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTableRow with FlutterBenchTableRowMappable { + const FlutterBenchTableRow({ + required this.label, + this.detail, + required this.description, + this.anchor, + }); + + final String label; + final String? detail; + final String description; + final String? anchor; + + /// Parses a table row from JSON/YAML map. + static FlutterBenchTableRow fromJson(Map json) => + FlutterBenchTableRowMapper.fromMap(json); +} + +/// A simple table section made up of [FlutterBenchTableRow]s, reused for the +/// Grader Implementation Tiers and Diagnostic Telemetry tables. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTableSection with FlutterBenchTableSectionMappable { + const FlutterBenchTableSection({required this.rows}); + + final List rows; + + /// Parses a table section from JSON/YAML map. + static FlutterBenchTableSection fromJson(Map json) => + FlutterBenchTableSectionMapper.fromMap(json); +} + +/// A label/detail pair, reused for the Human Root-Cause Audits list. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchLabeledDetail with FlutterBenchLabeledDetailMappable { + const FlutterBenchLabeledDetail({required this.label, required this.detail}); + + final String label; + final String detail; + + /// Parses a labeled detail item from JSON/YAML map. + static FlutterBenchLabeledDetail fromJson(Map json) => + FlutterBenchLabeledDetailMapper.fromMap(json); +} + +/// A list of [FlutterBenchLabeledDetail] items, used for the Human +/// Root-Cause Audits section. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchItemList with FlutterBenchItemListMappable { + const FlutterBenchItemList({required this.items}); + + final List items; + + /// Parses an item list from JSON/YAML map. + static FlutterBenchItemList fromJson(Map json) => + FlutterBenchItemListMapper.fromMap(json); +} + +/// The Transparency & Reproducibility chapter's Harbor CLI example values. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTransparency with FlutterBenchTransparencyMappable { + const FlutterBenchTransparency({required this.harborExample}); + + final FlutterBenchHarborExample harborExample; + + /// Parses transparency data from JSON/YAML map. + static FlutterBenchTransparency fromJson(Map json) => + FlutterBenchTransparencyMapper.fromMap(json); +} + +/// Values interpolated into the Harbor CLI reproduction example. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchHarborExample with FlutterBenchHarborExampleMappable { + const FlutterBenchHarborExample({ + required this.task, + required this.agent, + required this.model, + required this.mcp, + }); + + final String task; + final String agent; + final String model; + final String mcp; + + /// Parses a Harbor example from JSON/YAML map. + static FlutterBenchHarborExample fromJson(Map json) => + FlutterBenchHarborExampleMapper.fromMap(json); +} + +/// The "Interactive task anatomy" file-tree section. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTaskAnatomy with FlutterBenchTaskAnatomyMappable { + const FlutterBenchTaskAnatomy({ + required this.introText, + required this.rootId, + required this.rootLabel, + required this.tree, + }); + + final String introText; + final String rootId; + final String rootLabel; + final List tree; + + /// Parses task anatomy data from JSON/YAML map. + static FlutterBenchTaskAnatomy fromJson(Map json) => + FlutterBenchTaskAnatomyMapper.fromMap(json); +} + +/// A single file or folder node in the task anatomy file tree. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchTaskTreeNode with FlutterBenchTaskTreeNodeMappable { + const FlutterBenchTaskTreeNode({ + required this.type, + required this.id, + required this.label, + this.subtitle, + this.badge, + this.badgeColor, + this.isDefaultPage = false, + this.startsClosed = true, + this.body, + this.code, + this.children = const [], + }); + + final String type; // 'folder' | 'file' + final String id; + final String label; + final String? subtitle; + final String? badge; + final String? badgeColor; + final bool isDefaultPage; + final bool startsClosed; + final String? body; + final FlutterBenchCodeSample? code; + final List children; + + /// Parses a task tree node from JSON/YAML map. + static FlutterBenchTaskTreeNode fromJson(Map json) => + FlutterBenchTaskTreeNodeMapper.fromMap(json); +} + +/// A fenced code sample attached to a task tree node's detail body. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchCodeSample with FlutterBenchCodeSampleMappable { + const FlutterBenchCodeSample({required this.lang, required this.text}); + + final String lang; + final String text; + + /// Parses a code sample from JSON/YAML map. + static FlutterBenchCodeSample fromJson(Map json) => + FlutterBenchCodeSampleMapper.fromMap(json); +} + +/// The critical user journey (CUJ) catalog loaded from `data.flutterbench.cujs`. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchCujsData with FlutterBenchCujsDataMappable { + const FlutterBenchCujsData({required this.cujs}); + + final List cujs; + + /// Parses the CUJ catalog from JSON/YAML map. + static FlutterBenchCujsData fromJson(Map json) => + FlutterBenchCujsDataMapper.fromMap(json); +} + +/// A single critical user journey and the tasks that make it up. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchCujItem with FlutterBenchCujItemMappable { + const FlutterBenchCujItem({ + required this.id, + required this.goal, + required this.persona, + this.tasks = const [], + }); + + final int id; + final String goal; + final String persona; + final List tasks; + + /// Parses a CUJ item from JSON/YAML map. + static FlutterBenchCujItem fromJson(Map json) => + FlutterBenchCujItemMapper.fromMap(json); +} + +/// A concrete task contributing to a [FlutterBenchCujItem]'s goal. +@MappableClass(caseStyle: CaseStyle.snakeCase) +class FlutterBenchCujTaskItem with FlutterBenchCujTaskItemMappable { + const FlutterBenchCujTaskItem({ + required this.id, + required this.name, + required this.task, + }); + + final int id; + final String name; + final String task; + + /// Parses a CUJ task item from JSON/YAML map. + static FlutterBenchCujTaskItem fromJson(Map json) => + FlutterBenchCujTaskItemMapper.fromMap(json); +} diff --git a/sites/www/lib/src/models/content/flutterbench_content.mapper.dart b/sites/www/lib/src/models/content/flutterbench_content.mapper.dart new file mode 100644 index 00000000000..8e84ba78c70 --- /dev/null +++ b/sites/www/lib/src/models/content/flutterbench_content.mapper.dart @@ -0,0 +1,4986 @@ +// coverage:ignore-file +// GENERATED CODE - DO NOT MODIFY BY HAND +// dart format off +// ignore_for_file: type=lint +// ignore_for_file: invalid_use_of_protected_member +// ignore_for_file: unused_element, unnecessary_cast, override_on_non_overriding_member +// ignore_for_file: strict_raw_type, inference_failure_on_untyped_parameter + +part of 'flutterbench_content.dart'; + +class FlutterBenchJobDataMapper extends ClassMapperBase { + FlutterBenchJobDataMapper._(); + + static FlutterBenchJobDataMapper? _instance; + static FlutterBenchJobDataMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchJobDataMapper._()); + FlutterBenchEvalItemMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchJobData'; + + static String _$id(FlutterBenchJobData v) => v.id; + static const Field _f$id = Field('id', _$id); + static String _$startedAt(FlutterBenchJobData v) => v.startedAt; + static const Field _f$startedAt = Field( + 'startedAt', + _$startedAt, + key: r'started_at', + ); + static String _$finishedAt(FlutterBenchJobData v) => v.finishedAt; + static const Field _f$finishedAt = Field( + 'finishedAt', + _$finishedAt, + key: r'finished_at', + ); + static int _$nTotalTrials(FlutterBenchJobData v) => v.nTotalTrials; + static const Field _f$nTotalTrials = Field( + 'nTotalTrials', + _$nTotalTrials, + key: r'n_total_trials', + ); + static int _$nCompletedTrials(FlutterBenchJobData v) => v.nCompletedTrials; + static const Field _f$nCompletedTrials = Field( + 'nCompletedTrials', + _$nCompletedTrials, + key: r'n_completed_trials', + ); + static int _$nErroredTrials(FlutterBenchJobData v) => v.nErroredTrials; + static const Field _f$nErroredTrials = Field( + 'nErroredTrials', + _$nErroredTrials, + key: r'n_errored_trials', + ); + static double _$costUsd(FlutterBenchJobData v) => v.costUsd; + static const Field _f$costUsd = Field( + 'costUsd', + _$costUsd, + key: r'cost_usd', + ); + static int _$nInputTokens(FlutterBenchJobData v) => v.nInputTokens; + static const Field _f$nInputTokens = Field( + 'nInputTokens', + _$nInputTokens, + key: r'n_input_tokens', + ); + static int _$nCacheTokens(FlutterBenchJobData v) => v.nCacheTokens; + static const Field _f$nCacheTokens = Field( + 'nCacheTokens', + _$nCacheTokens, + key: r'n_cache_tokens', + ); + static int _$nOutputTokens(FlutterBenchJobData v) => v.nOutputTokens; + static const Field _f$nOutputTokens = Field( + 'nOutputTokens', + _$nOutputTokens, + key: r'n_output_tokens', + ); + static String _$topModelName(FlutterBenchJobData v) => v.topModelName; + static const Field _f$topModelName = Field( + 'topModelName', + _$topModelName, + key: r'top_model_name', + ); + static double _$topModelReward(FlutterBenchJobData v) => v.topModelReward; + static const Field _f$topModelReward = Field( + 'topModelReward', + _$topModelReward, + key: r'top_model_reward', + ); + static double _$overallAverageReward(FlutterBenchJobData v) => + v.overallAverageReward; + static const Field _f$overallAverageReward = + Field( + 'overallAverageReward', + _$overallAverageReward, + key: r'overall_average_reward', + ); + static List _$evals(FlutterBenchJobData v) => v.evals; + static const Field> _f$evals = + Field('evals', _$evals); + + @override + final MappableFields fields = const { + #id: _f$id, + #startedAt: _f$startedAt, + #finishedAt: _f$finishedAt, + #nTotalTrials: _f$nTotalTrials, + #nCompletedTrials: _f$nCompletedTrials, + #nErroredTrials: _f$nErroredTrials, + #costUsd: _f$costUsd, + #nInputTokens: _f$nInputTokens, + #nCacheTokens: _f$nCacheTokens, + #nOutputTokens: _f$nOutputTokens, + #topModelName: _f$topModelName, + #topModelReward: _f$topModelReward, + #overallAverageReward: _f$overallAverageReward, + #evals: _f$evals, + }; + + static FlutterBenchJobData _instantiate(DecodingData data) { + return FlutterBenchJobData( + id: data.dec(_f$id), + startedAt: data.dec(_f$startedAt), + finishedAt: data.dec(_f$finishedAt), + nTotalTrials: data.dec(_f$nTotalTrials), + nCompletedTrials: data.dec(_f$nCompletedTrials), + nErroredTrials: data.dec(_f$nErroredTrials), + costUsd: data.dec(_f$costUsd), + nInputTokens: data.dec(_f$nInputTokens), + nCacheTokens: data.dec(_f$nCacheTokens), + nOutputTokens: data.dec(_f$nOutputTokens), + topModelName: data.dec(_f$topModelName), + topModelReward: data.dec(_f$topModelReward), + overallAverageReward: data.dec(_f$overallAverageReward), + evals: data.dec(_f$evals), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchJobData fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchJobData fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchJobDataMappable { + String toJson() { + return FlutterBenchJobDataMapper.ensureInitialized() + .encodeJson(this as FlutterBenchJobData); + } + + Map toMap() { + return FlutterBenchJobDataMapper.ensureInitialized() + .encodeMap(this as FlutterBenchJobData); + } + + FlutterBenchJobDataCopyWith< + FlutterBenchJobData, + FlutterBenchJobData, + FlutterBenchJobData + > + get copyWith => + _FlutterBenchJobDataCopyWithImpl< + FlutterBenchJobData, + FlutterBenchJobData + >(this as FlutterBenchJobData, $identity, $identity); + @override + String toString() { + return FlutterBenchJobDataMapper.ensureInitialized().stringifyValue( + this as FlutterBenchJobData, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchJobDataMapper.ensureInitialized().equalsValue( + this as FlutterBenchJobData, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchJobDataMapper.ensureInitialized().hashValue( + this as FlutterBenchJobData, + ); + } +} + +extension FlutterBenchJobDataValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchJobData, $Out> { + FlutterBenchJobDataCopyWith<$R, FlutterBenchJobData, $Out> + get $asFlutterBenchJobData => $base.as( + (v, t, t2) => _FlutterBenchJobDataCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchJobDataCopyWith< + $R, + $In extends FlutterBenchJobData, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchEvalItem, + FlutterBenchEvalItemCopyWith<$R, FlutterBenchEvalItem, FlutterBenchEvalItem> + > + get evals; + $R call({ + String? id, + String? startedAt, + String? finishedAt, + int? nTotalTrials, + int? nCompletedTrials, + int? nErroredTrials, + double? costUsd, + int? nInputTokens, + int? nCacheTokens, + int? nOutputTokens, + String? topModelName, + double? topModelReward, + double? overallAverageReward, + List? evals, + }); + FlutterBenchJobDataCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchJobDataCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchJobData, $Out> + implements FlutterBenchJobDataCopyWith<$R, FlutterBenchJobData, $Out> { + _FlutterBenchJobDataCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchJobDataMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchEvalItem, + FlutterBenchEvalItemCopyWith<$R, FlutterBenchEvalItem, FlutterBenchEvalItem> + > + get evals => ListCopyWith( + $value.evals, + (v, t) => v.copyWith.$chain(t), + (v) => call(evals: v), + ); + @override + $R call({ + String? id, + String? startedAt, + String? finishedAt, + int? nTotalTrials, + int? nCompletedTrials, + int? nErroredTrials, + double? costUsd, + int? nInputTokens, + int? nCacheTokens, + int? nOutputTokens, + String? topModelName, + double? topModelReward, + double? overallAverageReward, + List? evals, + }) => $apply( + FieldCopyWithData({ + if (id != null) #id: id, + if (startedAt != null) #startedAt: startedAt, + if (finishedAt != null) #finishedAt: finishedAt, + if (nTotalTrials != null) #nTotalTrials: nTotalTrials, + if (nCompletedTrials != null) #nCompletedTrials: nCompletedTrials, + if (nErroredTrials != null) #nErroredTrials: nErroredTrials, + if (costUsd != null) #costUsd: costUsd, + if (nInputTokens != null) #nInputTokens: nInputTokens, + if (nCacheTokens != null) #nCacheTokens: nCacheTokens, + if (nOutputTokens != null) #nOutputTokens: nOutputTokens, + if (topModelName != null) #topModelName: topModelName, + if (topModelReward != null) #topModelReward: topModelReward, + if (overallAverageReward != null) + #overallAverageReward: overallAverageReward, + if (evals != null) #evals: evals, + }), + ); + @override + FlutterBenchJobData $make(CopyWithData data) => FlutterBenchJobData( + id: data.get(#id, or: $value.id), + startedAt: data.get(#startedAt, or: $value.startedAt), + finishedAt: data.get(#finishedAt, or: $value.finishedAt), + nTotalTrials: data.get(#nTotalTrials, or: $value.nTotalTrials), + nCompletedTrials: data.get(#nCompletedTrials, or: $value.nCompletedTrials), + nErroredTrials: data.get(#nErroredTrials, or: $value.nErroredTrials), + costUsd: data.get(#costUsd, or: $value.costUsd), + nInputTokens: data.get(#nInputTokens, or: $value.nInputTokens), + nCacheTokens: data.get(#nCacheTokens, or: $value.nCacheTokens), + nOutputTokens: data.get(#nOutputTokens, or: $value.nOutputTokens), + topModelName: data.get(#topModelName, or: $value.topModelName), + topModelReward: data.get(#topModelReward, or: $value.topModelReward), + overallAverageReward: data.get( + #overallAverageReward, + or: $value.overallAverageReward, + ), + evals: data.get(#evals, or: $value.evals), + ); + + @override + FlutterBenchJobDataCopyWith<$R2, FlutterBenchJobData, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchJobDataCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchEvalItemMapper extends ClassMapperBase { + FlutterBenchEvalItemMapper._(); + + static FlutterBenchEvalItemMapper? _instance; + static FlutterBenchEvalItemMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchEvalItemMapper._()); + FlutterBenchCujSummaryMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchEvalItem'; + + static String _$evalKey(FlutterBenchEvalItem v) => v.evalKey; + static const Field _f$evalKey = Field( + 'evalKey', + _$evalKey, + key: r'eval_key', + ); + static String _$agentName(FlutterBenchEvalItem v) => v.agentName; + static const Field _f$agentName = Field( + 'agentName', + _$agentName, + key: r'agent_name', + ); + static String _$modelName(FlutterBenchEvalItem v) => v.modelName; + static const Field _f$modelName = Field( + 'modelName', + _$modelName, + key: r'model_name', + ); + static String _$modelShortName(FlutterBenchEvalItem v) => v.modelShortName; + static const Field _f$modelShortName = Field( + 'modelShortName', + _$modelShortName, + key: r'model_short_name', + ); + static String _$provider(FlutterBenchEvalItem v) => v.provider; + static const Field _f$provider = Field( + 'provider', + _$provider, + ); + static String _$variant(FlutterBenchEvalItem v) => v.variant; + static const Field _f$variant = Field( + 'variant', + _$variant, + ); + static int _$nTrials(FlutterBenchEvalItem v) => v.nTrials; + static const Field _f$nTrials = Field( + 'nTrials', + _$nTrials, + key: r'n_trials', + ); + static int _$nErrors(FlutterBenchEvalItem v) => v.nErrors; + static const Field _f$nErrors = Field( + 'nErrors', + _$nErrors, + key: r'n_errors', + ); + static double _$meanReward(FlutterBenchEvalItem v) => v.meanReward; + static const Field _f$meanReward = Field( + 'meanReward', + _$meanReward, + key: r'mean_reward', + ); + static double _$minReward(FlutterBenchEvalItem v) => v.minReward; + static const Field _f$minReward = Field( + 'minReward', + _$minReward, + key: r'min_reward', + ); + static double _$maxReward(FlutterBenchEvalItem v) => v.maxReward; + static const Field _f$maxReward = Field( + 'maxReward', + _$maxReward, + key: r'max_reward', + ); + static double _$medianReward(FlutterBenchEvalItem v) => v.medianReward; + static const Field _f$medianReward = Field( + 'medianReward', + _$medianReward, + key: r'median_reward', + ); + static double _$passAt1(FlutterBenchEvalItem v) => v.passAt1; + static const Field _f$passAt1 = Field( + 'passAt1', + _$passAt1, + key: r'pass_at_1', + ); + static double _$costUsd(FlutterBenchEvalItem v) => v.costUsd; + static const Field _f$costUsd = Field( + 'costUsd', + _$costUsd, + key: r'cost_usd', + ); + static int _$inputTokens(FlutterBenchEvalItem v) => v.inputTokens; + static const Field _f$inputTokens = Field( + 'inputTokens', + _$inputTokens, + key: r'input_tokens', + ); + static int _$outputTokens(FlutterBenchEvalItem v) => v.outputTokens; + static const Field _f$outputTokens = Field( + 'outputTokens', + _$outputTokens, + key: r'output_tokens', + ); + static bool _$hasDartTooling(FlutterBenchEvalItem v) => v.hasDartTooling; + static const Field _f$hasDartTooling = Field( + 'hasDartTooling', + _$hasDartTooling, + key: r'has_dart_tooling', + ); + static double? _$outcomeScore(FlutterBenchEvalItem v) => v.outcomeScore; + static const Field _f$outcomeScore = Field( + 'outcomeScore', + _$outcomeScore, + key: r'outcome_score', + opt: true, + ); + static double? _$qualityScore(FlutterBenchEvalItem v) => v.qualityScore; + static const Field _f$qualityScore = Field( + 'qualityScore', + _$qualityScore, + key: r'quality_score', + opt: true, + ); + static double? _$dxScore(FlutterBenchEvalItem v) => v.dxScore; + static const Field _f$dxScore = Field( + 'dxScore', + _$dxScore, + key: r'dx_score', + opt: true, + ); + static List _$bestCujs(FlutterBenchEvalItem v) => + v.bestCujs; + static const Field> + _f$bestCujs = Field( + 'bestCujs', + _$bestCujs, + key: r'best_cujs', + opt: true, + def: const [], + ); + static List _$worstCujs(FlutterBenchEvalItem v) => + v.worstCujs; + static const Field> + _f$worstCujs = Field( + 'worstCujs', + _$worstCujs, + key: r'worst_cujs', + opt: true, + def: const [], + ); + + @override + final MappableFields fields = const { + #evalKey: _f$evalKey, + #agentName: _f$agentName, + #modelName: _f$modelName, + #modelShortName: _f$modelShortName, + #provider: _f$provider, + #variant: _f$variant, + #nTrials: _f$nTrials, + #nErrors: _f$nErrors, + #meanReward: _f$meanReward, + #minReward: _f$minReward, + #maxReward: _f$maxReward, + #medianReward: _f$medianReward, + #passAt1: _f$passAt1, + #costUsd: _f$costUsd, + #inputTokens: _f$inputTokens, + #outputTokens: _f$outputTokens, + #hasDartTooling: _f$hasDartTooling, + #outcomeScore: _f$outcomeScore, + #qualityScore: _f$qualityScore, + #dxScore: _f$dxScore, + #bestCujs: _f$bestCujs, + #worstCujs: _f$worstCujs, + }; + + static FlutterBenchEvalItem _instantiate(DecodingData data) { + return FlutterBenchEvalItem( + evalKey: data.dec(_f$evalKey), + agentName: data.dec(_f$agentName), + modelName: data.dec(_f$modelName), + modelShortName: data.dec(_f$modelShortName), + provider: data.dec(_f$provider), + variant: data.dec(_f$variant), + nTrials: data.dec(_f$nTrials), + nErrors: data.dec(_f$nErrors), + meanReward: data.dec(_f$meanReward), + minReward: data.dec(_f$minReward), + maxReward: data.dec(_f$maxReward), + medianReward: data.dec(_f$medianReward), + passAt1: data.dec(_f$passAt1), + costUsd: data.dec(_f$costUsd), + inputTokens: data.dec(_f$inputTokens), + outputTokens: data.dec(_f$outputTokens), + hasDartTooling: data.dec(_f$hasDartTooling), + outcomeScore: data.dec(_f$outcomeScore), + qualityScore: data.dec(_f$qualityScore), + dxScore: data.dec(_f$dxScore), + bestCujs: data.dec(_f$bestCujs), + worstCujs: data.dec(_f$worstCujs), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchEvalItem fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchEvalItem fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchEvalItemMappable { + String toJson() { + return FlutterBenchEvalItemMapper.ensureInitialized() + .encodeJson(this as FlutterBenchEvalItem); + } + + Map toMap() { + return FlutterBenchEvalItemMapper.ensureInitialized() + .encodeMap(this as FlutterBenchEvalItem); + } + + FlutterBenchEvalItemCopyWith< + FlutterBenchEvalItem, + FlutterBenchEvalItem, + FlutterBenchEvalItem + > + get copyWith => + _FlutterBenchEvalItemCopyWithImpl< + FlutterBenchEvalItem, + FlutterBenchEvalItem + >(this as FlutterBenchEvalItem, $identity, $identity); + @override + String toString() { + return FlutterBenchEvalItemMapper.ensureInitialized().stringifyValue( + this as FlutterBenchEvalItem, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchEvalItemMapper.ensureInitialized().equalsValue( + this as FlutterBenchEvalItem, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchEvalItemMapper.ensureInitialized().hashValue( + this as FlutterBenchEvalItem, + ); + } +} + +extension FlutterBenchEvalItemValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchEvalItem, $Out> { + FlutterBenchEvalItemCopyWith<$R, FlutterBenchEvalItem, $Out> + get $asFlutterBenchEvalItem => $base.as( + (v, t, t2) => _FlutterBenchEvalItemCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchEvalItemCopyWith< + $R, + $In extends FlutterBenchEvalItem, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchCujSummary, + FlutterBenchCujSummaryCopyWith< + $R, + FlutterBenchCujSummary, + FlutterBenchCujSummary + > + > + get bestCujs; + ListCopyWith< + $R, + FlutterBenchCujSummary, + FlutterBenchCujSummaryCopyWith< + $R, + FlutterBenchCujSummary, + FlutterBenchCujSummary + > + > + get worstCujs; + $R call({ + String? evalKey, + String? agentName, + String? modelName, + String? modelShortName, + String? provider, + String? variant, + int? nTrials, + int? nErrors, + double? meanReward, + double? minReward, + double? maxReward, + double? medianReward, + double? passAt1, + double? costUsd, + int? inputTokens, + int? outputTokens, + bool? hasDartTooling, + double? outcomeScore, + double? qualityScore, + double? dxScore, + List? bestCujs, + List? worstCujs, + }); + FlutterBenchEvalItemCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchEvalItemCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchEvalItem, $Out> + implements FlutterBenchEvalItemCopyWith<$R, FlutterBenchEvalItem, $Out> { + _FlutterBenchEvalItemCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchEvalItemMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchCujSummary, + FlutterBenchCujSummaryCopyWith< + $R, + FlutterBenchCujSummary, + FlutterBenchCujSummary + > + > + get bestCujs => ListCopyWith( + $value.bestCujs, + (v, t) => v.copyWith.$chain(t), + (v) => call(bestCujs: v), + ); + @override + ListCopyWith< + $R, + FlutterBenchCujSummary, + FlutterBenchCujSummaryCopyWith< + $R, + FlutterBenchCujSummary, + FlutterBenchCujSummary + > + > + get worstCujs => ListCopyWith( + $value.worstCujs, + (v, t) => v.copyWith.$chain(t), + (v) => call(worstCujs: v), + ); + @override + $R call({ + String? evalKey, + String? agentName, + String? modelName, + String? modelShortName, + String? provider, + String? variant, + int? nTrials, + int? nErrors, + double? meanReward, + double? minReward, + double? maxReward, + double? medianReward, + double? passAt1, + double? costUsd, + int? inputTokens, + int? outputTokens, + bool? hasDartTooling, + Object? outcomeScore = $none, + Object? qualityScore = $none, + Object? dxScore = $none, + List? bestCujs, + List? worstCujs, + }) => $apply( + FieldCopyWithData({ + if (evalKey != null) #evalKey: evalKey, + if (agentName != null) #agentName: agentName, + if (modelName != null) #modelName: modelName, + if (modelShortName != null) #modelShortName: modelShortName, + if (provider != null) #provider: provider, + if (variant != null) #variant: variant, + if (nTrials != null) #nTrials: nTrials, + if (nErrors != null) #nErrors: nErrors, + if (meanReward != null) #meanReward: meanReward, + if (minReward != null) #minReward: minReward, + if (maxReward != null) #maxReward: maxReward, + if (medianReward != null) #medianReward: medianReward, + if (passAt1 != null) #passAt1: passAt1, + if (costUsd != null) #costUsd: costUsd, + if (inputTokens != null) #inputTokens: inputTokens, + if (outputTokens != null) #outputTokens: outputTokens, + if (hasDartTooling != null) #hasDartTooling: hasDartTooling, + if (outcomeScore != $none) #outcomeScore: outcomeScore, + if (qualityScore != $none) #qualityScore: qualityScore, + if (dxScore != $none) #dxScore: dxScore, + if (bestCujs != null) #bestCujs: bestCujs, + if (worstCujs != null) #worstCujs: worstCujs, + }), + ); + @override + FlutterBenchEvalItem $make(CopyWithData data) => FlutterBenchEvalItem( + evalKey: data.get(#evalKey, or: $value.evalKey), + agentName: data.get(#agentName, or: $value.agentName), + modelName: data.get(#modelName, or: $value.modelName), + modelShortName: data.get(#modelShortName, or: $value.modelShortName), + provider: data.get(#provider, or: $value.provider), + variant: data.get(#variant, or: $value.variant), + nTrials: data.get(#nTrials, or: $value.nTrials), + nErrors: data.get(#nErrors, or: $value.nErrors), + meanReward: data.get(#meanReward, or: $value.meanReward), + minReward: data.get(#minReward, or: $value.minReward), + maxReward: data.get(#maxReward, or: $value.maxReward), + medianReward: data.get(#medianReward, or: $value.medianReward), + passAt1: data.get(#passAt1, or: $value.passAt1), + costUsd: data.get(#costUsd, or: $value.costUsd), + inputTokens: data.get(#inputTokens, or: $value.inputTokens), + outputTokens: data.get(#outputTokens, or: $value.outputTokens), + hasDartTooling: data.get(#hasDartTooling, or: $value.hasDartTooling), + outcomeScore: data.get(#outcomeScore, or: $value.outcomeScore), + qualityScore: data.get(#qualityScore, or: $value.qualityScore), + dxScore: data.get(#dxScore, or: $value.dxScore), + bestCujs: data.get(#bestCujs, or: $value.bestCujs), + worstCujs: data.get(#worstCujs, or: $value.worstCujs), + ); + + @override + FlutterBenchEvalItemCopyWith<$R2, FlutterBenchEvalItem, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchEvalItemCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchCujSummaryMapper + extends ClassMapperBase { + FlutterBenchCujSummaryMapper._(); + + static FlutterBenchCujSummaryMapper? _instance; + static FlutterBenchCujSummaryMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchCujSummaryMapper._()); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchCujSummary'; + + static String _$taskSlug(FlutterBenchCujSummary v) => v.taskSlug; + static const Field _f$taskSlug = Field( + 'taskSlug', + _$taskSlug, + key: r'task_slug', + ); + static String _$taskName(FlutterBenchCujSummary v) => v.taskName; + static const Field _f$taskName = Field( + 'taskName', + _$taskName, + key: r'task_name', + ); + static double? _$reward(FlutterBenchCujSummary v) => v.reward; + static const Field _f$reward = Field( + 'reward', + _$reward, + opt: true, + ); + static String _$status(FlutterBenchCujSummary v) => v.status; + static const Field _f$status = Field( + 'status', + _$status, + ); + + @override + final MappableFields fields = const { + #taskSlug: _f$taskSlug, + #taskName: _f$taskName, + #reward: _f$reward, + #status: _f$status, + }; + + static FlutterBenchCujSummary _instantiate(DecodingData data) { + return FlutterBenchCujSummary( + taskSlug: data.dec(_f$taskSlug), + taskName: data.dec(_f$taskName), + reward: data.dec(_f$reward), + status: data.dec(_f$status), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchCujSummary fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchCujSummary fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchCujSummaryMappable { + String toJson() { + return FlutterBenchCujSummaryMapper.ensureInitialized() + .encodeJson(this as FlutterBenchCujSummary); + } + + Map toMap() { + return FlutterBenchCujSummaryMapper.ensureInitialized() + .encodeMap(this as FlutterBenchCujSummary); + } + + FlutterBenchCujSummaryCopyWith< + FlutterBenchCujSummary, + FlutterBenchCujSummary, + FlutterBenchCujSummary + > + get copyWith => + _FlutterBenchCujSummaryCopyWithImpl< + FlutterBenchCujSummary, + FlutterBenchCujSummary + >(this as FlutterBenchCujSummary, $identity, $identity); + @override + String toString() { + return FlutterBenchCujSummaryMapper.ensureInitialized().stringifyValue( + this as FlutterBenchCujSummary, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchCujSummaryMapper.ensureInitialized().equalsValue( + this as FlutterBenchCujSummary, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchCujSummaryMapper.ensureInitialized().hashValue( + this as FlutterBenchCujSummary, + ); + } +} + +extension FlutterBenchCujSummaryValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchCujSummary, $Out> { + FlutterBenchCujSummaryCopyWith<$R, FlutterBenchCujSummary, $Out> + get $asFlutterBenchCujSummary => $base.as( + (v, t, t2) => _FlutterBenchCujSummaryCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchCujSummaryCopyWith< + $R, + $In extends FlutterBenchCujSummary, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + $R call({String? taskSlug, String? taskName, double? reward, String? status}); + FlutterBenchCujSummaryCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchCujSummaryCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchCujSummary, $Out> + implements + FlutterBenchCujSummaryCopyWith<$R, FlutterBenchCujSummary, $Out> { + _FlutterBenchCujSummaryCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchCujSummaryMapper.ensureInitialized(); + @override + $R call({ + String? taskSlug, + String? taskName, + Object? reward = $none, + String? status, + }) => $apply( + FieldCopyWithData({ + if (taskSlug != null) #taskSlug: taskSlug, + if (taskName != null) #taskName: taskName, + if (reward != $none) #reward: reward, + if (status != null) #status: status, + }), + ); + @override + FlutterBenchCujSummary $make(CopyWithData data) => FlutterBenchCujSummary( + taskSlug: data.get(#taskSlug, or: $value.taskSlug), + taskName: data.get(#taskName, or: $value.taskName), + reward: data.get(#reward, or: $value.reward), + status: data.get(#status, or: $value.status), + ); + + @override + FlutterBenchCujSummaryCopyWith<$R2, FlutterBenchCujSummary, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchCujSummaryCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchTasksDataMapper + extends ClassMapperBase { + FlutterBenchTasksDataMapper._(); + + static FlutterBenchTasksDataMapper? _instance; + static FlutterBenchTasksDataMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchTasksDataMapper._()); + FlutterBenchTaskItemMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTasksData'; + + static List _$tasks(FlutterBenchTasksData v) => v.tasks; + static const Field> + _f$tasks = Field('tasks', _$tasks); + + @override + final MappableFields fields = const {#tasks: _f$tasks}; + + static FlutterBenchTasksData _instantiate(DecodingData data) { + return FlutterBenchTasksData(tasks: data.dec(_f$tasks)); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTasksData fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTasksData fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTasksDataMappable { + String toJson() { + return FlutterBenchTasksDataMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTasksData); + } + + Map toMap() { + return FlutterBenchTasksDataMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTasksData); + } + + FlutterBenchTasksDataCopyWith< + FlutterBenchTasksData, + FlutterBenchTasksData, + FlutterBenchTasksData + > + get copyWith => + _FlutterBenchTasksDataCopyWithImpl< + FlutterBenchTasksData, + FlutterBenchTasksData + >(this as FlutterBenchTasksData, $identity, $identity); + @override + String toString() { + return FlutterBenchTasksDataMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTasksData, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTasksDataMapper.ensureInitialized().equalsValue( + this as FlutterBenchTasksData, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTasksDataMapper.ensureInitialized().hashValue( + this as FlutterBenchTasksData, + ); + } +} + +extension FlutterBenchTasksDataValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTasksData, $Out> { + FlutterBenchTasksDataCopyWith<$R, FlutterBenchTasksData, $Out> + get $asFlutterBenchTasksData => $base.as( + (v, t, t2) => _FlutterBenchTasksDataCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTasksDataCopyWith< + $R, + $In extends FlutterBenchTasksData, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchTaskItem, + FlutterBenchTaskItemCopyWith<$R, FlutterBenchTaskItem, FlutterBenchTaskItem> + > + get tasks; + $R call({List? tasks}); + FlutterBenchTasksDataCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTasksDataCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTasksData, $Out> + implements FlutterBenchTasksDataCopyWith<$R, FlutterBenchTasksData, $Out> { + _FlutterBenchTasksDataCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTasksDataMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchTaskItem, + FlutterBenchTaskItemCopyWith<$R, FlutterBenchTaskItem, FlutterBenchTaskItem> + > + get tasks => ListCopyWith( + $value.tasks, + (v, t) => v.copyWith.$chain(t), + (v) => call(tasks: v), + ); + @override + $R call({List? tasks}) => + $apply(FieldCopyWithData({if (tasks != null) #tasks: tasks})); + @override + FlutterBenchTasksData $make(CopyWithData data) => + FlutterBenchTasksData(tasks: data.get(#tasks, or: $value.tasks)); + + @override + FlutterBenchTasksDataCopyWith<$R2, FlutterBenchTasksData, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTasksDataCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchTaskItemMapper extends ClassMapperBase { + FlutterBenchTaskItemMapper._(); + + static FlutterBenchTaskItemMapper? _instance; + static FlutterBenchTaskItemMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchTaskItemMapper._()); + FlutterBenchTrialSummaryMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTaskItem'; + + static String _$slug(FlutterBenchTaskItem v) => v.slug; + static const Field _f$slug = Field( + 'slug', + _$slug, + ); + static String _$taskName(FlutterBenchTaskItem v) => v.taskName; + static const Field _f$taskName = Field( + 'taskName', + _$taskName, + key: r'task_name', + ); + static String _$displayName(FlutterBenchTaskItem v) => v.displayName; + static const Field _f$displayName = Field( + 'displayName', + _$displayName, + key: r'display_name', + ); + static String _$category(FlutterBenchTaskItem v) => v.category; + static const Field _f$category = Field( + 'category', + _$category, + ); + static String _$description(FlutterBenchTaskItem v) => v.description; + static const Field _f$description = Field( + 'description', + _$description, + ); + static List _$trials(FlutterBenchTaskItem v) => + v.trials; + static const Field> + _f$trials = Field('trials', _$trials, opt: true, def: const []); + static Map _$scoresByEval(FlutterBenchTaskItem v) => + v.scoresByEval; + static const Field> + _f$scoresByEval = Field( + 'scoresByEval', + _$scoresByEval, + key: r'scores_by_eval', + opt: true, + def: const {}, + ); + + @override + final MappableFields fields = const { + #slug: _f$slug, + #taskName: _f$taskName, + #displayName: _f$displayName, + #category: _f$category, + #description: _f$description, + #trials: _f$trials, + #scoresByEval: _f$scoresByEval, + }; + + static FlutterBenchTaskItem _instantiate(DecodingData data) { + return FlutterBenchTaskItem( + slug: data.dec(_f$slug), + taskName: data.dec(_f$taskName), + displayName: data.dec(_f$displayName), + category: data.dec(_f$category), + description: data.dec(_f$description), + trials: data.dec(_f$trials), + scoresByEval: data.dec(_f$scoresByEval), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTaskItem fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTaskItem fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTaskItemMappable { + String toJson() { + return FlutterBenchTaskItemMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTaskItem); + } + + Map toMap() { + return FlutterBenchTaskItemMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTaskItem); + } + + FlutterBenchTaskItemCopyWith< + FlutterBenchTaskItem, + FlutterBenchTaskItem, + FlutterBenchTaskItem + > + get copyWith => + _FlutterBenchTaskItemCopyWithImpl< + FlutterBenchTaskItem, + FlutterBenchTaskItem + >(this as FlutterBenchTaskItem, $identity, $identity); + @override + String toString() { + return FlutterBenchTaskItemMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTaskItem, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTaskItemMapper.ensureInitialized().equalsValue( + this as FlutterBenchTaskItem, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTaskItemMapper.ensureInitialized().hashValue( + this as FlutterBenchTaskItem, + ); + } +} + +extension FlutterBenchTaskItemValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTaskItem, $Out> { + FlutterBenchTaskItemCopyWith<$R, FlutterBenchTaskItem, $Out> + get $asFlutterBenchTaskItem => $base.as( + (v, t, t2) => _FlutterBenchTaskItemCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTaskItemCopyWith< + $R, + $In extends FlutterBenchTaskItem, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchTrialSummary, + FlutterBenchTrialSummaryCopyWith< + $R, + FlutterBenchTrialSummary, + FlutterBenchTrialSummary + > + > + get trials; + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get scoresByEval; + $R call({ + String? slug, + String? taskName, + String? displayName, + String? category, + String? description, + List? trials, + Map? scoresByEval, + }); + FlutterBenchTaskItemCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTaskItemCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTaskItem, $Out> + implements FlutterBenchTaskItemCopyWith<$R, FlutterBenchTaskItem, $Out> { + _FlutterBenchTaskItemCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTaskItemMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchTrialSummary, + FlutterBenchTrialSummaryCopyWith< + $R, + FlutterBenchTrialSummary, + FlutterBenchTrialSummary + > + > + get trials => ListCopyWith( + $value.trials, + (v, t) => v.copyWith.$chain(t), + (v) => call(trials: v), + ); + @override + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get scoresByEval => MapCopyWith( + $value.scoresByEval, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(scoresByEval: v), + ); + @override + $R call({ + String? slug, + String? taskName, + String? displayName, + String? category, + String? description, + List? trials, + Map? scoresByEval, + }) => $apply( + FieldCopyWithData({ + if (slug != null) #slug: slug, + if (taskName != null) #taskName: taskName, + if (displayName != null) #displayName: displayName, + if (category != null) #category: category, + if (description != null) #description: description, + if (trials != null) #trials: trials, + if (scoresByEval != null) #scoresByEval: scoresByEval, + }), + ); + @override + FlutterBenchTaskItem $make(CopyWithData data) => FlutterBenchTaskItem( + slug: data.get(#slug, or: $value.slug), + taskName: data.get(#taskName, or: $value.taskName), + displayName: data.get(#displayName, or: $value.displayName), + category: data.get(#category, or: $value.category), + description: data.get(#description, or: $value.description), + trials: data.get(#trials, or: $value.trials), + scoresByEval: data.get(#scoresByEval, or: $value.scoresByEval), + ); + + @override + FlutterBenchTaskItemCopyWith<$R2, FlutterBenchTaskItem, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTaskItemCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchTrialSummaryMapper + extends ClassMapperBase { + FlutterBenchTrialSummaryMapper._(); + + static FlutterBenchTrialSummaryMapper? _instance; + static FlutterBenchTrialSummaryMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchTrialSummaryMapper._(), + ); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTrialSummary'; + + static String _$trialName(FlutterBenchTrialSummary v) => v.trialName; + static const Field _f$trialName = Field( + 'trialName', + _$trialName, + key: r'trial_name', + ); + static String _$status(FlutterBenchTrialSummary v) => v.status; + static const Field _f$status = Field( + 'status', + _$status, + ); + static double? _$reward(FlutterBenchTrialSummary v) => v.reward; + static const Field _f$reward = Field( + 'reward', + _$reward, + opt: true, + ); + static String _$modelName(FlutterBenchTrialSummary v) => v.modelName; + static const Field _f$modelName = Field( + 'modelName', + _$modelName, + key: r'model_name', + ); + static String _$modelShortName(FlutterBenchTrialSummary v) => + v.modelShortName; + static const Field _f$modelShortName = + Field('modelShortName', _$modelShortName, key: r'model_short_name'); + static String _$agentName(FlutterBenchTrialSummary v) => v.agentName; + static const Field _f$agentName = Field( + 'agentName', + _$agentName, + key: r'agent_name', + ); + static bool _$hasDartTooling(FlutterBenchTrialSummary v) => v.hasDartTooling; + static const Field _f$hasDartTooling = Field( + 'hasDartTooling', + _$hasDartTooling, + key: r'has_dart_tooling', + ); + static String? _$exceptionType(FlutterBenchTrialSummary v) => v.exceptionType; + static const Field _f$exceptionType = Field( + 'exceptionType', + _$exceptionType, + key: r'exception_type', + opt: true, + ); + + @override + final MappableFields fields = const { + #trialName: _f$trialName, + #status: _f$status, + #reward: _f$reward, + #modelName: _f$modelName, + #modelShortName: _f$modelShortName, + #agentName: _f$agentName, + #hasDartTooling: _f$hasDartTooling, + #exceptionType: _f$exceptionType, + }; + + static FlutterBenchTrialSummary _instantiate(DecodingData data) { + return FlutterBenchTrialSummary( + trialName: data.dec(_f$trialName), + status: data.dec(_f$status), + reward: data.dec(_f$reward), + modelName: data.dec(_f$modelName), + modelShortName: data.dec(_f$modelShortName), + agentName: data.dec(_f$agentName), + hasDartTooling: data.dec(_f$hasDartTooling), + exceptionType: data.dec(_f$exceptionType), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTrialSummary fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTrialSummary fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTrialSummaryMappable { + String toJson() { + return FlutterBenchTrialSummaryMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTrialSummary); + } + + Map toMap() { + return FlutterBenchTrialSummaryMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTrialSummary); + } + + FlutterBenchTrialSummaryCopyWith< + FlutterBenchTrialSummary, + FlutterBenchTrialSummary, + FlutterBenchTrialSummary + > + get copyWith => + _FlutterBenchTrialSummaryCopyWithImpl< + FlutterBenchTrialSummary, + FlutterBenchTrialSummary + >(this as FlutterBenchTrialSummary, $identity, $identity); + @override + String toString() { + return FlutterBenchTrialSummaryMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTrialSummary, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTrialSummaryMapper.ensureInitialized().equalsValue( + this as FlutterBenchTrialSummary, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTrialSummaryMapper.ensureInitialized().hashValue( + this as FlutterBenchTrialSummary, + ); + } +} + +extension FlutterBenchTrialSummaryValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTrialSummary, $Out> { + FlutterBenchTrialSummaryCopyWith<$R, FlutterBenchTrialSummary, $Out> + get $asFlutterBenchTrialSummary => $base.as( + (v, t, t2) => _FlutterBenchTrialSummaryCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTrialSummaryCopyWith< + $R, + $In extends FlutterBenchTrialSummary, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + $R call({ + String? trialName, + String? status, + double? reward, + String? modelName, + String? modelShortName, + String? agentName, + bool? hasDartTooling, + String? exceptionType, + }); + FlutterBenchTrialSummaryCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTrialSummaryCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTrialSummary, $Out> + implements + FlutterBenchTrialSummaryCopyWith<$R, FlutterBenchTrialSummary, $Out> { + _FlutterBenchTrialSummaryCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTrialSummaryMapper.ensureInitialized(); + @override + $R call({ + String? trialName, + String? status, + Object? reward = $none, + String? modelName, + String? modelShortName, + String? agentName, + bool? hasDartTooling, + Object? exceptionType = $none, + }) => $apply( + FieldCopyWithData({ + if (trialName != null) #trialName: trialName, + if (status != null) #status: status, + if (reward != $none) #reward: reward, + if (modelName != null) #modelName: modelName, + if (modelShortName != null) #modelShortName: modelShortName, + if (agentName != null) #agentName: agentName, + if (hasDartTooling != null) #hasDartTooling: hasDartTooling, + if (exceptionType != $none) #exceptionType: exceptionType, + }), + ); + @override + FlutterBenchTrialSummary $make(CopyWithData data) => FlutterBenchTrialSummary( + trialName: data.get(#trialName, or: $value.trialName), + status: data.get(#status, or: $value.status), + reward: data.get(#reward, or: $value.reward), + modelName: data.get(#modelName, or: $value.modelName), + modelShortName: data.get(#modelShortName, or: $value.modelShortName), + agentName: data.get(#agentName, or: $value.agentName), + hasDartTooling: data.get(#hasDartTooling, or: $value.hasDartTooling), + exceptionType: data.get(#exceptionType, or: $value.exceptionType), + ); + + @override + FlutterBenchTrialSummaryCopyWith<$R2, FlutterBenchTrialSummary, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTrialSummaryCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchTrialsDataMapper + extends ClassMapperBase { + FlutterBenchTrialsDataMapper._(); + + static FlutterBenchTrialsDataMapper? _instance; + static FlutterBenchTrialsDataMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchTrialsDataMapper._()); + FlutterBenchTrialDetailMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTrialsData'; + + static List _$trials(FlutterBenchTrialsData v) => + v.trials; + static const Field> + _f$trials = Field('trials', _$trials); + + @override + final MappableFields fields = const { + #trials: _f$trials, + }; + + static FlutterBenchTrialsData _instantiate(DecodingData data) { + return FlutterBenchTrialsData(trials: data.dec(_f$trials)); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTrialsData fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTrialsData fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTrialsDataMappable { + String toJson() { + return FlutterBenchTrialsDataMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTrialsData); + } + + Map toMap() { + return FlutterBenchTrialsDataMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTrialsData); + } + + FlutterBenchTrialsDataCopyWith< + FlutterBenchTrialsData, + FlutterBenchTrialsData, + FlutterBenchTrialsData + > + get copyWith => + _FlutterBenchTrialsDataCopyWithImpl< + FlutterBenchTrialsData, + FlutterBenchTrialsData + >(this as FlutterBenchTrialsData, $identity, $identity); + @override + String toString() { + return FlutterBenchTrialsDataMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTrialsData, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTrialsDataMapper.ensureInitialized().equalsValue( + this as FlutterBenchTrialsData, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTrialsDataMapper.ensureInitialized().hashValue( + this as FlutterBenchTrialsData, + ); + } +} + +extension FlutterBenchTrialsDataValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTrialsData, $Out> { + FlutterBenchTrialsDataCopyWith<$R, FlutterBenchTrialsData, $Out> + get $asFlutterBenchTrialsData => $base.as( + (v, t, t2) => _FlutterBenchTrialsDataCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTrialsDataCopyWith< + $R, + $In extends FlutterBenchTrialsData, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchTrialDetail, + FlutterBenchTrialDetailCopyWith< + $R, + FlutterBenchTrialDetail, + FlutterBenchTrialDetail + > + > + get trials; + $R call({List? trials}); + FlutterBenchTrialsDataCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTrialsDataCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTrialsData, $Out> + implements + FlutterBenchTrialsDataCopyWith<$R, FlutterBenchTrialsData, $Out> { + _FlutterBenchTrialsDataCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTrialsDataMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchTrialDetail, + FlutterBenchTrialDetailCopyWith< + $R, + FlutterBenchTrialDetail, + FlutterBenchTrialDetail + > + > + get trials => ListCopyWith( + $value.trials, + (v, t) => v.copyWith.$chain(t), + (v) => call(trials: v), + ); + @override + $R call({List? trials}) => + $apply(FieldCopyWithData({if (trials != null) #trials: trials})); + @override + FlutterBenchTrialsData $make(CopyWithData data) => + FlutterBenchTrialsData(trials: data.get(#trials, or: $value.trials)); + + @override + FlutterBenchTrialsDataCopyWith<$R2, FlutterBenchTrialsData, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTrialsDataCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchTrialDetailMapper + extends ClassMapperBase { + FlutterBenchTrialDetailMapper._(); + + static FlutterBenchTrialDetailMapper? _instance; + static FlutterBenchTrialDetailMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchTrialDetailMapper._(), + ); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTrialDetail'; + + static String _$trialName(FlutterBenchTrialDetail v) => v.trialName; + static const Field _f$trialName = Field( + 'trialName', + _$trialName, + key: r'trial_name', + ); + static String _$taskName(FlutterBenchTrialDetail v) => v.taskName; + static const Field _f$taskName = Field( + 'taskName', + _$taskName, + key: r'task_name', + ); + static String _$taskSlug(FlutterBenchTrialDetail v) => v.taskSlug; + static const Field _f$taskSlug = Field( + 'taskSlug', + _$taskSlug, + key: r'task_slug', + ); + static String _$agentName(FlutterBenchTrialDetail v) => v.agentName; + static const Field _f$agentName = Field( + 'agentName', + _$agentName, + key: r'agent_name', + ); + static String _$modelName(FlutterBenchTrialDetail v) => v.modelName; + static const Field _f$modelName = Field( + 'modelName', + _$modelName, + key: r'model_name', + ); + static String _$modelShortName(FlutterBenchTrialDetail v) => v.modelShortName; + static const Field _f$modelShortName = Field( + 'modelShortName', + _$modelShortName, + key: r'model_short_name', + ); + static String _$provider(FlutterBenchTrialDetail v) => v.provider; + static const Field _f$provider = Field( + 'provider', + _$provider, + ); + static List _$skills(FlutterBenchTrialDetail v) => v.skills; + static const Field> _f$skills = Field( + 'skills', + _$skills, + opt: true, + def: const [], + ); + static List _$mcpServers(FlutterBenchTrialDetail v) => v.mcpServers; + static const Field> _f$mcpServers = + Field( + 'mcpServers', + _$mcpServers, + key: r'mcp_servers', + opt: true, + def: const [], + ); + static bool _$hasDartTooling(FlutterBenchTrialDetail v) => v.hasDartTooling; + static const Field _f$hasDartTooling = Field( + 'hasDartTooling', + _$hasDartTooling, + key: r'has_dart_tooling', + ); + static String _$status(FlutterBenchTrialDetail v) => v.status; + static const Field _f$status = Field( + 'status', + _$status, + ); + static double? _$reward(FlutterBenchTrialDetail v) => v.reward; + static const Field _f$reward = Field( + 'reward', + _$reward, + opt: true, + ); + static String? _$exceptionType(FlutterBenchTrialDetail v) => v.exceptionType; + static const Field _f$exceptionType = Field( + 'exceptionType', + _$exceptionType, + key: r'exception_type', + opt: true, + ); + static String? _$exceptionMessage(FlutterBenchTrialDetail v) => + v.exceptionMessage; + static const Field _f$exceptionMessage = + Field( + 'exceptionMessage', + _$exceptionMessage, + key: r'exception_message', + opt: true, + ); + static String? _$exceptionTraceback(FlutterBenchTrialDetail v) => + v.exceptionTraceback; + static const Field _f$exceptionTraceback = + Field( + 'exceptionTraceback', + _$exceptionTraceback, + key: r'exception_traceback', + opt: true, + ); + static Map _$durations(FlutterBenchTrialDetail v) => + v.durations; + static const Field> + _f$durations = Field('durations', _$durations, opt: true, def: const {}); + static int _$inputTokens(FlutterBenchTrialDetail v) => v.inputTokens; + static const Field _f$inputTokens = Field( + 'inputTokens', + _$inputTokens, + key: r'input_tokens', + ); + static int _$cacheTokens(FlutterBenchTrialDetail v) => v.cacheTokens; + static const Field _f$cacheTokens = Field( + 'cacheTokens', + _$cacheTokens, + key: r'cache_tokens', + ); + static int _$outputTokens(FlutterBenchTrialDetail v) => v.outputTokens; + static const Field _f$outputTokens = Field( + 'outputTokens', + _$outputTokens, + key: r'output_tokens', + ); + static double _$costUsd(FlutterBenchTrialDetail v) => v.costUsd; + static const Field _f$costUsd = Field( + 'costUsd', + _$costUsd, + key: r'cost_usd', + ); + static Map? _$rewardTree(FlutterBenchTrialDetail v) => + v.rewardTree; + static const Field> + _f$rewardTree = Field( + 'rewardTree', + _$rewardTree, + key: r'reward_tree', + opt: true, + ); + static Map _$diagnosticTree(FlutterBenchTrialDetail v) => + v.diagnosticTree; + static const Field> + _f$diagnosticTree = Field( + 'diagnosticTree', + _$diagnosticTree, + key: r'diagnostic_tree', + opt: true, + def: const {}, + ); + static List>? _$trajectory(FlutterBenchTrialDetail v) => + v.trajectory; + static const Field>> + _f$trajectory = Field('trajectory', _$trajectory, opt: true); + static List> _$artifacts(FlutterBenchTrialDetail v) => + v.artifacts; + static const Field>> + _f$artifacts = Field('artifacts', _$artifacts, opt: true, def: const []); + static String? _$testStdout(FlutterBenchTrialDetail v) => v.testStdout; + static const Field _f$testStdout = Field( + 'testStdout', + _$testStdout, + key: r'test_stdout', + opt: true, + ); + static String? _$exceptionLog(FlutterBenchTrialDetail v) => v.exceptionLog; + static const Field _f$exceptionLog = Field( + 'exceptionLog', + _$exceptionLog, + key: r'exception_log', + opt: true, + ); + + @override + final MappableFields fields = const { + #trialName: _f$trialName, + #taskName: _f$taskName, + #taskSlug: _f$taskSlug, + #agentName: _f$agentName, + #modelName: _f$modelName, + #modelShortName: _f$modelShortName, + #provider: _f$provider, + #skills: _f$skills, + #mcpServers: _f$mcpServers, + #hasDartTooling: _f$hasDartTooling, + #status: _f$status, + #reward: _f$reward, + #exceptionType: _f$exceptionType, + #exceptionMessage: _f$exceptionMessage, + #exceptionTraceback: _f$exceptionTraceback, + #durations: _f$durations, + #inputTokens: _f$inputTokens, + #cacheTokens: _f$cacheTokens, + #outputTokens: _f$outputTokens, + #costUsd: _f$costUsd, + #rewardTree: _f$rewardTree, + #diagnosticTree: _f$diagnosticTree, + #trajectory: _f$trajectory, + #artifacts: _f$artifacts, + #testStdout: _f$testStdout, + #exceptionLog: _f$exceptionLog, + }; + + static FlutterBenchTrialDetail _instantiate(DecodingData data) { + return FlutterBenchTrialDetail( + trialName: data.dec(_f$trialName), + taskName: data.dec(_f$taskName), + taskSlug: data.dec(_f$taskSlug), + agentName: data.dec(_f$agentName), + modelName: data.dec(_f$modelName), + modelShortName: data.dec(_f$modelShortName), + provider: data.dec(_f$provider), + skills: data.dec(_f$skills), + mcpServers: data.dec(_f$mcpServers), + hasDartTooling: data.dec(_f$hasDartTooling), + status: data.dec(_f$status), + reward: data.dec(_f$reward), + exceptionType: data.dec(_f$exceptionType), + exceptionMessage: data.dec(_f$exceptionMessage), + exceptionTraceback: data.dec(_f$exceptionTraceback), + durations: data.dec(_f$durations), + inputTokens: data.dec(_f$inputTokens), + cacheTokens: data.dec(_f$cacheTokens), + outputTokens: data.dec(_f$outputTokens), + costUsd: data.dec(_f$costUsd), + rewardTree: data.dec(_f$rewardTree), + diagnosticTree: data.dec(_f$diagnosticTree), + trajectory: data.dec(_f$trajectory), + artifacts: data.dec(_f$artifacts), + testStdout: data.dec(_f$testStdout), + exceptionLog: data.dec(_f$exceptionLog), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTrialDetail fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTrialDetail fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTrialDetailMappable { + String toJson() { + return FlutterBenchTrialDetailMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTrialDetail); + } + + Map toMap() { + return FlutterBenchTrialDetailMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTrialDetail); + } + + FlutterBenchTrialDetailCopyWith< + FlutterBenchTrialDetail, + FlutterBenchTrialDetail, + FlutterBenchTrialDetail + > + get copyWith => + _FlutterBenchTrialDetailCopyWithImpl< + FlutterBenchTrialDetail, + FlutterBenchTrialDetail + >(this as FlutterBenchTrialDetail, $identity, $identity); + @override + String toString() { + return FlutterBenchTrialDetailMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTrialDetail, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTrialDetailMapper.ensureInitialized().equalsValue( + this as FlutterBenchTrialDetail, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTrialDetailMapper.ensureInitialized().hashValue( + this as FlutterBenchTrialDetail, + ); + } +} + +extension FlutterBenchTrialDetailValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTrialDetail, $Out> { + FlutterBenchTrialDetailCopyWith<$R, FlutterBenchTrialDetail, $Out> + get $asFlutterBenchTrialDetail => $base.as( + (v, t, t2) => _FlutterBenchTrialDetailCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTrialDetailCopyWith< + $R, + $In extends FlutterBenchTrialDetail, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith<$R, String, ObjectCopyWith<$R, String, String>> get skills; + ListCopyWith<$R, String, ObjectCopyWith<$R, String, String>> get mcpServers; + MapCopyWith<$R, String, double, ObjectCopyWith<$R, double, double>> + get durations; + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?>? + get rewardTree; + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get diagnosticTree; + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + >? + get trajectory; + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + > + get artifacts; + $R call({ + String? trialName, + String? taskName, + String? taskSlug, + String? agentName, + String? modelName, + String? modelShortName, + String? provider, + List? skills, + List? mcpServers, + bool? hasDartTooling, + String? status, + double? reward, + String? exceptionType, + String? exceptionMessage, + String? exceptionTraceback, + Map? durations, + int? inputTokens, + int? cacheTokens, + int? outputTokens, + double? costUsd, + Map? rewardTree, + Map? diagnosticTree, + List>? trajectory, + List>? artifacts, + String? testStdout, + String? exceptionLog, + }); + FlutterBenchTrialDetailCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTrialDetailCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTrialDetail, $Out> + implements + FlutterBenchTrialDetailCopyWith<$R, FlutterBenchTrialDetail, $Out> { + _FlutterBenchTrialDetailCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTrialDetailMapper.ensureInitialized(); + @override + ListCopyWith<$R, String, ObjectCopyWith<$R, String, String>> get skills => + ListCopyWith( + $value.skills, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(skills: v), + ); + @override + ListCopyWith<$R, String, ObjectCopyWith<$R, String, String>> get mcpServers => + ListCopyWith( + $value.mcpServers, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(mcpServers: v), + ); + @override + MapCopyWith<$R, String, double, ObjectCopyWith<$R, double, double>> + get durations => MapCopyWith( + $value.durations, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(durations: v), + ); + @override + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?>? + get rewardTree => $value.rewardTree != null + ? MapCopyWith( + $value.rewardTree!, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(rewardTree: v), + ) + : null; + @override + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get diagnosticTree => MapCopyWith( + $value.diagnosticTree, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(diagnosticTree: v), + ); + @override + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + >? + get trajectory => $value.trajectory != null + ? ListCopyWith( + $value.trajectory!, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(trajectory: v), + ) + : null; + @override + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + > + get artifacts => ListCopyWith( + $value.artifacts, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(artifacts: v), + ); + @override + $R call({ + String? trialName, + String? taskName, + String? taskSlug, + String? agentName, + String? modelName, + String? modelShortName, + String? provider, + List? skills, + List? mcpServers, + bool? hasDartTooling, + String? status, + Object? reward = $none, + Object? exceptionType = $none, + Object? exceptionMessage = $none, + Object? exceptionTraceback = $none, + Map? durations, + int? inputTokens, + int? cacheTokens, + int? outputTokens, + double? costUsd, + Object? rewardTree = $none, + Map? diagnosticTree, + Object? trajectory = $none, + List>? artifacts, + Object? testStdout = $none, + Object? exceptionLog = $none, + }) => $apply( + FieldCopyWithData({ + if (trialName != null) #trialName: trialName, + if (taskName != null) #taskName: taskName, + if (taskSlug != null) #taskSlug: taskSlug, + if (agentName != null) #agentName: agentName, + if (modelName != null) #modelName: modelName, + if (modelShortName != null) #modelShortName: modelShortName, + if (provider != null) #provider: provider, + if (skills != null) #skills: skills, + if (mcpServers != null) #mcpServers: mcpServers, + if (hasDartTooling != null) #hasDartTooling: hasDartTooling, + if (status != null) #status: status, + if (reward != $none) #reward: reward, + if (exceptionType != $none) #exceptionType: exceptionType, + if (exceptionMessage != $none) #exceptionMessage: exceptionMessage, + if (exceptionTraceback != $none) #exceptionTraceback: exceptionTraceback, + if (durations != null) #durations: durations, + if (inputTokens != null) #inputTokens: inputTokens, + if (cacheTokens != null) #cacheTokens: cacheTokens, + if (outputTokens != null) #outputTokens: outputTokens, + if (costUsd != null) #costUsd: costUsd, + if (rewardTree != $none) #rewardTree: rewardTree, + if (diagnosticTree != null) #diagnosticTree: diagnosticTree, + if (trajectory != $none) #trajectory: trajectory, + if (artifacts != null) #artifacts: artifacts, + if (testStdout != $none) #testStdout: testStdout, + if (exceptionLog != $none) #exceptionLog: exceptionLog, + }), + ); + @override + FlutterBenchTrialDetail $make(CopyWithData data) => FlutterBenchTrialDetail( + trialName: data.get(#trialName, or: $value.trialName), + taskName: data.get(#taskName, or: $value.taskName), + taskSlug: data.get(#taskSlug, or: $value.taskSlug), + agentName: data.get(#agentName, or: $value.agentName), + modelName: data.get(#modelName, or: $value.modelName), + modelShortName: data.get(#modelShortName, or: $value.modelShortName), + provider: data.get(#provider, or: $value.provider), + skills: data.get(#skills, or: $value.skills), + mcpServers: data.get(#mcpServers, or: $value.mcpServers), + hasDartTooling: data.get(#hasDartTooling, or: $value.hasDartTooling), + status: data.get(#status, or: $value.status), + reward: data.get(#reward, or: $value.reward), + exceptionType: data.get(#exceptionType, or: $value.exceptionType), + exceptionMessage: data.get(#exceptionMessage, or: $value.exceptionMessage), + exceptionTraceback: data.get( + #exceptionTraceback, + or: $value.exceptionTraceback, + ), + durations: data.get(#durations, or: $value.durations), + inputTokens: data.get(#inputTokens, or: $value.inputTokens), + cacheTokens: data.get(#cacheTokens, or: $value.cacheTokens), + outputTokens: data.get(#outputTokens, or: $value.outputTokens), + costUsd: data.get(#costUsd, or: $value.costUsd), + rewardTree: data.get(#rewardTree, or: $value.rewardTree), + diagnosticTree: data.get(#diagnosticTree, or: $value.diagnosticTree), + trajectory: data.get(#trajectory, or: $value.trajectory), + artifacts: data.get(#artifacts, or: $value.artifacts), + testStdout: data.get(#testStdout, or: $value.testStdout), + exceptionLog: data.get(#exceptionLog, or: $value.exceptionLog), + ); + + @override + FlutterBenchTrialDetailCopyWith<$R2, FlutterBenchTrialDetail, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTrialDetailCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchMethodologyDataMapper + extends ClassMapperBase { + FlutterBenchMethodologyDataMapper._(); + + static FlutterBenchMethodologyDataMapper? _instance; + static FlutterBenchMethodologyDataMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchMethodologyDataMapper._(), + ); + FlutterBenchMethodologyOverviewMapper.ensureInitialized(); + FlutterBenchTaskAnatomyMapper.ensureInitialized(); + FlutterBenchTableSectionMapper.ensureInitialized(); + FlutterBenchItemListMapper.ensureInitialized(); + FlutterBenchTransparencyMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchMethodologyData'; + + static FlutterBenchMethodologyOverview _$overview( + FlutterBenchMethodologyData v, + ) => v.overview; + static const Field< + FlutterBenchMethodologyData, + FlutterBenchMethodologyOverview + > + _f$overview = Field('overview', _$overview); + static List> _$cujExample( + FlutterBenchMethodologyData v, + ) => v.cujExample; + static const Field>> + _f$cujExample = Field( + 'cujExample', + _$cujExample, + key: r'cuj_example', + opt: true, + def: const [], + ); + static List> _$taskSpecifications( + FlutterBenchMethodologyData v, + ) => v.taskSpecifications; + static const Field>> + _f$taskSpecifications = Field( + 'taskSpecifications', + _$taskSpecifications, + key: r'task_specifications', + opt: true, + def: const [], + ); + static FlutterBenchTaskAnatomy _$taskAnatomy(FlutterBenchMethodologyData v) => + v.taskAnatomy; + static const Field + _f$taskAnatomy = Field('taskAnatomy', _$taskAnatomy, key: r'task_anatomy'); + static Map _$evaluationMatrix( + FlutterBenchMethodologyData v, + ) => v.evaluationMatrix; + static const Field> + _f$evaluationMatrix = Field( + 'evaluationMatrix', + _$evaluationMatrix, + key: r'evaluation_matrix', + opt: true, + def: const {}, + ); + static List> _$dimensions( + FlutterBenchMethodologyData v, + ) => v.dimensions; + static const Field>> + _f$dimensions = Field('dimensions', _$dimensions, opt: true, def: const []); + static Map _$graderMatrix(FlutterBenchMethodologyData v) => + v.graderMatrix; + static const Field> + _f$graderMatrix = Field( + 'graderMatrix', + _$graderMatrix, + key: r'grader_matrix', + opt: true, + def: const {}, + ); + static FlutterBenchTableSection _$graderTiers( + FlutterBenchMethodologyData v, + ) => v.graderTiers; + static const Field + _f$graderTiers = Field('graderTiers', _$graderTiers, key: r'grader_tiers'); + static FlutterBenchTableSection _$diagnosticTelemetry( + FlutterBenchMethodologyData v, + ) => v.diagnosticTelemetry; + static const Field + _f$diagnosticTelemetry = Field( + 'diagnosticTelemetry', + _$diagnosticTelemetry, + key: r'diagnostic_telemetry', + ); + static Map _$reliability(FlutterBenchMethodologyData v) => + v.reliability; + static const Field> + _f$reliability = Field( + 'reliability', + _$reliability, + opt: true, + def: const {}, + ); + static Map _$scoreTriage(FlutterBenchMethodologyData v) => + v.scoreTriage; + static const Field> + _f$scoreTriage = Field( + 'scoreTriage', + _$scoreTriage, + key: r'score_triage', + opt: true, + def: const {}, + ); + static FlutterBenchItemList _$rootCauseAudits( + FlutterBenchMethodologyData v, + ) => v.rootCauseAudits; + static const Field + _f$rootCauseAudits = Field( + 'rootCauseAudits', + _$rootCauseAudits, + key: r'root_cause_audits', + ); + static FlutterBenchTransparency _$transparency( + FlutterBenchMethodologyData v, + ) => v.transparency; + static const Field + _f$transparency = Field('transparency', _$transparency); + + @override + final MappableFields fields = const { + #overview: _f$overview, + #cujExample: _f$cujExample, + #taskSpecifications: _f$taskSpecifications, + #taskAnatomy: _f$taskAnatomy, + #evaluationMatrix: _f$evaluationMatrix, + #dimensions: _f$dimensions, + #graderMatrix: _f$graderMatrix, + #graderTiers: _f$graderTiers, + #diagnosticTelemetry: _f$diagnosticTelemetry, + #reliability: _f$reliability, + #scoreTriage: _f$scoreTriage, + #rootCauseAudits: _f$rootCauseAudits, + #transparency: _f$transparency, + }; + + static FlutterBenchMethodologyData _instantiate(DecodingData data) { + return FlutterBenchMethodologyData( + overview: data.dec(_f$overview), + cujExample: data.dec(_f$cujExample), + taskSpecifications: data.dec(_f$taskSpecifications), + taskAnatomy: data.dec(_f$taskAnatomy), + evaluationMatrix: data.dec(_f$evaluationMatrix), + dimensions: data.dec(_f$dimensions), + graderMatrix: data.dec(_f$graderMatrix), + graderTiers: data.dec(_f$graderTiers), + diagnosticTelemetry: data.dec(_f$diagnosticTelemetry), + reliability: data.dec(_f$reliability), + scoreTriage: data.dec(_f$scoreTriage), + rootCauseAudits: data.dec(_f$rootCauseAudits), + transparency: data.dec(_f$transparency), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchMethodologyData fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchMethodologyData fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchMethodologyDataMappable { + String toJson() { + return FlutterBenchMethodologyDataMapper.ensureInitialized() + .encodeJson( + this as FlutterBenchMethodologyData, + ); + } + + Map toMap() { + return FlutterBenchMethodologyDataMapper.ensureInitialized() + .encodeMap( + this as FlutterBenchMethodologyData, + ); + } + + FlutterBenchMethodologyDataCopyWith< + FlutterBenchMethodologyData, + FlutterBenchMethodologyData, + FlutterBenchMethodologyData + > + get copyWith => + _FlutterBenchMethodologyDataCopyWithImpl< + FlutterBenchMethodologyData, + FlutterBenchMethodologyData + >(this as FlutterBenchMethodologyData, $identity, $identity); + @override + String toString() { + return FlutterBenchMethodologyDataMapper.ensureInitialized().stringifyValue( + this as FlutterBenchMethodologyData, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchMethodologyDataMapper.ensureInitialized().equalsValue( + this as FlutterBenchMethodologyData, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchMethodologyDataMapper.ensureInitialized().hashValue( + this as FlutterBenchMethodologyData, + ); + } +} + +extension FlutterBenchMethodologyDataValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchMethodologyData, $Out> { + FlutterBenchMethodologyDataCopyWith<$R, FlutterBenchMethodologyData, $Out> + get $asFlutterBenchMethodologyData => $base.as( + (v, t, t2) => _FlutterBenchMethodologyDataCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchMethodologyDataCopyWith< + $R, + $In extends FlutterBenchMethodologyData, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + FlutterBenchMethodologyOverviewCopyWith< + $R, + FlutterBenchMethodologyOverview, + FlutterBenchMethodologyOverview + > + get overview; + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + > + get cujExample; + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + > + get taskSpecifications; + FlutterBenchTaskAnatomyCopyWith< + $R, + FlutterBenchTaskAnatomy, + FlutterBenchTaskAnatomy + > + get taskAnatomy; + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get evaluationMatrix; + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + > + get dimensions; + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get graderMatrix; + FlutterBenchTableSectionCopyWith< + $R, + FlutterBenchTableSection, + FlutterBenchTableSection + > + get graderTiers; + FlutterBenchTableSectionCopyWith< + $R, + FlutterBenchTableSection, + FlutterBenchTableSection + > + get diagnosticTelemetry; + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get reliability; + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get scoreTriage; + FlutterBenchItemListCopyWith<$R, FlutterBenchItemList, FlutterBenchItemList> + get rootCauseAudits; + FlutterBenchTransparencyCopyWith< + $R, + FlutterBenchTransparency, + FlutterBenchTransparency + > + get transparency; + $R call({ + FlutterBenchMethodologyOverview? overview, + List>? cujExample, + List>? taskSpecifications, + FlutterBenchTaskAnatomy? taskAnatomy, + Map? evaluationMatrix, + List>? dimensions, + Map? graderMatrix, + FlutterBenchTableSection? graderTiers, + FlutterBenchTableSection? diagnosticTelemetry, + Map? reliability, + Map? scoreTriage, + FlutterBenchItemList? rootCauseAudits, + FlutterBenchTransparency? transparency, + }); + FlutterBenchMethodologyDataCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchMethodologyDataCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchMethodologyData, $Out> + implements + FlutterBenchMethodologyDataCopyWith< + $R, + FlutterBenchMethodologyData, + $Out + > { + _FlutterBenchMethodologyDataCopyWithImpl( + super.value, + super.then, + super.then2, + ); + + @override + late final ClassMapperBase $mapper = + FlutterBenchMethodologyDataMapper.ensureInitialized(); + @override + FlutterBenchMethodologyOverviewCopyWith< + $R, + FlutterBenchMethodologyOverview, + FlutterBenchMethodologyOverview + > + get overview => $value.overview.copyWith.$chain((v) => call(overview: v)); + @override + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + > + get cujExample => ListCopyWith( + $value.cujExample, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(cujExample: v), + ); + @override + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + > + get taskSpecifications => ListCopyWith( + $value.taskSpecifications, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(taskSpecifications: v), + ); + @override + FlutterBenchTaskAnatomyCopyWith< + $R, + FlutterBenchTaskAnatomy, + FlutterBenchTaskAnatomy + > + get taskAnatomy => + $value.taskAnatomy.copyWith.$chain((v) => call(taskAnatomy: v)); + @override + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get evaluationMatrix => MapCopyWith( + $value.evaluationMatrix, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(evaluationMatrix: v), + ); + @override + ListCopyWith< + $R, + Map, + ObjectCopyWith<$R, Map, Map> + > + get dimensions => ListCopyWith( + $value.dimensions, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(dimensions: v), + ); + @override + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get graderMatrix => MapCopyWith( + $value.graderMatrix, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(graderMatrix: v), + ); + @override + FlutterBenchTableSectionCopyWith< + $R, + FlutterBenchTableSection, + FlutterBenchTableSection + > + get graderTiers => + $value.graderTiers.copyWith.$chain((v) => call(graderTiers: v)); + @override + FlutterBenchTableSectionCopyWith< + $R, + FlutterBenchTableSection, + FlutterBenchTableSection + > + get diagnosticTelemetry => $value.diagnosticTelemetry.copyWith.$chain( + (v) => call(diagnosticTelemetry: v), + ); + @override + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get reliability => MapCopyWith( + $value.reliability, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(reliability: v), + ); + @override + MapCopyWith<$R, String, Object?, ObjectCopyWith<$R, Object?, Object?>?> + get scoreTriage => MapCopyWith( + $value.scoreTriage, + (v, t) => ObjectCopyWith(v, $identity, t), + (v) => call(scoreTriage: v), + ); + @override + FlutterBenchItemListCopyWith<$R, FlutterBenchItemList, FlutterBenchItemList> + get rootCauseAudits => + $value.rootCauseAudits.copyWith.$chain((v) => call(rootCauseAudits: v)); + @override + FlutterBenchTransparencyCopyWith< + $R, + FlutterBenchTransparency, + FlutterBenchTransparency + > + get transparency => + $value.transparency.copyWith.$chain((v) => call(transparency: v)); + @override + $R call({ + FlutterBenchMethodologyOverview? overview, + List>? cujExample, + List>? taskSpecifications, + FlutterBenchTaskAnatomy? taskAnatomy, + Map? evaluationMatrix, + List>? dimensions, + Map? graderMatrix, + FlutterBenchTableSection? graderTiers, + FlutterBenchTableSection? diagnosticTelemetry, + Map? reliability, + Map? scoreTriage, + FlutterBenchItemList? rootCauseAudits, + FlutterBenchTransparency? transparency, + }) => $apply( + FieldCopyWithData({ + if (overview != null) #overview: overview, + if (cujExample != null) #cujExample: cujExample, + if (taskSpecifications != null) #taskSpecifications: taskSpecifications, + if (taskAnatomy != null) #taskAnatomy: taskAnatomy, + if (evaluationMatrix != null) #evaluationMatrix: evaluationMatrix, + if (dimensions != null) #dimensions: dimensions, + if (graderMatrix != null) #graderMatrix: graderMatrix, + if (graderTiers != null) #graderTiers: graderTiers, + if (diagnosticTelemetry != null) + #diagnosticTelemetry: diagnosticTelemetry, + if (reliability != null) #reliability: reliability, + if (scoreTriage != null) #scoreTriage: scoreTriage, + if (rootCauseAudits != null) #rootCauseAudits: rootCauseAudits, + if (transparency != null) #transparency: transparency, + }), + ); + @override + FlutterBenchMethodologyData $make(CopyWithData data) => + FlutterBenchMethodologyData( + overview: data.get(#overview, or: $value.overview), + cujExample: data.get(#cujExample, or: $value.cujExample), + taskSpecifications: data.get( + #taskSpecifications, + or: $value.taskSpecifications, + ), + taskAnatomy: data.get(#taskAnatomy, or: $value.taskAnatomy), + evaluationMatrix: data.get( + #evaluationMatrix, + or: $value.evaluationMatrix, + ), + dimensions: data.get(#dimensions, or: $value.dimensions), + graderMatrix: data.get(#graderMatrix, or: $value.graderMatrix), + graderTiers: data.get(#graderTiers, or: $value.graderTiers), + diagnosticTelemetry: data.get( + #diagnosticTelemetry, + or: $value.diagnosticTelemetry, + ), + reliability: data.get(#reliability, or: $value.reliability), + scoreTriage: data.get(#scoreTriage, or: $value.scoreTriage), + rootCauseAudits: data.get(#rootCauseAudits, or: $value.rootCauseAudits), + transparency: data.get(#transparency, or: $value.transparency), + ); + + @override + FlutterBenchMethodologyDataCopyWith<$R2, FlutterBenchMethodologyData, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchMethodologyDataCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchMethodologyOverviewMapper + extends ClassMapperBase { + FlutterBenchMethodologyOverviewMapper._(); + + static FlutterBenchMethodologyOverviewMapper? _instance; + static FlutterBenchMethodologyOverviewMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchMethodologyOverviewMapper._(), + ); + FlutterBenchTableRowMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchMethodologyOverview'; + + static String _$leadText(FlutterBenchMethodologyOverview v) => v.leadText; + static const Field _f$leadText = + Field('leadText', _$leadText, key: r'lead_text'); + static List _$rows(FlutterBenchMethodologyOverview v) => + v.rows; + static const Field< + FlutterBenchMethodologyOverview, + List + > + _f$rows = Field('rows', _$rows); + + @override + final MappableFields fields = const { + #leadText: _f$leadText, + #rows: _f$rows, + }; + + static FlutterBenchMethodologyOverview _instantiate(DecodingData data) { + return FlutterBenchMethodologyOverview( + leadText: data.dec(_f$leadText), + rows: data.dec(_f$rows), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchMethodologyOverview fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchMethodologyOverview fromJson(String json) { + return ensureInitialized().decodeJson( + json, + ); + } +} + +mixin FlutterBenchMethodologyOverviewMappable { + String toJson() { + return FlutterBenchMethodologyOverviewMapper.ensureInitialized() + .encodeJson( + this as FlutterBenchMethodologyOverview, + ); + } + + Map toMap() { + return FlutterBenchMethodologyOverviewMapper.ensureInitialized() + .encodeMap( + this as FlutterBenchMethodologyOverview, + ); + } + + FlutterBenchMethodologyOverviewCopyWith< + FlutterBenchMethodologyOverview, + FlutterBenchMethodologyOverview, + FlutterBenchMethodologyOverview + > + get copyWith => + _FlutterBenchMethodologyOverviewCopyWithImpl< + FlutterBenchMethodologyOverview, + FlutterBenchMethodologyOverview + >(this as FlutterBenchMethodologyOverview, $identity, $identity); + @override + String toString() { + return FlutterBenchMethodologyOverviewMapper.ensureInitialized() + .stringifyValue(this as FlutterBenchMethodologyOverview); + } + + @override + bool operator ==(Object other) { + return FlutterBenchMethodologyOverviewMapper.ensureInitialized() + .equalsValue(this as FlutterBenchMethodologyOverview, other); + } + + @override + int get hashCode { + return FlutterBenchMethodologyOverviewMapper.ensureInitialized().hashValue( + this as FlutterBenchMethodologyOverview, + ); + } +} + +extension FlutterBenchMethodologyOverviewValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchMethodologyOverview, $Out> { + FlutterBenchMethodologyOverviewCopyWith< + $R, + FlutterBenchMethodologyOverview, + $Out + > + get $asFlutterBenchMethodologyOverview => $base.as( + (v, t, t2) => + _FlutterBenchMethodologyOverviewCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchMethodologyOverviewCopyWith< + $R, + $In extends FlutterBenchMethodologyOverview, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchTableRow, + FlutterBenchTableRowCopyWith<$R, FlutterBenchTableRow, FlutterBenchTableRow> + > + get rows; + $R call({String? leadText, List? rows}); + FlutterBenchMethodologyOverviewCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchMethodologyOverviewCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchMethodologyOverview, $Out> + implements + FlutterBenchMethodologyOverviewCopyWith< + $R, + FlutterBenchMethodologyOverview, + $Out + > { + _FlutterBenchMethodologyOverviewCopyWithImpl( + super.value, + super.then, + super.then2, + ); + + @override + late final ClassMapperBase $mapper = + FlutterBenchMethodologyOverviewMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchTableRow, + FlutterBenchTableRowCopyWith<$R, FlutterBenchTableRow, FlutterBenchTableRow> + > + get rows => ListCopyWith( + $value.rows, + (v, t) => v.copyWith.$chain(t), + (v) => call(rows: v), + ); + @override + $R call({String? leadText, List? rows}) => $apply( + FieldCopyWithData({ + if (leadText != null) #leadText: leadText, + if (rows != null) #rows: rows, + }), + ); + @override + FlutterBenchMethodologyOverview $make(CopyWithData data) => + FlutterBenchMethodologyOverview( + leadText: data.get(#leadText, or: $value.leadText), + rows: data.get(#rows, or: $value.rows), + ); + + @override + FlutterBenchMethodologyOverviewCopyWith< + $R2, + FlutterBenchMethodologyOverview, + $Out2 + > + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchMethodologyOverviewCopyWithImpl<$R2, $Out2>( + $value, + $cast, + t, + ); +} + +class FlutterBenchTableRowMapper extends ClassMapperBase { + FlutterBenchTableRowMapper._(); + + static FlutterBenchTableRowMapper? _instance; + static FlutterBenchTableRowMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchTableRowMapper._()); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTableRow'; + + static String _$label(FlutterBenchTableRow v) => v.label; + static const Field _f$label = Field( + 'label', + _$label, + ); + static String? _$detail(FlutterBenchTableRow v) => v.detail; + static const Field _f$detail = Field( + 'detail', + _$detail, + opt: true, + ); + static String _$description(FlutterBenchTableRow v) => v.description; + static const Field _f$description = Field( + 'description', + _$description, + ); + static String? _$anchor(FlutterBenchTableRow v) => v.anchor; + static const Field _f$anchor = Field( + 'anchor', + _$anchor, + opt: true, + ); + + @override + final MappableFields fields = const { + #label: _f$label, + #detail: _f$detail, + #description: _f$description, + #anchor: _f$anchor, + }; + + static FlutterBenchTableRow _instantiate(DecodingData data) { + return FlutterBenchTableRow( + label: data.dec(_f$label), + detail: data.dec(_f$detail), + description: data.dec(_f$description), + anchor: data.dec(_f$anchor), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTableRow fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTableRow fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTableRowMappable { + String toJson() { + return FlutterBenchTableRowMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTableRow); + } + + Map toMap() { + return FlutterBenchTableRowMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTableRow); + } + + FlutterBenchTableRowCopyWith< + FlutterBenchTableRow, + FlutterBenchTableRow, + FlutterBenchTableRow + > + get copyWith => + _FlutterBenchTableRowCopyWithImpl< + FlutterBenchTableRow, + FlutterBenchTableRow + >(this as FlutterBenchTableRow, $identity, $identity); + @override + String toString() { + return FlutterBenchTableRowMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTableRow, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTableRowMapper.ensureInitialized().equalsValue( + this as FlutterBenchTableRow, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTableRowMapper.ensureInitialized().hashValue( + this as FlutterBenchTableRow, + ); + } +} + +extension FlutterBenchTableRowValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTableRow, $Out> { + FlutterBenchTableRowCopyWith<$R, FlutterBenchTableRow, $Out> + get $asFlutterBenchTableRow => $base.as( + (v, t, t2) => _FlutterBenchTableRowCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTableRowCopyWith< + $R, + $In extends FlutterBenchTableRow, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + $R call({String? label, String? detail, String? description, String? anchor}); + FlutterBenchTableRowCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTableRowCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTableRow, $Out> + implements FlutterBenchTableRowCopyWith<$R, FlutterBenchTableRow, $Out> { + _FlutterBenchTableRowCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTableRowMapper.ensureInitialized(); + @override + $R call({ + String? label, + Object? detail = $none, + String? description, + Object? anchor = $none, + }) => $apply( + FieldCopyWithData({ + if (label != null) #label: label, + if (detail != $none) #detail: detail, + if (description != null) #description: description, + if (anchor != $none) #anchor: anchor, + }), + ); + @override + FlutterBenchTableRow $make(CopyWithData data) => FlutterBenchTableRow( + label: data.get(#label, or: $value.label), + detail: data.get(#detail, or: $value.detail), + description: data.get(#description, or: $value.description), + anchor: data.get(#anchor, or: $value.anchor), + ); + + @override + FlutterBenchTableRowCopyWith<$R2, FlutterBenchTableRow, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTableRowCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchTaskAnatomyMapper + extends ClassMapperBase { + FlutterBenchTaskAnatomyMapper._(); + + static FlutterBenchTaskAnatomyMapper? _instance; + static FlutterBenchTaskAnatomyMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchTaskAnatomyMapper._(), + ); + FlutterBenchTaskTreeNodeMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTaskAnatomy'; + + static String _$introText(FlutterBenchTaskAnatomy v) => v.introText; + static const Field _f$introText = Field( + 'introText', + _$introText, + key: r'intro_text', + ); + static String _$rootId(FlutterBenchTaskAnatomy v) => v.rootId; + static const Field _f$rootId = Field( + 'rootId', + _$rootId, + key: r'root_id', + ); + static String _$rootLabel(FlutterBenchTaskAnatomy v) => v.rootLabel; + static const Field _f$rootLabel = Field( + 'rootLabel', + _$rootLabel, + key: r'root_label', + ); + static List _$tree(FlutterBenchTaskAnatomy v) => + v.tree; + static const Field> + _f$tree = Field('tree', _$tree); + + @override + final MappableFields fields = const { + #introText: _f$introText, + #rootId: _f$rootId, + #rootLabel: _f$rootLabel, + #tree: _f$tree, + }; + + static FlutterBenchTaskAnatomy _instantiate(DecodingData data) { + return FlutterBenchTaskAnatomy( + introText: data.dec(_f$introText), + rootId: data.dec(_f$rootId), + rootLabel: data.dec(_f$rootLabel), + tree: data.dec(_f$tree), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTaskAnatomy fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTaskAnatomy fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTaskAnatomyMappable { + String toJson() { + return FlutterBenchTaskAnatomyMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTaskAnatomy); + } + + Map toMap() { + return FlutterBenchTaskAnatomyMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTaskAnatomy); + } + + FlutterBenchTaskAnatomyCopyWith< + FlutterBenchTaskAnatomy, + FlutterBenchTaskAnatomy, + FlutterBenchTaskAnatomy + > + get copyWith => + _FlutterBenchTaskAnatomyCopyWithImpl< + FlutterBenchTaskAnatomy, + FlutterBenchTaskAnatomy + >(this as FlutterBenchTaskAnatomy, $identity, $identity); + @override + String toString() { + return FlutterBenchTaskAnatomyMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTaskAnatomy, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTaskAnatomyMapper.ensureInitialized().equalsValue( + this as FlutterBenchTaskAnatomy, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTaskAnatomyMapper.ensureInitialized().hashValue( + this as FlutterBenchTaskAnatomy, + ); + } +} + +extension FlutterBenchTaskAnatomyValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTaskAnatomy, $Out> { + FlutterBenchTaskAnatomyCopyWith<$R, FlutterBenchTaskAnatomy, $Out> + get $asFlutterBenchTaskAnatomy => $base.as( + (v, t, t2) => _FlutterBenchTaskAnatomyCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTaskAnatomyCopyWith< + $R, + $In extends FlutterBenchTaskAnatomy, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNodeCopyWith< + $R, + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNode + > + > + get tree; + $R call({ + String? introText, + String? rootId, + String? rootLabel, + List? tree, + }); + FlutterBenchTaskAnatomyCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTaskAnatomyCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTaskAnatomy, $Out> + implements + FlutterBenchTaskAnatomyCopyWith<$R, FlutterBenchTaskAnatomy, $Out> { + _FlutterBenchTaskAnatomyCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTaskAnatomyMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNodeCopyWith< + $R, + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNode + > + > + get tree => ListCopyWith( + $value.tree, + (v, t) => v.copyWith.$chain(t), + (v) => call(tree: v), + ); + @override + $R call({ + String? introText, + String? rootId, + String? rootLabel, + List? tree, + }) => $apply( + FieldCopyWithData({ + if (introText != null) #introText: introText, + if (rootId != null) #rootId: rootId, + if (rootLabel != null) #rootLabel: rootLabel, + if (tree != null) #tree: tree, + }), + ); + @override + FlutterBenchTaskAnatomy $make(CopyWithData data) => FlutterBenchTaskAnatomy( + introText: data.get(#introText, or: $value.introText), + rootId: data.get(#rootId, or: $value.rootId), + rootLabel: data.get(#rootLabel, or: $value.rootLabel), + tree: data.get(#tree, or: $value.tree), + ); + + @override + FlutterBenchTaskAnatomyCopyWith<$R2, FlutterBenchTaskAnatomy, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTaskAnatomyCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchTaskTreeNodeMapper + extends ClassMapperBase { + FlutterBenchTaskTreeNodeMapper._(); + + static FlutterBenchTaskTreeNodeMapper? _instance; + static FlutterBenchTaskTreeNodeMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchTaskTreeNodeMapper._(), + ); + FlutterBenchCodeSampleMapper.ensureInitialized(); + FlutterBenchTaskTreeNodeMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTaskTreeNode'; + + static String _$type(FlutterBenchTaskTreeNode v) => v.type; + static const Field _f$type = Field( + 'type', + _$type, + ); + static String _$id(FlutterBenchTaskTreeNode v) => v.id; + static const Field _f$id = Field( + 'id', + _$id, + ); + static String _$label(FlutterBenchTaskTreeNode v) => v.label; + static const Field _f$label = Field( + 'label', + _$label, + ); + static String? _$subtitle(FlutterBenchTaskTreeNode v) => v.subtitle; + static const Field _f$subtitle = Field( + 'subtitle', + _$subtitle, + opt: true, + ); + static String? _$badge(FlutterBenchTaskTreeNode v) => v.badge; + static const Field _f$badge = Field( + 'badge', + _$badge, + opt: true, + ); + static String? _$badgeColor(FlutterBenchTaskTreeNode v) => v.badgeColor; + static const Field _f$badgeColor = Field( + 'badgeColor', + _$badgeColor, + key: r'badge_color', + opt: true, + ); + static bool _$isDefaultPage(FlutterBenchTaskTreeNode v) => v.isDefaultPage; + static const Field _f$isDefaultPage = Field( + 'isDefaultPage', + _$isDefaultPage, + key: r'is_default_page', + opt: true, + def: false, + ); + static bool _$startsClosed(FlutterBenchTaskTreeNode v) => v.startsClosed; + static const Field _f$startsClosed = Field( + 'startsClosed', + _$startsClosed, + key: r'starts_closed', + opt: true, + def: true, + ); + static String? _$body(FlutterBenchTaskTreeNode v) => v.body; + static const Field _f$body = Field( + 'body', + _$body, + opt: true, + ); + static FlutterBenchCodeSample? _$code(FlutterBenchTaskTreeNode v) => v.code; + static const Field _f$code = + Field('code', _$code, opt: true); + static List _$children( + FlutterBenchTaskTreeNode v, + ) => v.children; + static const Field> + _f$children = Field('children', _$children, opt: true, def: const []); + + @override + final MappableFields fields = const { + #type: _f$type, + #id: _f$id, + #label: _f$label, + #subtitle: _f$subtitle, + #badge: _f$badge, + #badgeColor: _f$badgeColor, + #isDefaultPage: _f$isDefaultPage, + #startsClosed: _f$startsClosed, + #body: _f$body, + #code: _f$code, + #children: _f$children, + }; + + static FlutterBenchTaskTreeNode _instantiate(DecodingData data) { + return FlutterBenchTaskTreeNode( + type: data.dec(_f$type), + id: data.dec(_f$id), + label: data.dec(_f$label), + subtitle: data.dec(_f$subtitle), + badge: data.dec(_f$badge), + badgeColor: data.dec(_f$badgeColor), + isDefaultPage: data.dec(_f$isDefaultPage), + startsClosed: data.dec(_f$startsClosed), + body: data.dec(_f$body), + code: data.dec(_f$code), + children: data.dec(_f$children), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTaskTreeNode fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTaskTreeNode fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTaskTreeNodeMappable { + String toJson() { + return FlutterBenchTaskTreeNodeMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTaskTreeNode); + } + + Map toMap() { + return FlutterBenchTaskTreeNodeMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTaskTreeNode); + } + + FlutterBenchTaskTreeNodeCopyWith< + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNode + > + get copyWith => + _FlutterBenchTaskTreeNodeCopyWithImpl< + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNode + >(this as FlutterBenchTaskTreeNode, $identity, $identity); + @override + String toString() { + return FlutterBenchTaskTreeNodeMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTaskTreeNode, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTaskTreeNodeMapper.ensureInitialized().equalsValue( + this as FlutterBenchTaskTreeNode, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTaskTreeNodeMapper.ensureInitialized().hashValue( + this as FlutterBenchTaskTreeNode, + ); + } +} + +extension FlutterBenchTaskTreeNodeValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTaskTreeNode, $Out> { + FlutterBenchTaskTreeNodeCopyWith<$R, FlutterBenchTaskTreeNode, $Out> + get $asFlutterBenchTaskTreeNode => $base.as( + (v, t, t2) => _FlutterBenchTaskTreeNodeCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTaskTreeNodeCopyWith< + $R, + $In extends FlutterBenchTaskTreeNode, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + FlutterBenchCodeSampleCopyWith< + $R, + FlutterBenchCodeSample, + FlutterBenchCodeSample + >? + get code; + ListCopyWith< + $R, + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNodeCopyWith< + $R, + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNode + > + > + get children; + $R call({ + String? type, + String? id, + String? label, + String? subtitle, + String? badge, + String? badgeColor, + bool? isDefaultPage, + bool? startsClosed, + String? body, + FlutterBenchCodeSample? code, + List? children, + }); + FlutterBenchTaskTreeNodeCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTaskTreeNodeCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTaskTreeNode, $Out> + implements + FlutterBenchTaskTreeNodeCopyWith<$R, FlutterBenchTaskTreeNode, $Out> { + _FlutterBenchTaskTreeNodeCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTaskTreeNodeMapper.ensureInitialized(); + @override + FlutterBenchCodeSampleCopyWith< + $R, + FlutterBenchCodeSample, + FlutterBenchCodeSample + >? + get code => $value.code?.copyWith.$chain((v) => call(code: v)); + @override + ListCopyWith< + $R, + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNodeCopyWith< + $R, + FlutterBenchTaskTreeNode, + FlutterBenchTaskTreeNode + > + > + get children => ListCopyWith( + $value.children, + (v, t) => v.copyWith.$chain(t), + (v) => call(children: v), + ); + @override + $R call({ + String? type, + String? id, + String? label, + Object? subtitle = $none, + Object? badge = $none, + Object? badgeColor = $none, + bool? isDefaultPage, + bool? startsClosed, + Object? body = $none, + Object? code = $none, + List? children, + }) => $apply( + FieldCopyWithData({ + if (type != null) #type: type, + if (id != null) #id: id, + if (label != null) #label: label, + if (subtitle != $none) #subtitle: subtitle, + if (badge != $none) #badge: badge, + if (badgeColor != $none) #badgeColor: badgeColor, + if (isDefaultPage != null) #isDefaultPage: isDefaultPage, + if (startsClosed != null) #startsClosed: startsClosed, + if (body != $none) #body: body, + if (code != $none) #code: code, + if (children != null) #children: children, + }), + ); + @override + FlutterBenchTaskTreeNode $make(CopyWithData data) => FlutterBenchTaskTreeNode( + type: data.get(#type, or: $value.type), + id: data.get(#id, or: $value.id), + label: data.get(#label, or: $value.label), + subtitle: data.get(#subtitle, or: $value.subtitle), + badge: data.get(#badge, or: $value.badge), + badgeColor: data.get(#badgeColor, or: $value.badgeColor), + isDefaultPage: data.get(#isDefaultPage, or: $value.isDefaultPage), + startsClosed: data.get(#startsClosed, or: $value.startsClosed), + body: data.get(#body, or: $value.body), + code: data.get(#code, or: $value.code), + children: data.get(#children, or: $value.children), + ); + + @override + FlutterBenchTaskTreeNodeCopyWith<$R2, FlutterBenchTaskTreeNode, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTaskTreeNodeCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchCodeSampleMapper + extends ClassMapperBase { + FlutterBenchCodeSampleMapper._(); + + static FlutterBenchCodeSampleMapper? _instance; + static FlutterBenchCodeSampleMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchCodeSampleMapper._()); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchCodeSample'; + + static String _$lang(FlutterBenchCodeSample v) => v.lang; + static const Field _f$lang = Field( + 'lang', + _$lang, + ); + static String _$text(FlutterBenchCodeSample v) => v.text; + static const Field _f$text = Field( + 'text', + _$text, + ); + + @override + final MappableFields fields = const { + #lang: _f$lang, + #text: _f$text, + }; + + static FlutterBenchCodeSample _instantiate(DecodingData data) { + return FlutterBenchCodeSample( + lang: data.dec(_f$lang), + text: data.dec(_f$text), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchCodeSample fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchCodeSample fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchCodeSampleMappable { + String toJson() { + return FlutterBenchCodeSampleMapper.ensureInitialized() + .encodeJson(this as FlutterBenchCodeSample); + } + + Map toMap() { + return FlutterBenchCodeSampleMapper.ensureInitialized() + .encodeMap(this as FlutterBenchCodeSample); + } + + FlutterBenchCodeSampleCopyWith< + FlutterBenchCodeSample, + FlutterBenchCodeSample, + FlutterBenchCodeSample + > + get copyWith => + _FlutterBenchCodeSampleCopyWithImpl< + FlutterBenchCodeSample, + FlutterBenchCodeSample + >(this as FlutterBenchCodeSample, $identity, $identity); + @override + String toString() { + return FlutterBenchCodeSampleMapper.ensureInitialized().stringifyValue( + this as FlutterBenchCodeSample, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchCodeSampleMapper.ensureInitialized().equalsValue( + this as FlutterBenchCodeSample, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchCodeSampleMapper.ensureInitialized().hashValue( + this as FlutterBenchCodeSample, + ); + } +} + +extension FlutterBenchCodeSampleValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchCodeSample, $Out> { + FlutterBenchCodeSampleCopyWith<$R, FlutterBenchCodeSample, $Out> + get $asFlutterBenchCodeSample => $base.as( + (v, t, t2) => _FlutterBenchCodeSampleCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchCodeSampleCopyWith< + $R, + $In extends FlutterBenchCodeSample, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + $R call({String? lang, String? text}); + FlutterBenchCodeSampleCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchCodeSampleCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchCodeSample, $Out> + implements + FlutterBenchCodeSampleCopyWith<$R, FlutterBenchCodeSample, $Out> { + _FlutterBenchCodeSampleCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchCodeSampleMapper.ensureInitialized(); + @override + $R call({String? lang, String? text}) => $apply( + FieldCopyWithData({ + if (lang != null) #lang: lang, + if (text != null) #text: text, + }), + ); + @override + FlutterBenchCodeSample $make(CopyWithData data) => FlutterBenchCodeSample( + lang: data.get(#lang, or: $value.lang), + text: data.get(#text, or: $value.text), + ); + + @override + FlutterBenchCodeSampleCopyWith<$R2, FlutterBenchCodeSample, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchCodeSampleCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchTableSectionMapper + extends ClassMapperBase { + FlutterBenchTableSectionMapper._(); + + static FlutterBenchTableSectionMapper? _instance; + static FlutterBenchTableSectionMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchTableSectionMapper._(), + ); + FlutterBenchTableRowMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTableSection'; + + static List _$rows(FlutterBenchTableSection v) => + v.rows; + static const Field> + _f$rows = Field('rows', _$rows); + + @override + final MappableFields fields = const { + #rows: _f$rows, + }; + + static FlutterBenchTableSection _instantiate(DecodingData data) { + return FlutterBenchTableSection(rows: data.dec(_f$rows)); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTableSection fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTableSection fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTableSectionMappable { + String toJson() { + return FlutterBenchTableSectionMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTableSection); + } + + Map toMap() { + return FlutterBenchTableSectionMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTableSection); + } + + FlutterBenchTableSectionCopyWith< + FlutterBenchTableSection, + FlutterBenchTableSection, + FlutterBenchTableSection + > + get copyWith => + _FlutterBenchTableSectionCopyWithImpl< + FlutterBenchTableSection, + FlutterBenchTableSection + >(this as FlutterBenchTableSection, $identity, $identity); + @override + String toString() { + return FlutterBenchTableSectionMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTableSection, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTableSectionMapper.ensureInitialized().equalsValue( + this as FlutterBenchTableSection, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTableSectionMapper.ensureInitialized().hashValue( + this as FlutterBenchTableSection, + ); + } +} + +extension FlutterBenchTableSectionValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTableSection, $Out> { + FlutterBenchTableSectionCopyWith<$R, FlutterBenchTableSection, $Out> + get $asFlutterBenchTableSection => $base.as( + (v, t, t2) => _FlutterBenchTableSectionCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTableSectionCopyWith< + $R, + $In extends FlutterBenchTableSection, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchTableRow, + FlutterBenchTableRowCopyWith<$R, FlutterBenchTableRow, FlutterBenchTableRow> + > + get rows; + $R call({List? rows}); + FlutterBenchTableSectionCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTableSectionCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTableSection, $Out> + implements + FlutterBenchTableSectionCopyWith<$R, FlutterBenchTableSection, $Out> { + _FlutterBenchTableSectionCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTableSectionMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchTableRow, + FlutterBenchTableRowCopyWith<$R, FlutterBenchTableRow, FlutterBenchTableRow> + > + get rows => ListCopyWith( + $value.rows, + (v, t) => v.copyWith.$chain(t), + (v) => call(rows: v), + ); + @override + $R call({List? rows}) => + $apply(FieldCopyWithData({if (rows != null) #rows: rows})); + @override + FlutterBenchTableSection $make(CopyWithData data) => + FlutterBenchTableSection(rows: data.get(#rows, or: $value.rows)); + + @override + FlutterBenchTableSectionCopyWith<$R2, FlutterBenchTableSection, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTableSectionCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchItemListMapper extends ClassMapperBase { + FlutterBenchItemListMapper._(); + + static FlutterBenchItemListMapper? _instance; + static FlutterBenchItemListMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchItemListMapper._()); + FlutterBenchLabeledDetailMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchItemList'; + + static List _$items(FlutterBenchItemList v) => + v.items; + static const Field> + _f$items = Field('items', _$items); + + @override + final MappableFields fields = const {#items: _f$items}; + + static FlutterBenchItemList _instantiate(DecodingData data) { + return FlutterBenchItemList(items: data.dec(_f$items)); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchItemList fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchItemList fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchItemListMappable { + String toJson() { + return FlutterBenchItemListMapper.ensureInitialized() + .encodeJson(this as FlutterBenchItemList); + } + + Map toMap() { + return FlutterBenchItemListMapper.ensureInitialized() + .encodeMap(this as FlutterBenchItemList); + } + + FlutterBenchItemListCopyWith< + FlutterBenchItemList, + FlutterBenchItemList, + FlutterBenchItemList + > + get copyWith => + _FlutterBenchItemListCopyWithImpl< + FlutterBenchItemList, + FlutterBenchItemList + >(this as FlutterBenchItemList, $identity, $identity); + @override + String toString() { + return FlutterBenchItemListMapper.ensureInitialized().stringifyValue( + this as FlutterBenchItemList, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchItemListMapper.ensureInitialized().equalsValue( + this as FlutterBenchItemList, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchItemListMapper.ensureInitialized().hashValue( + this as FlutterBenchItemList, + ); + } +} + +extension FlutterBenchItemListValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchItemList, $Out> { + FlutterBenchItemListCopyWith<$R, FlutterBenchItemList, $Out> + get $asFlutterBenchItemList => $base.as( + (v, t, t2) => _FlutterBenchItemListCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchItemListCopyWith< + $R, + $In extends FlutterBenchItemList, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchLabeledDetail, + FlutterBenchLabeledDetailCopyWith< + $R, + FlutterBenchLabeledDetail, + FlutterBenchLabeledDetail + > + > + get items; + $R call({List? items}); + FlutterBenchItemListCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchItemListCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchItemList, $Out> + implements FlutterBenchItemListCopyWith<$R, FlutterBenchItemList, $Out> { + _FlutterBenchItemListCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchItemListMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchLabeledDetail, + FlutterBenchLabeledDetailCopyWith< + $R, + FlutterBenchLabeledDetail, + FlutterBenchLabeledDetail + > + > + get items => ListCopyWith( + $value.items, + (v, t) => v.copyWith.$chain(t), + (v) => call(items: v), + ); + @override + $R call({List? items}) => + $apply(FieldCopyWithData({if (items != null) #items: items})); + @override + FlutterBenchItemList $make(CopyWithData data) => + FlutterBenchItemList(items: data.get(#items, or: $value.items)); + + @override + FlutterBenchItemListCopyWith<$R2, FlutterBenchItemList, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchItemListCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchLabeledDetailMapper + extends ClassMapperBase { + FlutterBenchLabeledDetailMapper._(); + + static FlutterBenchLabeledDetailMapper? _instance; + static FlutterBenchLabeledDetailMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchLabeledDetailMapper._(), + ); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchLabeledDetail'; + + static String _$label(FlutterBenchLabeledDetail v) => v.label; + static const Field _f$label = Field( + 'label', + _$label, + ); + static String _$detail(FlutterBenchLabeledDetail v) => v.detail; + static const Field _f$detail = Field( + 'detail', + _$detail, + ); + + @override + final MappableFields fields = const { + #label: _f$label, + #detail: _f$detail, + }; + + static FlutterBenchLabeledDetail _instantiate(DecodingData data) { + return FlutterBenchLabeledDetail( + label: data.dec(_f$label), + detail: data.dec(_f$detail), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchLabeledDetail fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchLabeledDetail fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchLabeledDetailMappable { + String toJson() { + return FlutterBenchLabeledDetailMapper.ensureInitialized() + .encodeJson( + this as FlutterBenchLabeledDetail, + ); + } + + Map toMap() { + return FlutterBenchLabeledDetailMapper.ensureInitialized() + .encodeMap( + this as FlutterBenchLabeledDetail, + ); + } + + FlutterBenchLabeledDetailCopyWith< + FlutterBenchLabeledDetail, + FlutterBenchLabeledDetail, + FlutterBenchLabeledDetail + > + get copyWith => + _FlutterBenchLabeledDetailCopyWithImpl< + FlutterBenchLabeledDetail, + FlutterBenchLabeledDetail + >(this as FlutterBenchLabeledDetail, $identity, $identity); + @override + String toString() { + return FlutterBenchLabeledDetailMapper.ensureInitialized().stringifyValue( + this as FlutterBenchLabeledDetail, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchLabeledDetailMapper.ensureInitialized().equalsValue( + this as FlutterBenchLabeledDetail, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchLabeledDetailMapper.ensureInitialized().hashValue( + this as FlutterBenchLabeledDetail, + ); + } +} + +extension FlutterBenchLabeledDetailValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchLabeledDetail, $Out> { + FlutterBenchLabeledDetailCopyWith<$R, FlutterBenchLabeledDetail, $Out> + get $asFlutterBenchLabeledDetail => $base.as( + (v, t, t2) => _FlutterBenchLabeledDetailCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchLabeledDetailCopyWith< + $R, + $In extends FlutterBenchLabeledDetail, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + $R call({String? label, String? detail}); + FlutterBenchLabeledDetailCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchLabeledDetailCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchLabeledDetail, $Out> + implements + FlutterBenchLabeledDetailCopyWith<$R, FlutterBenchLabeledDetail, $Out> { + _FlutterBenchLabeledDetailCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchLabeledDetailMapper.ensureInitialized(); + @override + $R call({String? label, String? detail}) => $apply( + FieldCopyWithData({ + if (label != null) #label: label, + if (detail != null) #detail: detail, + }), + ); + @override + FlutterBenchLabeledDetail $make(CopyWithData data) => + FlutterBenchLabeledDetail( + label: data.get(#label, or: $value.label), + detail: data.get(#detail, or: $value.detail), + ); + + @override + FlutterBenchLabeledDetailCopyWith<$R2, FlutterBenchLabeledDetail, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchLabeledDetailCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchTransparencyMapper + extends ClassMapperBase { + FlutterBenchTransparencyMapper._(); + + static FlutterBenchTransparencyMapper? _instance; + static FlutterBenchTransparencyMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchTransparencyMapper._(), + ); + FlutterBenchHarborExampleMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchTransparency'; + + static FlutterBenchHarborExample _$harborExample( + FlutterBenchTransparency v, + ) => v.harborExample; + static const Field + _f$harborExample = Field( + 'harborExample', + _$harborExample, + key: r'harbor_example', + ); + + @override + final MappableFields fields = const { + #harborExample: _f$harborExample, + }; + + static FlutterBenchTransparency _instantiate(DecodingData data) { + return FlutterBenchTransparency(harborExample: data.dec(_f$harborExample)); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchTransparency fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchTransparency fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchTransparencyMappable { + String toJson() { + return FlutterBenchTransparencyMapper.ensureInitialized() + .encodeJson(this as FlutterBenchTransparency); + } + + Map toMap() { + return FlutterBenchTransparencyMapper.ensureInitialized() + .encodeMap(this as FlutterBenchTransparency); + } + + FlutterBenchTransparencyCopyWith< + FlutterBenchTransparency, + FlutterBenchTransparency, + FlutterBenchTransparency + > + get copyWith => + _FlutterBenchTransparencyCopyWithImpl< + FlutterBenchTransparency, + FlutterBenchTransparency + >(this as FlutterBenchTransparency, $identity, $identity); + @override + String toString() { + return FlutterBenchTransparencyMapper.ensureInitialized().stringifyValue( + this as FlutterBenchTransparency, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchTransparencyMapper.ensureInitialized().equalsValue( + this as FlutterBenchTransparency, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchTransparencyMapper.ensureInitialized().hashValue( + this as FlutterBenchTransparency, + ); + } +} + +extension FlutterBenchTransparencyValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchTransparency, $Out> { + FlutterBenchTransparencyCopyWith<$R, FlutterBenchTransparency, $Out> + get $asFlutterBenchTransparency => $base.as( + (v, t, t2) => _FlutterBenchTransparencyCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchTransparencyCopyWith< + $R, + $In extends FlutterBenchTransparency, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + FlutterBenchHarborExampleCopyWith< + $R, + FlutterBenchHarborExample, + FlutterBenchHarborExample + > + get harborExample; + $R call({FlutterBenchHarborExample? harborExample}); + FlutterBenchTransparencyCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchTransparencyCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchTransparency, $Out> + implements + FlutterBenchTransparencyCopyWith<$R, FlutterBenchTransparency, $Out> { + _FlutterBenchTransparencyCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchTransparencyMapper.ensureInitialized(); + @override + FlutterBenchHarborExampleCopyWith< + $R, + FlutterBenchHarborExample, + FlutterBenchHarborExample + > + get harborExample => + $value.harborExample.copyWith.$chain((v) => call(harborExample: v)); + @override + $R call({FlutterBenchHarborExample? harborExample}) => $apply( + FieldCopyWithData({ + if (harborExample != null) #harborExample: harborExample, + }), + ); + @override + FlutterBenchTransparency $make(CopyWithData data) => FlutterBenchTransparency( + harborExample: data.get(#harborExample, or: $value.harborExample), + ); + + @override + FlutterBenchTransparencyCopyWith<$R2, FlutterBenchTransparency, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchTransparencyCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchHarborExampleMapper + extends ClassMapperBase { + FlutterBenchHarborExampleMapper._(); + + static FlutterBenchHarborExampleMapper? _instance; + static FlutterBenchHarborExampleMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchHarborExampleMapper._(), + ); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchHarborExample'; + + static String _$task(FlutterBenchHarborExample v) => v.task; + static const Field _f$task = Field( + 'task', + _$task, + ); + static String _$agent(FlutterBenchHarborExample v) => v.agent; + static const Field _f$agent = Field( + 'agent', + _$agent, + ); + static String _$model(FlutterBenchHarborExample v) => v.model; + static const Field _f$model = Field( + 'model', + _$model, + ); + static String _$mcp(FlutterBenchHarborExample v) => v.mcp; + static const Field _f$mcp = Field( + 'mcp', + _$mcp, + ); + + @override + final MappableFields fields = const { + #task: _f$task, + #agent: _f$agent, + #model: _f$model, + #mcp: _f$mcp, + }; + + static FlutterBenchHarborExample _instantiate(DecodingData data) { + return FlutterBenchHarborExample( + task: data.dec(_f$task), + agent: data.dec(_f$agent), + model: data.dec(_f$model), + mcp: data.dec(_f$mcp), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchHarborExample fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchHarborExample fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchHarborExampleMappable { + String toJson() { + return FlutterBenchHarborExampleMapper.ensureInitialized() + .encodeJson( + this as FlutterBenchHarborExample, + ); + } + + Map toMap() { + return FlutterBenchHarborExampleMapper.ensureInitialized() + .encodeMap( + this as FlutterBenchHarborExample, + ); + } + + FlutterBenchHarborExampleCopyWith< + FlutterBenchHarborExample, + FlutterBenchHarborExample, + FlutterBenchHarborExample + > + get copyWith => + _FlutterBenchHarborExampleCopyWithImpl< + FlutterBenchHarborExample, + FlutterBenchHarborExample + >(this as FlutterBenchHarborExample, $identity, $identity); + @override + String toString() { + return FlutterBenchHarborExampleMapper.ensureInitialized().stringifyValue( + this as FlutterBenchHarborExample, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchHarborExampleMapper.ensureInitialized().equalsValue( + this as FlutterBenchHarborExample, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchHarborExampleMapper.ensureInitialized().hashValue( + this as FlutterBenchHarborExample, + ); + } +} + +extension FlutterBenchHarborExampleValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchHarborExample, $Out> { + FlutterBenchHarborExampleCopyWith<$R, FlutterBenchHarborExample, $Out> + get $asFlutterBenchHarborExample => $base.as( + (v, t, t2) => _FlutterBenchHarborExampleCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchHarborExampleCopyWith< + $R, + $In extends FlutterBenchHarborExample, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + $R call({String? task, String? agent, String? model, String? mcp}); + FlutterBenchHarborExampleCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchHarborExampleCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchHarborExample, $Out> + implements + FlutterBenchHarborExampleCopyWith<$R, FlutterBenchHarborExample, $Out> { + _FlutterBenchHarborExampleCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchHarborExampleMapper.ensureInitialized(); + @override + $R call({String? task, String? agent, String? model, String? mcp}) => $apply( + FieldCopyWithData({ + if (task != null) #task: task, + if (agent != null) #agent: agent, + if (model != null) #model: model, + if (mcp != null) #mcp: mcp, + }), + ); + @override + FlutterBenchHarborExample $make(CopyWithData data) => + FlutterBenchHarborExample( + task: data.get(#task, or: $value.task), + agent: data.get(#agent, or: $value.agent), + model: data.get(#model, or: $value.model), + mcp: data.get(#mcp, or: $value.mcp), + ); + + @override + FlutterBenchHarborExampleCopyWith<$R2, FlutterBenchHarborExample, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchHarborExampleCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchCujsDataMapper extends ClassMapperBase { + FlutterBenchCujsDataMapper._(); + + static FlutterBenchCujsDataMapper? _instance; + static FlutterBenchCujsDataMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchCujsDataMapper._()); + FlutterBenchCujItemMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchCujsData'; + + static List _$cujs(FlutterBenchCujsData v) => v.cujs; + static const Field> _f$cujs = + Field('cujs', _$cujs); + + @override + final MappableFields fields = const {#cujs: _f$cujs}; + + static FlutterBenchCujsData _instantiate(DecodingData data) { + return FlutterBenchCujsData(cujs: data.dec(_f$cujs)); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchCujsData fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchCujsData fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchCujsDataMappable { + String toJson() { + return FlutterBenchCujsDataMapper.ensureInitialized() + .encodeJson(this as FlutterBenchCujsData); + } + + Map toMap() { + return FlutterBenchCujsDataMapper.ensureInitialized() + .encodeMap(this as FlutterBenchCujsData); + } + + FlutterBenchCujsDataCopyWith< + FlutterBenchCujsData, + FlutterBenchCujsData, + FlutterBenchCujsData + > + get copyWith => + _FlutterBenchCujsDataCopyWithImpl< + FlutterBenchCujsData, + FlutterBenchCujsData + >(this as FlutterBenchCujsData, $identity, $identity); + @override + String toString() { + return FlutterBenchCujsDataMapper.ensureInitialized().stringifyValue( + this as FlutterBenchCujsData, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchCujsDataMapper.ensureInitialized().equalsValue( + this as FlutterBenchCujsData, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchCujsDataMapper.ensureInitialized().hashValue( + this as FlutterBenchCujsData, + ); + } +} + +extension FlutterBenchCujsDataValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchCujsData, $Out> { + FlutterBenchCujsDataCopyWith<$R, FlutterBenchCujsData, $Out> + get $asFlutterBenchCujsData => $base.as( + (v, t, t2) => _FlutterBenchCujsDataCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchCujsDataCopyWith< + $R, + $In extends FlutterBenchCujsData, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchCujItem, + FlutterBenchCujItemCopyWith<$R, FlutterBenchCujItem, FlutterBenchCujItem> + > + get cujs; + $R call({List? cujs}); + FlutterBenchCujsDataCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchCujsDataCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchCujsData, $Out> + implements FlutterBenchCujsDataCopyWith<$R, FlutterBenchCujsData, $Out> { + _FlutterBenchCujsDataCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchCujsDataMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchCujItem, + FlutterBenchCujItemCopyWith<$R, FlutterBenchCujItem, FlutterBenchCujItem> + > + get cujs => ListCopyWith( + $value.cujs, + (v, t) => v.copyWith.$chain(t), + (v) => call(cujs: v), + ); + @override + $R call({List? cujs}) => + $apply(FieldCopyWithData({if (cujs != null) #cujs: cujs})); + @override + FlutterBenchCujsData $make(CopyWithData data) => + FlutterBenchCujsData(cujs: data.get(#cujs, or: $value.cujs)); + + @override + FlutterBenchCujsDataCopyWith<$R2, FlutterBenchCujsData, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchCujsDataCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchCujItemMapper extends ClassMapperBase { + FlutterBenchCujItemMapper._(); + + static FlutterBenchCujItemMapper? _instance; + static FlutterBenchCujItemMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use(_instance = FlutterBenchCujItemMapper._()); + FlutterBenchCujTaskItemMapper.ensureInitialized(); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchCujItem'; + + static int _$id(FlutterBenchCujItem v) => v.id; + static const Field _f$id = Field('id', _$id); + static String _$goal(FlutterBenchCujItem v) => v.goal; + static const Field _f$goal = Field( + 'goal', + _$goal, + ); + static String _$persona(FlutterBenchCujItem v) => v.persona; + static const Field _f$persona = Field( + 'persona', + _$persona, + ); + static List _$tasks(FlutterBenchCujItem v) => + v.tasks; + static const Field> + _f$tasks = Field('tasks', _$tasks, opt: true, def: const []); + + @override + final MappableFields fields = const { + #id: _f$id, + #goal: _f$goal, + #persona: _f$persona, + #tasks: _f$tasks, + }; + + static FlutterBenchCujItem _instantiate(DecodingData data) { + return FlutterBenchCujItem( + id: data.dec(_f$id), + goal: data.dec(_f$goal), + persona: data.dec(_f$persona), + tasks: data.dec(_f$tasks), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchCujItem fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchCujItem fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchCujItemMappable { + String toJson() { + return FlutterBenchCujItemMapper.ensureInitialized() + .encodeJson(this as FlutterBenchCujItem); + } + + Map toMap() { + return FlutterBenchCujItemMapper.ensureInitialized() + .encodeMap(this as FlutterBenchCujItem); + } + + FlutterBenchCujItemCopyWith< + FlutterBenchCujItem, + FlutterBenchCujItem, + FlutterBenchCujItem + > + get copyWith => + _FlutterBenchCujItemCopyWithImpl< + FlutterBenchCujItem, + FlutterBenchCujItem + >(this as FlutterBenchCujItem, $identity, $identity); + @override + String toString() { + return FlutterBenchCujItemMapper.ensureInitialized().stringifyValue( + this as FlutterBenchCujItem, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchCujItemMapper.ensureInitialized().equalsValue( + this as FlutterBenchCujItem, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchCujItemMapper.ensureInitialized().hashValue( + this as FlutterBenchCujItem, + ); + } +} + +extension FlutterBenchCujItemValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchCujItem, $Out> { + FlutterBenchCujItemCopyWith<$R, FlutterBenchCujItem, $Out> + get $asFlutterBenchCujItem => $base.as( + (v, t, t2) => _FlutterBenchCujItemCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchCujItemCopyWith< + $R, + $In extends FlutterBenchCujItem, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + ListCopyWith< + $R, + FlutterBenchCujTaskItem, + FlutterBenchCujTaskItemCopyWith< + $R, + FlutterBenchCujTaskItem, + FlutterBenchCujTaskItem + > + > + get tasks; + $R call({ + int? id, + String? goal, + String? persona, + List? tasks, + }); + FlutterBenchCujItemCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchCujItemCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchCujItem, $Out> + implements FlutterBenchCujItemCopyWith<$R, FlutterBenchCujItem, $Out> { + _FlutterBenchCujItemCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchCujItemMapper.ensureInitialized(); + @override + ListCopyWith< + $R, + FlutterBenchCujTaskItem, + FlutterBenchCujTaskItemCopyWith< + $R, + FlutterBenchCujTaskItem, + FlutterBenchCujTaskItem + > + > + get tasks => ListCopyWith( + $value.tasks, + (v, t) => v.copyWith.$chain(t), + (v) => call(tasks: v), + ); + @override + $R call({ + int? id, + String? goal, + String? persona, + List? tasks, + }) => $apply( + FieldCopyWithData({ + if (id != null) #id: id, + if (goal != null) #goal: goal, + if (persona != null) #persona: persona, + if (tasks != null) #tasks: tasks, + }), + ); + @override + FlutterBenchCujItem $make(CopyWithData data) => FlutterBenchCujItem( + id: data.get(#id, or: $value.id), + goal: data.get(#goal, or: $value.goal), + persona: data.get(#persona, or: $value.persona), + tasks: data.get(#tasks, or: $value.tasks), + ); + + @override + FlutterBenchCujItemCopyWith<$R2, FlutterBenchCujItem, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchCujItemCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + +class FlutterBenchCujTaskItemMapper + extends ClassMapperBase { + FlutterBenchCujTaskItemMapper._(); + + static FlutterBenchCujTaskItemMapper? _instance; + static FlutterBenchCujTaskItemMapper ensureInitialized() { + if (_instance == null) { + MapperContainer.globals.use( + _instance = FlutterBenchCujTaskItemMapper._(), + ); + } + return _instance!; + } + + @override + final String id = 'FlutterBenchCujTaskItem'; + + static int _$id(FlutterBenchCujTaskItem v) => v.id; + static const Field _f$id = Field('id', _$id); + static String _$name(FlutterBenchCujTaskItem v) => v.name; + static const Field _f$name = Field( + 'name', + _$name, + ); + static String _$task(FlutterBenchCujTaskItem v) => v.task; + static const Field _f$task = Field( + 'task', + _$task, + ); + + @override + final MappableFields fields = const { + #id: _f$id, + #name: _f$name, + #task: _f$task, + }; + + static FlutterBenchCujTaskItem _instantiate(DecodingData data) { + return FlutterBenchCujTaskItem( + id: data.dec(_f$id), + name: data.dec(_f$name), + task: data.dec(_f$task), + ); + } + + @override + final Function instantiate = _instantiate; + + static FlutterBenchCujTaskItem fromMap(Map map) { + return ensureInitialized().decodeMap(map); + } + + static FlutterBenchCujTaskItem fromJson(String json) { + return ensureInitialized().decodeJson(json); + } +} + +mixin FlutterBenchCujTaskItemMappable { + String toJson() { + return FlutterBenchCujTaskItemMapper.ensureInitialized() + .encodeJson(this as FlutterBenchCujTaskItem); + } + + Map toMap() { + return FlutterBenchCujTaskItemMapper.ensureInitialized() + .encodeMap(this as FlutterBenchCujTaskItem); + } + + FlutterBenchCujTaskItemCopyWith< + FlutterBenchCujTaskItem, + FlutterBenchCujTaskItem, + FlutterBenchCujTaskItem + > + get copyWith => + _FlutterBenchCujTaskItemCopyWithImpl< + FlutterBenchCujTaskItem, + FlutterBenchCujTaskItem + >(this as FlutterBenchCujTaskItem, $identity, $identity); + @override + String toString() { + return FlutterBenchCujTaskItemMapper.ensureInitialized().stringifyValue( + this as FlutterBenchCujTaskItem, + ); + } + + @override + bool operator ==(Object other) { + return FlutterBenchCujTaskItemMapper.ensureInitialized().equalsValue( + this as FlutterBenchCujTaskItem, + other, + ); + } + + @override + int get hashCode { + return FlutterBenchCujTaskItemMapper.ensureInitialized().hashValue( + this as FlutterBenchCujTaskItem, + ); + } +} + +extension FlutterBenchCujTaskItemValueCopy<$R, $Out> + on ObjectCopyWith<$R, FlutterBenchCujTaskItem, $Out> { + FlutterBenchCujTaskItemCopyWith<$R, FlutterBenchCujTaskItem, $Out> + get $asFlutterBenchCujTaskItem => $base.as( + (v, t, t2) => _FlutterBenchCujTaskItemCopyWithImpl<$R, $Out>(v, t, t2), + ); +} + +abstract class FlutterBenchCujTaskItemCopyWith< + $R, + $In extends FlutterBenchCujTaskItem, + $Out +> + implements ClassCopyWith<$R, $In, $Out> { + $R call({int? id, String? name, String? task}); + FlutterBenchCujTaskItemCopyWith<$R2, $In, $Out2> $chain<$R2, $Out2>( + Then<$Out2, $R2> t, + ); +} + +class _FlutterBenchCujTaskItemCopyWithImpl<$R, $Out> + extends ClassCopyWithBase<$R, FlutterBenchCujTaskItem, $Out> + implements + FlutterBenchCujTaskItemCopyWith<$R, FlutterBenchCujTaskItem, $Out> { + _FlutterBenchCujTaskItemCopyWithImpl(super.value, super.then, super.then2); + + @override + late final ClassMapperBase $mapper = + FlutterBenchCujTaskItemMapper.ensureInitialized(); + @override + $R call({int? id, String? name, String? task}) => $apply( + FieldCopyWithData({ + if (id != null) #id: id, + if (name != null) #name: name, + if (task != null) #task: task, + }), + ); + @override + FlutterBenchCujTaskItem $make(CopyWithData data) => FlutterBenchCujTaskItem( + id: data.get(#id, or: $value.id), + name: data.get(#name, or: $value.name), + task: data.get(#task, or: $value.task), + ); + + @override + FlutterBenchCujTaskItemCopyWith<$R2, FlutterBenchCujTaskItem, $Out2> + $chain<$R2, $Out2>(Then<$Out2, $R2> t) => + _FlutterBenchCujTaskItemCopyWithImpl<$R2, $Out2>($value, $cast, t); +} + diff --git a/sites/www/lib/src/pages/flutter_bench/flutterbench_cujs_page.dart b/sites/www/lib/src/pages/flutter_bench/flutterbench_cujs_page.dart new file mode 100644 index 00000000000..61d697046f5 --- /dev/null +++ b/sites/www/lib/src/pages/flutter_bench/flutterbench_cujs_page.dart @@ -0,0 +1,67 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../../components/flutterbench/cuj_catalog.dart'; +import '../../models/content/flutterbench_content.dart'; +import '../../utils/data_utils.dart'; +import 'flutterbench_nav.dart'; + +/// FlutterBench critical user journey (CUJ) catalog page. +/// +/// Mounted by `/ai/flutterbench/cujs/index.md`. +class FlutterBenchCujsPage extends StatelessComponent { + const FlutterBenchCujsPage({super.key}); + + @override + Component build(BuildContext context) { + final cujsData = context.decodeJsonObject( + 'data.flutterbench.cujs', + FlutterBenchCujsData.fromJson, + ); + + final cujMaps = cujsData.cujs.map((c) => c.toMap()).toList(); + + return main_(classes: 'bench-page cujs-page', [ + // Hero Header with Navigation Tabs + section(classes: 'bench-hero-header', [ + div(classes: 'bench-container', [ + div(classes: 'hero-badge-row', [ + const span(classes: 'hero-category-tag', [.text('CUJ CATALOG')]), + span(classes: 'job-id-tag', [ + .text('${cujsData.cujs.length} Journeys'), + ]), + ]), + const h1(classes: 'bench-hero-title', [ + .text('Flutter Critical User Journeys'), + ]), + const p(classes: 'bench-hero-subtitle', [ + .text( + 'Browse the catalog of canonical Flutter and Dart critical ' + 'user journeys that the FlutterBench evaluations test.', + ), + ]), + + const FlutterBenchNav(current: FlutterBenchNavItem.cujs), + ]), + ]), + + div(classes: 'bench-container content-area cujs-content', [ + const p(classes: 'methodology-lead', [ + .text( + 'A critical user journey (CUJ) is a goal that a developer sets ' + 'out to accomplish, such as "make an application accessible to ' + 'all users" or "diagnose and resolve layout overflow errors". ' + 'Each CUJ is broken down into the concrete tasks required to ' + 'complete it. The Flutter team uses CUJs to derive evaluation ' + 'tasks and prompts for FlutterBench.', + ), + ]), + CujCatalog(cujs: cujMaps), + ]), + ]); + } +} diff --git a/sites/www/lib/src/pages/flutter_bench/flutterbench_leaderboard_page.dart b/sites/www/lib/src/pages/flutter_bench/flutterbench_leaderboard_page.dart new file mode 100644 index 00000000000..674544e3e7b --- /dev/null +++ b/sites/www/lib/src/pages/flutter_bench/flutterbench_leaderboard_page.dart @@ -0,0 +1,107 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../../components/flutterbench/benchmark_scores.dart'; +import '../../components/flutterbench/leaderboard_table.dart'; +import '../../models/content/flutterbench_content.dart'; +import '../../utils/data_utils.dart'; +import 'flutterbench_nav.dart'; + +/// FlutterBench Overview & Leaderboard page component. +/// +/// Mounted by `/ai/flutterbench/index.md`. +class FlutterBenchLeaderboardPage extends StatelessComponent { + const FlutterBenchLeaderboardPage({super.key}); + + @override + Component build(BuildContext context) { + final job = context.decodeJsonObject( + 'data.flutterbench.job', + FlutterBenchJobData.fromJson, + ); + final tasksData = context.decodeJsonObject( + 'data.flutterbench.tasks', + FlutterBenchTasksData.fromJson, + ); + final trialsData = context.decodeJsonObject( + 'data.flutterbench.trials', + FlutterBenchTrialsData.fromJson, + ); + + final benchmarks = buildBenchmarkRows( + tasks: tasksData, + trials: trialsData, + ); + + // Convert evals to serializable maps for the client-hydrated LeaderboardTable + final evalsMaps = job.evals.map((e) => e.toMap()).toList(); + + return main_(classes: 'bench-page leaderboard-page', [ + // Hero Header + section(classes: 'bench-hero-header', [ + div(classes: 'bench-container', [ + div(classes: 'hero-badge-row', [ + const span(classes: 'hero-category-tag', [.text('AI BENCHMARK')]), + span(classes: 'job-id-tag', [ + .text('Job: ${job.id.substring(0, 8)}'), + ]), + ]), + const h1(classes: 'bench-hero-title', [ + .text('FlutterBench Leaderboard'), + ]), + const p(classes: 'bench-hero-subtitle', [ + .text( + 'Evaluating how autonomous AI coding agents perform on real-world Dart and Flutter tasks. ' + 'Scores reflect composite functional correctness, code quality, and developer experience.', + ), + ]), + + const FlutterBenchNav(current: FlutterBenchNavItem.leaderboard), + ]), + ]), + + div(classes: 'bench-container content-area', [ + // Leaderboard Table with interactive filter bar + section(classes: 'bench-section', [ + const div(classes: 'section-title-row', [ + h2(classes: 'section-h2', [.text('Model Rankings')]), + span(classes: 'section-note', [ + .text( + 'Sorted by mean reward descending. Click any column header to reorder.', + ), + ]), + ]), + LeaderboardTable( + evals: evalsMaps, + benchmarks: benchmarkRowsToMaps(benchmarks), + ), + ]), + + // Methodology highlight card + const section(classes: 'bench-section methodology-callout', [ + div(classes: 'callout-card', [ + div(classes: 'callout-text', [ + h3([.text('How are agents evaluated?')]), + p([ + .text( + 'Every FlutterBench trial runs in an isolated Docker container testing real Flutter features. ' + 'Scoring measures 60% Outcome (passing tests & builds), 30% Quality (idiomatic patterns & analyzer diagnostics), ' + 'and 10% Developer Experience (tool accuracy & minimal friction). Diagnostic metrics like token efficiency are captured separately.', + ), + ]), + ]), + div(classes: 'callout-action', [ + a(href: '/ai/flutterbench/methodology', classes: 'btn', [ + .text('Read Methodology →'), + ]), + ]), + ]), + ]), + ]), + ]); + } +} diff --git a/sites/www/lib/src/pages/flutter_bench/flutterbench_methodology_page.dart b/sites/www/lib/src/pages/flutter_bench/flutterbench_methodology_page.dart new file mode 100644 index 00000000000..7361a8ee60b --- /dev/null +++ b/sites/www/lib/src/pages/flutter_bench/flutterbench_methodology_page.dart @@ -0,0 +1,542 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; +import 'package:site_shared/components/common/ide_explorer/ide_explorer.dart'; +import 'package:site_shared/components/common/ide_explorer/models.dart'; + +import '../../components/flutterbench/methodology_components.dart'; +import '../../components/flutterbench/story_chapter.dart'; +import '../../components/flutterbench/task_anatomy.dart'; +import '../../models/content/flutterbench_content.dart'; +import '../../utils/data_utils.dart'; +import 'flutterbench_nav.dart'; + +/// Full FlutterBench Methodology page mounted at `/ai/flutterbench/methodology`. +class FlutterBenchMethodologyPage extends StatelessComponent { + const FlutterBenchMethodologyPage({super.key}); + + @override + Component build(BuildContext context) { + final data = context.decodeJsonObject( + 'data.flutterbench.methodology', + FlutterBenchMethodologyData.fromJson, + ); + + final graderMatrixMap = data.graderMatrix; + final graderMatrixTitle = + graderMatrixMap['title'] as String? ?? 'Grader Matrix'; + final graderMatrixDesc = graderMatrixMap['description'] as String? ?? ''; + final graderFilters = + (graderMatrixMap['filters'] as List? ?? const []) + .whereType>() + .toList(); + final graders = (graderMatrixMap['graders'] as List? ?? const []) + .whereType>() + .toList(); + + final reliabilityMap = data.reliability; + final reliabilityCards = + (reliabilityMap['cards'] as List? ?? const []) + .whereType>() + .toList(); + + final scoreTriageMap = data.scoreTriage; + final scoreTriageTitle = + scoreTriageMap['title'] as String? ?? 'Score Triage & Action Matrix'; + final scoreTriageDesc = scoreTriageMap['description'] as String? ?? ''; + final scoreTriageTiers = + (scoreTriageMap['tiers'] as List? ?? const []) + .whereType>() + .map( + (tier) => { + 'id': tier['id'], + 'primary_label': tier['score'], + 'secondary_label': tier['name'], + 'heading': tier['heading'], + 'badge': tier['badge'], + 'variant': tier['id'], + 'overview': tier['criteria'], + 'items_label': tier['actions_label'], + 'items': tier['actions'], + }, + ) + .toList(); + + final evalMatrixMap = data.evaluationMatrix; + final evalMatrixTitle = + evalMatrixMap['title'] as String? ?? '4-Axis Evaluation Matrix'; + final evalMatrixDesc = evalMatrixMap['description'] as String? ?? ''; + final evalMatrixAxes = (evalMatrixMap['axes'] as List? ?? const []) + .whereType>() + .map( + (axis) => { + 'id': axis['id'], + 'primary_label': axis['tab_label'], + 'secondary_label': axis['tab_sublabel'], + 'heading': axis['heading'], + 'badge': axis['badge'], + 'variant': axis['variant'] ?? 'blue', + 'overview': axis['overview'], + 'items_label': axis['items_label'], + 'items': axis['items'], + 'footer_text': axis['footer_text'], + }, + ) + .toList(); + + final harbor = data.transparency.harborExample; + + return main_(classes: 'bench-page methodology-page', [ + // Hero Header with Navigation Tabs + const section(classes: 'bench-hero-header', [ + div(classes: 'bench-container', [ + div(classes: 'hero-badge-row', [ + span(classes: 'hero-category-tag', [.text('AI BENCHMARK')]), + span(classes: 'job-id-tag', [.text('METHODOLOGY & EVALUATION')]), + ]), + h1(classes: 'bench-hero-title', [ + .text('FlutterBench Methodology'), + ]), + p(classes: 'bench-hero-subtitle', [ + .text( + 'Learn about Dart and Flutter\'s evaluation frameworks for measuring AI tooling reliability.', + ), + ]), + + FlutterBenchNav(current: FlutterBenchNavItem.methodology), + ]), + ]), + + div(classes: 'bench-container content-area methodology-content', [ + const div(classes: 'story-reading-progress', [ + div(classes: 'story-reading-progress-bar', []), + ]), + div(classes: 'story-canvas', [ + _buildOverviewChapter(data), + _buildDatasetTasksChapter(data), + StoryChapter( + number: '03', + title: 'Evaluation test matrix', + anchorId: 'evaluation-test-matrix', + children: [ + if (evalMatrixAxes.isNotEmpty) + InteractiveDetailCard( + title: evalMatrixTitle, + description: evalMatrixDesc, + classes: 'interactive-detail-card evaluation-matrix-card', + tabs: evalMatrixAxes, + ), + ], + ), + _buildScoringArchitectureChapter( + data, + graderMatrixTitle: graderMatrixTitle, + graderMatrixDesc: graderMatrixDesc, + graderFilters: graderFilters, + graders: graders, + ), + const StoryChapter( + number: '05', + title: 'Evaluation harness', + anchorId: 'evaluation-harness', + ), + _buildReliabilityTriageChapter( + data, + reliabilityCards: reliabilityCards, + scoreTriageTitle: scoreTriageTitle, + scoreTriageDesc: scoreTriageDesc, + scoreTriageTiers: scoreTriageTiers, + ), + _buildTransparencyChapter(harbor), + ]), + ]), + ]); + } + + Component _buildOverviewChapter(FlutterBenchMethodologyData data) { + return StoryChapter( + number: '01', + title: 'Overview', + anchorId: 'overview', + children: [ + p(classes: 'methodology-lead', [.text(data.overview.leadText)]), + div(classes: 'table-wrapper', [ + table(classes: 'bench-table methodology-table', [ + const thead([ + tr([ + th([.text('Component')]), + th([.text('Description')]), + ]), + ]), + tbody([ + for (final row in data.overview.rows) + tr([ + td([ + a(href: '#${row.anchor}', [ + strong([.text(row.label)]), + ]), + ]), + td([.text(row.description)]), + ]), + ]), + ]), + ]), + ], + ); + } + + Component _buildDatasetTasksChapter(FlutterBenchMethodologyData data) { + return StoryChapter( + number: '02', + title: 'Dataset & tasks', + anchorId: 'dataset-tasks', + children: [ + storyH3('Task derivation'), + const p([ + .text('Evaluation tasks derive directly from Flutter\'s canonical '), + a(href: '/ai/flutterbench/tasks', [ + .text('Critical User Journeys (CUJs)'), + ]), + .text( + ', which are the core workflows that developers perform ' + 'regularly. This approach ensures evaluations reflect ' + 'real-world developer needs rather than synthetic puzzles.', + ), + ]), + const p([.text('Each CUJ represents a combination of:')]), + const ul([ + li([ + .text( + 'A Flutter developer persona (e.g., app developer, plugin developer, full-stack developer)', + ), + ]), + li([.text('A high-level goal')]), + li([ + .text( + 'The specific sequential steps required to achieve that goal', + ), + ]), + ]), + + if (data.cujExample.isNotEmpty) CujDiagram(sections: data.cujExample), + + const div(classes: 'methodology-note-box', [ + p([ + strong([.text('Note: ')]), + .text( + 'We can\'t open-source the full evaluation tasks without ' + 'contaminating the benchmark dataset, but we publish our ' + 'canonical list of CUJs. With this list, along with the ' + 'example task below, you can understand our evaluation ' + 'philosophy for FlutterBench.', + ), + ]), + ]), + + const p([ + .text('These CUJs are converted into '), + a( + href: 'https://harborframework.com/', + target: Target.blank, + [.text('Harbor')], + ), + .text( + ' tasks. Harbor is the framework used to run containerized ' + 'evaluation tasks.', + ), + ]), + const p([ + .text( + 'CUJs and Harbor tasks don\'t map cleanly one-to-one. Instead, ' + 'the CUJ list serves as a guide to verify that core developer ' + 'workflows are evaluated. In some cases, several CUJs combine ' + 'into a single task, and vice-versa. Our most ambitious ' + 'evaluations combine multiple Harbor tasks, and thus cover ' + 'many CUJs.', + ), + ]), + + storyH3('Task specifications'), + if (data.taskSpecifications.isNotEmpty) + TaskSpecifications(specs: data.taskSpecifications), + + storyH3('Interactive task anatomy'), + p([.text(data.taskAnatomy.introText)]), + if (data.taskAnatomy.tree.isNotEmpty) + IdeExplorer( + roots: [ + IdeExplorerProjectRoot( + id: data.taskAnatomy.rootId, + label: data.taskAnatomy.rootLabel, + children: buildIdeTreeNodes(data.taskAnatomy.tree), + ), + ], + customContents: buildIdeCustomContents(data.taskAnatomy.tree), + ), + ], + ); + } + + Component _buildScoringArchitectureChapter( + FlutterBenchMethodologyData data, { + required String graderMatrixTitle, + required String graderMatrixDesc, + required List> graderFilters, + required List> graders, + }) { + return StoryChapter( + number: '04', + title: 'Scoring architecture', + anchorId: 'scoring-architecture', + children: [ + storyH3('Scoring philosophy'), + const p([ + .text( + 'When evaluating AI coding agents, execution friction—such as tool failures, endless retries, and ' + 'hallucinations—is often attributed entirely to model capability. However, AI coding systems follow a core equation:', + ), + ]), + const div(classes: 'scoring-formula-card', [ + div(classes: 'formula-math', [ + .text('Agent = Model + Harness'), + ]), + div(classes: 'formula-explainer', [ + .text( + 'While the Dart and Flutter teams do not train the underlying LLMs, we build and maintain the ' + 'Dart and Flutter AI Harness (skills, MCP tools, compiler diagnostics, and sandboxes). ' + 'Therefore, Developer Experience (DX) is directly within our engineering control and belongs in ' + 'our primary benchmark score alongside functional outcomes and code quality.', + ), + ]), + ]), + const ul([ + li([ + strong([.text('Primary focus: ')]), + .text('The quality and correctness of the final code artifact.'), + ]), + li([ + strong([.text('First-class signal: ')]), + .text( + 'Developer experience friction, tool accuracy, and recovery efficiency.', + ), + ]), + ]), + + storyH3('Three core evaluation dimensions'), + const p([ + .text( + 'Each evaluation produces three independent dimensions that compute the composite Result Score in Harbor\'s reward:', + ), + ]), + const div(classes: 'composite-reward-bar', [ + div(classes: 'reward-weight outcome-weight', [ + span(classes: 'weight-val', [.text('60%')]), + span(classes: 'weight-name', [.text('Outcome')]), + ]), + div(classes: 'reward-weight quality-weight', [ + span(classes: 'weight-val', [.text('30%')]), + span(classes: 'weight-name', [.text('Code Quality')]), + ]), + div(classes: 'reward-weight dx-weight', [ + span(classes: 'weight-val', [.text('10%')]), + span(classes: 'weight-name', [.text('Developer Experience')]), + ]), + ]), + + if (data.dimensions.isNotEmpty) + ThreeDimensionsCards(dimensions: data.dimensions), + + storyH3('Grader matrix'), + const p([ + .text( + 'FlutterBench deploys a mix of deterministic, LLM-as-a-judge, and heuristic graders across all three dimensions:', + ), + ]), + if (graders.isNotEmpty) + GraderMatrix( + title: graderMatrixTitle, + description: graderMatrixDesc, + filters: graderFilters, + graders: graders, + ), + + storyH3('Grader implementation tiers'), + div(classes: 'table-wrapper', [ + table(classes: 'bench-table methodology-table', [ + const thead([ + tr([ + th([.text('Tier')]), + th([.text('Grader Type')]), + th([.text('Evaluation Role')]), + ]), + ]), + tbody([ + for (final row in data.graderTiers.rows) + tr([ + td([ + strong([.text(row.label)]), + ]), + td([.text(row.detail ?? '')]), + td([.text(row.description)]), + ]), + ]), + ]), + ]), + + storyH3('Diagnostic telemetry (excluded from Result Score)'), + const p([ + .text( + 'To avoid penalizing capability scores on complex tasks that naturally require more reasoning steps or tokens, ' + 'FlutterBench tracks diagnostic telemetry separately from the Result Score:', + ), + ]), + div(classes: 'table-wrapper', [ + table(classes: 'bench-table methodology-table', [ + const thead([ + tr([ + th([.text('Metric')]), + th([.text('Measurement Target')]), + ]), + ]), + tbody([ + for (final row in data.diagnosticTelemetry.rows) + tr([ + td([ + strong([.text(row.label)]), + ]), + td([.text(row.description)]), + ]), + ]), + ]), + ]), + ], + ); + } + + Component _buildReliabilityTriageChapter( + FlutterBenchMethodologyData data, { + required List> reliabilityCards, + required String scoreTriageTitle, + required String scoreTriageDesc, + required List> scoreTriageTiers, + }) { + return StoryChapter( + number: '06', + title: 'Reliability & triage', + anchorId: 'reliability-triage', + children: [ + storyH3('Multi-run reliability metrics'), + const p([ + .text( + 'Single-run trials only sample luck. True agent trust requires measuring multi-trial stability across repeated runs:', + ), + ]), + if (reliabilityCards.isNotEmpty) + ReliabilityCards(cards: reliabilityCards), + + storyH3('Score interpretation & triage'), + const p([ + .text( + 'Click a score tier to view its grading criteria and actionable engineering triage steps:', + ), + ]), + if (scoreTriageTiers.isNotEmpty) + InteractiveDetailCard( + title: scoreTriageTitle, + description: scoreTriageDesc, + classes: 'interactive-detail-card score-triage', + tabs: scoreTriageTiers, + ), + + storyH4('Human root-cause audits'), + const p([ + .text( + 'When an evaluation task receives a low Result Score, human expert reviewers inspect the diagnostic process data:', + ), + ]), + ul([ + for (final item in data.rootCauseAudits.items) + li([ + strong([.text('${item.label}: ')]), + .text(item.detail), + ]), + ]), + ], + ); + } + + Component _buildTransparencyChapter(FlutterBenchHarborExample harbor) { + return StoryChapter( + number: '07', + title: 'Transparency & dataset integrity', + anchorId: 'transparency-dataset-integrity', + children: [ + const p([ + .text( + 'To prevent model training contamination, raw datasets and reference solutions cannot be open-sourced. ' + 'However, the Flutter team maintains transparency by:', + ), + ]), + const ul([ + li([ + .text( + 'Publishing the comprehensive evaluation methodology on this page.', + ), + ]), + li([ + .text( + 'Sharing task prompts and Critical User Journey (CUJ) lists.', + ), + ]), + li([ + .text('Publishing regular blog posts with analysis and insights.'), + ]), + li([ + .text( + 'Open-sourcing verification tooling that does not risk dataset compromise.', + ), + ]), + ]), + const p([ + .text( + 'To run benchmark tasks locally using the Harbor evaluation runner:', + ), + ]), + div(classes: 'code-block-wrapper', [ + pre([ + code([ + .text( + '# Run an individual trial with Harbor\n' + 'harbor run \\\n' + ' --task ${harbor.task} \\\n' + ' --agent ${harbor.agent} \\\n' + ' --model ${harbor.model} \\\n' + ' --mcp ${harbor.mcp}', + ), + ]), + ]), + ]), + const p([ + .text( + 'This methodology will evolve as more data is gathered and analyzed. ' + 'Expect updates and refinements in future blog posts and documentation.', + ), + ]), + const p([ + .text( + 'For questions or to contribute new CUJ evaluation tasks, visit the ', + ), + a( + href: 'https://github.com/flutter/flutter', + target: Target.blank, + [.text('Flutter repository on GitHub')], + ), + .text('.'), + ]), + ], + ); + } +} diff --git a/sites/www/lib/src/pages/flutter_bench/flutterbench_models_page.dart b/sites/www/lib/src/pages/flutter_bench/flutterbench_models_page.dart new file mode 100644 index 00000000000..91b338453c0 --- /dev/null +++ b/sites/www/lib/src/pages/flutter_bench/flutterbench_models_page.dart @@ -0,0 +1,69 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../../components/flutterbench/benchmark_scores.dart'; +import '../../components/flutterbench/models_explorer.dart'; +import '../../models/content/flutterbench_content.dart'; +import '../../utils/data_utils.dart'; +import 'flutterbench_nav.dart'; + +/// FlutterBench model directory page. +/// +/// Mounted by `/ai/flutterbench/models/index.md`. +class FlutterBenchModelsPage extends StatelessComponent { + const FlutterBenchModelsPage({super.key}); + + @override + Component build(BuildContext context) { + final job = context.decodeJsonObject( + 'data.flutterbench.job', + FlutterBenchJobData.fromJson, + ); + final tasksData = context.decodeJsonObject( + 'data.flutterbench.tasks', + FlutterBenchTasksData.fromJson, + ); + final trialsData = context.decodeJsonObject( + 'data.flutterbench.trials', + FlutterBenchTrialsData.fromJson, + ); + + final benchmarks = buildBenchmarkRows( + tasks: tasksData, + trials: trialsData, + ); + + return main_(classes: 'bench-page models-page', [ + section(classes: 'bench-hero-header', [ + div(classes: 'bench-container', [ + div(classes: 'hero-badge-row', [ + const span(classes: 'hero-category-tag', [.text('MODELS')]), + span(classes: 'job-id-tag', [ + .text('${job.evals.length} configurations evaluated'), + ]), + ]), + const h1(classes: 'bench-hero-title', [.text('Model Directory')]), + const p(classes: 'bench-hero-subtitle', [ + .text( + 'Every agent and model configuration FlutterBench has evaluated, ' + 'with its full accuracy, cost, and latency profile across each ' + 'Critical User Journey.', + ), + ]), + const FlutterBenchNav(current: FlutterBenchNavItem.models), + ]), + ]), + + div(classes: 'bench-container content-area', [ + ModelsExplorer( + evals: [for (final eval in job.evals) eval.toMap()], + benchmarks: benchmarkRowsToMaps(benchmarks), + ), + ]), + ]); + } +} diff --git a/sites/www/lib/src/pages/flutter_bench/flutterbench_nav.dart b/sites/www/lib/src/pages/flutter_bench/flutterbench_nav.dart new file mode 100644 index 00000000000..ef2a98cff67 --- /dev/null +++ b/sites/www/lib/src/pages/flutter_bench/flutterbench_nav.dart @@ -0,0 +1,44 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +/// A top-level FlutterBench section. +enum FlutterBenchNavItem { + leaderboard(label: 'Leaderboard', href: '/ai/flutterbench'), + models(label: 'Models', href: '/ai/flutterbench/models'), + tasks(label: 'Tasks & CUJs', href: '/ai/flutterbench/tasks'), + methodology(label: 'Methodology', href: '/ai/flutterbench/methodology'), + cujs(label: 'CUJs', href: '/ai/flutterbench/cujs'); + + const FlutterBenchNavItem({required this.label, required this.href}); + + final String label; + final String href; +} + +/// The sub-navigation shared by every FlutterBench page. +class FlutterBenchNav extends StatelessComponent { + const FlutterBenchNav({required this.current, super.key}); + + /// The section currently being viewed. + final FlutterBenchNavItem current; + + @override + Component build(BuildContext context) { + return nav(classes: 'bench-tab-nav', [ + for (final item in FlutterBenchNavItem.values) + a( + href: item.href, + classes: [ + 'bench-nav-link', + if (item == current) 'active', + ].join(' '), + attributes: {if (item == current) 'aria-current': 'page'}, + [.text(item.label)], + ), + ]); + } +} diff --git a/sites/www/lib/src/pages/flutter_bench/flutterbench_task_detail_page.dart b/sites/www/lib/src/pages/flutter_bench/flutterbench_task_detail_page.dart new file mode 100644 index 00000000000..5170d48dbea --- /dev/null +++ b/sites/www/lib/src/pages/flutter_bench/flutterbench_task_detail_page.dart @@ -0,0 +1,169 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../../components/flutterbench/error_state_badge.dart'; +import '../../components/flutterbench/model_name_formatter.dart'; +import '../../models/content/flutterbench_content.dart'; +import '../../utils/data_utils.dart'; + +/// Single task / CUJ detail page showing cross-model results. +/// +/// Mounted by `/ai/flutterbench/tasks/.md`. +class FlutterBenchTaskDetailPage extends StatelessComponent { + const FlutterBenchTaskDetailPage({required this.taskSlug, super.key}); + + factory FlutterBenchTaskDetailPage.fromAttrs(Map attrs) { + return FlutterBenchTaskDetailPage(taskSlug: attrs['task'] as String); + } + + final String taskSlug; + + @override + Component build(BuildContext context) { + final tasksData = context.decodeJsonObject( + 'data.flutterbench.tasks', + FlutterBenchTasksData.fromJson, + ); + + final task = tasksData.tasks.firstWhere( + (t) => t.slug == taskSlug, + orElse: () => tasksData.tasks.first, + ); + + return main_(classes: 'bench-page task-detail-page', [ + div(classes: 'bench-container content-area', [ + // Breadcrumbs + div(classes: 'bench-breadcrumbs', [ + const a(href: '/ai', [.text('AI')]), + const span(classes: 'breadcrumb-sep', [.text(' / ')]), + const a(href: '/ai/flutterbench', [.text('FlutterBench')]), + const span(classes: 'breadcrumb-sep', [.text(' / ')]), + const a(href: '/ai/flutterbench/tasks', [.text('Tasks')]), + const span(classes: 'breadcrumb-sep', [.text(' / ')]), + span(classes: 'current-page', [.text(task.displayName)]), + ]), + + // Task Header Card + section(classes: 'bench-card task-info-card', [ + div(classes: 'task-header-row', [ + div(classes: 'task-title-area', [ + span(classes: 'category-pill', [.text(task.category)]), + h1(classes: 'task-page-title', [.text(task.displayName)]), + span(classes: 'task-repo-identifier', [ + .text('ID: ${task.taskName}'), + ]), + ]), + ]), + p(classes: 'task-description-lead', [.text(task.description)]), + ]), + + // Cross-model trials comparison table + section(classes: 'bench-section', [ + const h2(classes: 'section-h2', [ + .text('Cross-Model Benchmark Trials'), + ]), + const p(classes: 'section-intro-text', [ + .text('Comparison of agent configurations evaluated on this task:'), + ]), + + div(classes: 'bench-table-wrapper', [ + table(classes: 'bench-table', [ + const thead([ + tr([ + th([.text('Model & Harness')]), + th([.text('Tooling')]), + th([.text('Status')]), + th([.text('Composite Reward')]), + th([.text('Actions')]), + ]), + ]), + tbody([ + for (final trial in task.trials) + tr(classes: 'task-trial-row', [ + td([ + div(classes: 'trial-model-cell', [ + span(classes: 'model-name-bold', [ + .text(formatModelName(trial.modelShortName)), + ]), + span(classes: 'agent-sub', [.text(trial.agentName)]), + ]), + ]), + td([ + if (trial.hasDartTooling) + const span(classes: 'tooling-pill', [ + .text('Dart MCP + Skills'), + ]) + else + const span(classes: 'tooling-pill uninstrumented', [ + .text('Baseline'), + ]), + ]), + td([ + if (trial.status == 'error') + ErrorStateBadge( + exceptionType: trial.exceptionType ?? 'Error', + compact: true, + ) + else + span( + classes: [ + 'trial-status-badge', + if (trial.status == 'pass') + 'status-pass' + else if (trial.status == 'partial') + 'status-partial' + else + 'status-fail', + ].join(' '), + [.text(trial.status.toUpperCase())], + ), + ]), + td([ + if (trial.reward != null) + span( + classes: [ + 'reward-main-score', + if (trial.reward! >= 0.8) + 'score-high' + else if (trial.reward! >= 0.5) + 'score-mid' + else + 'score-low', + ].join(' '), + [.text(trial.reward!.toStringAsFixed(2))], + ) + else + const span(classes: 'text-muted', [ + .text('— (Errored)'), + ]), + ]), + td([ + a( + href: '/ai/flutterbench/trials/${trial.trialName}', + classes: 'bench-btn-outline', + const [.text('Inspect Trial Details →')], + ), + ]), + ]), + ]), + ]), + ]), + ]), + + // Back link navigation + const div(classes: 'bench-bottom-nav', [ + a(href: '/ai/flutterbench/tasks', classes: 'btn quiet', [ + .text('← Back to Task Matrix'), + ]), + a(href: '/ai/flutterbench', classes: 'btn quiet', [ + .text('← Back to Leaderboard'), + ]), + ]), + ]), + ]); + } +} diff --git a/sites/www/lib/src/pages/flutter_bench/flutterbench_tasks_page.dart b/sites/www/lib/src/pages/flutter_bench/flutterbench_tasks_page.dart new file mode 100644 index 00000000000..eaae426b8e1 --- /dev/null +++ b/sites/www/lib/src/pages/flutter_bench/flutterbench_tasks_page.dart @@ -0,0 +1,110 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../../components/flutterbench/task_model_heatmap.dart'; +import '../../models/content/flutterbench_content.dart'; +import '../../utils/data_utils.dart'; +import 'flutterbench_nav.dart'; + +/// FlutterBench Tasks & CUJ Matrix explorer page. +/// +/// Mounted by `/ai/flutterbench/tasks/index.md`. +class FlutterBenchTasksPage extends StatelessComponent { + const FlutterBenchTasksPage({super.key}); + + @override + Component build(BuildContext context) { + final job = context.decodeJsonObject( + 'data.flutterbench.job', + FlutterBenchJobData.fromJson, + ); + final tasksData = context.decodeJsonObject( + 'data.flutterbench.tasks', + FlutterBenchTasksData.fromJson, + ); + + return main_(classes: 'bench-page tasks-page', [ + // Hero Header + section(classes: 'bench-hero-header', [ + div(classes: 'bench-container', [ + div(classes: 'hero-badge-row', [ + const span(classes: 'hero-category-tag', [.text('TASK MATRIX')]), + span(classes: 'job-id-tag', [ + .text('${tasksData.tasks.length} CUJs Evaluated'), + ]), + ]), + const h1(classes: 'bench-hero-title', [.text('Task & CUJ Explorer')]), + const p(classes: 'bench-hero-subtitle', [ + .text( + 'Explore cross-model performance across authentic Flutter and Dart developer workflows. ' + 'Click any cell in the matrix to inspect the full trial execution, rubric, trajectory, and logs.', + ), + ]), + + // Sub-nav tabs + const FlutterBenchNav(current: FlutterBenchNavItem.tasks), + ]), + ]), + + div(classes: 'bench-container content-area', [ + // Heatmap Matrix + section(classes: 'bench-section', [ + const div(classes: 'section-title-row', [ + h2(classes: 'section-h2', [ + .text('Cross-Model Performance Heatmap'), + ]), + span(classes: 'section-note', [ + .text( + 'Color scaled by reward (Green ≥ 0.80, Yellow 0.50–0.79, Red < 0.50). Errored trials hatched in gray.', + ), + ]), + ]), + TaskModelHeatmap( + tasks: tasksData.tasks, + evals: job.evals, + ), + ]), + + // Task Cards Directory + section(classes: 'bench-section', [ + const h2(classes: 'section-h2', [ + .text('Evaluated Critical User Journeys'), + ]), + const p(classes: 'section-intro-text', [ + .text( + 'Each task is grounded in realistic development workflows and includes automated grading suites:', + ), + ]), + div(classes: 'tasks-cards-grid', [ + for (final task in tasksData.tasks) + div(classes: 'task-card', [ + div(classes: 'task-card-header', [ + span(classes: 'category-pill', [.text(task.category)]), + h3(classes: 'task-card-title', [ + a(href: '/ai/flutterbench/tasks/${task.slug}', [ + .text(task.displayName), + ]), + ]), + ]), + p(classes: 'task-card-description', [.text(task.description)]), + div(classes: 'task-card-footer', [ + span(classes: 'trial-count-sub', [ + .text('${task.trials.length} trials recorded'), + ]), + a( + href: '/ai/flutterbench/tasks/${task.slug}', + classes: 'bench-btn-sm', + const [.text('View Task Details →')], + ), + ]), + ]), + ]), + ]), + ]), + ]); + } +} diff --git a/sites/www/lib/src/pages/flutter_bench/flutterbench_trial_detail_page.dart b/sites/www/lib/src/pages/flutter_bench/flutterbench_trial_detail_page.dart new file mode 100644 index 00000000000..65b962bf546 --- /dev/null +++ b/sites/www/lib/src/pages/flutter_bench/flutterbench_trial_detail_page.dart @@ -0,0 +1,57 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'package:jaspr/dom.dart'; +import 'package:jaspr/jaspr.dart'; + +import '../../components/flutterbench/trial_detail_view.dart'; +import '../../models/content/flutterbench_content.dart'; +import '../../utils/data_utils.dart'; + +/// Single trial detail page showing verifier scoring, trajectory, artifacts, and logs. +/// +/// Mounted by `/ai/flutterbench/trials/.md`. +class FlutterBenchTrialDetailPage extends StatelessComponent { + const FlutterBenchTrialDetailPage({required this.trialName, super.key}); + + factory FlutterBenchTrialDetailPage.fromAttrs(Map attrs) { + return FlutterBenchTrialDetailPage(trialName: attrs['trial'] as String); + } + + final String trialName; + + @override + Component build(BuildContext context) { + final trialsData = context.decodeJsonObject( + 'data.flutterbench.trials', + FlutterBenchTrialsData.fromJson, + ); + + final trial = trialsData.trials.firstWhere( + (t) => t.trialName == trialName, + orElse: () => trialsData.trials.first, + ); + + return main_(classes: 'bench-page trial-detail-page', [ + div(classes: 'bench-container content-area', [ + TrialDetailView(trial: trial), + div(classes: 'bench-bottom-nav', [ + a( + href: '/ai/flutterbench/tasks/${trial.taskSlug}', + classes: 'btn quiet', + [ + .text('← Back to ${trial.taskSlug} Task'), + ], + ), + const a(href: '/ai/flutterbench/tasks', classes: 'btn quiet', [ + .text('← Back to Tasks Matrix'), + ]), + const a(href: '/ai/flutterbench', classes: 'btn quiet', [ + .text('← Back to Leaderboard'), + ]), + ]), + ]), + ]); + } +} diff --git a/sites/www/lib/src/pages/home_page.dart b/sites/www/lib/src/pages/home_page.dart index 0778b472681..d03c6a28d6f 100644 --- a/sites/www/lib/src/pages/home_page.dart +++ b/sites/www/lib/src/pages/home_page.dart @@ -214,8 +214,7 @@ class HomePage extends StatelessComponent { ]), a( classes: 'btn', - href: - 'https://dartpad.dev/?id=e66e420f2f0201c772f73819711bf290', + href: 'https://dartpad.dev/?id=e66e420f2f0201c772f73819711bf290', attributes: {'target': '_blank'}, [.text('Try it in DartPad')], ), @@ -257,8 +256,7 @@ class HomePage extends StatelessComponent { ]), a( classes: 'btn', - href: - 'https://dartpad.dev/?id=bbd3f10c2593f0add04dd770318b33f7', + href: 'https://dartpad.dev/?id=bbd3f10c2593f0add04dd770318b33f7', attributes: {'target': '_blank'}, [.text('Try it in DartPad')], ), @@ -299,8 +297,7 @@ class HomePage extends StatelessComponent { ]), a( classes: 'btn', - href: - 'https://dartpad.dev/?id=1ab1b78a18039bbbd5cfbb4b835b5b8d', + href: 'https://dartpad.dev/?id=1ab1b78a18039bbbd5cfbb4b835b5b8d', attributes: {'target': '_blank'}, [.text('Try it in DartPad')], ), diff --git a/sites/www/lib/styles/components/_drawer.scss b/sites/www/lib/styles/components/_drawer.scss new file mode 100644 index 00000000000..5accd08934e --- /dev/null +++ b/sites/www/lib/styles/components/_drawer.scss @@ -0,0 +1,133 @@ +// Slide-in drawer overlay. +// +// The markup is produced by `src/components/common/drawer.dart`. +// The drawer stays in the DOM while closed so that +// both the enter and exit transitions can run. + +.drawer { + --drawer-width: 46rem; + --drawer-transition-duration: 250ms; + + position: fixed; + inset: 0; + z-index: var(--site-z-top, 1000); + visibility: hidden; + // Delay hiding until the panel has finished sliding out. + transition: visibility 0s linear var(--drawer-transition-duration); + + &.drawer--open { + visibility: visible; + transition-delay: 0s; + } +} + +.drawer__scrim { + position: absolute; + inset: 0; + background-color: rgba(0, 0, 0, 0.45); + opacity: 0; + transition: opacity var(--drawer-transition-duration) var(--ui-anim-func); + + .drawer--open & { + opacity: 1; + } +} + +.drawer__panel { + position: absolute; + inset-block: 0; + inset-inline-end: 0; + display: flex; + flex-direction: column; + width: min(var(--drawer-width), 100vw); + background-color: var(--site-base-bgColor); + color: var(--site-base-fgColor); + box-shadow: var(--ui-drop-shadow); + // Nudged past the edge so the shadow doesn't peek through while closed. + transform: translateX(calc(100% + 2rem)); + transition: transform var(--drawer-transition-duration) var(--ui-anim-func); + + &:focus-visible { + outline: none; + } + + .drawer--start & { + inset-inline: 0 auto; + transform: translateX(calc(-100% - 2rem)); + } + + .drawer--open & { + transform: translateX(0); + } +} + +.drawer__header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--gutter-sm); + flex: 0 0 auto; + padding: 1.25rem 1.5rem; + border-bottom: 1px solid var(--site-inset-borderColor); + + // When the body supplies its own heading, the header collapses to just the + // close button and hands its bottom edge over to the content. + &.drawer__header--bare { + justify-content: flex-end; + padding-bottom: 0; + border-bottom: none; + } +} + +.drawer__title { + margin: 0; + font-family: var(--font-ui); + font-size: var(--font-size-heading-4); + font-weight: var(--font-weight-bolder); + line-height: 1.3; +} + +.drawer__subtitle { + margin: 0.25rem 0 0; + font-size: var(--font-size-default); + color: var(--site-base-fgColor-alt); +} + +.drawer__close { + display: flex; + align-items: center; + justify-content: center; + flex: 0 0 auto; + height: var(--ui-btn-height); + width: var(--ui-btn-height); + padding: 0; + border: none; + border-radius: 100%; + background: none; + color: var(--site-base-fgColor-alt); + cursor: pointer; + transition: background-color 150ms var(--ui-anim-func); + + &:hover { + background-color: var(--site-inset-bgColor); + color: var(--site-base-fgColor); + } +} + +.drawer__body { + flex: 1 1 auto; + overflow-y: auto; + overscroll-behavior: contain; + padding: 1.5rem; +} + +@media (prefers-reduced-motion: reduce) { + .drawer { + transition-delay: 0s; + } + + .drawer__scrim, + .drawer__panel { + transition-duration: 1ms; + } +} diff --git a/sites/www/lib/styles/pages/_flutterbench-story.scss b/sites/www/lib/styles/pages/_flutterbench-story.scss new file mode 100644 index 00000000000..61754f79cab --- /dev/null +++ b/sites/www/lib/styles/pages/_flutterbench-story.scss @@ -0,0 +1,345 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +// Numbered "story chapter" treatment for the FlutterBench methodology page, +// ported from the original docs.flutter.dev version of this page +// (sites/docs/lib/_sass/pages/_story-layout.scss). + +@use 'sass:list'; +@use 'sass:map'; + +@function _pad-chapter-number($i) { + @if $i < 10 { + @return "0#{$i}"; + } + + @return "#{$i}"; +} + +// ----------------------------------------------------------------------------- +// Chapter accent colors +// ----------------------------------------------------------------------------- +// NOTE: These colors intentionally match the category colors defined in +// _flutterbench.scss ($color-functional, $color-outcome, $color-llm, +// $color-quality, $color-dx, $color-grey) for visual consistency. +$story-progress-gradient: linear-gradient(90deg, #0284c7 0%, #6366f1 50%, #059669 100%); + +$chapter-colors: ( + (rgb: "2, 132, 199", hex: #0284c7), + // Sky blue (functional) + (rgb: "5, 150, 105", hex: #059669), + // Green (outcome) + (rgb: "124, 58, 237", hex: #7c3aed), + // Violet (llm) + (rgb: "99, 102, 241", hex: #6366f1), + // Indigo (quality) + (rgb: "217, 119, 6", hex: #d97706), + // Amber (dx) + (rgb: "100, 116, 139", hex: #64748b) // Slate (grey) +); + +// ----------------------------------------------------------------------------- +// Scroll-driven reading progress line +// ----------------------------------------------------------------------------- +.story-reading-progress { + position: sticky; + top: var(--site-header-height, 4rem); + width: 100%; + height: 3px; + z-index: var(--site-z-top, 1000); + background: rgb(216, 216, 216); + pointer-events: none; + + .story-reading-progress-bar { + height: 100%; + width: 100%; + transform-origin: 0% 50%; + transform: scaleX(0); + background: $story-progress-gradient; + box-shadow: 0 1px 4px rgba(2, 132, 199, 0.25); + + @supports ((animation-timeline: scroll()) and (animation-range: 0% 100%)) { + animation: story-reading-progress-grow linear both; + animation-timeline: scroll(root block); + } + } +} + +@keyframes story-reading-progress-grow { + from { + transform: scaleX(0); + } + + to { + transform: scaleX(1); + } +} + +// ----------------------------------------------------------------------------- +// Chapter layout +// ----------------------------------------------------------------------------- +.methodology-page { + scroll-padding-top: calc(var(--site-header-height, 4rem) + 1.5rem); + + .story-canvas { + display: flex; + flex-direction: column; + width: 100%; + } + + .story-chapter { + position: relative; + padding-block: 3rem; + scroll-snap-align: start; + scroll-margin-top: calc(var(--site-header-height, 4rem) + 1.5rem); + transition: background-color 0.4s ease, border-color 0.4s ease; + + // Default fallback accent color + --chapter-accent: #0284c7; + --chapter-accent-rgb: 2, 132, 199; + --chapter-next-rgb: 5, 150, 105; + + // Generate chapter accent colors from palette + @for $i from 0 through 20 { + $formatted: _pad-chapter-number($i); + $curr-idx: ($i % list.length($chapter-colors)) + 1; + $next-idx: (($i + 1) % list.length($chapter-colors)) + 1; + + $curr: list.nth($chapter-colors, $curr-idx); + $next: list.nth($chapter-colors, $next-idx); + + &[data-chapter="#{$formatted}"] { + --chapter-accent: #{map.get($curr, hex)}; + --chapter-accent-rgb: #{map.get($curr, rgb)}; + --chapter-next-rgb: #{map.get($next, rgb)}; + } + } + + // Ambient atmosphere vignette + &::before { + content: ''; + position: absolute; + top: 0; + left: -2rem; + right: -2rem; + height: 100%; + background: + radial-gradient(650px circle at 50% 0%, rgba(var(--chapter-accent-rgb), 0.08), transparent 90%), + radial-gradient(650px circle at 50% 100%, rgba(var(--chapter-next-rgb), 0.08), transparent 90%); + pointer-events: none; + z-index: 0; + } + + &:first-of-type { + padding-top: clamp(2rem, 5vh, 4rem); + } + + &:last-of-type { + border-bottom: none; + padding-bottom: clamp(6rem, 12vh, 12rem); + } + + // Sticky chapter header & eyebrow waypoint + .chapter-header-group { + position: sticky; + top: calc(var(--site-header-height, 4rem) + 3px); + z-index: var(--site-z-floating, 10); + padding-block: 0.85rem 0.5rem; + margin-bottom: 2rem; + border-bottom: 1px solid var(--site-outline-variant); + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + background: var(--site-base-bgColor); + + .chapter-kicker, + > .header-wrapper { + width: 100%; + margin-inline: auto; + } + + .chapter-kicker { + display: flex; + align-items: center; + gap: 0.75rem; + margin-bottom: 0.35rem; + text-transform: uppercase; + letter-spacing: 0.08em; + font-size: 0.75rem; + font-weight: 700; + color: var(--chapter-accent); + + .chapter-number { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0.15rem 0.55rem; + border-radius: 9999px; + background: rgba(var(--chapter-accent-rgb), 0.08); + border: 1px solid rgba(var(--chapter-accent-rgb), 0.25); + font-family: var(--site-code-fontFamily); + font-size: 0.8rem; + line-height: 1.2; + color: var(--chapter-accent); + } + + .chapter-rule { + display: inline-block; + width: 2rem; + height: 1px; + background: currentColor; + opacity: 0.35; + } + } + + > .header-wrapper { + margin: 0; + + h2 { + font-weight: 700; + } + } + } + + // Prose centering vs. component breakouts + .chapter-content { + position: relative; + z-index: 1; + display: flex; + flex-direction: column; + gap: 0.5rem; + width: 100%; + + > p, + > ul, + > ol, + > blockquote, + > .table-wrapper, + > .alert, + > .methodology-note-box, + > .callout { + width: 100%; + max-width: 700px; + line-height: 1.75; + margin-left: 0.5rem; + margin-right: 0.5rem; + } + + // Sub-headings inside a chapter, centered to match prose + > .header-wrapper { + width: 100%; + + h3 { + font-size: clamp(1.3rem, 1.8vw, 1.55rem); + font-weight: 600; + margin-top: 2rem; + margin-bottom: 0.5rem; + } + + h4 { + font-size: 1.15rem; + font-weight: 600; + margin-top: 1.5rem; + margin-bottom: 0.5rem; + } + } + } + } + + // Heading anchor links, matching the site's article.content treatment + // (see components/_content.scss) for pages, like this one, that aren't + // wrapped in article.content. + .header-wrapper { + display: flex; + margin-block-start: 1.5rem; + margin-block-end: 0.75rem; + align-items: center; + + > h2, + h3, + h4 { + margin: 0; + } + + .heading-link { + border-radius: 0.125rem; + margin-left: 0.4rem; + font-size: 1.3rem; + line-height: 1; + transition: all 0.1s ease-in-out; + overflow: hidden; + color: var(--site-base-fgColor-alt); + opacity: 0; + text-decoration: none; + + &:hover { + color: var(--site-link-fgColor); + } + + &:focus { + opacity: 1; + } + + &:active { + color: var(--site-link-fgColor-active); + } + } + + &:hover { + .heading-link { + opacity: 1; + } + } + } + + // Dark mode adjustments + body.dark-mode & { + .story-reading-progress { + background: rgba(255, 255, 255, 0.06); + } + + .story-chapter { + border-bottom-color: rgba(255, 255, 255, 0.08); + + .chapter-header-group { + background: var(--site-base-bgColor); + border-bottom-color: rgba(255, 255, 255, 0.08); + } + + &::before { + background: transparent; + } + + .chapter-kicker .chapter-number { + background: rgba(var(--chapter-accent-rgb), 0.15); + border-color: rgba(var(--chapter-accent-rgb), 0.35); + } + } + } +} + +// Accessibility: respect user motion preferences +@media (prefers-reduced-motion: reduce) { + html:has(.methodology-page) { + scroll-snap-type: none; + } + + .methodology-page { + scroll-behavior: auto; + + .story-reading-progress .story-reading-progress-bar { + animation: none !important; + display: none; + } + + .story-chapter { + transition: none !important; + + .chapter-header-group { + position: static; + backdrop-filter: none; + -webkit-backdrop-filter: none; + } + } + } +} diff --git a/sites/www/lib/styles/pages/_flutterbench.scss b/sites/www/lib/styles/pages/_flutterbench.scss new file mode 100644 index 00000000000..28baf4f0a06 --- /dev/null +++ b/sites/www/lib/styles/pages/_flutterbench.scss @@ -0,0 +1,3839 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +@use '../core/breakpoints'; +@use 'package:site_shared/_sass/base/breakpoints' as shared-bp; + +// ----------------------------------------------------------------------------- +// Category Colors for Methodology +// ----------------------------------------------------------------------------- +$color-outcome: #059669; // Green - successful outcomes +$color-quality: #6366f1; // Indigo - code quality metrics +$color-dx: #d97706; // Amber - developer experience +$color-functional: #0284c7; // Sky blue - functional correctness +$color-llm: #7c3aed; // Violet - LLM grading +$color-heuristic: #2563eb; // Blue - heuristic grading +$color-poor: #ea580c; // Orange - poor performance +$color-failure: #e11d48; // Rose - failures +$color-grey: #64748b; // Slate - neutral/baseline +$color-white: #ffffff; +$color-gold: #fbbf24; + +$hero-bg-light: linear-gradient(135deg, #0d2040 0%, #1e1b4b 50%, #0f172a 100%); +$hero-bg-dark: linear-gradient(135deg, #091728 0%, #131230 50%, #090e17 100%); +$north-star-bg-light: linear-gradient(135deg, #102a4e 0%, #1e1b4b 100%); +$north-star-bg-dark: linear-gradient(135deg, #0d1e38 0%, #151433 100%); + +$font-size-xs: 0.75rem; +$font-size-sm: 0.875rem; +$font-size-md: 1rem; +$font-size-lg: 1.25rem; +$font-size-xl: 1.5rem; +$font-size-xxl: 2.5rem; + +$spacing-xs: 0.25rem; +$spacing-sm: 0.5rem; +$spacing-md: 1rem; +$spacing-lg: 1.5rem; +$spacing-xl: 2rem; + +$border-width: 2px; +$icon-size: 2.75rem; +$card-width: 280px; +$bench-content-max-width: 1000px; + +$shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.08); +$shadow-md: 0 4px 12px rgba(0, 0, 0, 0.08); +$shadow-hero: 0 4px 12px rgba(0, 0, 0, 0.2); +$shadow-hero-dark: 0 4px 12px rgba(0, 0, 0, 0.4); +$transition-normal: 0.2s ease; + +@mixin bench-element-width($max-width: $bench-content-max-width) { + width: 100%; + max-width: $max-width; + margin-left: auto; + margin-right: auto; +} + +@mixin custom-scrollbar($orientation: 'vertical') { + scrollbar-width: thin; + scrollbar-color: var(--site-inset-borderColor) transparent; + -webkit-overflow-scrolling: touch; + + &::-webkit-scrollbar { + @if $orientation == 'horizontal' { + height: 6px; + } @else { + width: 6px; + } + } + + &::-webkit-scrollbar-thumb { + background-color: var(--site-inset-borderColor); + border-radius: 3px; + } +} + +@mixin category-badge($with-border: false) { + font-size: $font-size-xs; + font-weight: 600; + padding: $spacing-xs $spacing-sm; + border-radius: var(--site-radius, $spacing-xs); + + &.badge-outcome, + &.outcome { + background: rgba($color-outcome, 0.12); + color: $color-outcome; + @if $with-border { + border: 1px solid rgba($color-outcome, 0.25); + } + } + + &.badge-quality, + &.quality { + background: rgba($color-quality, 0.12); + color: $color-quality; + @if $with-border { + border: 1px solid rgba($color-quality, 0.25); + } + } + + &.badge-dx, + &.dx { + background: rgba($color-dx, 0.12); + color: $color-dx; + @if $with-border { + border: 1px solid rgba($color-dx, 0.25); + } + } + + &.badge-functional, + &.badge-blue { + background: rgba($color-functional, 0.12); + color: $color-functional; + @if $with-border { + border: 1px solid rgba($color-functional, 0.25); + } + } + + &.badge-llm { + background: rgba($color-llm, 0.12); + color: $color-llm; + @if $with-border { + border: 1px solid rgba($color-llm, 0.25); + } + } + + &.badge-heuristic { + background: rgba($color-heuristic, 0.12); + color: $color-heuristic; + @if $with-border { + border: 1px solid rgba($color-heuristic, 0.25); + } + } + + &.badge-poor { + background: rgba($color-poor, 0.12); + color: $color-poor; + @if $with-border { + border: 1px solid rgba($color-poor, 0.25); + } + } + + &.badge-failure { + background: rgba($color-failure, 0.12); + color: $color-failure; + @if $with-border { + border: 1px solid rgba($color-failure, 0.25); + } + } + + &.badge-grey { + background: rgba($color-grey, 0.12); + color: $color-grey; + @if $with-border { + border: 1px solid rgba($color-grey, 0.25); + } + } + + &.badge-deterministic, + &.badge-perfect, + &.badge-green { + background: rgba($color-outcome, 0.12); + color: $color-outcome; + @if $with-border { + border: 1px solid rgba($color-outcome, 0.25); + } + } + + &.badge-purple { + background: rgba($color-quality, 0.12); + color: $color-quality; + @if $with-border { + border: 1px solid rgba($color-quality, 0.25); + } + } + + &.badge-amber { + background: rgba($color-dx, 0.12); + color: $color-dx; + @if $with-border { + border: 1px solid rgba($color-dx, 0.25); + } + } +} + +@mixin category-icon-background { + width: $icon-size; + height: $icon-size; + border-radius: $spacing-sm; + display: flex; + align-items: center; + justify-content: center; + + &.outcome, + &.variant-outcome, + &.variant-perfect, + &.variant-green { + background-color: rgba($color-outcome, 0.15); + color: $color-outcome; + } + + &.quality, + &.variant-quality, + &.variant-purple { + background-color: rgba($color-quality, 0.15); + color: $color-quality; + } + + &.dx, + &.variant-dx, + &.variant-amber { + background-color: rgba($color-dx, 0.15); + color: $color-dx; + } + + &.variant-blue, + &.variant-functional { + background-color: rgba($color-functional, 0.12); + color: $color-functional; + } + + &.variant-grey { + background-color: rgba($color-grey, 0.15); + color: $color-grey; + } + + .material-symbols { + font-size: $font-size-xl; + } +} + +:root { + --bench-success-bg: rgba(20, 194, 173, 0.14); + --bench-success-fg: #00796b; + --bench-success-border: rgba(20, 194, 173, 0.4); + + --bench-warning-bg: rgba(242, 180, 0, 0.16); + --bench-warning-fg: #b06000; + --bench-warning-border: rgba(242, 180, 0, 0.45); + + --bench-danger-bg: rgba(212, 51, 36, 0.12); + --bench-danger-fg: #c5221f; + --bench-danger-border: rgba(212, 51, 36, 0.4); + + --bench-error-bg: rgba(100, 116, 139, 0.12); + --bench-error-fg: #d93025; + --bench-error-border: #ea8600; + + --bench-diagnostic-bg: rgba(121, 108, 235, 0.08); + --bench-diagnostic-border: rgba(121, 108, 235, 0.3); + --bench-diagnostic-fg: #5b4cdb; +} + +@media (prefers-color-scheme: dark) { + :root { + --bench-success-bg: rgba(28, 218, 197, 0.2); + --bench-success-fg: #80cbc4; + --bench-success-border: rgba(28, 218, 197, 0.5); + + --bench-warning-bg: rgba(242, 221, 34, 0.2); + --bench-warning-fg: #ffe082; + --bench-warning-border: rgba(242, 221, 34, 0.5); + + --bench-danger-bg: rgba(242, 93, 80, 0.2); + --bench-danger-fg: #ef9a9a; + --bench-danger-border: rgba(242, 93, 80, 0.5); + + --bench-error-bg: rgba(148, 163, 184, 0.18); + --bench-error-fg: #ff8a80; + --bench-error-border: #ffab40; + + --bench-diagnostic-bg: rgba(198, 186, 250, 0.15); + --bench-diagnostic-border: rgba(198, 186, 250, 0.4); + --bench-diagnostic-fg: #c6bafa; + } +} + +body.flutterbench { + background-color: var(--site-base-bgColor); + color: var(--site-base-fgColor); + font-family: var(--font-body); + + .bench-container { + max-width: 1200px; + margin: 0 auto; + padding: 0 1.5rem; + + &.content-area { + padding-top: 2rem; + padding-bottom: 4rem; + } + } + + // --------------------------------------------------------------------------- + // Hero Header & Navigation Tabs + // --------------------------------------------------------------------------- + .bench-hero-header { + background: linear-gradient(135deg, #041e3c 0%, #0d2a54 60%, #042b59 100%); + color: #ffffff; + padding: 3.5rem 0 0 0; + border-bottom: 1px solid var(--site-inset-borderColor); + + .hero-badge-row { + display: flex; + align-items: center; + gap: 0.75rem; + margin-bottom: 1rem; + + .hero-category-tag { + font-size: 0.75rem; + font-weight: 700; + letter-spacing: 0.08em; + padding: 0.25rem 0.6rem; + background-color: var(--blue-5); + color: #ffffff; + border-radius: 4px; + } + + .job-id-tag { + font-size: 0.8rem; + color: var(--blue-3); + font-family: var(--font-code); + } + } + + .bench-hero-title { + font-size: 2.5rem; + font-weight: 700; + line-height: 1.15; + margin-bottom: 1rem; + color: #ffffff; + } + + .bench-hero-subtitle { + font-size: 1.15rem; + color: var(--blue-1); + max-width: 800px; + line-height: 1.6; + margin-bottom: 2.5rem; + } + + .bench-tab-nav { + display: flex; + gap: 0.5rem; + overflow-x: auto; + border-bottom: none; + + .bench-nav-link { + display: inline-block; + padding: 0.75rem 1.25rem; + font-size: 0.95rem; + font-weight: 600; + color: var(--blue-2); + text-decoration: none; + border-bottom: 3px solid transparent; + transition: all 0.15s ease; + + &:hover { + color: #ffffff; + border-bottom-color: rgba(255, 255, 255, 0.4); + } + + &.active { + color: #ffffff; + border-bottom-color: var(--blue-4); + } + } + } + } + + // --------------------------------------------------------------------------- + // Breadcrumbs + // --------------------------------------------------------------------------- + .bench-breadcrumbs { + font-size: 0.85rem; + color: var(--site-base-fgColor-alt); + margin-bottom: 1.5rem; + + a { + color: var(--site-link-fgColor); + text-decoration: none; + &:hover { + text-decoration: underline; + } + } + + .breadcrumb-sep { + margin: 0 0.35rem; + color: var(--site-inset-borderColor); + } + + .current-page { + color: var(--site-base-fgColor); + font-weight: 600; + } + } + + // --------------------------------------------------------------------------- + // Summary Stats Bar + // --------------------------------------------------------------------------- + .bench-stats-bar-container { + margin-bottom: 2.5rem; + + .bench-stats-grid { + display: grid; + grid-template-columns: 1fr; + gap: 1rem; + + @include breakpoints.screen(md) { + grid-template-columns: repeat(2, 1fr); + } + + @include breakpoints.screen(lg) { + grid-template-columns: repeat(4, 1fr); + } + } + + .bench-stat-card { + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + padding: 1.25rem; + box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04); + display: flex; + flex-direction: column; + justify-content: space-between; + + &.card-primary { + border-left: 4px solid var(--blue-5); + } + + &.card-warning { + border-left: 4px solid var(--coral); + } + + .stat-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 0.5rem; + + .stat-label { + font-size: 0.8rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--site-base-fgColor-alt); + } + + .stat-badge { + font-size: 0.7rem; + font-weight: 700; + padding: 0.15rem 0.45rem; + border-radius: 4px; + + &.badge-blue { + background-color: var(--blue-1); + color: var(--blue-7); + } + &.badge-neutral { + background-color: var(--grey-2); + color: var(--grey-6); + } + &.badge-green { + background-color: var(--bench-success-bg); + color: var(--bench-success-fg); + } + } + } + + .stat-value { + font-size: 1.75rem; + font-weight: 700; + color: var(--site-base-fgColor); + line-height: 1.2; + margin-bottom: 0.5rem; + + &.error-value { + display: flex; + align-items: center; + } + } + + .stat-meta { + font-size: 0.8rem; + color: var(--site-base-fgColor-alt); + + .meta-highlight { + font-weight: 700; + color: var(--bench-success-fg); + } + } + } + } + + // --------------------------------------------------------------------------- + // Filter Bar + // --------------------------------------------------------------------------- + .bench-filter-bar { + display: flex; + align-items: center; + justify-content: end; + gap: 1.25rem; + margin: 1.25rem 0; + + .dropdown { + .filters-dropdown__button { + color: var(--site-base-fgColor); + border-color: var(--site-inset-borderColor); + + &:hover { + color: var(--blue-6); + border-color: var(--blue-5); + } + + &.primary { + background-color: var(--blue-6); + border-color: var(--blue-6); + color: #ffffff !important; + } + } + + .filters-dropdown-content { + left: 0; + right: auto; + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + + .filters-dropdown__filters-container { + grid-template-columns: 16rem 1px 16rem; + + > .separator { + background-color: var(--site-inset-borderColor); + + &:last-child { + display: none; + } + } + + .filter-title { + color: var(--site-base-fgColor); + } + + .filters-dropdown__checkbox-container > label { + color: var(--site-base-fgColor); + } + } + + &.mobile-open { + .filters-dropdown__filters-container { + grid-template-columns: none; + } + } + + .filters-dropdown__buttons-container { + > button:nth-child(1) { + color: var(--site-base-fgColor) !important; + border-color: var(--site-inset-borderColor) !important; + } + } + } + } + + .filter-group { + display: flex; + align-items: center; + gap: 0.5rem; + + .filter-label { + font-size: 0.85rem; + font-weight: 600; + color: var(--site-base-fgColor-alt); + } + } + + .bench-chip { + background: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: 16px; + padding: 0.35rem 0.85rem; + font-size: 0.85rem; + font-weight: 500; + color: var(--site-base-fgColor); + cursor: pointer; + transition: all 0.15s ease; + + &:hover { + border-color: var(--blue-5); + color: var(--blue-6); + } + + &.active { + background-color: var(--blue-6); + border-color: var(--blue-6); + color: #ffffff; + } + } + + } + + // Reused by the leaderboard filter bar and the model detail view's + // accuracy/cost/latency switcher, so it isn't scoped to either one. + .bench-segmented-control { + display: inline-flex; + border: 1px solid var(--site-inset-borderColor); + border-radius: 6px; + overflow: hidden; + background-color: var(--site-base-bgColor); + + .segment-btn { + background: none; + border: none; + padding: 0.35rem 0.75rem; + font-size: 0.85rem; + font-weight: 500; + color: var(--site-base-fgColor); + cursor: pointer; + transition: background-color 0.15s ease; + + &:not(:last-child) { + border-right: 1px solid var(--site-inset-borderColor); + } + + &:hover { + background-color: var(--site-inset-bgColor); + } + + &.active { + background-color: var(--blue-6); + color: #ffffff; + } + } + } + + .bench-search-input { + padding: 0.4rem 0.85rem; + border: 1px solid var(--site-inset-borderColor); + border-radius: 6px; + font-size: 0.85rem; + background-color: var(--site-base-bgColor); + color: var(--site-base-fgColor); + outline: none; + width: 180px; + + &:focus { + border-color: var(--blue-5); + box-shadow: 0 0 0 2px rgba(4, 104, 215, 0.2); + } + } + + // --------------------------------------------------------------------------- + // Leaderboard & Heatmap Tables + // --------------------------------------------------------------------------- + .bench-table-wrapper { + overflow-x: auto; + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + background-color: var(--site-base-bgColor); + box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04); + margin-bottom: 2rem; + + table.bench-table, + table.bench-heatmap-table { + width: 100%; + border-collapse: collapse; + text-align: left; + font-size: 0.7rem; + + thead { + background-color: var(--site-inset-bgColor); + border-bottom: 1px solid var(--site-inset-borderColor); + + tr th { + padding: 0.2rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--site-base-fgColor-alt); + white-space: nowrap; + + &.sortable { + cursor: pointer; + user-select: none; + &:hover { + color: var(--blue-6); + } + } + + .sort-indicator { + font-size: 0.7rem; + margin-left: 0.25rem; + &.active { + color: var(--blue-6); + font-weight: 700; + } + &.inactive { + color: var(--site-inset-borderColor); + } + } + } + } + + tbody tr { + border-bottom: 1px solid var(--site-inset-borderColor); + transition: background-color 0.1s ease; + + &:last-child { + border-bottom: none; + } + + &.bench-row { + cursor: pointer; + &:hover { + background-color: var(--site-inset-bgColor); + } + &:focus-visible { + outline: 2px solid var(--blue-5); + outline-offset: -2px; + } + &.selected { + background-color: rgba(4, 104, 215, 0.08); + box-shadow: inset 3px 0 0 var(--blue-6); + } + } + + td { + padding: 0.85rem 1rem; + vertical-align: middle; + } + } + + th.col-model, + td.col-model { + min-width: 220px; + } + + th.col-outcome, + td.col-outcome, + th.col-quality, + td.col-quality, + th.col-dx, + td.col-dx, + th.col-tokens, + td.col-tokens, + th.col-cost, + td.col-cost, + th.col-overall, + td.col-overall { + white-space: nowrap; + font-variant-numeric: tabular-nums; + } + } + } + + // --------------------------------------------------------------------------- + // Model detail profile + // + // Shared by the leaderboard's details drawer and the models explorer, so + // these are scoped to the profile itself rather than to either host. + // The drawer shell lives in `styles/components/_drawer.scss`. + // --------------------------------------------------------------------------- + .model-detail { + display: flex; + flex-direction: column; + gap: $spacing-lg; + } + + .model-detail__actions { + display: flex; + } + + .model-detail__header { + display: flex; + flex-direction: column; + gap: 0.2rem; + padding-bottom: $spacing-md; + border-bottom: 1px solid var(--site-inset-borderColor); + } + + .model-detail__eyebrow { + margin: 0; + font-size: $font-size-xs; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.09em; + color: var(--site-base-fgColor-alt); + } + + .model-detail__title { + margin: 0; + font-size: var(--font-size-heading-4); + font-weight: 700; + line-height: 1.15; + } + + .model-detail__release { + margin: 0; + font-size: $font-size-xs; + font-weight: 500; + letter-spacing: 0.03em; + color: var(--site-base-fgColor-alt); + // Eval keys are long and unbroken, so let them wrap inside the drawer. + overflow-wrap: anywhere; + } + + // Specifications + .model-spec-block { + display: flex; + flex-direction: column; + gap: $spacing-md; + } + + .model-spec-grid { + display: grid; + grid-template-columns: max-content minmax(0, 1fr); + margin: 0; + } + + .model-spec-grid__label, + .model-spec-grid__value { + padding: 0.45rem 0; + border-bottom: 1px solid var(--site-inset-borderColor); + } + + .model-spec-grid__label { + font-size: $font-size-xs; + font-weight: 600; + letter-spacing: 0.06em; + color: var(--site-base-fgColor-alt); + } + + .model-spec-grid__value { + // Indents from the label rather than using a column gap, so the row's + // bottom rule stays unbroken. + padding-left: $spacing-lg; + margin: 0; + font-size: $font-size-sm; + font-weight: 500; + font-variant-numeric: tabular-nums; + overflow-wrap: anywhere; + } + + // The last pair closes the list, so it doesn't need a separator. + .model-spec-grid__label:nth-last-child(2), + .model-spec-grid__value:last-child { + border-bottom: none; + } + + // Headline statistics + .model-stat-cards { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr)); + gap: $spacing-sm; + } + + .model-stat-card { + display: flex; + flex-direction: column; + gap: 0.15rem; + padding: $spacing-md; + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + background-color: var(--site-inset-bgColor); + box-shadow: inset 0 3px 0 var(--model-stat-accent); + + &--accuracy { + --model-stat-accent: var(--blue-6); + } + + &--cost { + --model-stat-accent: var(--bench-warning-fg); + } + + &--latency { + --model-stat-accent: var(--bench-diagnostic-fg); + } + + // Keeps the distribution strips aligned when the cards differ in height. + .distribution { + margin-top: auto; + padding-top: $spacing-sm; + } + } + + .model-stat-card__label { + font-size: $font-size-xs; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--site-base-fgColor-alt); + } + + .model-stat-card__value { + font-size: $font-size-xl; + font-weight: 700; + line-height: 1.1; + font-variant-numeric: tabular-nums; + + // Override the compact sizing the pills use inside the leaderboard table. + .score-pill, + .bench-error-badge { + font-size: inherit; + } + } + + .model-stat-card__detail { + font-size: $font-size-xs; + color: var(--site-base-fgColor-alt); + } + + // A strip of ticks, one per scored configuration, with this one marked. + .distribution { + width: 100%; + min-width: 6rem; + } + + .distribution__track { + position: relative; + height: 1.1rem; + overflow: hidden; + border: 1px solid var(--site-inset-borderColor); + border-radius: 999px; + background-color: var(--site-base-bgColor); + } + + .distribution__tick, + .distribution__marker { + position: absolute; + top: 50%; + transform: translate(-50%, -50%); + border-radius: 1px; + } + + .distribution__tick { + width: 2px; + height: 0.5rem; + background-color: var(--site-base-fgColor-alt); + opacity: 0.3; + } + + .distribution__marker { + width: 3px; + height: 100%; + background-color: var(--model-stat-accent, var(--blue-6)); + } + + // Hyperparameters + .model-hyperparams { + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + background-color: var(--site-base-bgColor); + } + + .model-hyperparams__summary { + display: flex; + align-items: center; + gap: $spacing-xs; + padding: $spacing-sm $spacing-md; + font-size: $font-size-sm; + font-weight: 600; + cursor: pointer; + list-style: none; + + &::-webkit-details-marker { + display: none; + } + + &:focus-visible { + outline: 2px solid var(--blue-5); + outline-offset: -2px; + } + + .material-symbols-rounded { + transition: transform $transition-normal; + } + } + + .model-hyperparams[open] .model-hyperparams__summary + .material-symbols-rounded { + transform: rotate(90deg); + } + + .model-hyperparams__body { + padding: 0 $spacing-md $spacing-md; + } + + // Per-benchmark breakdown + .model-benchmarks { + display: flex; + flex-direction: column; + gap: $spacing-sm; + align-items: flex-start; + + // The table still needs the full width; only the switcher shrinks to fit. + .bench-table-wrapper, + .bench-coming-soon { + align-self: stretch; + } + } + + .model-benchmarks__hint { + margin: 0; + font-size: $font-size-xs; + line-height: 1.5; + color: var(--site-base-fgColor-alt); + } + + .model-benchmarks__table { + th.col-distribution, + td.col-distribution { + width: 40%; + min-width: 7rem; + } + + th.col-value, + td.col-value { + white-space: nowrap; + font-weight: 600; + font-variant-numeric: tabular-nums; + } + + th.col-ranking, + td.col-ranking { + white-space: nowrap; + text-align: right; + } + } + + .benchmark-row__link { + font-weight: 600; + color: var(--site-link-fgColor); + text-decoration: none; + + &:hover { + text-decoration: underline; + } + } + + .benchmark-row__rank { + font-size: $font-size-xs; + font-variant-numeric: tabular-nums; + color: var(--site-base-fgColor-alt); + } + + .benchmark-row__rank-value { + font-size: $font-size-md; + font-weight: 700; + color: var(--site-base-fgColor); + } + + .score-bar { + height: 0.5rem; + min-width: 5rem; + overflow: hidden; + border-radius: 999px; + background-color: var(--site-inset-bgColor); + } + + .score-bar__fill { + height: 100%; + border-radius: inherit; + background-color: var(--blue-6); + + &.score-high { + background-color: var(--bench-success-fg); + } + + &.score-mid { + background-color: var(--bench-warning-fg); + } + + &.score-low { + background-color: var(--bench-danger-fg); + } + } + + // Placeholder for data FlutterBench doesn't collect yet. + .bench-coming-soon { + display: flex; + flex-direction: column; + gap: 0.2rem; + padding: $spacing-md; + border: 1px dashed var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + background-color: var(--site-inset-bgColor); + } + + .bench-coming-soon__label { + font-size: $font-size-xs; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--site-base-fgColor-alt); + } + + .bench-coming-soon__note { + font-size: $font-size-sm; + line-height: 1.5; + color: var(--site-base-fgColor-alt); + } + + // --------------------------------------------------------------------------- + // Models explorer + // --------------------------------------------------------------------------- + .models-explorer { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: $spacing-xl; + align-items: start; + + @include breakpoints.screen(lg) { + grid-template-columns: minmax(15rem, 20rem) minmax(0, 1fr); + } + } + + .models-explorer__sidebar { + display: flex; + flex-direction: column; + gap: $spacing-sm; + min-width: 0; + + // Keeps the list reachable while the much taller detail pane scrolls. + @include breakpoints.screen(lg) { + position: sticky; + top: $spacing-lg; + max-height: calc(100vh - #{$spacing-lg} * 2); + } + } + + .models-explorer__controls { + display: flex; + flex-wrap: wrap; + gap: $spacing-sm; + } + + .models-explorer__field { + display: flex; + flex: 1 1 8rem; + flex-direction: column; + gap: 0.25rem; + + .bench-search-input { + width: 100%; + } + } + + .models-explorer__field-label { + font-size: $font-size-xs; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--site-base-fgColor-alt); + } + + .models-explorer__select { + padding: 0.4rem 0.6rem; + border: 1px solid var(--site-inset-borderColor); + border-radius: 6px; + font-size: 0.85rem; + background-color: var(--site-base-bgColor); + color: var(--site-base-fgColor); + + &:focus-visible { + outline: 2px solid var(--blue-5); + outline-offset: 1px; + } + } + + .models-explorer__list-header { + display: flex; + justify-content: space-between; + padding: $spacing-xs $spacing-sm 0; + font-size: $font-size-xs; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--site-base-fgColor-alt); + } + + .models-explorer__list { + display: flex; + flex-direction: column; + gap: 2px; + margin: 0; + padding: 0; + overflow-y: auto; + list-style: none; + + @include custom-scrollbar; + } + + .models-explorer__item { + display: flex; + align-items: center; + justify-content: space-between; + gap: $spacing-sm; + width: 100%; + padding: $spacing-sm; + border: 1px solid transparent; + border-radius: var(--ui-border-radius-sm); + background: none; + color: inherit; + text-align: left; + cursor: pointer; + transition: background-color 0.15s ease; + + &:hover { + background-color: var(--site-inset-bgColor); + } + + &:focus-visible { + outline: 2px solid var(--blue-5); + outline-offset: -2px; + } + + &.selected { + border-color: var(--blue-6); + background-color: rgba(4, 104, 215, 0.08); + } + } + + .models-explorer__item-main { + display: flex; + flex-direction: column; + gap: 0.1rem; + min-width: 0; + } + + .models-explorer__item-name { + font-size: $font-size-sm; + font-weight: 600; + } + + .models-explorer__item-meta { + font-size: $font-size-xs; + color: var(--site-base-fgColor-alt); + } + + .models-explorer__item-score { + font-weight: 700; + font-variant-numeric: tabular-nums; + } + + .models-explorer__empty { + padding: $spacing-md $spacing-sm; + } + + .models-explorer__detail { + min-width: 0; + } + + .text-success { + color: var(--bench-success-fg); + } + + .text-danger { + color: var(--bench-danger-fg); + } + + .text-muted { + margin: 0; + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + } + + .score-pill { + display: inline-block; + font-weight: 700; + font-size: 0.7rem; + font-variant-numeric: tabular-nums; + } + + // Model info cell + .model-info-cell { + .model-title-row { + display: flex; + align-items: center; + gap: 0.5rem; + margin-bottom: 0.2rem; + + .model-name-text { + font-weight: 700; + color: var(--site-base-fgColor); + } + + .provider-tag { + font-size: 0.7rem; + font-weight: 600; + padding: 0.1rem 0.4rem; + background-color: var(--grey-2); + color: var(--grey-6); + border-radius: 4px; + } + + .tooling-badge { + font-size: 0.7rem; + font-weight: 600; + padding: 0.1rem 0.45rem; + background-color: var(--bench-success-bg); + color: var(--bench-success-fg); + border: 1px solid var(--bench-success-border); + border-radius: 4px; + } + } + + .agent-subtext { + font-size: 0.78rem; + color: var(--site-base-fgColor-alt); + } + } + + // Rank Pill + .rank-pill { + font-weight: 700; + font-size: 0.85rem; + color: var(--site-base-fgColor-alt); + } + + // Reward Main Score + .reward-cell { + display: flex; + align-items: baseline; + gap: 0.25rem; + + .reward-main-score { + font-weight: 700; + font-size: 0.7rem; + } + + .reward-range { + font-size: 0.8rem; + color: var(--site-base-fgColor-alt); + } + } + + // Semantic Score Classes + .score-high { + color: var(--bench-success-fg) !important; + } + .score-mid { + color: var(--bench-warning-fg) !important; + } + .score-low { + color: var(--bench-danger-fg) !important; + } + .score-error { + color: var(--bench-error-fg) !important; + } + + // --------------------------------------------------------------------------- + // ErrorStateBadge + // --------------------------------------------------------------------------- + .bench-error-badge { + display: inline-flex; + align-items: center; + gap: 0.35rem; + padding: 0.25rem 0.6rem; + border-radius: 4px; + background-color: var(--bench-error-bg); + border: 1px dashed var(--bench-error-border); + color: var(--bench-error-fg); + font-size: 0.8rem; + font-weight: 600; + line-height: 1.2; + + // Distinct hatched background for error status + background-image: repeating-linear-gradient( + -45deg, + rgba(217, 48, 37, 0.05), + rgba(217, 48, 37, 0.05) 6px, + transparent 6px, + transparent 12px + ); + + .bench-error-icon { + font-size: 0.85rem; + } + + &.compact { + padding: 0.15rem 0.4rem; + font-size: 0.75rem; + } + } + + // --------------------------------------------------------------------------- + // Heatmap Specifics + // --------------------------------------------------------------------------- + .bench-heatmap-table { + .col-model-header { + text-align: center; + + .model-header-content { + display: flex; + flex-direction: column; + align-items: center; + gap: 0.15rem; + + .model-name-title { + font-weight: 700; + color: var(--site-base-fgColor); + } + + .agent-tag { + font-size: 0.7rem; + color: var(--site-base-fgColor-alt); + text-transform: none; + } + + .tooling-icon-badge { + font-size: 0.68rem; + color: var(--bench-success-fg); + font-weight: 600; + display: inline-flex; + align-items: center; + gap: 0.2rem; + + .tooling-dot { + width: 6px; + height: 6px; + border-radius: 50%; + background-color: var(--bench-success-fg); + } + } + } + } + + .heatmap-cell { + text-align: center; + padding: 0 !important; + + .cell-link { + display: block; + padding: 1rem 0.75rem; + text-decoration: none; + color: inherit; + font-weight: 700; + transition: transform 0.1s ease, box-shadow 0.1s ease; + + &:hover { + transform: scale(1.03); + box-shadow: inset 0 0 0 2px var(--blue-5); + } + } + + &.cell-scored { + &.score-high { + background-color: var(--bench-success-bg); + .cell-score-value { + color: var(--bench-success-fg); + } + } + &.score-mid { + background-color: var(--bench-warning-bg); + .cell-score-value { + color: var(--bench-warning-fg); + } + } + &.score-low { + background-color: var(--bench-danger-bg); + .cell-score-value { + color: var(--bench-danger-fg); + } + } + } + + &.cell-error { + background-color: var(--bench-error-bg); + background-image: repeating-linear-gradient( + -45deg, + rgba(217, 48, 37, 0.08), + rgba(217, 48, 37, 0.08) 8px, + transparent 8px, + transparent 16px + ); + border: 1px dashed var(--bench-error-border); + + .cell-link { + padding: 0.85rem 0.5rem; + } + } + } + } + + // Heatmap Best & Worst CUJs summaries + .bench-cuj-model-summaries { + margin-top: 2rem; + + .summaries-title { + font-size: 1.25rem; + font-weight: 700; + margin-bottom: 1rem; + } + + .summaries-grid { + display: grid; + grid-template-columns: 1fr; + gap: 1.25rem; + + @include breakpoints.screen(md) { + grid-template-columns: repeat(2, 1fr); + } + } + + .model-summary-card { + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + padding: 1.25rem; + + .card-header { + margin-bottom: 0.75rem; + border-bottom: 1px solid var(--site-inset-borderColor); + padding-bottom: 0.5rem; + + .model-title { + font-size: 1.1rem; + font-weight: 700; + margin-bottom: 0.2rem; + } + .agent-subtitle { + font-size: 0.8rem; + color: var(--site-base-fgColor-alt); + } + } + + .cuj-list-group { + margin-bottom: 0.75rem; + + .group-label { + font-size: 0.8rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.04em; + display: block; + margin-bottom: 0.35rem; + } + + .cuj-bullet-list { + list-style: none; + padding: 0; + margin: 0; + + li { + font-size: 0.85rem; + margin-bottom: 0.25rem; + a { + color: var(--site-link-fgColor); + text-decoration: none; + &:hover { + text-decoration: underline; + } + } + } + } + } + } + } + + // --------------------------------------------------------------------------- + // Trial Detail Page Components + // --------------------------------------------------------------------------- + .bench-card { + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + padding: 1.75rem; + box-shadow: 0 1px 4px rgba(0, 0, 0, 0.04); + margin-bottom: 2rem; + + .section-heading { + font-size: 1.35rem; + font-weight: 700; + color: var(--site-base-fgColor); + margin-bottom: 0.5rem; + } + + .section-intro-text { + font-size: 0.95rem; + color: var(--site-base-fgColor-alt); + margin-bottom: 1.25rem; + } + } + + .trial-summary-card { + .summary-top-row { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + justify-content: space-between; + gap: 1.5rem; + + .sub-tag { + font-size: 0.75rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--blue-6); + display: block; + margin-bottom: 0.25rem; + } + + .trial-title { + font-size: 1.75rem; + font-weight: 700; + line-height: 1.2; + margin-bottom: 0.5rem; + word-break: break-word; + } + + .task-link-row { + font-size: 0.95rem; + color: var(--site-base-fgColor-alt); + + .task-anchor { + color: var(--site-link-fgColor); + font-weight: 600; + text-decoration: none; + &:hover { + text-decoration: underline; + } + } + } + + .status-badge-container { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 0.5rem; + + .trial-status-badge { + font-size: 0.85rem; + font-weight: 700; + letter-spacing: 0.05em; + padding: 0.35rem 0.85rem; + border-radius: 20px; + + &.status-pass { + background-color: var(--bench-success-bg); + color: var(--bench-success-fg); + border: 1px solid var(--bench-success-border); + } + &.status-partial { + background-color: var(--bench-warning-bg); + color: var(--bench-warning-fg); + border: 1px solid var(--bench-warning-border); + } + &.status-fail { + background-color: var(--bench-danger-bg); + color: var(--bench-danger-fg); + border: 1px solid var(--bench-danger-border); + } + } + + .hero-score-badge { + display: flex; + align-items: baseline; + gap: 0.15rem; + + .score-num { + font-size: 2.25rem; + font-weight: 800; + color: var(--site-base-fgColor); + line-height: 1; + } + .score-pct { + font-size: 1.15rem; + font-weight: 700; + color: var(--site-base-fgColor-alt); + } + .score-caption { + font-size: 0.75rem; + color: var(--site-base-fgColor-alt); + margin-left: 0.5rem; + } + } + } + } + + .meta-divider { + height: 1px; + background-color: var(--site-inset-borderColor); + margin: 1.5rem 0; + } + + .trial-meta-grid { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 1.25rem; + + @include breakpoints.screen(md) { + grid-template-columns: repeat(4, 1fr); + } + + .meta-col { + display: flex; + flex-direction: column; + gap: 0.2rem; + + .meta-label { + font-size: 0.75rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--site-base-fgColor-alt); + } + + .meta-val { + font-size: 1.05rem; + color: var(--site-base-fgColor); + + &.bold { + font-weight: 700; + } + } + + .meta-sub { + font-size: 0.8rem; + color: var(--site-base-fgColor-alt); + } + + .tooling-pill { + font-size: 0.72rem; + font-weight: 600; + padding: 0.1rem 0.45rem; + border-radius: 4px; + background-color: var(--bench-success-bg); + color: var(--bench-success-fg); + align-self: flex-start; + + &.uninstrumented { + background-color: var(--grey-2); + color: var(--grey-6); + } + } + } + } + + // Phase Timeline + .phase-durations-section { + .phase-title { + font-size: 0.8rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--site-base-fgColor-alt); + display: block; + margin-bottom: 0.75rem; + } + + .phase-bars-row { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 0.75rem; + + @include breakpoints.screen(md) { + grid-template-columns: repeat(4, 1fr); + } + + .phase-pill { + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: 6px; + padding: 0.6rem 0.85rem; + display: flex; + justify-content: space-between; + align-items: center; + + .phase-label { + font-size: 0.8rem; + color: var(--site-base-fgColor); + } + .phase-time { + font-size: 0.85rem; + font-weight: 700; + font-family: var(--font-code); + color: var(--blue-6); + } + } + } + } + } + + // --------------------------------------------------------------------------- + // Reward Breakdown Accordion & LLM Reasoning + // --------------------------------------------------------------------------- + .reward-breakdown-section { + .section-header-row { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + align-items: center; + gap: 0.75rem; + margin-bottom: 0.5rem; + + .rubric-formula-badge { + font-size: 0.78rem; + font-family: var(--font-code); + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + padding: 0.25rem 0.6rem; + border-radius: 4px; + color: var(--site-base-fgColor-alt); + } + } + + .criteria-tree { + display: flex; + flex-direction: column; + gap: 0.75rem; + margin-bottom: 2rem; + } + + .criterion-accordion { + border: 1px solid var(--site-inset-borderColor); + border-radius: 6px; + background-color: var(--site-base-bgColor); + overflow: hidden; + + .criterion-summary { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0.85rem 1.25rem; + background-color: var(--site-inset-bgColor); + cursor: pointer; + user-select: none; + + .summary-left { + display: flex; + align-items: center; + gap: 0.75rem; + + .criterion-name { + font-weight: 700; + font-size: 0.95rem; + color: var(--site-base-fgColor); + } + + .weight-pill { + font-size: 0.75rem; + font-weight: 600; + padding: 0.15rem 0.45rem; + border-radius: 4px; + background-color: var(--grey-3); + color: var(--grey-6); + } + } + } + + .criterion-body { + padding: 1.25rem; + border-top: 1px solid var(--site-inset-borderColor); + + .description-box { + font-size: 0.85rem; + font-family: var(--font-code); + background-color: var(--site-inset-bgColor); + padding: 0.85rem; + border-radius: 4px; + margin: 0; + white-space: pre-wrap; + } + } + } + + .sub-criteria-list { + display: flex; + flex-direction: column; + gap: 1rem; + + .sub-criterion-row { + border-bottom: 1px solid var(--site-inset-borderColor); + padding-bottom: 0.85rem; + + &:last-child { + border-bottom: none; + padding-bottom: 0; + } + + .sub-header-line { + display: flex; + align-items: center; + gap: 0.75rem; + margin-bottom: 0.35rem; + + .sub-name { + font-weight: 600; + font-size: 0.9rem; + font-family: var(--font-code); + color: var(--site-base-fgColor); + } + + .sub-weight { + font-size: 0.75rem; + color: var(--site-base-fgColor-alt); + } + + .sub-score-badge { + font-weight: 700; + font-size: 0.85rem; + margin-left: auto; + } + } + + .sub-description { + font-size: 0.85rem; + color: var(--site-base-fgColor-alt); + margin-bottom: 0.5rem; + } + + .llm-reasoning-card { + background-color: var(--bench-diagnostic-bg); + border: 1px solid var(--bench-diagnostic-border); + border-radius: 6px; + padding: 0.85rem; + margin-top: 0.5rem; + + .reasoning-badge { + font-size: 0.7rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--bench-diagnostic-fg); + margin-bottom: 0.35rem; + display: inline-block; + } + + .reasoning-text { + font-size: 0.85rem; + font-style: italic; + color: var(--site-base-fgColor); + margin: 0; + line-height: 1.5; + } + } + } + } + + // Diagnostic section + .diagnostic-breakdown-panel { + border: 1px solid var(--bench-diagnostic-border); + background-color: var(--bench-diagnostic-bg); + border-radius: var(--ui-border-radius-sm); + padding: 1.25rem; + + .diagnostic-banner { + display: flex; + gap: 0.75rem; + margin-bottom: 1.25rem; + + .diag-icon { + font-size: 1.25rem; + color: var(--bench-diagnostic-fg); + } + + .diag-banner-text { + h3 { + font-size: 1.05rem; + font-weight: 700; + color: var(--bench-diagnostic-fg); + margin-bottom: 0.25rem; + } + p { + font-size: 0.85rem; + color: var(--site-base-fgColor-alt); + margin: 0; + line-height: 1.4; + } + } + } + + .diagnostic-grids { + display: grid; + grid-template-columns: 1fr; + gap: 1rem; + + @include breakpoints.screen(md) { + grid-template-columns: repeat(2, 1fr); + } + + .diagnostic-card { + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: 6px; + padding: 1rem; + + .diag-card-header { + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid var(--site-inset-borderColor); + padding-bottom: 0.4rem; + margin-bottom: 0.75rem; + + .diag-title { + font-size: 0.85rem; + font-weight: 700; + letter-spacing: 0.04em; + color: var(--site-base-fgColor); + } + .diag-score { + font-weight: 700; + font-size: 0.9rem; + color: var(--bench-diagnostic-fg); + } + } + + .diag-criteria-list { + list-style: none; + padding: 0; + margin: 0; + + li { + margin-bottom: 0.5rem; + + .diag-item-row { + display: flex; + justify-content: space-between; + font-size: 0.82rem; + font-weight: 600; + + .diag-item-name { + font-family: var(--font-code); + } + } + + .diag-item-desc { + font-size: 0.78rem; + color: var(--site-base-fgColor-alt); + margin: 0.15rem 0 0 0; + } + } + } + } + } + } + } + + // --------------------------------------------------------------------------- + // Trajectory Timeline + // --------------------------------------------------------------------------- + .trajectory-section { + .trajectory-timeline { + list-style: none; + padding: 0; + margin: 0; + position: relative; + + &::before { + content: ''; + position: absolute; + top: 15px; + bottom: 15px; + left: 15px; + width: 2px; + background-color: var(--site-inset-borderColor); + } + + .trajectory-step-item { + display: flex; + gap: 1.25rem; + margin-bottom: 1.25rem; + position: relative; + + .step-marker { + width: 32px; + height: 32px; + border-radius: 50%; + background-color: var(--blue-6); + color: #ffffff; + display: flex; + align-items: center; + justify-content: center; + font-weight: 700; + font-size: 0.85rem; + z-index: 1; + flex-shrink: 0; + } + + .step-content { + flex: 1; + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: 6px; + padding: 0.85rem 1rem; + + .step-top-line { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 0.35rem; + + .action-badge { + font-size: 0.75rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.04em; + padding: 0.15rem 0.5rem; + border-radius: 4px; + background-color: var(--blue-1); + color: var(--blue-7); + } + + .duration-pill { + font-size: 0.75rem; + font-family: var(--font-code); + color: var(--site-base-fgColor-alt); + } + } + + .step-input-code { + font-size: 0.82rem; + font-family: var(--font-code); + display: block; + color: var(--site-base-fgColor); + } + } + } + } + } + + // --------------------------------------------------------------------------- + // Artifacts Code Viewer + // --------------------------------------------------------------------------- + .artifacts-section { + .artifacts-list { + display: flex; + flex-direction: column; + gap: 1rem; + } + + .artifact-card { + border: 1px solid var(--site-inset-borderColor); + border-radius: 6px; + overflow: hidden; + + .artifact-header { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.75rem 1rem; + background-color: var(--site-inset-bgColor); + cursor: pointer; + user-select: none; + + .file-path-icon { + font-size: 1rem; + } + + .artifact-path { + font-family: var(--font-code); + font-weight: 600; + font-size: 0.85rem; + color: var(--site-base-fgColor); + flex: 1; + } + + .artifact-status-pill { + font-size: 0.7rem; + font-weight: 600; + padding: 0.1rem 0.4rem; + border-radius: 4px; + background-color: var(--bench-success-bg); + color: var(--bench-success-fg); + } + } + + .artifact-content { + border-top: 1px solid var(--site-inset-borderColor); + + .code-viewer { + margin: 0; + padding: 1rem; + background-color: var(--site-base-bgColor); + font-family: var(--font-code); + font-size: 0.82rem; + line-height: 1.5; + overflow-x: auto; + } + } + } + } + + // --------------------------------------------------------------------------- + // Raw Logs Section + // --------------------------------------------------------------------------- + .logs-section { + .log-details { + border: 1px solid var(--site-inset-borderColor); + border-radius: 6px; + margin-bottom: 0.75rem; + overflow: hidden; + + &.error-log { + border-color: var(--bench-danger-border); + } + + .log-summary { + padding: 0.75rem 1rem; + background-color: var(--site-inset-bgColor); + cursor: pointer; + font-size: 0.85rem; + font-weight: 600; + } + + .raw-log-pre { + margin: 0; + padding: 1rem; + background-color: var(--site-base-bgColor); + font-family: var(--font-code); + font-size: 0.8rem; + overflow-x: auto; + max-height: 400px; + white-space: pre-wrap; + } + } + } + + // --------------------------------------------------------------------------- + // Bottom Nav + // --------------------------------------------------------------------------- + .bench-bottom-nav { + display: flex; + gap: 1rem; + margin-top: 2rem; + } + + .bench-btn-outline { + display: inline-block; + padding: 0.35rem 0.75rem; + border: 1px solid var(--site-inset-borderColor); + border-radius: 4px; + font-size: 0.8rem; + font-weight: 600; + color: var(--site-link-fgColor); + text-decoration: none; + transition: all 0.15s ease; + + &:hover { + background-color: var(--site-inset-bgColor); + border-color: var(--blue-5); + } + } + + .bench-btn-sm { + font-size: 0.8rem; + font-weight: 600; + color: var(--site-link-fgColor); + text-decoration: none; + &:hover { + text-decoration: underline; + } + } + + // Task Cards Directory in Tasks Page + .tasks-cards-grid { + display: grid; + grid-template-columns: 1fr; + gap: 1.25rem; + + @include breakpoints.screen(md) { + grid-template-columns: repeat(2, 1fr); + } + + @include breakpoints.screen(lg) { + grid-template-columns: repeat(3, 1fr); + } + + .task-card { + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + padding: 1.25rem; + display: flex; + flex-direction: column; + justify-content: space-between; + + .task-card-header { + margin-bottom: 0.5rem; + + .category-pill { + font-size: 0.7rem; + font-weight: 600; + padding: 0.15rem 0.45rem; + border-radius: 4px; + background-color: var(--grey-2); + color: var(--grey-6); + display: inline-block; + margin-bottom: 0.35rem; + } + + .task-card-title { + font-size: 1.15rem; + font-weight: 700; + margin: 0; + + a { + color: var(--site-base-fgColor); + text-decoration: none; + &:hover { + color: var(--blue-6); + } + } + } + } + + .task-card-description { + font-size: 0.85rem; + color: var(--site-base-fgColor-alt); + line-height: 1.5; + margin-bottom: 1rem; + flex: 1; + } + + .task-card-footer { + display: flex; + justify-content: space-between; + align-items: center; + border-top: 1px solid var(--site-inset-borderColor); + padding-top: 0.75rem; + + .trial-count-sub { + font-size: 0.75rem; + color: var(--site-base-fgColor-alt); + } + } + } + } + + // --------------------------------------------------------------------------- + // CUJ Catalog (CUJs page) + // --------------------------------------------------------------------------- + .cujs-content { + .cuj-result-count { + font-size: 0.8rem; + color: var(--site-base-fgColor-alt); + white-space: nowrap; + } + } + + .empty-table-message { + display: block; + padding: 2rem 1rem; + text-align: center; + color: var(--site-base-fgColor-alt); + } + + .persona-tag { + display: inline-block; + font-size: $font-size-xs; + font-weight: 600; + padding: $spacing-xs $spacing-sm; + border-radius: var(--site-radius, $spacing-xs); + white-space: nowrap; + + &.color-blue { + background: rgba($color-functional, 0.12); + color: $color-functional; + } + + &.color-purple { + background: rgba($color-llm, 0.12); + color: $color-llm; + } + + &.color-teal { + background: rgba(#0d9488, 0.12); + color: #0d9488; + } + + &.color-magenta { + background: rgba(#db2777, 0.12); + color: #db2777; + } + + &.color-amber { + background: rgba($color-dx, 0.12); + color: $color-dx; + } + + &.color-grey { + background: rgba($color-grey, 0.12); + color: $color-grey; + } + } + + .cuj-card-list { + display: flex; + flex-direction: column; + gap: 0.75rem; + } + + .cuj-card { + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + overflow: hidden; + + .cuj-card-header { + width: 100%; + display: flex; + align-items: center; + gap: 1rem; + padding: 1rem 1.25rem; + background: none; + border: none; + cursor: pointer; + text-align: left; + font: inherit; + color: inherit; + + &:hover { + background-color: var(--site-inset-bgColor); + } + + .cuj-goal { + flex: 1; + margin: 0; + font-size: 1rem; + font-weight: 600; + } + + .cuj-task-count { + font-size: 0.8rem; + color: var(--site-base-fgColor-alt); + white-space: nowrap; + } + } + + .cuj-task-list { + margin: 0; + padding: 0 1.25rem 1.25rem 3rem; + display: flex; + flex-direction: column; + gap: 0.5rem; + color: var(--site-base-fgColor-alt); + font-size: 0.9rem; + line-height: 1.5; + } + + &.expanded .cuj-card-header { + border-bottom: 1px solid var(--site-inset-borderColor); + } + } + + // Methodology callout in Leaderboard + .methodology-callout { + .callout-card { + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--ui-border-radius-sm); + padding: 1.5rem 1.75rem; + display: flex; + flex-direction: column; + gap: 1rem; + + @include breakpoints.screen(md) { + flex-direction: row; + align-items: center; + justify-content: space-between; + } + + .callout-text { + h3 { + font-size: 1.25rem; + font-weight: 700; + margin-bottom: 0.35rem; + } + p { + font-size: 0.9rem; + color: var(--site-base-fgColor-alt); + margin: 0; + line-height: 1.5; + } + } + + .callout-action { + flex-shrink: 0; + } + } + } + + // --------------------------------------------------------------------------- + // Methodology Page Styles & Components + // --------------------------------------------------------------------------- + + .methodology-page { + .bench-hero-header { + margin-bottom: 0; + } + } + + .methodology-content { + .methodology-lead { + font-size: 1.15rem; + color: var(--site-base-fgColor-alt); + line-height: 1.6; + margin-bottom: 1.5rem; + } + + .methodology-note-box { + background-color: var(--site-inset-bgColor); + border-left: 4px solid var(--site-primary-color, #0284c7); + border-radius: var(--site-radius, 4px); + padding: 1rem 1.25rem; + margin-block: 1.25rem; + + p { + margin: 0; + font-size: $font-size-sm; + color: var(--site-base-fgColor); + line-height: 1.6; + } + + strong { + color: var(--site-primary-color, #0284c7); + } + } + + .scoring-formula-card { + background: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius, 8px); + padding: 1.5rem; + margin-block: 1.25rem 1.5rem; + text-align: center; + + .formula-math { + font-family: var(--site-code-fontFamily, monospace); + font-size: 1.35rem; + font-weight: 700; + color: var(--site-base-fgColor); + margin-bottom: 0.75rem; + letter-spacing: 0.02em; + } + + .formula-explainer { + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + max-width: 750px; + margin: 0 auto; + line-height: 1.6; + } + } + + .composite-reward-bar { + display: flex; + border-radius: var(--site-radius, 8px); + overflow: hidden; + margin-block: 1.25rem 1.75rem; + height: 3rem; + box-shadow: $shadow-sm; + + .reward-weight { + display: flex; + align-items: center; + justify-content: center; + gap: 0.5rem; + color: #ffffff; + font-weight: 600; + font-size: $font-size-sm; + transition: flex $transition-normal; + + .weight-val { + font-weight: 800; + font-size: 1rem; + } + + .weight-name { + opacity: 0.95; + } + + &.outcome-weight { + flex: 60; + background-color: $color-outcome; + } + + &.quality-weight { + flex: 30; + background-color: $color-quality; + } + + &.dx-weight { + flex: 10; + background-color: $color-dx; + } + } + } + + .code-block-wrapper { + pre { + background-color: #0f172a; + color: #e2e8f0; + padding: 1.25rem 1.5rem; + border-radius: var(--site-radius, 8px); + font-family: var(--site-code-fontFamily, monospace); + font-size: 0.9rem; + line-height: 1.6; + overflow-x: auto; + margin-block: 1rem 1.5rem; + + code { + background: transparent; + padding: 0; + color: inherit; + } + } + } + + .bench-section { + margin-bottom: 3.5rem; + + .section-title-row { + margin-bottom: 1.25rem; + border-bottom: 1px solid var(--site-inset-borderColor); + padding-bottom: 0.5rem; + + .section-h2 { + font-size: 1.85rem; + font-weight: 700; + margin: 0; + color: var(--site-base-fgColor); + } + } + + h3 { + font-size: 1.3rem; + font-weight: 600; + margin-top: 2rem; + margin-bottom: 0.75rem; + color: var(--site-base-fgColor); + } + + p { + font-size: 1rem; + color: var(--site-base-fgColor-alt); + line-height: 1.65; + margin-bottom: 1rem; + + code { + font-size: 0.875em; + background-color: var(--site-raised-bgColor); + padding: 0.2rem 0.4rem; + border-radius: var(--site-radius, 4px); + font-family: var(--site-code-fontFamily, monospace); + } + + strong { + color: var(--site-base-fgColor); + } + } + + ul { + margin: 0 0 1.25rem 1.5rem; + padding: 0; + + li { + font-size: 1rem; + color: var(--site-base-fgColor-alt); + line-height: 1.6; + margin-bottom: 0.5rem; + + strong { + color: var(--site-base-fgColor); + } + + code { + font-size: 0.875em; + background-color: var(--site-raised-bgColor); + padding: 0.2rem 0.4rem; + border-radius: var(--site-radius, 4px); + } + } + } + } + + .table-wrapper:not(.task-specs-list *) { + @include bench-element-width; + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + margin-block: $spacing-md $spacing-lg; + overflow-x: auto; + box-shadow: $shadow-sm; + + table { + width: 100%; + border-collapse: collapse; + border-spacing: 0; + margin: 0; + + thead { + background-color: var(--site-raised-bgColor); + border-bottom: 1px solid var(--site-inset-borderColor); + + tr { + th { + padding: $spacing-sm $spacing-md; + font-family: var(--site-ui-fontFamily); + font-size: $font-size-xs; + font-weight: 700; + text-transform: uppercase; + color: var(--site-base-fgColor); + border: none; + text-align: left; + + &:first-child { + padding-left: $spacing-md; + } + + &:last-child { + padding-right: $spacing-md; + } + } + } + } + + tbody { + tr { + background-color: var(--site-base-bgColor); + border-top: 1px solid var(--site-inset-borderColor); + transition: background-color $transition-normal; + + &:first-child { + border-top: none; + } + + &:hover { + background-color: var(--site-inset-bgColor); + } + + td { + padding: $spacing-sm $spacing-md; + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + border: none; + vertical-align: top; + + strong { + font-family: var(--site-ui-fontFamily); + font-size: $font-size-sm; + font-weight: 600; + color: var(--site-base-fgColor); + } + + code { + font-size: 0.85em; + background-color: var(--site-raised-bgColor); + padding: $spacing-xs $spacing-sm; + border-radius: var(--site-radius); + } + + &:first-child { + padding-left: $spacing-md; + font-weight: 600; + color: var(--site-base-fgColor); + white-space: nowrap; + } + + &:last-child { + padding-right: $spacing-md; + } + } + } + } + } + } + } + + // Three Core Dimensions Grid + .dimension-cards-grid { + @include bench-element-width; + display: grid; + grid-template-columns: 1fr; + gap: $spacing-md; + margin-block: $spacing-lg $spacing-xl; + + @include breakpoints.screen(md) { + grid-template-columns: repeat(3, 1fr); + } + + .dimension-card { + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + padding: $spacing-md; + display: flex; + flex-direction: column; + + .card-header-row { + display: flex; + align-items: center; + + .card-icon-wrap { + @include category-icon-background; + margin: 0 $spacing-md $spacing-md 0; + } + + h4 { + flex: 2; + font-size: $font-size-md; + font-weight: 600; + margin: 0 0 $spacing-sm 0; + color: var(--site-base-fgColor); + } + } + + p { + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + margin: 0; + flex-grow: 1; + line-height: 1.5; + + code { + font-size: 0.85em; + background-color: var(--site-raised-bgColor); + padding: $spacing-xs $spacing-sm; + border-radius: var(--site-radius); + } + } + + .card-footer-info { + margin-top: $spacing-md; + padding-top: $spacing-sm; + border-top: 1px solid var(--site-inset-borderColor); + display: flex; + align-items: center; + justify-content: space-between; + font-size: $font-size-xs; + color: var(--site-base-fgColor-lighter); + + .badge { + @include category-badge; + } + } + } + } + + // Grader Matrix Carousel + .grader-matrix { + @include bench-element-width; + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + padding: $spacing-md; + margin-block: $spacing-md; + position: relative; + + .matrix-header { + display: flex; + flex-direction: column; + gap: $spacing-md; + margin-bottom: $spacing-md; + + @include breakpoints.screen(md) { + flex-direction: row; + align-items: center; + justify-content: space-between; + } + + .matrix-title-area { + h3 { + margin: 0 0 $spacing-xs 0; + font-size: $font-size-lg; + font-weight: 600; + color: var(--site-base-fgColor); + } + + p { + margin: 0; + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + } + } + + .matrix-filters { + display: flex; + flex-wrap: wrap; + gap: $spacing-xs; + background-color: var(--site-raised-bgColor); + padding: $spacing-xs; + border-radius: $spacing-sm; + + .filter-btn { + background: transparent; + border: none; + border-radius: var(--site-radius); + padding: $spacing-xs $spacing-sm; + font-size: $font-size-xs; + font-weight: 500; + color: var(--site-base-fgColor-alt); + cursor: pointer; + transition: $transition-normal; + + &:hover { + color: var(--site-base-fgColor); + } + + &.active { + background-color: var(--site-base-bgColor); + color: var(--site-primary-color); + font-weight: 600; + box-shadow: $shadow-sm; + } + } + } + } + + .grader-carousel-wrapper { + position: relative; + width: 100%; + + .carousel-nav-btn { + position: absolute; + top: 50%; + transform: translateY(-50%); + display: inline-flex; + align-items: center; + justify-content: center; + width: 2rem; + height: 2rem; + border-radius: 50%; + border: 1px solid var(--site-inset-borderColor); + background-color: var(--site-base-bgColor); + color: var(--site-base-fgColor); + cursor: pointer; + transition: $transition-normal; + box-shadow: $shadow-sm; + z-index: var(--site-z-floating, 10); + + &.prev { + left: -$spacing-md; + } + + &.next { + right: -$spacing-md; + } + + &:hover:not(:disabled) { + background-color: var(--site-raised-bgColor); + border-color: var(--site-primary-color); + color: var(--site-primary-color); + transform: translateY(-50%) scale(1.05); + } + + &:active:not(:disabled) { + transform: translateY(-50%) scale(0.95); + } + + &:disabled { + opacity: 0.35; + cursor: not-allowed; + } + + .material-symbols { + font-size: $font-size-xl; + } + } + + .grader-cards-track { + @include custom-scrollbar('horizontal'); + width: 100%; + display: flex; + flex-direction: row; + flex-wrap: nowrap; + gap: $spacing-md; + overflow-x: auto; + scroll-behavior: smooth; + scroll-snap-type: x mandatory; + padding: $spacing-sm; + background-color: var(--site-base-bgColor); + border-radius: var(--site-radius); + border-left: $spacing-sm solid var(--site-base-bgColor); + border-right: $spacing-sm solid var(--site-base-bgColor); + + .grader-card { + flex: 0 0 $card-width; + max-width: $card-width; + min-width: 260px; + scroll-snap-align: start; + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + padding: $spacing-md; + display: flex; + flex-direction: column; + gap: $spacing-sm; + transition: border-color $transition-normal; + + &:hover { + border-color: var(--site-primary-color); + } + + &.hidden { + display: none !important; + } + + .grader-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: $spacing-sm; + + .grader-cat { + font-size: $font-size-xs; + font-weight: 700; + text-transform: uppercase; + + &.outcome { + color: $color-outcome; + } + + &.quality { + color: $color-quality; + } + + &.dx { + color: $color-dx; + } + } + + .grader-badge { + @include category-badge(true); + border-radius: 9999px; + } + } + + h4 { + font-size: $font-size-md; + font-weight: 600; + margin: 0; + color: var(--site-base-fgColor); + } + + p { + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + margin: 0; + flex-grow: 1; + + code { + font-size: 0.9em; + background-color: rgba(0, 0, 0, 0.05); + padding: $spacing-xs; + border-radius: var(--site-radius); + } + } + } + } + } + } + + // Multi-Run Reliability + .reliability-comparison-grid { + @include bench-element-width; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr)); + gap: $spacing-md; + margin-block: $spacing-lg $spacing-xl; + + .reliability-card { + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + padding: $spacing-lg; + display: flex; + flex-direction: column; + gap: $spacing-sm; + + .reliability-header { + display: flex; + align-items: center; + justify-content: space-between; + + .tag { + font-size: $font-size-xs; + font-weight: 700; + text-transform: uppercase; + color: var(--site-primary-color); + } + + .math-pill { + font-family: var(--site-code-fontFamily, monospace); + font-size: $font-size-sm; + font-weight: 700; + background-color: rgba(4, 104, 215, 0.1); + color: var(--site-primary-color); + padding: $spacing-xs $spacing-sm; + border-radius: var(--site-radius); + } + } + + h4 { + font-size: $font-size-lg; + font-weight: 600; + margin: 0; + color: var(--site-base-fgColor); + } + + p { + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + margin: 0; + + strong, + em { + color: var(--site-base-fgColor); + } + } + + &.north-star { + background: $north-star-bg-light; + border: 1px solid rgba(59, 130, 246, 0.4); + color: $color-white; + box-shadow: $shadow-md; + + .reliability-header { + .tag { + color: $color-gold; + display: flex; + align-items: center; + gap: $spacing-xs; + + .material-symbols { + font-size: $font-size-md; + } + } + + .math-pill { + background-color: rgba(59, 130, 246, 0.25); + border: 1px solid rgba(147, 197, 253, 0.4); + color: #bfdbfe; + } + } + + h4 { + color: $color-white; + } + + p { + color: #e2e8f0; + + strong, + em { + color: $color-white; + } + } + } + } + } + + // Interactive Detail Card & Score Triage + .interactive-detail-card, + .score-triage { + @include bench-element-width; + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + padding: $spacing-lg; + margin-block: $spacing-lg $spacing-xl; + + .card-header-area, + .triage-header { + margin-bottom: $spacing-md; + + h3 { + font-size: $font-size-lg; + font-weight: 600; + margin: 0 0 $spacing-xs 0; + color: var(--site-base-fgColor); + } + + p { + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + margin: 0; + } + } + + .card-tabs-grid, + .triage-tiers-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(8.5rem, 1fr)); + gap: $spacing-sm; + margin-bottom: $spacing-md; + + .card-tab-btn, + .triage-tier-btn { + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + padding: $spacing-sm $spacing-md; + text-align: left; + cursor: pointer; + transition: $transition-normal; + opacity: 0.75; + + &:hover { + opacity: 1; + } + + .tab-primary-label, + .tier-score { + font-size: $font-size-sm; + font-weight: 800; + } + + .tab-secondary-label, + .tier-name { + font-size: $font-size-xs; + font-weight: 600; + margin-top: $spacing-xs; + color: var(--site-base-fgColor); + } + + // Color variants for tabs + &.variant-perfect, + &.tier-perfect { + .tab-primary-label, + .tier-score { + color: $color-outcome; + } + + &.active { + border: $border-width solid $color-outcome; + background-color: rgba($color-outcome, 0.08); + opacity: 1; + } + } + + &.variant-functional, + &.variant-blue, + &.tier-functional { + .tab-primary-label, + .tier-score { + color: $color-functional; + } + + &.active { + border: $border-width solid $color-functional; + background-color: rgba($color-functional, 0.08); + opacity: 1; + } + } + + &.variant-quality, + &.variant-purple { + .tab-primary-label, + .tier-score { + color: $color-quality; + } + + &.active { + border: $border-width solid $color-quality; + background-color: rgba($color-quality, 0.08); + opacity: 1; + } + } + + &.variant-partial, + &.variant-dx, + &.variant-amber, + &.tier-partial { + .tab-primary-label, + .tier-score { + color: $color-dx; + } + + &.active { + border: $border-width solid $color-dx; + background-color: rgba($color-dx, 0.08); + opacity: 1; + } + } + + &.variant-poor, + &.tier-poor { + .tab-primary-label, + .tier-score { + color: $color-poor; + } + + &.active { + border: $border-width solid $color-poor; + background-color: rgba($color-poor, 0.08); + opacity: 1; + } + } + + &.variant-failure, + &.tier-failure { + .tab-primary-label, + .tier-score { + color: $color-failure; + } + + &.active { + border: $border-width solid $color-failure; + background-color: rgba($color-failure, 0.08); + opacity: 1; + } + } + } + } + + .card-panels-container, + .triage-detail-card { + @include custom-scrollbar; + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + padding: $spacing-md; + position: relative; + min-height: 220px; + max-height: 380px; + overflow-y: auto; + + .card-panel, + .triage-panel { + display: none; + flex-direction: column; + gap: $spacing-sm; + + &.active { + display: flex; + } + + .panel-heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: $spacing-sm; + + h4 { + font-size: $font-size-md; + font-weight: 700; + margin: 0; + color: var(--site-base-fgColor); + } + + .panel-badge { + @include category-badge; + } + } + + .panel-overview, + .criteria-text { + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + margin: 0; + } + + .panel-items-section, + .actions-section { + border-top: 1px solid var(--site-inset-borderColor); + padding-top: $spacing-sm; + + .items-label, + .actions-label { + font-size: $font-size-xs; + font-weight: 700; + text-transform: uppercase; + color: var(--site-base-fgColor); + margin-bottom: $spacing-xs; + } + + ul { + margin: 0; + padding-left: $spacing-md; + + li { + font-size: $font-size-sm; + color: var(--site-base-fgColor); + margin-bottom: $spacing-xs; + + strong { + color: var(--site-primary-color); + } + + code { + font-size: 0.85em; + background-color: var(--site-raised-bgColor); + padding: $spacing-xs; + border-radius: var(--site-radius); + } + + &:last-child { + margin-bottom: 0; + } + } + } + } + + .panel-footer-text { + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + margin: $spacing-sm 0 0 0; + font-style: italic; + } + } + } + } + + // CUJ Diagram Component + .cuj-diagram-card { + @include bench-element-width; + background-color: var(--site-base-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + padding: $spacing-md; + margin-block: $spacing-lg $spacing-xl; + box-shadow: $shadow-sm; + display: flex; + flex-direction: column; + gap: $spacing-sm; + + .cuj-diagram-section { + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + padding: $spacing-md $spacing-lg; + display: grid; + grid-template-columns: 4.5rem 1fr; + align-items: center; + gap: $spacing-md; + + @media (max-width: 639px) { + grid-template-columns: 1fr; + justify-items: center; + text-align: center; + padding: $spacing-md; + gap: $spacing-sm; + } + + .section-sidebar { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: $spacing-xs; + + .section-icon { + display: flex; + align-items: center; + justify-content: center; + + .material-symbols { + font-size: $font-size-xxl; + font-variation-settings: 'FILL' 1; + } + + &.variant-grey, + &.variant-blue { + color: $color-functional; + } + + &.variant-amber { + color: $color-dx; + } + + &.variant-green { + color: $color-outcome; + } + + &.variant-purple { + color: $color-quality; + } + } + + .section-label { + font-family: var(--site-ui-fontFamily); + font-size: $font-size-sm; + font-weight: 700; + color: var(--site-base-fgColor); + } + } + + .section-items { + display: flex; + flex-direction: column; + gap: $spacing-sm; + width: 100%; + + .cuj-pill { + width: 100%; + padding: $spacing-md $spacing-lg; + border-radius: 9999px; + font-size: $font-size-sm; + font-weight: 500; + text-align: center; + display: flex; + align-items: center; + justify-content: center; + transition: transform $transition-normal; + + &.pill-grey { + background-color: #d1d5db; + border: $border-width solid #858d99; + color: #1f2937; + } + + &.pill-blue { + background-color: #bae6fd; + border: $border-width solid $color-functional; + color: #0c4a6e; + } + + &.pill-amber { + background-color: #fef08a; + border: $border-width solid #eab308; + color: #713f12; + } + + &.pill-green { + background-color: #bbf7d0; + border: $border-width solid #16a34a; + color: #14532d; + } + + &.pill-purple { + background-color: #e9d5ff; + border: $border-width solid #9333ea; + color: #581c87; + } + } + } + } + } + + // Task Specifications Expansion Panels + .task-specs-list { + @include bench-element-width; + background-color: var(--site-inset-bgColor); + border: 1px solid var(--site-inset-borderColor); + border-radius: var(--site-radius); + margin-block: $spacing-lg $spacing-xl; + overflow: hidden; + box-shadow: $shadow-sm; + + .task-spec-panel { + border-bottom: 1px solid var(--site-inset-borderColor); + + &:last-child { + border-bottom: none; + } + + > a.collapsible { + display: flex; + align-items: center; + justify-content: space-between; + padding: $spacing-md $spacing-lg; + background-color: var(--site-base-bgColor); + color: var(--site-base-fgColor); + text-decoration: none; + cursor: pointer; + gap: $spacing-md; + transition: background-color $transition-normal; + + &:hover { + background-color: var(--site-inset-bgColor); + text-decoration: none; + } + + &:not(.collapsed) { + background-color: var(--site-inset-bgColor); + + &::after { + transform: rotate(180deg); + } + } + + &::after { + content: 'keyboard_arrow_down'; + content: 'keyboard_arrow_down' / ''; + font-family: var(--site-icon-fontFamily, 'Material Symbols Outlined'); + font-variation-settings: 'FILL' 0, 'wght' 400, 'GRAD' 0, 'opsz' 24; + font-weight: normal; + font-style: normal; + font-size: $font-size-xl; + transition: transform $transition-normal; + color: var(--site-base-fgColor-lighter); + flex-shrink: 0; + } + + .panel-header-left { + display: flex; + align-items: center; + gap: $spacing-md; + flex: 1; + min-width: 0; + + @media (max-width: 639px) { + align-items: flex-start; + } + + .panel-icon-wrap { + @include category-icon-background; + flex-shrink: 0; + } + + .panel-header-content { + display: flex; + flex-direction: column; + gap: $spacing-xs; + flex: 1; + min-width: 0; + + .panel-title-row { + display: flex; + align-items: center; + gap: $spacing-sm; + flex-wrap: wrap; + + h4 { + font-family: var(--site-ui-fontFamily); + font-size: $font-size-md; + font-weight: 600; + margin: 0; + color: var(--site-base-fgColor); + } + + .badge { + @include category-badge; + } + } + + .panel-description { + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + margin: 0; + } + } + } + } + + .task-spec-body { + padding: $spacing-md $spacing-lg; + background-color: var(--site-base-bgColor); + border-top: 1px solid var(--site-inset-borderColor); + display: none; + + &.show { + display: block; + } + + .lead-text { + margin: 0 0 $spacing-md 0; + + code { + font-size: 0.85em; + background-color: var(--site-raised-bgColor); + padding: $spacing-xs $spacing-sm; + border-radius: var(--site-radius); + } + } + + .table-subheading { + font-size: $font-size-sm; + font-weight: 700; + margin: $spacing-md 0 $spacing-xs 0; + + &:first-of-type { + margin-top: 0; + } + } + + .table-wrapper { + table.spec-table, + table { + width: 100%; + border-spacing: $spacing-sm; + border: none; + font-size: $font-size-sm; + + tbody { + background: transparent; + border: none; + + tr { + td { + vertical-align: top; + + strong { + font-weight: 600; + } + + code { + font-size: 0.85em; + padding: $spacing-xs $spacing-sm; + border-radius: var(--site-radius); + } + + &:first-child { + padding-left: 0; + font-weight: 600; + width: 200px; + } + + &:last-child { + color: var(--site-base-fgColor-alt); + padding-right: 0; + } + } + } + } + } + } + + .footer-text { + font-size: $font-size-sm; + color: var(--site-base-fgColor-alt); + margin: $spacing-sm 0 0 0; + font-style: italic; + + code { + font-size: 0.85em; + background-color: var(--site-raised-bgColor); + padding: $spacing-xs $spacing-sm; + border-radius: var(--site-radius); + font-style: normal; + } + } + } + } + } + + // Dark mode overrides for methodology components + @media (prefers-color-scheme: dark) { + .cuj-diagram-card { + .cuj-diagram-section { + background-color: rgba(255, 255, 255, 0.04); + border-color: rgba(255, 255, 255, 0.08); + + .section-items { + .cuj-pill { + &.pill-grey { + background-color: rgba(148, 163, 184, 0.18); + border-color: rgba(148, 163, 184, 0.45); + color: #f1f5f9; + } + + &.pill-blue { + background-color: rgba(56, 189, 248, 0.18); + border-color: rgba(56, 189, 248, 0.55); + color: #e0f2fe; + } + + &.pill-amber { + background-color: rgba(234, 179, 8, 0.18); + border-color: rgba(234, 179, 8, 0.55); + color: #fef08a; + } + + &.pill-green { + background-color: rgba(34, 197, 94, 0.18); + border-color: rgba(34, 197, 94, 0.55); + color: #dcfce7; + } + + &.pill-purple { + background-color: rgba(168, 85, 247, 0.18); + border-color: rgba(168, 85, 247, 0.55); + color: #f3e8ff; + } + } + } + } + } + + .grader-matrix { + .grader-cards-track .grader-card.cat-llm, + .grader-cards-grid .grader-card.cat-llm { + background-color: rgba($color-llm, 0.08); + } + } + + .reliability-comparison-grid .reliability-card.north-star { + background: $north-star-bg-dark; + border-color: rgba(59, 130, 246, 0.35); + } + } + + body.dark-mode & { + .cuj-diagram-card { + .cuj-diagram-section { + background-color: rgba(255, 255, 255, 0.04); + border-color: rgba(255, 255, 255, 0.08); + + .section-items { + .cuj-pill { + &.pill-grey { + background-color: rgba(148, 163, 184, 0.18); + border-color: rgba(148, 163, 184, 0.45); + color: #f1f5f9; + } + + &.pill-blue { + background-color: rgba(56, 189, 248, 0.18); + border-color: rgba(56, 189, 248, 0.55); + color: #e0f2fe; + } + + &.pill-amber { + background-color: rgba(234, 179, 8, 0.18); + border-color: rgba(234, 179, 8, 0.55); + color: #fef08a; + } + + &.pill-green { + background-color: rgba(34, 197, 94, 0.18); + border-color: rgba(34, 197, 94, 0.55); + color: #dcfce7; + } + + &.pill-purple { + background-color: rgba(168, 85, 247, 0.18); + border-color: rgba(168, 85, 247, 0.55); + color: #f3e8ff; + } + } + } + } + } + + .grader-matrix { + .grader-cards-track .grader-card.cat-llm, + .grader-cards-grid .grader-card.cat-llm { + background-color: rgba($color-llm, 0.08); + } + } + + .reliability-comparison-grid .reliability-card.north-star { + background: $north-star-bg-dark; + border-color: rgba(59, 130, 246, 0.35); + } + } +} diff --git a/sites/www/lib/styles/styles.scss b/sites/www/lib/styles/styles.scss index 966964cb6bd..409e10c168d 100644 --- a/sites/www/lib/styles/styles.scss +++ b/sites/www/lib/styles/styles.scss @@ -14,6 +14,7 @@ @use 'components/carousel'; @use 'components/cookies'; @use 'components/content'; +@use 'components/drawer'; @use 'components/embeds'; @use 'components/features'; @use 'components/filters-dropdown'; @@ -47,10 +48,13 @@ @use 'pages/partner'; @use 'pages/showcase'; @use 'pages/why_flutter'; +@use 'pages/flutterbench'; +@use 'pages/flutterbench-story'; @use 'package:site_shared/_sass/components/breadcrumbs'; @use 'package:site_shared/_sass/components/blog'; @use 'package:site_shared/_sass/components/code'; @use 'package:site_shared/_sass/components/cookie-notice'; @use 'package:site_shared/_sass/components/dropdown'; +@use 'package:site_shared/_sass/components/ide-explorer'; @use 'package:site_shared/_sass/components/mermaid'; diff --git a/sites/www/test/models/content/flutterbench_content_test.dart b/sites/www/test/models/content/flutterbench_content_test.dart new file mode 100644 index 00000000000..e79ac99c965 --- /dev/null +++ b/sites/www/test/models/content/flutterbench_content_test.dart @@ -0,0 +1,140 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_website/src/models/content/flutterbench_content.dart'; +import 'package:test/test.dart'; + +void main() { + group('FlutterBenchJobData.fromJson', () { + test('decodes production job.json file successfully', () { + final file = File('content/data/flutterbench/job.json'); + expect(file.existsSync(), isTrue); + + final json = jsonDecode(file.readAsStringSync()) as Map; + final job = FlutterBenchJobData.fromJson(json); + + expect(job.id, isNotEmpty); + expect(job.startedAt, isNotEmpty); + expect(job.nTotalTrials, greaterThan(0)); + expect(job.evals, isNotEmpty); + + final firstEval = job.evals.first; + expect(firstEval.evalKey, isNotEmpty); + expect(firstEval.modelName, isNotEmpty); + expect(firstEval.passAt1, greaterThanOrEqualTo(0.0)); + expect(firstEval.meanReward, greaterThanOrEqualTo(0.0)); + expect(firstEval.outcomeScore, isNotNull); + expect(firstEval.qualityScore, isNotNull); + expect(firstEval.dxScore, isNotNull); + }); + }); + + group('FlutterBenchTasksData.fromJson', () { + test('decodes production tasks.json file successfully', () { + final file = File('content/data/flutterbench/tasks.json'); + expect(file.existsSync(), isTrue); + + final json = jsonDecode(file.readAsStringSync()) as Map; + final tasksData = FlutterBenchTasksData.fromJson(json); + + expect(tasksData.tasks, isNotEmpty); + final task = tasksData.tasks.first; + expect(task.slug, isNotEmpty); + expect(task.displayName, isNotEmpty); + expect(task.category, isNotEmpty); + expect(task.trials, isNotEmpty); + }); + }); + + group('FlutterBenchTrialsData.fromJson', () { + test('decodes production trials.json file successfully', () { + final file = File('content/data/flutterbench/trials.json'); + expect(file.existsSync(), isTrue); + + final json = jsonDecode(file.readAsStringSync()) as Map; + final trialsData = FlutterBenchTrialsData.fromJson(json); + + expect(trialsData.trials, isNotEmpty); + + // Verify that errored trial has distinct exception properties + final errorTrial = trialsData.trials.firstWhere( + (t) => t.status == 'error', + orElse: () => throw StateError('Expected an error trial in test data'), + ); + expect(errorTrial.exceptionType, isNotNull); + expect(errorTrial.reward, isNull); + + // Verify passing / partial trial has valid scores and durations + final passedTrial = trialsData.trials.firstWhere( + (t) => t.status != 'error', + ); + expect(passedTrial.reward, isNotNull); + expect(passedTrial.durations, isNotEmpty); + }); + }); + + group('FlutterBenchMethodologyData.fromJson', () { + test('decodes production methodology.json file successfully', () { + final file = File('content/data/flutterbench/methodology.json'); + expect(file.existsSync(), isTrue); + + final json = jsonDecode(file.readAsStringSync()) as Map; + final data = FlutterBenchMethodologyData.fromJson(json); + + expect(data.overview.leadText, isNotEmpty); + expect(data.overview.rows, isNotEmpty); + expect(data.overview.rows.first.anchor, isNotEmpty); + + expect(data.taskAnatomy.introText, isNotEmpty); + expect(data.taskAnatomy.rootId, isNotEmpty); + expect(data.taskAnatomy.tree, isNotEmpty); + + final envNode = data.taskAnatomy.tree.firstWhere( + (n) => n.id == 'environment', + ); + expect(envNode.type, 'folder'); + expect(envNode.children, isNotEmpty); + + final analysisOptionsNode = envNode.children.firstWhere( + (n) => n.id == 'analysis-options', + ); + expect(analysisOptionsNode.code, isNotNull); + expect(analysisOptionsNode.code!.lang, 'yaml'); + + expect(data.graderTiers.rows, isNotEmpty); + expect(data.diagnosticTelemetry.rows, isNotEmpty); + expect(data.rootCauseAudits.items, isNotEmpty); + + expect(data.transparency.harborExample.task, isNotEmpty); + expect(data.transparency.harborExample.agent, isNotEmpty); + expect(data.transparency.harborExample.model, isNotEmpty); + expect(data.transparency.harborExample.mcp, isNotEmpty); + + expect(data.cujExample, isNotEmpty); + expect(data.taskSpecifications, isNotEmpty); + expect(data.dimensions, isNotEmpty); + }); + }); + + group('FlutterBenchCujsData.fromJson', () { + test('decodes production cujs.json file successfully', () { + final file = File('content/data/flutterbench/cujs.json'); + expect(file.existsSync(), isTrue); + + final json = jsonDecode(file.readAsStringSync()) as Map; + final data = FlutterBenchCujsData.fromJson(json); + + expect(data.cujs, isNotEmpty); + + final first = data.cujs.first; + expect(first.goal, isNotEmpty); + expect(first.persona, isNotEmpty); + expect(first.tasks, isNotEmpty); + expect(first.tasks.first.task, isNotEmpty); + }); + }); +} diff --git a/sites/www/tool/import_flutterbench.dart b/sites/www/tool/import_flutterbench.dart new file mode 100644 index 00000000000..1d4e8bf6bb7 --- /dev/null +++ b/sites/www/tool/import_flutterbench.dart @@ -0,0 +1,881 @@ +// Copyright 2026, the Flutter authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that +// can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as p; + +void main(List args) async { + final repoRoot = _findRepoRoot(); + final wwwDir = p.join(repoRoot, 'sites', 'www'); + + // Determine source directory + String? sourceDir; + for (final arg in args) { + if (arg.startsWith('--data=')) { + sourceDir = arg.substring('--data='.length); + } + } + + if (sourceDir == null) { + final candidate1 = p.join( + wwwDir, + 'lib', + 'src', + 'data', + 'raw_flutterbench_data', + ); + final candidate2 = p.join(repoRoot, 'eaw_flutter_bench'); + final candidate3 = p.join(repoRoot, '..', 'flutter-bench', 'mock_data'); + if (Directory(candidate1).existsSync()) { + sourceDir = candidate1; + } else if (Directory(candidate2).existsSync()) { + sourceDir = candidate2; + } else if (Directory(candidate3).existsSync()) { + sourceDir = candidate3; + } else { + stderr.writeln('Error: Could not find raw flutterbench data directory.'); + exit(1); + } + } + + stdout.writeln('Importing FlutterBench data from: $sourceDir'); + + final rawDir = Directory(sourceDir); + if (!rawDir.existsSync()) { + stderr.writeln('Directory does not exist: $sourceDir'); + exit(1); + } + + // 1. Read job result.json + final jobResultFile = File(p.join(sourceDir, 'result.json')); + if (!jobResultFile.existsSync()) { + stderr.writeln('Missing job result.json in $sourceDir'); + exit(1); + } + + final jobJson = + jsonDecode(jobResultFile.readAsStringSync()) as Map; + final jobId = jobJson['id'] as String? ?? 'unknown-job'; + final startedAt = jobJson['started_at'] as String? ?? ''; + final finishedAt = jobJson['finished_at'] as String? ?? ''; + final nTotalTrials = (jobJson['n_total_trials'] as num?)?.toInt() ?? 0; + final stats = (jobJson['stats'] as Map?) ?? {}; + final nCompletedTrials = (stats['n_completed_trials'] as num?)?.toInt() ?? 0; + final nErroredTrials = (stats['n_errored_trials'] as num?)?.toInt() ?? 0; + final jobCostUsd = (stats['cost_usd'] as num?)?.toDouble() ?? 0.0; + final jobInputTokens = (stats['n_input_tokens'] as num?)?.toInt() ?? 0; + final jobCacheTokens = (stats['n_cache_tokens'] as num?)?.toInt() ?? 0; + final jobOutputTokens = (stats['n_output_tokens'] as num?)?.toInt() ?? 0; + + final rawEvals = (stats['evals'] as Map?) ?? {}; + + // 2. Discover and parse all trials + final trials = >[]; + final entities = rawDir.listSync(); + + for (final entity in entities) { + if (entity is Directory) { + final dirName = p.basename(entity.path); + final trialResultFile = File(p.join(entity.path, 'result.json')); + if (!trialResultFile.existsSync()) continue; + + final trialJson = jsonDecode( + trialResultFile.readAsStringSync(), + ) as Map; + final trialName = trialJson['trial_name'] as String? ?? dirName; + final taskName = trialJson['task_name'] as String? ?? ''; + final taskSlug = _slugFromTrial(trialName, taskName); + + final config = (trialJson['config'] as Map?) ?? {}; + final agentConfig = (config['agent'] as Map?) ?? {}; + final agentName = agentConfig['name'] as String? ?? 'unknown-agent'; + final modelName = agentConfig['model_name'] as String? ?? 'unknown-model'; + final skills = + (agentConfig['skills'] as List?)?.cast() ?? + []; + final mcpServersRaw = + (agentConfig['mcp_servers'] as List?) ?? []; + final mcpServers = []; + for (final mcp in mcpServersRaw) { + if (mcp is Map && mcp['name'] != null) { + mcpServers.add(mcp['name'] as String); + } else if (mcp is String) { + mcpServers.add(mcp); + } + } + + final hasDartTooling = skills.isNotEmpty || mcpServers.isNotEmpty; + + // Agent tokens and cost + final agentResult = trialJson['agent_result'] as Map?; + final inputTokens = + (agentResult?['n_input_tokens'] as num?)?.toInt() ?? 0; + final cacheTokens = + (agentResult?['n_cache_tokens'] as num?)?.toInt() ?? 0; + final outputTokens = + (agentResult?['n_output_tokens'] as num?)?.toInt() ?? 0; + final costUsd = (agentResult?['cost_usd'] as num?)?.toDouble() ?? 0.0; + + // Verifier result + final verifierResult = + trialJson['verifier_result'] as Map?; + final rewardsMap = verifierResult?['rewards'] as Map?; + final reward = (rewardsMap?['reward'] as num?)?.toDouble(); + + // Exception info + final exceptionInfo = + trialJson['exception_info'] as Map?; + final exceptionType = exceptionInfo?['exception_type'] as String?; + final exceptionMessage = exceptionInfo?['exception_message'] as String?; + final exceptionTraceback = + exceptionInfo?['exception_traceback'] as String?; + + // Determine status + final String status; + if (exceptionInfo != null || exceptionType != null || reward == null) { + status = 'error'; + } else if (reward >= 0.80) { + status = 'pass'; + } else if (reward >= 0.25) { + status = 'partial'; + } else { + status = 'fail'; + } + + // Phase durations + final durations = {}; + for (final phase in [ + 'environment_setup', + 'agent_setup', + 'agent_execution', + 'verifier', + ]) { + final phaseData = trialJson[phase] as Map?; + if (phaseData != null && + phaseData['started_at'] != null && + phaseData['finished_at'] != null) { + final start = DateTime.tryParse(phaseData['started_at'] as String); + final end = DateTime.tryParse(phaseData['finished_at'] as String); + if (start != null && end != null) { + durations[phase] = end.difference(start).inMilliseconds / 1000.0; + } + } + } + + // Reward details + Map? rewardTree; + final diagnosticTree = {}; + final rewardDetailsFile = File( + p.join(entity.path, 'verifier', 'reward-details.json'), + ); + if (rewardDetailsFile.existsSync()) { + try { + final rewardDetails = jsonDecode( + rewardDetailsFile.readAsStringSync(), + ) as Map; + rewardTree = rewardDetails; + for (final entry in rewardDetails.entries) { + final val = entry.value; + if (val is Map && val['diagnostic'] == true) { + diagnosticTree[entry.key] = val; + } + } + } catch (e) { + stdout.writeln( + 'Warning: Failed to parse reward-details.json for $trialName: $e', + ); + } + } + + // Trajectory + List>? trajectory; + final trajectoryFile = File( + p.join(entity.path, 'agent', 'trajectory.json'), + ); + if (trajectoryFile.existsSync()) { + try { + final trajJson = jsonDecode( + trajectoryFile.readAsStringSync(), + ) as Map; + final steps = trajJson['steps'] as List?; + if (steps != null) { + trajectory = steps.whereType>().toList(); + } + } catch (e) { + stdout.writeln( + 'Warning: Failed to parse trajectory.json for $trialName: $e', + ); + } + } + + // Artifacts + final artifacts = >[]; + final manifestFile = File( + p.join(entity.path, 'artifacts', 'manifest.json'), + ); + if (manifestFile.existsSync()) { + try { + final manifestList = + jsonDecode(manifestFile.readAsStringSync()) as List; + for (final item in manifestList) { + if (item is Map) { + final dest = item['destination'] as String? ?? ''; + final src = item['source'] as String? ?? ''; + final type = item['type'] as String? ?? 'file'; + final fileOnDisk = File(p.join(entity.path, dest)); + String? content; + if (fileOnDisk.existsSync()) { + try { + content = fileOnDisk.readAsStringSync(); + } catch (_) {} + } + artifacts.add({ + 'source': src, + 'destination': dest, + 'type': type, + 'status': item['status'] ?? 'ok', + 'content': content, + }); + } + } + } catch (e) { + stdout.writeln( + 'Warning: Failed to parse manifest.json for $trialName: $e', + ); + } + } + + // Raw logs + String? testStdout; + final testStdoutFile = File( + p.join(entity.path, 'verifier', 'test-stdout.txt'), + ); + if (testStdoutFile.existsSync()) { + testStdout = testStdoutFile.readAsStringSync(); + } + + String? exceptionLog; + final exceptionFile = File(p.join(entity.path, 'exception.txt')); + if (exceptionFile.existsSync()) { + exceptionLog = exceptionFile.readAsStringSync(); + } + + final outcomeScore = _extractDimensionScore(rewardTree, 'outcome'); + final qualityScore = _extractDimensionScore(rewardTree, 'quality'); + final dxScore = _extractDimensionScore(rewardTree, 'dx'); + + final evalKey = (config['eval_key'] as String?) ?? + (trialJson['eval_key'] as String?); + + trials.add({ + 'trial_name': trialName, + 'task_name': taskName, + 'task_slug': taskSlug, + 'eval_key': evalKey, + 'agent_name': agentName, + 'model_name': modelName, + 'model_short_name': _shortModelName(modelName), + 'provider': _providerFromModel(modelName), + 'skills': skills, + 'mcp_servers': mcpServers, + 'has_dart_tooling': hasDartTooling, + 'status': status, + 'reward': reward, + 'outcome_score': outcomeScore, + 'quality_score': qualityScore, + 'dx_score': dxScore, + 'exception_type': exceptionType, + 'exception_message': exceptionMessage, + 'exception_traceback': exceptionTraceback, + 'durations': durations, + 'input_tokens': inputTokens, + 'cache_tokens': cacheTokens, + 'output_tokens': outputTokens, + 'cost_usd': costUsd, + 'reward_tree': rewardTree, + 'diagnostic_tree': diagnosticTree, + 'trajectory': trajectory, + 'artifacts': artifacts, + 'test_stdout': testStdout, + 'exception_log': exceptionLog, + }); + } + } + + // 3. Aggregate evals data + final evalsList = >[]; + for (final entry in rawEvals.entries) { + final evalKey = entry.key; + final evalData = entry.value as Map; + final nTrials = (evalData['n_trials'] as num?)?.toInt() ?? 0; + final nErrors = (evalData['n_errors'] as num?)?.toInt() ?? 0; + + final metricsList = (evalData['metrics'] as List?) ?? []; + var meanReward = 0.0; + var minReward = 0.0; + var maxReward = 0.0; + var medianReward = 0.0; + + if (metricsList.isNotEmpty && metricsList.first is Map) { + final m = metricsList.first as Map; + meanReward = + (m['mean'] as num?)?.toDouble() ?? + (m['reward'] as num?)?.toDouble() ?? + 0.0; + minReward = (m['min'] as num?)?.toDouble() ?? meanReward; + maxReward = (m['max'] as num?)?.toDouble() ?? meanReward; + medianReward = (m['median'] as num?)?.toDouble() ?? meanReward; + } + + final passAtK = (evalData['pass_at_k'] as Map?) ?? {}; + final passAt1 = (passAtK['1'] as num?)?.toDouble() ?? 0.0; + + // Split evalKey: {agent}__{model}__{variant} + final parts = evalKey.split('__'); + final agentName = parts.isNotEmpty ? parts[0] : 'unknown-agent'; + final modelName = parts.length > 1 ? parts[1] : 'unknown-model'; + final variant = parts.length > 2 ? parts[2] : ''; + + // Find trials for this eval + final evalTrials = trials + .where( + (t) => + (t['eval_key'] != null && t['eval_key'] == evalKey) || + ((t['model_short_name'] == modelName || + t['model_name'] == modelName || + (t['model_name'] as String).endsWith('/$modelName')) && + ((t['agent_name'] as String).contains(agentName) || + agentName.contains(t['agent_name'] as String))), + ) + .toList(); + + var cost = 0.0; + var inTokens = 0; + var outTokens = 0; + var hasTooling = false; + var outcomeScoreSum = 0.0; + var qualityScoreSum = 0.0; + var dxScoreSum = 0.0; + var scoredTrialCount = 0; + + for (final t in evalTrials) { + cost += (t['cost_usd'] as num?)?.toDouble() ?? 0.0; + inTokens += (t['input_tokens'] as num?)?.toInt() ?? 0; + outTokens += (t['output_tokens'] as num?)?.toInt() ?? 0; + if (t['has_dart_tooling'] == true) hasTooling = true; + + if (t['status'] != 'error') { + final o = t['outcome_score'] as num?; + final q = t['quality_score'] as num?; + final d = t['dx_score'] as num?; + if (o != null || q != null || d != null) { + if (o != null) outcomeScoreSum += o.toDouble(); + if (q != null) qualityScoreSum += q.toDouble(); + if (d != null) dxScoreSum += d.toDouble(); + scoredTrialCount++; + } + } + } + + final meanOutcome = + scoredTrialCount > 0 ? (outcomeScoreSum / scoredTrialCount) : null; + final meanQuality = + scoredTrialCount > 0 ? (qualityScoreSum / scoredTrialCount) : null; + final meanDx = + scoredTrialCount > 0 ? (dxScoreSum / scoredTrialCount) : null; + + evalsList.add({ + 'eval_key': evalKey, + 'agent_name': agentName, + 'model_name': modelName, + 'model_short_name': _shortModelName(modelName), + 'provider': _providerFromModel(modelName), + 'variant': variant, + 'n_trials': nTrials, + 'n_errors': nErrors, + 'mean_reward': meanReward, + 'outcome_score': meanOutcome != null + ? double.parse(meanOutcome.toStringAsFixed(2)) + : null, + 'quality_score': meanQuality != null + ? double.parse(meanQuality.toStringAsFixed(2)) + : null, + 'dx_score': meanDx != null + ? double.parse(meanDx.toStringAsFixed(2)) + : null, + 'min_reward': minReward, + 'max_reward': maxReward, + 'median_reward': medianReward, + 'pass_at_1': passAt1, + 'cost_usd': cost > 0 ? cost : (nTrials > 0 ? jobCostUsd : 0.0), + 'input_tokens': inTokens > 0 + ? inTokens + : (nTrials > 0 ? jobInputTokens : 0), + 'output_tokens': outTokens > 0 + ? outTokens + : (nTrials > 0 ? jobOutputTokens : 0), + 'has_dart_tooling': hasTooling, + }); + } + + // Sort evals by mean_reward descending + evalsList.sort( + (a, b) => (b['mean_reward'] as num).toDouble().compareTo( + (a['mean_reward'] as num).toDouble(), + ), + ); + + // Compute top model and overall average reward + final topModel = evalsList.isNotEmpty ? evalsList.first : null; + final topModelName = topModel != null + ? (topModel['model_short_name'] as String) + : 'None'; + final topModelReward = topModel != null + ? (topModel['mean_reward'] as num).toDouble() + : 0.0; + + var totalScoreSum = 0.0; + var completedCount = 0; + for (final t in trials) { + final r = t['reward'] as num?; + if (r != null && t['status'] != 'error') { + totalScoreSum += r.toDouble(); + completedCount++; + } + } + final overallAverageReward = completedCount > 0 + ? totalScoreSum / completedCount + : 0.0; + + // 4. Organize tasks (CUJs) + final tasksMap = >{}; + for (final t in trials) { + final slug = t['task_slug'] as String; + final taskName = t['task_name'] as String; + if (!tasksMap.containsKey(slug)) { + tasksMap[slug] = { + 'slug': slug, + 'task_name': taskName, + 'display_name': _displayNameForTask(slug), + 'category': _categoryForTask(slug), + 'description': _descriptionForTask(slug), + 'trials': >[], + 'scores_by_eval': {}, + }; + } + final taskEntry = tasksMap[slug]!; + (taskEntry['trials'] as List>).add({ + 'trial_name': t['trial_name'], + 'status': t['status'], + 'reward': t['reward'], + 'model_name': t['model_name'], + 'model_short_name': t['model_short_name'], + 'agent_name': t['agent_name'], + 'has_dart_tooling': t['has_dart_tooling'], + 'exception_type': t['exception_type'], + }); + + // Score for eval + final tEvalKey = t['eval_key'] as String?; + final evalKey = (tEvalKey != null && + evalsList.any((e) => e['eval_key'] == tEvalKey)) + ? tEvalKey + : evalsList.firstWhere( + (e) { + final em = e['model_name'] as String; + final ea = e['agent_name'] as String; + final tm = t['model_name'] as String; + final tsm = t['model_short_name'] as String; + final ta = t['agent_name'] as String; + final modelMatch = tm == em || tsm == em || tm.endsWith('/$em'); + final agentMatch = ta.contains(ea) || ea.contains(ta); + return modelMatch && agentMatch; + }, + orElse: () => evalsList.firstWhere( + (e) { + final em = e['model_name'] as String; + final tm = t['model_name'] as String; + final tsm = t['model_short_name'] as String; + return tm == em || tsm == em || tm.endsWith('/$em'); + }, + orElse: () => evalsList.first, + ), + )['eval_key'] as String; + + (taskEntry['scores_by_eval'] as Map)[evalKey] = { + 'trial_name': t['trial_name'], + 'status': t['status'], + 'reward': t['reward'], + 'exception_type': t['exception_type'], + }; + } + + final tasksList = tasksMap.values.toList(); + + // 5. Best CUJs / Worst CUJs per eval + for (final eval in evalsList) { + final evalKey = eval['eval_key'] as String; + final scoredTasks = >[]; + for (final task in tasksList) { + final scores = task['scores_by_eval'] as Map; + if (scores.containsKey(evalKey)) { + final sc = scores[evalKey] as Map; + scoredTasks.add({ + 'task_slug': task['slug'], + 'task_name': task['display_name'], + 'reward': sc['reward'], + 'status': sc['status'], + }); + } + } + scoredTasks.sort((a, b) { + final ra = (a['reward'] as num?)?.toDouble() ?? -1.0; + final rb = (b['reward'] as num?)?.toDouble() ?? -1.0; + return rb.compareTo(ra); + }); + + eval['best_cujs'] = scoredTasks + .where( + (t) => (t['reward'] as num?) != null && (t['reward'] as num) > 0.5, + ) + .take(3) + .toList(); + eval['worst_cujs'] = scoredTasks.reversed + .where( + (t) => + t['status'] == 'error' || + ((t['reward'] as num?) != null && (t['reward'] as num) < 0.5), + ) + .take(3) + .toList(); + } + + // 6. Output structured data + final dataDir = Directory(p.join(wwwDir, 'content', 'data', 'flutterbench')); + dataDir.createSync(recursive: true); + + final jobData = { + 'id': jobId, + 'started_at': startedAt, + 'finished_at': finishedAt, + 'n_total_trials': nTotalTrials, + 'n_completed_trials': nCompletedTrials, + 'n_errored_trials': nErroredTrials, + 'cost_usd': jobCostUsd, + 'n_input_tokens': jobInputTokens, + 'n_cache_tokens': jobCacheTokens, + 'n_output_tokens': jobOutputTokens, + 'top_model_name': topModelName, + 'top_model_reward': topModelReward, + 'overall_average_reward': overallAverageReward, + 'evals': evalsList, + }; + + File(p.join(dataDir.path, 'job.json')).writeAsStringSync( + const JsonEncoder.withIndent(' ').convert(jobData), + ); + + File(p.join(dataDir.path, 'tasks.json')).writeAsStringSync( + const JsonEncoder.withIndent(' ').convert({'tasks': tasksList}), + ); + + File(p.join(dataDir.path, 'trials.json')).writeAsStringSync( + const JsonEncoder.withIndent(' ').convert({'trials': trials}), + ); + + stdout.writeln('Wrote structured JSON data to: ${dataDir.path}'); + + // 7. Generate static Markdown pages + final fbPagesDir = Directory(p.join(wwwDir, 'content', 'ai', 'flutterbench')); + fbPagesDir.createSync(recursive: true); + + // /ai/flutterbench/index.md + File(p.join(fbPagesDir.path, 'index.md')).writeAsStringSync('''--- +title: FlutterBench Leaderboard +bodyTags: interior flutterbench +description: Benchmark results for AI coding agents on Dart and Flutter developer tasks. +publishDate: "2026-09-10" +--- + + +'''); + + // /ai/flutterbench/tasks/index.md + final tasksDir = Directory(p.join(fbPagesDir.path, 'tasks')); + tasksDir.createSync(recursive: true); + File(p.join(tasksDir.path, 'index.md')).writeAsStringSync('''--- +title: FlutterBench Tasks & CUJs +bodyTags: interior flutterbench +description: Explore Critical User Journeys (CUJs) and task performance across AI coding models in FlutterBench. +publishDate: "2026-09-10" +--- + + +'''); + + // One page per task: /ai/flutterbench/tasks/.md + for (final task in tasksList) { + final slug = task['slug'] as String; + final displayName = task['display_name'] as String; + File(p.join(tasksDir.path, '$slug.md')).writeAsStringSync('''--- +title: "FlutterBench Task: $displayName" +bodyTags: interior flutterbench +description: "Cross-model benchmark results and details for the $displayName task." +publishDate: "2026-09-10" +--- + + +'''); + } + + // One page per trial: /ai/flutterbench/trials/.md + final trialsDir = Directory(p.join(fbPagesDir.path, 'trials')); + trialsDir.createSync(recursive: true); + for (final trial in trials) { + final trialName = trial['trial_name'] as String; + final taskSlug = trial['task_slug'] as String; + final displayName = _displayNameForTask(taskSlug); + File(p.join(trialsDir.path, '$trialName.md')).writeAsStringSync('''--- +title: "FlutterBench Trial: $trialName" +bodyTags: interior flutterbench +description: "Full scoring rubric, trajectory, artifacts, and logs for $displayName trial $trialName." +publishDate: "2026-09-10" +--- + + +'''); + } + + // /ai/flutterbench/methodology.md + File(p.join(fbPagesDir.path, 'methodology.md')).writeAsStringSync('''--- +title: FlutterBench Methodology +bodyTags: interior flutterbench methodology +description: Detailed explanation of the FlutterBench harness, three-dimensional scoring rubric, and reproduction steps. +publishDate: "2026-09-10" +--- + +# FlutterBench Methodology + +FlutterBench is Flutter's dedicated evaluation harness for measuring how autonomous AI coding agents perform on real-world Dart and Flutter development tasks. + +Unlike general code benchmarks that rely on isolated algorithm puzzles or synthetic docstring completions, FlutterBench tests agents against authentic developer workflows grounded in Flutter's canonical **Critical User Journeys (CUJs)**. + +--- + +## Core Principles + +1. **Realistic Tasks over Synthetic Puzzles**: Every task tests a real feature implementation, bug fix, or refactoring in a realistic Flutter or Dart codebase. +2. **Developer Experience (DX) is a First-Class Signal**: We measure not just whether the final code compiles, but tool accuracy, trajectory efficiency, and recovery from errors. +3. **Containerized Sandboxing**: All trials run inside isolated Docker containers with pre-installed Flutter SDKs and the official Dart MCP server. +4. **Weighted Multi-Dimensional Scoring**: A composite reward evaluates outcome, quality, and developer experience, while capturing diagnostic telemetry separately. + +--- + +## Three-Dimensional Scoring Rubric + +Each trial produces a normalized composite reward between `0.0` and `1.0`: + +\$\$\\text{Reward} = 0.60 \\times \\text{Outcome} + 0.30 \\times \\text{Quality} + 0.10 \\times \\text{DX}\$\$ + +| Dimension | Weight | Description | Evaluation Mechanism | +| :--- | :--- | :--- | :--- | +| **Outcome** | 60% | Functional correctness and feature completion | `flutter test`, `flutter build bundle`, heuristic event assertion | +| **Quality** | 30% | Architectural conventions, idiomatic code, maintainability | `dart analyze`, DCM, LLM judge rubrics | +| **DX (Developer Experience)** | 10% | Tool interaction accuracy, minimal thrash, clean trajectory | Dart MCP telemetry, plan adherence, error loops | + +### Diagnostic Telemetry (Unscored) + +Separate telemetry blocks track **Process** and **Efficiency**: +- **Process**: Number of prompt turns, reasoning quality, and recovery velocity. +- **Efficiency**: Token usage (input, cache, and output tokens) and wall-clock execution time. + +These diagnostic metrics are reported alongside trials for observability, but are strictly excluded from the composite reward. + +--- + +## Task Anatomy + +Each FlutterBench task directory contains: +- **Instruction**: Natural developer prompt describing the target user journey. +- **Codebase**: Seed workspace repository containing initial project structure and dependencies. +- **Graders**: Verification scripts (`tests/graders.dart`, unit test suites, static analysis checks). +- **Environment**: Docker container specification defining SDK constraints and tool permissions. + +--- + +## Trial Execution Phases + +Each trial proceeds through four distinct, timed phases: + +1. **Environment Setup**: Container initialization, caching, and volume mounts. +2. **Agent Setup**: Workspace cloning, dependency resolution, and MCP tool initialization. +3. **Agent Execution**: Autonomous problem-solving, code generation, and iterative refinement. +4. **Verifier**: Automated test execution, static analysis grading, and LLM rubric evaluation. + +--- + +## Reproducing Results + +To reproduce benchmark trials locally using the Harbor evaluation runner: + +```bash +# Clone the evaluation task repository +git clone https://github.com/flutter/evals.git +cd evals + +# Run an individual trial +harbor run \\ + --task dataset/flutter/manage-state-with-bloc \\ + --agent antigravity-sdk \\ + --model google/gemini-3.5-flash \\ + --mcp dart +``` + +For questions or to contribute new CUJ evaluation tasks, visit the [Flutter repository on GitHub](https://github.com/flutter/flutter). +'''); + + stdout.writeln('Successfully generated FlutterBench content and data pages.'); +} + +String _slugFromTrial(String trialName, String taskName) { + if (trialName.contains('__')) { + return trialName.split('__').first; + } + if (taskName.contains('/')) { + return taskName.split('/').last; + } + return taskName.replaceAll(' ', '-').toLowerCase(); +} + +String _displayNameForTask(String slug) { + switch (slug) { + case 'flutter-manage-state-with-bloc': + return 'Manage State with BLoC'; + case 'flutter-adaptive-material-cupertino': + return 'Adaptive Material & Cupertino UI'; + case 'dart-build-cli-app': + return 'Build Command-Line CLI App'; + case 'flutter-custom-render-object': + return 'Custom RenderObject & Canvas'; + case 'flutter-offline-sync-sqlite': + return 'Offline SQLite Sync Repository'; + default: + return slug + .split('-') + .map((w) => w.isEmpty ? '' : '${w[0].toUpperCase()}${w.substring(1)}') + .join(' '); + } +} + +String _categoryForTask(String slug) { + switch (slug) { + case 'flutter-manage-state-with-bloc': + return 'State Management'; + case 'flutter-adaptive-material-cupertino': + return 'Multi-Platform UI'; + case 'dart-build-cli-app': + return 'Dart Utilities'; + case 'flutter-custom-render-object': + return 'Advanced Rendering'; + case 'flutter-offline-sync-sqlite': + return 'Data & Storage'; + default: + return 'General'; + } +} + +String _descriptionForTask(String slug) { + switch (slug) { + case 'flutter-manage-state-with-bloc': + return 'Implement an immutable state management layer using package:flutter_bloc, connecting UI events to business logic with unit tests.'; + case 'flutter-adaptive-material-cupertino': + return 'Build adaptive Flutter widgets that render Material 3 on Android/Web and native Cupertino patterns on iOS/macOS.'; + case 'dart-build-cli-app': + return 'Create a robust Dart command-line interface application with argument parsing, formatted output, and exit code handling.'; + case 'flutter-custom-render-object': + return 'Implement a custom RenderBox with layout constraints, custom painting, intrinsic dimensions, and pointer hit-testing.'; + case 'flutter-offline-sync-sqlite': + return 'Build an offline-first repository using SQLite with background synchronization, retry queues, and conflict resolution.'; + default: + return 'Benchmark evaluation task for Flutter and Dart AI agent capabilities.'; + } +} + +String _shortModelName(String modelName) { + var name = modelName; + if (name.contains('/')) { + name = name.split('/').last; + } + return name + .replaceAll('google/', '') + .replaceAll('anthropic/', '') + .replaceAll('openai/', '') + .replaceAll('deepseek/', ''); +} + +String _providerFromModel(String modelName) { + final lower = modelName.toLowerCase(); + if (lower.contains('gemini') || lower.contains('google')) { + return 'Google'; + } + if (lower.contains('claude') || lower.contains('anthropic')) { + return 'Anthropic'; + } + if (lower.contains('deepseek')) { + return 'DeepSeek'; + } + if (lower.contains('gpt') || + lower.contains('openai') || + lower.startsWith('o1') || + lower.startsWith('o3') || + lower.contains('/o1') || + lower.contains('/o3')) { + return 'OpenAI'; + } + return 'Community'; +} + +String _findRepoRoot() { + var dir = Directory.current; + while (true) { + if (File(p.join(dir.path, 'AGENTS.md')).existsSync() && + Directory(p.join(dir.path, 'sites', 'www')).existsSync()) { + return dir.path; + } + if (dir.path == dir.parent.path) break; + dir = dir.parent; + } + return '/Users/ewindmill/development/website'; +} + +double? _extractDimensionScore( + Map? rewardTree, + String dimName, +) { + if (rewardTree == null) return null; + if (rewardTree[dimName] is Map) { + final dimNode = rewardTree[dimName] as Map; + if (dimNode['score'] is num) { + return (dimNode['score'] as num).toDouble(); + } + } + if (rewardTree['reward'] is Map) { + final rewardNode = rewardTree['reward'] as Map; + if (rewardNode['criteria'] is List) { + for (final crit in rewardNode['criteria'] as List) { + if (crit is Map && crit['name'] == dimName) { + if (crit['value'] is num) { + return (crit['value'] as num).toDouble(); + } + if (crit['score'] is num) { + return (crit['score'] as num).toDouble(); + } + } + } + } + } + return null; +}