From 8ee4114163dc18053491e1fbba79da9092db63eb Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 23:26:21 -0400 Subject: [PATCH] Make release Experiments opt-in (#1423) --- docs/guide.md | 19 +++--- scripts/release.sh | 24 ++++---- site/architecture/index.html | 2 +- site/docs/index.html | 17 +++--- site/how-it-works/index.html | 2 +- site/index.html | 2 +- tests/release/experiments.rs | 110 ++++++++++++++++++++++++----------- 7 files changed, 111 insertions(+), 65 deletions(-) diff --git a/docs/guide.md b/docs/guide.md index 670bbe59..65ebdd40 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -2007,8 +2007,12 @@ cargo test ## Releasing -Before pushing anything, the release script runs `thirdshift experiment ---release ` using this checkout's runner and private Eval suites. +The release script skips Experiments by default, printing a note. Opt in with +`scripts/release.sh --with-experiment 0.4.0`, adding `--with-experiment` again +on each resume that should run an Experiment. + +With `--with-experiment`, before pushing anything, the script runs +`thirdshift experiment --release ` using this checkout's runner and private Eval suites. It evaluates the content commit beneath the version-only bump, so reruns use the same Experiment even before a bump branch exists. It pins that commit in a local Git ref before starting, so a merge to `main` during interruption does @@ -2019,18 +2023,15 @@ intervals and asks `release anyway? [y/N]`, including without `--review`. Only `y` or `yes` continues; EOF or any other answer stops. Pending borderline and decisive Scores are listed with their recheck commands and polled until you have rechecked them. No touched suite or a last release with no Trial flag -prints a note and carries on. `--no-experiment` skips the release Experiment -outright, on the first run and on every resume, and prints a line saying so: -use it only when an Experiment can't give a Verdict worth its Trials, such as -while every Trial is known to be void. An interrupted invocation resumes its saved +prints a note and carries on. An interrupted invocation resumes its saved Trials. A pushed bump branch is checked again before opening its PR, and an open or merged bump PR before merging or tagging. An already pushed tag only resumes the release workflow. Python 3 is needed to read the Experiment results. If `main` gains different content after the Candidate was selected, the script -refuses to push or merge that release under its old Experiment; choose a fresh -release version to measure current `main`. It also checks the merged tree before -tagging, so a concurrent merge cannot publish unmeasured content. +refuses to push or merge that release; choose a fresh release version for current +`main`. It also checks the merged tree before tagging, so a concurrent merge +cannot publish different content. These checks apply even without an Experiment. From a clone of this repo, signed in to `gh` and the User config's default Harness, run the release script with the new version: diff --git a/scripts/release.sh b/scripts/release.sh index 5a801c03..b31fd1f7 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -5,7 +5,7 @@ # workflow, and the script waits for that workflow run to publish the GitHub # Release. # -# Usage: scripts/release.sh [--review] [--no-experiment] +# Usage: scripts/release.sh [--review] [--with-experiment] # # Run it in a clone of this repo, signed in to gh and the default Harness. It # works from origin/main in a temporary worktree, so the branch checked out @@ -24,13 +24,13 @@ # didn't succeed. It waits for that tip's run to appear and finish, following # main if it moves and naming each move. The version bump starts from the tip # whose CI succeeded, with the requested version checked against that tip. -# Then an Experiment compares that content commit with its last release tag, -# selecting touched suites. A pass carries on; pending Scores are listed and -# waited for; fail/inconclusive Verdicts show intervals and require an override. +# With --with-experiment, an Experiment compares that content commit with its +# last release tag, selecting touched suites. A pass carries on; pending Scores +# are listed and waited for; fail/inconclusive Verdicts show intervals and +# require an override. # Untouched suites or a Baseline with no Trial flag need no Experiment. -# --no-experiment skips the release Experiment outright, on a fresh run and on -# every resume, and says so; use it only when an Experiment can't give a -# Verdict worth its Trials. +# Experiments are skipped by default, on a fresh run and on every resume, +# with a progress note. Supply --with-experiment on each invocation to run one. # # If a run is interrupted, running it again with the same version carries on # from the first step not yet done: it opens the PR for a pushed branch, waits @@ -74,17 +74,17 @@ poll_seconds=${RELEASE_POLL_SECONDS:-15} main() { review= - no_experiment= + with_experiment= while :; do case ${1-} in --review) review=1 ;; - --no-experiment) no_experiment=1 ;; + --with-experiment) with_experiment=1 ;; *) break ;; esac shift done if [ "$#" -ne 1 ]; then - echo "usage: release.sh [--review] [--no-experiment] " >&2 + echo "usage: release.sh [--review] [--with-experiment] " >&2 exit 2 fi version=$1 @@ -262,8 +262,8 @@ commit_bump() { # Use this checkout's runner and private Eval suites, as for summary_agent. wait_for_experiment() { local snapshot status outcome id answer - if [ -n "$no_experiment" ]; then - progress "skipping the release Experiment (--no-experiment) for ${1:0:7}" + if [ -z "$with_experiment" ]; then + progress "skipping the release Experiment (use --with-experiment to run it) for ${1:0:7}" return fi progress "checking release Experiment for ${1:0:7}" diff --git a/site/architecture/index.html b/site/architecture/index.html index b3013435..d0938dc7 100644 --- a/site/architecture/index.html +++ b/site/architecture/index.html @@ -216,7 +216,7 @@

Architecture

Trial completion removes Harness caches from each attempt's Session home, retaining native transcripts and Session evidence. Once grading and all rechecks finish, the Experiment removes every attempt's repository copy. Paused and interrupted Experiments retain repositories for resume. Requests, results, traces, reports, Grader packets and Scores stay readable; pruning failures only warn.

Each Trial, code Grader and model Grader gets its own systemd-run --user --scope with MemoryHigh, MemoryMax, MemorySwapMax and OOMPolicy=kill. Its process raises oom_score_adj before starting work. The User config defaults are 1 GiB, 2 GiB and 1 GiB; scopes require a reachable user manager with lingering enabled for logged-out work.

The Experiment owns recheck history beside its immutable Grader Scores. thirdshift experiments <id> --json exposes pending borderline and decisive Scores. thirdshift recheck <id> <trial> pass|fail --reason '…' takes the Experiment's exclusive lock and atomically saves the history, recalculated Verdict and pending list. It also updates the latest invocation's Command outcome and Inbox group, preserving its completion time. The Verdict waits until the list is empty; flipping either pass/fail or a wrong-finding flag can make a Score decisive. Waiting rechecks place that invocation in Needs you; a confirmed pass moves it to Done.

-

thirdshift experiment --release <commit> compares the content commit with the last release tag on touched suites. The release script waits for rechecks and asks permission for a fail or inconclusive Verdict before pushing. Reruns reuse the Experiment, including before opening a pushed branch's PR. It refuses different main content before pushing or merging and checks the merged tree before tagging, so the release cannot contain unmeasured changes. No touched suite or a last release without the Trial flag carries on with a note. Releasing.

+

thirdshift experiment --release <commit> compares the content commit with the last release tag on touched suites. The release script skips Experiments by default. Add --with-experiment to scripts/release.sh on each invocation to run one: it waits for rechecks and asks permission for a fail or inconclusive Verdict before pushing. Reruns with the flag reuse the Experiment, including before opening a pushed branch's PR. It refuses different main content before pushing or merging and checks the merged tree before tagging, including without an Experiment. No touched suite or a last release without the Trial flag carries on with a note. Releasing.

Each Ticket run's own process saves its Run metrics at completion in its Command record, including Ticket runs and Base fixes: outcome, Repairs, CI rounds, Unaddressed findings by Standards, Spec and Security, stage times against their Usual ranges, and minutes by stage and in total. Unaddressed findings counts retain Delivery's final PR body and the Security review's count, so later edits do not change them. Each Session stage keeps the review's Standards and Spec finding counts from its summary, with missing counts left missing. thirdshift show <command> --json includes these retained facts. The Console reads the same details, showing each Ticket run's available metrics and marking unavailable signals as Missing. thirdshift metrics <command> [--json] reads a Ticket run with its children; without a Command it groups weekly trends by repository, Harness, requested Model and Effort, Ticket run kind and session kind. Week and repository filters narrow that view. Historical metrics are computed from Command records without writing back, and missing signals stay missing. No combined score is produced.

Run metrics and Experiments share one Session usage module for tokens by observed Model and Harness-reported cost. Its normalized-event and retained-span readers resolve call amendments and partial totals, retaining fields a total omits. Input includes cache reads and writes. Reported Session cost wins; otherwise each Model's reported cost replaces its call costs. The metrics reader separately counts tool errors, permission denials, killed work and repeated tool calls within that session. Completion metrics sum these by session kind, and weekly Session trends split by the Harness version from each trace's resource. Saved completion metrics stay fixed. Historical metrics also read converted Session traces, leaving missing traces and unreported measurements missing.

Command details opens with the same subject and issue title as the Inbox, before Stop and Dismiss. The subject links to the issue and long titles wrap. Unknown titles say “(title unavailable)”; Commands without an issue show only their Inbox subject. The recorded issue title comes from the same details JSON as thirdshift show <command> --json.

diff --git a/site/docs/index.html b/site/docs/index.html index 9b910529..7083266e 100644 --- a/site/docs/index.html +++ b/site/docs/index.html @@ -1691,7 +1691,11 @@

Releasing
-

Before pushing anything, the release script runs thirdshift experiment --release <commit> using this checkout's runner and private Eval suites. +

The release script skips Experiments by default, printing a note. Opt in with +scripts/release.sh --with-experiment 0.4.0, adding --with-experiment again +on each resume that should run an Experiment.

+

With --with-experiment, before pushing anything, the script runs +thirdshift experiment --release <commit> using this checkout's runner and private Eval suites. It evaluates the content commit beneath the version-only bump, so reruns use the same Experiment even before a bump branch exists. It pins that commit in a local Git ref before starting, so a merge to main during interruption does @@ -1702,17 +1706,14 @@ Only y or yes continues; EOF or any other answer stops. Pending borderline and decisive Scores are listed with their recheck commands and polled until you have rechecked them. No touched suite or a last release with no Trial flag -prints a note and carries on. --no-experiment skips the release Experiment -outright, on the first run and on every resume, and prints a line saying so: -use it only when an Experiment can't give a Verdict worth its Trials, such as -while every Trial is known to be void. An interrupted invocation resumes its saved +prints a note and carries on. An interrupted invocation resumes its saved Trials. A pushed bump branch is checked again before opening its PR, and an open or merged bump PR before merging or tagging. An already pushed tag only resumes the release workflow. Python 3 is needed to read the Experiment results.

If main gains different content after the Candidate was selected, the script -refuses to push or merge that release under its old Experiment; choose a fresh -release version to measure current main. It also checks the merged tree before -tagging, so a concurrent merge cannot publish unmeasured content.

+refuses to push or merge that release; choose a fresh release version for current +main. It also checks the merged tree before tagging, so a concurrent merge +cannot publish different content. These checks apply even without an Experiment.

From a clone of this repo, signed in to gh and the User config's default Harness, run the release script with the new version:

scripts/release.sh 0.4.0
diff --git a/site/how-it-works/index.html b/site/how-it-works/index.html index 0889543f..20a9961c 100644 --- a/site/how-it-works/index.html +++ b/site/how-it-works/index.html @@ -240,7 +240,7 @@

How thirdshift works

Proposed Eval tasks appear on the Experiments list and, for its suites, on each Experiment page. Approve one there or with thirdshift evals approve <suite> <task> --json to change its state in its suite, preserving its inputs. Approval starts no work and affects future Experiments; existing Experiments keep their fixed Eval tasks on resume. Unapproved proposals never run.

Each Trial and each Grader runs in its own memory-capped scope and raises its OOM score to give way before the factory. The User config defaults to 1 GiB for throttling, 2 GiB for killing and 1 GiB of swap per scope. Enable lingering once with loginctl enable-linger <user>; an Experiment stops before dispatch if it cannot reach the systemd user manager.

A Verdict stays waiting on rechecks until you have checked every borderline Score and every decisive Score, whose flip would change the Verdict. The pending list gives Trial IDs for thirdshift recheck <id> <trial> pass|fail --reason '…'. Your latest decision feeds the calculation; the Grader's Score and your timestamped history remain. Each recheck updates the pending list and the latest invocation's Command outcome and Inbox group. A confirmed pass moves from Needs you to Done.

-

thirdshift experiment --release <commit> compares touched suites with the last release tag. The release script waits for rechecks before pushing and asks whether to release on a fail or inconclusive Verdict. Reruns reuse the Experiment, including before opening a pushed branch’s PR. If main gains different content, the script refuses to publish it under the old Candidate's Experiment; choose a fresh release version. No touched suite or a last release with no Trial flag carries on with a note. Release checks.

+

thirdshift experiment --release <commit> compares touched suites with the last release tag. The release script skips Experiments by default. Add --with-experiment to scripts/release.sh on each invocation to run one: it waits for rechecks before pushing and asks whether to release on a fail or inconclusive Verdict. Reruns with the flag reuse the Experiment, including before opening a pushed branch’s PR. If main gains different content, the script refuses to publish it; choose a fresh release version. No touched suite or a last release with no Trial flag carries on with a note. Release checks.

Architect runs every few minutes and Security runs on a schedule, with fixing allowed, started by your operating system.

Next: inspect every prompt and skill, see a real Ticket run's recorded output, see the architecture in six figures, use the manual for installation, commands and configuration, or watch the explainer videos.

diff --git a/site/index.html b/site/index.html index f09d4fd0..4744ee57 100644 --- a/site/index.html +++ b/site/index.html @@ -364,7 +364,7 @@

Six workflows

Each Trial and each Grader has its own memory cap. A Trial its cap kills is void; repeated cap kills are reported for its Arm. The User config sets the caps, with defaults of 1 GiB for throttling, 2 GiB for killing and 1 GiB of swap.

The Verdict waits for your rechecks of borderline and decisive Scores. thirdshift experiments <id> --json lists those waiting; thirdshift recheck <id> <trial> pass --reason '…' records your decision beside the Grader's Score and keeps its history.

Each Trial removes its Harness caches when its session ends. After grading and all rechecks finish, the Experiment removes every attempt's repository copy. Paused Experiments keep repositories for resume; Session evidence, reports, packets and Scores remain readable.

-

thirdshift experiment --release <commit> compares touched suites with the last release tag. The release script waits for rechecks before pushing and asks whether to release on a fail or inconclusive Verdict. Reruns reuse the Experiment, including before opening a pushed branch’s PR. If main gains different content, the script refuses to publish it under the old Candidate's Experiment; choose a fresh release version. No touched suite or a last release with no Trial flag carries on with a note. Release checks.

+

thirdshift experiment --release <commit> compares touched suites with the last release tag. The release script skips Experiments by default. Add --with-experiment to scripts/release.sh on each invocation to run one: it waits for rechecks before pushing and asks whether to release on a fail or inconclusive Verdict. Reruns with the flag reuse the Experiment, including before opening a pushed branch’s PR. If main gains different content, the script refuses to publish it; choose a fresh release version. No touched suite or a last release with no Trial flag carries on with a note. Release checks.

A Scoring run, started with thirdshift score, samples merged factory pull requests across configured GitHub owners, leaving out Security fixes. Each gets a blind packet with its issue and repository at merge time, its diff, pull request and CI conclusions. thirdshift scores --json lists the private samples by week. An independent Model grades each packet on Correct, Scoped, Tested, Honest and Standards, with a pass or fail and a reason for each. Completed Scores are kept; the next Scoring run finishes pending packets after a plan-limit stop. It also proposes Bug origins for closed bugs, excluding Security fixes, using a sandboxed session and history ending at the fix. The first proposal run covers all closed bugs; later runs cover their window. Use thirdshift bug-origins [issue] --json to read them; --confirm, --correct <pull request> or --none settles one. Confirmed origins count against their retained Ticket run in show --json and create proposed code-review Eval tasks with pinned PR diffs and draft catch criteria. Rejected and unsettled origins count for nothing. Eval tasks stay out of Experiments until approved.

Scoring samples, Bug-origin proposals and settlements, and Run metrics share retained pull-request attribution. Matching ignores ASCII letter case and trailing slashes. Latest means chronological RFC 3339 time, including UTC offsets, then greatest lexical record path and Command name for ties; invalid times sort before valid ones. Saved local record-path/name pairs select that exact Command under the matching pull request. Missing or mismatched local identities count for nothing; only absent or cross-host attribution selects the latest local Command, so Continuations never multiply a count.

diff --git a/tests/release/experiments.rs b/tests/release/experiments.rs index 56f162ff..93252430 100644 --- a/tests/release/experiments.rs +++ b/tests/release/experiments.rs @@ -50,7 +50,7 @@ fn calls(release: &Release) -> Vec { fn a_resumed_release_cannot_ship_content_absent_from_its_experiment() { let release = Release::new(); cargo_reports(&release, &[json!(1)]); - let interrupted = release.run_script("0.2.0"); + let interrupted = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(!interrupted.status.success()); release.merge_pr( "arrived-during-pause", @@ -60,7 +60,7 @@ fn a_resumed_release_cannot_ship_content_absent_from_its_experiment() { release.main_ci("completed", "success"); release.ci_reports(GREEN); cargo_reports(&release, &[verdict("pass")]); - let resumed = release.run_script("0.2.0"); + let resumed = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(!resumed.status.success()); assert!( stderr(&resumed).contains("main contains content absent from the measured Candidate"), @@ -80,7 +80,7 @@ fn a_merged_release_cannot_tag_content_absent_from_its_experiment() { release.merge_bump_pr("0.2.0", &head); cargo_reports(&release, &[verdict("pass")]); - let resumed = release.run_script("0.2.0"); + let resumed = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(!resumed.status.success()); assert!( @@ -94,21 +94,27 @@ fn a_merged_release_cannot_tag_content_absent_from_its_experiment() { #[test] fn a_passing_experiment_is_checked_before_any_push() { - let release = Release::new(); - cargo_reports(&release, &[verdict("pass")]); - release.ci_reports(GREEN); - let candidate = release.origin(&["rev-parse", "main"]); - let output = release.run_script("0.2.0"); - assert!(output.status.success(), "{}", stderr(&output)); - assert!(unstamped(&output).contains(&format!("Experiment {ID}: pass"))); - let calls = calls(&release); - assert_eq!(calls.len(), 1); - assert_eq!( - calls[0]["args"], - json!(["experiment", "--release", candidate, "--json"]) - ); - assert_eq!(calls[0]["branch"], ""); - assert!(release.origin_tag("v0.2.0").is_some()); + for args in [ + &["--with-experiment", "0.2.0"][..], + &["--review", "--with-experiment", "0.2.0"][..], + &["--with-experiment", "--review", "0.2.0"][..], + ] { + let release = Release::new(); + cargo_reports(&release, &[verdict("pass")]); + release.ci_reports(GREEN); + let candidate = release.origin(&["rev-parse", "main"]); + let output = release.run_script_with(args, "y\n", &[]); + assert!(output.status.success(), "{args:?}: {}", stderr(&output)); + assert!(unstamped(&output).contains(&format!("Experiment {ID}: pass"))); + let calls = calls(&release); + assert_eq!(calls.len(), 1); + assert_eq!( + calls[0]["args"], + json!(["experiment", "--release", candidate, "--json"]) + ); + assert_eq!(calls[0]["branch"], ""); + assert!(release.origin_tag("v0.2.0").is_some()); + } } #[test] @@ -119,7 +125,7 @@ fn failed_or_inconclusive_experiments_show_intervals_and_require_permission() { let before = release.origin_refs(); for answer in ["", "n\n", "yes\n"] { cargo_reports(&release, &[verdict(outcome)]); - let output = release.run_script_with(&["0.2.0"], answer, &[]); + let output = release.run_script_with(&["--with-experiment", "0.2.0"], answer, &[]); let err = unstamped(&output); assert!(err.contains("release anyway?"), "{err}"); assert!(err.contains("accuracy [-0.25, 0.12]"), "{err}"); @@ -151,7 +157,7 @@ fn pending_rechecks_are_listed_and_waited_for_before_pushing() { "trial": 0, "borderline": true, "decisive": false }]); cargo_reports(&release, &[waiting.clone(), waiting, verdict("pass")]); - let output = release.run_script("0.2.0"); + let output = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(output.status.success(), "{}", stderr(&output)); let err = unstamped(&output); assert!(err.contains("trial-1"), "{err}"); @@ -171,17 +177,17 @@ fn resume_reuses_the_content_commit_and_checks_a_pushed_branch_before_opening_it let candidate = release.origin(&["rev-parse", "main"]); let before = release.origin_refs(); cargo_reports(&release, &[json!(1)]); - let stopped = release.run_script("0.2.0"); + let stopped = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(!stopped.status.success()); assert_eq!(release.origin_refs(), before); release.push_bump("0.2.0"); cargo_reports(&release, &[verdict("fail")]); - let stopped = release.run_script("0.2.0"); + let stopped = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(!stopped.status.success()); assert!(!release.pr_opened("release-0.2.0")); cargo_reports(&release, &[verdict("pass")]); release.ci_reports(GREEN); - let resumed = release.run_script("0.2.0"); + let resumed = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(resumed.status.success(), "{}", stderr(&resumed)); let checked = calls(&release); assert_eq!(checked.len(), 3); @@ -196,12 +202,8 @@ fn resume_reuses_the_content_commit_and_checks_a_pushed_branch_before_opening_it } #[test] -fn no_experiment_releases_without_running_one_and_says_so() { - for args in [ - &["--no-experiment", "0.2.0"][..], - &["--review", "--no-experiment", "0.2.0"][..], - &["--no-experiment", "--review", "0.2.0"][..], - ] { +fn releases_skip_experiments_by_default_and_say_so() { + for args in [&["0.2.0"][..], &["--review", "0.2.0"][..]] { let release = Release::new(); cargo_reports(&release, &[]); release.ci_reports(GREEN); @@ -209,7 +211,7 @@ fn no_experiment_releases_without_running_one_and_says_so() { let err = unstamped(&output); assert!(output.status.success(), "{args:?}: {err}"); assert!( - err.contains("skipping the release Experiment (--no-experiment)"), + err.contains("skipping the release Experiment (use --with-experiment to run it)"), "{err}" ); assert!(calls(&release).is_empty(), "{args:?} ran an Experiment"); @@ -217,6 +219,48 @@ fn no_experiment_releases_without_running_one_and_says_so() { } } +#[test] +fn resumed_releases_skip_experiments_by_default() { + for state in ["pushed_branch", "open_pr", "merged_pr"] { + let release = Release::new(); + cargo_reports(&release, &[]); + release.ci_reports(GREEN); + let head = release.push_bump("0.2.0"); + if state != "pushed_branch" { + release.open_bump_pr("0.2.0"); + } + if state == "merged_pr" { + release.merge_bump_pr("0.2.0", &head); + } + + let output = release.run_script("0.2.0"); + let err = unstamped(&output); + assert!(output.status.success(), "{state}: {err}"); + assert!( + err.contains("skipping the release Experiment (use --with-experiment to run it)"), + "{state}: {err}" + ); + assert!(calls(&release).is_empty(), "{state} ran an Experiment"); + assert!(release.origin_tag("v0.2.0").is_some()); + } +} + +#[test] +fn an_interrupted_experiment_can_be_skipped_on_resume() { + let release = Release::new(); + cargo_reports(&release, &[json!(1)]); + let interrupted = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); + assert!(!interrupted.status.success()); + assert_eq!(calls(&release).len(), 1); + + cargo_reports(&release, &[]); + release.ci_reports(GREEN); + let resumed = release.run_script("0.2.0"); + assert!(resumed.status.success(), "{}", stderr(&resumed)); + assert_eq!(calls(&release).len(), 1, "resume ran another Experiment"); + assert!(release.origin_tag("v0.2.0").is_some()); +} + #[test] fn skipped_experiments_carry_on_with_a_note() { for (code, reason) in [ @@ -229,7 +273,7 @@ fn skipped_experiments_carry_on_with_a_note() { let release = Release::new(); cargo_reports(&release, &[json!({"release_skip": code, "reason": reason})]); release.ci_reports(GREEN); - let output = release.run_script("0.2.0"); + let output = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(output.status.success(), "{}", stderr(&output)); assert!(unstamped(&output).contains(reason), "{}", stderr(&output)); assert!(release.origin_tag("v0.2.0").is_some()); @@ -244,7 +288,7 @@ fn interrupted_release_before_push_keeps_its_experiment_when_main_moves() { // Stand in for an Experiment interrupted before any release branch exists. cargo_reports(&release, &[json!(1)]); - let interrupted = release.run_script("0.2.0"); + let interrupted = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(!interrupted.status.success(), "{}", stderr(&interrupted)); assert_eq!(release.origin_refs(), before); assert!(!release.origin_branch("release-0.2.0")); @@ -259,7 +303,7 @@ fn interrupted_release_before_push_keeps_its_experiment_when_main_moves() { // Stop again at the Experiment call so later release steps cannot interfere. cargo_reports(&release, &[json!(1)]); - let resumed = release.run_script("0.2.0"); + let resumed = release.run_script_with(&["--with-experiment", "0.2.0"], "", &[]); assert!(!resumed.status.success(), "{}", stderr(&resumed)); let checked = calls(&release); assert_eq!(checked.len(), 2);