From bf5dcc5bdb5d9b3738abcabf368f7b48be31e55a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Mon, 26 Oct 2026 08:48:27 +0000 Subject: [PATCH 1/5] acc: add CLI-only test for clearing a Genie space description bundle/invariant/no_drift with genie_space.yml.tmpl flakes on cloud: after an update that clears the description, the next GET still returns the old one. Replay the direct engine's create/update requests with plain genie commands and poll GET to tell a stale read from a lost update. Co-authored-by: Isaac --- .../genie-clear-description/out.test.toml | 1 + .../genie-clear-description/output.txt | 4 + .../workspace/genie-clear-description/script | 93 +++++++++++++++++++ .../genie-clear-description/test.toml | 1 + 4 files changed, 99 insertions(+) create mode 100644 acceptance/cmd/workspace/genie-clear-description/out.test.toml create mode 100644 acceptance/cmd/workspace/genie-clear-description/output.txt create mode 100644 acceptance/cmd/workspace/genie-clear-description/script create mode 100644 acceptance/cmd/workspace/genie-clear-description/test.toml diff --git a/acceptance/cmd/workspace/genie-clear-description/out.test.toml b/acceptance/cmd/workspace/genie-clear-description/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/workspace/genie-clear-description/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/workspace/genie-clear-description/output.txt b/acceptance/cmd/workspace/genie-clear-description/output.txt new file mode 100644 index 00000000000..a32b4cb5ff5 --- /dev/null +++ b/acceptance/cmd/workspace/genie-clear-description/output.txt @@ -0,0 +1,4 @@ +round 1: CONVERGED +round 2: CONVERGED +round 3: CONVERGED +round 4: CONVERGED diff --git a/acceptance/cmd/workspace/genie-clear-description/script b/acceptance/cmd/workspace/genie-clear-description/script new file mode 100644 index 00000000000..6186c47bc40 --- /dev/null +++ b/acceptance/cmd/workspace/genie-clear-description/script @@ -0,0 +1,93 @@ +# Clearing a Genie space description sometimes does not stick on cloud: right after an update that +# sends description="", a GET can still return the old value (seen as a flaky drift in +# bundle/invariant/no_drift with genie_space.yml.tmpl). This test replays the same requests the +# direct engine sends, using plain CLI commands, and polls GET afterwards. The poll pattern tells +# a stale read (old value, then cleared for good) from a lost update (cleared, then the old value +# comes back), e.g. from the backend migrating serialized_space v1 -> v2 in the background. +# +# Expected output is CONVERGED for every round; anything else prints the timeline. + +me=$($CLI current-user me -o json | jq -r .userName) +parent_path="/Workspace/Users/$me/genie-clear-description-$UNIQUE_NAME" +$CLI workspace mkdirs "$parent_path" + +space_ids=() +cleanup() { + for id in "${space_ids[@]}"; do + $CLI genie trash-space "$id" > /dev/null + done + $CLI workspace delete --recursive "$parent_path" +} +trap cleanup EXIT + +if [ -n "${CLOUD_ENV:-}" ]; then + delays="0 3 6 10" + polls=15 + poll_interval=2 +else + delays="0 0 0 0" + polls=3 + poll_interval=0 +fi + +round=0 +for delay in $delays; do + round=$((round + 1)) + timeline="LOG.timeline.$round" + + # Same create body as the direct engine (ResourceGenieSpace.DoCreate). + space_id=$($CLI genie create-space -o json --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"title\": \"test-genie-space-$UNIQUE_NAME-$round\", + \"description\": \"This is a test genie space\", + \"parent_path\": \"$parent_path\", + \"serialized_space\": \"{\\\"version\\\":1}\" + }" | jq -r .space_id) + space_ids+=("$space_id") + start=$SECONDS + + sleep "$delay" + + # Same update body as ResourceGenieSpace.DoUpdate after the description is removed from config: + # every field is sent, description is force-sent as empty, etag is omitted. + update_desc=$($CLI genie update-space "$space_id" -o json --json "{ + \"description\": \"\", + \"title\": \"test-genie-space-$UNIQUE_NAME-$round\", + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"parent_path\": \"$parent_path\", + \"serialized_space\": \"{\\\"version\\\":1}\" + }" | jq -r '.description // ""') + echo "t=$((SECONDS - start)) update-response desc='$update_desc'" > "$timeline" + + # One letter per poll: C = description cleared, O = old description returned. + pattern="" + for i in $(seq "$polls"); do + $CLI genie get-space "$space_id" --include-serialized-space -o json > LOG.get + desc=$(jq -r '.description // ""' LOG.get) + version=$(jq -r '.serialized_space // "{}" | fromjson | .version' LOG.get) + etag=$(jq -r '.etag // ""' LOG.get) + echo "t=$((SECONDS - start)) get desc='$desc' serialized_space.version=$version etag=$etag" >> "$timeline" + if [ -z "$desc" ]; then + pattern="${pattern}C" + else + pattern="${pattern}O" + fi + sleep "$poll_interval" + done + + case "$pattern" in + *C*O*) result="LOST_UPDATE" ;; + *C*) [[ "$pattern" == C* ]] && result="CONVERGED" || result="STALE_READ" ;; + *) result="NOT_APPLIED" ;; + esac + if [ -n "$update_desc" ]; then + result="$result (update response still had the old description)" + fi + + echo "round $round: $result" + if [ "$result" != "CONVERGED" ]; then + echo " delay before update: ${delay}s, polls: $pattern" + sed 's/^/ /' "$timeline" + fi +done +rm -f LOG.get diff --git a/acceptance/cmd/workspace/genie-clear-description/test.toml b/acceptance/cmd/workspace/genie-clear-description/test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/workspace/genie-clear-description/test.toml @@ -0,0 +1 @@ +Cloud = true From 6a97c8a73e38512f11140365e1e705cff5dc6714 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 28 Oct 2026 09:57:31 +0000 Subject: [PATCH 2/5] acc: disable MSYS path conversion in genie-clear-description Co-authored-by: Isaac --- acceptance/cmd/workspace/genie-clear-description/script | 3 +++ 1 file changed, 3 insertions(+) diff --git a/acceptance/cmd/workspace/genie-clear-description/script b/acceptance/cmd/workspace/genie-clear-description/script index 6186c47bc40..f0bbb4c698d 100644 --- a/acceptance/cmd/workspace/genie-clear-description/script +++ b/acceptance/cmd/workspace/genie-clear-description/script @@ -7,6 +7,9 @@ # # Expected output is CONVERGED for every round; anything else prints the timeline. +# Keep Git Bash on Windows from rewriting /Workspace/... into a C:/Program Files/Git/... path. +export MSYS_NO_PATHCONV=1 + me=$($CLI current-user me -o json | jq -r .userName) parent_path="/Workspace/Users/$me/genie-clear-description-$UNIQUE_NAME" $CLI workspace mkdirs "$parent_path" From 7e5526f37db1238b7b45377452bc54fc7a499576 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 29 Oct 2026 15:57:11 +0000 Subject: [PATCH 3/5] acc: replay permissions set and plan read in genie-clear-description Co-authored-by: Isaac --- acceptance/cmd/workspace/genie-clear-description/script | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/acceptance/cmd/workspace/genie-clear-description/script b/acceptance/cmd/workspace/genie-clear-description/script index f0bbb4c698d..b68a5d9d900 100644 --- a/acceptance/cmd/workspace/genie-clear-description/script +++ b/acceptance/cmd/workspace/genie-clear-description/script @@ -49,6 +49,11 @@ for delay in $delays; do space_ids+=("$space_id") start=$SECONDS + # The bundle deploy sets permissions right after creating the space, and the drift-check plan + # reads the space before the field-removal update; replay both. + $CLI permissions set genie "$space_id" --json '{"access_control_list": [{"group_name": "users", "permission_level": "CAN_READ"}]}' > /dev/null + $CLI genie get-space "$space_id" --include-serialized-space > /dev/null + sleep "$delay" # Same update body as ResourceGenieSpace.DoUpdate after the description is removed from config: From f8d742102005374ffb107a847a2435da23101ba4 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 30 Oct 2026 09:27:55 +0000 Subject: [PATCH 4/5] acc: run 30 parallel rounds in genie-clear-description probe Co-authored-by: Isaac --- .../genie-clear-description/output.txt | 26 ++++++ .../workspace/genie-clear-description/script | 83 ++++++++++++------- 2 files changed, 77 insertions(+), 32 deletions(-) diff --git a/acceptance/cmd/workspace/genie-clear-description/output.txt b/acceptance/cmd/workspace/genie-clear-description/output.txt index a32b4cb5ff5..18595383013 100644 --- a/acceptance/cmd/workspace/genie-clear-description/output.txt +++ b/acceptance/cmd/workspace/genie-clear-description/output.txt @@ -2,3 +2,29 @@ round 1: CONVERGED round 2: CONVERGED round 3: CONVERGED round 4: CONVERGED +round 5: CONVERGED +round 6: CONVERGED +round 7: CONVERGED +round 8: CONVERGED +round 9: CONVERGED +round 10: CONVERGED +round 11: CONVERGED +round 12: CONVERGED +round 13: CONVERGED +round 14: CONVERGED +round 15: CONVERGED +round 16: CONVERGED +round 17: CONVERGED +round 18: CONVERGED +round 19: CONVERGED +round 20: CONVERGED +round 21: CONVERGED +round 22: CONVERGED +round 23: CONVERGED +round 24: CONVERGED +round 25: CONVERGED +round 26: CONVERGED +round 27: CONVERGED +round 28: CONVERGED +round 29: CONVERGED +round 30: CONVERGED diff --git a/acceptance/cmd/workspace/genie-clear-description/script b/acceptance/cmd/workspace/genie-clear-description/script index b68a5d9d900..be7213b386a 100644 --- a/acceptance/cmd/workspace/genie-clear-description/script +++ b/acceptance/cmd/workspace/genie-clear-description/script @@ -1,9 +1,8 @@ -# Clearing a Genie space description sometimes does not stick on cloud: right after an update that -# sends description="", a GET can still return the old value (seen as a flaky drift in -# bundle/invariant/no_drift with genie_space.yml.tmpl). This test replays the same requests the -# direct engine sends, using plain CLI commands, and polls GET afterwards. The poll pattern tells -# a stale read (old value, then cleared for good) from a lost update (cleared, then the old value -# comes back), e.g. from the backend migrating serialized_space v1 -> v2 in the background. +# Probe, not meant to be merged: clearing a Genie space description sometimes does not stick on +# cloud (flaky drift in bundle/invariant/no_drift with genie_space.yml.tmpl, ~3% of CI runs). This +# replays the requests the direct engine sends, using plain CLI commands, and polls GET afterwards. +# The poll pattern tells a stale read (old value, then cleared for good) from a lost update +# (cleared, then the old value comes back). # # Expected output is CONVERGED for every round; anything else prints the timeline. @@ -14,40 +13,40 @@ me=$($CLI current-user me -o json | jq -r .userName) parent_path="/Workspace/Users/$me/genie-clear-description-$UNIQUE_NAME" $CLI workspace mkdirs "$parent_path" -space_ids=() cleanup() { - for id in "${space_ids[@]}"; do + for id in $(cat LOG.space_ids 2>/dev/null); do $CLI genie trash-space "$id" > /dev/null done $CLI workspace delete --recursive "$parent_path" } trap cleanup EXIT +# The flake is rare, so take many samples: rounds run in parallel batches. +rounds=30 +batch=10 if [ -n "${CLOUD_ENV:-}" ]; then - delays="0 3 6 10" polls=15 - poll_interval=2 + poll_interval=3 else - delays="0 0 0 0" polls=3 poll_interval=0 fi -round=0 -for delay in $delays; do - round=$((round + 1)) - timeline="LOG.timeline.$round" +run_round() { + local round=$1 delay=$2 + local timeline="LOG.timeline.$round" title="test-genie-space-$UNIQUE_NAME-$round" # Same create body as the direct engine (ResourceGenieSpace.DoCreate). + local space_id space_id=$($CLI genie create-space -o json --json "{ \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", - \"title\": \"test-genie-space-$UNIQUE_NAME-$round\", + \"title\": \"$title\", \"description\": \"This is a test genie space\", \"parent_path\": \"$parent_path\", \"serialized_space\": \"{\\\"version\\\":1}\" }" | jq -r .space_id) - space_ids+=("$space_id") - start=$SECONDS + echo "$space_id" >> LOG.space_ids + local start=$SECONDS # The bundle deploy sets permissions right after creating the space, and the drift-check plan # reads the space before the field-removal update; replay both. @@ -58,22 +57,24 @@ for delay in $delays; do # Same update body as ResourceGenieSpace.DoUpdate after the description is removed from config: # every field is sent, description is force-sent as empty, etag is omitted. - update_desc=$($CLI genie update-space "$space_id" -o json --json "{ + $CLI genie update-space "$space_id" -o json --json "{ \"description\": \"\", - \"title\": \"test-genie-space-$UNIQUE_NAME-$round\", + \"title\": \"$title\", \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", \"parent_path\": \"$parent_path\", \"serialized_space\": \"{\\\"version\\\":1}\" - }" | jq -r '.description // ""') - echo "t=$((SECONDS - start)) update-response desc='$update_desc'" > "$timeline" + }" > "LOG.upd.$round" + local update_desc + update_desc=$(jq -r '.description // ""' "LOG.upd.$round") + echo "t=$((SECONDS - start)) update-response desc='$update_desc' etag=$(jq -r '.etag // ""' "LOG.upd.$round")" > "$timeline" # One letter per poll: C = description cleared, O = old description returned. - pattern="" + local pattern="" desc version etag for i in $(seq "$polls"); do - $CLI genie get-space "$space_id" --include-serialized-space -o json > LOG.get - desc=$(jq -r '.description // ""' LOG.get) - version=$(jq -r '.serialized_space // "{}" | fromjson | .version' LOG.get) - etag=$(jq -r '.etag // ""' LOG.get) + $CLI genie get-space "$space_id" --include-serialized-space -o json > "LOG.get.$round" + desc=$(jq -r '.description // ""' "LOG.get.$round") + version=$(jq -r '.serialized_space // "{}" | fromjson | .version' "LOG.get.$round") + etag=$(jq -r '.etag // ""' "LOG.get.$round") echo "t=$((SECONDS - start)) get desc='$desc' serialized_space.version=$version etag=$etag" >> "$timeline" if [ -z "$desc" ]; then pattern="${pattern}C" @@ -82,7 +83,9 @@ for delay in $delays; do fi sleep "$poll_interval" done + rm -f "LOG.get.$round" "LOG.upd.$round" + local result case "$pattern" in *C*O*) result="LOST_UPDATE" ;; *C*) [[ "$pattern" == C* ]] && result="CONVERGED" || result="STALE_READ" ;; @@ -92,10 +95,26 @@ for delay in $delays; do result="$result (update response still had the old description)" fi - echo "round $round: $result" - if [ "$result" != "CONVERGED" ]; then - echo " delay before update: ${delay}s, polls: $pattern" - sed 's/^/ /' "$timeline" + { + echo "round $round: $result" + if [ "$result" != "CONVERGED" ]; then + echo " delay before update: ${delay}s, polls: $pattern" + sed 's/^/ /' "$timeline" + fi + } > "LOG.result.$round" +} + +for round in $(seq "$rounds"); do + # Spread the create->update delay; in the bundle flow plans and uploads sit in between. + delay=0 + [ -n "${CLOUD_ENV:-}" ] && delay=$(( (round - 1) % 4 * 4 )) + run_round "$round" "$delay" & + if (( round % batch == 0 )); then + wait fi done -rm -f LOG.get +wait + +for round in $(seq "$rounds"); do + cat "LOG.result.$round" +done From 74582610675a731e3168bdc508e741e522080501 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 30 Oct 2026 10:08:45 +0000 Subject: [PATCH 5/5] acc: replay the missing-parent create path in the genie probe Co-authored-by: Isaac --- .../workspace/genie-clear-description/script | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/acceptance/cmd/workspace/genie-clear-description/script b/acceptance/cmd/workspace/genie-clear-description/script index be7213b386a..04c03ade6c0 100644 --- a/acceptance/cmd/workspace/genie-clear-description/script +++ b/acceptance/cmd/workspace/genie-clear-description/script @@ -36,15 +36,23 @@ run_round() { local round=$1 delay=$2 local timeline="LOG.timeline.$round" title="test-genie-space-$UNIQUE_NAME-$round" - # Same create body as the direct engine (ResourceGenieSpace.DoCreate). - local space_id - space_id=$($CLI genie create-space -o json --json "{ + # Same create body and missing-parent handling as ResourceGenieSpace.DoCreate. Each round gets a + # parent folder that does not exist yet, as on a bundle's first deploy: the first create fails, + # then the folder is created and the create retried. + local parent="$parent_path/r$round" body space_id + body="{ \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", \"title\": \"$title\", \"description\": \"This is a test genie space\", - \"parent_path\": \"$parent_path\", + \"parent_path\": \"$parent\", \"serialized_space\": \"{\\\"version\\\":1}\" - }" | jq -r .space_id) + }" + if ! $CLI genie create-space -o json --json "$body" > "LOG.create.$round" 2> "LOG.create.err.$round"; then + $CLI workspace mkdirs "$parent" + $CLI genie create-space -o json --json "$body" > "LOG.create.$round" + fi + space_id=$(jq -r .space_id "LOG.create.$round") + rm -f "LOG.create.$round" "LOG.create.err.$round" echo "$space_id" >> LOG.space_ids local start=$SECONDS @@ -61,7 +69,7 @@ run_round() { \"description\": \"\", \"title\": \"$title\", \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", - \"parent_path\": \"$parent_path\", + \"parent_path\": \"$parent\", \"serialized_space\": \"{\\\"version\\\":1}\" }" > "LOG.upd.$round" local update_desc