Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions crates/buzz-backend-kubernetes/src/sandbox.rs
Original file line number Diff line number Diff line change
Expand Up @@ -841,6 +841,31 @@ fn container_started(pod: &Pod) -> bool {
})
}

/// Why the controller has not created the Sandbox's Pod, from its `Ready`
/// condition. A quota refusal is the usual cause: say which limit is full.
fn pod_not_created(sandbox: &DynamicObject) -> String {
let message = sandbox.data["status"]["conditions"]
.as_array()
.and_then(|conditions| {
conditions
.iter()
.find(|c| c["type"] == "Ready" && c["status"] != "True")
})
.and_then(|ready| ready["message"].as_str())
.unwrap_or_default();
if message.contains("exceeded quota: primary-sessions") {
"All of your agent sessions are in use. End an idle session, or wait for one to finish, then retry".into()
} else if let Some(quota) = message
.split("exceeded quota: ")
.nth(1)
.and_then(|rest| rest.split(',').next())
{
format!("Your agent namespace quota `{quota}` is full. End an idle session, or wait for one to finish, then retry")
} else {
"Sandbox controller has not created its Pod; inspect the Sandbox and retry".into()
}
}

/// Bind credentials to the Sandbox's first Pod and wait for it to start.
///
/// Resumable: a session bound to its first Pod whose Secret is missing and
Expand Down Expand Up @@ -873,10 +898,9 @@ async fn bind(
));
}
if expired {
return Err(
"Sandbox controller has not created its Pod; inspect the Sandbox and retry"
.into(),
);
// The controller records why it could not create the Pod.
let current = session.sandboxes.get_opt(name).await.ok().flatten();
return Err(pod_not_created(current.as_ref().unwrap_or(&sandbox)).into());
}
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
Expand Down
33 changes: 33 additions & 0 deletions crates/buzz-backend-kubernetes/src/sandbox_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1418,3 +1418,36 @@ fn direct_messages_have_one_isolated_workspace_and_owner_gated_conversation_env(
assert!(parse(invalid).is_err());
}
}

#[test]
fn a_pod_the_quota_refused_names_the_full_limit() {
let sandbox = |ready: serde_json::Value| -> DynamicObject {
serde_json::from_value(serde_json::json!({
"apiVersion": "agents.x-k8s.io/v1alpha1", "kind": "Sandbox",
"metadata": {"name": "buzz-agent-a"},
"status": {"conditions": [
{"type": "Suspended", "status": "False", "message": "Sandbox is not suspended"},
ready,
]},
}))
.unwrap()
};
// The controller's message from staging.
let sessions = sandbox(serde_json::json!({"type": "Ready", "status": "False",
"reason": "ReconcilerError",
"message": "Error seen: pods \"buzz-agent-a\" is forbidden: exceeded quota: primary-sessions, requested: pods=1, used: pods=5, limited: pods=5"}));
assert!(pod_not_created(&sessions).starts_with("All of your agent sessions are in use"));
let capacity = sandbox(serde_json::json!({"type": "Ready", "status": "False",
"message": "pods \"x\" is forbidden: exceeded quota: agent-capacity, requested: requests.cpu=2"}));
assert!(pod_not_created(&capacity).contains("quota `agent-capacity` is full"));
for unknown in [
sandbox(
serde_json::json!({"type": "Ready", "status": "False", "message": "image pull backoff"}),
),
sandbox(
serde_json::json!({"type": "Ready", "status": "True", "message": "exceeded quota: primary-sessions"}),
),
] {
assert!(pod_not_created(&unknown).starts_with("Sandbox controller has not created its Pod"));
}
}
Loading