mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-31 07:44:57 +08:00
fix(cli): clarify gpu sandbox timeout errors
This commit is contained in:
+116
-16
@@ -62,6 +62,47 @@ fn phase_name(phase: i32) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
fn ready_false_condition_message(
|
||||
status: Option<&navigator_core::proto::SandboxStatus>,
|
||||
) -> Option<String> {
|
||||
let condition = status?.conditions.iter().find(|condition| {
|
||||
condition.r#type == "Ready" && condition.status.eq_ignore_ascii_case("false")
|
||||
})?;
|
||||
|
||||
if condition.message.is_empty() {
|
||||
if condition.reason.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(condition.reason.clone())
|
||||
}
|
||||
} else if condition.reason.is_empty() {
|
||||
Some(condition.message.clone())
|
||||
} else {
|
||||
Some(format!("{}: {}", condition.reason, condition.message))
|
||||
}
|
||||
}
|
||||
|
||||
fn provisioning_timeout_message(
|
||||
timeout_secs: u64,
|
||||
requested_gpu: bool,
|
||||
condition_message: Option<&str>,
|
||||
) -> String {
|
||||
let mut message = format!("sandbox provisioning timed out after {timeout_secs}s");
|
||||
|
||||
if let Some(condition_message) = condition_message.filter(|msg| !msg.is_empty()) {
|
||||
message.push_str(". Last reported status: ");
|
||||
message.push_str(condition_message);
|
||||
}
|
||||
|
||||
if requested_gpu {
|
||||
message.push_str(
|
||||
". Hint: this may be because the gateway's available GPU is already in use by another sandbox.",
|
||||
);
|
||||
}
|
||||
|
||||
message
|
||||
}
|
||||
|
||||
/// Format milliseconds since Unix epoch as a `YYYY-MM-DD HH:MM:SS` UTC string.
|
||||
fn format_epoch_ms(ms: i64) -> String {
|
||||
use std::time::UNIX_EPOCH;
|
||||
@@ -296,11 +337,6 @@ impl ProvisioningDisplay {
|
||||
let _ = self.mp.println(msg);
|
||||
}
|
||||
|
||||
/// Return a description of the current active step for error messages.
|
||||
fn current_step_description(&self) -> &str {
|
||||
&self.active_label
|
||||
}
|
||||
|
||||
/// Clear all progress output (spinner, spacer, and completed step lines).
|
||||
fn clear(&mut self) {
|
||||
self.spacer.finish_and_clear();
|
||||
@@ -1953,6 +1989,7 @@ pub async fn sandbox_create(
|
||||
|
||||
let mut last_phase = sandbox.phase;
|
||||
let mut last_error_reason = String::new();
|
||||
let mut last_condition_message = ready_false_condition_message(sandbox.status.as_ref());
|
||||
// Track whether we have seen a non-Ready phase during the watch.
|
||||
let mut saw_non_ready = SandboxPhase::try_from(sandbox.phase) != Ok(SandboxPhase::Ready);
|
||||
let start_time = Instant::now();
|
||||
@@ -1963,18 +2000,16 @@ pub async fn sandbox_create(
|
||||
while let Some(item) = stream.next().await {
|
||||
// Check for timeout
|
||||
if start_time.elapsed() > provision_timeout {
|
||||
let timeout_message = provisioning_timeout_message(
|
||||
provision_timeout.as_secs(),
|
||||
requested_gpu,
|
||||
last_condition_message.as_deref(),
|
||||
);
|
||||
if let Some(d) = display.as_mut() {
|
||||
let step_desc = d.current_step_description().to_string();
|
||||
d.finish_error(&format!(
|
||||
"Timed out after {}s (stuck at: {step_desc})",
|
||||
provision_timeout.as_secs()
|
||||
));
|
||||
d.finish_error(&timeout_message);
|
||||
}
|
||||
println!();
|
||||
return Err(miette::miette!(
|
||||
"sandbox provisioning timed out after {}s",
|
||||
provision_timeout.as_secs()
|
||||
));
|
||||
return Err(miette::miette!(timeout_message));
|
||||
}
|
||||
|
||||
let evt = item.into_diagnostic()?;
|
||||
@@ -1982,6 +2017,9 @@ pub async fn sandbox_create(
|
||||
Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(s)) => {
|
||||
let phase = SandboxPhase::try_from(s.phase).unwrap_or(SandboxPhase::Unknown);
|
||||
last_phase = s.phase;
|
||||
if let Some(message) = ready_false_condition_message(s.status.as_ref()) {
|
||||
last_condition_message = Some(message);
|
||||
}
|
||||
|
||||
if phase != SandboxPhase::Ready {
|
||||
saw_non_ready = true;
|
||||
@@ -4221,8 +4259,8 @@ mod tests {
|
||||
GatewayControlTarget, TlsOptions, format_gateway_select_header,
|
||||
format_gateway_select_items, gateway_auth_label, gateway_select_with, gateway_type_label,
|
||||
git_sync_files, http_health_check, image_requests_gpu, inferred_provider_type,
|
||||
parse_credential_pairs, resolve_gateway_control_target_from, sandbox_should_persist,
|
||||
source_requests_gpu,
|
||||
parse_credential_pairs, provisioning_timeout_message, ready_false_condition_message,
|
||||
resolve_gateway_control_target_from, sandbox_should_persist, source_requests_gpu,
|
||||
};
|
||||
use crate::TEST_ENV_LOCK;
|
||||
use hyper::StatusCode;
|
||||
@@ -4235,6 +4273,7 @@ mod tests {
|
||||
use std::thread;
|
||||
|
||||
use openshell_bootstrap::GatewayMetadata;
|
||||
use openshell_core::proto::{SandboxCondition, SandboxStatus};
|
||||
|
||||
struct EnvVarGuard {
|
||||
key: &'static str,
|
||||
@@ -4427,6 +4466,67 @@ mod tests {
|
||||
assert!(!source_requests_gpu("base"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_false_condition_message_prefers_reason_and_message() {
|
||||
let status = SandboxStatus {
|
||||
sandbox_name: "gpu".to_string(),
|
||||
agent_pod: "gpu-pod".to_string(),
|
||||
agent_fd: String::new(),
|
||||
sandbox_fd: String::new(),
|
||||
conditions: vec![SandboxCondition {
|
||||
r#type: "Ready".to_string(),
|
||||
status: "False".to_string(),
|
||||
reason: "Unschedulable".to_string(),
|
||||
message: "Another GPU sandbox may already be using the available GPU.".to_string(),
|
||||
last_transition_time: String::new(),
|
||||
}],
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
ready_false_condition_message(Some(&status)).as_deref(),
|
||||
Some("Unschedulable: Another GPU sandbox may already be using the available GPU.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_false_condition_message_ignores_non_ready_conditions() {
|
||||
let status = SandboxStatus {
|
||||
sandbox_name: "gpu".to_string(),
|
||||
agent_pod: "gpu-pod".to_string(),
|
||||
agent_fd: String::new(),
|
||||
sandbox_fd: String::new(),
|
||||
conditions: vec![SandboxCondition {
|
||||
r#type: "Scheduled".to_string(),
|
||||
status: "True".to_string(),
|
||||
reason: "Scheduled".to_string(),
|
||||
message: "Sandbox scheduled".to_string(),
|
||||
last_transition_time: String::new(),
|
||||
}],
|
||||
};
|
||||
|
||||
assert!(ready_false_condition_message(Some(&status)).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provisioning_timeout_message_includes_condition_and_gpu_hint() {
|
||||
let message = provisioning_timeout_message(
|
||||
120,
|
||||
true,
|
||||
Some("DependenciesNotReady: Pod exists with phase: Pending; Service Exists"),
|
||||
);
|
||||
|
||||
assert!(message.contains("sandbox provisioning timed out after 120s"));
|
||||
assert!(message.contains("Last reported status: DependenciesNotReady: Pod exists with phase: Pending; Service Exists"));
|
||||
assert!(message.contains("available GPU is already in use by another sandbox"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provisioning_timeout_message_omits_gpu_hint_for_non_gpu_requests() {
|
||||
let message = provisioning_timeout_message(120, false, None);
|
||||
|
||||
assert_eq!(message, "sandbox provisioning timed out after 120s");
|
||||
}
|
||||
|
||||
fn init_git_repo(path: &Path) {
|
||||
let status = Command::new("git")
|
||||
.args(["init"])
|
||||
|
||||
Reference in New Issue
Block a user