diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 3d0c7ebdb..df23c27bc 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -62,6 +62,47 @@ fn phase_name(phase: i32) -> &'static str { } } +fn ready_false_condition_message( + status: Option<&navigator_core::proto::SandboxStatus>, +) -> Option { + let condition = status?.conditions.iter().find(|condition| { + condition.r#type == "Ready" && condition.status.eq_ignore_ascii_case("false") + })?; + + if condition.message.is_empty() { + if condition.reason.is_empty() { + None + } else { + Some(condition.reason.clone()) + } + } else if condition.reason.is_empty() { + Some(condition.message.clone()) + } else { + Some(format!("{}: {}", condition.reason, condition.message)) + } +} + +fn provisioning_timeout_message( + timeout_secs: u64, + requested_gpu: bool, + condition_message: Option<&str>, +) -> String { + let mut message = format!("sandbox provisioning timed out after {timeout_secs}s"); + + if let Some(condition_message) = condition_message.filter(|msg| !msg.is_empty()) { + message.push_str(". Last reported status: "); + message.push_str(condition_message); + } + + if requested_gpu { + message.push_str( + ". Hint: this may be because the gateway's available GPU is already in use by another sandbox.", + ); + } + + message +} + /// Format milliseconds since Unix epoch as a `YYYY-MM-DD HH:MM:SS` UTC string. fn format_epoch_ms(ms: i64) -> String { use std::time::UNIX_EPOCH; @@ -296,11 +337,6 @@ impl ProvisioningDisplay { let _ = self.mp.println(msg); } - /// Return a description of the current active step for error messages. - fn current_step_description(&self) -> &str { - &self.active_label - } - /// Clear all progress output (spinner, spacer, and completed step lines). fn clear(&mut self) { self.spacer.finish_and_clear(); @@ -1953,6 +1989,7 @@ pub async fn sandbox_create( let mut last_phase = sandbox.phase; let mut last_error_reason = String::new(); + let mut last_condition_message = ready_false_condition_message(sandbox.status.as_ref()); // Track whether we have seen a non-Ready phase during the watch. let mut saw_non_ready = SandboxPhase::try_from(sandbox.phase) != Ok(SandboxPhase::Ready); let start_time = Instant::now(); @@ -1963,18 +2000,16 @@ pub async fn sandbox_create( while let Some(item) = stream.next().await { // Check for timeout if start_time.elapsed() > provision_timeout { + let timeout_message = provisioning_timeout_message( + provision_timeout.as_secs(), + requested_gpu, + last_condition_message.as_deref(), + ); if let Some(d) = display.as_mut() { - let step_desc = d.current_step_description().to_string(); - d.finish_error(&format!( - "Timed out after {}s (stuck at: {step_desc})", - provision_timeout.as_secs() - )); + d.finish_error(&timeout_message); } println!(); - return Err(miette::miette!( - "sandbox provisioning timed out after {}s", - provision_timeout.as_secs() - )); + return Err(miette::miette!(timeout_message)); } let evt = item.into_diagnostic()?; @@ -1982,6 +2017,9 @@ pub async fn sandbox_create( Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(s)) => { let phase = SandboxPhase::try_from(s.phase).unwrap_or(SandboxPhase::Unknown); last_phase = s.phase; + if let Some(message) = ready_false_condition_message(s.status.as_ref()) { + last_condition_message = Some(message); + } if phase != SandboxPhase::Ready { saw_non_ready = true; @@ -4221,8 +4259,8 @@ mod tests { GatewayControlTarget, TlsOptions, format_gateway_select_header, format_gateway_select_items, gateway_auth_label, gateway_select_with, gateway_type_label, git_sync_files, http_health_check, image_requests_gpu, inferred_provider_type, - parse_credential_pairs, resolve_gateway_control_target_from, sandbox_should_persist, - source_requests_gpu, + parse_credential_pairs, provisioning_timeout_message, ready_false_condition_message, + resolve_gateway_control_target_from, sandbox_should_persist, source_requests_gpu, }; use crate::TEST_ENV_LOCK; use hyper::StatusCode; @@ -4235,6 +4273,7 @@ mod tests { use std::thread; use openshell_bootstrap::GatewayMetadata; + use openshell_core::proto::{SandboxCondition, SandboxStatus}; struct EnvVarGuard { key: &'static str, @@ -4427,6 +4466,67 @@ mod tests { assert!(!source_requests_gpu("base")); } + #[test] + fn ready_false_condition_message_prefers_reason_and_message() { + let status = SandboxStatus { + sandbox_name: "gpu".to_string(), + agent_pod: "gpu-pod".to_string(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "Unschedulable".to_string(), + message: "Another GPU sandbox may already be using the available GPU.".to_string(), + last_transition_time: String::new(), + }], + }; + + assert_eq!( + ready_false_condition_message(Some(&status)).as_deref(), + Some("Unschedulable: Another GPU sandbox may already be using the available GPU.") + ); + } + + #[test] + fn ready_false_condition_message_ignores_non_ready_conditions() { + let status = SandboxStatus { + sandbox_name: "gpu".to_string(), + agent_pod: "gpu-pod".to_string(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![SandboxCondition { + r#type: "Scheduled".to_string(), + status: "True".to_string(), + reason: "Scheduled".to_string(), + message: "Sandbox scheduled".to_string(), + last_transition_time: String::new(), + }], + }; + + assert!(ready_false_condition_message(Some(&status)).is_none()); + } + + #[test] + fn provisioning_timeout_message_includes_condition_and_gpu_hint() { + let message = provisioning_timeout_message( + 120, + true, + Some("DependenciesNotReady: Pod exists with phase: Pending; Service Exists"), + ); + + assert!(message.contains("sandbox provisioning timed out after 120s")); + assert!(message.contains("Last reported status: DependenciesNotReady: Pod exists with phase: Pending; Service Exists")); + assert!(message.contains("available GPU is already in use by another sandbox")); + } + + #[test] + fn provisioning_timeout_message_omits_gpu_hint_for_non_gpu_requests() { + let message = provisioning_timeout_message(120, false, None); + + assert_eq!(message, "sandbox provisioning timed out after 120s"); + } + fn init_git_repo(path: &Path) { let status = Command::new("git") .args(["init"])