mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-11-01 07:53:42 +08:00
refactor(podman): keep WORKDIR change focused on workspace behavior
Remove the new cross-driver workspace failure classification path and its monitor/readiness workaround. Preserve Podman WORKDIR resolution, mount safety, and workspace rejection without requiring a distinct condition reason. Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
This commit is contained in:
@@ -58,22 +58,11 @@ pub const CONDITION_WORKSPACE_VALIDATION_FAILED: &str = "WorkspaceValidationFail
|
||||
|
||||
/// Supervisor exit status reserved for OCI workspace validation failures.
|
||||
///
|
||||
/// Local container drivers translate a supervisor exit with this status into
|
||||
/// Local container drivers translate this status into
|
||||
/// [`CONDITION_WORKSPACE_VALIDATION_FAILED`] so users receive the specific
|
||||
/// provisioning failure rather than a generic container exit.
|
||||
pub const SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED: i32 = 78;
|
||||
|
||||
/// Error context for a rejected image-provided OCI working directory.
|
||||
///
|
||||
/// The supervisor recognizes it in a failed agent start and exits with
|
||||
/// [`SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED`].
|
||||
pub const WORKSPACE_VALIDATION_ERROR_CONTEXT: &str = "image workspace validation failed";
|
||||
|
||||
/// Driver condition message for [`CONDITION_WORKSPACE_VALIDATION_FAILED`].
|
||||
/// Fixed text, because supervisor output may contain secrets.
|
||||
pub const WORKSPACE_VALIDATION_FAILED_MESSAGE: &str =
|
||||
"OCI WorkingDir is not usable by the sandbox identity";
|
||||
|
||||
/// Ready-condition reason when a container was terminated by an external signal.
|
||||
///
|
||||
/// SIGKILL/SIGTERM (exit 137/143) is what a Podman/Docker machine or daemon
|
||||
|
||||
@@ -12,9 +12,9 @@ use bollard::Docker;
|
||||
use bollard::errors::Error as BollardError;
|
||||
use bollard::models::{
|
||||
ContainerCreateBody, ContainerState, ContainerStateStatusEnum, ContainerSummary,
|
||||
ContainerSummaryStateEnum, ContainerWaitResponse, CreateImageInfo, DeviceRequest, HealthConfig,
|
||||
HealthStatusEnum, HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, MountVolumeOptions,
|
||||
ProgressDetail, SystemInfo, VolumeCreateRequest,
|
||||
ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, HealthConfig, HealthStatusEnum,
|
||||
HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, ProgressDetail,
|
||||
SystemInfo, VolumeCreateRequest,
|
||||
};
|
||||
use bollard::query_parameters::{
|
||||
CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder,
|
||||
@@ -26,12 +26,10 @@ use futures::{Stream, StreamExt};
|
||||
use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS;
|
||||
use openshell_core::driver_mounts;
|
||||
use openshell_core::driver_utils::{
|
||||
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_WORKSPACE_VALIDATION_FAILED,
|
||||
LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME,
|
||||
LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SANDBOX_RUNTIME_IMAGE_BINARY_PATH,
|
||||
SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_FAILED_MESSAGE,
|
||||
extract_first_tar_entry, supervisor_image_should_refresh, temp_extract_container_name,
|
||||
validate_linux_elf_binary,
|
||||
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE,
|
||||
LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE,
|
||||
SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh,
|
||||
temp_extract_container_name, validate_linux_elf_binary,
|
||||
};
|
||||
use openshell_core::gpu::{
|
||||
CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements,
|
||||
@@ -1865,7 +1863,7 @@ impl DockerComputeDriver {
|
||||
.await;
|
||||
cleanup_docker_boundary_state(sandbox, &self.config);
|
||||
return Err(DockerProvisioningFailure::new(
|
||||
supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"),
|
||||
"ControlSupervisorStartFailed",
|
||||
status.message(),
|
||||
));
|
||||
}
|
||||
@@ -2157,7 +2155,7 @@ impl DockerComputeDriver {
|
||||
Err(status) => {
|
||||
handle_docker_runtime_failure(
|
||||
failure_context,
|
||||
supervisor_start_failure_reason(&status, "ControlSupervisorExited"),
|
||||
"ControlSupervisorExited",
|
||||
format!(
|
||||
"failed to start Docker control supervisor: {}",
|
||||
status.message()
|
||||
@@ -5240,9 +5238,7 @@ async fn spawn_docker_control_process(
|
||||
let supervisor_id = created.id;
|
||||
let monitored_supervisor_id = supervisor_id.clone();
|
||||
let monitored_docker = failure_context.docker.clone();
|
||||
let readiness_container_id = failure_context.container_id.clone();
|
||||
let readiness_sandbox_id = sandbox_id.clone();
|
||||
let readiness_failures = failure_context.failures.clone();
|
||||
let readiness_sandbox_id = failure_context.container_id.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let wait = async {
|
||||
let mut stream = monitored_docker.wait_container(
|
||||
@@ -5286,13 +5282,26 @@ async fn spawn_docker_control_process(
|
||||
).await;
|
||||
return;
|
||||
}
|
||||
let (reason, mut message) = docker_supervisor_wait_failure(result);
|
||||
warn!(%sandbox_id, %reason, %message, "Docker supervisor container exited unexpectedly");
|
||||
let mut message = match result {
|
||||
Some(Ok(status)) => {
|
||||
warn!(%sandbox_id, status = status.status_code, "Docker supervisor container exited unexpectedly");
|
||||
format!("Docker supervisor container exited with status {}", status.status_code)
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
warn!(%sandbox_id, %error, "Failed to wait for Docker supervisor container");
|
||||
format!("failed to wait for Docker supervisor container: {error}")
|
||||
}
|
||||
None => "Docker supervisor wait stream ended unexpectedly".to_string(),
|
||||
};
|
||||
let log_tail =
|
||||
docker_container_log_tail(&monitored_docker, &monitored_supervisor_id).await;
|
||||
if !log_tail.is_empty() {
|
||||
write!(message, "; log tail: {log_tail}").ok();
|
||||
}
|
||||
let _ = monitored_docker.remove_container(
|
||||
&monitored_supervisor_id,
|
||||
Some(RemoveContainerOptionsBuilder::default().force(true).build()),
|
||||
).await;
|
||||
// The gateway can close the supervisor session as soon as it
|
||||
// commits Stopping. Re-check after collecting diagnostics so
|
||||
// an overlapping driver stop cannot be published as an
|
||||
@@ -5300,13 +5309,12 @@ async fn spawn_docker_control_process(
|
||||
if monitored_shutdown.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
handle_docker_runtime_failure(failure_context, reason, message).await;
|
||||
// Publish the reason before removing the container. Readiness
|
||||
// may inspect it after cleanup and must still see exit 78.
|
||||
let _ = monitored_docker.remove_container(
|
||||
&monitored_supervisor_id,
|
||||
Some(RemoveContainerOptionsBuilder::default().force(true).build()),
|
||||
).await;
|
||||
handle_docker_runtime_failure(
|
||||
failure_context,
|
||||
"ControlSupervisorExited",
|
||||
message,
|
||||
)
|
||||
.await;
|
||||
},
|
||||
}
|
||||
});
|
||||
@@ -5315,14 +5323,8 @@ async fn spawn_docker_control_process(
|
||||
intentional_shutdown,
|
||||
task,
|
||||
};
|
||||
if let Err(error) = Box::pin(wait_for_docker_supervisor_ready(
|
||||
docker,
|
||||
&supervisor_id,
|
||||
&readiness_container_id,
|
||||
&readiness_sandbox_id,
|
||||
&readiness_failures,
|
||||
))
|
||||
.await
|
||||
if let Err(error) =
|
||||
wait_for_docker_supervisor_ready(docker, &supervisor_id, &readiness_sandbox_id).await
|
||||
{
|
||||
stop_docker_control_process(process).await;
|
||||
return Err(error);
|
||||
@@ -5333,60 +5335,33 @@ async fn spawn_docker_control_process(
|
||||
async fn wait_for_docker_supervisor_ready(
|
||||
docker: &Docker,
|
||||
supervisor_id: &str,
|
||||
workload_container_id: &str,
|
||||
sandbox_id: &str,
|
||||
failures: &Arc<Mutex<HashMap<String, DockerRuntimeFailure>>>,
|
||||
) -> Result<(), Status> {
|
||||
// Gateway provisioning deadlines own startup expiration, including policy
|
||||
// repairs that extend the window. Unhealthy means not ready while the
|
||||
// supervisor is quarantined; only a stopped process is a startup failure.
|
||||
loop {
|
||||
if let Some(status) = recorded_workspace_validation_failure(failures, sandbox_id).await {
|
||||
return Err(status);
|
||||
}
|
||||
let sandbox = match docker.inspect_container(workload_container_id, None).await {
|
||||
Ok(sandbox) => sandbox,
|
||||
Err(error) => {
|
||||
return Err(recorded_workspace_validation_failure(failures, sandbox_id)
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
Status::internal(format!("inspect Docker sandbox container: {error}"))
|
||||
}));
|
||||
}
|
||||
};
|
||||
let sandbox = docker
|
||||
.inspect_container(sandbox_id, None)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
Status::internal(format!("inspect Docker sandbox container: {error}"))
|
||||
})?;
|
||||
if sandbox.state.unwrap_or_default().running == Some(false) {
|
||||
return Err(recorded_workspace_validation_failure(failures, sandbox_id)
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
Status::unavailable("Docker sandbox exited before supervisor became ready")
|
||||
}));
|
||||
return Err(Status::unavailable(
|
||||
"Docker sandbox exited before supervisor became ready",
|
||||
));
|
||||
}
|
||||
let inspected = match docker.inspect_container(supervisor_id, None).await {
|
||||
Ok(inspected) => inspected,
|
||||
Err(error) => {
|
||||
return Err(recorded_workspace_validation_failure(failures, sandbox_id)
|
||||
.await
|
||||
.unwrap_or_else(|| {
|
||||
Status::internal(format!("inspect Docker supervisor container: {error}"))
|
||||
}));
|
||||
}
|
||||
};
|
||||
let inspected = docker
|
||||
.inspect_container(supervisor_id, None)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
Status::internal(format!("inspect Docker supervisor container: {error}"))
|
||||
})?;
|
||||
let state = inspected.state.unwrap_or_default();
|
||||
match state.health.and_then(|health| health.status) {
|
||||
Some(HealthStatusEnum::HEALTHY) => return Ok(()),
|
||||
_ if state.running == Some(false)
|
||||
&& state.exit_code
|
||||
== Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)) =>
|
||||
{
|
||||
let log_tail = docker_container_log_tail(docker, supervisor_id).await;
|
||||
return Err(workspace_validation_status(&log_tail));
|
||||
}
|
||||
_ if state.running == Some(false) => {
|
||||
if let Some(status) =
|
||||
recorded_workspace_validation_failure(failures, sandbox_id).await
|
||||
{
|
||||
return Err(status);
|
||||
}
|
||||
let log_tail = docker_container_log_tail(docker, supervisor_id).await;
|
||||
warn!(sandbox_id, supervisor_id, supervisor_logs = %log_tail, "Docker supervisor exited before becoming ready");
|
||||
return Err(Status::unavailable(format!(
|
||||
@@ -5399,77 +5374,6 @@ async fn wait_for_docker_supervisor_ready(
|
||||
}
|
||||
}
|
||||
|
||||
/// Bollard reports nonzero Docker wait statuses as errors, not successful
|
||||
/// responses. Both forms must preserve the sandbox's workspace failure reason.
|
||||
fn docker_supervisor_wait_failure(
|
||||
result: Option<Result<ContainerWaitResponse, BollardError>>,
|
||||
) -> (&'static str, String) {
|
||||
match result {
|
||||
Some(Ok(status)) => (
|
||||
supervisor_exit_reason(status.status_code),
|
||||
format!(
|
||||
"Docker supervisor container exited with status {}",
|
||||
status.status_code
|
||||
),
|
||||
),
|
||||
Some(Err(BollardError::DockerContainerWaitError { code, error })) => (
|
||||
supervisor_exit_reason(code),
|
||||
format!("Docker supervisor container exited with status {code}: {error}"),
|
||||
),
|
||||
Some(Err(error)) => (
|
||||
"ControlSupervisorExited",
|
||||
format!("failed to wait for Docker supervisor container: {error}"),
|
||||
),
|
||||
None => (
|
||||
"ControlSupervisorExited",
|
||||
"Docker supervisor wait stream ended unexpectedly".to_string(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
async fn recorded_workspace_validation_failure(
|
||||
failures: &Arc<Mutex<HashMap<String, DockerRuntimeFailure>>>,
|
||||
sandbox_id: &str,
|
||||
) -> Option<Status> {
|
||||
let failures = failures.lock().await;
|
||||
failures
|
||||
.get(sandbox_id)
|
||||
.filter(|failure| failure.reason == CONDITION_WORKSPACE_VALIDATION_FAILED)
|
||||
.map(|failure| workspace_validation_status(&failure.message))
|
||||
}
|
||||
|
||||
/// Startup failure for a supervisor that exited because the sandbox rejected
|
||||
/// the image working directory. The fixed message prefix identifies it for
|
||||
/// [`supervisor_start_failure_reason`].
|
||||
fn workspace_validation_status(log_tail: &str) -> Status {
|
||||
Status::failed_precondition(format!(
|
||||
"{WORKSPACE_VALIDATION_FAILED_MESSAGE}{}",
|
||||
format_log_tail(log_tail)
|
||||
))
|
||||
}
|
||||
|
||||
/// Condition reason for a supervisor that failed before becoming ready.
|
||||
fn supervisor_start_failure_reason(status: &Status, default: &'static str) -> &'static str {
|
||||
if status.code() == tonic::Code::FailedPrecondition
|
||||
&& status
|
||||
.message()
|
||||
.starts_with(WORKSPACE_VALIDATION_FAILED_MESSAGE)
|
||||
{
|
||||
CONDITION_WORKSPACE_VALIDATION_FAILED
|
||||
} else {
|
||||
default
|
||||
}
|
||||
}
|
||||
|
||||
/// Condition reason for a supervisor that exited with `status_code`.
|
||||
fn supervisor_exit_reason(status_code: i64) -> &'static str {
|
||||
if status_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) {
|
||||
CONDITION_WORKSPACE_VALIDATION_FAILED
|
||||
} else {
|
||||
"ControlSupervisorExited"
|
||||
}
|
||||
}
|
||||
|
||||
fn format_log_tail(log_tail: &str) -> String {
|
||||
// gRPC status messages travel in HTTP/2 headers. A full 16 KiB container
|
||||
// tail can exceed the client's 16 KiB header budget and hide the real
|
||||
|
||||
@@ -3653,101 +3653,6 @@ fn docker_oom_kill_stays_terminal_despite_137() {
|
||||
assert_eq!(ready_reason(&sandbox), CONDITION_EXITED);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn supervisor_workspace_validation_exit_is_reported_explicitly() {
|
||||
let status = workspace_validation_status("image workspace validation failed: denied");
|
||||
assert!(status.message().contains("log tail: image workspace"));
|
||||
assert_eq!(
|
||||
supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"),
|
||||
CONDITION_WORKSPACE_VALIDATION_FAILED
|
||||
);
|
||||
assert_eq!(
|
||||
supervisor_start_failure_reason(
|
||||
&Status::failed_precondition("provider SPIFFE socket has no parent directory"),
|
||||
"ControlSupervisorStartFailed"
|
||||
),
|
||||
"ControlSupervisorStartFailed"
|
||||
);
|
||||
assert_eq!(
|
||||
supervisor_exit_reason(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)),
|
||||
CONDITION_WORKSPACE_VALIDATION_FAILED
|
||||
);
|
||||
assert_eq!(supervisor_exit_reason(1), "ControlSupervisorExited");
|
||||
|
||||
let (reason, message) =
|
||||
docker_supervisor_wait_failure(Some(Err(BollardError::DockerContainerWaitError {
|
||||
code: i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED),
|
||||
error: "image workspace validation failed".into(),
|
||||
})));
|
||||
assert_eq!(reason, CONDITION_WORKSPACE_VALIDATION_FAILED);
|
||||
assert!(message.contains("status 78"));
|
||||
assert_eq!(
|
||||
docker_supervisor_wait_failure(Some(Err(BollardError::DockerContainerWaitError {
|
||||
code: 1,
|
||||
error: String::new(),
|
||||
})))
|
||||
.0,
|
||||
"ControlSupervisorExited"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn docker_wait_exit_78_keeps_workspace_failure_reason() {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut request = [0; 4096];
|
||||
let size = socket.read(&mut request).await.unwrap();
|
||||
assert!(String::from_utf8_lossy(&request[..size]).contains("/containers/supervisor/wait"));
|
||||
let body = r#"{"StatusCode":78,"Error":null}"#;
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
socket.write_all(response.as_bytes()).await.unwrap();
|
||||
});
|
||||
let docker =
|
||||
Docker::connect_with_http(&address.to_string(), 5, bollard::API_DEFAULT_VERSION).unwrap();
|
||||
let mut wait = docker.wait_container(
|
||||
"supervisor",
|
||||
None::<bollard::query_parameters::WaitContainerOptions>,
|
||||
);
|
||||
let (reason, message) = docker_supervisor_wait_failure(wait.next().await);
|
||||
assert_eq!(reason, CONDITION_WORKSPACE_VALIDATION_FAILED);
|
||||
assert!(message.contains("status 78"));
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn workspace_failure_survives_supervisor_cleanup_before_readiness_inspection() {
|
||||
let failures = Arc::new(Mutex::new(HashMap::from([(
|
||||
"sandbox-1".to_string(),
|
||||
DockerRuntimeFailure {
|
||||
reason: CONDITION_WORKSPACE_VALIDATION_FAILED,
|
||||
message: "Docker supervisor container exited with status 78".into(),
|
||||
},
|
||||
)])));
|
||||
// There is no Docker daemon at this address: a monitor that has removed
|
||||
// the supervisor must not need another inspection to retain the reason.
|
||||
let docker = Docker::connect_with_http("127.0.0.1:1", 1, bollard::API_DEFAULT_VERSION).unwrap();
|
||||
let status = wait_for_docker_supervisor_ready(
|
||||
&docker,
|
||||
"removed-supervisor",
|
||||
"workload-container-1",
|
||||
"sandbox-1",
|
||||
&failures,
|
||||
)
|
||||
.await
|
||||
.expect_err("the recorded workspace failure takes precedence over inspection");
|
||||
assert_eq!(
|
||||
supervisor_start_failure_reason(&status, "ControlSupervisorStartFailed"),
|
||||
CONDITION_WORKSPACE_VALIDATION_FAILED
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_container_removal_is_idempotent() {
|
||||
let removing = BollardError::DockerResponseServerError {
|
||||
|
||||
@@ -121,10 +121,6 @@ pub struct ContainerState {
|
||||
/// container-log marker. It is never deserialized from Podman.
|
||||
#[serde(skip)]
|
||||
pub startup_diagnostic: Option<String>,
|
||||
/// Exit status of the sandbox's supervisor companion when it stopped
|
||||
/// before the workload. It is never deserialized from Podman.
|
||||
#[serde(skip)]
|
||||
pub supervisor_exit_code: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
|
||||
@@ -29,7 +29,6 @@ const CONDITION_STARTING: &str = "ContainerStarting";
|
||||
use openshell_core::driver_utils::{
|
||||
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_STOPPED,
|
||||
CONDITION_WORKSPACE_VALIDATION_FAILED, SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED,
|
||||
WORKSPACE_VALIDATION_FAILED_MESSAGE,
|
||||
};
|
||||
|
||||
pub type WatchStream =
|
||||
@@ -391,12 +390,7 @@ pub async fn inspect_workload(
|
||||
// Both containers exist before initial start. A missing or exited
|
||||
// companion therefore requires containment, including after a
|
||||
// gateway restart that missed the original Podman exit event.
|
||||
Ok(supervisor) => {
|
||||
client.stop_container(&workload.id, 0).await?;
|
||||
workload = client.inspect_container(&workload.id).await?;
|
||||
workload.state.supervisor_exit_code = exited_exit_code(&supervisor.state);
|
||||
}
|
||||
Err(PodmanApiError::NotFound(_)) => {
|
||||
Ok(_) | Err(PodmanApiError::NotFound(_)) => {
|
||||
client.stop_container(&workload.id, 0).await?;
|
||||
workload = client.inspect_container(&workload.id).await?;
|
||||
}
|
||||
@@ -408,17 +402,10 @@ pub async fn inspect_workload(
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|logs| boundary_startup_termination_marker(&logs));
|
||||
workload.state.supervisor_exit_code = supervisor
|
||||
.ok()
|
||||
.and_then(|supervisor| exited_exit_code(&supervisor.state));
|
||||
}
|
||||
Ok(workload)
|
||||
}
|
||||
|
||||
fn exited_exit_code(state: &ContainerState) -> Option<i64> {
|
||||
matches!(state.status.as_str(), "exited" | "stopped").then_some(state.exit_code)
|
||||
}
|
||||
|
||||
/// Extract only fixed, OpenShell-owned startup diagnostics from container
|
||||
/// output. Workload and supervisor output may contain secrets, so it must not
|
||||
/// be propagated to driver conditions or tracing.
|
||||
@@ -566,12 +553,10 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition {
|
||||
"OOMKilled",
|
||||
"Container was killed by the OOM killer".to_string(),
|
||||
)
|
||||
} else if state.supervisor_exit_code
|
||||
== Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED))
|
||||
{
|
||||
} else if state.exit_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) {
|
||||
(
|
||||
CONDITION_WORKSPACE_VALIDATION_FAILED,
|
||||
WORKSPACE_VALIDATION_FAILED_MESSAGE.to_string(),
|
||||
"OCI WorkingDir is not usable by the sandbox identity".to_string(),
|
||||
)
|
||||
} else if matches!(state.exit_code, 137 | 143) {
|
||||
(
|
||||
@@ -707,7 +692,6 @@ mod tests {
|
||||
health: None,
|
||||
started_at: Some("2026-08-12T16:38:58Z".to_string()),
|
||||
finished_at: Some("2026-08-12T16:39:13Z".to_string()),
|
||||
supervisor_exit_code: None,
|
||||
startup_diagnostic: None,
|
||||
};
|
||||
|
||||
@@ -757,7 +741,6 @@ mod tests {
|
||||
}),
|
||||
started_at: Some("2026-04-14T10:00:00Z".to_string()),
|
||||
finished_at: None,
|
||||
supervisor_exit_code: None,
|
||||
startup_diagnostic: None,
|
||||
};
|
||||
let cond = condition_from_state(&state);
|
||||
@@ -777,7 +760,6 @@ mod tests {
|
||||
health: None,
|
||||
started_at: Some("2026-04-14T10:00:00Z".to_string()),
|
||||
finished_at: None,
|
||||
supervisor_exit_code: None,
|
||||
startup_diagnostic: None,
|
||||
};
|
||||
let cond = condition_from_state(&state);
|
||||
@@ -800,7 +782,6 @@ mod tests {
|
||||
}),
|
||||
started_at: Some("2026-04-14T10:00:00Z".to_string()),
|
||||
finished_at: None,
|
||||
supervisor_exit_code: None,
|
||||
startup_diagnostic: None,
|
||||
};
|
||||
let condition = condition_from_state(&state);
|
||||
@@ -819,7 +800,6 @@ mod tests {
|
||||
health: None,
|
||||
started_at: None,
|
||||
finished_at: Some("2026-04-14T11:00:00Z".to_string()),
|
||||
supervisor_exit_code: None,
|
||||
startup_diagnostic: None,
|
||||
};
|
||||
let cond = condition_from_state(&state);
|
||||
@@ -838,7 +818,6 @@ mod tests {
|
||||
health: None,
|
||||
started_at: None,
|
||||
finished_at: Some("2026-04-14T12:00:00Z".to_string()),
|
||||
supervisor_exit_code: None,
|
||||
startup_diagnostic: None,
|
||||
};
|
||||
let cond = condition_from_state(&state);
|
||||
@@ -857,7 +836,6 @@ mod tests {
|
||||
health: None,
|
||||
started_at: None,
|
||||
finished_at: Some("2026-04-14T12:00:00Z".to_string()),
|
||||
supervisor_exit_code: None,
|
||||
startup_diagnostic: boundary_startup_termination_marker(
|
||||
b"untrusted workload output\nsandbox boundary received SIGTERM before supervisor confirmation\n",
|
||||
),
|
||||
@@ -882,30 +860,21 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn condition_workspace_validation_exit_is_reported_explicitly() {
|
||||
// The supervisor exits with the reserved status and the watcher then
|
||||
// stops the workload, so the workload itself reports SIGKILL.
|
||||
let mut state = ContainerState {
|
||||
let state = ContainerState {
|
||||
status: "exited".to_string(),
|
||||
running: false,
|
||||
exit_code: 137,
|
||||
exit_code: i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED),
|
||||
oom_killed: false,
|
||||
health: None,
|
||||
started_at: None,
|
||||
finished_at: Some("2026-04-14T12:00:00Z".to_string()),
|
||||
supervisor_exit_code: Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)),
|
||||
startup_diagnostic: None,
|
||||
};
|
||||
|
||||
let cond = condition_from_state(&state);
|
||||
|
||||
assert_eq!(cond.reason, CONDITION_WORKSPACE_VALIDATION_FAILED);
|
||||
assert_eq!(cond.message, WORKSPACE_VALIDATION_FAILED_MESSAGE);
|
||||
|
||||
// A workload's own exit with the reserved status is not a supervisor
|
||||
// workspace rejection.
|
||||
state.exit_code = i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED);
|
||||
state.supervisor_exit_code = Some(1);
|
||||
assert_eq!(condition_from_state(&state).reason, CONDITION_EXITED);
|
||||
assert!(cond.message.contains("WorkingDir"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -923,7 +892,6 @@ mod tests {
|
||||
health: None,
|
||||
started_at: None,
|
||||
finished_at: Some("2026-04-14T12:30:00Z".to_string()),
|
||||
supervisor_exit_code: None,
|
||||
startup_diagnostic: None,
|
||||
};
|
||||
let cond = condition_from_state(&state);
|
||||
|
||||
@@ -54,7 +54,7 @@ pub async fn spawn_workload(
|
||||
crate::process::validate_oci_workspace_as_effective_identity(std::path::Path::new(
|
||||
workspace_root,
|
||||
))
|
||||
.wrap_err(openshell_core::driver_utils::WORKSPACE_VALIDATION_ERROR_CONTEXT)?;
|
||||
.wrap_err("image workspace validation failed")?;
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
|
||||
@@ -11,7 +11,7 @@ use clap::{Parser, ValueEnum};
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use openshell_isolation_interface::contract::BackendDescriptor;
|
||||
use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer};
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::{info, warn};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::filter::LevelFilter;
|
||||
use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _};
|
||||
@@ -492,7 +492,6 @@ fn main() -> Result<()> {
|
||||
args.main_exit_marker,
|
||||
))
|
||||
.await
|
||||
.or_else(workspace_validation_exit)
|
||||
}
|
||||
SupervisorRole::NetworkProxy => {
|
||||
let listen = args.listen.unwrap_or_else(|| ([127, 0, 0, 1], 3128).into());
|
||||
@@ -533,24 +532,6 @@ fn otlp_span_filter(log_level: &str) -> EnvFilter {
|
||||
EnvFilter::new(format!("info,openshell={level}"))
|
||||
}
|
||||
|
||||
/// Exit with the reserved status when the sandbox rejects the image working
|
||||
/// directory, so the compute driver can report the specific failure.
|
||||
fn workspace_validation_exit(error: miette::Report) -> Result<i32> {
|
||||
use openshell_core::driver_utils::{
|
||||
SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_ERROR_CONTEXT,
|
||||
};
|
||||
// Display keeps the message on one line; Debug may wrap it.
|
||||
if !error
|
||||
.to_string()
|
||||
.contains(WORKSPACE_VALIDATION_ERROR_CONTEXT)
|
||||
{
|
||||
return Err(error);
|
||||
}
|
||||
error!("Image workspace validation failed");
|
||||
eprintln!("{error:?}");
|
||||
Ok(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -564,29 +545,6 @@ mod tests {
|
||||
assert_eq!(directives("info,h2=debug"), "openshell=info,info");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_validation_failure_exits_with_reserved_status() {
|
||||
use miette::WrapErr as _;
|
||||
use openshell_core::driver_utils::{
|
||||
SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_ERROR_CONTEXT,
|
||||
};
|
||||
// Mirror how the sandbox reports the rejection across the boundary.
|
||||
let sandbox_error = Err::<(), _>(miette::miette!(
|
||||
"workspace path component '/workspace/project' is not writable by the sandbox \
|
||||
identity in the image: Permission denied (os error 13)"
|
||||
))
|
||||
.wrap_err(WORKSPACE_VALIDATION_ERROR_CONTEXT)
|
||||
.unwrap_err();
|
||||
let error = miette::miette!(
|
||||
"process error: boundary process leaf: start process supervisor leaf: {sandbox_error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
workspace_validation_exit(error).expect("reserved exit status"),
|
||||
SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED
|
||||
);
|
||||
assert!(workspace_validation_exit(miette::miette!("connect failed")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn isolation_backend_is_the_default_role() {
|
||||
let directory = tempfile::tempdir().expect("temporary runtime descriptor directory");
|
||||
|
||||
Reference in New Issue
Block a user