// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

#![cfg(feature = "e2e")]

#[cfg(target_os = "linux")]
use std::fs;
use std::process::Stdio;
use std::time::Duration;

use openshell_e2e::harness::binary::{openshell_cmd, openshell_tty_cmd};
use openshell_e2e::harness::cli::{
    run_cli, wait_for_sandbox_exec_contains, wait_for_sandbox_phase,
};
use openshell_e2e::harness::output::{extract_field, strip_ansi};
use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, SandboxGuard, unique_sandbox_name};
use serial_test::serial;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::time::{Instant, sleep};

const SANDBOX_PRESENCE_TIMEOUT: Duration = Duration::from_secs(30);
const SANDBOX_LIST_POLL_INTERVAL: Duration = Duration::from_millis(500);
const SANDBOX_RESTART_TIMEOUT: Duration = Duration::from_secs(60);

fn normalize_output(output: &str) -> String {
    let stripped = strip_ansi(output).replace('\r', "");
    let mut cleaned = String::with_capacity(stripped.len());

    for ch in stripped.chars() {
        match ch {
            '\u{8}' => {
                cleaned.pop();
            }
            '\u{4}' => {}
            _ => cleaned.push(ch),
        }
    }

    cleaned
}

fn extract_sandbox_name(output: &str) -> Option<String> {
    if let Some((_, rest)) = output.split_once("Created sandbox:") {
        return rest.split_whitespace().next().map(ToOwned::to_owned);
    }

    extract_field(output, "Created sandbox").or_else(|| extract_field(output, "Name"))
}

async fn sandbox_list_names(deadline: Instant) -> Option<Vec<String>> {
    if Instant::now() >= deadline {
        return None;
    }

    let mut cmd = openshell_cmd();
    cmd.args(["sandbox", "list", "--names"])
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());

    let output = match tokio::time::timeout_at(deadline, cmd.output()).await {
        Ok(output) => output.expect("spawn openshell sandbox list"),
        Err(_) => return None,
    };
    let stdout = String::from_utf8_lossy(&output.stdout).to_string();
    let stderr = String::from_utf8_lossy(&output.stderr).to_string();
    let combined = normalize_output(&format!("{stdout}{stderr}"));
    assert!(
        output.status.success(),
        "sandbox list should succeed (exit {:?}):\n{combined}",
        output.status.code()
    );

    Some(
        combined
            .lines()
            .map(str::trim)
            .filter(|line| !line.is_empty())
            .map(ToOwned::to_owned)
            .collect(),
    )
}

async fn assert_sandbox_presence_eventually(
    sandbox_name: &str,
    should_exist: bool,
) -> Result<(), Vec<String>> {
    let deadline = Instant::now() + SANDBOX_PRESENCE_TIMEOUT;
    let mut last_sandbox_names = Vec::new();

    loop {
        let Some(sandbox_names) = sandbox_list_names(deadline).await else {
            return Err(last_sandbox_names);
        };
        let exists = sandbox_names.iter().any(|name| name == sandbox_name);
        if exists == should_exist {
            return Ok(());
        }

        let now = Instant::now();
        if now >= deadline {
            return Err(sandbox_names);
        }

        last_sandbox_names = sandbox_names;
        sleep(SANDBOX_LIST_POLL_INTERVAL.min(deadline - now)).await;
    }
}

async fn delete_sandbox(name: &str) {
    let mut cmd = openshell_cmd();
    cmd.args(["sandbox", "delete", name])
        .stdout(Stdio::null())
        .stderr(Stdio::null());
    let _ = cmd.status().await;
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn sandbox_exec_outlives_startup_recovery_deadline() {
    let mut sandbox = SandboxGuard::create(&[])
        .await
        .expect("create sandbox for exec recovery deadline");
    let output = tokio::time::timeout(
        Duration::from_secs(60),
        sandbox.exec(&["sh", "-c", "sleep 31; printf past-recovery-deadline"]),
    )
    .await;
    let next = sandbox.exec(&["printf", "fresh-exec"]).await;
    sandbox.cleanup().await;

    assert_eq!(
        output.expect("exec timed out").expect("exec failed"),
        "past-recovery-deadline",
        "expiration must not terminate an admitted command"
    );
    assert_eq!(next.expect("fresh exec after expiration"), "fresh-exec");
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn sandbox_exec_large_output_is_complete() {
    const BYTES: usize = 8 * 1024 * 1024;
    let mut sandbox = SandboxGuard::create(&[])
        .await
        .expect("create sandbox for large output");

    for stderr in [false, true] {
        let script = if stderr {
            format!("yes A | head -c {BYTES} >&2")
        } else {
            format!("yes A | head -c {BYTES}")
        };
        let mut command = openshell_cmd();
        command
            .args([
                "sandbox",
                "exec",
                "--name",
                &sandbox.name,
                "--no-tty",
                "--no-login-shell",
                "--",
                "sh",
                "-c",
                &script,
            ])
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped());
        let output = tokio::time::timeout(Duration::from_secs(30), command.output())
            .await
            .expect("large exec output timed out")
            .expect("run large exec output");
        assert!(output.status.success(), "large exec exited unsuccessfully");
        let bytes = if stderr { output.stderr } else { output.stdout };
        assert_eq!(bytes.len(), BYTES, "exec output was truncated");
        assert!(bytes.chunks_exact(2).all(|pair| pair == b"A\n"));
    }

    let mut early_exit = openshell_cmd();
    let mut early_child = early_exit
        .args([
            "sandbox",
            "exec",
            "--name",
            &sandbox.name,
            "--no-tty",
            "--no-login-shell",
            "--",
            "head",
            "-n1",
        ])
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()
        .expect("start early-exit exec");
    let mut input = b"first\n".to_vec();
    input.extend(vec![b'x'; 128 * 1024]);
    let write_result = early_child
        .stdin
        .take()
        .expect("early-exit stdin")
        .write_all(&input)
        .await;
    if let Err(error) = write_result {
        assert_eq!(error.kind(), std::io::ErrorKind::BrokenPipe);
    }
    let early_output =
        tokio::time::timeout(Duration::from_secs(30), early_child.wait_with_output())
            .await
            .expect("early-exit exec timed out")
            .expect("wait for early-exit exec");
    assert!(early_output.status.success(), "early-exit exec failed");
    assert_eq!(early_output.stdout, b"first\n");

    let mut command = openshell_cmd();
    let output = tokio::time::timeout(
        Duration::from_secs(10),
        command
            .args([
                "sandbox",
                "exec",
                "--name",
                &sandbox.name,
                "--no-tty",
                "--no-login-shell",
                "--",
                "sh",
                "-c",
                "sleep 5 & echo ok",
            ])
            .stdin(Stdio::null())
            .output(),
    )
    .await
    .expect("exec with background pipe holder timed out")
    .expect("run exec with background pipe holder");
    assert!(output.status.success(), "unexpected output failure");
    assert_eq!(output.stdout, b"ok\n");

    sandbox.cleanup().await;
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn piped_exec_stdin_crosses_grpc_message_limit() {
    let mut sandbox = SandboxGuard::create(&[])
        .await
        .expect("create sandbox for streamed stdin");

    for size in [5, 1_048_576, 4_194_304] {
        let mut command = openshell_cmd();
        command
            .args([
                "sandbox",
                "exec",
                "--name",
                &sandbox.name,
                "--no-tty",
                "--no-login-shell",
                "--",
                "wc",
                "-c",
            ])
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped());
        let mut child = command.spawn().expect("spawn sandbox exec");
        let mut input = child.stdin.take().expect("piped stdin");
        input
            .write_all(&vec![b'x'; size])
            .await
            .expect("write piped stdin");
        drop(input);
        let output = tokio::time::timeout(Duration::from_secs(30), child.wait_with_output())
            .await
            .expect("streamed exec timed out")
            .expect("wait for streamed exec");
        assert!(
            output.status.success(),
            "streamed exec failed: {}",
            String::from_utf8_lossy(&output.stderr)
        );
        assert_eq!(
            String::from_utf8_lossy(&output.stdout).trim(),
            size.to_string()
        );
    }

    let mut oversized = openshell_cmd();
    let mut oversized_child = oversized
        .args([
            "sandbox",
            "exec",
            "--name",
            &sandbox.name,
            "--no-tty",
            "--no-login-shell",
            "--",
            "wc",
            "-c",
        ])
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()
        .expect("spawn oversized stdin exec");
    oversized_child
        .stdin
        .take()
        .expect("oversized stdin pipe")
        .write_all(&vec![b'x'; 4_194_305])
        .await
        .expect("write oversized stdin");
    let oversized_output = oversized_child
        .wait_with_output()
        .await
        .expect("wait for oversized stdin error");
    assert!(!oversized_output.status.success());
    assert!(oversized_output.stdout.is_empty());
    assert!(String::from_utf8_lossy(&oversized_output.stderr).contains("4 MiB limit"));

    // A forced PTY with a small pipe must keep using the unary RPC for
    // compatibility with older gateways.
    let mut tty_command = openshell_cmd();
    let mut tty_child = tty_command
        .args([
            "sandbox",
            "exec",
            "--name",
            &sandbox.name,
            "--tty",
            "--no-login-shell",
            "--",
            "sh",
            "-c",
            "printf tty-ok",
        ])
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()
        .expect("spawn forced TTY exec");
    tty_child
        .stdin
        .take()
        .expect("piped TTY stdin")
        .write_all(b"x")
        .await
        .expect("write TTY stdin");
    let tty_output = tokio::time::timeout(Duration::from_secs(30), tty_child.wait_with_output())
        .await
        .expect("forced TTY exec timed out")
        .expect("wait for forced TTY exec");
    assert!(
        tty_output.status.success(),
        "forced TTY exec failed: {}",
        String::from_utf8_lossy(&tty_output.stderr)
    );
    assert!(String::from_utf8_lossy(&tty_output.stdout).contains("tty-ok"));

    #[cfg(unix)]
    {
        let directory = std::fs::File::open("/").expect("open directory as stdin");
        let mut command = openshell_cmd();
        let output = command
            .args([
                "sandbox",
                "exec",
                "--name",
                &sandbox.name,
                "--no-tty",
                "--no-login-shell",
                "--",
                "wc",
                "-c",
            ])
            .stdin(Stdio::from(directory))
            .output()
            .await
            .expect("run exec with unreadable stdin");
        assert!(!output.status.success(), "stdin read error was ignored");
        assert!(output.stdout.is_empty());
    }

    sandbox.cleanup().await;
}

async fn run_sandbox_lifecycle_command(operation: &str, name: &str) -> String {
    let mut cmd = openshell_cmd();
    cmd.args(["sandbox", operation, name])
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());

    let output = cmd
        .output()
        .await
        .unwrap_or_else(|error| panic!("spawn openshell sandbox {operation}: {error}"));
    let combined = normalize_output(&format!(
        "{}{}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr),
    ));
    assert!(
        output.status.success(),
        "sandbox {operation} should succeed (exit {:?}):\n{combined}",
        output.status.code(),
    );
    combined
}

async fn sandbox_details(name: &str) -> String {
    let (details, exit_code) = run_cli(&["sandbox", "get", name]).await;
    let details = normalize_output(&details);
    assert_eq!(
        exit_code, 0,
        "sandbox get should succeed for {name}:\n{details}"
    );
    details
}

async fn reconnect_with_input_ownership(
    sandbox_name: &str,
) -> (tokio::process::Child, Vec<String>) {
    let reconnect_deadline = Instant::now() + Duration::from_secs(30);
    let mut attempt = 0;

    loop {
        attempt += 1;
        let token = format!("reconnect-{attempt}");
        let mut reconnect_cmd = openshell_cmd();
        reconnect_cmd
            .args(["sandbox", "connect", sandbox_name])
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped());
        let mut candidate = reconnect_cmd.spawn().expect("spawn reconnect attachment");
        let mut candidate_stdin = candidate.stdin.take().expect("reconnect stdin");
        candidate_stdin
            .write_all(format!("{token}\n").as_bytes())
            .await
            .expect("write reconnect ownership probe");
        candidate_stdin
            .flush()
            .await
            .expect("flush reconnect ownership probe");

        let reconnect_stdout = candidate.stdout.take().expect("reconnect stdout");
        let mut reconnect_lines = BufReader::new(reconnect_stdout).lines();
        let mut candidate_replay = Vec::new();
        let remaining = reconnect_deadline.saturating_duration_since(Instant::now());
        assert!(
            !remaining.is_zero(),
            "reconnect did not acquire input ownership"
        );
        let attempt_timeout = Duration::from_secs(2).min(remaining);
        let acquired = tokio::time::timeout(attempt_timeout, async {
            loop {
                let Some(line) = reconnect_lines
                    .next_line()
                    .await
                    .expect("read reconnect output")
                else {
                    return false;
                };
                if line.contains("main_pid=") {
                    candidate_replay.push(line.clone());
                }
                if line.contains(&format!("input={token}")) {
                    return true;
                }
            }
        })
        .await
        .unwrap_or(false);

        if acquired {
            return (candidate, candidate_replay);
        }

        let _ = candidate.kill().await;
        let _ = candidate.wait().await;
        assert!(
            Instant::now() < reconnect_deadline,
            "reconnect did not acquire input ownership"
        );
        sleep(Duration::from_millis(100)).await;
    }
}

#[cfg(target_os = "linux")]
fn find_process_with_args(expected_args: &[&str]) -> Option<u32> {
    for entry in fs::read_dir("/proc").ok()?.filter_map(Result::ok) {
        let Ok(pid) = entry.file_name().to_string_lossy().parse::<u32>() else {
            continue;
        };
        let cmdline = fs::read(entry.path().join("cmdline")).unwrap_or_default();
        let args = cmdline.split(|byte| *byte == 0).collect::<Vec<_>>();
        if expected_args
            .iter()
            .all(|expected| args.contains(&expected.as_bytes()))
        {
            return Some(pid);
        }
    }
    None
}

#[cfg(target_os = "linux")]
fn find_child_process_with_args(parent_pid: u32, expected_args: &[&str]) -> Option<u32> {
    for entry in fs::read_dir("/proc").ok()?.filter_map(Result::ok) {
        let Ok(pid) = entry.file_name().to_string_lossy().parse::<u32>() else {
            continue;
        };
        let status = fs::read_to_string(entry.path().join("status")).unwrap_or_default();
        let process_parent = status.lines().find_map(|line| {
            line.strip_prefix("PPid:")
                .and_then(|value| value.trim().parse::<u32>().ok())
        });
        if process_parent != Some(parent_pid) {
            continue;
        }
        let cmdline = fs::read(entry.path().join("cmdline")).unwrap_or_default();
        let args = cmdline.split(|byte| *byte == 0).collect::<Vec<_>>();
        if expected_args
            .iter()
            .all(|expected| args.contains(&expected.as_bytes()))
        {
            return Some(pid);
        }
    }
    None
}

#[cfg(target_os = "linux")]
async fn wait_for_process_with_args(expected_args: &[&str]) -> u32 {
    tokio::time::timeout(Duration::from_secs(10), async {
        loop {
            if let Some(pid) = find_process_with_args(expected_args) {
                return pid;
            }
            sleep(Duration::from_millis(50)).await;
        }
    })
    .await
    .unwrap_or_else(|_| panic!("process with arguments {expected_args:?} did not start"))
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn sandbox_stop_start_preserves_workspace() {
    const SENTINEL: &str = "openshell-stop-start-sentinel";
    const SENTINEL_PATH: &str = "/sandbox/.openshell-stop-start-e2e";
    const RUN_COUNT_PATH: &str = "/sandbox/.openshell-main-run-count";
    let write_sentinel = format!("printf '%s\\n' '{SENTINEL}' > '{SENTINEL_PATH}'");
    let main = format!(
        "count=0; test ! -f '{RUN_COUNT_PATH}' || count=$(cat '{RUN_COUNT_PATH}'); \
         count=$((count + 1)); printf '%s\\n' \"$count\" > '{RUN_COUNT_PATH}'; \
         echo lifecycle-ready-$count; exec sleep infinity"
    );

    let mut sandbox = SandboxGuard::create_keep(&["sh", "-c", &main], "lifecycle-ready")
        .await
        .expect("sandbox create should start a durable main process");
    sandbox
        .exec(&["sh", "-lc", &write_sentinel])
        .await
        .expect("sandbox exec should write the workspace sentinel");

    let stop_output = run_sandbox_lifecycle_command("stop", &sandbox.name).await;
    assert!(
        stop_output.contains("Stopped sandbox"),
        "expected stop confirmation in:\n{stop_output}",
    );

    let mut exec_cmd = openshell_cmd();
    exec_cmd
        .args([
            "sandbox",
            "exec",
            "--name",
            &sandbox.name,
            "--no-tty",
            "--",
            "cat",
            SENTINEL_PATH,
        ])
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let stopped_exec = exec_cmd
        .output()
        .await
        .expect("spawn openshell sandbox exec while stopped");
    assert!(
        !stopped_exec.status.success(),
        "sandbox exec should fail while stopped"
    );

    let start_output = run_sandbox_lifecycle_command("start", &sandbox.name).await;
    assert!(
        start_output.contains("Started sandbox"),
        "expected start confirmation in:\n{start_output}",
    );

    let sentinel = sandbox
        .exec(&["cat", SENTINEL_PATH])
        .await
        .expect("sandbox exec should succeed after start");
    assert!(
        sentinel.lines().any(|line| line.trim() == SENTINEL),
        "workspace sentinel should survive stop and start:\n{sentinel}",
    );
    let run_count = sandbox
        .exec(&["cat", RUN_COUNT_PATH])
        .await
        .expect("sandbox exec should read the canonical main run count");
    assert_eq!(
        run_count.trim(),
        "2",
        "start should launch a fresh canonical main instance:\n{run_count}",
    );

    sandbox.cleanup().await;
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn sandbox_can_be_deleted_while_stopped() {
    let mut sandbox = SandboxGuard::create_keep(
        &["sh", "-c", "echo stop-ready; exec sleep infinity"],
        "stop-ready",
    )
    .await
    .expect("sandbox create should start a durable main process");

    let stop_output = run_sandbox_lifecycle_command("stop", &sandbox.name).await;
    assert!(
        stop_output.contains("Stopped sandbox"),
        "expected stop confirmation in:\n{stop_output}",
    );

    let delete_output = run_sandbox_lifecycle_command("delete", &sandbox.name).await;
    // Deletion may return before the owned cleanup worker finishes. Both
    // outcomes must still reach absence, which is checked below.
    assert!(
        delete_output.contains(&format!("Deleted sandbox {}", sandbox.name))
            || delete_output.contains(&format!(
                "Sandbox {} deletion accepted; cleanup is pending",
                sandbox.name
            )),
        "expected completed or accepted deletion in:\n{delete_output}",
    );

    if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox.name, false).await {
        sandbox.cleanup().await;
        panic!(
            "stopped sandbox {} should be deleted without starting after \
             {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}",
            sandbox.name,
        );
    }

    // Mark the guard cleaned up. Its idempotent delete is harmless now that
    // the lifecycle operation above has removed the sandbox.
    sandbox.cleanup().await;
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn canonical_main_exit_zero_completes_persistent_sandbox() {
    // Armed before create so a failed create or parse still cleans up.
    let sandbox_name = unique_sandbox_name();
    let _cleanup = SandboxGuard::manage_existing(sandbox_name.clone());

    let mut cmd = openshell_tty_cmd(&[
        "sandbox",
        "create",
        "--name",
        &sandbox_name,
        "--",
        "echo",
        "OK",
    ]);
    cmd.stdout(Stdio::piped()).stderr(Stdio::piped());

    let output = cmd.output().await.expect("spawn openshell sandbox create");
    let stdout = String::from_utf8_lossy(&output.stdout).to_string();
    let stderr = String::from_utf8_lossy(&output.stderr).to_string();
    let combined = normalize_output(&format!("{stdout}{stderr}"));

    assert!(output.status.success(), "create failed:\n{combined}");
    assert!(
        combined.contains("OK"),
        "main output was not streamed:\n{combined}"
    );

    if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox_name, true).await {
        panic!(
            "sandbox {sandbox_name} should still exist by default after {SANDBOX_PRESENCE_TIMEOUT:?}; \
             last observed sandbox list: {last_sandbox_list:?}"
        );
    }

    let mut get_cmd = openshell_cmd();
    get_cmd
        .args(["sandbox", "get", &sandbox_name])
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let get_output = get_cmd.output().await.expect("spawn openshell sandbox get");
    let details = normalize_output(&format!(
        "{}{}",
        String::from_utf8_lossy(&get_output.stdout),
        String::from_utf8_lossy(&get_output.stderr),
    ));
    assert!(
        get_output.status.success(),
        "sandbox get failed:\n{details}"
    );
    assert!(
        details.contains("Phase: Completed"),
        "expected terminal sandbox phase:\n{details}"
    );
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn canonical_main_nonzero_exit_preserves_status() {
    // Armed before create so a failed create or parse still cleans up.
    let sandbox_name = unique_sandbox_name();
    let _cleanup = SandboxGuard::manage_existing(sandbox_name.clone());

    let mut cmd = openshell_tty_cmd(&[
        "sandbox",
        "create",
        "--name",
        &sandbox_name,
        "--",
        "sh",
        "-c",
        "echo failed-main; exit 7",
    ]);
    cmd.stdout(Stdio::piped()).stderr(Stdio::piped());

    let output = cmd.output().await.expect("spawn openshell sandbox create");
    let combined = normalize_output(&format!(
        "{}{}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr),
    ));
    assert_eq!(
        output.status.code(),
        Some(7),
        "unexpected result:\n{combined}"
    );
    assert!(
        combined.contains("failed-main"),
        "main output was not streamed:\n{combined}"
    );

    let mut get_cmd = openshell_cmd();
    get_cmd
        .args(["sandbox", "get", &sandbox_name])
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let get_output = get_cmd.output().await.expect("spawn openshell sandbox get");
    let details = normalize_output(&format!(
        "{}{}",
        String::from_utf8_lossy(&get_output.stdout),
        String::from_utf8_lossy(&get_output.stderr),
    ));
    assert!(
        details.contains("Phase: Error"),
        "unexpected phase:\n{details}"
    );
    assert!(
        details.contains("Exit Code: 7"),
        "missing exit code:\n{details}"
    );
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn detached_canonical_main_exit_zero_reaches_completed() {
    const RELEASE_PATH: &str = "/sandbox/.openshell-detached-success-release";
    let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 0");
    let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script])
        .await
        .expect("create detached successful canonical main");
    sandbox
        .exec(&["touch", RELEASE_PATH])
        .await
        .expect("release detached successful canonical main");

    wait_for_sandbox_phase(&sandbox.name, "Completed", SANDBOX_PRESENCE_TIMEOUT)
        .await
        .unwrap_or_else(|err| panic!("detached successful main did not complete:\n{err}"));
    let details = sandbox_details(&sandbox.name).await;
    assert!(
        details.contains("Phase: Completed"),
        "detached successful main should retain Completed status:\n{details}"
    );

    sandbox.cleanup().await;
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn detached_canonical_main_nonzero_exit_reaches_error() {
    const RELEASE_PATH: &str = "/sandbox/.openshell-detached-failure-release";
    let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 11");
    let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script])
        .await
        .expect("create detached failing canonical main");
    sandbox
        .exec(&["touch", RELEASE_PATH])
        .await
        .expect("release detached failing canonical main");

    wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT)
        .await
        .unwrap_or_else(|err| panic!("detached failing main did not reach Error:\n{err}"));
    let details = sandbox_details(&sandbox.name).await;
    assert!(
        details.contains("Phase: Error") && details.contains("Exit Code: 11"),
        "detached failing main should retain its terminal result:\n{details}"
    );

    sandbox.cleanup().await;
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn canonical_main_and_exec_receive_declared_environment() {
    for mode in ["--tty", "--no-tty"] {
        let script = r#"printf 'declared_env=%s\n' "${REPRO_SENTINEL:-missing}"; while true; do sleep 1; done"#;
        let mut sandbox = SandboxGuard::create_keep_with_args(
            &[
                mode,
                "--no-auto-providers",
                "--env",
                "REPRO_SENTINEL=present",
            ],
            &["sh", "-c", script],
            "declared_env=",
        )
        .await
        .expect("create canonical process with declared environment");
        let initial = normalize_output(&sandbox.create_output);
        let later = sandbox
            .exec(&[
                "sh",
                "-c",
                r#"printf 'declared_env=%s\n' "${REPRO_SENTINEL:-missing}""#,
            ])
            .await;
        sandbox.cleanup().await;

        assert!(
            initial.lines().any(|line| line == "declared_env=present"),
            "initial process must receive declared environment ({mode}): {initial}"
        );
        let later = normalize_output(&later.expect("exec environment probe"));
        assert!(
            later.lines().any(|line| line == "declared_env=present"),
            "exec must receive the same declared environment ({mode}): {later}"
        );
    }
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn detached_main_exit_during_provisioning_is_classified_as_workload_result() {
    let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", "exit 11"])
        .await
        .expect("fast detached main exit should not be reported as a provisioning failure");

    wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT)
        .await
        .unwrap_or_else(|err| panic!("fast detached main did not reach Error:\n{err}"));

    let mut get_cmd = openshell_cmd();
    get_cmd
        .args(["sandbox", "get", &sandbox.name])
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let get_output = get_cmd.output().await.expect("spawn openshell sandbox get");
    let details = normalize_output(&format!(
        "{}{}",
        String::from_utf8_lossy(&get_output.stdout),
        String::from_utf8_lossy(&get_output.stderr),
    ));
    assert!(
        get_output.status.success(),
        "sandbox get failed:\n{details}"
    );
    assert!(
        details.contains("Phase: Error") && details.contains("Exit Code: 11"),
        "fast detached main should retain its workload result:\n{details}"
    );

    sandbox.cleanup().await;
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn canonical_tty_main_uses_sandbox_environment() {
    let script = r#"printf 'canonical_env home=%s user=%s term=%s\n' "$HOME" "$USER" "$TERM"; while true; do sleep 1; done"#;
    let mut sandbox =
        SandboxGuard::create_keep_with_args(&["--tty"], &["sh", "-lc", script], "canonical_env")
            .await
            .expect("create canonical TTY process");

    let output = normalize_output(&sandbox.create_output);
    let environment = output
        .lines()
        .find(|line| line.contains("canonical_env"))
        .expect("canonical environment output");
    let field = |name: &str| {
        environment
            .split_whitespace()
            .find_map(|value| value.strip_prefix(&format!("{name}=")))
            .unwrap_or_default()
    };

    assert!(
        !field("home").is_empty() && field("home") != "/root",
        "canonical process must not inherit the supervisor HOME: {environment}"
    );
    assert!(
        !field("user").is_empty(),
        "canonical process USER must identify the sandbox user: {environment}"
    );
    assert!(
        !field("term").is_empty() && field("term") != "dumb",
        "canonical TTY process must receive a usable TERM: {environment}"
    );

    sandbox.cleanup().await;
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn canonical_main_disconnect_reconnect_replays_history_for_same_process() {
    const FIRST_MARKER: &str = "sequence=0001";
    let script = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'main_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'main_pid=%s input=%s\n' "$$" "$line"; done"#;
    let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-lc", script])
        .await
        .expect("create retained canonical main process");

    let mut owner_cmd = openshell_cmd();
    owner_cmd
        .args(["sandbox", "connect", &sandbox.name])
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let mut owner = owner_cmd.spawn().expect("spawn input-owning attachment");
    let owner_stdout = owner.stdout.take().expect("owner stdout");
    let mut owner_lines = BufReader::new(owner_stdout).lines();
    let owner_line = tokio::time::timeout(Duration::from_secs(30), owner_lines.next_line())
        .await
        .expect("owner output timeout")
        .expect("read owner output")
        .expect("owner output should remain open");
    assert!(
        owner_line.contains("main_pid="),
        "unexpected owner output: {owner_line}"
    );

    let mut observer_cmd = openshell_cmd();
    observer_cmd
        .args(["sandbox", "connect", &sandbox.name])
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let mut observer = observer_cmd.spawn().expect("spawn competing attachment");
    let observer_stderr = observer.stderr.take().expect("observer stderr");
    let mut observer_errors = BufReader::new(observer_stderr).lines();
    let warning = tokio::time::timeout(Duration::from_secs(30), observer_errors.next_line())
        .await
        .expect("observer warning timeout")
        .expect("read observer warning")
        .expect("observer warning should remain open");
    assert!(
        warning.contains("already has an input owner") && warning.contains("read-only"),
        "competing attachment should become read-only: {warning}"
    );
    let observer_stdout = observer.stdout.take().expect("observer stdout");
    let mut observer_lines = BufReader::new(observer_stdout).lines();
    let observer_output_line =
        tokio::time::timeout(Duration::from_secs(30), observer_lines.next_line())
            .await
            .expect("observer output timeout")
            .expect("read observer output")
            .expect("observer output should remain open");
    assert!(
        observer_output_line.contains("main_pid="),
        "read-only attachment should observe output: {observer_output_line}"
    );

    observer
        .stdin
        .as_mut()
        .expect("observer stdin")
        .write_all(b"\x03")
        .await
        .expect("send Ctrl-C to viewer");
    let observer_status = tokio::time::timeout(Duration::from_secs(30), observer.wait())
        .await
        .expect("Ctrl-C should exit viewer")
        .expect("wait for viewer");
    assert!(observer_status.success(), "viewer should exit successfully");

    owner
        .stdin
        .as_mut()
        .expect("owner stdin")
        .write_all(b"owner-after-viewer-exit\n")
        .await
        .expect("send input after viewer exits");
    tokio::time::timeout(Duration::from_secs(30), async {
        loop {
            let line = owner_lines
                .next_line()
                .await
                .expect("read owner output after viewer exits")
                .expect("main process should remain running");
            if line.contains("input=owner-after-viewer-exit") {
                break;
            }
        }
    })
    .await
    .expect("owner should retain working stdin after viewer exits");

    owner.kill().await.expect("disconnect input owner");
    owner.wait().await.expect("wait for input owner disconnect");

    let (mut reconnect, replay) = reconnect_with_input_ownership(&sandbox.name).await;

    let first_pid = owner_line
        .split_whitespace()
        .find_map(|field| field.strip_prefix("main_pid="))
        .expect("owner output pid");
    assert!(
        replay.iter().any(|line| line.contains(FIRST_MARKER)),
        "reconnect should replay the beginning of retained history: {replay:?}"
    );
    assert!(
        replay
            .iter()
            .all(|line| line.contains(&format!("main_pid={first_pid}"))),
        "reconnect should target the same canonical process: {replay:?}"
    );
    assert!(
        replay.iter().any(|line| !line.contains(FIRST_MARKER)),
        "reconnect should observe output beyond the first record: {replay:?}"
    );

    reconnect
        .kill()
        .await
        .expect("disconnect reconnect attachment");
    reconnect
        .wait()
        .await
        .expect("wait for reconnect disconnect");
    sandbox.cleanup().await;
}

#[cfg(target_os = "linux")]
#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn canonical_main_connect_recovers_its_ssh_transport() {
    let script = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'transport_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'transport_pid=%s input=%s\n' "$$" "$line"; done"#;
    let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-lc", script])
        .await
        .expect("create retained canonical main process");

    let mut connect_cmd = openshell_cmd();
    connect_cmd
        .args(["sandbox", "connect", &sandbox.name])
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let mut connect = connect_cmd.spawn().expect("spawn supervised attachment");
    let mut connect_stdin = connect.stdin.take().expect("connect stdin");
    let connect_stdout = connect.stdout.take().expect("connect stdout");
    let mut connect_lines = BufReader::new(connect_stdout).lines();
    let connect_stderr = connect.stderr.take().expect("connect stderr");
    let mut connect_errors = BufReader::new(connect_stderr).lines();

    let initial_line = tokio::time::timeout(Duration::from_secs(30), connect_lines.next_line())
        .await
        .expect("initial attachment output timeout")
        .expect("read initial attachment output")
        .expect("initial attachment output should remain open");
    assert!(
        initial_line.contains("transport_pid="),
        "unexpected initial attachment output: {initial_line}"
    );

    // Recovery intentionally starts only for an established attachment, so
    // let the initial SSH process live beyond that setup guard before killing
    // only its ProxyCommand transport.
    sleep(Duration::from_secs(3)).await;
    let proxy_pid = wait_for_process_with_args(&["ssh-proxy", "--sandbox", &sandbox.name]).await;
    let kill_status = tokio::process::Command::new("kill")
        .args(["-TERM", &proxy_pid.to_string()])
        .status()
        .await
        .expect("terminate SSH proxy transport");
    assert!(kill_status.success(), "terminate SSH proxy transport");

    tokio::time::timeout(Duration::from_secs(30), async {
        loop {
            let line = connect_errors
                .next_line()
                .await
                .expect("read supervised attachment diagnostics")
                .expect("supervised attachment diagnostics should remain open");
            if normalize_output(&line).contains("Connection to sandbox lost; reconnecting") {
                break;
            }
        }
    })
    .await
    .expect("supervised attachment did not enter recovery");

    let input_token = format!("after-transport-recovery-{:x}", rand::random::<u64>());
    connect_stdin
        .write_all(format!("{input_token}\n").as_bytes())
        .await
        .expect("write input after transport recovery");
    connect_stdin
        .flush()
        .await
        .expect("flush input after transport recovery");

    tokio::time::timeout(Duration::from_secs(30), async {
        loop {
            let line = connect_lines
                .next_line()
                .await
                .expect("read recovered attachment output")
                .expect("recovered attachment output should remain open");
            if line.contains(&format!("input={input_token}")) {
                break;
            }
        }
    })
    .await
    .expect("replacement SSH session did not reattach to canonical main");
    assert!(
        connect
            .try_wait()
            .expect("inspect supervised attachment")
            .is_none(),
        "sandbox connect parent should remain alive after recovery"
    );

    connect
        .kill()
        .await
        .expect("disconnect recovered attachment");
    connect
        .wait()
        .await
        .expect("wait for recovered attachment disconnect");
    sandbox.cleanup().await;
}

#[cfg(target_os = "linux")]
#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn canonical_main_connect_forwards_pid_targeted_termination_and_reaps_ssh() {
    use std::os::fd::OwnedFd;

    let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-lc", "exec sleep infinity"])
        .await
        .expect("create retained canonical main process");
    let pty = nix::pty::openpty(None, None).expect("open pseudo-terminal");
    let controller: OwnedFd = pty.master;
    let follower: OwnedFd = pty.slave;

    let mut connect_cmd = openshell_cmd();
    connect_cmd
        .args(["sandbox", "connect", &sandbox.name])
        .stdin(
            follower
                .try_clone()
                .expect("duplicate PTY follower for stdin"),
        )
        .stdout(
            follower
                .try_clone()
                .expect("duplicate PTY follower for stdout"),
        )
        .stderr(
            follower
                .try_clone()
                .expect("duplicate PTY follower for stderr"),
        );
    let mut connect = connect_cmd
        .spawn()
        .expect("spawn supervised PTY attachment");
    let connect_pid = connect.id().expect("connect process ID");
    drop(follower);

    let ssh_pid = tokio::time::timeout(Duration::from_secs(30), async {
        loop {
            if let Some(pid) =
                find_child_process_with_args(connect_pid, &["-s", "sandbox", "openshell-main"])
            {
                return pid;
            }
            sleep(Duration::from_millis(50)).await;
        }
    })
    .await
    .expect("supervised SSH child did not start");

    nix::sys::signal::kill(
        nix::unistd::Pid::from_raw(i32::try_from(connect_pid).expect("connect PID fits i32")),
        nix::sys::signal::Signal::SIGTERM,
    )
    .expect("send SIGTERM to only the OpenShell parent");

    let status = tokio::time::timeout(Duration::from_secs(10), connect.wait())
        .await
        .expect("OpenShell parent did not terminate")
        .expect("wait for OpenShell parent");
    assert_eq!(status.code(), Some(143));
    tokio::time::timeout(Duration::from_secs(5), async {
        while fs::metadata(format!("/proc/{ssh_pid}")).is_ok() {
            sleep(Duration::from_millis(25)).await;
        }
    })
    .await
    .expect("SSH child was not reaped after parent termination");

    drop(controller);
    sandbox.cleanup().await;
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn canonical_main_exit_255_is_not_retried_as_transport_failure() {
    const READY_MARKER: &str = "exit-255-ready";
    const RELEASE_PATH: &str = "/sandbox/.openshell-exit-255-release";
    let script = format!(
        "echo {READY_MARKER}; while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 255"
    );
    let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script])
        .await
        .expect("create retained canonical main process");

    let mut connect_cmd = openshell_cmd();
    connect_cmd
        .args(["sandbox", "connect", &sandbox.name])
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped());
    let mut connect = connect_cmd.spawn().expect("spawn supervised attachment");
    let connect_stdout = connect.stdout.take().expect("connect stdout");
    let mut connect_lines = BufReader::new(connect_stdout).lines();

    tokio::time::timeout(Duration::from_secs(30), async {
        loop {
            let line = connect_lines
                .next_line()
                .await
                .expect("read attachment output")
                .expect("attachment output should remain open");
            if line.contains(READY_MARKER) {
                break;
            }
        }
    })
    .await
    .expect("attachment did not observe the canonical main process");

    // Keep the attachment alive beyond the setup guard used to distinguish
    // initial SSH failures from established transport failures.
    sleep(Duration::from_secs(3)).await;
    sandbox
        .exec(&["touch", RELEASE_PATH])
        .await
        .expect("release canonical main process");

    let status = tokio::time::timeout(Duration::from_secs(10), connect.wait()).await;
    if status.is_err() {
        connect.kill().await.expect("stop stuck attachment");
    }
    sandbox.cleanup().await;
    let status = status
        .expect("exit status 255 must not enter the transport recovery loop")
        .expect("wait for canonical main attachment");
    assert_eq!(status.code(), Some(255));
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn on_failure_policy_replaces_runtime_and_preserves_workspace() {
    const SCRIPT: &str = r#"
marker=/sandbox/.openshell-restart-e2e
if [ -e "$marker" ]; then
  printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run
  sleep 300
else
  touch "$marker"
  printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run
  while [ ! -e /sandbox/.openshell-restart-release ]; do sleep 0.05; done
  exit 17
fi
"#;

    let mut sandbox = SandboxGuard::manage_existing(unique_sandbox_name());
    let (output, exit_code) = run_cli(&[
        "sandbox",
        "create",
        "--name",
        &sandbox.name,
        "--from",
        E2E_WORKLOAD_IMAGE,
        "--detach",
        "--restart-policy",
        "on-failure",
        "--",
        "sh",
        "-lc",
        SCRIPT,
    ])
    .await;
    assert_eq!(
        exit_code, 0,
        "create sandbox with OnFailure policy: {output}"
    );
    wait_for_sandbox_exec_contains(
        &sandbox.name,
        &["cat", "/sandbox/initial-run"],
        "initial-",
        SANDBOX_RESTART_TIMEOUT,
    )
    .await
    .expect("initial main process should write its workspace marker");

    let initial_details = sandbox_details(&sandbox.name).await;
    let initial_instance = extract_field(&initial_details, "Main process instance")
        .expect("initial main process instance");
    sandbox
        .exec(&["touch", "/sandbox/.openshell-restart-release"])
        .await
        .expect("release initial main process");

    let deadline = Instant::now() + SANDBOX_RESTART_TIMEOUT;
    let mut last_runs = Err("replacement is not ready".to_string());
    let final_details = loop {
        let details = sandbox_details(&sandbox.name).await;
        let replacement_instance = extract_field(&details, "Main process instance")
            .is_some_and(|instance| instance != "-" && instance != initial_instance);
        if details.contains("Phase: Ready")
            && extract_field(&details, "Restart count").as_deref() == Some("1")
            && replacement_instance
        {
            last_runs = sandbox
                .exec(&["cat", "/sandbox/initial-run", "/sandbox/replacement-run"])
                .await;
            if let Ok(runs) = &last_runs
                && runs.contains("initial-")
                && runs.contains("replacement-")
            {
                break details;
            }
        }
        assert!(
            Instant::now() < deadline,
            "sandbox did not complete its policy-driven restart within \
             {SANDBOX_RESTART_TIMEOUT:?}; last details:\n{details}\nworkspace: {last_runs:?}"
        );
        sleep(Duration::from_millis(250)).await;
    };

    assert!(
        final_details.contains("Restart policy: on-failure"),
        "restart policy should remain visible after replacement:\n{final_details}"
    );
    sandbox.cleanup().await;
}

#[tokio::test]
async fn sandbox_create_with_no_keep_cleans_up_after_tty_command() {
    let name = format!("tty-{:015x}", rand::random::<u64>() & 0x0fff_ffff_ffff_ffff);
    // Capture startup diagnostics before --no-keep removes a failed container.
    // This is best-effort: the lifecycle assertions also run on other drivers.
    let log_name = name.clone();
    let diagnostics = tokio::spawn(async move {
        tokio::time::timeout(Duration::from_secs(30), async {
            loop {
                let containers = tokio::process::Command::new("docker")
                    .args(["ps", "--all", "--quiet", "--filter"])
                    .arg(format!("label=openshell.ai/sandbox-name={log_name}"))
                    .kill_on_drop(true)
                    .output()
                    .await
                    .ok()?;
                if !containers.status.success() {
                    return None;
                }
                let ids = String::from_utf8_lossy(&containers.stdout);
                if let Some(id) = ids.split_whitespace().next() {
                    let logs = tokio::process::Command::new("docker")
                        .args(["logs", "--follow", id])
                        .kill_on_drop(true)
                        .output()
                        .await
                        .ok()?;
                    return Some(normalize_output(&format!(
                        "{}{}",
                        String::from_utf8_lossy(&logs.stdout),
                        String::from_utf8_lossy(&logs.stderr)
                    )));
                }
                sleep(Duration::from_millis(100)).await;
            }
        })
        .await
        .ok()
        .flatten()
    });
    let mut cmd = openshell_tty_cmd(&[
        "sandbox",
        "create",
        "--name",
        &name,
        "--no-keep",
        "--",
        "echo",
        "OK",
    ]);
    cmd.stdout(Stdio::piped()).stderr(Stdio::piped());

    let output = cmd.output().await.expect("spawn openshell sandbox create");
    let stdout = String::from_utf8_lossy(&output.stdout).to_string();
    let stderr = String::from_utf8_lossy(&output.stderr).to_string();
    let combined = normalize_output(&format!("{stdout}{stderr}"));

    let startup_logs = if output.status.success() {
        diagnostics.abort();
        None
    } else {
        diagnostics.await.ok().flatten()
    };
    assert!(
        output.status.success(),
        "create failed:\n{combined}\nsupervisor logs:\n{}",
        startup_logs.as_deref().unwrap_or("unavailable")
    );
    assert!(
        combined.contains("OK"),
        "main output was not streamed:\n{combined}"
    );
    let sandbox_name =
        extract_sandbox_name(&combined).expect("sandbox name should be present in output");

    if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox_name, false).await {
        delete_sandbox(&sandbox_name).await;
        panic!(
            "sandbox {sandbox_name} should have been deleted automatically after \
             {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}"
        );
    }
}

#[tokio::test]
#[serial(sandbox_lifecycle)]
async fn sandbox_create_with_no_keep_preserves_failure_then_cleans_up() {
    let mut cmd = openshell_tty_cmd(&[
        "sandbox",
        "create",
        "--no-keep",
        "--",
        "sh",
        "-c",
        "echo ephemeral-failure; exit 17",
    ]);
    cmd.stdout(Stdio::piped()).stderr(Stdio::piped());

    let output = cmd.output().await.expect("spawn openshell sandbox create");
    let combined = normalize_output(&format!(
        "{}{}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr),
    ));

    assert_eq!(
        output.status.code(),
        Some(17),
        "ephemeral main should preserve its failure code:\n{combined}"
    );
    assert!(
        combined.contains("ephemeral-failure"),
        "ephemeral main output was not streamed:\n{combined}"
    );
    let sandbox_name =
        extract_sandbox_name(&combined).expect("sandbox name should be present in output");

    if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox_name, false).await {
        delete_sandbox(&sandbox_name).await;
        panic!(
            "failed ephemeral sandbox {sandbox_name} should have been deleted automatically after \
             {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}"
        );
    }
}