From 2ea4927d2ee699ae9d9d244ce855f4de5e373234 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 13 Mar 2026 07:47:21 -0700 Subject: [PATCH] fix(bootstrap): detect missing sandbox supervisor binary during gateway health check The cluster health check now verifies that /opt/openshell/bin/openshell-sandbox exists in the gateway container. Without this binary, every sandbox pod crashes with 'no such file or directory' but the gateway previously reported healthy. - Add HEALTHCHECK_MISSING_SUPERVISOR marker to cluster-healthcheck.sh - Add early detection in bootstrap polling loop (runtime.rs) - Add structured error diagnosis with recovery steps (errors.rs) - Update debug-openshell-cluster skill with new failure pattern --- .../skills/debug-openshell-cluster/SKILL.md | 6 ++++ crates/openshell-bootstrap/src/errors.rs | 33 +++++++++++++++++++ crates/openshell-bootstrap/src/runtime.rs | 30 +++++++++++++++++ deploy/docker/cluster-healthcheck.sh | 12 +++++++ 4 files changed, 81 insertions(+) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 2d46ea550..81b1b53e5 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -26,6 +26,7 @@ Use **only** `openshell` CLI commands (`openshell status`, `openshell doctor log - k3s API server readiness (`/readyz`) - `openshell` statefulset ready in `openshell` namespace - TLS secrets `openshell-server-tls` and `openshell-client-tls` exist in `openshell` namespace + - Sandbox supervisor binary exists at `/opt/openshell/bin/openshell-sandbox` (emits `HEALTHCHECK_MISSING_SUPERVISOR` marker if absent) For local deploys, metadata endpoint selection now depends on Docker connectivity: @@ -311,6 +312,8 @@ If DNS is broken, all image pulls from the distribution registry will fail, as w | `metrics-server` errors in logs | Normal k3s noise, not the root cause | These errors are benign — look for the actual failing health check component | | Stale NotReady nodes from previous deploys | Volume reused across container recreations | The deploy flow now auto-cleans stale nodes; if it still fails, manually delete NotReady nodes (see Step 2) or choose "Recreate" when prompted | | gRPC `UNIMPLEMENTED` for newer RPCs in push mode | Helm values still point at older pulled images instead of the pushed refs | Verify rendered `openshell-helmchart.yaml` uses the expected push refs (`server`, `sandbox`, `pki-job`) and not `:latest` | +| Sandbox pods crash with `/opt/openshell/bin/openshell-sandbox: no such file or directory` | Supervisor binary missing from cluster image | The cluster image was built/published without the `supervisor-builder` stage. Rebuild with `mise run docker:build:cluster` and recreate gateway. Bootstrap auto-detects via `HEALTHCHECK_MISSING_SUPERVISOR` marker | +| `HEALTHCHECK_MISSING_SUPERVISOR` in health check logs | `/opt/openshell/bin/openshell-sandbox` not found in gateway container | Rebuild cluster image: `mise run docker:build:cluster`, then `openshell gateway destroy && openshell gateway start` | ## Full Diagnostic Dump @@ -359,6 +362,9 @@ openshell doctor exec -- kubectl -n kube-system logs -l job-name=helm-install-op echo "=== Registry Configuration ===" openshell doctor exec -- cat /etc/rancher/k3s/registries.yaml +echo "=== Supervisor Binary ===" +openshell doctor exec -- ls -la /opt/openshell/bin/openshell-sandbox + echo "=== DNS Configuration ===" openshell doctor exec -- cat /etc/rancher/k3s/resolv.conf ``` diff --git a/crates/openshell-bootstrap/src/errors.rs b/crates/openshell-bootstrap/src/errors.rs index 40aac1a74..39088ffb9 100644 --- a/crates/openshell-bootstrap/src/errors.rs +++ b/crates/openshell-bootstrap/src/errors.rs @@ -139,6 +139,12 @@ const FAILURE_PATTERNS: &[FailurePattern] = &[ match_mode: MatchMode::Any, diagnose: diagnose_node_pressure, }, + // Missing sandbox supervisor binary + FailurePattern { + matchers: &["HEALTHCHECK_MISSING_SUPERVISOR"], + match_mode: MatchMode::Any, + diagnose: diagnose_missing_supervisor, + }, // TLS/certificate issues FailurePattern { matchers: &[ @@ -342,6 +348,33 @@ fn diagnose_node_pressure(gateway_name: &str) -> GatewayFailureDiagnosis { } } +fn diagnose_missing_supervisor(gateway_name: &str) -> GatewayFailureDiagnosis { + GatewayFailureDiagnosis { + summary: "Sandbox supervisor binary missing from cluster image".to_string(), + explanation: "The sandbox supervisor binary (/opt/openshell/bin/openshell-sandbox) \ + was not found in the gateway container. This binary is side-loaded into every \ + sandbox pod via a hostPath volume mount. Without it, all sandbox pods will \ + crash immediately with \"no such file or directory\". This typically means the \ + cluster image was built or published without the supervisor-builder stage." + .to_string(), + recovery_steps: vec![ + RecoveryStep::with_command( + "Rebuild the cluster image with the supervisor binary included", + "mise run docker:build:cluster", + ), + RecoveryStep::with_command( + "Destroy and recreate the gateway with the updated image", + format!("openshell gateway destroy {gateway_name} && openshell gateway start"), + ), + RecoveryStep::new( + "Or set OPENSHELL_CLUSTER_IMAGE to a cluster image version that includes \ + the supervisor binary", + ), + ], + retryable: false, + } +} + fn diagnose_certificate_issue(gateway_name: &str) -> GatewayFailureDiagnosis { GatewayFailureDiagnosis { summary: "TLS certificate issue".to_string(), diff --git a/crates/openshell-bootstrap/src/runtime.rs b/crates/openshell-bootstrap/src/runtime.rs index 0bc893d84..271fde8d4 100644 --- a/crates/openshell-bootstrap/src/runtime.rs +++ b/crates/openshell-bootstrap/src/runtime.rs @@ -24,6 +24,13 @@ const DNS_FAILURE_MARKERS: &[&str] = &["DNS_PROBE_FAILED", "HEALTHCHECK_DNS_FAIL /// new scheduling, so the cluster will never become healthy on its own. const NODE_PRESSURE_MARKER: &str = "HEALTHCHECK_NODE_PRESSURE"; +/// Log marker emitted by the health-check script when the sandbox supervisor +/// binary (`/opt/openshell/bin/openshell-sandbox`) is missing from the node +/// filesystem. Without this binary, every sandbox pod will crash immediately +/// with "no such file or directory". This is a permanent error that requires +/// rebuilding or updating the cluster image. +const MISSING_SUPERVISOR_MARKER: &str = "HEALTHCHECK_MISSING_SUPERVISOR"; + /// Number of consecutive polling iterations that must observe DNS failure /// markers before we treat the failure as persistent and abort. A small /// grace period avoids false positives on transient hiccups during startup. @@ -116,6 +123,29 @@ where } } + // -- Missing supervisor binary detection ---------------------------- + // The health-check script verifies that /opt/openshell/bin/openshell-sandbox + // exists on the node filesystem. If missing, every sandbox pod will crash. + // This is a permanent error — fail immediately with actionable guidance. + if recent_logs + .iter() + .any(|line| line.contains(MISSING_SUPERVISOR_MARKER)) + { + result = Some(Err(miette::miette!( + "The sandbox supervisor binary is missing from the cluster image.\n\ + The file /opt/openshell/bin/openshell-sandbox was not found in the gateway \ + container. Without it, sandbox pods cannot start.\n\n\ + This usually means the cluster image was built or published without the \ + supervisor-builder stage.\n\n\ + To fix:\n \ + 1. Rebuild the cluster image: mise run docker:build:cluster\n \ + 2. Or update to a cluster image that includes the supervisor binary\n \ + 3. Then recreate the gateway: openshell gateway destroy && openshell gateway start\n\n{}", + format_recent_logs(&recent_logs) + ))); + break; + } + let inspect = docker .inspect_container(&container_name, None::) .await diff --git a/deploy/docker/cluster-healthcheck.sh b/deploy/docker/cluster-healthcheck.sh index a3b373030..68210b456 100644 --- a/deploy/docker/cluster-healthcheck.sh +++ b/deploy/docker/cluster-healthcheck.sh @@ -50,6 +50,18 @@ done kubectl -n openshell get statefulset/openshell >/dev/null 2>&1 || exit 1 kubectl -n openshell wait --for=jsonpath='{.status.readyReplicas}'=1 statefulset/openshell --timeout=1s >/dev/null 2>&1 || exit 1 +# --------------------------------------------------------------------------- +# Verify the sandbox supervisor binary exists on the node filesystem. +# Sandbox pods mount /opt/openshell/bin as a read-only hostPath volume and +# exec /opt/openshell/bin/openshell-sandbox as their entrypoint. If the binary +# is missing (e.g. cluster image was built without the supervisor-builder +# stage), every sandbox pod will crash with "no such file or directory". +# --------------------------------------------------------------------------- +if [ ! -x /opt/openshell/bin/openshell-sandbox ]; then + echo "HEALTHCHECK_MISSING_SUPERVISOR: /opt/openshell/bin/openshell-sandbox not found" >&2 + exit 1 +fi + # Verify TLS secrets exist (created by openshell-bootstrap before the StatefulSet starts) # Skip when TLS is disabled — secrets are not required. if [ "${DISABLE_TLS:-}" != "true" ]; then