From 14859fd48e65e78404a0638154e0ca596e2e8d03 Mon Sep 17 00:00:00 2001
From: Drew Newberry <anewberry@nvidia.com>
Date: Thu, 19 Feb 2026 21:39:46 -0800
Subject: [PATCH] fix(cluster): remove stale image on destroy and verify
 architecture after pull

Destroy now removes the cluster image so the next deploy always pulls the
latest version from the registry.  After pulling on a remote host the
image architecture is verified against the host platform, surfacing a
clear error instead of a cryptic 'exec format error' at container start.
---
 crates/navigator-bootstrap/src/docker.rs | 33 +++++++++++++++++++++++-
 crates/navigator-bootstrap/src/image.rs  | 23 +++++++++++++++++
 2 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/crates/navigator-bootstrap/src/docker.rs b/crates/navigator-bootstrap/src/docker.rs
index 391b8ae82..e12af7cd3 100644
--- a/crates/navigator-bootstrap/src/docker.rs
+++ b/crates/navigator-bootstrap/src/docker.rs
@@ -9,7 +9,7 @@ use bollard::models::{
 };
 use bollard::query_parameters::{
     CreateContainerOptions, CreateImageOptions, InspectContainerOptions, InspectNetworkOptions,
-    RemoveContainerOptions, RemoveVolumeOptions, StartContainerOptions,
+    RemoveContainerOptions, RemoveImageOptions, RemoveVolumeOptions, StartContainerOptions,
 };
 use futures::StreamExt;
 use miette::{IntoDiagnostic, Result, WrapErr};
@@ -427,6 +427,15 @@ pub async fn destroy_cluster_resources(
     let container_name = container_name(name);
     let volume_name = volume_name(name);
 
+    // Capture the container's image reference before removing the container so
+    // we can clean it up afterwards.  This prevents stale images from being
+    // re-used on subsequent deploys.
+    let container_image = docker
+        .inspect_container(&container_name, None::<InspectContainerOptions>)
+        .await
+        .ok()
+        .and_then(|info| info.image);
+
     let _ = stop_container(docker, &container_name).await;
 
     let remove_container = docker
@@ -444,6 +453,28 @@ pub async fn destroy_cluster_resources(
         return Err(err).into_diagnostic();
     }
 
+    // Remove the cluster image so the next deploy always pulls the latest
+    // version from the registry instead of reusing a stale local copy.
+    if let Some(ref image_id) = container_image {
+        tracing::debug!("Removing cluster image: {}", image_id);
+        let result = docker
+            .remove_image(
+                image_id,
+                Some(RemoveImageOptions {
+                    force: true,
+                    noprune: true,
+                    ..Default::default()
+                }),
+                None,
+            )
+            .await;
+        if let Err(err) = result
+            && !is_not_found(&err)
+        {
+            tracing::warn!("Failed to remove cluster image {}: {}", image_id, err);
+        }
+    }
+
     let remove_volume = docker
         .remove_volume(&volume_name, Some(RemoveVolumeOptions { force: true }))
         .await;
diff --git a/crates/navigator-bootstrap/src/image.rs b/crates/navigator-bootstrap/src/image.rs
index 150c6e01e..4e2d178ca 100644
--- a/crates/navigator-bootstrap/src/image.rs
+++ b/crates/navigator-bootstrap/src/image.rs
@@ -252,6 +252,29 @@ pub async fn pull_remote_image(
             format!("failed to tag {registry_image} as {target_repo}:{target_tag} on remote")
         })?;
 
+    // Verify that the pulled image matches the expected architecture.
+    // This catches cases where the registry returned the wrong platform
+    // variant (e.g., amd64 on an arm64 host) which would cause an
+    // "exec format error" at container start time.
+    let inspect = remote
+        .inspect_image(image_ref)
+        .await
+        .into_diagnostic()
+        .wrap_err_with(|| format!("failed to inspect pulled image {image_ref} on remote"))?;
+
+    let actual_arch = inspect.architecture.as_deref().unwrap_or("unknown");
+    if actual_arch != remote_platform.arch {
+        return Err(miette::miette!(
+            "architecture mismatch: pulled image {image_ref} is {actual_arch} but remote host is {expected}; \
+             try removing stale images on the remote host and re-deploying",
+            expected = remote_platform.arch,
+        ));
+    }
+    info!(
+        "Verified image architecture: {} matches remote host",
+        actual_arch
+    );
+
     on_progress(format!("[status] Image {image_ref} ready on remote host"));
     info!("Remote image pull and tag complete: {}", image_ref);