chore(ci): speed up ci builds and improve caching (!20)

2026-10-01 23:28:42 +08:00 · 2026-02-13 11:06:25 -08:00
parent ae7378e5ce
commit 4ef5c0d2dd
17 changed files with 755 additions and 133 deletions
@@ -17,7 +17,7 @@ Diagnose why a navigator cluster failed to start after `nav cluster admin deploy
 4. Create and start a privileged Docker container (`navigator-cluster-{name}`)
 5. Wait for k3s to generate kubeconfig (up to 60s)
 6. **Clean stale nodes**: Remove any `NotReady` k3s nodes left over from previous container instances that reused the same persistent volume
-7. **Push local images** (if `NAVIGATOR_PUSH_IMAGES` is set): Export locally-built component images from the Docker daemon and import them into the k3s containerd runtime via `k3s ctr -n k8s.io images import`. This "push" path is used by `mise run cluster` so the cluster uses locally-built server/sandbox/pki-job images instead of pulling from the remote registry.
+7. **Prepare local images** (if `NAVIGATOR_PUSH_IMAGES` is set): In `internal` registry mode, bootstrap waits for the in-cluster registry and pushes tagged images there. In `external` mode, bootstrap uses legacy `ctr -n k8s.io images import` push-mode behavior.
 8. Wait for cluster health checks to pass (up to 6 min):
   - k3s API server readiness (`/readyz`)
   - `navigator` deployment available in `navigator` namespace
@@ -149,7 +149,7 @@ docker exec navigator-cluster-<name> sh -lc 'KUBECONFIG=/etc/rancher/k3s/k3s.yam

 Common issues:

- **ImagePullBackOff**: The component image failed to pull. When using push mode (`mise run cluster`), verify that images were imported into the k8s.io containerd namespace (see Step 6). When using pull mode (remote deploy or manual `nav cluster admin deploy`), check that `/etc/rancher/k3s/registries.yaml` exists with correct credentials and that DNS is working (Step 8). The remote registry is `d1i0nduu2f6qxk.cloudfront.net/navigator/`.
+- **ImagePullBackOff**: The component image failed to pull. In `internal` mode, verify internal registry readiness and pushed image tags (Step 6). In `external` mode, check `/etc/rancher/k3s/registries.yaml` credentials/endpoints and DNS (Step 8). Default external registry is `d1i0nduu2f6qxk.cloudfront.net/navigator/`.
 - **CrashLoopBackOff**: The server is crashing. Check pod logs for the actual error.
 - **Pending**: Insufficient resources or scheduling constraints.

@@ -184,9 +184,19 @@ If using Docker-in-Docker (`DOCKER_HOST=tcp://docker:2375`), verify metadata poi

 ### Step 6: Check Image Availability

-Component images (server, sandbox, pki-job) can reach k3s containerd via two paths:
+Component images (server, sandbox, pki-job) can reach kubelet via two paths:

-**Push mode** (local development via `mise run cluster` or `mise run cluster:deploy`): Images are built locally and imported into the k3s containerd `k8s.io` namespace. The HelmChart is configured with `pullPolicy: IfNotPresent` and uses the `IMAGE_TAG` (default `dev`).
+**Local/external pull mode** (default local via `mise run cluster`): Images are built locally, tagged to the configured local registry base (default `127.0.0.1:5000/navigator/*`), pushed to that registry, and pulled by k3s via `registries.yaml` mirror endpoint (typically `host.docker.internal:5000`).
+
+```bash
+# Verify image refs currently used by navigator deployment
+docker exec navigator-cluster-<name> sh -lc 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n navigator get deploy navigator -o jsonpath="{.spec.template.spec.containers[*].image}"'
+
+# Verify registry mirror/auth endpoint configuration
+docker exec navigator-cluster-<name> cat /etc/rancher/k3s/registries.yaml
+```
+
+**Legacy push mode** (`mise run cluster:push`): Images are imported into the k3s containerd `k8s.io` namespace.

 ```bash
 # Check if images were imported into containerd (k3s default namespace is k8s.io)
@@ -199,7 +209,7 @@ If images are missing, re-import with:
 docker save <image-ref> | docker exec -i navigator-cluster-<name> ctr -a /run/k3s/containerd/containerd.sock images import -
 ```

-**Pull mode** (remote deploy or manual `nav cluster admin deploy` without `NAVIGATOR_PUSH_IMAGES`): Images are pulled from the distribution registry at runtime. The entrypoint generates `/etc/rancher/k3s/registries.yaml`.
+**External pull mode** (remote deploy, or local with `NAVIGATOR_REGISTRY_HOST`/`IMAGE_REPO_BASE` pointing at a non-local registry): Images are pulled from an external registry at runtime. The entrypoint generates `/etc/rancher/k3s/registries.yaml`.

 ```bash
 # Verify registries.yaml exists and has credentials
@@ -209,7 +219,7 @@ docker exec navigator-cluster-<name> cat /etc/rancher/k3s/registries.yaml
 docker exec navigator-cluster-<name> sh -lc 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml crictl pull d1i0nduu2f6qxk.cloudfront.net/navigator/pki-job:latest'
 ```

-If `registries.yaml` is missing or has wrong credentials, the cluster image may need to be rebuilt. The file should contain auth for `d1i0nduu2f6qxk.cloudfront.net`.
+If `registries.yaml` is missing or has wrong values, verify env wiring (`NAVIGATOR_REGISTRY_HOST`, `NAVIGATOR_REGISTRY_INSECURE`, username/password for authenticated registries).

 ### Step 7: Check mTLS / PKI

@@ -25,7 +25,6 @@ htmlcov

 # Build/dist artifacts
 dist
-build
 *.egg

 # Kubernetes/local dev
@@ -13,7 +13,12 @@ workflow:

 variables:
  CARGO_TERM_COLOR: "always"
+  CARGO_INCREMENTAL: "0"
  CI_IMAGE: $CI_REGISTRY_IMAGE/ci:latest
+  CI_IMAGE_CACHE: $CI_REGISTRY_IMAGE/ci:buildcache
+  MISE_AUTO_INSTALL: "0"
+  MISE_TASK_RUN_AUTO_INSTALL: "0"
+  MISE_NOT_FOUND_AUTO_INSTALL: "0"

 default:
  image: $CI_IMAGE
@@ -21,14 +26,74 @@ default:
    - agent-dev-kit-build

 cache:
-  key: "$CI_COMMIT_REF_SLUG"
-  paths:
-    - target/
-    - .venv/
+  - key:
+      files:
+        - uv.lock
+    paths:
+      - .venv/
+      - .cache/uv/
+  - key:
+      files:
+        - Cargo.lock
+      prefix: "sccache-$CI_RUNNER_EXECUTABLE_ARCH"
+    paths:
+      - .cache/sccache/
+  - key:
+      files:
+        - Cargo.lock
+        - build/rust.toml
+      prefix: "target-$CI_RUNNER_EXECUTABLE_ARCH"
+    paths:
+      - target/
+  - key: "buildkit-$CI_COMMIT_REF_SLUG"
+    paths:
+      - .cache/buildkit/

-# Install Python dependencies (cached in .venv/)
-before_script:
-  - uv sync --frozen
+.rust_job_rules:
+  rules:
+    - changes:
+        - Cargo.toml
+        - Cargo.lock
+        - crates/**/*
+        - proto/**/*
+        - build/rust.toml
+        - build/test.toml
+        - build/ci.toml
+        - mise.toml
+        - .gitlab-ci.yml
+    - when: never
+
+.python_job_rules:
+  rules:
+    - changes:
+        - pyproject.toml
+        - uv.lock
+        - python/**/*
+        - scripts/**/*
+        - proto/**/*
+        - build/python.toml
+        - build/test.toml
+        - build/ci.toml
+        - mise.toml
+        - .gitlab-ci.yml
+    - when: never
+
+.e2e_job_rules:
+  rules:
+    - changes:
+        - e2e/**/*
+        - deploy/docker/**/*
+        - deploy/helm/**/*
+        - deploy/kube/**/*
+        - build/cluster.toml
+        - build/docker.toml
+        - build/test.toml
+        - build/scripts/**/*
+        - crates/**/*
+        - proto/**/*
+        - mise.toml
+        - .gitlab-ci.yml
+    - when: never

 # =============================================================================
 # CI Image Build (runs only when dependencies change)
@@ -38,40 +103,58 @@ build_ci_image:
  image: docker:24
  services:
    - docker:24-dind
+  cache: []
  tags:
    - agent-dev-kit-build
  variables:
-    DOCKER_TLS_CERTDIR: "/certs"
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_TLS_CERTDIR: ""
+    DOCKER_TLS_VERIFY: ""
+    DOCKER_CERT_PATH: ""
    DOCKER_BUILDKIT: "1"
  before_script: []  # Override default before_script (docker:24 doesn't have uv)
-  when: manual
-  allow_failure: true
+  rules:
+    - changes:
+        - deploy/docker/Dockerfile.ci
+        - mise.toml
+        - build/**/*
+        - .gitlab-ci.yml
+    - when: never
  script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - until docker info >/dev/null 2>&1; do sleep 1; done
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
+    - docker buildx create --use --name ci-builder || docker buildx use ci-builder
    - |
-      docker build \
-        --cache-from $CI_IMAGE \
+      docker buildx build \
+        --pull \
+        --cache-from type=registry,ref=$CI_IMAGE \
+        --cache-to type=registry,ref=$CI_IMAGE_CACHE,mode=max \
        --build-arg BUILDKIT_INLINE_CACHE=1 \
        -f deploy/docker/Dockerfile.ci \
        -t $CI_IMAGE \
+        --push \
        .
-    - docker push $CI_IMAGE

 # =============================================================================
 # Lint Jobs
 # =============================================================================
 fmt_check:
+  extends: .rust_job_rules
  stage: lint
  script:
    - mise run fmt:check

 clippy:
+  extends: .rust_job_rules
  stage: lint
  script:
    - mise run clippy

 python_lint:
+  extends: .python_job_rules
  stage: lint
+  before_script:
+    - uv sync --frozen
  script:
    - mise run python:lint

@@ -79,16 +162,21 @@ python_lint:
 # Test Jobs
 # =============================================================================
 rust_test:
+  extends: .rust_job_rules
  stage: test
  script:
    - mise run test:rust

 python_test:
+  extends: .python_job_rules
  stage: test
+  before_script:
+    - uv sync --frozen
  script:
    - mise run test:python

 python_e2e_sandbox_test:
+  extends: .e2e_job_rules
  stage: test
  services:
    - docker:24-dind
@@ -99,13 +187,12 @@ python_e2e_sandbox_test:
    DOCKER_TLS_CERTDIR: ""
    DOCKER_TLS_VERIFY: ""
    DOCKER_CERT_PATH: ""
+    IMAGE_TAG: "$CI_COMMIT_SHA"
+    IMAGE_REPO_BASE: "$CI_REGISTRY_IMAGE"
+    NAVIGATOR_REGISTRY_INSECURE: "false"
  before_script:
    - uv sync --frozen
-    - apt-get update -qq && apt-get install -y -qq socat >/dev/null
-    - curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz | tar xz --strip-components=1 -C /usr/local/bin docker/docker
-    - mkdir -p /usr/local/lib/docker/cli-plugins
-    - curl -fsSL https://github.com/docker/buildx/releases/download/v0.21.1/buildx-v0.21.1.linux-amd64 -o /usr/local/lib/docker/cli-plugins/docker-buildx
-    - chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
  script:
    - socat UNIX-LISTEN:/var/run/docker.sock,fork,reuseaddr TCP:docker:2375 &
    - sleep 1
@@ -131,15 +218,6 @@ publish_ecr_images:
    - if: $CI_COMMIT_BRANCH == "main"
  before_script:
    - uv sync --frozen
-    # Install Docker CLI + buildx plugin (CI image has mise/helm but no Docker client)
-    - curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-27.5.1.tgz | tar xz --strip-components=1 -C /usr/local/bin docker/docker
-    - mkdir -p /usr/local/lib/docker/cli-plugins
-    - curl -fsSL https://github.com/docker/buildx/releases/download/v0.21.1/buildx-v0.21.1.linux-amd64 -o /usr/local/lib/docker/cli-plugins/docker-buildx
-    - chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx
-    # Install AWS CLI
-    - apt-get update -qq && apt-get install -y -qq unzip >/dev/null
-    - curl -fsSL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o /tmp/awscliv2.zip
-    - unzip -q /tmp/awscliv2.zip -d /tmp && /tmp/aws/install && rm -rf /tmp/aws /tmp/awscliv2.zip
    # Authenticate to ECR
    - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 012345678901.dkr.ecr.us-west-2.amazonaws.com
  script:
@@ -150,7 +150,38 @@ The project uses the Navigator CLI to provision a local k3s-in-container cluster

 ```bash
 mise run cluster          # Build and deploy local k3s cluster with Navigator
-mise run cluster:deploy   # Deploy changes to existing cluster (rebuilds images and upgrades helm release)
+mise run cluster:deploy   # Fast deploy: rebuild changed components and skip unnecessary helm work
+mise run cluster:push:server    # Push local server image to configured pull registry
+mise run cluster:push:sandbox   # Push local sandbox image to configured pull registry
+mise run cluster:push:pki-job   # Push local pki-job image to configured pull registry
+mise run cluster:deploy:pull    # Force full pull-mode deploy flow
+mise run cluster:push           # Legacy image-import fallback workflow
+```
+
+Default local cluster workflow uses pull mode with a local Docker registry at `127.0.0.1:5000`.
+You can override repository settings with:
+
+- `IMAGE_REPO_BASE` (for example `127.0.0.1:5000/navigator`)
+- `NAVIGATOR_REGISTRY_HOST`, `NAVIGATOR_REGISTRY_NAMESPACE`
+- `NAVIGATOR_REGISTRY_ENDPOINT` (optional mirror endpoint override, e.g. `host.docker.internal:5000`)
+- `NAVIGATOR_REGISTRY_USERNAME`, `NAVIGATOR_REGISTRY_PASSWORD`
+- `NAVIGATOR_REGISTRY_INSECURE=true|false`
+
+Useful env flags for fast deploy:
+
+- `FORCE_HELM_UPGRADE=1` - run Helm upgrade even when chart files are unchanged
+- `DEPLOY_FAST_HELM_WAIT=1` - wait for Helm upgrade completion (`helm --wait`)
+- `DEPLOY_FAST_MODE=full` - force full component rebuild behavior through fast deploy
+- `DOCKER_BUILD_CACHE_DIR=.cache/buildkit` - local BuildKit cache directory used by component image builds
+
+GitLab Container Registry mapping (CI or shared dev):
+
+```bash
+export NAVIGATOR_REGISTRY_HOST=${CI_REGISTRY}
+export NAVIGATOR_REGISTRY_NAMESPACE=${CI_PROJECT_PATH}
+export NAVIGATOR_REGISTRY_USERNAME=${CI_REGISTRY_USER}
+export NAVIGATOR_REGISTRY_PASSWORD=${CI_REGISTRY_PASSWORD}
+export IMAGE_REPO_BASE=${CI_REGISTRY}/${CI_PROJECT_PATH}
 ```

 The cluster exposes ports 80/443 for gateway traffic and 6443 for the Kubernetes API.
@@ -168,7 +199,7 @@ defaults to `navigator`).

 ### Debugging Cluster Issues

-If a cluster fails to start or is unhealthy after `nav cluster admin deploy`, use the `debug-navigator-cluster` skill (located at `.claude/skills/debug-navigator-cluster/SKILL.md`) to diagnose the issue. This skill provides step-by-step instructions for troubleshooting cluster bootstrap failures, health check errors, and other infrastructure problems.
+If a cluster fails to start or is unhealthy after `nav cluster admin deploy`, use the `debug-navigator-cluster` skill (located at `.agent/skills/debug-navigator-cluster/SKILL.md`) to diagnose the issue. This skill provides step-by-step instructions for troubleshooting cluster bootstrap failures, health check errors, and other infrastructure problems.

 ### Docker Build Tasks

@@ -110,6 +110,34 @@ All builds use mise tasks defined in `build/*.toml` (included from `mise.toml`):
 | `K3S_VERSION`        | `v1.29.8-k3s1` | k3s version for cluster image           |
 | `CLUSTER_NAME`       | `navigator`    | Name for local cluster deployment       |

+### Build Caching
+
+Container builds use Docker BuildKit local caches under `.cache/buildkit/`:
+
+- `build/scripts/docker-build-component.sh` stores per-component caches in `.cache/buildkit/<component>`
+- `build/scripts/docker-build-cluster.sh` stores the cluster image cache in `.cache/buildkit/cluster`
+- Rust-heavy Dockerfiles use BuildKit cache mounts for cargo registry and target directories keyed by image and target architecture, with `sharing=locked` to avoid concurrent cache corruption in parallel CI builds
+- When the active buildx driver is `docker` (instead of `docker-container`), local cache import/export flags are skipped automatically because that driver cannot export local caches
+
+In CI, caching `.cache/buildkit/` between pipeline runs avoids recompiling unchanged Rust dependencies and reduces repeated image rebuild time.
+
+The `python_e2e_sandbox_test` job does not use a localhost registry. It tags and pushes component images to the GitLab project registry (`$CI_REGISTRY_IMAGE`) and configures cluster bootstrap to pull from that remote registry with CI credentials.
+
+The `build_ci_image` job also publishes and reuses a registry-backed BuildKit cache at `$CI_REGISTRY_IMAGE/ci:buildcache`, so layer cache survives across runners and pipelines even when local cache directories are cold.
+
+Rust lint/test jobs also cache `.cache/sccache/` and `target/` with keys derived from `Cargo.lock` and Rust task config files (scoped per runner architecture) so branches can reuse compilation artifacts. CI sets `CARGO_INCREMENTAL=0` to favor deterministic clean builds over incremental metadata churn.
+
+### CI Runner Image
+
+`deploy/docker/Dockerfile.ci` pre-installs tools used by pipeline jobs so they do not download at runtime:
+
+- Docker CLI and buildx plugin for DinD-based image build/publish jobs
+- AWS CLI v2 for ECR authentication and image publishing
+- `uv` installed directly from Astral's installer script (avoids GitHub API rate-limit failures during image builds)
+- `sccache` installed on amd64 CI images (skipped on arm64 where the pinned aqua package is unavailable)
+- `socat` for Docker socket forwarding in sandbox e2e tests
+- The CI image build context must include `build/` because `Dockerfile.ci` copies build task includes from that directory
+
 ## Helm Chart

 The Navigator Helm chart (`deploy/helm/navigator/`) deploys the server to Kubernetes.
@@ -8,84 +8,24 @@ depends = [
  "docker:build:sandbox",
  "docker:build:pki-job",
 ]
-run = """
-#!/usr/bin/env bash
-set -euo pipefail
-
-CLUSTER_NAME=${CLUSTER_NAME:-navigator}
-IMAGE_TAG=${IMAGE_TAG:-dev}
-REGISTRY=${NAVIGATOR_REGISTRY:-d1i0nduu2f6qxk.cloudfront.net/navigator}
-
-# Tag locally-built component images with distribution registry paths.
-# These tags match what the HelmChart manifest expects, so when the
-# bootstrap code imports them into k3s containerd the pods can resolve
-# images locally without pulling from the network.
-for component in server sandbox pki-job; do
-  docker tag "navigator-${component}:${IMAGE_TAG}" "${REGISTRY}/${component}:${IMAGE_TAG}"
-done
-
-# Tell the bootstrap code which images to push into k3s containerd.
-export NAVIGATOR_PUSH_IMAGES="${REGISTRY}/server:${IMAGE_TAG},${REGISTRY}/sandbox:${IMAGE_TAG},${REGISTRY}/pki-job:${IMAGE_TAG}"
-
-# Deploy the cluster using the locally built image
-nav cluster admin deploy --name "${CLUSTER_NAME}" --update-kube-config
-
-echo ""
-echo "Cluster '${CLUSTER_NAME}' is ready."
-echo "KUBECONFIG has been updated."
-"""
+run = "build/scripts/cluster-bootstrap.sh"

 ["cluster:deploy"]
-description = "Build and deploy changes to existing cluster using local images"
-depends = [
-  "docker:build:server",
-  "docker:build:sandbox",
-  "docker:build:pki-job",
-]
-run = """
-#!/usr/bin/env bash
-set -euo pipefail
+description = "Fast deploy: rebuild changed components and skip unnecessary helm work"
+run = "build/scripts/cluster-deploy-fast.sh"

-CLUSTER_NAME=${CLUSTER_NAME:-navigator}
-CONTAINER_NAME="navigator-cluster-${CLUSTER_NAME}"
-REGISTRY=${NAVIGATOR_REGISTRY:-d1i0nduu2f6qxk.cloudfront.net/navigator}
-IMAGE_TAG=${IMAGE_TAG:-dev}
+["cluster:deploy:all"]
+description = "Pull-mode deploy using local registry pushes"
+run = "build/scripts/cluster-deploy-fast.sh all"

-# Check if cluster container is running
-if ! docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then
-  echo "Error: Cluster container '${CONTAINER_NAME}' is not running."
-  echo "Start the cluster first with: mise run cluster"
-  exit 1
-fi
+["cluster:push:server"]
+description = "Tag and push server image to pull registry"
+run = "build/scripts/cluster-push-component.sh server"

-# Tag locally-built images with distribution registry paths
-for component in server sandbox pki-job; do
-  docker tag "navigator-${component}:${IMAGE_TAG}" "${REGISTRY}/${component}:${IMAGE_TAG}"
-done
+["cluster:push:sandbox"]
+description = "Tag and push sandbox image to pull registry"
+run = "build/scripts/cluster-push-component.sh sandbox"

-# Import images into k3s containerd
-echo "Importing local component images into cluster..."
-docker save \
-  "${REGISTRY}/server:${IMAGE_TAG}" \
-  "${REGISTRY}/sandbox:${IMAGE_TAG}" \
-  "${REGISTRY}/pki-job:${IMAGE_TAG}" \
-  | docker exec -i "${CONTAINER_NAME}" ctr -a /run/k3s/containerd/containerd.sock images import -
-
-echo "Upgrading helm release to use local images..."
-helm upgrade navigator deploy/helm/navigator \
-  --namespace navigator \
-  --set image.repository=${REGISTRY}/server \
-  --set image.tag=${IMAGE_TAG} \
-  --set image.pullPolicy=IfNotPresent \
-  --set server.sandboxImage=${REGISTRY}/sandbox:${IMAGE_TAG} \
-  --set gateway.tls.enabled=true \
-  --set gateway.tls.listenerPort=443 \
-  --set gateway.tls.jobImage=${REGISTRY}/pki-job:${IMAGE_TAG} \
-  --wait
-
-echo "Restarting deployment to pick up new images..."
-kubectl rollout restart deployment/navigator -n navigator
-kubectl rollout status deployment/navigator -n navigator
-
-echo "Deploy complete!"
-"""
+["cluster:push:pki-job"]
+description = "Tag and push pki-job image to pull registry"
+run = "build/scripts/cluster-push-component.sh pki-job"
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CLUSTER_NAME=${CLUSTER_NAME:-navigator}
+IMAGE_TAG=${IMAGE_TAG:-dev}
+
+if [ -n "${CI:-}" ] && [ -n "${CI_REGISTRY_IMAGE:-}" ]; then
+  IMAGE_REPO_BASE_DEFAULT=${CI_REGISTRY_IMAGE}
+else
+  IMAGE_REPO_BASE_DEFAULT=localhost:5000/navigator
+fi
+
+IMAGE_REPO_BASE=${IMAGE_REPO_BASE:-${NAVIGATOR_REGISTRY:-${IMAGE_REPO_BASE_DEFAULT}}}
+REGISTRY_HOST=${NAVIGATOR_REGISTRY_HOST:-${IMAGE_REPO_BASE%%/*}}
+REGISTRY_NAMESPACE_DEFAULT=${IMAGE_REPO_BASE#*/}
+
+if [ "${REGISTRY_NAMESPACE_DEFAULT}" = "${IMAGE_REPO_BASE}" ]; then
+  REGISTRY_NAMESPACE_DEFAULT=navigator
+fi
+
+is_local_registry_host() {
+  [ "${REGISTRY_HOST}" = "127.0.0.1:5000" ] || [ "${REGISTRY_HOST}" = "localhost:5000" ]
+}
+
+registry_reachable() {
+  curl -4 -fsS --max-time 2 "http://127.0.0.1:5000/v2/" >/dev/null 2>&1 || \
+    curl -4 -fsS --max-time 2 "http://localhost:5000/v2/" >/dev/null 2>&1
+}
+
+ensure_local_registry() {
+  if registry_reachable; then
+    return
+  fi
+
+  if ! docker inspect navigator-local-registry >/dev/null 2>&1; then
+    docker run -d --restart=always --name navigator-local-registry -p 5000:5000 registry:2 >/dev/null
+  else
+    if ! docker ps --filter "name=^navigator-local-registry$" --filter "status=running" -q | grep -q .; then
+      docker start navigator-local-registry >/dev/null
+    fi
+
+    port_map=$(docker port navigator-local-registry 5000/tcp 2>/dev/null || true)
+    case "${port_map}" in
+      *:5000*)
+        ;;
+      *)
+        docker rm -f navigator-local-registry >/dev/null 2>&1 || true
+        docker run -d --restart=always --name navigator-local-registry -p 5000:5000 registry:2 >/dev/null
+        ;;
+    esac
+  fi
+
+  if registry_reachable; then
+    return
+  fi
+
+  echo "Error: local registry is not reachable at ${REGISTRY_HOST}." >&2
+  echo "       Ensure a registry is running on port 5000 (e.g. docker run -d --name navigator-local-registry -p 5000:5000 registry:2)." >&2
+  docker ps -a >&2 || true
+  docker logs navigator-local-registry >&2 || true
+  exit 1
+}
+
+REGISTRY_ENDPOINT_DEFAULT=${REGISTRY_HOST}
+if is_local_registry_host; then
+  REGISTRY_ENDPOINT_DEFAULT=host.docker.internal:5000
+fi
+
+REGISTRY_INSECURE_DEFAULT=false
+if is_local_registry_host; then
+  REGISTRY_INSECURE_DEFAULT=true
+fi
+
+export NAVIGATOR_REGISTRY_HOST=${NAVIGATOR_REGISTRY_HOST:-${REGISTRY_HOST}}
+export NAVIGATOR_REGISTRY_ENDPOINT=${NAVIGATOR_REGISTRY_ENDPOINT:-${REGISTRY_ENDPOINT_DEFAULT}}
+export NAVIGATOR_REGISTRY_NAMESPACE=${NAVIGATOR_REGISTRY_NAMESPACE:-${REGISTRY_NAMESPACE_DEFAULT}}
+export NAVIGATOR_REGISTRY_INSECURE=${NAVIGATOR_REGISTRY_INSECURE:-${REGISTRY_INSECURE_DEFAULT}}
+export IMAGE_REPO_BASE
+export IMAGE_TAG
+
+if [ -n "${CI:-}" ] && [ -n "${CI_REGISTRY:-}" ] && [ -n "${CI_REGISTRY_USER:-}" ] && [ -n "${CI_REGISTRY_PASSWORD:-}" ]; then
+  printf '%s' "${CI_REGISTRY_PASSWORD}" | docker login -u "${CI_REGISTRY_USER}" --password-stdin "${CI_REGISTRY}"
+  export NAVIGATOR_REGISTRY_USERNAME=${NAVIGATOR_REGISTRY_USERNAME:-${CI_REGISTRY_USER}}
+  export NAVIGATOR_REGISTRY_PASSWORD=${NAVIGATOR_REGISTRY_PASSWORD:-${CI_REGISTRY_PASSWORD}}
+fi
+
+if is_local_registry_host; then
+  ensure_local_registry
+fi
+
+for component in server sandbox pki-job; do
+  build/scripts/cluster-push-component.sh "${component}"
+done
+
+nav cluster admin deploy --name "${CLUSTER_NAME}" --update-kube-config
+
+echo ""
+echo "Cluster '${CLUSTER_NAME}' is ready."
+echo "KUBECONFIG has been updated."
@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CLUSTER_NAME=${CLUSTER_NAME:-navigator}
+CONTAINER_NAME="navigator-cluster-${CLUSTER_NAME}"
+IMAGE_REPO_BASE=${IMAGE_REPO_BASE:-${NAVIGATOR_REGISTRY:-localhost:5000/navigator}}
+IMAGE_TAG=${IMAGE_TAG:-dev}
+RUST_BUILD_PROFILE=${RUST_BUILD_PROFILE:-debug}
+DEPLOY_FAST_MODE=${DEPLOY_FAST_MODE:-auto}
+FORCE_HELM_UPGRADE=${FORCE_HELM_UPGRADE:-0}
+DEPLOY_FAST_HELM_WAIT=${DEPLOY_FAST_HELM_WAIT:-0}
+
+overall_start=$(date +%s)
+
+log_duration() {
+  local label=$1
+  local start=$2
+  local end=$3
+  echo "${label} took $((end - start))s"
+}
+
+if ! docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then
+  echo "Error: Cluster container '${CONTAINER_NAME}' is not running."
+  echo "Start the cluster first with: mise run cluster"
+  exit 1
+fi
+
+build_server=0
+build_sandbox=0
+build_pki_job=0
+needs_helm_upgrade=0
+explicit_target=0
+
+if [[ "$#" -gt 0 ]]; then
+  explicit_target=1
+  build_server=0
+  build_sandbox=0
+  build_pki_job=0
+  needs_helm_upgrade=0
+
+  for target in "$@"; do
+    case "${target}" in
+      server)
+        build_server=1
+        ;;
+      sandbox)
+        build_sandbox=1
+        ;;
+      pki-job)
+        build_pki_job=1
+        ;;
+      chart|helm)
+        needs_helm_upgrade=1
+        ;;
+      all)
+        build_server=1
+        build_sandbox=1
+        build_pki_job=1
+        needs_helm_upgrade=1
+        ;;
+      *)
+        echo "Unknown target '${target}'. Use server, sandbox, pki-job, chart, or all."
+        exit 1
+        ;;
+    esac
+  done
+fi
+
+declare -a changed_files=()
+if [[ "${explicit_target}" == "0" ]]; then
+  detect_start=$(date +%s)
+  mapfile -t changed_files < <(
+    {
+      git diff --name-only
+      git diff --name-only --cached
+      git ls-files --others --exclude-standard
+    } | sort -u
+  )
+  detect_end=$(date +%s)
+  log_duration "Change detection" "${detect_start}" "${detect_end}"
+fi
+
+if [[ "${explicit_target}" == "0" && "${DEPLOY_FAST_MODE}" == "full" ]]; then
+  build_server=1
+  build_sandbox=1
+  build_pki_job=1
+  needs_helm_upgrade=1
+elif [[ "${explicit_target}" == "0" ]]; then
+  for path in "${changed_files[@]}"; do
+    case "${path}" in
+      Cargo.toml|Cargo.lock|proto/*|deploy/docker/cross-build.sh)
+        build_server=1
+        build_sandbox=1
+        ;;
+      crates/navigator-core/*)
+        build_server=1
+        build_sandbox=1
+        ;;
+      crates/navigator-router/*)
+        build_server=1
+        ;;
+      crates/navigator-server/*|deploy/docker/Dockerfile.server)
+        build_server=1
+        ;;
+      crates/navigator-sandbox/*|deploy/docker/Dockerfile.sandbox|python/*|pyproject.toml|uv.lock|dev-sandbox-policy.rego)
+        build_sandbox=1
+        ;;
+      deploy/docker/Dockerfile.pki-job)
+        build_pki_job=1
+        ;;
+      deploy/helm/navigator/*)
+        needs_helm_upgrade=1
+        ;;
+    esac
+  done
+fi
+
+if [[ "${FORCE_HELM_UPGRADE}" == "1" ]]; then
+  needs_helm_upgrade=1
+fi
+
+echo "Fast deploy plan:"
+echo "  build server:  ${build_server}"
+echo "  build sandbox: ${build_sandbox}"
+echo "  build pki-job: ${build_pki_job}"
+echo "  helm upgrade:  ${needs_helm_upgrade}"
+
+if [[ "${explicit_target}" == "0" && "${#changed_files[@]}" -eq 0 && "${DEPLOY_FAST_MODE}" != "full" ]]; then
+  echo "No local changes detected."
+fi
+
+build_start=$(date +%s)
+
+server_pid=""
+sandbox_pid=""
+
+if [[ "${build_server}" == "1" ]]; then
+  if [[ "${build_sandbox}" == "1" ]]; then
+    build/scripts/docker-build-component.sh server &
+    server_pid=$!
+  else
+    build/scripts/docker-build-component.sh server
+  fi
+fi
+
+if [[ "${build_sandbox}" == "1" ]]; then
+  if [[ -n "${server_pid}" ]]; then
+    build/scripts/docker-build-component.sh sandbox --build-arg RUST_BUILD_PROFILE=${RUST_BUILD_PROFILE} &
+    sandbox_pid=$!
+  else
+    build/scripts/docker-build-component.sh sandbox --build-arg RUST_BUILD_PROFILE=${RUST_BUILD_PROFILE}
+  fi
+fi
+
+if [[ -n "${server_pid}" ]]; then
+  wait "${server_pid}"
+fi
+
+if [[ -n "${sandbox_pid}" ]]; then
+  wait "${sandbox_pid}"
+fi
+
+if [[ "${build_pki_job}" == "1" ]]; then
+  build/scripts/docker-build-component.sh pki-job
+fi
+
+build_end=$(date +%s)
+log_duration "Image builds" "${build_start}" "${build_end}"
+
+declare -a pushed_images=()
+
+if [[ "${build_server}" == "1" ]]; then
+  docker tag "navigator-server:${IMAGE_TAG}" "${IMAGE_REPO_BASE}/server:${IMAGE_TAG}"
+  pushed_images+=("${IMAGE_REPO_BASE}/server:${IMAGE_TAG}")
+fi
+
+if [[ "${build_sandbox}" == "1" ]]; then
+  docker tag "navigator-sandbox:${IMAGE_TAG}" "${IMAGE_REPO_BASE}/sandbox:${IMAGE_TAG}"
+  pushed_images+=("${IMAGE_REPO_BASE}/sandbox:${IMAGE_TAG}")
+fi
+
+if [[ "${build_pki_job}" == "1" ]]; then
+  docker tag "navigator-pki-job:${IMAGE_TAG}" "${IMAGE_REPO_BASE}/pki-job:${IMAGE_TAG}"
+  pushed_images+=("${IMAGE_REPO_BASE}/pki-job:${IMAGE_TAG}")
+fi
+
+if [[ "${#pushed_images[@]}" -gt 0 ]]; then
+  push_start=$(date +%s)
+  echo "Pushing updated images to local registry..."
+  for image_ref in "${pushed_images[@]}"; do
+    docker push "${image_ref}"
+  done
+  push_end=$(date +%s)
+  log_duration "Image push" "${push_start}" "${push_end}"
+fi
+
+if [[ "${needs_helm_upgrade}" == "1" ]]; then
+  helm_start=$(date +%s)
+  echo "Upgrading helm release..."
+  helm_wait_args=()
+  if [[ "${DEPLOY_FAST_HELM_WAIT}" == "1" ]]; then
+    helm_wait_args+=(--wait)
+  fi
+
+  helm upgrade navigator deploy/helm/navigator \
+    --namespace navigator \
+    --set image.repository=${IMAGE_REPO_BASE}/server \
+    --set image.tag=${IMAGE_TAG} \
+    --set image.pullPolicy=Always \
+    --set server.sandboxImage=${IMAGE_REPO_BASE}/sandbox:${IMAGE_TAG} \
+    --set gateway.tls.enabled=true \
+    --set gateway.tls.listenerPort=443 \
+    --set gateway.tls.jobImage=${IMAGE_REPO_BASE}/pki-job:${IMAGE_TAG} \
+    "${helm_wait_args[@]}"
+  helm_end=$(date +%s)
+  log_duration "Helm upgrade" "${helm_start}" "${helm_end}"
+fi
+
+if [[ "${#pushed_images[@]}" -gt 0 ]]; then
+  rollout_start=$(date +%s)
+  echo "Restarting deployment to pick up updated images..."
+  kubectl rollout restart deployment/navigator -n navigator
+  kubectl rollout status deployment/navigator -n navigator
+  rollout_end=$(date +%s)
+  log_duration "Rollout" "${rollout_start}" "${rollout_end}"
+else
+  echo "No image updates to roll out."
+fi
+
+overall_end=$(date +%s)
+log_duration "Total deploy" "${overall_start}" "${overall_end}"
+
+echo "Deploy complete!"
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+component=${1:-}
+if [ -z "${component}" ]; then
+  echo "usage: $0 <server|sandbox|pki-job>" >&2
+  exit 1
+fi
+
+case "${component}" in
+  server|sandbox|pki-job)
+    ;;
+  *)
+    echo "invalid component '${component}'; expected server, sandbox, or pki-job" >&2
+    exit 1
+    ;;
+esac
+
+IMAGE_TAG=${IMAGE_TAG:-dev}
+IMAGE_REPO_BASE=${IMAGE_REPO_BASE:-${NAVIGATOR_REGISTRY:-localhost:5000/navigator}}
+
+docker tag "navigator-${component}:${IMAGE_TAG}" "${IMAGE_REPO_BASE}/${component}:${IMAGE_TAG}"
+docker push "${IMAGE_REPO_BASE}/${component}:${IMAGE_TAG}"
@@ -9,6 +9,22 @@
 set -euo pipefail

 IMAGE_TAG=${IMAGE_TAG:-dev}
+DOCKER_BUILD_CACHE_DIR=${DOCKER_BUILD_CACHE_DIR:-.cache/buildkit}
+CACHE_PATH="${DOCKER_BUILD_CACHE_DIR}/cluster"
+
+mkdir -p "${CACHE_PATH}"
+
+CACHE_ARGS=()
+if [[ -n "${CI:-}" ]]; then
+  echo "CI environment detected; skipping local build cache export options."
+elif docker buildx inspect 2>/dev/null | grep -q "Driver: docker-container"; then
+  CACHE_ARGS=(
+    --cache-from "type=local,src=${CACHE_PATH}"
+    --cache-to "type=local,dest=${CACHE_PATH},mode=max"
+  )
+else
+  echo "Buildx driver does not support local cache export; skipping local build cache options."
+fi

 # Create build directory for charts
 mkdir -p deploy/docker/.build/charts
@@ -29,6 +45,7 @@ helm pull oci://docker.io/envoyproxy/gateway-helm \
 echo "Building cluster image..."
 docker buildx build \
  ${DOCKER_PLATFORM:+--platform ${DOCKER_PLATFORM}} \
+  "${CACHE_ARGS[@]}" \
  -f deploy/docker/Dockerfile.cluster \
  -t navigator-cluster:${IMAGE_TAG} \
  --build-arg K3S_VERSION=${K3S_VERSION} \
@@ -11,9 +11,26 @@ COMPONENT=${1:?"Usage: docker-build-component.sh <component> [extra-args...]"}
 shift

 IMAGE_TAG=${IMAGE_TAG:-dev}
+DOCKER_BUILD_CACHE_DIR=${DOCKER_BUILD_CACHE_DIR:-.cache/buildkit}
+CACHE_PATH="${DOCKER_BUILD_CACHE_DIR}/${COMPONENT}"
+
+mkdir -p "${CACHE_PATH}"
+
+CACHE_ARGS=()
+if [[ -n "${CI:-}" ]]; then
+  echo "CI environment detected; skipping local build cache export options."
+elif docker buildx inspect 2>/dev/null | grep -q "Driver: docker-container"; then
+  CACHE_ARGS=(
+    --cache-from "type=local,src=${CACHE_PATH}"
+    --cache-to "type=local,dest=${CACHE_PATH},mode=max"
+  )
+else
+  echo "Buildx driver does not support local cache export; skipping local build cache options."
+fi

 docker buildx build \
  ${DOCKER_PLATFORM:+--platform ${DOCKER_PLATFORM}} \
+  "${CACHE_ARGS[@]}" \
  -f "deploy/docker/Dockerfile.${COMPONENT}" \
  -t "navigator-${COMPONENT}:${IMAGE_TAG}" \
  "$@" \
@@ -16,6 +16,26 @@ use miette::{IntoDiagnostic, Result, WrapErr};
 use std::collections::HashMap;
 use std::path::Path;

+const REGISTRY_NAMESPACE_DEFAULT: &str = "navigator";
+
+const REGISTRY_MODE_EXTERNAL: &str = "external";
+
+fn env_non_empty(key: &str) -> Option<String> {
+    std::env::var(key)
+        .ok()
+        .map(|v| v.trim().to_string())
+        .filter(|v| !v.is_empty())
+}
+
+fn env_bool(key: &str) -> Option<bool> {
+    env_non_empty(key).map(|value| {
+        matches!(
+            value.to_ascii_lowercase().as_str(),
+            "1" | "true" | "yes" | "on"
+        )
+    })
+}
+
 /// Platform information for a Docker daemon host.
 #[derive(Debug, Clone)]
 pub struct HostPlatform {
@@ -245,7 +265,6 @@ pub async fn ensure_container(
            host_port: Some("443".to_string()),
        }]),
    );
-
    let exposed_ports = vec![
        "6443/tcp".to_string(),
        "80/tcp".to_string(),
@@ -278,11 +297,43 @@ pub async fn ensure_container(
    // Pass extra SANs, SSH gateway config, and registry credentials to the
    // entrypoint so they can be injected into the HelmChart manifest and
    // k3s registries.yaml.
+    let registry_host = env_non_empty("NAVIGATOR_REGISTRY_HOST").unwrap_or_else(pull_registry);
+    let registry_namespace = env_non_empty("NAVIGATOR_REGISTRY_NAMESPACE")
+        .unwrap_or_else(|| REGISTRY_NAMESPACE_DEFAULT.to_string());
+    let image_repo_base = env_non_empty("IMAGE_REPO_BASE")
+        .or_else(|| env_non_empty("NAVIGATOR_IMAGE_REPO_BASE"))
+        .unwrap_or_else(|| format!("{registry_host}/{registry_namespace}"));
+    let registry_insecure = env_bool("NAVIGATOR_REGISTRY_INSECURE").unwrap_or(false);
+    let registry_endpoint = env_non_empty("NAVIGATOR_REGISTRY_ENDPOINT");
+
+    let registry_username = env_non_empty("NAVIGATOR_REGISTRY_USERNAME").or_else(|| {
+        if registry_host == pull_registry() {
+            Some(pull_registry_username())
+        } else {
+            None
+        }
+    });
+    let registry_password = env_non_empty("NAVIGATOR_REGISTRY_PASSWORD").or_else(|| {
+        if registry_host == pull_registry() {
+            Some(pull_registry_password())
+        } else {
+            None
+        }
+    });
+
    let mut env_vars: Vec<String> = vec![
-        format!("REGISTRY_HOST={}", pull_registry()),
-        format!("REGISTRY_USERNAME={}", pull_registry_username()),
-        format!("REGISTRY_PASSWORD={}", pull_registry_password()),
+        format!("REGISTRY_MODE={REGISTRY_MODE_EXTERNAL}"),
+        format!("REGISTRY_HOST={registry_host}"),
+        format!("REGISTRY_INSECURE={registry_insecure}"),
+        format!("IMAGE_REPO_BASE={image_repo_base}"),
    ];
+    if let Some(endpoint) = registry_endpoint {
+        env_vars.push(format!("REGISTRY_ENDPOINT={endpoint}"));
+    }
+    if let (Some(username), Some(password)) = (registry_username, registry_password) {
+        env_vars.push(format!("REGISTRY_USERNAME={username}"));
+        env_vars.push(format!("REGISTRY_PASSWORD={password}"));
+    }
    if !extra_sans.is_empty() {
        env_vars.push(format!("EXTRA_SANS={}", extra_sans.join(",")));
    }
@@ -4,6 +4,14 @@

 FROM ubuntu:24.04

+ARG DOCKER_VERSION=27.5.1
+ARG BUILDX_VERSION=v0.21.1
+ARG TARGETARCH
+ARG KUBECTL_VERSION=v1.35.1
+ARG HELM_VERSION=v4.1.1
+ARG PROTOC_VERSION=29.6
+ARG SCCACHE_VERSION=v0.14.0
+
 ENV DEBIAN_FRONTEND=noninteractive
 ENV MISE_DATA_DIR=/opt/mise
 ENV MISE_CACHE_DIR=/opt/mise/cache
@@ -21,17 +29,77 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
    python3-venv \
    cmake \
    protobuf-compiler \
+    socat \
+    unzip \
    xz-utils \
    && rm -rf /var/lib/apt/lists/*

+# Install Docker CLI and buildx plugin used by CI jobs
+RUN case "$TARGETARCH" in \
+      amd64) docker_arch=x86_64; buildx_arch=amd64 ;; \
+      arm64) docker_arch=aarch64; buildx_arch=arm64 ;; \
+      *) echo "Unsupported TARGETARCH: $TARGETARCH"; exit 1 ;; \
+    esac \
+    && curl -fsSL "https://download.docker.com/linux/static/stable/${docker_arch}/docker-${DOCKER_VERSION}.tgz" \
+    | tar xz --strip-components=1 -C /usr/local/bin docker/docker \
+    && mkdir -p /usr/local/lib/docker/cli-plugins \
+    && curl -fsSL "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-${buildx_arch}" \
+      -o /usr/local/lib/docker/cli-plugins/docker-buildx \
+    && chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx
+
+# Install AWS CLI v2 used for ECR publishing
+RUN case "$TARGETARCH" in \
+      amd64) aws_arch=x86_64 ;; \
+      arm64) aws_arch=aarch64 ;; \
+      *) echo "Unsupported TARGETARCH: $TARGETARCH"; exit 1 ;; \
+    esac \
+    && curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}.zip" -o /tmp/awscliv2.zip \
+    && unzip -q /tmp/awscliv2.zip -d /tmp \
+    && /tmp/aws/install \
+    && rm -rf /tmp/aws /tmp/awscliv2.zip
+
+# Install kubectl, helm, and protoc without GitHub API lookups
+RUN case "$TARGETARCH" in \
+      amd64) karch=amd64; helm_arch=amd64; protoc_arch=x86_64 ;; \
+      arm64) karch=arm64; helm_arch=arm64; protoc_arch=aarch_64 ;; \
+      *) echo "Unsupported TARGETARCH: $TARGETARCH"; exit 1 ;; \
+    esac \
+    && curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${karch}/kubectl" -o /usr/local/bin/kubectl \
+    && chmod +x /usr/local/bin/kubectl \
+    && curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${helm_arch}.tar.gz" -o /tmp/helm.tgz \
+    && tar -xzf /tmp/helm.tgz -C /tmp \
+    && mv "/tmp/linux-${helm_arch}/helm" /usr/local/bin/helm \
+    && chmod +x /usr/local/bin/helm \
+    && rm -rf /tmp/helm.tgz "/tmp/linux-${helm_arch}" \
+    && curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${protoc_arch}.zip" -o /tmp/protoc.zip \
+    && unzip -q /tmp/protoc.zip -d /tmp/protoc \
+    && mv /tmp/protoc/bin/protoc /usr/local/bin/protoc \
+    && chmod +x /usr/local/bin/protoc \
+    && rm -rf /tmp/protoc /tmp/protoc.zip
+
+# Install sccache directly on amd64 (mise/aqua plugin is arch-limited)
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+      curl -fsSL "https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" -o /tmp/sccache.tgz \
+      && tar -xzf /tmp/sccache.tgz -C /tmp \
+      && mv "/tmp/sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" /usr/local/bin/sccache \
+      && chmod +x /usr/local/bin/sccache \
+      && rm -rf /tmp/sccache.tgz "/tmp/sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl"; \
+    else \
+      echo "Skipping sccache install on $TARGETARCH"; \
+    fi
+
 # Install mise
 RUN curl https://mise.run | sh

-# Copy mise.toml and build task includes, then install all tools
+# Copy mise.toml and build task includes, then install core tools
 COPY mise.toml /opt/mise/mise.toml
 COPY build/ /opt/mise/build/
 WORKDIR /opt/mise
-RUN mise trust /opt/mise/mise.toml && mise install
+RUN mise trust /opt/mise/mise.toml && \
+    env -u RUSTC_WRAPPER mise install python rust
+
+# Install uv directly to avoid GitHub API rate limits in CI image builds
+RUN curl -LsSf https://astral.sh/uv/0.10.2/install.sh | sh

 # Set working directory for CI jobs
 WORKDIR /builds
@@ -25,8 +25,8 @@ COPY dev-sandbox-policy.rego ./
 # Build with cross-compilation support.
 # Cache mounts are keyed by TARGETARCH so parallel multi-platform builds
 # don't corrupt each other's cargo registry or build artifacts.
-RUN --mount=type=cache,id=cargo-registry-${TARGETARCH},target=/usr/local/cargo/registry \
-    --mount=type=cache,id=cargo-target-${TARGETARCH},target=/build/target \
+RUN --mount=type=cache,id=cargo-registry-sandbox-${TARGETARCH},sharing=locked,target=/usr/local/cargo/registry \
+    --mount=type=cache,id=cargo-target-sandbox-${TARGETARCH},sharing=locked,target=/build/target \
    . cross-build.sh && \
    if [ "$RUST_BUILD_PROFILE" = "release" ]; then \
        cargo_cross_build --release -p navigator-sandbox; \
@@ -37,7 +37,9 @@ RUN mkdir -p crates/navigator-cli/src crates/navigator-core/src crates/navigator
 COPY proto/ proto/

 # Build dependencies only (cached unless Cargo.toml/lock changes)
-RUN . cross-build.sh && cargo_cross_build --release -p navigator-server 2>/dev/null || true
+RUN --mount=type=cache,id=cargo-registry-server-${TARGETARCH},sharing=locked,target=/usr/local/cargo/registry \
+    --mount=type=cache,id=cargo-target-server-${TARGETARCH},sharing=locked,target=/build/target \
+    . cross-build.sh && cargo_cross_build --release -p navigator-server 2>/dev/null || true

 # Copy actual source code
 COPY crates/ crates/
@@ -46,7 +48,9 @@ COPY crates/ crates/
 RUN touch crates/navigator-server/src/main.rs

 # Build the actual application
-RUN . cross-build.sh && \
+RUN --mount=type=cache,id=cargo-registry-server-${TARGETARCH},sharing=locked,target=/usr/local/cargo/registry \
+    --mount=type=cache,id=cargo-target-server-${TARGETARCH},sharing=locked,target=/build/target \
+    . cross-build.sh && \
    cargo_cross_build --release -p navigator-server && \
    cp "$(cross_output_dir release)/navigator-server" /build/navigator-server

@@ -92,13 +92,25 @@ fi
 # component images from the distribution registry at runtime.
 # Credentials are passed as environment variables by the bootstrap code.
 REGISTRIES_YAML="/etc/rancher/k3s/registries.yaml"
-if [ -n "$REGISTRY_HOST" ] && [ -n "$REGISTRY_USERNAME" ] && [ -n "$REGISTRY_PASSWORD" ]; then
-    echo "Configuring registry credentials for distribution registry"
+if [ -n "${REGISTRY_HOST:-}" ]; then
+    REGISTRY_SCHEME="https"
+    REGISTRY_ENDPOINT="${REGISTRY_ENDPOINT:-${REGISTRY_HOST}}"
+    insecure_value=$(printf '%s' "${REGISTRY_INSECURE:-false}" | tr '[:upper:]' '[:lower:]')
+    if [ "$insecure_value" = "true" ] || [ "$insecure_value" = "1" ] || [ "$insecure_value" = "yes" ] || [ "$insecure_value" = "on" ]; then
+        REGISTRY_SCHEME="http"
+    fi
+
+    echo "Configuring registry mirror for ${REGISTRY_HOST} via ${REGISTRY_ENDPOINT} (${REGISTRY_SCHEME})"
    cat > "$REGISTRIES_YAML" <<REGEOF
 mirrors:
  "${REGISTRY_HOST}":
    endpoint:
-      - "https://${REGISTRY_HOST}"
+      - "${REGISTRY_SCHEME}://${REGISTRY_ENDPOINT}"
+
+REGEOF
+
+    if [ -n "${REGISTRY_USERNAME:-}" ] && [ -n "${REGISTRY_PASSWORD:-}" ]; then
+        cat >> "$REGISTRIES_YAML" <<REGEOF

 configs:
  "${REGISTRY_HOST}":
@@ -106,8 +118,9 @@ configs:
      username: ${REGISTRY_USERNAME}
      password: ${REGISTRY_PASSWORD}
 REGEOF
+    fi
 else
-    echo "Warning: REGISTRY_HOST, REGISTRY_USERNAME, or REGISTRY_PASSWORD not set; skipping registry config"
+    echo "Warning: REGISTRY_HOST not set; skipping registry config"
 fi

 # Copy bundled manifests to k3s manifests directory.
@@ -115,7 +128,10 @@ fi
 # on /var/lib/rancher/k3s overwrites any files baked into that path.
 if [ -d "/opt/navigator/manifests" ]; then
    echo "Copying bundled manifests to k3s..."
-    cp /opt/navigator/manifests/*.yaml /var/lib/rancher/k3s/server/manifests/ 2>/dev/null || true
+    for manifest in /opt/navigator/manifests/*.yaml; do
+        [ ! -f "$manifest" ] && continue
+        cp "$manifest" /var/lib/rancher/k3s/server/manifests/
+    done
 fi

 # ---------------------------------------------------------------------------
@@ -127,6 +143,14 @@ fi
 # images already present in containerd instead of pulling from the registry.
 HELMCHART="/var/lib/rancher/k3s/server/manifests/navigator-helmchart.yaml"

+if [ -n "${IMAGE_REPO_BASE:-}" ] && [ -f "$HELMCHART" ]; then
+    target_tag="${IMAGE_TAG:-latest}"
+    echo "Setting image repository base: ${IMAGE_REPO_BASE}"
+    sed -i -E "s|repository:[[:space:]]*[^[:space:]]+|repository: ${IMAGE_REPO_BASE}/server|" "$HELMCHART"
+    sed -i -E "s|sandboxImage:[[:space:]]*[^[:space:]]+|sandboxImage: ${IMAGE_REPO_BASE}/sandbox:${target_tag}|" "$HELMCHART"
+    sed -i -E "s|jobImage:[[:space:]]*[^[:space:]]+|jobImage: ${IMAGE_REPO_BASE}/pki-job:${target_tag}|" "$HELMCHART"
+fi
+
 # In push mode, use the exact image references that were imported into cluster
 # containerd so the Helm release cannot drift back to remote ":latest" tags.
 if [ -n "${PUSH_IMAGE_REFS:-}" ] && [ -f "$HELMCHART" ]; then
@@ -11,12 +11,12 @@ experimental = true
 [tools]
 python = "3.12"
 rust = "stable"
-kubectl = "latest"
-uv = "latest"
-protoc = "29"
-helm = "latest"
-sccache = "latest"
-"cargo:cargo-edit" = "latest"
+kubectl = "1.35.1"
+uv = "0.10.2"
+protoc = "29.6"
+helm = "4.1.1"
+sccache = "0.14.0"
+"cargo:cargo-edit" = "0.13.8"

 [env]
 _.path = ["{{config_root}}/scripts/bin"]