OpenShell/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh

#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Minimal init for sandbox VMs. Runs as PID 1 inside the guest, mounts the
# essential filesystems, configures networking (gvproxy DHCP or TAP static),
# optionally loads NVIDIA GPU drivers, then execs the OpenShell sandbox
# supervisor.

set -euo pipefail

# libkrun consumes this driver-owned switch before exec'ing this script as
# PID 1. Do not leak the runtime control into the supervisor or workloads.
unset KRUN_INIT_PID1

BOOT_START=$(date +%s%3N 2>/dev/null || date +%s)
# gvisor-tap-vsock subnet layout:
#   192.168.127.1   — gateway: gvproxy's DNS / DHCP / HTTP API. Does NOT
#                     proxy arbitrary host ports.
#   192.168.127.254 — host-loopback: NAT-rewritten to host's 127.0.0.1 by
#                     gvproxy's TCP/UDP/ICMP forwarder. Use this address
#                     (or any of the host.* hostnames below) to reach a
#                     service the host is listening on.
# The host.openshell.internal / host.containers.internal /
# host.docker.internal DNS records served by gvproxy's embedded resolver
# point at 192.168.127.254. We mirror that in /etc/hosts so the supervisor
# can reach the gateway even when gvproxy's DNS is not in resolv.conf
# (e.g. DHCP failed and we fell back to 8.8.8.8).
GVPROXY_GATEWAY_IP="192.168.127.1"
GVPROXY_HOST_LOOPBACK_IP="192.168.127.254"
GATEWAY_IP="$GVPROXY_GATEWAY_IP"

GPU_ENABLED="${GPU_ENABLED:-false}"
VM_NET_IP="${VM_NET_IP:-}"
VM_NET_GW="${VM_NET_GW:-}"
VM_NET_DNS="${VM_NET_DNS:-}"

ts() {
    local now
    now=$(date +%s%3N 2>/dev/null || date +%s)
    local elapsed=$((now - BOOT_START))
    printf "[%d.%03ds] %s\n" $((elapsed / 1000)) $((elapsed % 1000)) "$*"
}

mount_initial_fs() {
    mount -t proc proc /proc 2>/dev/null || true
    mount -t sysfs sysfs /sys 2>/dev/null || true
    mount -t tmpfs tmpfs /tmp 2>/dev/null || true
    mount -t tmpfs tmpfs /run 2>/dev/null || true
    mount -t devtmpfs devtmpfs /dev 2>/dev/null || true
}

bind_mount_into_newroot() {
    local source="$1"
    local target="/newroot${source}"

    mkdir -p "$target" 2>/dev/null || true
    mount --rbind "$source" "$target" 2>/dev/null \
        || mount --bind "$source" "$target" 2>/dev/null \
        || true
}

root_path() {
    local path="$1"
    printf '%s%s\n' "${ROOT_PREFIX:-}" "$path"
}

sandbox_owner() {
    sandbox_owner_from_passwd "$(root_path /etc/passwd)"
}

sandbox_owner_for_root() {
    local root="$1"
    sandbox_owner_from_passwd "$root/etc/passwd"
}

sandbox_owner_from_passwd() {
    local passwd_path name uid gid rest
    passwd_path="$1"
    if [ -f "$passwd_path" ]; then
        while IFS=: read -r name _ uid gid rest; do
            _="${rest:-}"
            if [ "$name" = "sandbox" ] \
                && [[ "$uid" =~ ^[0-9]+$ ]] \
                && [[ "$gid" =~ ^[0-9]+$ ]]; then
                printf '%s:%s\n' "$uid" "$gid"
                return
            fi
        done < "$passwd_path"
    fi

    # New images use the conventional sandbox UID. Existing images retain the
    # sandbox account read above, including the legacy 10001:10001 identity.
    printf '1000:1000\n'
}

source_overlay_env_if_present() {
    local env_file="/overlay/upper/srv/openshell-env.sh"
    if [ -f "$env_file" ]; then
        # shellcheck source=/dev/null
        source "$env_file"
    fi
}

ensure_target_runtime() {
    local image_root="$1"
    local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}"
    local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}"
    local replace_account=0

    # An omitted identity means the image owns its sandbox account contract.
    # Fall back to 1000 only when the image has no sandbox account at all.
    if [ -n "$sandbox_uid" ] || [ -n "$sandbox_gid" ]; then
        sandbox_uid="${sandbox_uid:-1000}"
        sandbox_gid="${sandbox_gid:-$sandbox_uid}"
        replace_account=1
    elif ! grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then
        sandbox_uid=1000
        sandbox_gid=1000
        replace_account=1
    fi

    mkdir -p \
        "$image_root/srv" \
        "$image_root/opt/openshell/bin" \
        "$image_root/sandbox" \
        "$image_root/etc"

    cp /srv/openshell-vm-sandbox-init.sh "$image_root/srv/openshell-vm-sandbox-init.sh"
    chmod 0755 "$image_root/srv/openshell-vm-sandbox-init.sh"

    if [ -x /opt/openshell/bin/openshell-sandbox ]; then
        cp /opt/openshell/bin/openshell-sandbox "$image_root/opt/openshell/bin/openshell-sandbox"
        chmod 0755 "$image_root/opt/openshell/bin/openshell-sandbox"
    fi

    touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow"
    if [ "$replace_account" -eq 1 ]; then
        if grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then
            sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$image_root/etc/group"
        else
            printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$image_root/etc/group"
        fi
        if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then
            printf 'sandbox:!::\n' >> "$image_root/etc/gshadow"
        fi
        if grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then
            sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$image_root/etc/passwd"
        else
            printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$image_root/etc/passwd"
        fi
        if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then
            printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow"
        fi
    fi
    local owner
    owner="$(sandbox_owner_for_root "$image_root")"
    if ! chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then
        ts "FATAL: failed to apply sandbox image ownership (${owner})"
        exit 1
    fi
    chmod 0755 "$image_root/sandbox"
}

prepare_guest_image_rootfs() {
    local payload_dir="/overlay/config/openshell-image"
    local image_root="/overlay/image-rootfs"
    local partial_root="/overlay/image-rootfs.partial"
    local source

    [ -d "$payload_dir" ] || return 0

    source="$(cat "$payload_dir/source" 2>/dev/null || true)"
    ts "preparing sandbox image rootfs in guest (${source:-unknown})"

    rm -rf "$image_root" "$partial_root"

    case "$source" in
        local-docker)
            mkdir -p "$image_root"
            tar -xpf "$payload_dir/source-rootfs.tar" -C "$image_root"
            ;;
        oci-layout)
            if [ ! -x /opt/openshell/bin/umoci ]; then
                ts "FATAL: umoci not found in VM bootstrap image"
                exit 1
            fi
            /opt/openshell/bin/umoci raw unpack \
                --image "$payload_dir/oci:openshell" \
                "$partial_root"
            if [ ! -d "$partial_root/rootfs" ]; then
                ts "FATAL: umoci unpack did not produce rootfs directory"
                exit 1
            fi
            mv "$partial_root/rootfs" "$image_root"
            rm -rf "$partial_root"
            ;;
        *)
            ts "FATAL: unknown guest image payload source: ${source:-missing}"
            exit 1
            ;;
    esac

    ensure_target_runtime "$image_root"
    if [ -f "$payload_dir/identity" ]; then
        cp "$payload_dir/identity" "$image_root/.openshell-rootfs-variant"
    fi
    rm -rf "$payload_dir"
}

# Driver-owned arguments appended to the supervisor's command line.
#
# The VM driver cannot build the supervisor's argv the way the container
# drivers do, so it writes the arguments it chose into the overlay upperdir
# and this script appends them verbatim. Populated by
# read_supervisor_extra_args; empty until then.
SUPERVISOR_EXTRA_ARGS=()

# Upper bound on driver-supplied supervisor arguments.
#
# The corporate proxy settings are the only producer today and top out at ten
# entries. The cap exists so a corrupt or oversized file cannot expand into an
# unbounded command line.
SUPERVISOR_EXTRA_ARGS_MAX=32

read_supervisor_extra_args() {
    # Read the driver-authored supervisor argument list, one argument per
    # line, verbatim -- no word splitting, globbing, or expansion, so values
    # containing spaces (e.g. a NO_PROXY list) survive intact.
    #
    # Security: this is the operator-owned egress boundary. The driver writes
    # this file into the overlay upperdir on every launch, including an empty
    # file when it has no arguments to pass, so the upperdir copy always
    # shadows the read-only image layer. A sandbox image can therefore neither
    # supply its own supervisor arguments by baking a file at this path nor
    # disable the operator's by omitting one. A missing file means the driver
    # passed nothing; a file it cannot read means the overlay is broken, and
    # we fail closed rather than start a supervisor with a silently truncated
    # egress configuration.
    local args_file
    args_file="$(root_path /opt/openshell/supervisor-args)"

    SUPERVISOR_EXTRA_ARGS=()
    if [ ! -f "$args_file" ]; then
        return 0
    fi
    if [ ! -r "$args_file" ]; then
        ts "FATAL: supervisor argument list ${args_file} is not readable"
        exit 1
    fi

    local arg
    while IFS= read -r arg; do
        # render_guest_supervisor_args never emits a blank line, so one means
        # the file was truncated or tampered with after the driver wrote it.
        if [ -z "$arg" ]; then
            ts "FATAL: empty entry in supervisor argument list"
            exit 1
        fi
        if [ "${#SUPERVISOR_EXTRA_ARGS[@]}" -ge "$SUPERVISOR_EXTRA_ARGS_MAX" ]; then
            ts "FATAL: supervisor argument list exceeds ${SUPERVISOR_EXTRA_ARGS_MAX} entries"
            exit 1
        fi
        SUPERVISOR_EXTRA_ARGS+=("$arg")
    done < "$args_file"

    if [ "${#SUPERVISOR_EXTRA_ARGS[@]}" -gt 0 ]; then
        ts "supervisor arguments from driver: ${#SUPERVISOR_EXTRA_ARGS[@]} entries"
    fi
}

exec_supervisor_in_newroot() {
    local chroot_bin
    local bootstrap="/.openshell-bootstrap"
    local supervisor="${bootstrap}/opt/openshell/bin/openshell-sandbox"
    local loader
    local lib_path

    for chroot_bin in /usr/sbin/chroot /usr/bin/chroot /sbin/chroot /bin/chroot; do
        [ -x "$chroot_bin" ] || continue

        if [ -x "/newroot${supervisor}" ]; then
            for loader in \
                "${bootstrap}/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2" \
                "${bootstrap}/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2" \
                "${bootstrap}/lib64/ld-linux-x86-64.so.2" \
                "${bootstrap}/lib/ld-linux-x86-64.so.2" \
                "${bootstrap}/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1" \
                "${bootstrap}/usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1" \
                "${bootstrap}/lib/ld-linux-aarch64.so.1" \
                "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do
                if [ -x "/newroot${loader}" ]; then
                    lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu"
                    exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" "$@"
                fi
            done
            exec "$chroot_bin" /newroot "$supervisor" "$@"
        fi

        if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then
            exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox "$@"
        fi
    done

    ts "FATAL: unable to exec openshell-sandbox in guest rootfs"
    exit 1
}

setup_overlay_root() {
    ts "setting up writable overlay root"
    mount_initial_fs

    if [ ! -b /dev/vdb ]; then
        ts "FATAL: writable overlay disk /dev/vdb not found"
        exit 1
    fi

    mkdir -p /overlay /lower /newroot /image-cache
    mount -o remount,ro / 2>/dev/null || true
    mount -t ext4 -o rw /dev/vdb /overlay
    mkdir -p /overlay/upper /overlay/work
    source_overlay_env_if_present

    if [ "${OPENSHELL_VM_INIT_MODE:-sandbox}" = "image-prep" ]; then
        prepare_guest_image_rootfs
        sync
        if ! umount /overlay; then
            ts "FATAL: failed to unmount /overlay cleanly after image-prep; refusing to produce a dirty image-cache disk"
            exit 1
        fi
        ts "image-prep complete"
        exit 0
    fi

    mount --bind / /lower
    mount -o remount,bind,ro /lower 2>/dev/null || true

    local lower_root="/lower"
    if [ -b /dev/vdc ]; then
        mount -t ext4 -o ro,noload /dev/vdc /image-cache
        if [ -d /image-cache/image-rootfs ]; then
            lower_root="/image-cache/image-rootfs"
            ts "using prepared image rootfs lowerdir"
        else
            ts "FATAL: prepared image disk missing /image-rootfs"
            exit 1
        fi
    fi

    mount -t overlay overlay \
        -o lowerdir="$lower_root",upperdir=/overlay/upper,workdir=/overlay/work \
        /newroot
    mkdir -p /newroot/.openshell-bootstrap
    mount --bind /lower /newroot/.openshell-bootstrap

    # GPU setup runs against the bootstrap runtime and its mounted /dev, /proc,
    # and /run before those filesystems are mirrored into the target root.
    if [ "${GPU_ENABLED}" = "true" ]; then
        setup_gpu || ts "WARNING: GPU init failed; continuing without GPU"
    fi

    bind_mount_into_newroot /proc
    bind_mount_into_newroot /sys
    bind_mount_into_newroot /tmp
    bind_mount_into_newroot /dev
    bind_mount_into_newroot /run

    ROOT_PREFIX="/newroot"
    run_post_overlay_setup
}

parse_endpoint() {
    local endpoint="$1"
    local scheme rest authority path host port

    case "$endpoint" in
        *://*)
            scheme="${endpoint%%://*}"
            rest="${endpoint#*://}"
            ;;
        *)
            return 1
            ;;
    esac

    authority="${rest%%/*}"
    path="${rest#"$authority"}"
    if [ "$path" = "$rest" ]; then
        path=""
    fi

    if [[ "$authority" =~ ^\[([^]]+)\]:(.+)$ ]]; then
        host="${BASH_REMATCH[1]}"
        port="${BASH_REMATCH[2]}"
    elif [[ "$authority" =~ ^\[([^]]+)\]$ ]]; then
        host="${BASH_REMATCH[1]}"
        port=""
    elif [[ "$authority" == *:* ]]; then
        host="${authority%%:*}"
        port="${authority##*:}"
    else
        host="$authority"
        port=""
    fi

    if [ -z "$port" ]; then
        case "$scheme" in
            https) port="443" ;;
            *) port="80" ;;
        esac
    fi

    printf '%s\n%s\n%s\n%s\n' "$scheme" "$host" "$port" "$path"
}

tcp_probe() {
    local host="$1"
    local port="$2"

    if command -v timeout >/dev/null 2>&1; then
        timeout 2 bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1
    else
        bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1
    fi
}

ensure_host_gateway_aliases() {
    # Seed /etc/hosts with the well-known gvproxy hostnames so the supervisor
    # can reach the OpenShell server even when gvproxy's built-in DNS is not
    # in resolv.conf (e.g. when DHCP fails and we fall back to 8.8.8.8).
    #
    # Critical distinction: host.* aliases point at the gvproxy *host-loopback*
    # IP (192.168.127.254), not the gateway IP (192.168.127.1). Only the
    # host-loopback IP carries NAT rewriting to the host's 127.0.0.1 — the
    # gateway IP only listens on gvproxy's own service ports (DNS:53, DHCP,
    # HTTP API:80). Pinning host.containers.internal to the gateway IP
    # silently breaks guest→host port reachability for arbitrary ports.
    local host_aliases="host.openshell.internal host.containers.internal host.docker.internal"
    local gateway_aliases="gateway.containers.internal"
    local filter='(^|[[:space:]])(host\.openshell\.internal|host\.containers\.internal|host\.docker\.internal|gateway\.containers\.internal)([[:space:]]|$)'

    write_host_gateway_aliases "$(root_path /etc/hosts)" "$(root_path "/tmp/openshell-hosts.$$.tmp")" || true
    if [ -n "${ROOT_PREFIX:-}" ]; then
        write_host_gateway_aliases "/etc/hosts" "/tmp/openshell-hosts.$$.tmp" || true
    fi
}

write_host_gateway_aliases() {
    local hosts_path="$1"
    local hosts_tmp="$2"
    mkdir -p "$(dirname "$hosts_path")" 2>/dev/null || true
    mkdir -p "$(dirname "$hosts_tmp")" 2>/dev/null || true
    if [ -f "$hosts_path" ]; then
        grep -vE "$filter" "$hosts_path" > "$hosts_tmp" || true
    else
        : > "$hosts_tmp"
    fi

    # In TAP/GPU mode, GATEWAY_IP is overridden to VM_NET_GW (the host-side
    # of the TAP), and the gateway is reachable directly there. In gvproxy
    # mode, host.openshell.internal etc. need GVPROXY_HOST_LOOPBACK_IP
    # (192.168.127.254) which is gvproxy's host-NAT entry, while
    # gateway.containers.internal points at the gvproxy gateway itself.
    if [ "${GATEWAY_IP}" = "${GVPROXY_GATEWAY_IP}" ]; then
        printf '%s %s\n' "$GVPROXY_HOST_LOOPBACK_IP" "$host_aliases" >> "$hosts_tmp"
        printf '%s %s\n' "$GVPROXY_GATEWAY_IP" "$gateway_aliases" >> "$hosts_tmp"
    else
        # TAP networking: gateway and host are both reachable at GATEWAY_IP.
        printf '%s %s %s\n' "$GATEWAY_IP" "$host_aliases" "$gateway_aliases" >> "$hosts_tmp"
    fi
    if ! cat "$hosts_tmp" > "$hosts_path" 2>/dev/null; then
        rm -f "$hosts_tmp"
        ts "WARNING: could not update ${hosts_path}"
        return 1
    fi
    rm -f "$hosts_tmp"
}

rewrite_openshell_endpoint_if_needed() {
    local endpoint="${OPENSHELL_ENDPOINT:-}"
    [ -n "$endpoint" ] || return 0

    local parsed
    if ! parsed="$(parse_endpoint "$endpoint")"; then
        ts "WARNING: could not parse OPENSHELL_ENDPOINT=$endpoint"
        return 0
    fi

    local scheme host port path
    scheme="$(printf '%s\n' "$parsed" | sed -n '1p')"
    host="$(printf '%s\n' "$parsed" | sed -n '2p')"
    port="$(printf '%s\n' "$parsed" | sed -n '3p')"
    path="$(printf '%s\n' "$parsed" | sed -n '4p')"

    if tcp_probe "$host" "$port"; then
        return 0
    fi

    # Probe candidates in preference order. Hostnames first for informative
    # log output, then a bare IP as a final safety net. In gvproxy mode the
    # bare IP is the host-loopback (192.168.127.254). In TAP/GPU mode it's
    # the TAP host gateway.
    local fallback_ip="$GVPROXY_HOST_LOOPBACK_IP"
    if [ "${GATEWAY_IP}" != "${GVPROXY_GATEWAY_IP}" ]; then
        fallback_ip="$GATEWAY_IP"
    fi
    local candidates="host.openshell.internal host.containers.internal host.docker.internal"
    if [ "$scheme" != "https" ]; then
        candidates="${candidates} ${fallback_ip}"
    fi

    for candidate in $candidates; do
        if [ "$candidate" = "$host" ]; then
            continue
        fi
        if tcp_probe "$candidate" "$port"; then
            local authority="$candidate"
            if ! { [ "$scheme" = "http" ] && [ "$port" = "80" ]; } \
                && ! { [ "$scheme" = "https" ] && [ "$port" = "443" ]; }; then
                authority="${authority}:${port}"
            fi
            export OPENSHELL_ENDPOINT="${scheme}://${authority}${path}"
            ts "rewrote OPENSHELL_ENDPOINT to ${OPENSHELL_ENDPOINT}"
            return 0
        fi
    done

    if [ "$scheme" = "https" ]; then
        ts "WARNING: could not preflight HTTPS OpenShell endpoint ${host}:${port}; preserving hostname for TLS verification"
        return 0
    fi

    ts "WARNING: could not reach OpenShell endpoint ${host}:${port}"
}

create_gpu_device_nodes_mknod() {
    # Mode 666 is intentional: single-tenant microVM with the VM itself as the
    # isolation boundary. The sandbox user is the only non-root user.
    local nv_major
    nv_major=$(awk '$2 == "nvidia" {print $1}' /proc/devices 2>/dev/null || true)
    if [ -n "$nv_major" ]; then
        mknod -m 666 /dev/nvidiactl c "$nv_major" 255 2>/dev/null || true

        local gpu_count=0
        if [ -d /proc/driver/nvidia/gpus ]; then
            for gpu_dir in /proc/driver/nvidia/gpus/*/; do
                [ -d "$gpu_dir" ] || continue
                mknod -m 666 "/dev/nvidia${gpu_count}" c "$nv_major" "$gpu_count" 2>/dev/null || true
                gpu_count=$((gpu_count + 1))
            done
        fi
        if [ "$gpu_count" -eq 0 ]; then
            mknod -m 666 /dev/nvidia0 c "$nv_major" 0 2>/dev/null || true
        fi

        local modeset_major
        modeset_major=$(awk '$2 == "nvidia-modeset" {print $1}' /proc/devices 2>/dev/null || true)
        if [ -n "$modeset_major" ]; then
            mknod -m 666 /dev/nvidia-modeset c "$modeset_major" 254 2>/dev/null || true
        fi

        local uvm_major
        uvm_major=$(awk '$2 == "nvidia-uvm" {print $1}' /proc/devices 2>/dev/null || true)
        if [ -n "$uvm_major" ]; then
            mknod -m 666 /dev/nvidia-uvm c "$uvm_major" 0 2>/dev/null || true
            mknod -m 666 /dev/nvidia-uvm-tools c "$uvm_major" 1 2>/dev/null || true
        fi

        local caps_major
        caps_major=$(awk '$2 == "nvidia-caps" {print $1}' /proc/devices 2>/dev/null || true)
        if [ -n "$caps_major" ]; then
            mkdir -p /dev/nvidia-caps 2>/dev/null || true
            mknod -m 666 /dev/nvidia-caps/nvidia-cap1 c "$caps_major" 1 2>/dev/null || true
            mknod -m 666 /dev/nvidia-caps/nvidia-cap2 c "$caps_major" 2 2>/dev/null || true
        fi

        ts "GPU device nodes created via mknod (${gpu_count} GPU(s), major=${nv_major})"
    else
        ts "WARNING: 'nvidia' not in /proc/devices; device nodes unavailable"
    fi
}

setup_gpu() {
    ts "GPU_ENABLED=true — initializing GPU passthrough"

    if ! command -v modprobe >/dev/null 2>&1; then
        ts "FATAL: modprobe not found; cannot load nvidia kernel modules"
        return 1
    fi

    # Stage GSP firmware to tmpfs so module loading reads it from a stable
    # early-boot path.
    if [ -d /lib/firmware/nvidia ]; then
        ts "staging GPU firmware to tmpfs"
        mkdir -p /run/firmware/nvidia
        cp -a /lib/firmware/nvidia/* /run/firmware/nvidia/ 2>/dev/null || true
        if [ -e /sys/module/firmware_class/parameters/path ]; then
            echo /run/firmware > /sys/module/firmware_class/parameters/path
        fi
    fi

    ts "loading nvidia kernel modules"
    modprobe nvidia || { ts "FATAL: modprobe nvidia failed"; return 1; }
    modprobe nvidia_uvm 2>/dev/null || true
    modprobe nvidia_modeset 2>/dev/null || true

    rm -rf /run/firmware 2>/dev/null || true

    if command -v nvidia-smi >/dev/null 2>&1; then
        ts "running nvidia-smi to create device nodes and validate GPU"
        local smi_rc=0
        nvidia-smi >/dev/null 2>&1 || smi_rc=$?
        if [ "$smi_rc" -eq 0 ]; then
            nvidia-smi -L 2>/dev/null | while read -r line; do ts "  $line"; done
            ts "GPU initialization successful"
        else
            ts "WARNING: nvidia-smi failed (exit ${smi_rc}); falling back to mknod"
            create_gpu_device_nodes_mknod
        fi
    else
        ts "nvidia-smi not found; creating device nodes via mknod"
        create_gpu_device_nodes_mknod
    fi
}

reconcile_sandbox_account() {
    local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}"
    local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}"
    local etc

    [ -n "$sandbox_uid" ] && [ -n "$sandbox_gid" ] || return 0
    [[ "$sandbox_uid" =~ ^[0-9]+$ ]] && [[ "$sandbox_gid" =~ ^[0-9]+$ ]] || {
        ts "FATAL: invalid requested sandbox identity"
        exit 1
    }
    etc="$(root_path /etc)"
    mkdir -p "$etc"
    touch "$etc/passwd" "$etc/group" "$etc/shadow" "$etc/gshadow"
    if grep -q '^sandbox:' "$etc/group"; then
        if ! awk -F: -v OFS=: -v gid="$sandbox_gid" \
            '$1 == "sandbox" { $3 = gid } { print }' \
            "$etc/group" >"$etc/group.openshell"; then
            rm -f "$etc/group.openshell"
            ts "FATAL: failed to reconcile sandbox group"
            exit 1
        fi
        mv "$etc/group.openshell" "$etc/group"
    else
        printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$etc/group"
    fi
    if grep -q '^sandbox:' "$etc/passwd"; then
        # Preserve image-owned account metadata (home, shell, and description)
        # while restoring the UID/GID contract recorded for this overlay.
        if ! awk -F: -v OFS=: -v uid="$sandbox_uid" -v gid="$sandbox_gid" \
            '$1 == "sandbox" { $3 = uid; $4 = gid } { print }' \
            "$etc/passwd" >"$etc/passwd.openshell"; then
            rm -f "$etc/passwd.openshell"
            ts "FATAL: failed to reconcile sandbox account"
            exit 1
        fi
        mv "$etc/passwd.openshell" "$etc/passwd"
    else
        printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$etc/passwd"
    fi
    grep -q '^sandbox:' "$etc/gshadow" || printf 'sandbox:!::\n' >> "$etc/gshadow"
    grep -q '^sandbox:' "$etc/shadow" || printf 'sandbox:!:20123:0:99999:7:::\n' >> "$etc/shadow"
    ts "reconciled sandbox account (${sandbox_uid}:${sandbox_gid})"
}

setup_sandbox_workdir() {
    local sandbox_dir
    local owner
    local current_owner
    sandbox_dir="$(root_path /sandbox)"
    owner="$(sandbox_owner)"
    mkdir -p "$sandbox_dir"
    current_owner="$(stat -c '%u:%g' "$sandbox_dir" 2>/dev/null || true)"
    if [ "$owner" = "10001:10001" ]; then
        ts "preserving legacy sandbox ownership (10001:10001)"
    fi
    if [ "$current_owner" != "$owner" ]; then
        if ! chown -R "$owner" "$sandbox_dir" 2>/dev/null; then
            ts "FATAL: failed to apply sandbox ownership (${owner})"
            exit 1
        fi
    fi
    chmod 0755 "$sandbox_dir"
    ts "prepared /sandbox ownership (${owner})"
}

configure_hostname() {
    local sandbox_hostname="${OPENSHELL_SANDBOX:-openshell-sandbox-vm}"
    sandbox_hostname="$(printf '%s' "$sandbox_hostname" | tr -c 'A-Za-z0-9.-' '-')"
    sandbox_hostname="$(printf '%s' "$sandbox_hostname" | sed 's/^[.-][.-]*//; s/[.-][.-]*$//')"
    sandbox_hostname="$(printf '%.63s' "$sandbox_hostname")"
    if [ -z "$sandbox_hostname" ]; then
        sandbox_hostname="openshell-sandbox-vm"
    fi

    hostname "$sandbox_hostname" 2>/dev/null || true
    printf '%s\n' "$sandbox_hostname" >"$(root_path /etc/hostname)" 2>/dev/null || true
    ts "hostname=${sandbox_hostname}"
}

run_openshell_init_dropins() {
    # Run executable drop-ins from /opt/openshell/init.d in deterministic
    # ASCII-sorted order. Drop-ins are *executed* in a child shell rather
    # than sourced, so they cannot mutate parent shell state or exit the
    # caller. They inherit `OPENSHELL_VM_INIT_PHASE`, `ROOT_PREFIX`, and
    # any `OPENSHELL_VM_DATA_*` env vars set by lifecycle extensions.
    #
    # Security: this runs as root before the supervisor enforces policy, so
    # it executes *only* the drop-ins the VM driver explicitly injected this
    # launch, as enumerated in the driver-authored manifest. Anything else
    # found under init.d (e.g. files baked into a user-controlled guest
    # image) is ignored. The manifest lives in the overlay upperdir, which
    # the driver owns, so a guest image cannot forge or shadow it. A missing
    # manifest is treated as "run nothing" (fail-closed).
    local init_dir manifest
    init_dir="$(root_path /opt/openshell/init.d)"
    manifest="$(root_path /opt/openshell/init.d.manifest)"

    if [ ! -f "$manifest" ]; then
        if [ -d "$init_dir" ]; then
            ts "no OpenShell VM init drop-in manifest present; skipping init.d"
        fi
        return 0
    fi

    export OPENSHELL_VM_INIT_PHASE="before-supervisor"
    export ROOT_PREFIX="${ROOT_PREFIX:-}"

    # Fail closed on any inconsistency: every drop-in below is trusted,
    # required setup the driver injected on purpose, so we abort the boot
    # rather than run a half-configured sandbox. Aborting exits this init
    # non-zero; the VM helper then exits and the driver runs
    # lifecycle-extension cleanup, so a failed init does not leak resources.
    # The loop runs in the main shell (process substitution, not a pipe), so
    # `exit` here terminates init as intended.
    local name dropin rc
    while IFS= read -r name; do
        [ -n "$name" ] || continue
        # Manifest entries are bare file names that the driver already
        # validated. A separator or traversal here means the manifest was
        # tampered with after the driver wrote it.
        case "$name" in
            */* | . | ..)
                ts "FATAL: unsafe init drop-in manifest entry '${name}'"
                exit 1
                ;;
        esac

        # The driver writes each entry's file and marks it 0755 in the same
        # operation, so a missing or non-executable entry means the overlay
        # was tampered with or provisioning is broken.
        dropin="${init_dir}/${name}"
        if [ ! -f "$dropin" ]; then
            ts "FATAL: init drop-in '${name}' listed in manifest but not found"
            exit 1
        fi
        if [ ! -x "$dropin" ]; then
            ts "FATAL: init drop-in '${name}' is not executable"
            exit 1
        fi

        ts "running OpenShell VM init drop-in ${name}"
        rc=0
        set +e
        "$dropin"
        rc=$?
        set -e
        if [ "$rc" -ne 0 ]; then
            ts "FATAL: OpenShell VM init drop-in ${name} failed with exit code ${rc}"
            exit 1
        fi
    done < <(LC_ALL=C sort -u "$manifest")
}

run_post_overlay_setup() {
    # Source QEMU-injected environment variables if present. The file lives in
    # the overlay upperdir so the cached bootstrap rootfs remains immutable.
    local env_file
    env_file="$(root_path /srv/openshell-env.sh)"
    if [ -f "$env_file" ]; then
        # shellcheck source=/dev/null
        source "$env_file"
    fi

    if [ -z "${ROOT_PREFIX:-}" ]; then
        mount -t proc proc /proc 2>/dev/null &
        mount -t sysfs sysfs /sys 2>/dev/null &
        mount -t tmpfs tmpfs /tmp 2>/dev/null &
        mount -t tmpfs tmpfs /run 2>/dev/null &
        mount -t devtmpfs devtmpfs /dev 2>/dev/null &
        wait
    fi

    mkdir -p "$(root_path /dev/pts)" "$(root_path /dev/shm)" "$(root_path /sys/fs/cgroup)"
    mount -t devpts devpts "$(root_path /dev/pts)" 2>/dev/null &
    mount -t tmpfs tmpfs "$(root_path /dev/shm)" 2>/dev/null &
    mount -t cgroup2 cgroup2 "$(root_path /sys/fs/cgroup)" 2>/dev/null &
    wait

    # Allow nftables LOG rules to work in non-init network namespaces.
    # Without this, the kernel's nf_log_syslog silently suppresses output
    # from the sandbox's network namespace.
    if [ -f /proc/sys/net/netfilter/nf_log_all_netns ]; then
        echo 1 > /proc/sys/net/netfilter/nf_log_all_netns 2>/dev/null || true
    fi

    reconcile_sandbox_account
    setup_sandbox_workdir

    configure_hostname
    ip link set lo up 2>/dev/null || true

# Networking: use TAP static config if VM_NET_IP is set (QEMU path),
# otherwise fall back to gvproxy DHCP on eth0 (libkrun path).
if [ -n "${VM_NET_IP}" ] && [ -n "${VM_NET_GW}" ]; then
    ts "configuring TAP networking (static ${VM_NET_IP} gw ${VM_NET_GW})"
    GATEWAY_IP="${VM_NET_GW}"

    TAP_NIC=""
    NIC_WAIT=0
    while [ -z "$TAP_NIC" ] && [ "$NIC_WAIT" -lt 10 ]; do
        for candidate in eth0 ens3 enp0s2; do
            if ip link show "$candidate" >/dev/null 2>&1 && [ "$candidate" != "lo" ]; then
                TAP_NIC="$candidate"
                break
            fi
        done
        if [ -z "$TAP_NIC" ]; then
            for sys_nic in /sys/class/net/*; do
                [ -e "$sys_nic" ] || continue
                candidate="${sys_nic##*/}"
                if ip link show "$candidate" >/dev/null 2>&1 && [ "$candidate" != "lo" ]; then
                    TAP_NIC="$candidate"
                    break
                fi
            done
        fi
        if [ -z "$TAP_NIC" ]; then
            sleep 1
            NIC_WAIT=$((NIC_WAIT + 1))
        fi
    done

    if [ -n "$TAP_NIC" ]; then
        ts "using NIC ${TAP_NIC} for TAP networking"
        ip link set "$TAP_NIC" up 2>/dev/null || true
        ip addr add "${VM_NET_IP}/30" dev "$TAP_NIC" 2>/dev/null || true
        ip route add default via "${VM_NET_GW}" 2>/dev/null || true
    else
        ts "WARNING: no network interface found for TAP networking"
    fi

    if [ -n "${VM_NET_DNS}" ]; then
        echo "nameserver ${VM_NET_DNS}" > "$(root_path /etc/resolv.conf)"
    elif [ ! -s "$(root_path /etc/resolv.conf)" ]; then
        echo "nameserver 8.8.8.8" > "$(root_path /etc/resolv.conf)"
        echo "nameserver 8.8.4.4" >> "$(root_path /etc/resolv.conf)"
    fi

    ensure_host_gateway_aliases
elif ip link show eth0 >/dev/null 2>&1; then
    ts "detected eth0 (gvproxy networking)"
    ip link set eth0 up 2>/dev/null || true

    if command -v udhcpc >/dev/null 2>&1; then
        UDHCPC_SCRIPT="$(root_path /run/openshell-udhcpc.script)"
        mkdir -p "$(dirname "$UDHCPC_SCRIPT")"
        cat > "$UDHCPC_SCRIPT" <<'DHCP_SCRIPT'
#!/bin/sh
case "$1" in
    bound|renew)
        ip addr flush dev "$interface"
        ip addr add "$ip/$mask" dev "$interface"
        if [ -n "$router" ]; then
            ip route add default via "$router" dev "$interface"
        fi
        if [ -n "$dns" ]; then
            resolv_conf="${OPENSHELL_RESOLV_CONF:-/etc/resolv.conf}"
            mkdir -p "$(dirname "$resolv_conf")" 2>/dev/null || true
            : > "$resolv_conf" 2>/dev/null || true
            for d in $dns; do
                echo "nameserver $d" >> "$resolv_conf" 2>/dev/null || true
            done
        fi
        ;;
esac
DHCP_SCRIPT
        chmod +x "$UDHCPC_SCRIPT"

        if ! OPENSHELL_RESOLV_CONF="$(root_path /etc/resolv.conf)" \
            udhcpc -i eth0 -f -q -n -T 1 -t 3 -A 1 -s "$UDHCPC_SCRIPT" 2>&1; then
            ts "WARNING: DHCP failed, falling back to static config"
            ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true
            ip route add default via "$GVPROXY_GATEWAY_IP" 2>/dev/null || true
        fi
    else
        ts "no DHCP client, using static config"
        ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true
        ip route add default via "$GVPROXY_GATEWAY_IP" 2>/dev/null || true
    fi

    if [ ! -s "$(root_path /etc/resolv.conf)" ]; then
        echo "nameserver 8.8.8.8" > "$(root_path /etc/resolv.conf)"
        echo "nameserver 8.8.4.4" >> "$(root_path /etc/resolv.conf)"
    fi

    ensure_host_gateway_aliases
else
    ts "WARNING: no network interface found; supervisor will start without guest egress"
fi

export HOME=/sandbox
export USER=sandbox

# Fix /sandbox ownership. The host-side CLI extracts OCI layers as a non-root
# user (e.g. UID 501 on macOS), so /sandbox may be owned by the host UID.
#
# On macOS (Hypervisor.framework), guest root has real root privileges and
# chown succeeds. On Linux non-root hosts with virtiofs, guest root maps to
# the host user, so chown is denied — this is non-fatal because the
# supervisor's own filesystem preparation handles the paths that matter.
if [ -d /sandbox ]; then
    _sb_uid=$(id -u sandbox 2>/dev/null || true)
    _sb_gid=$(id -g sandbox 2>/dev/null || true)
    if [ -n "$_sb_uid" ] && [ -n "$_sb_gid" ]; then
        _cur_uid=$(stat -c '%u' /sandbox 2>/dev/null || true)
        if [ -n "$_cur_uid" ] && [ "$_cur_uid" != "$_sb_uid" ]; then
            ts "fixing /sandbox ownership (was uid=${_cur_uid}, setting to sandbox=${_sb_uid}:${_sb_gid})"
            chown -R "${_sb_uid}:${_sb_gid}" /sandbox 2>/dev/null || \
                ts "chown /sandbox denied (virtiofs rootless host), continuing"
        fi
    fi
fi

run_openshell_init_dropins

rewrite_openshell_endpoint_if_needed

# Log supervisor connectivity state for debugging stuck-in-Provisioning issues
if [ -n "${OPENSHELL_ENDPOINT:-}" ]; then
    _ep_parsed="$(parse_endpoint "$OPENSHELL_ENDPOINT" 2>/dev/null || true)"
    if [ -n "$_ep_parsed" ]; then
        _ep_host="$(printf '%s\n' "$_ep_parsed" | sed -n '2p')"
        _ep_port="$(printf '%s\n' "$_ep_parsed" | sed -n '3p')"
        if tcp_probe "$_ep_host" "$_ep_port"; then
            ts "gateway reachable at ${_ep_host}:${_ep_port}"
        else
            ts "WARNING: gateway NOT reachable at ${_ep_host}:${_ep_port} — supervisor may fail to connect"
        fi
    fi
    ts "OPENSHELL_ENDPOINT=${OPENSHELL_ENDPOINT}"
fi
if [ -n "${OPENSHELL_SANDBOX_ID:-}" ]; then
    ts "OPENSHELL_SANDBOX_ID=${OPENSHELL_SANDBOX_ID}"
fi

read_supervisor_extra_args
set -- --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}"

ts "starting openshell-sandbox supervisor"
if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then
    exec_supervisor_in_newroot "$@"
fi
exec /opt/openshell/bin/openshell-sandbox "$@"
}

if [ "${1:-}" != "--post-overlay" ]; then
    setup_overlay_root
fi

shift || true
run_post_overlay_setup