From 2b3d95df962b4cfc760fa0ab8a759cec5e626dc9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 13 Oct 2026 11:43:29 -0700 Subject: [PATCH] refactor(docker): resolve host gateway in the driver Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/README.md | 17 ++- crates/openshell-driver-docker/src/lib.rs | 112 ++++++++++++++++--- crates/openshell-driver-docker/src/tests.rs | 84 +++++++++++++- crates/openshell-supervisor/src/main.rs | 88 +-------------- docs/how-it-works/gateways/configuration.mdx | 15 ++- docs/how-it-works/sandboxes/runtimes.mdx | 4 +- skills/debug-openshell-cluster/SKILL.md | 6 +- 7 files changed, 210 insertions(+), 116 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 42a5d2182..b1d64de38 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -182,12 +182,17 @@ uses its sandbox-scoped bearer token to authenticate RPCs. User client certificates and private keys are not delivered to either container. Host mode on Docker Desktop requires host networking enabled in Settings. -Bridge mode uses Docker-resolved `host-gateway` entries for both -host aliases. The supervisor's `--host-gateway-from-hosts` option pins the -numeric address from its own driver-owned `/etc/hosts` into the runtime -descriptor before attaching the workload; workload DNS and hosts files cannot -select this address. Gateway endpoints and host aliases are independent in -bridge mode. +In bridge mode, the driver inspects the default `bridge` network's IPAM gateway +at gateway startup, preferring a unique IPv4 address and otherwise a unique +IPv6 address. Set the operator-owned `host_gateway_ip` to override this +heuristic. Docker Desktop and custom daemon host-gateway settings can use a +different host address and require the override. Missing or ambiguous gateway +metadata produces an actionable configuration error. + +The driver uses the selected address for both host aliases and the runtime +descriptor's trust pin; workload DNS and hosts files cannot select it. Host +mode retains its gateway-address pin unless `host_gateway_ip` is configured. +No helper container or supervisor-specific resolution flag is required. When `grpc_endpoint` is omitted, bridge mode uses `host.docker.internal` on Desktop or for a wildcard gateway listener, and the concrete bind address for diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index dc27488e7..3796e2da6 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -196,6 +196,9 @@ pub struct DockerComputeConfig { /// Networking mode for the supervisor only. Workloads always use network=none. pub supervisor_network_mode: DockerSupervisorNetworkMode, + /// Override the trusted host address. Bridge mode otherwise uses the default bridge gateway. + pub host_gateway_ip: Option, + /// Image containing the trusted `openshell-sandbox` binary. pub sandbox_runtime_image: Option, @@ -296,6 +299,7 @@ impl Default for DockerComputeConfig { grpc_endpoint: String::new(), supervisor_otlp_endpoint: None, supervisor_network_mode: DockerSupervisorNetworkMode::Host, + host_gateway_ip: None, sandbox_runtime_image: None, supervisor_bin: None, supervisor_image: None, @@ -334,6 +338,7 @@ struct DockerDriverRuntimeConfig { supervisor_network_mode: DockerSupervisorNetworkMode, supervisor_endpoint_explicit: bool, bridge_default_endpoint: Option, + host_gateway_ip: Option, ssh_socket_path: String, guest_tls: Option, gpu: DockerGpuRuntimeCapabilities, @@ -928,6 +933,26 @@ impl DockerComputeDriver { )) })?; let supervisor_grpc_endpoint = docker_config.grpc_endpoint.clone(); + let host_gateway_ip = if let Some(ip) = docker_config.host_gateway_ip { + Some(validate_docker_host_gateway_ip( + ip, + docker_config.supervisor_network_mode, + )?) + } else if docker_config.supervisor_network_mode == DockerSupervisorNetworkMode::Bridge { + let network = docker + .inspect_network("bridge", None) + .await + .map_err(|error| { + Error::config(format!( + "inspect Docker bridge gateway: {error}; set host_gateway_ip explicitly" + )) + })?; + let ip = docker_bridge_host_gateway_ip(&network)?; + info!(host_gateway_ip = %ip, "Using Docker bridge gateway for host aliases; configure host_gateway_ip if the Docker host uses a different address"); + Some(ip) + } else { + docker_supervisor_host_address(&supervisor_grpc_endpoint) + }; let supervisor_image = docker_config .supervisor_image .clone() @@ -975,6 +1000,7 @@ impl DockerComputeDriver { supervisor_network_mode: docker_config.supervisor_network_mode, supervisor_endpoint_explicit, bridge_default_endpoint, + host_gateway_ip, ssh_socket_path: docker_config.ssh_socket_path.clone(), guest_tls, gpu, @@ -4599,10 +4625,7 @@ async fn prepare_docker_boundary_files( server_name: tls.server_name.clone(), trust_anchor_pem: tls.trust_anchor_pem.clone(), }, - // Host mode pins the alias to the gateway address, normally loopback. - // In bridge mode the companion replaces this value from its own - // Docker-injected hosts file before attaching the descriptor. - host_gateway_ip: docker_supervisor_host_address(&config.supervisor_grpc_endpoint), + host_gateway_ip: config.host_gateway_ip, workload_identity: workload_identity.clone(), child_env: docker_child_environment(sandbox), } @@ -5012,6 +5035,7 @@ fn docker_supervisor_host_config( mounts: Vec, grpc_endpoint: &str, network_mode: DockerSupervisorNetworkMode, + host_gateway_ip: Option, ) -> HostConfig { HostConfig { // The supervisor is trusted infrastructure and originates every @@ -5026,16 +5050,18 @@ fn docker_supervisor_host_config( } .to_string(), ), - extra_hosts: if network_mode == DockerSupervisorNetworkMode::Bridge { - // Docker resolves host-gateway on the daemon, including Desktop's VM. - // The gateway endpoint may be remote and must not define host aliases. - Some(vec![ - format!("{HOST_OPEN_SHELL_INTERNAL}:host-gateway"), - format!("{HOST_DOCKER_INTERNAL}:host-gateway"), - ]) - } else { - docker_supervisor_host_aliases(grpc_endpoint) - }, + extra_hosts: host_gateway_ip + .map(|ip| { + let address = match ip { + IpAddr::V4(ip) => ip.to_string(), + IpAddr::V6(ip) => format!("[{ip}]"), + }; + vec![ + format!("{HOST_OPEN_SHELL_INTERNAL}:{address}"), + format!("{HOST_DOCKER_INTERNAL}:{address}"), + ] + }) + .or_else(|| docker_supervisor_host_aliases(grpc_endpoint)), mounts: Some(mounts), cap_drop: Some(vec!["ALL".to_string()]), cap_add: None, @@ -5190,9 +5216,6 @@ async fn spawn_docker_control_process( workspace_root, format!("--health-socket-path={SUPERVISOR_HEALTH_SOCKET_PATH}"), ]; - if network_mode == DockerSupervisorNetworkMode::Bridge { - command.push("--host-gateway-from-hosts".to_string()); - } command.extend(docker_upstream_proxy_cli_args( &config.upstream_proxy, config.proxy_ca_bundle.is_some(), @@ -5258,6 +5281,7 @@ async fn spawn_docker_control_process( supervisor_mounts, &supervisor_endpoint, network_mode, + config.host_gateway_ip, )), ..Default::default() }; @@ -6628,6 +6652,60 @@ fn default_docker_bridge_endpoint( } } +fn validate_docker_host_gateway_ip( + ip: IpAddr, + mode: DockerSupervisorNetworkMode, +) -> CoreResult { + let normalized = match ip { + IpAddr::V6(ip) => ip.to_ipv4_mapped().map_or(IpAddr::V6(ip), IpAddr::V4), + ip @ IpAddr::V4(_) => ip, + }; + if normalized.is_unspecified() + || normalized.is_multicast() + || (mode == DockerSupervisorNetworkMode::Bridge && normalized.is_loopback()) + || matches!(normalized, IpAddr::V4(ip) if ip.is_link_local() || ip.is_broadcast()) + || matches!(normalized, IpAddr::V6(ip) if ip.is_unicast_link_local()) + { + return Err(Error::config(format!( + "invalid Docker host_gateway_ip '{ip}' for {mode:?} networking" + ))); + } + Ok(normalized) +} + +fn docker_bridge_host_gateway_ip(network: &bollard::models::NetworkInspect) -> CoreResult { + let addresses = network + .ipam + .as_ref() + .and_then(|ipam| ipam.config.as_ref()) + .into_iter() + .flatten() + .filter_map(|config| config.gateway.as_deref()) + .map(str::parse::) + .collect::, _>>() + .map_err(|error| { + Error::config(format!( + "invalid Docker bridge gateway: {error}; set host_gateway_ip explicitly" + )) + })?; + let ipv4 = addresses + .iter() + .copied() + .filter(IpAddr::is_ipv4) + .collect::>(); + let candidates = if ipv4.is_empty() { + addresses.into_iter().collect::>() + } else { + ipv4 + }; + let [ip] = candidates.as_slice() else { + return Err(Error::config( + "Docker bridge does not report a unique gateway; set host_gateway_ip explicitly", + )); + }; + validate_docker_host_gateway_ip(*ip, DockerSupervisorNetworkMode::Bridge) +} + fn docker_supervisor_networking( config: &DockerDriverRuntimeConfig, ) -> Result<(DockerSupervisorNetworkMode, String), Status> { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index c1feee151..8102e17a6 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -215,6 +215,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { supervisor_network_mode: DockerSupervisorNetworkMode::Host, supervisor_endpoint_explicit: true, bridge_default_endpoint: Some("https://host.docker.internal:8443".to_string()), + host_gateway_ip: None, ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), guest_tls: Some(DockerGuestTlsPaths { ca: PathBuf::from("/tmp/ca.crt"), @@ -3037,6 +3038,7 @@ fn docker_supervisor_uses_host_network() { Vec::new(), "https://127.0.0.1:17670", DockerSupervisorNetworkMode::Host, + None, ); assert_eq!(host.network_mode.as_deref(), Some("host")); @@ -3057,6 +3059,7 @@ fn docker_supervisor_maps_host_aliases_to_the_gateway_address() { Vec::new(), "https://172.20.0.4:17670", DockerSupervisorNetworkMode::Host, + None, ); assert_eq!( @@ -3078,6 +3081,7 @@ fn docker_supervisor_leaves_named_gateway_hosts_to_dns() { Vec::new(), "https://gateway.example.com:17670", DockerSupervisorNetworkMode::Host, + None, ); assert_eq!(host.extra_hosts, None); @@ -3194,13 +3198,14 @@ fn docker_bridge_host_aliases_are_independent_of_gateway_endpoint() { Vec::new(), "https://gateway.example.com:9443", DockerSupervisorNetworkMode::Bridge, + Some("172.17.0.1".parse().unwrap()), ); assert_eq!(host.network_mode.as_deref(), Some("bridge")); assert_eq!( host.extra_hosts, Some(vec![ - "host.openshell.internal:host-gateway".into(), - "host.docker.internal:host-gateway".into() + "host.openshell.internal:172.17.0.1".into(), + "host.docker.internal:172.17.0.1".into() ]) ); assert_eq!(host.cap_drop, Some(vec!["ALL".into()])); @@ -3220,6 +3225,81 @@ fn docker_bridge_host_aliases_are_independent_of_gateway_endpoint() { ); } +#[test] +fn docker_bridge_gateway_heuristic_prefers_ipv4_and_requires_one_safe_address() { + let network = |gateways: &[&str]| { + serde_json::from_value::( + serde_json::json!({"IPAM": {"Config": gateways.iter().map(|ip| serde_json::json!({"Gateway": ip})).collect::>()}}) + ).unwrap() + }; + assert_eq!( + docker_bridge_host_gateway_ip(&network(&["172.17.0.1", "fd00::1"])).unwrap(), + "172.17.0.1".parse::().unwrap() + ); + assert_eq!( + docker_bridge_host_gateway_ip(&network(&["fd00::1"])).unwrap(), + "fd00::1".parse::().unwrap() + ); + for gateways in [ + &[][..], + &["172.17.0.1", "172.18.0.1"][..], + &["bad-ip"][..], + &["169.254.169.254"][..], + &["127.0.0.1"][..], + &["::ffff:127.0.0.1"][..], + ] { + assert!( + docker_bridge_host_gateway_ip(&network(gateways)).is_err(), + "{gateways:?}" + ); + } +} + +#[test] +fn docker_host_gateway_override_is_operator_owned_and_matches_the_alias_pin() { + let config: DockerComputeConfig = + toml::from_str("supervisor_network_mode = 'bridge'\nhost_gateway_ip = '172.29.0.254'") + .unwrap(); + let ip = validate_docker_host_gateway_ip( + config.host_gateway_ip.unwrap(), + config.supervisor_network_mode, + ) + .unwrap(); + let host = docker_supervisor_host_config( + Vec::new(), + "https://gateway.example.com:8443", + config.supervisor_network_mode, + Some(ip), + ); + assert_eq!( + host.extra_hosts.unwrap(), + [ + "host.openshell.internal:172.29.0.254", + "host.docker.internal:172.29.0.254" + ] + ); + assert!( + serde_json::from_value::( + serde_json::json!({"host_gateway_ip": "172.29.0.254"}) + ) + .is_err() + ); + assert!( + validate_docker_host_gateway_ip( + "127.0.0.1".parse().unwrap(), + DockerSupervisorNetworkMode::Host + ) + .is_ok() + ); + assert!( + validate_docker_host_gateway_ip( + "0.0.0.0".parse().unwrap(), + DockerSupervisorNetworkMode::Host + ) + .is_err() + ); +} + #[test] fn docker_supervisor_network_mode_is_an_operator_config_enum() { assert_eq!( diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 2836ded07..7c062b5f2 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -121,10 +121,6 @@ struct Args { #[arg(long)] backend_descriptor_file: Option, - /// Resolve the Docker host alias from this supervisor's driver-owned hosts file. - #[arg(long)] - host_gateway_from_hosts: bool, - /// Protected gateway-issued credentials for this exact sandbox launch. #[arg(long)] auth_bundle_file: Option, @@ -181,68 +177,14 @@ fn backend_descriptor(args: &Args) -> Result { let path = args.backend_descriptor_file.as_deref().ok_or_else(|| { miette::miette!("--backend-descriptor-file is required for --role=isolation-backend") })?; - let mut payload = std::fs::read(path) + let payload = std::fs::read(path) .map_err(|error| miette::miette!("read backend descriptor {}: {error}", path.display()))?; - if args.host_gateway_from_hosts { - let hosts = std::fs::read_to_string("/etc/hosts") - .map_err(|error| miette::miette!("read supervisor Docker host aliases: {error}"))?; - let ip = docker_host_gateway_from_hosts(&hosts)?; - let mut descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor = - serde_json::from_slice(&payload).into_diagnostic()?; - // Resolve before any workload is admitted, from the supervisor's mount - // namespace. Workload /etc/hosts and DNS never supply this trusted IP. - descriptor.host_gateway_ip = Some(ip); - payload = serde_json::to_vec(&descriptor).into_diagnostic()?; - } Ok(BackendDescriptor { backend_name: openshell_sandbox_backend::BACKEND_NAME.to_string(), payload, }) } -fn docker_host_gateway_from_hosts(hosts: &str) -> Result { - let mut ips = std::collections::BTreeSet::new(); - for line in hosts.lines() { - let mut fields = line.split('#').next().unwrap_or("").split_whitespace(); - let Some(ip) = fields - .next() - .and_then(|value| value.parse::().ok()) - else { - continue; - }; - if fields.any(|host| host.eq_ignore_ascii_case("host.openshell.internal")) { - ips.insert(ip); - } - } - // Docker may expand host-gateway to both families. Prefer IPv4 consistently. - let ipv4 = ips - .iter() - .copied() - .filter(std::net::IpAddr::is_ipv4) - .collect::>(); - let candidates = if ipv4.is_empty() { - ips.into_iter().collect() - } else { - ipv4 - }; - let [ip] = candidates.as_slice() else { - return Err(miette::miette!( - "expected one Docker host-gateway address per family in supervisor /etc/hosts" - )); - }; - if ip.is_loopback() - || ip.is_unspecified() - || ip.is_multicast() - || matches!(ip, std::net::IpAddr::V4(ip) if ip.is_link_local() || ip.is_broadcast()) - || matches!(ip, std::net::IpAddr::V6(ip) if ip.is_unicast_link_local() || ip.to_ipv4_mapped().is_some()) - { - return Err(miette::miette!( - "unsafe Docker host-gateway address in supervisor /etc/hosts" - )); - } - Ok(*ip) -} - fn auth_bundle(args: &Args) -> Result { let path = args.auth_bundle_file.as_deref().ok_or_else(|| { miette::miette!("--auth-bundle-file is required for --role=isolation-backend") @@ -287,7 +229,6 @@ fn validate_role_arguments(args: &Args) -> Result<()> { } SupervisorRole::NetworkProxy => { if args.backend_descriptor_file.is_some() - || args.host_gateway_from_hosts || args.auth_bundle_file.is_some() || args.sandbox_id.is_some() || args.sandbox.is_some() @@ -624,33 +565,6 @@ mod tests { assert_eq!(directives("info,h2=debug"), "openshell=info,info"); } - #[test] - fn docker_host_gateway_uses_supervisor_alias_and_prefers_ipv4() { - let hosts = "127.0.0.1 localhost\n192.168.65.254 host.openshell.internal host.docker.internal\nfd00::1 host.openshell.internal\n192.168.65.254 host.openshell.internal # duplicate\n"; - assert_eq!( - docker_host_gateway_from_hosts(hosts).unwrap(), - "192.168.65.254".parse::().unwrap() - ); - assert_eq!( - docker_host_gateway_from_hosts("fd00::1 host.openshell.internal").unwrap(), - "fd00::1".parse::().unwrap() - ); - } - - #[test] - fn docker_host_gateway_rejects_missing_ambiguous_and_unsafe_aliases() { - for hosts in [ - "192.168.65.254 host.docker.internal", - "192.168.65.254 host.openshell.internal\n172.17.0.1 host.openshell.internal", - "127.0.0.1 host.openshell.internal", - "0.0.0.0 host.openshell.internal", - "169.254.169.254 host.openshell.internal", - "::ffff:127.0.0.1 host.openshell.internal", - ] { - assert!(docker_host_gateway_from_hosts(hosts).is_err(), "{hosts}"); - } - } - #[test] fn isolation_backend_is_the_default_role() { let directory = tempfile::tempdir().expect("temporary runtime descriptor directory"); diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 8441833d9..2dea4447c 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -968,6 +968,9 @@ sandbox_label = "docker-dev" # host (default) or bridge networking for the supervisor. # Operator-owned; sandbox driver config cannot override this choice. supervisor_network_mode = "host" +# Optional host-address override. Bridge mode otherwise uses Docker's default +# bridge gateway; Desktop and custom Docker setups may need a different IP. +# host_gateway_ip = "172.29.0.254" # Optional override. Omit for automatic host or bridge callback selection. # grpc_endpoint = "https://gateway.example.com:17670" # Workload-side runtime. Defaults to the gateway version. @@ -1017,8 +1020,16 @@ explicit endpoint for remapped ports. HTTPS certificates must cover the endpoint The local `gateway:docker` task uses automatic endpoint selection unless you set `OPENSHELL_GRPC_ENDPOINT`. -Bridge resolves both host aliases through Docker's `host-gateway`; native Linux -services bound only to loopback remain unreachable. Desktop host mode requires +In bridge mode, the driver uses the default Docker bridge network's gateway IP +for `host.openshell.internal` and `host.docker.internal`, preferring IPv4. +This heuristic works with standard native Linux Docker networking. Docker +Desktop and custom daemon host-gateway settings may use a different host IP: +set `host_gateway_ip` to the address reachable from the supervisor. This +operator-owned setting also overrides the host aliases in host mode. The +driver uses the same IP for the aliases and the sandbox's trusted host address. +If the bridge has no unique gateway, configure `host_gateway_ip` explicitly. + +Native Linux services bound only to loopback remain unreachable from bridge mode. Desktop host mode requires host networking enabled. The sandbox must pass Landlock and seccomp qualification in either mode. Configure the OTLP collector endpoint to be reachable from the selected supervisor network; a loopback collector URL in diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 2eb94ceb1..9c01d5052 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -112,12 +112,14 @@ compute_driver = "docker" socket_path = "/var/run/docker.sock" ``` -Common options in `[openshell.drivers.docker]` are `socket_path`, `grpc_endpoint`, `supervisor_network_mode`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, and `sandbox_pids_limit`. When `socket_path` is unset, the driver uses the socket found by auto-detection. +Common options in `[openshell.drivers.docker]` are `socket_path`, `grpc_endpoint`, `supervisor_network_mode`, `host_gateway_ip`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, and `sandbox_pids_limit`. When `socket_path` is unset, the driver uses the socket found by auto-detection. `supervisor_network_mode` defaults to `host`. Set `bridge` to use Docker bridge networking; `auto` also selects host. The workload always retains `network=none`. Host mode on Docker Desktop requires host networking enabled. Bridge mode automatically uses `host.docker.internal` for a gateway on the Desktop host. On native Linux, the gateway and host services must listen on bridge-reachable addresses. Set `grpc_endpoint` for remote gateways, remapped published ports, or other custom layouts. Endpoint selection preserves TLS verification and does not change the gateway's listener. See [Docker configuration](/how-it-works/gateways/configuration#docker). +Bridge mode maps the host aliases to Docker's default bridge gateway IP. Set `host_gateway_ip` when Docker Desktop or a custom daemon uses a different host address. + The sandbox must pass its Landlock and seccomp qualification in either network mode. For GPU sandboxes, configure Docker CDI before starting the gateway. ### Docker Corporate Proxy Egress diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 121a959dc..f61ec75fa 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -320,7 +320,11 @@ Common findings: `gateway:docker` task discovers its endpoint unless `OPENSHELL_GRPC_ENDPOINT` supplies an override. The OTLP collector endpoint must also be reachable from the supervisor's selected network. Mode selection does not bypass - Landlock/seccomp qualification. See the + Landlock/seccomp qualification. Bridge host aliases use the default Docker + bridge gateway as a heuristic; set the operator-owned `host_gateway_ip` if + Desktop or custom daemon networking uses a different host address. The driver + uses that same address for alias mapping and the sandbox's host trust pin. + Missing or ambiguous bridge gateway metadata requires an explicit override. See the published [Docker configuration](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration.md) for the current networking options. - Sandbox runtime image exits before printing `openshell-sandbox --version`: verify the configured image contains a static executable at `/openshell-sandbox`.