mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-02 07:34:45 +08:00
feat(gateway,cli): windows compilation support (#2496)
* chore(windows): gate Unix-only workspace code for MSVC Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * feat(windows): stub unsupported compute drivers Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * ci(windows): add MSVC mise build lane Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * docs(windows): document MSVC build-only design Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * docs(agent): add Windows MSVC build skill Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * feat(windows): add Windows build support Signed-off-by: Akber Raza <akberr@nvidia.com> * refactor(windows): consolidate Windows-specific dependencies and improve build logic Signed-off-by: Akber Raza <akberr@nvidia.com> * feat(windows): add libclang path resolution and update cargo commands with bundled Z3 features Signed-off-by: Akber Raza <akberr@nvidia.com> * chore(tooling): lock Windows tool artifacts Signed-off-by: Giedrius Burachas <gburachas@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * feat(windows): enhance libclang path resolution to support architecture-specific subdirectories Signed-off-by: Akber Raza <akberr@nvidia.com> * Fix Windows dependency gating after sync merge Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(z3): update Z3 header path requirements in Windows build documentation and scripts Signed-off-by: Akber Raza <akberr@nvidia.com> * docs(windows): relocate Windows MSVC build design to architecture/ Why: windows-msvc-build-design.mdx is a design document ("design decisions for the native Windows MSVC build lane"), but it lived in the published, user-facing docs/reference/ tree. Per AGENTS.md (Documentation) and architecture/README.md ("rfc/ vs architecture/"), design content belongs in architecture/ (or rfc/), not in published reference. It also shared Fern sidebar "position: 6" with the MXC compute-driver design page, colliding in the Reference nav ordering. What: - Move docs/reference/windows-msvc-build-design.mdx -> architecture/windows-msvc-build.md. - Strip the Fern publish frontmatter and add a plain H1, matching the other architecture docs. - Register it in the architecture doc index in architecture/README.md. - Repoint the inbound references (build-openshell-mxc-windows skill + reference, implement-openshell-mxc-driver skill) to the new path. With both design pages moved out of docs/reference/, the duplicate position-6 sidebar collision is resolved. Signed-off-by: Akber Raza <akberr@nvidia.com> * remove openshell-supervisor-network from unsupported driver package test exclusion list Signed-off-by: Akber Raza <akberr@nvidia.com> # Conflicts: # tasks/scripts/windows-msvc.ps1 * fix(interceptors): gate unix-only imports so the crate builds on Windows openshell-gateway-interceptors failed to compile on Windows (E0432: no UnixStream in tokio::net), breaking any Windows build of openshell-server (which depends on it unconditionally). The connect_unix_endpoint fn was already #[cfg(unix)]-gated, but the imports it uses (UnixStream, TokioIo, Uri, service_fn) were left ungated. Gate those four imports with #[cfg(unix)] too. No behavior change on unix; Windows now compiles (no errors, no unused-import warnings). Signed-off-by: Akber Raza <akberr@nvidia.com> * feat(windows): add native ARM64 test support Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(mise): skip Skaffold on Windows Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): harden ARM64 toolchain discovery Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): scope ARM64 toolchain preflight Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): restore compatibility after GitHub sync Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): avoid rate-limited Z3 source lookup Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(mise): skip Helm checks on Windows Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): support repository pre-commit checks Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): stabilize native MSVC validation Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): harden shared Z3 source cache Signed-off-by: Shailendra Singh <shailendras@nvidia.com> * fix(windows): avoid leaking MSVC flags into clang-cl Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): complete ARM64 migration audit Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): restore ARM64 Ninja discovery Signed-off-by: Akber Raza <akberr@nvidia.com> * refactor(windows): separate platform crate roots Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): restore proto include cfg gating Signed-off-by: Akber Raza <akberr@nvidia.com> * refactor: address lint errors * fix(windows): add preflight check for proxy auth file path * docs(windows): update GitHub checkout guidance Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): restore CI after dependency updates Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(mise): repair Windows sccache lock entry Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(windows): reconcile validation after rebase Signed-off-by: Akber Raza <akberr@nvidia.com> * refactor(server): exclude unsupported drivers on Windows Signed-off-by: Piotr Mlocek <pmlocek@nvidia.com> * refactor(server): isolate platform driver config Signed-off-by: Piotr Mlocek <pmlocek@nvidia.com> * fix(windows): repair unsupported driver contract test Signed-off-by: Akber Raza <akberr@nvidia.com> * fix(sandbox): remove stale dependencies Signed-off-by: Akber Raza <akberr@nvidia.com> * ci(windows): pin x64 workflow actions Signed-off-by: Akber Raza <akberr@nvidia.com> * ci(windows): align x64 Rust toolchain Signed-off-by: Akber Raza <akberr@nvidia.com> * ci(windows): align ARM64 workflow setup Signed-off-by: Akber Raza <akberr@nvidia.com> * refactor(windows): exclude unsupported runtime crates Signed-off-by: Akber Raza <akberr@nvidia.com> * refactor(windows): exclude unsupported crates at workspace boundary Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * refactor(server): gate builtin driver config by platform Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * fix(sandbox): restore crate documentation Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * ci(windows): make build workflow manual Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * ci(windows): temporarily enable pull request builds Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * ci(windows): cache Rust dependencies Signed-off-by: Akber Raza <akberr@nvidia.com> * refactor(windows): remove unnecessary platform changes Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * ci(windows): make build workflow manual Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * fix(ci): synchronize mise lockfile Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * fix(ci): normalize mise provenance metadata Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * refactor(python): isolate Windows atomic replace retry Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> * fix(python): type Windows permission test errors Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> --------- Signed-off-by: Shailendra Singh <shailendras@nvidia.com> Signed-off-by: Akber Raza <akberr@nvidia.com> Signed-off-by: Giedrius Burachas <gburachas@nvidia.com> Signed-off-by: Piotr Mlocek <pmlocek@nvidia.com> Signed-off-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com> Co-authored-by: Shailendra Singh <shailendras@nvidia.com> Co-authored-by: Giedrius Burachas <gburachas@nvidia.com> Co-authored-by: Jamie King <jamiek@nvidia.com> Co-authored-by: Piotr Mlocek <pmlocek@nvidia.com> Co-authored-by: Piotr Mlocek <1116309+pimlock@users.noreply.github.com>
This commit is contained in:
araza008
co-authored by
Shailendra Singh
Giedrius Burachas
Jamie King
Piotr Mlocek
Piotr Mlocek
Shailendra Singh
Giedrius Burachas
Jamie King
Piotr Mlocek
Piotr Mlocek
parent
0310cbed6c
commit
2f96c53b8c
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
@@ -1065,6 +1066,40 @@ def _xdg_config_home() -> pathlib.Path:
|
||||
# matches `openshell-bootstrap::oidc_token::is_token_expired`.
|
||||
_OIDC_TOKEN_EXPIRY_GRACE_SECONDS = 30
|
||||
|
||||
_IS_WINDOWS = os.name == "nt"
|
||||
_WINDOWS_REPLACE_RETRYABLE_ERRORS = frozenset({5, 32})
|
||||
_WINDOWS_REPLACE_TIMEOUT_SECONDS = 0.25
|
||||
_WINDOWS_REPLACE_INITIAL_DELAY_SECONDS = 0.005
|
||||
_WINDOWS_REPLACE_MAX_DELAY_SECONDS = 0.05
|
||||
_WINDOWS_REPLACE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _atomic_replace(source: pathlib.Path, destination: pathlib.Path) -> None:
|
||||
"""Atomically replace a file, retrying transient Windows sharing errors."""
|
||||
if not _IS_WINDOWS:
|
||||
source.replace(destination)
|
||||
return
|
||||
|
||||
# Serialize writers in this process. The retry still handles other
|
||||
# processes (including the Rust CLI) and filesystem scanners that briefly
|
||||
# open the destination without delete sharing.
|
||||
with _WINDOWS_REPLACE_LOCK:
|
||||
deadline = time.monotonic() + _WINDOWS_REPLACE_TIMEOUT_SECONDS
|
||||
delay = _WINDOWS_REPLACE_INITIAL_DELAY_SECONDS
|
||||
while True:
|
||||
try:
|
||||
source.replace(destination)
|
||||
return
|
||||
except PermissionError as error:
|
||||
winerror = getattr(error, "winerror", None)
|
||||
retryable = winerror in _WINDOWS_REPLACE_RETRYABLE_ERRORS or (
|
||||
winerror is None and error.errno == errno.EACCES
|
||||
)
|
||||
if not retryable or time.monotonic() >= deadline:
|
||||
raise
|
||||
time.sleep(delay)
|
||||
delay = min(delay * 2, _WINDOWS_REPLACE_MAX_DELAY_SECONDS)
|
||||
|
||||
|
||||
def _read_oidc_token_bundle(gateway_dir: pathlib.Path) -> dict | None:
|
||||
"""Read and parse `oidc_token.json` for a gateway.
|
||||
@@ -1531,7 +1566,7 @@ class _OidcRefresher:
|
||||
f.write(payload)
|
||||
with contextlib.suppress(OSError):
|
||||
tmp_path.chmod(0o600)
|
||||
tmp_path.replace(path)
|
||||
_atomic_replace(tmp_path, path)
|
||||
except BaseException:
|
||||
# Clean up our tmp on failure so we don't leave orphaned
|
||||
# `.oidc_token.<rand>.tmp` files lying around. The replace
|
||||
|
||||
@@ -16,6 +16,7 @@ from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
import openshell.sandbox as sandbox_module
|
||||
from openshell._proto import openshell_pb2
|
||||
from openshell.sandbox import (
|
||||
_PYTHON_CLOUDPICKLE_BOOTSTRAP,
|
||||
@@ -27,6 +28,7 @@ from openshell.sandbox import (
|
||||
SandboxRef,
|
||||
SandboxStatusRef,
|
||||
TlsConfig,
|
||||
_atomic_replace,
|
||||
_BearerAuthInterceptor,
|
||||
_load_cluster_bearer_token,
|
||||
_make_cluster_bearer_provider,
|
||||
@@ -1281,6 +1283,65 @@ def test_refresher_concurrent_write_back_does_not_trample(tmp_path: Path) -> Non
|
||||
r.close()
|
||||
|
||||
|
||||
class _WindowsPermissionError(PermissionError):
|
||||
winerror: int
|
||||
|
||||
|
||||
def test_atomic_replace_retries_windows_sharing_violations(
|
||||
tmp_path: Path, monkeypatch: Any
|
||||
) -> None:
|
||||
source = tmp_path / "source"
|
||||
destination = tmp_path / "destination"
|
||||
source.write_text("new")
|
||||
destination.write_text("old")
|
||||
attempts = 0
|
||||
delays: list[float] = []
|
||||
real_replace = Path.replace
|
||||
|
||||
def replace(path: Path, target: Path) -> Path:
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts < 3:
|
||||
error = _WindowsPermissionError("destination is busy")
|
||||
error.winerror = 32
|
||||
raise error
|
||||
return real_replace(path, target)
|
||||
|
||||
monkeypatch.setattr(sandbox_module, "_IS_WINDOWS", True)
|
||||
monkeypatch.setattr(Path, "replace", replace)
|
||||
monkeypatch.setattr(time, "sleep", delays.append)
|
||||
|
||||
_atomic_replace(source, destination)
|
||||
|
||||
assert attempts == 3
|
||||
assert delays == [0.005, 0.01]
|
||||
assert destination.read_text() == "new"
|
||||
|
||||
|
||||
def test_atomic_replace_does_not_retry_permanent_windows_errors(
|
||||
tmp_path: Path, monkeypatch: Any
|
||||
) -> None:
|
||||
source = tmp_path / "source"
|
||||
destination = tmp_path / "destination"
|
||||
source.write_text("new")
|
||||
attempts = 0
|
||||
|
||||
def replace(_path: Path, _target: Path) -> Path:
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
error = _WindowsPermissionError("access denied")
|
||||
error.winerror = 13
|
||||
raise error
|
||||
|
||||
monkeypatch.setattr(sandbox_module, "_IS_WINDOWS", True)
|
||||
monkeypatch.setattr(Path, "replace", replace)
|
||||
|
||||
with pytest.raises(PermissionError, match="access denied"):
|
||||
_atomic_replace(source, destination)
|
||||
|
||||
assert attempts == 1
|
||||
|
||||
|
||||
def test_sandbox_wrapper_forwards_auth_kwargs_to_from_active_cluster(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user