mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-28 07:13:04 +08:00
test(mxc): cover real sandbox TCP forwarding
Signed-off-by: Drew Newberry <anewberry@nvidia.com>
This commit is contained in:
Drew Newberry
@@ -363,6 +363,16 @@ test-internal `SKIP` messages from Cargo's passed count in the real integration
|
||||
suite, and report a verified unsupported-host rejection independently from
|
||||
positive admission coverage.
|
||||
|
||||
The real `forwarding` E2E scenario uses a native test-only echo agent inside MXC and
|
||||
`openshell forward service` to verify two exact request/reply exchanges through
|
||||
the gateway, supervisor, and boundary. Dynamic ports and a per-workload nonce
|
||||
avoid mistaking unrelated listeners for success. This is managed-ingress
|
||||
coverage, not evidence that direct host ingress is fenced. Use
|
||||
`-Scenario forwarding` on the existing local or remote runner for focused checks.
|
||||
The E2E mise task and remote runner build the `mxc-forwarding-agent` Cargo example
|
||||
using `windows:build:mxc-fixtures` (or its `:x64` / `:arm64` variant). This fixture
|
||||
is not one of the four production deliverables; do not package it in releases.
|
||||
|
||||
When reporting `windows:ci`, distinguish these categories:
|
||||
|
||||
- Passed tests from the full x64 workspace test log.
|
||||
|
||||
@@ -164,3 +164,11 @@ run incomplete and exits non-zero; a passing exit requires all selected scenario
|
||||
to execute successfully. Unit-test mocks are separate from E2E coverage.
|
||||
`network-policy` checks admission and workload execution, not real network
|
||||
enforcement. All-skipped real runs report `INCOMPLETE`, not `PASS`.
|
||||
|
||||
The `forwarding` scenario starts a TCP listener inside MXC, then verifies two
|
||||
nonce-bound request/reply exchanges through `openshell forward service`, the
|
||||
gateway, supervisor, and Windows boundary. Both listener ports are dynamically
|
||||
assigned. Its native echo agent is a test-only Cargo example, built automatically
|
||||
by the E2E mise task and remote runner, not a shipped runtime binary. It verifies
|
||||
managed ingress, not denial of direct host-to-workload connections. Select it with
|
||||
`-Scenario forwarding` on the local harness or remote runner.
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# The listener reports its dynamically assigned port in the per-run workdir.
|
||||
# No Internet egress policy is needed for this ingress-only test.
|
||||
version: 1
|
||||
|
||||
filesystem_policy:
|
||||
include_workdir: false
|
||||
read_only: []
|
||||
read_write:
|
||||
- "C:/work/openshell-mxc-e2e"
|
||||
@@ -0,0 +1,39 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Test-only TCP fixture for real MXC ingress E2E; not a shipped runtime binary.
|
||||
|
||||
use std::io::{BufRead, BufReader, Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::time::Duration;
|
||||
|
||||
fn main() -> std::io::Result<()> {
|
||||
let arguments: Vec<String> = std::env::args().collect();
|
||||
if arguments.len() != 3 {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"usage: mxc-forwarding-agent READY_FILE NONCE",
|
||||
));
|
||||
}
|
||||
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))?;
|
||||
let ready_file = std::path::Path::new(&arguments[1]);
|
||||
let staged_ready = ready_file.with_extension("tmp");
|
||||
std::fs::write(&staged_ready, listener.local_addr()?.port().to_string())?;
|
||||
std::fs::rename(staged_ready, ready_file)?;
|
||||
for connection in listener.incoming() {
|
||||
let mut socket = connection?;
|
||||
socket.set_nodelay(true)?;
|
||||
socket.set_read_timeout(Some(Duration::from_secs(10)))?;
|
||||
socket.set_write_timeout(Some(Duration::from_secs(10)))?;
|
||||
let mut request = String::new();
|
||||
BufReader::new((&socket).take(4096)).read_line(&mut request)?;
|
||||
if !request.ends_with('\n') {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"request must be a bounded newline-terminated string",
|
||||
));
|
||||
}
|
||||
write!(socket, "{}:{request}", arguments[2])?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -34,6 +34,7 @@
|
||||
# fs-default-deny - ungranted write is denied after the agent launches.
|
||||
# processcontainer only.
|
||||
# network-policy - supervisor-owned network policy permits admission.
|
||||
# forwarding - two TCP request/reply exchanges through OpenShell ingress.
|
||||
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
@@ -118,7 +119,7 @@ function Quote-NativeArgument([string]$value) {
|
||||
return $quoted.ToString()
|
||||
}
|
||||
|
||||
function Invoke-NativeCaptured([string]$filePath, [string[]]$argumentList) {
|
||||
function Invoke-NativeCaptured([string]$filePath, [string[]]$argumentList, [int]$timeoutSeconds = 0) {
|
||||
$startInfo = New-Object System.Diagnostics.ProcessStartInfo
|
||||
$startInfo.FileName = $filePath
|
||||
$startInfo.Arguments = (($argumentList | ForEach-Object { Quote-NativeArgument $_ }) -join ' ')
|
||||
@@ -142,6 +143,9 @@ function Invoke-NativeCaptured([string]$filePath, [string[]]$argumentList) {
|
||||
}
|
||||
$stdout = $process.StandardOutput.ReadToEndAsync()
|
||||
$stderr = $process.StandardError.ReadToEndAsync()
|
||||
if ($timeoutSeconds -gt 0 -and -not $process.WaitForExit($timeoutSeconds * 1000)) {
|
||||
$process.Kill()
|
||||
}
|
||||
$process.WaitForExit()
|
||||
|
||||
return @{
|
||||
@@ -271,6 +275,66 @@ function Wait-File([string]$path, [int]$seconds) {
|
||||
return (Test-Path $path)
|
||||
}
|
||||
|
||||
# Exercise the public gRPC forward path, never dial the workload port directly.
|
||||
# The nonce identifies this workload, not an unrelated host-loopback listener.
|
||||
function Test-Forwarding([string]$sandboxName, [string]$readyFile, [string]$nonce) {
|
||||
if (-not (Wait-File $readyFile 30)) { throw "MXC forwarding listener did not report readiness" }
|
||||
$targetPort = [int](Get-Content -LiteralPath $readyFile -Raw).Trim()
|
||||
if ($targetPort -lt 1 -or $targetPort -gt 65535) { throw "Invalid workload listener port" }
|
||||
$outLog = Join-Path $resultDir "forward.stdout.log"
|
||||
$errLog = Join-Path $resultDir "forward.stderr.log"
|
||||
$arguments = @("forward", "service", $sandboxName, "--target-port", "$targetPort", "--local", "127.0.0.1:0")
|
||||
$forwarder = $null
|
||||
try {
|
||||
$forwarder = Start-Process -FilePath $cli -WindowStyle Hidden -PassThru `
|
||||
-ArgumentList (($arguments | ForEach-Object { Quote-NativeArgument $_ }) -join ' ') `
|
||||
-RedirectStandardOutput $outLog -RedirectStandardError $errLog
|
||||
$deadline = (Get-Date).AddSeconds(30)
|
||||
$localPort = 0
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if ($forwarder.HasExited) { throw "CLI forwarder exited early (code $($forwarder.ExitCode)); see $errLog" }
|
||||
$text = (Get-Content -LiteralPath $errLog -Raw -ErrorAction SilentlyContinue)
|
||||
if ($text -match 'Forwarding 127\.0\.0\.1:(\d+) ->') {
|
||||
$localPort = [int]$Matches[1]
|
||||
break
|
||||
}
|
||||
Start-Sleep -Milliseconds 200
|
||||
}
|
||||
if ($localPort -eq 0) { throw "CLI forwarder did not bind within 30 seconds" }
|
||||
for ($exchange = 1; $exchange -le 2; $exchange++) {
|
||||
$socket = New-Object System.Net.Sockets.TcpClient
|
||||
$reader = $null
|
||||
try {
|
||||
$connect = $socket.BeginConnect('127.0.0.1', $localPort, $null, $null)
|
||||
try {
|
||||
if (-not $connect.AsyncWaitHandle.WaitOne(5000)) { throw "Forward client connection timed out" }
|
||||
$socket.EndConnect($connect)
|
||||
} finally { $connect.AsyncWaitHandle.Close() }
|
||||
$socket.NoDelay = $true
|
||||
$stream = $socket.GetStream()
|
||||
$stream.ReadTimeout = 10000
|
||||
$stream.WriteTimeout = 10000
|
||||
$request = "request-$exchange-$([guid]::NewGuid().ToString('N'))"
|
||||
$bytes = [Text.Encoding]::ASCII.GetBytes("$request`n")
|
||||
$stream.Write($bytes, 0, $bytes.Length)
|
||||
$socket.Client.Shutdown([Net.Sockets.SocketShutdown]::Send)
|
||||
$reader = New-Object IO.StreamReader($stream, [Text.Encoding]::ASCII)
|
||||
$reply = $reader.ReadLine()
|
||||
if ($reply -ne "$nonce`:$request") { throw "Forwarded exchange $exchange returned an unexpected response: '$reply'" }
|
||||
Info "forwarded exchange ${exchange}: exact workload response verified"
|
||||
} finally {
|
||||
if ($reader) { $reader.Dispose() }
|
||||
$socket.Close()
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if ($forwarder) {
|
||||
if (-not $forwarder.HasExited) { Stop-Process -Id $forwarder.Id -Force -ErrorAction SilentlyContinue }
|
||||
$forwarder.Dispose()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Detect an agent *launch* failure (binary not found / not implemented) vs a
|
||||
# legitimate policy denial. Used to avoid false-passing a deny scenario when the
|
||||
# agent never actually ran.
|
||||
@@ -508,6 +572,13 @@ try {
|
||||
Backends = "both"; Kind = "positive"
|
||||
PosTarget = (Join-Path $DemoDir "network-policy-result.txt")
|
||||
Description = "supervisor-owned network policy is accepted and workload runs (not an egress assertion)"
|
||||
},
|
||||
@{
|
||||
Name = "forwarding"; PolicyFile = Join-Path $policyDir "forwarding.yaml"
|
||||
SandboxId = "fwd"
|
||||
Backends = "process_container"; Kind = "forwarding"
|
||||
ReadyFile = (Join-Path $DemoDir "forwarding-port.txt")
|
||||
Description = "real MXC listener receives two TCP exchanges through authenticated OpenShell forwarding"
|
||||
}
|
||||
)
|
||||
|
||||
@@ -579,6 +650,19 @@ try {
|
||||
} else {
|
||||
$command = @($cmdExe, "/c", "echo denied 1> $denied")
|
||||
}
|
||||
} elseif ($sc.Kind -eq "forwarding") {
|
||||
Remove-Item -LiteralPath $sc.ReadyFile -Force -ErrorAction SilentlyContinue
|
||||
$forwardNonce = [guid]::NewGuid().ToString('N')
|
||||
$fixture = Join-Path $BinaryDir 'mxc-forwarding-agent.exe'
|
||||
if (-not (Test-Path -LiteralPath $fixture)) {
|
||||
$fixture = Join-Path $BinaryDir 'examples/mxc-forwarding-agent.exe'
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $fixture)) {
|
||||
throw "Missing forwarding fixture; run mise run --skip-tools windows:build:mxc-fixtures"
|
||||
}
|
||||
$workload = Join-Path $DemoDir 'mxc-forwarding-agent.exe'
|
||||
Copy-Item -LiteralPath $fixture -Destination $workload -Force
|
||||
$command = @($workload, $sc.ReadyFile, $forwardNonce)
|
||||
} else {
|
||||
$command = @($cmdExe, "/c", "exit 0")
|
||||
}
|
||||
@@ -635,6 +719,21 @@ try {
|
||||
if (Launch-Failed $gwText) { Info "gateway log shows an agent-launch failure (not a policy result)" }
|
||||
$results += [pscustomobject]@{ Scenario = $sc.Name; Result = "FAIL"; Reason = "artifact absent" }
|
||||
}
|
||||
} elseif ($sc.Kind -eq "forwarding") {
|
||||
try {
|
||||
if ($createExitCode -ne 0) { throw "sandbox create failed: $createOutStr" }
|
||||
Test-Forwarding $sandboxName $sc.ReadyFile $forwardNonce
|
||||
Ok "$($sc.Name): both forwarded request/reply exchanges succeeded"
|
||||
$results += [pscustomobject]@{ Scenario = $sc.Name; Result = "PASS"; Reason = "two exact TCP replies through OpenShell" }
|
||||
} catch {
|
||||
Bad "$($sc.Name): $($_.Exception.Message)"
|
||||
$results += [pscustomobject]@{ Scenario = $sc.Name; Result = "FAIL"; Reason = $_.Exception.Message }
|
||||
try {
|
||||
$diagnostic = Invoke-NativeCaptured $cli @('sandbox', 'get', $sandboxName, '-o', 'json') 5
|
||||
$diagnostic.Output | Set-Content -LiteralPath (Join-Path $resultDir 'forward.sandbox.json') -Encoding UTF8
|
||||
Info "sandbox state captured in forward.sandbox.json"
|
||||
} catch { Info "sandbox diagnostics unavailable: $($_.Exception.Message)" }
|
||||
}
|
||||
} else {
|
||||
# deny
|
||||
if ($sc.ControlTarget) {
|
||||
@@ -739,8 +838,10 @@ Files in this bundle ($resultDir):
|
||||
|
||||
What PASS means: every selected scenario ran against real MXC and met its expected verdict - positive
|
||||
writes produced their artifact, deny writes were blocked with either a control
|
||||
write or driver-launch evidence. Missing coverage exits non-zero;
|
||||
network-policy proves admission and execution, not network enforcement.
|
||||
write or driver-launch evidence, and forwarding verified two exact TCP replies
|
||||
through OpenShell. Missing coverage exits non-zero; network-policy proves
|
||||
admission and execution, not network enforcement. Forwarding proves managed
|
||||
ingress works, not that direct host ingress is blocked.
|
||||
"@
|
||||
Set-Content -Path (Join-Path $resultDir "summary.txt") -Value $summary -Encoding UTF8
|
||||
Write-Host $summary -ForegroundColor ($(if ($verdict -eq "PASS") { "Green" } else { "Red" }))
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Remote half of test-mxc-remote.ps1. No source checkout or build tools required.
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
$PSNativeCommandUseErrorActionPreference = $false
|
||||
$repo = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..'))
|
||||
$export = Join-Path $repo 'result-export'
|
||||
New-Item -ItemType Directory -Path $export | Out-Null
|
||||
$manifest = Get-Content -Raw (Join-Path $repo 'manifest.json') | ConvertFrom-Json
|
||||
Copy-Item -LiteralPath (Join-Path $repo 'manifest.json') -Destination $export
|
||||
$code = 1
|
||||
$transcript = $false
|
||||
function Get-PeMachine([string] $Path) {
|
||||
$stream = [IO.File]::OpenRead($Path)
|
||||
$reader = New-Object IO.BinaryReader($stream)
|
||||
try {
|
||||
if ($reader.ReadUInt16() -ne 0x5a4d) { throw "Not a PE executable: $Path" }
|
||||
$stream.Position = 0x3c
|
||||
$offset = $reader.ReadInt32()
|
||||
$stream.Position = $offset
|
||||
if ($reader.ReadUInt32() -ne 0x4550) { throw "Invalid PE signature: $Path" }
|
||||
return $reader.ReadUInt16()
|
||||
} finally { $reader.Dispose() }
|
||||
}
|
||||
try {
|
||||
Start-Transcript -Path (Join-Path $export 'remote-transcript.txt') | Out-Null
|
||||
$transcript = $true
|
||||
$native = Get-ItemPropertyValue 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' PROCESSOR_ARCHITECTURE
|
||||
$expected = if ($manifest.target -eq 'aarch64-pc-windows-msvc') { 0xaa64 } else { 0x8664 }
|
||||
if (($expected -eq 0xaa64 -and $native -ne 'ARM64') -or ($expected -eq 0x8664 -and $native -ne 'AMD64')) { throw 'Native host architecture changed.' }
|
||||
foreach ($name in @('openshell.exe', 'openshell-gateway.exe', 'openshell-supervisor.exe', 'openshell-windows-sandbox.exe', 'mxc-forwarding-agent.exe', 'libz3.dll')) {
|
||||
if ((Get-PeMachine (Join-Path $repo "bin/$name")) -ne $expected) { throw "Wrong executable architecture: $name" }
|
||||
}
|
||||
$wxc = $manifest.wxcExecPath
|
||||
# Check loader dependencies before the harness starts a gateway.
|
||||
$startInfo = New-Object Diagnostics.ProcessStartInfo
|
||||
$startInfo.FileName = Join-Path $repo 'bin/openshell-gateway.exe'
|
||||
$startInfo.Arguments = '--version'
|
||||
$startInfo.UseShellExecute = $false
|
||||
$startInfo.CreateNoWindow = $true
|
||||
$startInfo.RedirectStandardOutput = $true
|
||||
$startInfo.RedirectStandardError = $true
|
||||
$process = New-Object Diagnostics.Process
|
||||
$process.StartInfo = $startInfo
|
||||
try {
|
||||
[void]$process.Start()
|
||||
$stdout = $process.StandardOutput.ReadToEndAsync()
|
||||
$stderr = $process.StandardError.ReadToEndAsync()
|
||||
if (-not $process.WaitForExit(15000)) { $process.Kill(); throw 'Gateway loader check timed out.' }
|
||||
@($stdout.Result, $stderr.Result) | Set-Content (Join-Path $export 'gateway-version.txt') -Encoding UTF8
|
||||
if ($process.ExitCode -ne 0) { throw "Gateway loader check failed: exit $($process.ExitCode). Check the packaged DLLs and native Visual C++ runtime." }
|
||||
Write-Host "Gateway loader check passed: $($stdout.Result.Trim())"
|
||||
} finally { $process.Dispose() }
|
||||
if (-not $wxc) { $wxc = (Get-Command wxc-exec.exe -ErrorAction Stop).Source }
|
||||
$openssl = (Get-Command openssl.exe -ErrorAction Stop).Source
|
||||
$probe = @(& $wxc --probe)
|
||||
$probeCode = $LASTEXITCODE
|
||||
$probe | Set-Content -LiteralPath (Join-Path $export 'wxc-probe.json') -Encoding UTF8
|
||||
[ordered]@{ hostName = $env:COMPUTERNAME; nativeArchitecture = $native; shellArchitecture = $env:PROCESSOR_ARCHITECTURE;
|
||||
windowsVersion = [Environment]::OSVersion.Version.ToString(); wxcExecPath = $wxc;
|
||||
wxcVersion = (Get-Item -LiteralPath $wxc).VersionInfo.ProductVersion;
|
||||
wxcSha256 = (Get-FileHash -LiteralPath $wxc).Hash; wxcPeMachine = (Get-PeMachine $wxc);
|
||||
probeExitCode = $probeCode; openssl = $openssl } | ConvertTo-Json | Set-Content (Join-Path $export 'host.json') -Encoding UTF8
|
||||
if ($probeCode -ne 0) { throw "MXC probe failed: exit $probeCode" }
|
||||
Write-Host "Native host: $native; MXC: $wxc; running unchanged REAL E2E"
|
||||
$harness = Join-Path $repo 'crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1'
|
||||
$arguments = @('-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', $harness,
|
||||
'-BinaryDir', (Join-Path $repo 'bin'), '-WxcExecPath', $wxc, '-Backend', 'process_container', '-Port', "$($manifest.gatewayPort)")
|
||||
if ($manifest.scenario) { $arguments += @('-Scenario', $manifest.scenario) }
|
||||
& powershell.exe @arguments
|
||||
$code = $LASTEXITCODE
|
||||
} catch {
|
||||
Write-Warning $_.Exception.Message
|
||||
$_ | Out-String | Set-Content -LiteralPath (Join-Path $export 'error.txt')
|
||||
} finally {
|
||||
Set-Content -LiteralPath (Join-Path $export 'exit-code.txt') -Value $code
|
||||
$resultsRoot = Join-Path $repo 'target/windows-e2e-results'
|
||||
if (Test-Path $resultsRoot) {
|
||||
# Only copy completed harness bundles and summaries, never sibling keys-*.
|
||||
Get-ChildItem $resultsRoot -Filter 'results-e2e-*.zip' -File | Copy-Item -Destination $export
|
||||
foreach ($dir in Get-ChildItem $resultsRoot -Filter 'results-e2e-*' -Directory) {
|
||||
$summary = Join-Path $dir.FullName 'summary.txt'
|
||||
if (Test-Path $summary) { Copy-Item -LiteralPath $summary -Destination (Join-Path $export "$($dir.Name)-summary.txt") }
|
||||
}
|
||||
}
|
||||
if ($transcript) { Stop-Transcript | Out-Null }
|
||||
Compress-Archive -Path (Join-Path $export '*') -DestinationPath (Join-Path $repo 'results.zip')
|
||||
}
|
||||
exit $code
|
||||
@@ -0,0 +1,189 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Build locally, upload content-addressed artifacts, run unchanged real E2E,
|
||||
# and download diagnostic bundles even when tests return a nonzero exit code.
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)] [string] $HostName,
|
||||
[Parameter(Mandatory = $true)] [string] $UserName,
|
||||
[string] $IdentityFile = (Join-Path $env:USERPROFILE '.ssh/openshell-mxc-test'),
|
||||
[ValidateSet('auto', 'arm64', 'x64')] [string] $Architecture = 'auto',
|
||||
[string] $BuildDirectory,
|
||||
[string] $Z3DllPath,
|
||||
[string] $RemoteRoot,
|
||||
[string] $WxcExecPath,
|
||||
[string] $Scenario,
|
||||
[ValidateRange(1, 65535)] [int] $SshPort = 22,
|
||||
[ValidateRange(1, 65535)] [int] $GatewayPort = 17670,
|
||||
[switch] $SkipBuild
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$PSNativeCommandUseErrorActionPreference = $false
|
||||
$repo = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
|
||||
if ($HostName -notmatch '^[A-Za-z0-9.-]+$' -or $UserName -notmatch '^[A-Za-z0-9_.-]+$') {
|
||||
throw 'Use a hostname/IP and a simple SSH username (or configure an SSH alias).'
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) { throw "Missing SSH key: $IdentityFile" }
|
||||
$ssh = (Get-Command ssh.exe -ErrorAction Stop).Source
|
||||
$scp = (Get-Command scp.exe -ErrorAction Stop).Source
|
||||
$peer = "$UserName@$HostName"
|
||||
$sshOptions = @('-i', $IdentityFile, '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes',
|
||||
'-o', 'StrictHostKeyChecking=yes', '-o', 'ConnectTimeout=15',
|
||||
'-o', 'ServerAliveInterval=15', '-o', 'ServerAliveCountMax=3', '-p', "$SshPort")
|
||||
$scpOptions = @('-i', $IdentityFile, '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes',
|
||||
'-o', 'StrictHostKeyChecking=yes', '-o', 'ConnectTimeout=15', '-P', "$SshPort")
|
||||
function Quote-PS([string] $Value) { return "'" + $Value.Replace("'", "''") + "'" }
|
||||
function Find-Z3Runtime([string] $BuildRoot, [string] $RustTarget, [string] $ExplicitPath) {
|
||||
if ($ExplicitPath) {
|
||||
if (-not (Test-Path -LiteralPath $ExplicitPath -PathType Leaf)) { throw "Missing Z3 runtime: $ExplicitPath" }
|
||||
return (Resolve-Path -LiteralPath $ExplicitPath).Path
|
||||
}
|
||||
$release = Join-Path $BuildRoot "$RustTarget/release"
|
||||
$adjacent = Join-Path $release 'libz3.dll'
|
||||
if (Test-Path -LiteralPath $adjacent -PathType Leaf) { return $adjacent }
|
||||
# Windows prebuilt-z3 links an import library rather than a static library.
|
||||
$candidates = @(Get-ChildItem -Path (Join-Path $release 'build/z3-sys-*/out/z3-*/bin/libz3.dll') -File -ErrorAction SilentlyContinue)
|
||||
if ($candidates.Count -eq 0) { throw 'libz3.dll not found in the target build cache. Supply -Z3DllPath matching the linked Z3 build.' }
|
||||
$hashes = @($candidates | ForEach-Object { (Get-FileHash -LiteralPath $_.FullName).Hash } | Select-Object -Unique)
|
||||
if ($hashes.Count -ne 1) { throw 'Multiple different Z3 runtimes found. Supply -Z3DllPath matching the linked Z3 build.' }
|
||||
return $candidates[0].FullName
|
||||
}
|
||||
function Read-CacheHashes([string] $Json) {
|
||||
# Windows PowerShell 5.1 emits decoded JSON arrays as one pipeline object.
|
||||
# Explicitly enumerate so callers do not get a nested array with -contains.
|
||||
$decoded = $Json | ConvertFrom-Json
|
||||
foreach ($hash in $decoded) {
|
||||
if ($hash -notmatch '^[a-fA-F0-9]{64}$') { throw 'Invalid hash in remote cache response.' }
|
||||
Write-Output $hash
|
||||
}
|
||||
}
|
||||
function Remote-Command([string] $Code) {
|
||||
$encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes(
|
||||
'$ProgressPreference="SilentlyContinue"; $ErrorActionPreference="Stop"; ' + $Code))
|
||||
& $ssh @sshOptions $peer "powershell.exe -NoProfile -NonInteractive -EncodedCommand $encoded"
|
||||
}
|
||||
|
||||
# Registry architecture is native even if the SSH shell runs under emulation.
|
||||
$hostJson = @(Remote-Command @'
|
||||
[pscustomobject]@{
|
||||
name = $env:COMPUTERNAME
|
||||
architecture = (Get-ItemPropertyValue 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' PROCESSOR_ARCHITECTURE)
|
||||
profile = $env:USERPROFILE
|
||||
os = [Environment]::OSVersion.Version.ToString()
|
||||
} | ConvertTo-Json -Compress
|
||||
'@)
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Remote host inspection failed.' }
|
||||
$hostInfo = ($hostJson -join "`n") | ConvertFrom-Json
|
||||
$native = switch ($hostInfo.architecture) { 'ARM64' { 'arm64' } 'AMD64' { 'x64' } default { throw 'Unsupported remote architecture.' } }
|
||||
if ($Architecture -eq 'auto') { $Architecture = $native }
|
||||
if ($Architecture -ne $native) { throw "Requested $Architecture does not match native remote $native." }
|
||||
$target = if ($Architecture -eq 'arm64') { 'aarch64-pc-windows-msvc' } else { 'x86_64-pc-windows-msvc' }
|
||||
if (-not $BuildDirectory) { $BuildDirectory = Join-Path $repo 'target/remote-mxc-build' }
|
||||
$BuildDirectory = [IO.Path]::GetFullPath($BuildDirectory)
|
||||
if (-not $RemoteRoot) { $RemoteRoot = Join-Path $hostInfo.profile 'openshell-mxc-tests' }
|
||||
if ($RemoteRoot -notmatch '^[A-Za-z]:[\\/]' -or $RemoteRoot -match '[\r\n]') { throw 'RemoteRoot must be an absolute Windows directory.' }
|
||||
$RemoteRoot = $RemoteRoot.Replace('/', '\').TrimEnd('\')
|
||||
$commit = (& git -C $repo rev-parse HEAD).Trim()
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Cannot determine checkout commit.' }
|
||||
$dirty = @(& git -C $repo status --porcelain)
|
||||
$runId = (Get-Date -Format 'yyyyMMdd-HHmmss') + '-' + $commit.Substring(0, 12) + '-' + [guid]::NewGuid().ToString('N').Substring(0, 6)
|
||||
$localRun = Join-Path $repo "target/windows-remote-results/$runId"
|
||||
New-Item -ItemType Directory -Force -Path $localRun | Out-Null
|
||||
$transcript = $false
|
||||
$exitCode = 1
|
||||
try {
|
||||
Start-Transcript -Path (Join-Path $localRun 'local-transcript.txt') | Out-Null
|
||||
$transcript = $true
|
||||
Write-Host "Host: $($hostInfo.name), $native, Windows $($hostInfo.os); checkout: $commit"
|
||||
if (-not $SkipBuild) {
|
||||
$previousTarget = $env:CARGO_TARGET_DIR
|
||||
Push-Location $repo
|
||||
try {
|
||||
$env:CARGO_TARGET_DIR = $BuildDirectory
|
||||
& mise run --skip-tools "windows:build:$Architecture"
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Local Windows build failed; no tests launched.' }
|
||||
& mise run --skip-tools "windows:build:mxc-fixtures:$Architecture"
|
||||
if ($LASTEXITCODE -ne 0) { throw 'MXC E2E fixture build failed; no tests launched.' }
|
||||
} finally {
|
||||
$env:CARGO_TARGET_DIR = $previousTarget
|
||||
Pop-Location
|
||||
foreach ($logName in @("build-$target-release.log", "build-$target-mxc-fixtures.log")) {
|
||||
$buildLog = Join-Path $repo $logName
|
||||
if (Test-Path $buildLog) { Copy-Item -LiteralPath $buildLog -Destination $localRun }
|
||||
}
|
||||
}
|
||||
} else { Write-Warning 'SkipBuild: using existing binaries, not claiming they were rebuilt from this checkout.' }
|
||||
|
||||
$files = @()
|
||||
foreach ($name in @('openshell.exe', 'openshell-gateway.exe', 'openshell-supervisor.exe', 'openshell-windows-sandbox.exe')) {
|
||||
$files += @{ source = (Join-Path $BuildDirectory "$target/release/$name"); path = "bin/$name" }
|
||||
}
|
||||
$examples = 'crates/openshell-driver-mxc/examples'
|
||||
$files += @{ source = (Join-Path $BuildDirectory "$target/release/examples/mxc-forwarding-agent.exe"); path = 'bin/mxc-forwarding-agent.exe' }
|
||||
$files += @{ source = (Find-Z3Runtime $BuildDirectory $target $Z3DllPath); path = 'bin/libz3.dll' }
|
||||
foreach ($path in @("$examples/run-mxc-e2e.ps1", "$examples/mxc-gateway.toml", 'tasks/scripts/run-mxc-remote.ps1')) {
|
||||
$files += @{ source = (Join-Path $repo $path); path = $path }
|
||||
}
|
||||
foreach ($policy in Get-ChildItem (Join-Path $repo "$examples/e2e-policies") -File) {
|
||||
$files += @{ source = $policy.FullName; path = "$examples/e2e-policies/$($policy.Name)" }
|
||||
}
|
||||
$entries = @($files | ForEach-Object {
|
||||
if (-not (Test-Path -LiteralPath $_.source -PathType Leaf)) { throw "Missing artifact: $($_.source)" }
|
||||
[pscustomobject]@{ source = $_.source; path = $_.path; sha256 = (Get-FileHash -LiteralPath $_.source -Algorithm SHA256).Hash.ToLowerInvariant(); bytes = (Get-Item -LiteralPath $_.source).Length }
|
||||
})
|
||||
$manifest = [ordered]@{ runId = $runId; commit = $commit; dirty = $dirty; buildSkipped = [bool]$SkipBuild;
|
||||
target = $target; hostInfo = $hostInfo; wxcExecPath = $WxcExecPath; scenario = $Scenario; gatewayPort = $GatewayPort;
|
||||
files = @($entries | Select-Object path, sha256, bytes) }
|
||||
$manifestPath = Join-Path $localRun 'manifest.json'
|
||||
$manifest | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $manifestPath -Encoding UTF8
|
||||
$remoteRun = "$RemoteRoot\runs\$runId"
|
||||
$cache = "$RemoteRoot\cache"
|
||||
Remote-Command "New-Item -ItemType Directory -Force -Path $(Quote-PS $remoteRun),$(Quote-PS $cache) | Out-Null"
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Could not create remote test directories.' }
|
||||
& $scp @scpOptions $manifestPath "${peer}:$($remoteRun.Replace('\', '/'))/manifest.json"
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Manifest transfer failed.' }
|
||||
$missingJson = @(Remote-Command @"
|
||||
`$m = Get-Content -Raw $(Quote-PS "$remoteRun\manifest.json") | ConvertFrom-Json
|
||||
`$missing = @(`$m.files | Where-Object { `$p = Join-Path $(Quote-PS $cache) `$_.sha256; !(Test-Path -LiteralPath `$p) -or (Get-FileHash -LiteralPath `$p).Hash -ne `$_.sha256 } | ForEach-Object { `$_.sha256 })
|
||||
ConvertTo-Json -InputObject `$missing -Compress
|
||||
"@)
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Remote artifact cache inspection failed.' }
|
||||
$missing = @(Read-CacheHashes ($missingJson -join "`n"))
|
||||
foreach ($entry in $entries) {
|
||||
if ($missing -contains $entry.sha256) {
|
||||
Write-Host "Uploading $($entry.path) ($($entry.bytes) bytes)"
|
||||
& $scp @scpOptions $entry.source "${peer}:$($cache.Replace('\', '/'))/$($entry.sha256)"
|
||||
if ($LASTEXITCODE -ne 0) { throw "Transfer failed: $($entry.path)" }
|
||||
} else { Write-Host "Cached: $($entry.path)" }
|
||||
}
|
||||
Remote-Command @"
|
||||
`$m = Get-Content -Raw $(Quote-PS "$remoteRun\manifest.json") | ConvertFrom-Json
|
||||
foreach (`$f in `$m.files) {
|
||||
`$src = Join-Path $(Quote-PS $cache) `$f.sha256
|
||||
if ((Get-FileHash -LiteralPath `$src).Hash -ne `$f.sha256) { throw 'Remote artifact hash mismatch' }
|
||||
`$dst = Join-Path $(Quote-PS $remoteRun) `$f.path
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path `$dst) | Out-Null
|
||||
Copy-Item -LiteralPath `$src -Destination `$dst
|
||||
}
|
||||
"@
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Remote payload verification failed.' }
|
||||
$command = "& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $(Quote-PS "$remoteRun\tasks\scripts\run-mxc-remote.ps1"); exit `$LASTEXITCODE"
|
||||
Remote-Command $command | Tee-Object -FilePath (Join-Path $localRun 'ssh-console.log')
|
||||
$exitCode = $LASTEXITCODE
|
||||
# Download independently of test status. Never include the harness key directory.
|
||||
& $scp @scpOptions "${peer}:$($remoteRun.Replace('\', '/'))/results.zip" (Join-Path $localRun 'results.zip')
|
||||
if ($LASTEXITCODE -ne 0) { throw "Results download failed (remote test exit $exitCode). Artifacts remain in $remoteRun" }
|
||||
Expand-Archive -LiteralPath (Join-Path $localRun 'results.zip') -DestinationPath (Join-Path $localRun 'results')
|
||||
Write-Host "Remote test exit: $exitCode; results: $localRun"
|
||||
} catch {
|
||||
Write-Warning $_.Exception.Message
|
||||
$_ | Out-String | Set-Content -LiteralPath (Join-Path $localRun 'error.txt')
|
||||
$exitCode = 1
|
||||
} finally {
|
||||
if ($transcript) { Stop-Transcript | Out-Null }
|
||||
Write-Host "Local logs: $localRun"
|
||||
}
|
||||
exit $exitCode
|
||||
@@ -6,7 +6,7 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true, Position = 0)]
|
||||
[ValidateSet("check", "lint", "build", "test", "test-precommit", "test-unsupported", "test-mxc-real", "artifacts", "ci")]
|
||||
[ValidateSet("check", "lint", "build", "build-mxc-fixtures", "test", "test-precommit", "test-unsupported", "test-mxc-real", "artifacts", "ci")]
|
||||
[string] $Action,
|
||||
|
||||
[Parameter(Position = 1)]
|
||||
@@ -614,7 +614,7 @@ if ($Action -in @("test", "test-precommit", "test-unsupported", "test-mxc-real")
|
||||
}
|
||||
}
|
||||
|
||||
if ($Action -in @("check", "lint", "build", "test", "test-precommit", "test-unsupported", "test-mxc-real", "ci")) {
|
||||
if ($Action -in @("check", "lint", "build", "build-mxc-fixtures", "test", "test-precommit", "test-unsupported", "test-mxc-real", "ci")) {
|
||||
$z3Features = Configure-Z3
|
||||
$Z3WorkspaceFeatures = $z3Features.WorkspaceFeatures
|
||||
$Z3ServerFeatures = $z3Features.ServerFeatures
|
||||
@@ -641,6 +641,13 @@ switch ($Action) {
|
||||
}
|
||||
Show-Artifacts $targets
|
||||
}
|
||||
"build-mxc-fixtures" {
|
||||
foreach ($rustTarget in $targets) {
|
||||
Invoke-VsCargo -RustTarget $rustTarget `
|
||||
-CargoArgs "cargo build --release -p openshell-driver-mxc --example mxc-forwarding-agent --target $rustTarget" `
|
||||
-LogName "build-$rustTarget-mxc-fixtures.log"
|
||||
}
|
||||
}
|
||||
"test" {
|
||||
foreach ($rustTarget in $targets) {
|
||||
Invoke-Test $rustTarget
|
||||
|
||||
@@ -87,5 +87,21 @@ run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts
|
||||
|
||||
["windows:e2e:mxc"]
|
||||
description = "Run real MXC Tier-3 e2e; missing runtime coverage exits non-zero (no mocks)"
|
||||
depends = ["windows:build:mxc-fixtures"]
|
||||
run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1"
|
||||
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1"
|
||||
|
||||
["windows:build:mxc-fixtures"]
|
||||
description = "Build test-only MXC forwarding fixture for the native host"
|
||||
run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1"
|
||||
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 build-mxc-fixtures native"
|
||||
|
||||
["windows:build:mxc-fixtures:x64"]
|
||||
description = "Build test-only MXC forwarding fixture for x64"
|
||||
run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1"
|
||||
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 build-mxc-fixtures x86_64-pc-windows-msvc"
|
||||
|
||||
["windows:build:mxc-fixtures:arm64"]
|
||||
description = "Build test-only MXC forwarding fixture for ARM64"
|
||||
run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1"
|
||||
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 build-mxc-fixtures aarch64-pc-windows-msvc"
|
||||
|
||||
Reference in New Issue
Block a user