Files
openbot/scripts/linux-gpu-fallback-e2e.ts
T
4607005848 Use the GPU on Linux until it fails on this computer (#2308)
* Use the GPU on Linux until it fails on this computer

#1046 turned off hardware acceleration for all Linux computers (#992), so
surfaces with a blurred backdrop render on the CPU also where the GPU works.

Hardware acceleration now stays on. A GPU process crash (gpu-crash.json) or
a start that stops before it is confirmed (gpu-start.json, for the SIGTRAP
case where main stops first) turns on software rendering at later starts
with the same Electron version. OPENBOT_DISABLE_GPU=1 or 0 overrides it.

scripts/linux-gpu-fallback-e2e.ts runs the real Electron and crashes the
real GPU process.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Keep software rendering on Linux servers, and key GPU failures on the Chromium major version

- A server (`OPENBOT_SERVER=1` or a hosted server) runs under Xvfb with no GPU. It keeps software
  rendering, as before #2308, and writes no start file.
- A failure record keeps the Chromium major version, not the Electron version. An Electron patch
  release keeps Chromium's major version, so it does not make a failing computer stop again.
- Only `configureLinuxGpu` and its option and result types are exported (knip).
- The E2E test has a server step. docs/TROUBLESHOOTING.md tells how to try the GPU again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Confirm the GPU start when a quit starts, not after the teardown

`will-quit` fires only after the teardown. The teardown can take longer than a stop allows: the
shutdown deadline ends with `app.exit`, and `dev:stop` sends SIGKILL after a few seconds. Then a
normal quit in the first 30 s left `gpu-start.json`, and later starts used software rendering.
`before-quit` fires when the quit starts.

A second instance can read the start file of a first instance that is still new. It no longer logs
that the GPU failed before.

The E2E test has a step that kills the app during a teardown that does not end. With `will-quit`,
this step fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Keep the GPU on Linux after a GPU context loss

When the GPU context is lost, for example after suspend and resume, Chromium stops the GPU process on
purpose with exit code 34 (`viz::ExitCode::RESULT_CODE_GPU_EXIT_ON_CONTEXT_LOST`) and starts it
again. Electron reports this exit as `abnormal-exit`, so one context loss wrote `gpu-crash.json`, and
all later starts used software rendering until a new Chromium major version. This exit no longer
counts as a GPU failure.

The E2E test sends this exit to the handler and checks that no crash file is written. Without the
fix, this step fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Natan Pimentel <natanpimentel@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: NB😈 <norbertbodziony@gmail.com>
2026-10-16 19:25:59 +02:00

266 lines
12 KiB
TypeScript

// Runs the real Electron with the Linux GPU mode of `src/main/linux-gpu-mode.ts` (#992):
// 1. A first start uses the GPU. A crash of the real GPU process (SIGSEGV) writes `gpu-crash.json`.
// 2. The next start uses software rendering. `OPENBOT_DISABLE_GPU=0` uses the GPU again.
// 3. A crash file from another Chromium major version does not count, and a clean quit removes the start file.
// A GPU process exit after a GPU context loss, which Chromium restarts, writes no crash file.
// 4. A start that is killed before it is confirmed makes the next start use software rendering. A quit
// that starts confirms the start, also when the teardown is killed before it ends.
// 5. `OPENBOT_DISABLE_GPU=1` uses software rendering and writes no start file.
// 6. A server uses software rendering and writes no start file, also with `OPENBOT_DISABLE_GPU=0`.
//
// Run on a Linux desktop with a working GPU: bun scripts/linux-gpu-fallback-e2e.ts
// It writes .openbot-build/linux-gpu-fallback-e2e/report.json and exits non-zero on a failure.
import assert from "node:assert/strict";
import { type ChildProcess, spawn, spawnSync } from "node:child_process";
import { closeSync, existsSync, openSync, readdirSync, readFileSync } from "node:fs";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { withoutElectronRuntimeFlags } from "./electron-spawn-env";
const ROOT = resolve(import.meta.dirname, "..");
const OUT = join(ROOT, ".openbot-build", "linux-gpu-fallback-e2e");
const STATE_TIMEOUT_MS = 30_000;
// The file names of `src/main/linux-gpu-mode.ts`, which imports Electron and cannot load in Bun.
const GPU_START_FILE = "gpu-start.json";
const GPU_CRASH_FILE = "gpu-crash.json";
// The Electron binary itself, not the `.bin/electron` Node wrapper: the test kills the main process.
const electronPath = join(
ROOT,
"node_modules",
"electron",
"dist",
readFileSync(join(ROOT, "node_modules", "electron", "path.txt"), "utf8").trim(),
);
if (process.platform !== "linux") {
process.stdout.write("SKIP: Linux only\n");
process.exit(0);
}
interface ReadyLine {
pid: number;
mode: { hardwareAcceleration: boolean; reason: string };
hardwareAccelerationEnabled: boolean;
gpuCompositing: string;
}
interface Step {
step: string;
ready: ReadyLine;
gpuProcess?: number;
}
interface Launch {
child: ChildProcess;
ready: ReadyLine;
exited: Promise<number | null>;
}
await mkdir(OUT, { recursive: true });
const root = await mkdtemp(join(tmpdir(), "openbot-linux-gpu-"));
const bundle = join(root, "linux-gpu-fallback-electron.mjs");
const steps: Step[] = [];
const children: ChildProcess[] = [];
let launchCount = 0;
let passed = false;
let error: string | undefined;
try {
const build = spawnSync(process.execPath, [
"build",
join(ROOT, "scripts", "linux-gpu-fallback-electron.ts"),
"--target=node",
"--format=esm",
"--external=electron",
`--outfile=${bundle}`,
]);
assert.equal(build.status, 0, `bun build failed: ${build.stderr.toString()}`);
// 1. The first start uses the GPU, and a crash of the real GPU process is recorded.
const data = join(root, "user-data");
let launch = await start(data, 1_000);
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: true, reason: "default" });
assert.equal(launch.ready.hardwareAccelerationEnabled, true);
assert.equal(existsSync(join(data, GPU_START_FILE)), true, "the start file exists while the start is new");
await until(() => !existsSync(join(data, GPU_START_FILE)), "the confirmed start removes the start file");
const gpuProcess = await until(() => descendantWithType(launch.ready.pid, "gpu-process"), "a GPU process");
steps.push({ step: "gpu start", ready: launch.ready, gpuProcess });
process.kill(gpuProcess, "SIGSEGV");
await until(() => existsSync(join(data, GPU_CRASH_FILE)), "the GPU crash writes the crash file");
await quit(launch);
// 2. The next start uses software rendering, and OPENBOT_DISABLE_GPU=0 overrides it.
launch = await start(data, 1_000);
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: false, reason: "gpu-process-crash" });
assert.equal(launch.ready.hardwareAccelerationEnabled, false);
assert.equal(existsSync(join(data, GPU_START_FILE)), false, "a software start writes no start file");
steps.push({ step: "after gpu crash", ready: launch.ready });
await quit(launch);
launch = await start(data, 1_000, { OPENBOT_DISABLE_GPU: "0" });
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: true, reason: "environment-on" });
steps.push({ step: "OPENBOT_DISABLE_GPU=0", ready: launch.ready });
await quit(launch);
// 3. A crash with another Chromium major version does not count. A clean quit confirms the start.
await writeFile(join(data, GPU_CRASH_FILE), JSON.stringify({ version: 1, chromium: "0" }));
launch = await start(data, 600_000);
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: true, reason: "default" });
await quit(launch);
assert.equal(existsSync(join(data, GPU_START_FILE)), false, "a clean quit removes the start file");
steps.push({ step: "crash from another Chromium major version", ready: launch.ready });
const contextLost = join(root, "context-lost");
launch = await start(contextLost, 1_000, {}, ["--context-lost"]);
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: true, reason: "default" });
assert.equal(existsSync(join(contextLost, GPU_CRASH_FILE)), false, "a context-loss exit writes no crash file");
await quit(launch);
launch = await start(contextLost, 1_000);
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: true, reason: "default" });
steps.push({ step: "after a GPU context loss", ready: launch.ready });
await quit(launch);
// 4. A start that stops before it is confirmed (as with SIGTRAP in #992) turns off the GPU.
const killed = join(root, "killed-start");
launch = await start(killed, 600_000);
await killTree(launch.ready.pid, launch.exited);
assert.equal(existsSync(join(killed, GPU_START_FILE)), true, "a killed start leaves the start file");
launch = await start(killed, 1_000);
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: false, reason: "unfinished-start" });
assert.equal(launch.ready.hardwareAccelerationEnabled, false);
steps.push({ step: "after unfinished start", ready: launch.ready });
await quit(launch);
const hung = join(root, "hung-quit");
launch = await start(hung, 600_000, {}, ["--hang-on-quit"]);
launch.child.stdin?.write("quit\n");
await until(() => !existsSync(join(hung, GPU_START_FILE)), "a quit that starts removes the start file");
await killTree(launch.ready.pid, launch.exited);
launch = await start(hung, 1_000);
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: true, reason: "default" });
steps.push({ step: "after a killed teardown", ready: launch.ready });
await quit(launch);
// 5. OPENBOT_DISABLE_GPU=1 always uses software rendering.
const forced = join(root, "forced-software");
launch = await start(forced, 1_000, { OPENBOT_DISABLE_GPU: "1" });
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: false, reason: "environment-off" });
assert.equal(existsSync(join(forced, GPU_START_FILE)), false);
steps.push({ step: "OPENBOT_DISABLE_GPU=1", ready: launch.ready });
await quit(launch);
// 6. A server has no GPU: software rendering, and the environment value does not change it.
const server = join(root, "server");
launch = await start(server, 1_000, { OPENBOT_DISABLE_GPU: "0" }, ["--server"]);
assert.deepEqual(launch.ready.mode, { hardwareAcceleration: false, reason: "server" });
assert.equal(launch.ready.hardwareAccelerationEnabled, false);
assert.equal(existsSync(join(server, GPU_START_FILE)), false, "a server writes no start file");
steps.push({ step: "server", ready: launch.ready });
await quit(launch);
passed = true;
} catch (cause) {
error = cause instanceof Error ? cause.message : String(cause);
} finally {
for (const child of children) {
if (child.pid !== undefined && child.exitCode === null && child.signalCode === null) {
await killTree(child.pid, once(child));
}
}
await rm(root, { recursive: true, force: true });
await mkdir(OUT, { recursive: true });
await writeFile(join(OUT, "report.json"), `${JSON.stringify({ passed, steps, error }, null, 2)}\n`);
process.stdout.write(
`${passed ? "PASS" : "FAIL"}${error ? `: ${error}` : ""}: report in ${join(OUT, "report.json")}\n`,
);
if (!passed) process.exitCode = 1;
}
// The app's output goes to a log file, not to a pipe: a closed pipe makes a later write in the app
// fail with EPIPE, and Electron shows that as an error dialog on the desktop.
async function start(
userData: string,
confirmMs: number,
environment: Record<string, string> = {},
extraArguments: string[] = [],
): Promise<Launch> {
const { OPENBOT_DISABLE_GPU: _inherited, ...base } = withoutElectronRuntimeFlags(process.env);
launchCount += 1;
const readyFile = join(root, `ready-${launchCount}.json`);
const log = openSync(join(OUT, `electron-${launchCount}.log`), "w");
const child = spawn(
electronPath,
[bundle, `--user-data=${userData}`, `--confirm-ms=${confirmMs}`, `--ready-file=${readyFile}`, ...extraArguments],
{ env: { ...base, ...environment }, stdio: ["pipe", log, log] },
);
closeSync(log);
children.push(child);
const exited = once(child);
let stopped = false;
void exited.then(() => {
stopped = true;
});
await until(() => existsSync(readyFile) || stopped, "the app is ready");
assert.equal(stopped, false, `Electron stopped before it was ready: see ${OUT}`);
return { child, ready: JSON.parse(readFileSync(readyFile, "utf8")), exited };
}
function once(child: ChildProcess): Promise<number | null> {
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(child.exitCode);
return new Promise((done) => child.once("exit", (code) => done(code)));
}
/** Kills the main process as a crash would, then the child processes it leaves. */
async function killTree(pid: number, exited: Promise<number | null>): Promise<void> {
const descendants = descendantsOf(pid);
process.kill(pid, "SIGKILL");
await exited;
for (const pid of descendants) {
try {
process.kill(pid, "SIGKILL");
} catch {}
}
}
async function quit(launch: Launch): Promise<void> {
launch.child.stdin?.end("quit\n");
assert.equal(await launch.exited, 0, "Electron quits cleanly");
}
async function until<T>(read: () => T | undefined | false, what: string): Promise<T> {
const deadline = Date.now() + STATE_TIMEOUT_MS;
for (;;) {
const value = read();
if (value !== undefined && value !== false) return value;
if (Date.now() > deadline) throw new Error(`Timed out waiting for: ${what}`);
await sleep(100);
}
}
function descendantsOf(root: number): number[] {
const parents = new Map<number, number>();
for (const entry of readdirSync("/proc")) {
const pid = Number(entry);
if (!Number.isInteger(pid)) continue;
try {
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
parents.set(pid, Number(stat.slice(stat.lastIndexOf(")") + 2).split(" ")[1]));
} catch {}
}
return [...parents.keys()].filter((pid) => {
let parent = parents.get(pid);
while (parent !== undefined && parent !== root && parent > 1) parent = parents.get(parent);
return parent === root;
});
}
/** The PID of a process under `root` whose command line has `--type=<type>`. */
function descendantWithType(root: number, type: string): number | undefined {
return descendantsOf(root).find((pid) => {
try {
// Chromium rewrites its process title, so the arguments can be one string with spaces.
return readFileSync(`/proc/${pid}/cmdline`, "utf8").split(/[\0 ]/u).includes(`--type=${type}`);
} catch {
return false;
}
});
}