Health endpoint: return 503 for unexpected browser absence

Tracks _lastBrowserStopReason to distinguish intentional idle/admin
stops (200) from unexpected deaths like browser_disconnected,
memory_pressure, browser_rss_pressure (503 + warm retry).

Fly health checks will now detect and route around machines with
unexpectedly dead browsers, while intentional idle shutdown still
passes health checks as before.

server.js

@@ -580,6 +580,11 @@ let browserIdleTimer = null;
 let browserLaunchPromise = null;
 let browserWarmRetryTimer = null;
 
+// Tracks why the browser was last stopped. Intentional reasons (idle_shutdown, admin_stop)
+// keep /health returning 200. Unexpected reasons trigger 503 + warm retry.
+let _lastBrowserStopReason = null;
+const INTENTIONAL_STOP_REASONS = new Set(['idle_shutdown', 'admin_stop']);
+
 function scheduleBrowserIdleShutdown() {
   if (browserIdleTimer || sessions.size > 0 || !browser) return;
   browserIdleTimer = setTimeout(async () => {
@@ -788,6 +793,9 @@ async function _closeBrowserFullyImpl(reason) {
   const preCloseFds = _countOpenFds();
   const preCloseHandles = _countActiveHandles();
 
+  // Track stop reason for health semantics
+  _lastBrowserStopReason = reason;
+
   // Null the ref so new requests don't use a dying browser
   browser = null;
   _lastBrowserPid = null;
@@ -1013,6 +1021,7 @@ async function launchBrowserInstance() {
       browserLaunchProxy = launchProxy;
       _lastBrowserPid = candidateBrowser.process?.()?.pid ?? null;
       browser = candidateBrowser; // publish AFTER PID is captured
+      _lastBrowserStopReason = null; // clear — browser is healthy
       _lastBrowserRestartAt = Date.now();
       attachBrowserCleanup(browser, localVirtualDisplay);
       pluginEvents.emit('browser:launched', { browser, display: vdDisplay });
@@ -2370,6 +2379,23 @@ app.get('/health', (req, res) => {
   const rssMb = Math.round(mem.rss / 1048576);
   const heapUsedMb = Math.round(mem.heapUsed / 1048576);
   const nativeMemMb = rssMb - heapUsedMb;
+
+  // Browser not running: distinguish intentional idle stop from unexpected death
+  if (!running && _lastBrowserStopReason && !INTENTIONAL_STOP_REASONS.has(_lastBrowserStopReason)) {
+    // Unexpected browser absence — schedule recovery and report unhealthy
+    scheduleBrowserWarmRetry();
+    return res.status(503).json({
+      ok: false,
+      engine: 'camoufox',
+      browserRunning: false,
+      reason: _lastBrowserStopReason,
+      activeTabs: 0,
+      activeSessions: sessions.size,
+      memory: { rssMb, heapUsedMb, nativeMemMb },
+      ...(FLY_MACHINE_ID ? { machineId: FLY_MACHINE_ID } : {}),
+    });
+  }
+
   res.json({ 
     ok: true, 
     engine: 'camoufox',

tests/unit/healthSemantics.test.js

@@ -0,0 +1,70 @@
+'use strict';
+
+/**
+ * Tests for /health endpoint semantics:
+ * - 200 when browser running
+ * - 200 when browser intentionally idle-stopped (idle_shutdown, admin_stop)
+ * - 503 when browser unexpectedly missing (browser_disconnected, memory_pressure, etc.)
+ */
+
+const INTENTIONAL_STOP_REASONS = new Set(['idle_shutdown', 'admin_stop']);
+
+function computeHealthResponse({ browserConnected, lastStopReason, isRecovering }) {
+  if (isRecovering) {
+    return { status: 503, body: { ok: false, recovering: true } };
+  }
+  if (!browserConnected && lastStopReason && !INTENTIONAL_STOP_REASONS.has(lastStopReason)) {
+    return { status: 503, body: { ok: false, browserRunning: false, reason: lastStopReason } };
+  }
+  return { status: 200, body: { ok: true, browserRunning: browserConnected } };
+}
+
+describe('health endpoint semantics', () => {
+  test('returns 200 when browser is connected', () => {
+    const r = computeHealthResponse({ browserConnected: true, lastStopReason: null, isRecovering: false });
+    expect(r.status).toBe(200);
+    expect(r.body.ok).toBe(true);
+  });
+
+  test('returns 200 when browser idle-stopped intentionally', () => {
+    const r = computeHealthResponse({ browserConnected: false, lastStopReason: 'idle_shutdown', isRecovering: false });
+    expect(r.status).toBe(200);
+  });
+
+  test('returns 200 when browser admin-stopped', () => {
+    const r = computeHealthResponse({ browserConnected: false, lastStopReason: 'admin_stop', isRecovering: false });
+    expect(r.status).toBe(200);
+  });
+
+  test('returns 503 when browser disconnected unexpectedly', () => {
+    const r = computeHealthResponse({ browserConnected: false, lastStopReason: 'browser_disconnected', isRecovering: false });
+    expect(r.status).toBe(503);
+    expect(r.body.reason).toBe('browser_disconnected');
+  });
+
+  test('returns 503 for memory_pressure', () => {
+    const r = computeHealthResponse({ browserConnected: false, lastStopReason: 'memory_pressure', isRecovering: false });
+    expect(r.status).toBe(503);
+  });
+
+  test('returns 503 for browser_rss_pressure', () => {
+    const r = computeHealthResponse({ browserConnected: false, lastStopReason: 'browser_rss_pressure', isRecovering: false });
+    expect(r.status).toBe(503);
+  });
+
+  test('returns 503 when recovering', () => {
+    const r = computeHealthResponse({ browserConnected: false, lastStopReason: 'idle_shutdown', isRecovering: true });
+    expect(r.status).toBe(503);
+    expect(r.body.recovering).toBe(true);
+  });
+
+  test('returns 200 when no stop reason (fresh start, browser not yet launched)', () => {
+    const r = computeHealthResponse({ browserConnected: false, lastStopReason: null, isRecovering: false });
+    expect(r.status).toBe(200);
+  });
+
+  test('returns 503 for browser_restart reasons', () => {
+    const r = computeHealthResponse({ browserConnected: false, lastStopReason: 'browser_restart:nav_failures', isRecovering: false });
+    expect(r.status).toBe(503);
+  });
+});