From aa16bc2b0e4de005fa1192f4816d95ea9b3e0cce Mon Sep 17 00:00:00 2001 From: Pradeep Elankumaran Date: Fri, 18 Sep 2026 08:43:09 -0700 Subject: [PATCH] fix(openclaw): use direct Camoufox downloader --- AGENTS.md | 10 +- lib/camoufox-download.js | 20 +++ package.json | 39 +++--- postinstall.js | 96 ++++++++++++++ scripts/postinstall.js | 195 +--------------------------- scripts/postinstall.test.js | 56 +++----- tests/unit/openclawManifest.test.js | 12 ++ 7 files changed, 174 insertions(+), 254 deletions(-) create mode 100644 lib/camoufox-download.js create mode 100644 postinstall.js diff --git a/AGENTS.md b/AGENTS.md index 82f2693..1f99b24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -454,17 +454,17 @@ Use the plugin manager to install third-party plugins from git or local paths: ```bash # Install from git -npm run plugin install https://github.com/user/camofox-screenshot-plugin -npm run plugin install git:github.com/user/my-plugin +node scripts/plugin.js install https://github.com/user/camofox-screenshot-plugin +node scripts/plugin.js install git:github.com/user/my-plugin # Install from local directory -npm run plugin install ./path/to/my-plugin +node scripts/plugin.js install ./path/to/my-plugin # List installed plugins -npm run plugin list +node scripts/plugin.js list # Remove a plugin -npm run plugin remove my-plugin +node scripts/plugin.js remove my-plugin ``` The installer copies the plugin into `plugins/`, adds it to `camofox.config.json`, and runs `npm install` for any npm dependencies. System deps (`apt.txt`, `post-install.sh`) are flagged but must be installed manually or via Docker rebuild. diff --git a/lib/camoufox-download.js b/lib/camoufox-download.js new file mode 100644 index 0000000..b07027a --- /dev/null +++ b/lib/camoufox-download.js @@ -0,0 +1,20 @@ +import { DefaultAddons, maybeDownloadAddons } from 'camoufox-js/dist/addons.js'; +import { ALLOW_GEOIP, downloadMMDB } from 'camoufox-js/dist/locale.js'; +import { CamoufoxFetcher } from 'camoufox-js/dist/pkgman.js'; + +// camoufox-js does not currently expose its fetch command from the public +// entrypoint. Keep its pinned version in package.json while using the same +// downloader implementation as `npx camoufox-js fetch`, without spawning a +// shell or a second Node process during this package's lifecycle hook. +export async function downloadBundledCamoufox({ + createFetcher = () => new CamoufoxFetcher(), + shouldDownloadGeoIp = ALLOW_GEOIP, + downloadGeoIp = downloadMMDB, + downloadAddons = maybeDownloadAddons, +} = {}) { + const fetcher = createFetcher(); + await fetcher.install(); + + if (shouldDownloadGeoIp) downloadGeoIp(); + await downloadAddons(DefaultAddons); +} diff --git a/package.json b/package.json index ec010fa..0360fa0 100644 --- a/package.json +++ b/package.json @@ -36,29 +36,29 @@ }, "files": [ "server.js", - "lib/", - "plugins/", - "camofox.config.json", - "plugin.ts", "plugin.js", - "dist/plugin.js", - "tsconfig.json", "openclaw.plugin.json", - "mcp/", - "bin/", - "scripts/", - "run.sh", - "Dockerfile", - "README.md", - "LICENSE", - "AGENTS.md", - "!**/*.test.js", - "!**/*.test.ts", - "!**/*.spec.js" + "camofox.config.json", + "bin/camofox-browser.js", + "lib/", + "mcp/server.mjs", + "mcp/lib/config.mjs", + "mcp/lib/cookies.mjs", + "mcp/lib/tool-contracts.mjs", + "plugins/persistence/index.js", + "plugins/vnc/index.js", + "plugins/vnc/plugin.json", + "plugins/vnc/spawn.js", + "plugins/vnc/vnc-launcher.js", + "plugins/vnc/vnc-watcher.sh", + "plugins/vnc/vnc-watcher-lib.sh", + "plugins/youtube/index.js", + "plugins/youtube/youtube.js", + "postinstall.js" ], "openclaw": { "extensions": [ - "plugin.ts" + "plugin.js" ], "runtimeExtensions": [ "plugin.js" @@ -128,11 +128,10 @@ "test:plugins": "NODE_OPTIONS='--experimental-vm-modules' jest --forceExit plugins/", "test:live": "RUN_LIVE_TESTS=1 NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit tests/live", "test:debug": "DEBUG_SERVER=1 NODE_OPTIONS='--experimental-vm-modules' jest --runInBand --forceExit", - "plugin": "node scripts/plugin.js", "generate-openapi": "node scripts/generate-openapi.js", "version:sync": "node scripts/sync-version.js", "version": "node scripts/sync-version.js && node scripts/generate-openapi.js && git add openclaw.plugin.json mcp/package.json mcp/package-lock.json openapi.json", - "postinstall": "node scripts/postinstall.js", + "postinstall": "node postinstall.js", "fetch-bin": "npx camoufox-js fetch" }, "dependencies": { diff --git a/postinstall.js b/postinstall.js new file mode 100644 index 0000000..65a36ad --- /dev/null +++ b/postinstall.js @@ -0,0 +1,96 @@ +#!/usr/bin/env node +// Download Camoufox at installation time without spawning a shell or `npx`. +// Failures are warnings so restricted plugin-install environments can still +// install the JavaScript package and fetch the browser later. + +import { accessSync, constants, existsSync, statSync } from 'node:fs'; +import { homedir, platform } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { downloadBundledCamoufox } from './lib/camoufox-download.js'; + +const EXTERNAL_EXECUTABLE_ENV_VARS = [ + 'CAMOUFOX_EXECUTABLE', + 'CAMOUFOX_EXECUTABLE_PATH', + 'CAMOFOX_EXECUTABLE_PATH', +]; + +function camoufoxCacheDir() { + const home = homedir(); + const plat = platform(); + if (plat === 'darwin') return join(home, 'Library', 'Caches', 'camoufox'); + if (plat === 'win32') { + const base = process.env.LOCALAPPDATA || join(home, 'AppData', 'Local'); + return join(base, 'camoufox', 'camoufox', 'Cache'); + } + return join(process.env.XDG_CACHE_HOME || join(home, '.cache'), 'camoufox'); +} + +function warn(message) { + process.stderr.write(`[camofox-browser] postinstall warning: ${message}\n`); +} + +function fail(message) { + warn(message); + warn('The Camoufox browser binary may not have been downloaded.'); + warn('Run `npx camoufox-js fetch` manually before starting the server.'); + process.exit(0); +} + +export function externalExecutableFromEnv(env = process.env) { + for (const name of EXTERNAL_EXECUTABLE_ENV_VARS) { + const value = (env[name] || '').trim(); + if (value) return { name, value }; + } + return null; +} + +function assertExternalExecutable(path) { + if (!existsSync(path)) fail(`external Camoufox executable does not exist: ${path}`); + const stat = statSync(path); + if (!stat.isFile()) fail(`external Camoufox executable is not a file: ${path}`); + if (platform() !== 'win32') { + try { + accessSync(path, constants.X_OK); + } catch { + fail(`external Camoufox executable is not executable: ${path}`); + } + } +} + +export async function main() { + if (process.env.CAMOFOX_SKIP_DOWNLOAD === '1' || process.env.CAMOFOX_SKIP_DOWNLOAD === 'true') { + process.stderr.write('[camofox-browser] postinstall: skipping binary download (CAMOFOX_SKIP_DOWNLOAD=1)\n'); + return; + } + + const externalExecutable = externalExecutableFromEnv(); + if (externalExecutable) { + assertExternalExecutable(externalExecutable.value); + process.stdout.write( + `[camofox-browser] postinstall: ${externalExecutable.name} is set; skipping bundled Camoufox download.\n` + ); + return; + } + + const versionFile = join(camoufoxCacheDir(), 'version.json'); + if (existsSync(versionFile)) { + process.stdout.write('[camofox-browser] postinstall: Camoufox binary already cached.\n'); + return; + } + + try { + await downloadBundledCamoufox(); + } catch (error) { + fail(`failed to download Camoufox: ${error instanceof Error ? error.message : String(error)}`); + } + if (!existsSync(versionFile)) { + warn('Camoufox cache not populated after fetch.'); + warn(` Expected file: ${versionFile}`); + warn(' Manual fix: npx camoufox-js fetch'); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch(() => process.exit(0)); +} diff --git a/scripts/postinstall.js b/scripts/postinstall.js index 9c67417..e47c831 100644 --- a/scripts/postinstall.js +++ b/scripts/postinstall.js @@ -1,197 +1,10 @@ -#!/usr/bin/env node -// Postinstall: download Camoufox binaries and verify the cache is populated. -// -// Why a script instead of an inline `npx camoufox-js fetch`: -// 1. Cross-platform: avoids POSIX-only `VAR= cmd` shell syntax (Windows -// cmd.exe does not honor it). -// 2. Defends against PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 inherited from -// the user's shell or a CI/Docker base image. `camoufox-js` honors -// that flag by convention (same env name as `playwright`'s skip flag), -// which leaves the binary cache empty and makes the server crash at -// runtime with "Version information not found". -// 3. Verifies the cache after fetch and prints a warning with actionable -// remediation if the binary is still missing — the server will fail -// at startup, but install itself succeeds so plugin installs don't break. -// -// Exit behavior: -// Always exits 0. Download failures produce warnings, not hard errors. -// This ensures `npm install` succeeds in environments where the binary -// download is blocked (CI, firewalls, plugin installs that only need the -// JS tooling). The server prints a clear error at startup if the binary -// is missing. +// Compatibility entrypoint for repository tooling. Published installs run +// ../postinstall.js directly so only the lifecycle hook is shipped. +export { externalExecutableFromEnv, main } from '../postinstall.js'; -import { accessSync, constants, existsSync, statSync } from 'node:fs'; -import { homedir, platform } from 'node:os'; -import { join } from 'node:path'; +import { main } from '../postinstall.js'; import { pathToFileURL } from 'node:url'; -const EXTERNAL_EXECUTABLE_ENV_VARS = [ - 'CAMOUFOX_EXECUTABLE', - 'CAMOUFOX_EXECUTABLE_PATH', - 'CAMOFOX_EXECUTABLE_PATH', -]; - -const FETCH_CHILD_ENV_VARS = [ - 'PATH', - 'Path', - 'HOME', - 'USERPROFILE', - 'APPDATA', - 'LOCALAPPDATA', - 'XDG_CACHE_HOME', - 'TMPDIR', - 'TEMP', - 'TMP', - 'SystemRoot', - 'WINDIR', - 'ComSpec', - 'PATHEXT', - 'CI', - 'NODE_EXTRA_CA_CERTS', - 'SSL_CERT_FILE', - 'SSL_CERT_DIR', - 'HTTP_PROXY', - 'HTTPS_PROXY', - 'NO_PROXY', - 'ALL_PROXY', - 'http_proxy', - 'https_proxy', - 'no_proxy', - 'all_proxy', - 'npm_config_proxy', - 'npm_config_https_proxy', - 'npm_config_http_proxy', - 'npm_config_noproxy', - 'npm_config_no_proxy', - 'npm_config_registry', - 'npm_config_cache', - 'npm_config_fetch_retries', - 'npm_config_fetch_retry_factor', - 'npm_config_fetch_retry_mintimeout', - 'npm_config_fetch_retry_maxtimeout', - 'npm_config_fetch_timeout', - 'NPM_CONFIG_PROXY', - 'NPM_CONFIG_HTTPS_PROXY', - 'NPM_CONFIG_HTTP_PROXY', - 'NPM_CONFIG_NOPROXY', - 'NPM_CONFIG_NO_PROXY', - 'NPM_CONFIG_REGISTRY', - 'NPM_CONFIG_CACHE', - 'NPM_CONFIG_FETCH_RETRIES', - 'NPM_CONFIG_FETCH_RETRY_FACTOR', - 'NPM_CONFIG_FETCH_RETRY_MINTIMEOUT', - 'NPM_CONFIG_FETCH_RETRY_MAXTIMEOUT', - 'NPM_CONFIG_FETCH_TIMEOUT', - 'CAMOFOX_SKIP_DOWNLOAD', - 'CAMOUFOX_EXECUTABLE', - 'CAMOUFOX_EXECUTABLE_PATH', - 'CAMOFOX_EXECUTABLE_PATH', - 'CAMOUFOX_CACHE_DIR', - 'GITHUB_TOKEN', -]; - -function camoufoxCacheDir() { - const home = homedir(); - const plat = platform(); - if (plat === 'darwin') return join(home, 'Library', 'Caches', 'camoufox'); - if (plat === 'win32') { - // Matches camoufox-js/dist/pkgman.js:246 which nests the app name twice: - // %LOCALAPPDATA%\camoufox\camoufox\Cache - const base = process.env.LOCALAPPDATA || join(home, 'AppData', 'Local'); - return join(base, 'camoufox', 'camoufox', 'Cache'); - } - return join(process.env.XDG_CACHE_HOME || join(home, '.cache'), 'camoufox'); -} - -function warn(message) { - process.stderr.write(`[camofox-browser] postinstall warning: ${message}\n`); -} - -function fail(message) { - warn(message); - warn('The Camoufox browser binary may not have been downloaded.'); - warn('Run `npx camoufox-js fetch` manually before starting the server.'); - process.exit(0); -} - -export function externalExecutableFromEnv(env = process.env) { - for (const name of EXTERNAL_EXECUTABLE_ENV_VARS) { - const value = (env[name] || '').trim(); - if (value) return { name, value }; - } - return null; -} - -export function childEnvForFetch(env = process.env) { - const childEnv = {}; - for (const name of FETCH_CHILD_ENV_VARS) { - if (env[name] !== undefined) childEnv[name] = env[name]; - } - return childEnv; -} - -function assertExternalExecutable(path) { - if (!existsSync(path)) fail(`external Camoufox executable does not exist: ${path}`); - const stat = statSync(path); - if (!stat.isFile()) fail(`external Camoufox executable is not a file: ${path}`); - if (platform() !== 'win32') { - try { - accessSync(path, constants.X_OK); - } catch { - fail(`external Camoufox executable is not executable: ${path}`); - } - } -} - -export async function main() { - // Skip binary download entirely when CAMOFOX_SKIP_DOWNLOAD is set. - if (process.env.CAMOFOX_SKIP_DOWNLOAD === '1' || process.env.CAMOFOX_SKIP_DOWNLOAD === 'true') { - process.stderr.write('[camofox-browser] postinstall: skipping binary download (CAMOFOX_SKIP_DOWNLOAD=1)\n'); - return; - } - - const externalExecutable = externalExecutableFromEnv(); - if (externalExecutable) { - assertExternalExecutable(externalExecutable.value); - process.stdout.write( - `[camofox-browser] postinstall: ${externalExecutable.name} is set; skipping bundled Camoufox download.\n` - ); - return; - } - - // Check if binary is already cached — skip download entirely if so. - const versionFile = join(camoufoxCacheDir(), 'version.json'); - if (existsSync(versionFile)) { - process.stdout.write('[camofox-browser] postinstall: Camoufox binary already cached.\n'); - return; - } - - // Dynamic import with renamed binding to avoid triggering static code scanners - // (e.g. OpenClaw plugin security) that pattern-match on child_process function - // names like spawn/spawnSync/exec/execSync in the same file as "child_process". - const { spawnSync: run } = await import('node:child_process'); - - const isWindows = platform() === 'win32'; - const result = run(isWindows ? 'npx.cmd' : 'npx', ['camoufox-js', 'fetch'], { - stdio: 'inherit', - env: childEnvForFetch(), - shell: isWindows, - }); - - if (result.error) fail(`failed to spawn npx: ${result.error.message}`); - if (result.status !== 0) fail(`\`npx camoufox-js fetch\` exited with code ${result.status}`); - - if (!existsSync(versionFile)) { - warn('Camoufox cache not populated after fetch.'); - warn(` Expected file: ${versionFile}`); - warn(' Possible causes:'); - warn(' - Network failure during binary download (check your connection)'); - warn(' - GitHub API rate limit — set GITHUB_TOKEN in your env and retry'); - warn(' - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD re-exported by a wrapping process'); - warn(' Manual fix: npx camoufox-js fetch'); - } -} - if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { main().catch(() => process.exit(0)); } diff --git a/scripts/postinstall.test.js b/scripts/postinstall.test.js index e742fc2..d4285db 100644 --- a/scripts/postinstall.test.js +++ b/scripts/postinstall.test.js @@ -1,11 +1,12 @@ -import { afterEach, describe, expect, test } from '@jest/globals'; +import { afterEach, describe, expect, jest, test } from '@jest/globals'; import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; import { spawnSync } from 'child_process'; import { dirname } from 'path'; import { fileURLToPath } from 'url'; -import { childEnvForFetch, externalExecutableFromEnv } from './postinstall.js'; +import { downloadBundledCamoufox } from '../lib/camoufox-download.js'; +import { externalExecutableFromEnv } from '../postinstall.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const tempDirs = []; @@ -58,7 +59,7 @@ describe('postinstall external executable handling', () => { test('skips bundled download when an external executable is configured', () => { const executable = makeExecutable(); - const result = spawnSync(process.execPath, ['scripts/postinstall.js'], { + const result = spawnSync(process.execPath, ['postinstall.js'], { cwd: join(__dirname, '..'), encoding: 'utf8', env: postinstallTestEnv({ @@ -73,42 +74,21 @@ describe('postinstall external executable handling', () => { }); }); -describe('postinstall fetch child env', () => { - test('passes an explicit whitelist to the downloader child', () => { - const childEnv = childEnvForFetch({ - PATH: '/bin', - HOME: '/home/agent', - XDG_CACHE_HOME: '/home/agent/.cache', - LOCALAPPDATA: 'C:\\Users\\agent\\AppData\\Local', - HTTPS_PROXY: 'http://proxy.example:8080', - npm_config_https_proxy: 'http://npm-proxy.example:8080', - NPM_CONFIG_REGISTRY: 'https://registry.npmjs.org/', - CAMOFOX_SKIP_DOWNLOAD: '0', - CAMOUFOX_EXECUTABLE: '/opt/camoufox/camoufox-bin', - CAMOUFOX_EXECUTABLE_PATH: '/compat/camoufox-bin', - CAMOFOX_EXECUTABLE_PATH: '/legacy/camoufox-bin', - CAMOUFOX_CACHE_DIR: '/opt/camoufox-cache', - GITHUB_TOKEN: 'github-token', - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1', - AWS_ACCESS_KEY_ID: 'should-not-leak', - SOME_RANDOM_SECRET: 'should-not-leak', - npm_config__authToken: 'should-not-leak', +describe('postinstall downloader', () => { + test('uses camoufox-js without a package-owned child process', async () => { + const install = jest.fn().mockResolvedValue(undefined); + const downloadGeoIp = jest.fn(); + const downloadAddons = jest.fn().mockResolvedValue(undefined); + + await downloadBundledCamoufox({ + createFetcher: () => ({ install }), + shouldDownloadGeoIp: true, + downloadGeoIp, + downloadAddons, }); - expect(childEnv).toEqual({ - PATH: '/bin', - HOME: '/home/agent', - LOCALAPPDATA: 'C:\\Users\\agent\\AppData\\Local', - XDG_CACHE_HOME: '/home/agent/.cache', - HTTPS_PROXY: 'http://proxy.example:8080', - npm_config_https_proxy: 'http://npm-proxy.example:8080', - NPM_CONFIG_REGISTRY: 'https://registry.npmjs.org/', - CAMOFOX_SKIP_DOWNLOAD: '0', - CAMOUFOX_EXECUTABLE: '/opt/camoufox/camoufox-bin', - CAMOUFOX_EXECUTABLE_PATH: '/compat/camoufox-bin', - CAMOFOX_EXECUTABLE_PATH: '/legacy/camoufox-bin', - CAMOUFOX_CACHE_DIR: '/opt/camoufox-cache', - GITHUB_TOKEN: 'github-token', - }); + expect(install).toHaveBeenCalledTimes(1); + expect(downloadGeoIp).toHaveBeenCalledTimes(1); + expect(downloadAddons).toHaveBeenCalledTimes(1); }); }); diff --git a/tests/unit/openclawManifest.test.js b/tests/unit/openclawManifest.test.js index 991d33a..a277bba 100644 --- a/tests/unit/openclawManifest.test.js +++ b/tests/unit/openclawManifest.test.js @@ -14,9 +14,21 @@ describe('OpenClaw manifest', () => { test('pins the tested OpenClaw plugin API version in package metadata', () => { const pkg = readJson('package.json'); + expect(pkg.openclaw.extensions).toEqual(['plugin.js']); + expect(pkg.openclaw.runtimeExtensions).toEqual(['plugin.js']); expect(pkg.openclaw.compat.pluginApi).toBe('>=2026.9.4'); expect(pkg.openclaw.build.openclawVersion).toBe('2026.9.4'); }); + test('ships the compiled plugin entrypoint and no development script directory', () => { + const pkg = readJson('package.json'); + + expect(pkg.openclaw.extensions).toEqual(['plugin.js']); + expect(pkg.openclaw.runtimeExtensions).toEqual(['plugin.js']); + expect(pkg.files).toContain('postinstall.js'); + expect(pkg.files).not.toContain('scripts/'); + expect(pkg.files).not.toContain('plugin.ts'); + }); + test('declares ownership contracts for every canonical tool', () => { const manifest = readJson('openclaw.plugin.json'); const pkg = readJson('package.json');