import { test, type TestContext } from 'node:test'; import assert from 'node:assert/strict'; import { chmod, mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises'; import { once } from 'node:events'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import { basename, join } from 'node:path'; import { execFileSync } from 'node:child_process'; import * as core from '@actions/core'; import { DefaultArtifactClient } from '@actions/artifact'; import { BlobClient } from '@azure/storage-blob'; import { runAction } from '../src/main.ts'; import { parse } from 'yaml'; import { gitEnvironment } from '../src/targets.ts'; import { runtimeEnvironment, type Runtime } from '../src/runtime.ts'; import type { ProcessResult } from '../src/process.ts'; import { copyCompletedScan } from './fixtures.ts'; type Scenario = 'schedule' | 'pr' | 'policy-pr' | 'empty-pr'; const inputNames = Object.keys(parse(await readFile(new URL('../../action.yml', import.meta.url), 'utf8')).inputs); const inputKey = (name: string): string => `INPUT_${name.toUpperCase()}`; function assertAttribution(args: readonly string[], provider: string): void { assert.deepEqual(args.flatMap((arg, index) => arg === '--codex' ? [args[index + 1]] : []), [ 'analytics.enabled=false', 'responses_api_metadata.codex_security_integration="github_action"', `responses_api_metadata.codex_security_provider="${provider}"`, ]); } function outputValues(text: string): Record { const values: Record = {}; const lines = text.split(/\r?\n/u); for (let index = 0; index < lines.length; index++) { const line = lines[index]; const marker = line.indexOf('<<'); if (marker < 0) continue; const name = line.slice(0, marker); const delimiter = line.slice(marker + 2); const value: string[] = []; while (++index < lines.length && lines[index] !== delimiter) value.push(lines[index]); values[name] = value.join('\n'); } return values; } async function harness(t: TestContext, scenario: Scenario = 'schedule', serverUrl = 'https://github.com') { const root = await realpath(await mkdtemp(join(tmpdir(), 'codex-main-test-'))); const repository = join(root, 'repo'); await mkdir(repository); const git = (...args: string[]): string => execFileSync('/usr/bin/git', [ '-c', 'user.name=Offline Test', '-c', 'user.email=offline@example.invalid', '-c', 'core.hooksPath=/dev/null', ...args, ], { cwd: repository, env: gitEnvironment(), encoding: 'utf8' }).trim(); git('init', '-q', '-b', 'main'); git('remote', 'add', 'origin', `${serverUrl}/example/repo.git`); await writeFile(join(repository, 'app.txt'), 'synthetic source\n'); git('add', '.'); git('commit', '-qm', 'base'); const base = git('rev-parse', 'HEAD'); let sha = base; if (scenario !== 'schedule' && scenario !== 'empty-pr') { await writeFile(join(repository, scenario === 'policy-pr' ? 'SECURITY.md' : 'app.txt'), 'synthetic change\n'); git('add', '.'); git('commit', '-qm', 'PR change'); sha = git('rev-parse', 'HEAD'); } const previousEnv = { ...process.env }; const previousExitCode = process.exitCode; t.after(async () => { for (const key of Object.keys(process.env)) if (!(key in previousEnv)) delete process.env[key]; Object.assign(process.env, previousEnv); process.exitCode = previousExitCode; await rm(root, { recursive: true, force: true }); }); for (const name of inputNames) delete process.env[inputKey(name)]; const eventPath = join(root, 'event.json'); const outputPath = join(root, 'outputs'); const statePath = join(root, 'state'); const payload = scenario === 'schedule' ? {} : { number: 4, pull_request: { number: 4, head: { sha, repo: { full_name: 'example/repo' } }, base: { sha: base, repo: { full_name: 'example/repo' } } } }; await writeFile(eventPath, JSON.stringify(payload)); await writeFile(outputPath, ''); await writeFile(statePath, ''); Object.assign(process.env, { GITHUB_EVENT_PATH: eventPath, GITHUB_EVENT_NAME: scenario === 'schedule' ? 'schedule' : 'pull_request', GITHUB_REPOSITORY: 'example/repo', GITHUB_WORKSPACE: repository, GITHUB_SHA: sha, GITHUB_REF: scenario === 'schedule' ? 'refs/heads/main' : 'refs/pull/4/merge', GITHUB_ACTOR: 'trusted-maintainer', GITHUB_SERVER_URL: serverUrl, GITHUB_RUN_ID: '123456', GITHUB_OUTPUT: outputPath, GITHUB_STATE: statePath, RUNNER_TEMP: root, OPENAI_API_KEY: 'synthetic-offline-test-key', INPUT_SUMMARY: 'true', INPUT_ANNOTATIONS: 'false', INPUT_SCOPE: scenario === 'schedule' ? 'repository' : 'diff', }); delete process.env.CODEX_API_KEY; const runtime: Runtime = { root: join(root, 'runtime'), home: join(root, 'runtime/home'), codexHome: join(root, 'runtime/codex'), stateDirectory: join(root, 'runtime/state'), resultsDirectory: join(root, 'results'), nodePath: '/never-executed/node', cliPath: '/never-executed/cli.js', env: (credentials) => runtimeEnvironment(runtime, credentials), }; let setups = 0; let processes = 0; let cleanups = 0; const configuration: { exitCode?: number; missingSarif?: boolean; mutateCheckout?: boolean; cleanupFails?: boolean; cliFailure?: boolean; scanProcess?: Partial; scanResult?: (value: any) => void; missingReport?: string; prepare?: () => void | Promise } = {}; let scanOutput = ''; let summary = ''; t.mock.method(core.summary, 'write', async () => { summary = core.summary.stringify(); core.summary.emptyBuffer(); return core.summary; }); t.after(() => { core.summary.emptyBuffer(); }); let capturedArgs: readonly string[] = []; let capturedEnvironment: NodeJS.ProcessEnv = {}; async function reports(): Promise { await copyCompletedScan(runtime.resultsDirectory); const manifestPath = join(runtime.resultsDirectory, 'scan-manifest.json'); const coveragePath = join(runtime.resultsDirectory, 'coverage.json'); const sarifPath = join(runtime.resultsDirectory, 'exports/results.sarif'); const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); const coverage = JSON.parse(await readFile(coveragePath, 'utf8')); const sarif = JSON.parse(await readFile(sarifPath, 'utf8')); const scanSha = git('rev-parse', 'HEAD'); manifest.scan.target.revision = scanSha; if (scenario !== 'schedule') { Object.assign(manifest.scan.target, { kind: 'git_diff', baseRevision: base, headRevision: scanSha }); coverage.mode = 'branch_diff'; sarif.runs[0].properties.codexSecurityTargetKind = 'git_diff'; } if (process.env.INPUT_MODE === 'deep') coverage.mode = 'deep_repository'; sarif.runs[0].versionControlProvenance[0].revisionId = scanSha; sarif.runs[0].versionControlProvenance[0].repositoryUri = `${serverUrl}/example/repo`; await writeFile(sarifPath, JSON.stringify(sarif)); if (configuration.missingSarif) await rm(sarifPath); const value = {manifest, coverage, findings: JSON.parse(await readFile(join(runtime.resultsDirectory, 'findings.json'), 'utf8')), scanDir: runtime.resultsDirectory, sarifPath: configuration.missingSarif ? null : sarifPath, cost: {estimatedUsd: 0.125}}; configuration.scanResult?.(value); await writeFile(coveragePath, JSON.stringify(coverage)); await writeFile(manifestPath, JSON.stringify(manifest)); scanOutput = JSON.stringify(value); if (configuration.missingReport) await rm(join(runtime.resultsDirectory, configuration.missingReport)); } return { repository, sha, git, setInput: (name: string, value: string) => { process.env[inputKey(name)] = value; }, configure: (options: typeof configuration) => Object.assign(configuration, options), run: async () => { const logs: string[] = []; const originalWrite = process.stdout.write; process.stdout.write = ((chunk: string | Uint8Array, encodingOrCallback?: unknown, callback?: unknown): boolean => { logs.push(String(chunk)); if (typeof encodingOrCallback === 'function') encodingOrCallback(); else if (typeof callback === 'function') callback(); return true; }) as typeof process.stdout.write; let exitCode: typeof process.exitCode; try { process.exitCode = 0; await runAction(root, { setupRuntime: async (options) => { setups++; options.log?.('Synthetic provisioning output.'); await configuration.prepare?.(); await reports(); return runtime; }, cleanupRuntime: async () => { cleanups++; if (configuration.cleanupFails) throw new Error('Synthetic cleanup failure'); }, runProcess: async (_executable, args, options): Promise => { processes++; capturedArgs = args; capturedEnvironment = options.env; options.log?.('[codex-security] Synthetic live CLI progress.'); if (configuration.mutateCheckout) await writeFile(join(repository, 'app.txt'), 'modified while scanning\n'); return { exitCode: configuration.exitCode ?? (process.env['INPUT_FAIL-ON-SEVERITY'] === 'high' ? 1 : 0), signal: null, stdout: configuration.cliFailure ? JSON.stringify({status: 'failed', code: 'SCAN_FAILED', message: 'Synthetic API authentication failure.'}) : scanOutput, stderr: '', interrupted: false, timedOut: false, ...configuration.scanProcess }; }, }); exitCode = process.exitCode; } finally { process.stdout.write = originalWrite; process.exitCode = previousExitCode; } return { exitCode, setups, processes, cleanups, summary, args: capturedArgs, environment: capturedEnvironment, outputs: outputValues(await readFile(outputPath, 'utf8')), logs: logs.join('') }; }, }; } test('PR severity failure retains SARIF outputs despite unrelated scope notes', async (t) => { const app = await harness(t, 'pr'); app.setInput('fail-on-severity', 'high'); app.configure({ exitCode: 1, scanResult: value => { value.coverage.deferred = [{reason: 'Dependency validation remains unavailable.'}]; }}); app.setInput('model', 'gpt-5.6-luna'); app.setInput('effort', 'medium'); app.setInput('annotations', ''); // Use the default, as in the README's PR workflow. const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.setups, 1); assert.equal(result.cleanups, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.equal(result.outputs['analysis-ref'], 'refs/pull/4/head'); assert.equal(result.outputs['scanned-sha'], app.sha); assert.equal(result.outputs['high-count'], '1'); assert.equal(result.outputs['estimated-cost'], '0.125'); assert.ok(result.outputs['sarif-path']); assert.ok(result.args.includes('--model=gpt-5.6-luna')); assert.ok(result.args.includes('--effort=medium')); assert.ok(!result.args.includes('--max-cost')); assert.match(result.logs, /::error::Scan completed\. Findings meet the configured failure threshold\./); assert.match(result.summary, /^## Codex Security\n\n\*\*Scan completed\. Findings meet the configured failure threshold\.\*\*/); const annotation = /^::warning ([^\r\n]+)::([^\r\n]+)$/m.exec(result.logs); assert.ok(annotation, 'PR findings must emit a GitHub warning annotation even when the severity policy fails'); const properties = Object.fromEntries(annotation[1].split(',').map(property => property.split('='))); assert.equal(properties.file, 'src/extract.py'); assert.equal(properties.line, '41'); assert.equal(properties.endLine, '44'); assert.match(properties.title, /^HIGH%3A Unsafe archive extraction/); assert.match(annotation[2], /filesystem write without containment validation/); }); test('a Critical finding after 50 Low findings leads the PR summary and annotations', async (t) => { const app = await harness(t, 'pr'); app.setInput('fail-on-severity', 'high'); app.setInput('annotations', 'true'); app.configure({ scanResult: value => { const [finding] = value.findings.findings; const lowFindings = Array.from({ length: 50 }, (_, i) => ({ ...finding, title: `Low finding ${i + 1}`, severity: { level: 'low' }, })); value.findings.findings = [...lowFindings, { ...finding, title: 'Blocking finding', severity: { level: 'critical' } }]; }}); const result = await app.run(); assert.equal(result.outputs['policy-status'], 'failed'); const headings = [...result.summary.matchAll(/

([^<]+)<\/h3>/g)].map(match => match[1]); const annotations = result.logs.split(/\r?\n/).filter(line => line.startsWith('::warning ')); assert.equal(headings.length, 30); assert.equal(annotations.length, 50); assert.deepEqual(headings.slice(0, 2), ['CRITICAL: Blocking finding', 'LOW: Low finding 1']); assert.match(annotations[0], /title=CRITICAL%3A Blocking finding/); assert.match(annotations[1], /title=LOW%3A Low finding 1/); }); test('scheduled complete report-only scan succeeds', async (t) => { const app = await harness(t); process.env.INPUT_FOO = 'unrelated workflow value'; app.setInput('effort', 'medium'); app.configure({scanResult: value => { value.turn = {usage: {input_tokens: 1200, output_tokens: 50}}; }}); const result = await app.run(); assertAttribution(result.args, 'openai'); assert.ok(result.args.includes('--effort=medium')); assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.equal(result.outputs['report-status'], 'ready'); assert.equal(result.cleanups, 1); assert.equal(result.outputs['analysis-ref'], 'refs/heads/main'); assert.ok(result.args.includes('--verbose')); assert.match(result.logs, /Synthetic live CLI progress/); assert.match(result.logs, /CLI preparation completed in \d+m \d+s/); assert.match(result.logs, /Target commit: [a-f0-9]{40}/); assert.match(result.logs, /Security scan exited after \d+m \d+s; exit code: 0/); assert.match(result.logs, /Scan: completed; findings policy: passed; report: ready/); assert.match(result.logs, /Estimated cost: \$0.1250/); assert.equal(result.outputs['estimated-cost'], '0.125'); assert.ok(result.summary.includes('1,200 input (0 cached) · 50 output')); assert.match(result.logs, /Scan completed\. Findings are reported without failing the job\./); assert.match(result.summary, /\*\*Scan completed\. Findings are reported without failing the job\.\*\*/); }); for (const [scenario, mode] of [['schedule', 'standard'], ['pr', 'standard'], ['schedule', 'deep']] as const) { test(`enterprise ${scenario} ${mode} scans preserve reports and upload identity`, async t => { const serverUrl = 'https://example-enterprise.ghe.com'; const app = await harness(t, scenario, serverUrl); app.setInput('mode', mode); app.setInput('upload-artifacts', 'true'); app.setInput('annotations', 'true'); let uploadedFiles: string[] = []; t.mock.method(DefaultArtifactClient.prototype, 'uploadArtifact', async (_name: string, files: string[]) => { assert.equal(process.env.GITHUB_SERVER_URL, serverUrl); uploadedFiles = files.map(file => basename(file)); for (const file of files) assert.ok((await readFile(file)).length); return {id:1, size:1}; }); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.processes, 1); assert.equal(result.cleanups, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['report-status'], 'ready'); assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.equal(result.outputs['scanned-sha'], app.sha); assert.equal(result.outputs['analysis-ref'], scenario === 'pr' ? 'refs/pull/4/head' : 'refs/heads/main'); assert.deepEqual(uploadedFiles.sort(), ['coverage.json', 'findings.json', 'results.sarif', 'scan-manifest.json']); const sarif = JSON.parse(await readFile(result.outputs['sarif-path'], 'utf8')); assert.equal(sarif.runs[0].versionControlProvenance[0].repositoryUri, `${serverUrl}/example/repo`); assert.match(result.logs, /^::warning .*file=src\/extract\.py/m); assert.match(result.summary, /Scan completed/); }); } test('Deep scans forward their budget and publish completed reports with cleanup', async (t) => { const app = await harness(t); app.setInput('mode', 'deep'); app.setInput('max-time-hours', '1.5'); const result = await app.run(); assertAttribution(result.args, 'openai'); assert.equal(result.args[result.args.indexOf('--mode') + 1], 'deep'); assert.equal(result.args[result.args.indexOf('--max-time') + 1], '90'); assert.equal(result.exitCode, 0); assert.equal(result.cleanups, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['report-status'], 'ready'); assert.equal(result.outputs['high-count'], '1'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.match(result.logs, /mode: deep/); assert.match(result.summary, /\*\*Mode:\*\* deep/); }); test('findings below the threshold pass with an explicit outcome', async (t) => { const app = await harness(t); app.setInput('fail-on-severity', 'critical'); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.outputs['high-count'], '1'); assert.match(result.logs, /Scan completed\. No findings meet the failure threshold\./); assert.match(result.summary, /\*\*Scan completed\. No findings meet the failure threshold\.\*\*/); }); test('same-repository Dependabot PR scans with a supplied key', async (t) => { const app = await harness(t, 'pr'); process.env.GITHUB_ACTOR = 'dependabot[bot]'; const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.setups, 1); assert.equal(result.processes, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['scanned-sha'], app.sha); assert.equal(result.environment.OPENAI_API_KEY, 'synthetic-offline-test-key'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); }); for (const mode of ['standard', 'deep']) { test(`Bedrock ${mode} scans use captured AWS credentials without an OpenAI key`, async t => { const app = await harness(t); app.setInput('provider', 'amazon-bedrock'); app.setInput('model', 'openai.synthetic-model'); app.setInput('mode', mode); delete process.env.OPENAI_API_KEY; Object.assign(process.env, {AWS_ACCESS_KEY_ID:'synthetic-access-key', AWS_SECRET_ACCESS_KEY:'synthetic-secret-key', AWS_SESSION_TOKEN:'synthetic-session-token', AWS_REGION:'us-east-2', GITHUB_TOKEN:'must-not-reach-scan'}); app.configure({prepare: () => { process.env.AWS_SESSION_TOKEN = 'changed-during-setup'; }}); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.equal(result.args[result.args.indexOf('--provider') + 1], 'amazon-bedrock'); assertAttribution(result.args, 'amazon-bedrock'); assert.equal(result.args[result.args.indexOf('--auth') + 1], 'auto'); assert.equal(result.environment.AWS_ACCESS_KEY_ID, 'synthetic-access-key'); assert.equal(result.environment.AWS_SECRET_ACCESS_KEY, 'synthetic-secret-key'); assert.equal(result.environment.AWS_SESSION_TOKEN, 'synthetic-session-token'); assert.equal(result.environment.AWS_REGION, 'us-east-2'); assert.equal(result.environment.OPENAI_API_KEY, undefined); assert.equal(result.environment.GITHUB_TOKEN, undefined); }); } for (const [provider, key] of [['openrouter', 'OPENROUTER_API_KEY'], ['fireworks', 'FIREWORKS_API_KEY']]) { for (const mode of ['standard', 'deep']) test(`${provider} ${mode} scans use only the captured provider key`, async t => { const app = await harness(t); app.setInput('provider', provider); app.setInput('model', 'synthetic/model'); app.setInput('mode', mode); Object.assign(process.env, {OPENROUTER_API_KEY:'synthetic-openrouter', FIREWORKS_API_KEY:'synthetic-fireworks', AWS_SESSION_TOKEN:'unrelated-aws', GITHUB_TOKEN:'unrelated-github'}); const expectedKey = process.env[key]; app.configure({prepare: () => { process.env[key] = 'changed-during-setup'; }}); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.args[result.args.indexOf('--provider') + 1], provider); assertAttribution(result.args, provider); assert.equal(result.args[result.args.indexOf('--auth') + 1], 'api-key'); assert.equal(result.environment[key], expectedKey); for (const other of ['OPENAI_API_KEY', 'CODEX_API_KEY', 'OPENROUTER_API_KEY', 'FIREWORKS_API_KEY', 'AWS_SESSION_TOKEN', 'GITHUB_TOKEN']) if (other !== key) assert.equal(result.environment[other], undefined); }); test(`${provider} requires its own key and keeps dry-run credential-free`, async t => { const app = await harness(t); app.setInput('provider', provider); app.setInput('model', 'synthetic/model'); delete process.env[key]; const missing = await app.run(); assert.equal(missing.exitCode, 1); assert.equal(missing.setups, 0); assert.ok(missing.summary.includes(key)); process.env[key] = 'synthetic-provider-key'; app.setInput('dry-run', 'true'); const dry = await app.run(); assert.equal(dry.exitCode, 0); assert.equal(dry.outputs['skip-reason'], 'dry-run'); assert.equal(dry.environment[key], undefined); assert.equal(dry.environment.OPENAI_API_KEY, undefined); }); } test('Bedrock PR scans retain diff scope and severity failure outputs', async t => { const app = await harness(t, 'pr'); app.setInput('provider', 'amazon-bedrock'); app.setInput('model', 'openai.synthetic-model'); app.setInput('fail-on-severity', 'high'); process.env.AWS_BEARER_TOKEN_BEDROCK = 'synthetic-bedrock-bearer'; const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.ok(result.args.includes('--diff')); assert.ok(result.args.includes('--head')); assert.equal(result.environment.AWS_BEARER_TOKEN_BEDROCK, 'synthetic-bedrock-bearer'); assert.equal(result.environment.OPENAI_API_KEY, undefined); }); test('Bedrock dry-run forwards no credentials and remains a skipped scan', async t => { const app = await harness(t); app.setInput('provider', 'amazon-bedrock'); app.setInput('model', 'openai.synthetic-model'); app.setInput('dry-run', 'true'); process.env.AWS_BEARER_TOKEN_BEDROCK = 'synthetic-bedrock-bearer'; const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'skipped'); assert.equal(result.outputs['skip-reason'], 'dry-run'); assert.equal(result.environment.AWS_BEARER_TOKEN_BEDROCK, undefined); assert.equal(result.environment.OPENAI_API_KEY, undefined); }); for (const scenario of ['schedule', 'pr'] as const) { test(`missing API key fails a ${scenario === 'pr' ? 'Dependabot PR' : 'scheduled scan'} with setup guidance`, async (t) => { const app = await harness(t, scenario); delete process.env.OPENAI_API_KEY; process.env.AWS_BEARER_TOKEN_BEDROCK = 'synthetic-bedrock-bearer'; if (scenario === 'pr') process.env.GITHUB_ACTOR = 'dependabot[bot]'; const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.setups, 0); assert.equal(result.processes, 0); assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.match(result.logs, /::error::Scan could not complete\./); assert.match(result.summary, /\*\*Scan could not complete\.\*\*/); assert.match(result.summary, /Set CODEX_SECURITY_API_KEY in Actions secrets \(or Dependabot secrets for Dependabot runs\) and pass it as OPENAI_API_KEY/); assert.doesNotMatch(result.logs, /Findings meet the configured failure threshold/); }); } test('missing RUNNER_TEMP fails with setup guidance', async (t) => { const app = await harness(t); delete process.env.RUNNER_TEMP; const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.setups, 0); assert.equal(result.processes, 0); assert.match(result.summary, /RUNNER_TEMP is required\./); }); test('cleanup failure fails the job and appears in the summary and final error', async (t) => { const app = await harness(t); app.configure({cleanupFails: true}); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.cleanups, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['report-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.match(result.logs, /::error::Scan completed\..*Runtime cleanup failed\./); assert.match(result.summary, /Runtime cleanup failed\.\*\*/); }); for (const threshold of ['none', 'high']) { test(`summary write failure preserves the scan result with severity threshold ${threshold}`, async (t) => { const app = await harness(t); app.setInput('summary', 'true'); app.setInput('fail-on-severity', threshold); app.setInput('upload-artifacts', 'true'); const upload = t.mock.method(DefaultArtifactClient.prototype, 'uploadArtifact', async () => ({id: 2468, size: 1})); t.mock.method(core.summary, 'write', async () => { throw new Error('Synthetic summary write failure'); }); const result = await app.run(); assert.equal(result.exitCode, threshold === 'none' ? 0 : 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], threshold === 'none' ? 'passed' : 'failed'); assert.equal(result.outputs['report-status'], 'ready'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.ok(result.outputs['sarif-path']); assert.equal(result.cleanups, 1); assert.equal(upload.mock.callCount(), 1); assert.match(result.logs, /::warning::Could not write the job summary\./); }); } test('configuration diagnostics preserve values and cannot inject runner commands', async (t) => { const app = await harness(t); app.setInput('model', 'synthetic-offline-test-key\u2028::error::injected'); const result = await app.run(); const logs = result.logs; assert.match(logs, /synthetic-offline-test-key/); assert.doesNotMatch(logs, /^::error::injected/m); assert.doesNotMatch(logs, /\[REDACTED\]/); }); test('scope exclusions and deferred notes do not fail completed scans or make findings provisional', async (t) => { const app = await harness(t); app.configure({scanResult: value => { value.coverage.explicitExclusions = [{pattern: 'assets/', reason: 'Static assets outside review scope.'}]; value.coverage.deferred = [{id: 'unreviewed-route', reason: 'Dependency unavailable; validation deferred.'}]; }}); app.setInput('verbose', 'false'); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['exit-code'], '0'); assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.equal(result.outputs['high-count'], '1'); assert.ok(result.outputs['json-path']); assert.ok(!result.args.includes('--verbose')); assert.doesNotMatch(result.logs, /Synthetic live CLI progress|Synthetic provisioning output/); assert.match(result.logs, /Starting security scan/); assert.match(result.logs, /Scan: completed/); assert.match(result.logs, /Findings:.*high: 1/); assert.doesNotMatch(result.logs, /Provisional|Scan coverage is partial|::error::/); assert.doesNotMatch(result.summary, /provisional|not evaluated|not-evaluated/i); const coverage = JSON.parse(await readFile(result.outputs['coverage-path'], 'utf8')); assert.equal(coverage.deferred[0].reason, 'Dependency unavailable; validation deferred.'); }); test('CLI warnings fail a completed scan and cannot emit legacy runner commands', async (t) => { const app = await harness(t); app.configure({scanResult: value => { value.warnings = ['Synthetic diagnostic: ##[stop-commands]token']; }}); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.outputs['exit-code'], '0'); assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.match(result.logs, /Synthetic diagnostic: ##\\\[stop-commands\]token/); assert.doesNotMatch(result.logs, /##\[stop-commands\]/); assert.match(result.summary, /Synthetic diagnostic: ##[stop-commands]token/); }); test('exit 2 fails the step even when finalized reports were retained', async (t) => { const app = await harness(t); app.configure({exitCode: 2}); app.setInput('annotations', 'true'); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['exit-code'], '2'); assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.equal(result.outputs['high-count'], '1'); assert.ok(result.outputs['json-path']); assert.match(result.summary, /absence of findings is inconclusive/); assert.doesNotMatch(result.logs, /Provisional/); }); for (const [name, scanProcess] of [ ['timeout', {timedOut: true}], ['interruption', {interrupted: true}], ['signal', {signal: 'SIGTERM' as const}], ] as const) { test(`retained reports cannot hide scan ${name}`, async (t) => { const app = await harness(t); app.configure({scanProcess}); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); }); } for (const missingReport of ['scan-manifest.json', 'findings.json', 'coverage.json']) { test(`completed scan fails when required ${missingReport} is missing without artifact upload`, async (t) => { const app = await harness(t); app.configure({missingReport}); app.setInput('upload-artifacts', 'false'); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.outputs['report-status'], 'failed'); assert.equal(result.outputs['json-path'], ''); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.match(result.logs, /required reporting failed/); }); } test('wrong checkout fails before setup or scanner execution', async (t) => { const app = await harness(t); process.env.GITHUB_SHA = '0'.repeat(40); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.setups, 0); assert.equal(result.processes, 0); assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); }); for (const restoreCheckout of [false, true]) { test(`a scan of a commit checked out during preparation fails${restoreCheckout ? ' even after restoring the requested checkout' : ''}`, async (t) => { const app = await harness(t); app.git('commit', '--allow-empty', '-qm', 'Synthetic second revision'); const otherSha = app.git('rev-parse', 'HEAD'); app.git('checkout', '--detach', app.sha); app.configure({ prepare: () => { app.git('checkout', '--detach', otherSha); }, scanResult: () => { if (restoreCheckout) app.git('checkout', '--detach', app.sha); }, }); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['scanned-sha'], app.sha); assert.equal(result.outputs['json-path'], ''); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.match(result.summary, /revision does not match/); }); } test('a worktree changed during preparation cannot establish a scan of the requested commit', async (t) => { const app = await harness(t); app.configure({ prepare: () => writeFile(join(app.repository, 'app.txt'), 'Changed before the CLI snapshot.\n'), scanResult: value => { value.manifest.scan.target.kind = 'git_worktree'; }, }); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['json-path'], ''); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.match(result.summary, /revision does not match/); }); test('PR policy changes are scanned at the checked-out revision', async (t) => { const app = await harness(t, 'policy-pr'); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.setups, 1); assert.equal(result.processes, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['scanned-sha'], app.sha); assert.ok(result.args.includes('--diff')); assert.ok(result.args.includes(app.sha)); }); test('dry-run is explicitly skipped with no findings policy pass', async (t) => { const app = await harness(t); app.setInput('dry-run', 'true'); delete process.env.OPENAI_API_KEY; const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'skipped'); assert.equal(result.outputs['skip-reason'], 'dry-run'); assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.equal(result.outputs['high-count'], ''); assert.ok(result.args.includes('--dry-run')); assert.equal(result.environment.OPENAI_API_KEY, undefined); }); for (const threshold of ['low', 'medium', 'high', 'critical']) { test(`dry-run rejects ${threshold} enforcement before runtime setup`, async (t) => { const app = await harness(t); app.setInput('dry-run', 'true'); app.setInput('fail-on-severity', threshold); app.setInput('summary', 'false'); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.setups, 0); assert.equal(result.processes, 0); assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['scan-status'], 'failed'); assert.match(result.logs, /dry-run requires fail-on-severity: none/); assert.equal(result.summary, ''); }); } for (const threshold of ['none', 'high']) { test(`verified empty PR diff succeeds without a scan for ${threshold} enforcement`, async (t) => { const app = await harness(t, 'empty-pr'); app.setInput('fail-on-severity', threshold); delete process.env.OPENAI_API_KEY; const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.setups, 0); assert.equal(result.processes, 0); assert.equal(result.outputs['scan-status'], 'skipped'); assert.equal(result.outputs['skip-reason'], 'empty-diff'); assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['high-count'], ''); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.match(result.summary, /No committed changes to scan/); }); } test('successful artifact uploads link named reports and retention in the job summary', async (t) => { const app = await harness(t); app.setInput('upload-artifacts', 'true'); app.setInput('artifact-name', 'review-reports'); app.setInput('retention-days', '14'); const upload = t.mock.method(DefaultArtifactClient.prototype, 'uploadArtifact', async (...[name, files, _root, options]: Parameters) => { assert.equal(name, 'review-reports'); assert.equal(options?.retentionDays, 14); assert.ok(files.some(file => basename(file) === 'findings.json')); return {id: 2468, size: 1}; }); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(upload.mock.callCount(), 1); assert.match(result.summary, /review-reports<\/a> · Retention: up to 14 days/); }); test('disabled artifact upload neither uploads reports nor adds a download link', async (t) => { const app = await harness(t); const upload = t.mock.method(DefaultArtifactClient.prototype, 'uploadArtifact', async () => ({id: 2468, size: 1})); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(upload.mock.callCount(), 0); assert.doesNotMatch(result.summary, /\/artifacts\//); }); test('disabled summary leaves requested artifact upload and scan outcome intact', async (t) => { const app = await harness(t); app.setInput('summary', 'false'); app.setInput('upload-artifacts', 'true'); const upload = t.mock.method(DefaultArtifactClient.prototype, 'uploadArtifact', async () => ({id: 2468, size: 1})); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(upload.mock.callCount(), 1); assert.equal(result.summary, ''); assert.equal(result.outputs['report-status'], 'ready'); }); test('Action path input preserves leading and trailing filename spaces', async t => { const app = await harness(t); app.setInput('dry-run', 'true'); const requested = [' leading.txt', ' ', 'trailing.txt ']; const decoys = ['leading.txt', 'trailing.txt']; for (const filename of [...requested, ...decoys]) await writeFile(join(app.repository, filename), filename); app.git('add', '.'); app.git('commit', '-qm', 'Literal filename fixtures'); process.env.GITHUB_SHA = app.git('rev-parse', 'HEAD'); app.setInput('paths', requested.join('\r\n')); const result = await app.run(); assert.equal(result.exitCode, 0, result.logs); assert.deepEqual(result.args.filter(arg => arg.startsWith('--path=')), requested.map(path => '--path=./' + path)); }); for (const path of ['slash\\name.txt', '\tname.txt\t']) test('Action rejects an unsupported scan path before setup: ' + JSON.stringify(path), async t => { const app = await harness(t); app.setInput('paths', path); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.setups, 0); assert.match(result.logs, /backslashes or control characters/u); }); test('scope conflict fails without runtime setup', async (t) => { const app = await harness(t, 'pr'); app.setInput('paths', 'app.txt'); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.setups, 0); assert.match(result.logs, /paths cannot be combined/u); }); test('missing optional CLI SARIF preserves a successful scan and uploads the remaining reports', async (t) => { const app = await harness(t); app.configure({ missingSarif: true }); app.setInput('annotations', 'true'); app.setInput('upload-artifacts', 'true'); let uploadedFiles: string[] = []; t.mock.method(DefaultArtifactClient.prototype, 'uploadArtifact', async (_name: string, files: string[]) => { uploadedFiles = files.map(file => basename(file)); for (const file of files) assert.ok((await readFile(file)).length); return {id: 1, size: 1}; }); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.processes, 1); assert.equal(result.cleanups, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['report-status'], 'partial'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.equal(result.outputs['sarif-path'], ''); assert.equal(result.outputs['high-count'], '1'); assert.ok(result.outputs['json-path']); assert.ok(result.outputs['coverage-path']); assert.ok(result.outputs['results-directory']); assert.deepEqual(uploadedFiles.sort(), ['coverage.json', 'findings.json', 'scan-manifest.json']); assert.match(result.logs, /::warning::SARIF report is unavailable; scan results and other reports are still available\./); assert.match(result.logs, /^::warning .*file=src\/extract\.py/m); assert.doesNotMatch(result.logs, /::error::/); assert.match(result.summary, /\*\*Scan completed\. Findings are reported without failing the job\.\*\*/); assert.match(result.summary, /SARIF report is unavailable/); }); test('missing SARIF does not mask a findings threshold failure', async (t) => { const app = await harness(t); app.setInput('fail-on-severity', 'high'); app.configure({missingSarif: true}); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'failed'); assert.equal(result.outputs['report-status'], 'partial'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.equal(result.outputs['high-count'], '1'); assert.match(result.logs, /::error::Scan completed\. Findings meet the configured failure threshold\./); assert.match(result.summary, /\*\*Scan completed\. Findings meet the configured failure threshold\.\*\*/); }); test('requested artifact upload failure remains fatal without SARIF', async (t) => { const app = await harness(t); app.configure({missingSarif: true}); app.setInput('upload-artifacts', 'true'); const upload = t.mock.method(DefaultArtifactClient.prototype, 'uploadArtifact', async () => { throw new Error('Synthetic artifact upload failure'); }); const result = await app.run(); assert.equal(upload.mock.callCount(), 1); assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['report-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.ok(result.outputs['json-path']); assert.ok(result.outputs['coverage-path']); assert.ok(result.outputs['results-directory']); assert.equal(result.outputs['sarif-path'], ''); assert.match(result.logs, /::error::Scan completed, but required reporting failed\./); assert.match(result.summary, /Synthetic artifact upload failure/); assert.doesNotMatch(result.summary, /\/artifacts\//); }); test('archive read failure preserves scan reports, summary and cleanup', { skip: process.platform === 'win32' || process.getuid?.() === 0, }, async t => { const app = await harness(t); app.setInput('upload-artifacts', 'true'); const requests: string[] = []; const server = createServer(async (request, response) => { for await (const _chunk of request) { /* Read the artifact creation request. */ } requests.push(request.url ?? ''); response.writeHead(200, {'Content-Type': 'application/json'}); response.end(JSON.stringify({ok: true, signed_upload_url: 'https://example.invalid/artifact'})); }); server.listen(0, '127.0.0.1'); await once(server, 'listening'); t.after(() => new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve()))); const address = server.address(); assert.ok(address && typeof address !== 'string'); const payload = Buffer.from(JSON.stringify({scp: 'Actions.Results:synthetic-run:synthetic-job'})).toString('base64url'); Object.assign(process.env, { ACTIONS_RESULTS_URL: `http://127.0.0.1:${address.port}`, ACTIONS_RUNTIME_TOKEN: `e30.${payload}.synthetic-signature`, }); let signal: AbortSignal | undefined; t.mock.method(BlobClient.prototype, 'getBlockBlobClient', () => ({ uploadStream: async (input: AsyncIterable, _bufferSize: number, _concurrency: number, options: {abortSignal: AbortSignal}) => { signal = options.abortSignal; for await (const _chunk of input) { /* Consume the real archive stream. */ } }, })); let staging = ''; let unreadable = ''; const uploadArtifact = DefaultArtifactClient.prototype.uploadArtifact; t.mock.method(DefaultArtifactClient.prototype, 'uploadArtifact', async function (this: DefaultArtifactClient, ...args: Parameters) { staging = args[2]; unreadable = args[1][0]; // Fail the archive's actual file read without altering the saved scan report. await chmod(unreadable, 0); return uploadArtifact.apply(this, args); }); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.cleanups, 1); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['report-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); for (const name of ['json-path', 'coverage-path', 'sarif-path']) { assert.ok((await readFile(result.outputs[name])).length); } assert.match(result.summary, /EACCES/); const displayedSummary = result.summary.replace(/&#(\d+);/g, (_entity, value: string) => String.fromCodePoint(Number(value))); assert.ok(displayedSummary.includes(unreadable)); assert.match(result.logs, /::error::Scan completed, but required reporting failed\./); assert.deepEqual(requests, ['/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact']); assert.equal(signal?.aborted, true); await assert.rejects(stat(staging), {code: 'ENOENT'}); }); test('Bedrock CLI authentication failure leaves findings unavailable', async (t) => { const app = await harness(t); app.setInput('provider', 'amazon-bedrock'); app.setInput('model', 'openai.synthetic-model'); delete process.env.OPENAI_API_KEY; app.configure({exitCode: 2, cliFailure: true}); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.processes, 1); assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['policy-status'], 'not-evaluated'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.equal(result.outputs['json-path'], ''); assert.match(result.summary, /Synthetic API authentication failure/); assert.doesNotMatch(result.logs, /Findings meet the configured failure threshold/); for (const level of ['critical', 'high', 'medium', 'low', 'informational']) assert.equal(result.outputs[`${level}-count`], ''); assert.match(result.summary, /\*\*Findings:\*\* unavailable/); assert.match(result.logs, /Findings: unavailable/); }); test('valid empty findings publish zero counts', async (t) => { const app = await harness(t); app.configure({scanResult: value => { value.findings.findings = []; }}); const result = await app.run(); assert.equal(result.exitCode, 0); for (const level of ['critical', 'high', 'medium', 'low', 'informational']) assert.equal(result.outputs[`${level}-count`], '0'); assert.doesNotMatch(result.summary, /Scope limitations/); }); test('the job summary shows scope limitations from a completed scan with zero findings', async t => { const app = await harness(t); app.setInput('fail-on-severity', 'high'); app.configure({exitCode: 0, scanResult: value => { value.findings.findings = []; value.manifest.scan.scope.limitations = [ 'Discovery stopped before any review completed.', 'Unreviewed directory: .', ]; }}); const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.ok(result.summary.includes('**Scope limitations**')); assert.ok(result.summary.includes('Discovery stopped before any review completed.')); assert.ok(result.summary.includes('Unreviewed directory: <example>.')); }); test('changed checkout during a committed diff scan fails', async (t) => { const app = await harness(t, 'pr'); app.configure({mutateCheckout: true}); const result = await app.run(); assert.equal(result.exitCode, 1); assert.equal(result.outputs['scan-status'], 'failed'); assert.equal(result.outputs['sarif-upload-ready'], 'false'); assert.match(result.summary, /source checkout changed/); });