mirror of
https://github.com/openai/codex-security.git
synced 2026-11-21 02:12:29 +08:00
* fix(sdk): restore GPT-6 Sol cost estimates and budgets * feat(action)!: prepare hosted runners and require complete CI reviews * fix(action): handle containers and retargeted pull requests * Align the main Action quick start with sandbox and coverage behavior * test(packaging): await tar extraction in package fixtures * fix(action): preserve required scans across metadata edits * fix(action): verify sandbox startup before hosted setup * fix(action): only provision Ubuntu after sandbox startup failures * fix(action): preserve setup diagnostics and check native configuration * fix(action): preserve runner setup and PR check semantics * fix(action): verify live PR targets on the triggering host * docs(action): clarify check migration and SARIF permissions --------- Co-authored-by: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com>
227 lines
11 KiB
TypeScript
227 lines
11 KiB
TypeScript
import test, { type TestContext } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { readFileSync } from 'node:fs';
|
|
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { parse } from 'yaml';
|
|
|
|
const workflow = parse(readFileSync(new URL('../examples/pull-request.yml', import.meta.url), 'utf8'));
|
|
const scan = workflow.jobs.security;
|
|
const gate = workflow.jobs['required-review'];
|
|
const step = gate.steps.find((value: {name: string}) => value.name === 'Require a completed review or an empty diff');
|
|
|
|
function context(action: string, changes: Record<string, unknown> = {}, fork = false) {
|
|
return {run_id: 42, token: 'synthetic-read-only-token', repository: 'example/repo', server_url: 'https://github.com', event: {
|
|
action, changes, pull_request: {number: 7, head: {sha: 'a'.repeat(40), repo: {full_name: fork ? 'example/fork' : 'example/repo'}}, base: {ref: 'main', sha: 'b'.repeat(40)}},
|
|
}};
|
|
}
|
|
|
|
function evaluate(expression: string, github: ReturnType<typeof context>): unknown {
|
|
// These workflow expressions use the same boolean operators as JavaScript.
|
|
return new Function('github', 'always', `return (${expression});`)(github, () => true);
|
|
}
|
|
|
|
function render(value: string, github: ReturnType<typeof context>): string {
|
|
return value.replace(/\$\{\{([\s\S]*?)\}\}/g, (_, expression) => String(evaluate(expression, github)));
|
|
}
|
|
|
|
test('the required PR check runs even when the scan job is skipped', () => {
|
|
assert.ok(workflow.on.pull_request.types.includes('edited'), 'Retargeting a PR must rerun the scan against its new base.');
|
|
assert.equal(render(gate.name, context('synchronize')), 'Codex Security / main');
|
|
assert.equal(gate.needs, 'security');
|
|
assert.equal(evaluate(gate.if, context('synchronize', {}, true)), true);
|
|
assert.deepEqual(scan.outputs, {
|
|
'scan-status': '${{ steps.security.outputs.scan-status }}',
|
|
'skip-reason': '${{ steps.security.outputs.skip-reason }}',
|
|
});
|
|
assert.deepEqual(step.env, {
|
|
SCAN_JOB_RESULT: '${{ needs.security.result }}',
|
|
SCAN_STATUS: '${{ needs.security.outputs.scan-status }}',
|
|
SKIP_REASON: '${{ needs.security.outputs.skip-reason }}',
|
|
});
|
|
assert.equal(evaluate(scan.if, context('synchronize', {}, true)), false);
|
|
const checkout = scan.steps.find((entry: {uses?: string}) => entry.uses?.startsWith('actions/checkout@'));
|
|
assert.equal(checkout.with.ref, '${{ github.event.pull_request.head.sha }}');
|
|
assert.equal(checkout.with['fetch-depth'], 0);
|
|
assert.equal(checkout.with['persist-credentials'], false);
|
|
const action = scan.steps.find((entry: {id?: string}) => entry.id === 'security');
|
|
assert.equal(action.with['fail-on-severity'], 'high');
|
|
});
|
|
|
|
for (const [label, action, changes, review] of [
|
|
['opened', 'opened', {}, true],
|
|
['new commits', 'synchronize', {}, true],
|
|
['reopened', 'reopened', {}, true],
|
|
['ready for review', 'ready_for_review', {}, true],
|
|
['base retarget without a new head', 'edited', {base: {ref: {from: 'previous-base'}}}, true],
|
|
['title edit', 'edited', {title: {from: 'Previous title'}}, false],
|
|
['body edit', 'edited', {body: {from: 'Previous body'}}, false],
|
|
] as const) {
|
|
test(`PR workflow admission handles ${label}`, () => {
|
|
const github = context(action, changes);
|
|
assert.equal(evaluate(scan.if, github), review);
|
|
assert.equal(evaluate(gate.if, github), review);
|
|
if (review) {
|
|
assert.equal(render(gate.name, github), 'Codex Security / main');
|
|
}
|
|
});
|
|
}
|
|
|
|
for (const field of ['title', 'body']) {
|
|
test(`${field} edits cannot cancel a scan or replace its required result`, () => {
|
|
const github = context('edited', {[field]: {from: 'Previous value'}});
|
|
github.run_id = 43;
|
|
assert.equal(workflow.concurrency['cancel-in-progress'], true);
|
|
assert.notEqual(render(workflow.concurrency.group, github), render(workflow.concurrency.group, context('synchronize')));
|
|
assert.notEqual(render(gate.name, github), 'Codex Security / main');
|
|
});
|
|
}
|
|
|
|
for (const [jobResult, scanStatus, skipReason, accepted] of [
|
|
['success', 'completed', '', true],
|
|
['success', 'skipped', 'empty-diff', true],
|
|
['success', 'failed', '', false],
|
|
['success', 'skipped', 'dry-run', false],
|
|
['success', '', '', false],
|
|
['failure', 'completed', '', false],
|
|
['failure', 'skipped', 'empty-diff', false],
|
|
['cancelled', 'completed', '', false],
|
|
['skipped', '', '', false],
|
|
] as const) {
|
|
test(`required PR check handles ${jobResult}/${scanStatus || 'no scan'}/${skipReason || 'no skip reason'}`, () => {
|
|
const result = spawnSync('/bin/bash', ['--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', step.run], {
|
|
encoding: 'utf8', env: {PATH: process.env.PATH, SCAN_JOB_RESULT: jobResult, SCAN_STATUS: scanStatus, SKIP_REASON: skipReason},
|
|
});
|
|
assert.ifError(result.error);
|
|
assert.equal(result.status, accepted ? 0 : 1, result.stderr);
|
|
});
|
|
}
|
|
|
|
|
|
async function prBindingFixture(t: TestContext) {
|
|
const directory = await mkdtemp(join(tmpdir(), 'action-pr-binding-'));
|
|
t.after(() => rm(directory, {recursive:true, force:true}));
|
|
const gh = join(directory, 'gh');
|
|
await writeFile(gh, `#!${process.execPath}
|
|
const assert = require('node:assert/strict');
|
|
const args = process.argv.slice(2);
|
|
assert.equal(args.shift(), 'api');
|
|
const hostIndex = args.indexOf('--hostname');
|
|
const host = hostIndex < 0 ? 'github.com' : args.splice(hostIndex, 2)[1];
|
|
assert.equal(host, new URL(process.env.GITHUB_SERVER_URL).hostname);
|
|
assert.equal(process.env.GH_TOKEN, 'synthetic-read-only-token');
|
|
if (args[0] === 'repos/example/repo/pulls/7') {
|
|
assert.deepEqual(args.slice(1), ['--jq', '[.head.sha, .base.ref, .base.sha] | @tsv']);
|
|
console.log(process.env.SYNTHETIC_CURRENT_PR);
|
|
process.exitCode = Number(process.env.SYNTHETIC_API_EXIT);
|
|
} else {
|
|
const ref = encodeURIComponent(process.env.PR_BASE_REF).replace(/[!'()*]/g, c => '%' + c.charCodeAt(0).toString(16).toUpperCase());
|
|
assert.deepEqual(args, ['repos/example/repo/git/ref/heads/' + ref, '--jq', '.object.sha']);
|
|
console.log(process.env.SYNTHETIC_LIVE_BASE);
|
|
process.exitCode = Number(process.env.SYNTHETIC_REF_API_EXIT);
|
|
}
|
|
`, {mode:0o700});
|
|
const before = scan.steps[0];
|
|
const after = gate.steps[0];
|
|
assert.equal(workflow.permissions['pull-requests'], 'read');
|
|
assert.equal(before.name, 'Verify the PR still matches this scan');
|
|
assert.deepEqual(after, before);
|
|
return (github: ReturnType<typeof context>, current: ReturnType<typeof context>, accepting = false, apiExit = 0,
|
|
liveBase = current.event.pull_request.base.sha, refApiExit = 0) => {
|
|
const pr = current.event.pull_request;
|
|
const result = spawnSync('/bin/bash', ['--noprofile', '--norc', '-e', '-o', 'pipefail', '-c',
|
|
accepting ? gate.steps.map((value: {run: string}) => value.run).join('\n') : before.run], {
|
|
encoding:'utf8', env:{PATH:`${directory}:${process.env.PATH}`, GITHUB_REPOSITORY:github.repository,
|
|
GITHUB_SERVER_URL:github.server_url, GH_HOST:'unrelated.example.invalid',
|
|
...Object.fromEntries(Object.entries(before.env).map(([key, value]) => [key, render(String(value), github)])),
|
|
SYNTHETIC_CURRENT_PR:[pr.head.sha, pr.base.ref, pr.base.sha].join('\t'), SYNTHETIC_API_EXIT:String(apiExit),
|
|
SYNTHETIC_LIVE_BASE:liveBase, SYNTHETIC_REF_API_EXIT:String(refApiExit),
|
|
SCAN_JOB_RESULT:'success', SCAN_STATUS:'completed', SKIP_REASON:''},
|
|
});
|
|
assert.ifError(result.error);
|
|
return result;
|
|
};
|
|
}
|
|
|
|
for (const [label, currentHead, currentRef, currentBase, apiExit, accepted] of [
|
|
['current snapshot', 'a', 'main', 'b', 0, true],
|
|
['stale head', 'c', 'main', 'b', 0, false],
|
|
['retarget with the same base commit', 'a', 'release', 'b', 0, false],
|
|
['target advanced', 'a', 'main', 'c', 0, false],
|
|
['API failure', 'a', 'main', 'b', 7, false],
|
|
] as const) {
|
|
test(`PR snapshot binding fails closed before scanning and accepting: ${label}`, async t => {
|
|
const run = await prBindingFixture(t);
|
|
const original = context('synchronize');
|
|
const current = context('synchronize');
|
|
current.event.pull_request.head.sha = currentHead.repeat(40);
|
|
current.event.pull_request.base = {ref:currentRef, sha:currentBase.repeat(40)};
|
|
for (const accepting of [false, true]) {
|
|
const result = run(original, current, accepting, apiExit);
|
|
assert.equal(result.status === 0, accepted, result.stderr);
|
|
if (apiExit === 0 && !accepted) assert.match(result.stderr, /PR head or target changed/);
|
|
}
|
|
});
|
|
}
|
|
|
|
for (const accepting of [false, true]) {
|
|
test(`a cached pull base cannot accept an advanced live branch (${accepting ? 'acceptance' : 'scan'})`, async t => {
|
|
const run = await prBindingFixture(t);
|
|
const original = context('opened');
|
|
assert.equal(run(original, original, accepting, 0, 'c'.repeat(40)).status, 1);
|
|
assert.notEqual(run(original, original, accepting, 0, original.event.pull_request.base.sha, 7).status, 0);
|
|
});
|
|
|
|
test(`PR verification uses the triggering enterprise host (${accepting ? 'acceptance' : 'scan'})`, async t => {
|
|
const run = await prBindingFixture(t);
|
|
const github = context('opened');
|
|
github.server_url = 'https://octocat.ghe.com';
|
|
const result = run(github, github, accepting);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
});
|
|
}
|
|
|
|
test('live branch verification preserves slashes and URL-significant ref characters', async t => {
|
|
const run = await prBindingFixture(t);
|
|
const github = context('opened');
|
|
github.event.pull_request.base.ref = 'release/#canary%';
|
|
for (const accepting of [false, true]) {
|
|
const result = run(github, github, accepting);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
}
|
|
});
|
|
|
|
test('case-only retargets isolate an old rerun and reject its completed result', async t => {
|
|
const run = await prBindingFixture(t);
|
|
const original = context('opened');
|
|
const current = context('edited', {base: {ref: {from: 'main'}}});
|
|
current.event.pull_request.base.ref = 'Main';
|
|
current.run_id = 43;
|
|
const oldRerun = {...original, run_attempt:2};
|
|
const group = (github: ReturnType<typeof context>) => render(workflow.concurrency.group, github).toLowerCase();
|
|
assert.notEqual(group(original), group(current));
|
|
assert.equal(group(original), group(oldRerun));
|
|
assert.notEqual(render(gate.name, original), render(gate.name, current));
|
|
assert.equal(run(original, original).status, 0);
|
|
// The target changes after the initial check, while the first scan is running.
|
|
assert.equal(run(original, current, true).status, 1);
|
|
assert.equal(run(current, current).status, 0);
|
|
assert.equal(run(current, current, true).status, 0);
|
|
assert.equal(run(oldRerun, current).status, 1);
|
|
assert.equal(run(oldRerun, current, true).status, 1);
|
|
});
|
|
|
|
test('same-head PRs targeting different branches require distinct check names', () => {
|
|
const first = context('opened');
|
|
const second = context('opened');
|
|
second.event.pull_request.number = 8;
|
|
second.event.pull_request.base.ref = 'release';
|
|
second.run_id = 43;
|
|
assert.equal(first.event.pull_request.head.sha, second.event.pull_request.head.sha);
|
|
assert.equal(first.event.pull_request.base.sha, second.event.pull_request.base.sha);
|
|
assert.notEqual(render(gate.name, first), render(gate.name, second));
|
|
assert.notEqual(render(workflow.concurrency.group, first), render(workflow.concurrency.group, second));
|
|
});
|