Files
codex-security/.github/scripts/select-ci-checks.mjs
utsav-oaiandutsav-oai e224a3314b fix(cli): hillclimb dependency investigation eval & speed (#2737)
* feat(cli): add independently reviewed dependency checks

* Compare frozen Python advisory versions as releases

* fix(cli): bind unnamed root manifests to observed services

* test(cli): evaluate dependency investigations across public fixtures

* test(sdk): bound the budget wait after the mock scan starts

* test(cli): exercise MCP evidence and missing caller access

* Bind graded citations to the approved use evidence

* test: restore dependency investigation quality controls

* fix(cli): restore dependency investigation and report compatibility

* test: grade malformed dependency baseline answers

* fix(cli): scope headless tool access to the investigator

* test(cli): lock down reviewer and external evidence tool access

* fix(cli): keep linked evidence tied to its originating call

* Reconcile application mappings and expose original evidence hashes

* test: verify selected catalog rows against original replies

* fix(cli): recover incomplete reviews and preserve selected evidence rows

* fix(cli): preserve wire evidence and verify finding citations

* Finalize from approved submissions and explain remaining uses

* fix(cli): restore coordinator recovery, independent review and report contracts

* Review withdrawals under the original dependency assessment ID

* test(cli): compare trace progress with completed search scopes

* Recheck dependency coverage after concurrent investigation changes

* Preserve exploratory scope explanations during dependency review

* feat(cli): track shared source work in dependency investigations

* Include shared source work in final coverage checks

* Require deciding evidence on the caller it establishes

* Preserve generic MCP search outcomes and scopes in dependency reports

* Share pinned source notes between dependency investigators

* Ship the source worklist and search audit modules

* refactor(cli): name the dependency coordinator for its ability to resume

* refactor(cli): simplify the tools for dependency investigations

* fix(cli): keep investigation tool arguments visible to Codex and reuse the evidence store

* fix(cli): send missing verdict citations back to the reviewer

* fix(cli): verify caller bindings and input flow across investigation steps

* fix(cli): require direct citations for asserted runtime facts on every use

* test(cli): verify failed catalog lookups without a timed source scan

---------

Co-authored-by: utsav-oai <348536218+utsav-oai@users.noreply.github.com>
2026-11-19 11:10:07 -08:00

176 lines
6.6 KiB
JavaScript

import { execFileSync } from "node:child_process";
import { appendFileSync, readFileSync } from "node:fs";
import { pathToFileURL } from "node:url";
import { checkoutBase } from "./verify-pr-source.mjs";
const documentation =
/^(?:README\.md|CONTRIBUTING\.md|CODE_OF_CONDUCT\.md|docs\/.*\.md|github-action\/README\.md|docker\/README\.md)$/;
const ordinarySource =
/^(?:sdk\/typescript\/(?:src|tests-ts)|plugins\/codex-security\/mcp-app\/(?:src|tests))\/.*\.ts$/;
const pythonTest = /^plugins\/codex-security\/tests\/test_[^/]*\.py$/;
const sdkTest = /^sdk\/typescript\/tests-ts\/[^/]+\.test\.ts$/;
const actionFiles =
/^(?:action\.yml|github-action\/(?:README\.md|(?:runtime\/)?package(?:-lock)?\.json|dist\/(?:index|post)\.cjs|(?:src|scripts|tests)\/[^/]+\.(?:mjs|ts)))$/;
const evalFiles =
/^evals\/(?:(?:dependency-reachability|package-application-mapping)\/.+|standard-time-budget\/(?:[^/]+\.mts|experiment\.json|README\.md)|triage-finding\/(?:package\.json|pnpm-lock\.yaml|README\.md|(?:assertions|scripts)\/[^/]+\.mts))$/;
const packageManifest = JSON.parse(
readFileSync(
new URL("../../sdk/typescript/package.json", import.meta.url),
"utf8",
),
);
const packagedDocs = new Set(
packageManifest.files
.filter((file) => file.endsWith(".md"))
.map((file) => `sdk/typescript/${file}`),
);
const isDocumentation = (path) =>
documentation.test(path) || packagedDocs.has(path);
function scopeFor(paths, { event, baseChanged, modifiedOnly }) {
if (event !== "pull_request" || baseChanged || !paths.length) return "full";
if (modifiedOnly && paths.every((path) => pythonTest.test(path)))
return "python-tests";
if (paths.every((path) => isDocumentation(path) || path === "RELEASING.md")) {
// The Action profile already builds the package and installs it on Node 24.
// Select the separate Action job from its own inputs below.
return paths.some((path) => packagedDocs.has(path)) ? "action" : "docs";
}
if (paths.every((path) => actionFiles.test(path))) return "action";
if (paths.every((path) => evalFiles.test(path))) return "evals";
if (paths.every((path) => ordinarySource.test(path) || isDocumentation(path)))
return "source";
return "full";
}
export function selectLockfileAudits(paths, { event, baseChanged }) {
// Full runs have no reliable pull-request diff. Exercise changes to the audit itself, too.
if (
event !== "pull_request" ||
baseChanged ||
paths.includes(".github/actions/audit-pnpm-lockfiles/action.yml")
)
return "mcp evals tokenizer";
return [
["mcp", "plugins/codex-security/mcp-app/pnpm-lock.yaml"],
["evals", "evals/triage-finding/pnpm-lock.yaml"],
[
"tokenizer",
"sdk/typescript/pnpm-lock.yaml",
"sdk/typescript/src/server/embedding-tokenizer.ts",
"sdk/typescript/scripts/build-embedding-tokenizer.mjs",
],
]
.filter(([, ...inputs]) => inputs.some((input) => paths.includes(input)))
.map(([target]) => target)
.join(" ");
}
export function selectChecks(paths, options) {
const scope = scopeFor(paths, options);
const full = scope === "full";
const pythonTestsOnly = scope === "python-tests";
const allUbuntuVersions = full || pythonTestsOnly;
const source = full || scope === "source";
const sdkTestsOnly =
scope === "source" && paths.every((path) => sdkTest.test(path));
const sourceBeyondSdkTests = source && !sdkTestsOnly;
const action =
full ||
paths.some(
(path) =>
actionFiles.test(path) || path === "sdk/typescript/src/git-remote.ts",
);
const compatibility = ["22.13.0", "24.0.0", "24", "26.0.0", "26"]
.filter((version) => allUbuntuVersions || version !== "26")
.map((version) => ({ os: "ubuntu-latest", node: version }));
compatibility.push({ os: "macos-latest", node: "22.13.0" });
if (!pythonTestsOnly) {
compatibility.push({ os: "windows-latest", node: "22.13.0" });
if (full) compatibility.push({ os: "windows-latest", node: "24" });
}
const pythonMatrix = [
...["3.10", "3.12", "3.14"]
.filter((version) => allUbuntuVersions || version === "3.12")
.map((version) => ({
os: "ubuntu-latest",
python: version,
markers: "",
})),
{
os: "macos-latest",
python: "3.12",
markers: "cross_platform or native_macos",
},
{
os: "windows-latest",
python: "3.12",
markers: "cross_platform or native_windows",
},
];
return {
"source-required": String(source),
"container-required": String(sourceBeyondSdkTests),
"plugin-tests-required": String(sourceBeyondSdkTests || pythonTestsOnly),
"package-required": String(source || pythonTestsOnly || scope === "action"),
"action-ci": String(action),
"eval-contracts": String(
sourceBeyondSdkTests ||
scope === "evals" ||
paths.includes("sdk/typescript/tests-ts/codex-dependency-pins.test.ts"),
),
"workflow-quality": String(full || pythonTestsOnly),
"compatibility-matrix": JSON.stringify({
include:
scope === "action" || sdkTestsOnly
? [{ os: "ubuntu-latest", node: "24" }]
: compatibility,
}),
"python-matrix": JSON.stringify({ include: pythonMatrix }),
};
}
if (
process.argv[1] &&
import.meta.url === pathToFileURL(process.argv[1]).href
) {
const event = process.env.EVENT_NAME;
const baseChanged = process.env.BASE_CHANGED === "true";
if (event === "pull_request" && baseChanged) {
const { changes, pull_request: pr } = JSON.parse(
readFileSync(process.env.GITHUB_EVENT_PATH, "utf8"),
);
// Ordinary edits already proved their live target before falling back to full checks.
if (changes?.base) {
const base = checkoutBase(pr, process.env.GITHUB_SHA);
if (base !== pr.base.sha)
throw new Error(
`Stale test merge. Requested ${pr.base.ref} at ${pr.base.sha}, checked out base ${base} with head ${pr.head.sha}. Start a fresh pull request event after the merge ref updates.`,
);
}
}
const changes =
event === "pull_request" && !baseChanged
? execFileSync(
"git",
["diff", "--no-renames", "--name-status", "-z", "HEAD^1", "HEAD"],
{ encoding: "utf8" },
)
.split("\0")
.filter(Boolean)
: [];
const paths = changes.filter((_, i) => i % 2);
const modifiedOnly = changes.every((field, i) => i % 2 || field === "M");
const selected = {
...selectChecks(paths, { event, baseChanged, modifiedOnly }),
"audit-lockfiles": selectLockfileAudits(paths, { event, baseChanged }),
};
appendFileSync(
process.env.GITHUB_OUTPUT,
Object.entries(selected)
.map(([key, value]) => `${key}=${value}\n`)
.join(""),
);
}