fix(release): find npm's CLI when node and npm live in different directories

Review follow-ups on #4045:

- Windows npm lookup (Devin): the scripts assumed npm-cli.js sits next to
  node.exe, so a node shim or npm upgraded into the global prefix would make
  the release build throw. Look in npm_execpath first, then next to node,
  then beside every npm.cmd on PATH, and list every place searched if none
  is found. Still no shell involved.
- Benchmark redaction test (Devanshi): pass a fake secret through
  model_args and assert the log shows `--model_args <redacted>` without the
  secret while lm_eval still receives the original value.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-08 09:46:36 -07:00
co-authored by Claude Opus 5.5
parent 9538e97d87
commit 921de19b8e
3 changed files with 109 additions and 8 deletions
+23 -4
View File
@@ -94,11 +94,30 @@ function npmCommand(args) {
if (process.platform !== "win32") {
return ["npm", args];
}
const npmCli = path.join(path.dirname(process.execPath), "node_modules", "npm", "bin", "npm-cli.js");
if (!existsSync(npmCli)) {
throw new Error(`npm CLI not found next to node: ${npmCli}`);
return [process.execPath, [findNpmCli(), ...args]];
}
// Node and npm can live in different directories (a node shim, or npm
// upgraded into the global prefix), so look where npm itself says it is, then
// next to node, then next to every npm.cmd on PATH. Every npm.cmd install
// keeps its CLI at node_modules/npm/bin/npm-cli.js beside the shim.
function findNpmCli() {
const cliPath = path.join("node_modules", "npm", "bin", "npm-cli.js");
const candidates = [];
if (process.env.npm_execpath?.endsWith("npm-cli.js")) {
candidates.push(process.env.npm_execpath);
}
return [process.execPath, [npmCli, ...args]];
candidates.push(path.join(path.dirname(process.execPath), cliPath));
for (const dir of (process.env.PATH || "").split(path.delimiter)) {
if (dir && existsSync(path.join(dir, "npm.cmd"))) {
candidates.push(path.join(dir, cliPath));
}
}
const found = candidates.find((candidate) => existsSync(candidate));
if (!found) {
throw new Error(`npm CLI (npm-cli.js) not found. Looked in: ${candidates.join(", ")}`);
}
return found;
}
function runNpm(args, cwd) {
+23 -4
View File
@@ -73,11 +73,30 @@ function npmCommand(args) {
if (process.platform !== "win32") {
return ["npm", args];
}
const npmCli = path.join(path.dirname(process.execPath), "node_modules", "npm", "bin", "npm-cli.js");
if (!existsSync(npmCli)) {
throw new Error(`npm CLI not found next to node: ${npmCli}`);
return [process.execPath, [findNpmCli(), ...args]];
}
// Node and npm can live in different directories (a node shim, or npm
// upgraded into the global prefix), so look where npm itself says it is, then
// next to node, then next to every npm.cmd on PATH. Every npm.cmd install
// keeps its CLI at node_modules/npm/bin/npm-cli.js beside the shim.
function findNpmCli() {
const cliPath = path.join("node_modules", "npm", "bin", "npm-cli.js");
const candidates = [];
if (process.env.npm_execpath?.endsWith("npm-cli.js")) {
candidates.push(process.env.npm_execpath);
}
return [process.execPath, [npmCli, ...args]];
candidates.push(path.join(path.dirname(process.execPath), cliPath));
for (const dir of (process.env.PATH || "").split(path.delimiter)) {
if (dir && existsSync(path.join(dir, "npm.cmd"))) {
candidates.push(path.join(dir, cliPath));
}
}
const found = candidates.find((candidate) => existsSync(candidate));
if (!found) {
throw new Error(`npm CLI (npm-cli.js) not found. Looked in: ${candidates.join(", ")}`);
}
return found;
}
function runNpm(args, cwd) {
@@ -0,0 +1,63 @@
"""run_lm_eval must not log --model_args, which can carry an API key (CodeQL #177).
The redaction is log-only: lm_eval itself still has to receive the real value,
or the benchmark silently runs against the wrong endpoint/credentials.
"""
import logging
from types import SimpleNamespace
import pytest
from headroom.evals import comprehensive_benchmark
FAKE_SECRET = "sk-test-FAKE-0123456789abcdef"
def test_model_args_redacted_in_log_but_passed_to_lm_eval(monkeypatch, caplog, tmp_path):
calls: list[list[str]] = []
def fake_run(cmd, **_kwargs):
calls.append(list(cmd))
# Non-zero exit stops run_lm_eval right after the call, before it
# looks for result files we did not create.
return SimpleNamespace(returncode=1, stdout="", stderr="stopped by test")
monkeypatch.setattr(comprehensive_benchmark, "run", fake_run)
with caplog.at_level(logging.INFO, logger=comprehensive_benchmark.__name__):
with pytest.raises(RuntimeError, match="stopped by test"):
comprehensive_benchmark.run_lm_eval(
model_args=f"model=gpt-4o-mini,api_key={FAKE_SECRET}",
base_url="http://localhost:8787/v1",
tasks=["arc_easy"],
output_path=str(tmp_path),
)
running = [r.getMessage() for r in caplog.records if r.getMessage().startswith("Running:")]
assert len(running) == 1
assert "--model_args <redacted>" in running[0]
assert FAKE_SECRET not in caplog.text
assert len(calls) == 1
cmd = calls[0]
assert cmd[cmd.index("--model_args") + 1] == (
f"model=gpt-4o-mini,api_key={FAKE_SECRET},base_url=http://localhost:8787/v1"
)
def test_no_model_args_logs_no_placeholder(monkeypatch, caplog, tmp_path):
calls: list[list[str]] = []
def fake_run(cmd, **_kwargs):
calls.append(list(cmd))
return SimpleNamespace(returncode=1, stdout="", stderr="stopped by test")
monkeypatch.setattr(comprehensive_benchmark, "run", fake_run)
with caplog.at_level(logging.INFO, logger=comprehensive_benchmark.__name__):
with pytest.raises(RuntimeError):
comprehensive_benchmark.run_lm_eval(tasks=["arc_easy"], output_path=str(tmp_path))
assert "--model_args" not in caplog.text
assert "--model_args" not in calls[0]