From c32871b09f9f505453ec68540af2cd33bdea6564 Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Wed, 30 Sep 2026 10:19:38 +0100 Subject: [PATCH] feat: support pnpm package artifact consumers --- README.md | 23 ++++++++--- package-lock.json | 23 ++++++++++- package.json | 4 +- scripts/local-package-install.mjs | 27 +++++++++++- scripts/local-package-install.test.mjs | 50 ++++++++++++++++++++-- scripts/local-release.mjs | 57 +++++++++++--------------- scripts/pack-packages.mjs | 41 +++++++++--------- scripts/package-artifact-cli.test.mjs | 12 ++++-- scripts/package-artifacts.mjs | 2 +- scripts/package-artifacts.test.mjs | 10 ++--- scripts/use-local-packages.mjs | 51 ++++++++++++----------- 11 files changed, 200 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index 938c1f5b9..7c0188565 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,8 @@ npm run pack:packages -- --out .artifacts/pi-packages This refreshes model data before building `pi-ai`. To avoid network access when model data is already hydrated, pass `--offline-model-data`. -Then configure an external npm project to consume one package and resolve all -of its Pi dependencies from the same artifact set: +Then configure an external project to consume one package and resolve all of +its Pi dependencies from the same artifact set. npm is the default: ```bash node scripts/use-local-packages.mjs \ @@ -86,10 +86,23 @@ cd ../my-project npm install --ignore-scripts ``` +For a pnpm project, point `--consumer` at the workspace root: + +```bash +node scripts/use-local-packages.mjs \ + --manifest .artifacts/pi-packages/manifest.json \ + --consumer ../my-project \ + --package @earendil-works/pi-agent-core \ + --package-manager pnpm +cd ../my-project +pnpm install --ignore-scripts +``` + Repeat `--package` for each direct dependency. The command updates the -consumer's `package.json` with content-addressed local `file:` references and -npm overrides. Keep the artifact directory available while installing or -updating the consumer. Re-run both commands after changing Pi source. +consumer's `package.json` with content-addressed local `file:` references. It +writes transitive overrides to `package.json` for npm or `pnpm-workspace.yaml` +for pnpm. Keep the artifact directory available while installing or updating +the consumer. Re-run both commands after changing Pi source. ## Building standalone binaries from release source diff --git a/package-lock.json b/package-lock.json index 45a939e81..0135de425 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,8 +22,10 @@ "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", + "pnpm": "11.15.1", "shx": "0.4.0", - "typescript": "7.0.2" + "typescript": "7.0.2", + "yaml": "2.9.0" }, "engines": { "node": ">=22.19.0" @@ -4397,6 +4399,25 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pnpm": { + "version": "11.15.1", + "resolved": "https://registry.npmjs.org/pnpm/-/pnpm-11.15.1.tgz", + "integrity": "sha512-gTULB+U8lTigLx8jA7QpD6LXvgTlbiqXDEzEtBfcdh3hlu2r1J1Vx9yVgNuBAHxEFD5OPX5GKzAA0jwlUSLQZQ==", + "dev": true, + "license": "MIT", + "bin": { + "pn": "bin/pnpm.mjs", + "pnpm": "bin/pnpm.mjs", + "pnpx": "bin/pnpx.mjs", + "pnx": "bin/pnpx.mjs" + }, + "engines": { + "node": ">=22.13" + }, + "funding": { + "url": "https://opencollective.com/pnpm" + } + }, "node_modules/postcss": { "version": "8.5.24", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.24.tgz", diff --git a/package.json b/package.json index 9a7e7e474..bc48b0758 100644 --- a/package.json +++ b/package.json @@ -61,8 +61,10 @@ "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", + "pnpm": "11.15.1", "shx": "0.4.0", - "typescript": "7.0.2" + "typescript": "7.0.2", + "yaml": "2.9.0" }, "engines": { "node": ">=22.19.0" diff --git a/scripts/local-package-install.mjs b/scripts/local-package-install.mjs index e1028e866..f1dc3b872 100644 --- a/scripts/local-package-install.mjs +++ b/scripts/local-package-install.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { isAbsolute, join, relative, resolve } from "node:path"; import { pathToFileURL } from "node:url"; +import { parseDocument } from "yaml"; import { execNpmSync } from "./npm-command.mjs"; const dependencySections = ["dependencies", "devDependencies", "optionalDependencies"]; @@ -92,7 +93,27 @@ function verifyLocalResolutions(directory, artifactSet, directPackageName) { if (!resolvedNames.has(directPackageName)) throw new Error(`${directPackageName} is missing from the consumer lockfile`); } -export function wireConsumer({ artifactSet, consumerDirectory, packageNames }) { +function preparePnpmWorkspace(consumerDirectory, specifiers) { + const workspacePath = join(consumerDirectory, "pnpm-workspace.yaml"); + const contents = existsSync(workspacePath) ? readFileSync(workspacePath, "utf8") : ""; + const document = parseDocument(contents); + if (document.errors.length > 0) { + throw new Error(`Invalid pnpm workspace file ${workspacePath}: ${document.errors.map((error) => error.message).join("; ")}`); + } + const workspace = document.toJS(); + if (workspace !== null && (typeof workspace !== "object" || Array.isArray(workspace))) { + throw new Error(`Invalid pnpm workspace file ${workspacePath}: root must be a mapping`); + } + if (workspace?.overrides !== undefined && (typeof workspace.overrides !== "object" || workspace.overrides === null || Array.isArray(workspace.overrides))) { + throw new Error(`Invalid pnpm workspace file ${workspacePath}: overrides must be a mapping`); + } + if (document.contents === null) document.contents = document.createNode({}); + for (const [name, specifier] of Object.entries(specifiers)) document.setIn(["overrides", name], specifier); + return { contents: String(document), path: workspacePath }; +} + +export function wireConsumer({ artifactSet, consumerDirectory, packageNames, packageManager = "npm" }) { + if (packageManager !== "npm" && packageManager !== "pnpm") throw new Error(`Unsupported package manager: ${packageManager}`); if (packageNames.length === 0) throw new Error("At least one package is required"); const packageJsonPath = join(consumerDirectory, "package.json"); if (!existsSync(packageJsonPath)) throw new Error(`Consumer package.json does not exist: ${packageJsonPath}`); @@ -117,8 +138,10 @@ export function wireConsumer({ artifactSet, consumerDirectory, packageNames }) { } manifest[targetSection] = { ...manifest[targetSection], [name]: specifiers[name] }; } - manifest.overrides = { ...manifest.overrides, ...specifiers }; + if (packageManager === "npm") manifest.overrides = { ...manifest.overrides, ...specifiers }; + const pnpmWorkspace = packageManager === "pnpm" ? preparePnpmWorkspace(consumerDirectory, specifiers) : undefined; writeFileSync(packageJsonPath, `${JSON.stringify(manifest, null, detectIndentation(contents))}\n`); + if (pnpmWorkspace) writeFileSync(pnpmWorkspace.path, pnpmWorkspace.contents); return manifest; } diff --git a/scripts/local-package-install.test.mjs b/scripts/local-package-install.test.mjs index 08b832210..d5e27074d 100644 --- a/scripts/local-package-install.test.mjs +++ b/scripts/local-package-install.test.mjs @@ -2,8 +2,10 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { parse } from "yaml"; import { installConsumer, smokeTestNpmConsumer, wireConsumer } from "./local-package-install.mjs"; import { produceArtifactSet } from "./package-artifacts.mjs"; @@ -134,8 +136,8 @@ test("installs a package as the only direct dependency", (t) => { const { artifactSet, root } = createArtifactSet(t); const unsupportedDirectory = join(root, "unsupported-consumer"); assert.throws( - () => installConsumer({ artifactSet, directory: unsupportedDirectory, packageManager: "pnpm", packageNames: ["@pi-package-test/target"] }), - /Unsupported package manager: pnpm/, + () => installConsumer({ artifactSet, directory: unsupportedDirectory, packageManager: "yarn", packageNames: ["@pi-package-test/target"] }), + /Unsupported package manager: yarn/, ); assert.equal(existsSync(unsupportedDirectory), false); const consumerDirectory = join(root, "isolated-consumer"); @@ -169,3 +171,45 @@ test("installs a package as the only direct dependency", (t) => { /Invalid npm lockfile/, ); }); + +test("wires and installs a pnpm consumer without registry fallbacks", (t) => { + const { artifactSet, root } = createArtifactSet(t); + const consumerDirectory = join(root, "pnpm-consumer"); + mkdirSync(consumerDirectory); + writeFileSync(join(consumerDirectory, "package.json"), '{"private":true,"type":"module"}\n'); + writeFileSync(join(consumerDirectory, "pnpm-workspace.yaml"), "# existing workspace comment\npackages: []\noverrides:\n existing: 1.2.3\n"); + wireConsumer({ + artifactSet, + consumerDirectory, + packageManager: "pnpm", + packageNames: ["@pi-package-test/target"], + }); + + const manifest = JSON.parse(readFileSync(join(consumerDirectory, "package.json"), "utf8")); + assert.deepEqual(Object.keys(manifest.dependencies), ["@pi-package-test/target"]); + assert.equal(manifest.overrides, undefined); + const workspaceContents = readFileSync(join(consumerDirectory, "pnpm-workspace.yaml"), "utf8"); + assert.match(workspaceContents, /^# existing workspace comment/m); + const workspace = parse(workspaceContents); + assert.equal(workspace.overrides.existing, "1.2.3"); + for (const artifact of artifactSet.packages) { + assert.ok(workspace.overrides[artifact.name].endsWith(basename(artifact.tarballPath)), `${artifact.name} override must reference its artifact`); + } + + const pnpmCli = fileURLToPath(new URL("../node_modules/pnpm/bin/pnpm.mjs", import.meta.url)); + execFileSync(process.execPath, [pnpmCli, "install", "--prod", "--ignore-scripts", "--offline"], { + cwd: consumerDirectory, + stdio: "pipe", + timeout: 300_000, + }); + const marker = execFileSync(process.execPath, ["--input-type=module", "--eval", 'import("@pi-package-test/target").then(({ marker }) => console.log(marker))'], { + cwd: consumerDirectory, + encoding: "utf8", + }); + assert.equal(marker.trim(), "local artifact"); + const lockContents = readFileSync(join(consumerDirectory, "pnpm-lock.yaml"), "utf8"); + assert.doesNotMatch(lockContents, /https?:\/\//); + for (const packageName of ["@pi-package-test/target", "@pi-package-test/shared"]) { + assert.ok(lockContents.includes(basename(artifactSet.getPackage(packageName).tarballPath)), `${packageName} lock entry must reference its artifact`); + } +}); diff --git a/scripts/local-release.mjs b/scripts/local-release.mjs index 3ae9762c7..4a9858074 100644 --- a/scripts/local-release.mjs +++ b/scripts/local-release.mjs @@ -4,6 +4,7 @@ import { execFileSync } from "node:child_process"; import { cpSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { parseArgs } from "node:util"; import { installConsumer, packageConsumerDirectoryName, smokeTestNpmConsumer } from "./local-package-install.mjs"; import { produceArtifactSet } from "./package-artifacts.mjs"; import { codingAgentName, smokeTestCodingAgent } from "./coding-agent-smoke.mjs"; @@ -26,38 +27,6 @@ Options: `); } -function parseArgs() { - const options = { - force: false, - outDir: undefined, - skipBunInstall: false, - skipCheck: false, - skipInstall: false, - skipTest: false, - }; - const args = process.argv.slice(2); - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - if (arg === "--help") { - printUsage(); - process.exit(0); - } - if (arg === "--force") options.force = true; - else if (arg === "--skip-check") options.skipCheck = true; - else if (arg === "--skip-test") options.skipTest = true; - else if (arg === "--skip-install") options.skipInstall = true; - else if (arg === "--skip-bun-install") options.skipBunInstall = true; - else if (arg === "--out") { - const outDir = args[i + 1]; - if (!outDir || outDir.startsWith("--")) throw new Error("--out requires a directory"); - options.outDir = outDir; - i++; - } - else throw new Error(`Unknown option: ${arg}`); - } - return options; -} - function currentBinaryPlatform() { if (process.platform === "win32") return process.arch === "arm64" ? "windows-arm64" : "windows-x64"; if (process.platform === "darwin") return process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; @@ -83,7 +52,29 @@ function buildBunBinaryRelease(targetDirectory, archiveDirectory) { return platform; } -const options = parseArgs(); +const { values } = parseArgs({ + options: { + force: { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + out: { type: "string" }, + "skip-bun-install": { type: "boolean", default: false }, + "skip-check": { type: "boolean", default: false }, + "skip-install": { type: "boolean", default: false }, + "skip-test": { type: "boolean", default: false }, + }, +}); +if (values.help) { + printUsage(); + process.exit(0); +} +const options = { + force: values.force, + outDir: values.out, + skipBunInstall: values["skip-bun-install"], + skipCheck: values["skip-check"], + skipInstall: values["skip-install"], + skipTest: values["skip-test"], +}; const repoRoot = process.cwd(); const rootPackageJson = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")); if (rootPackageJson.name !== "pi-monorepo") throw new Error("Run this script from the repository root"); diff --git a/scripts/pack-packages.mjs b/scripts/pack-packages.mjs index 82c5997f5..b61a61320 100644 --- a/scripts/pack-packages.mjs +++ b/scripts/pack-packages.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node import { resolve } from "node:path"; +import { parseArgs } from "node:util"; import { produceArtifactSet } from "./package-artifacts.mjs"; function printUsage() { @@ -17,28 +18,28 @@ Options: `); } -const options = { build: true, force: false, offlineModelData: false, outDir: undefined }; -const args = process.argv.slice(2); -for (let i = 0; i < args.length; i++) { - const arg = args[i]; - if (arg === "--help") { - printUsage(); - process.exit(0); - } - if (arg === "--force") options.force = true; - else if (arg === "--offline-model-data") options.offlineModelData = true; - else if (arg === "--skip-build") options.build = false; - else if (arg === "--out") { - const outDir = args[i + 1]; - if (!outDir || outDir.startsWith("--")) throw new Error("--out requires a directory"); - options.outDir = outDir; - i++; - } - else throw new Error(`Unknown option: ${arg}`); +const { values } = parseArgs({ + options: { + force: { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + "offline-model-data": { type: "boolean", default: false }, + out: { type: "string" }, + "skip-build": { type: "boolean", default: false }, + }, +}); +if (values.help) { + printUsage(); + process.exit(0); } -if (!options.outDir) throw new Error("--out is required"); +if (!values.out) throw new Error("--out is required"); -const artifactSet = produceArtifactSet({ ...options, repoRoot: process.cwd() }); +const artifactSet = produceArtifactSet({ + build: !values["skip-build"], + force: values.force, + offlineModelData: values["offline-model-data"], + outDir: values.out, + repoRoot: process.cwd(), +}); console.log(`\nLocal package artifacts created: ${artifactSet.artifactDirectory}`); console.log(`Manifest: ${artifactSet.manifestPath}`); console.log("\nConnect an external npm project with:"); diff --git a/scripts/package-artifact-cli.test.mjs b/scripts/package-artifact-cli.test.mjs index 5ab37876b..3063353aa 100644 --- a/scripts/package-artifact-cli.test.mjs +++ b/scripts/package-artifact-cli.test.mjs @@ -19,12 +19,16 @@ function assertFailsBeforeWork(result, message) { test("rejects a missing local-release --out value before release work starts", () => { for (const args of [["--out"], ["--out", "--skip-check"]]) { - assertFailsBeforeWork(runScript("local-release.mjs", args), /--out requires a directory/); + assertFailsBeforeWork(runScript("local-release.mjs", args), /Option '--out.*argument (?:missing|is ambiguous)/); } }); test("rejects malformed package artifact options before doing work", () => { - assertFailsBeforeWork(runScript("pack-packages.mjs", ["--out", "--force"]), /--out requires a directory/); - assertFailsBeforeWork(runScript("use-local-packages.mjs", ["--manifest", "--consumer", "target"]), /--manifest requires a value/); - assertFailsBeforeWork(runScript("use-local-packages.mjs", ["--unknown", "value"]), /Unknown option: --unknown/); + assertFailsBeforeWork(runScript("pack-packages.mjs", ["--out", "--force"]), /Option '--out' argument is ambiguous/); + assertFailsBeforeWork(runScript("use-local-packages.mjs", ["--manifest", "--consumer", "target"]), /Option '--manifest' argument is ambiguous/); + assertFailsBeforeWork(runScript("use-local-packages.mjs", ["--unknown", "value"]), /Unknown option '--unknown'/); + assertFailsBeforeWork( + runScript("use-local-packages.mjs", ["--manifest", "missing.json", "--consumer", "target", "--package", "example", "--package-manager", "yarn"]), + /Unsupported package manager: yarn/, + ); }); diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index d52cc2767..e4d129b29 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -55,7 +55,7 @@ function packPackages(packages, tarballDirectory) { return packedPackages; } -export function prepareOutputDirectory(outDir, options) { +function prepareOutputDirectory(outDir, options) { const repoRoot = resolve(options.repoRoot); if (!outDir) return mkdtempSync(join(tmpdir(), "pi-package-artifacts-")); const outputDirectory = resolve(outDir); diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index fdee07b54..b5ec291f5 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -4,7 +4,7 @@ import { appendFileSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSy import { tmpdir } from "node:os"; import { dirname, join, parse } from "node:path"; import test from "node:test"; -import { prepareOutputDirectory, produceArtifactSet, readArtifactSet } from "./package-artifacts.mjs"; +import { produceArtifactSet, readArtifactSet } from "./package-artifacts.mjs"; function writePackage(directory, manifest, files) { mkdirSync(directory, { recursive: true }); @@ -57,20 +57,20 @@ test("produces a verified, content-addressed artifact set", (t) => { assert.throws(() => readArtifactSet(artifactSet.manifestPath), /integrity mismatch/); const packageJsonPath = join(repoRoot, "packages", "shared", "package.json"); assert.throws( - () => prepareOutputDirectory(join(repoRoot, "packages", "shared"), { force: true, repoRoot }), + () => produceArtifactSet({ build: false, force: true, outDir: join(repoRoot, "packages", "shared"), repoRoot }), /Repository-local output directory must be inside.*\.artifacts/, ); assert.equal(existsSync(packageJsonPath), true); assert.throws( - () => prepareOutputDirectory(repoRoot, { force: true, repoRoot }), + () => produceArtifactSet({ build: false, force: true, outDir: repoRoot, repoRoot }), /repository, its ancestor, or a filesystem root/, ); assert.throws( - () => prepareOutputDirectory(temporaryRoot, { force: true, repoRoot }), + () => produceArtifactSet({ build: false, force: true, outDir: temporaryRoot, repoRoot }), /repository, its ancestor, or a filesystem root/, ); assert.throws( - () => prepareOutputDirectory(parse(repoRoot).root, { force: true, repoRoot }), + () => produceArtifactSet({ build: false, force: true, outDir: parse(repoRoot).root, repoRoot }), /repository, its ancestor, or a filesystem root/, ); }); diff --git a/scripts/use-local-packages.mjs b/scripts/use-local-packages.mjs index 93ea4708e..6641bc488 100644 --- a/scripts/use-local-packages.mjs +++ b/scripts/use-local-packages.mjs @@ -1,40 +1,41 @@ #!/usr/bin/env node import { resolve } from "node:path"; +import { parseArgs } from "node:util"; import { wireConsumer } from "./local-package-install.mjs"; import { readArtifactSet } from "./package-artifacts.mjs"; function printUsage() { - console.log(`Usage: node scripts/use-local-packages.mjs --manifest --consumer --package [--package ...] + console.log(`Usage: node scripts/use-local-packages.mjs --manifest --consumer --package [--package ...] [--package-manager npm|pnpm] -Updates an external npm project's package.json to use direct packages and all -transitive Pi packages from a local package artifact set. +Updates an external project's package configuration to use direct packages and +all transitive Pi packages from a local package artifact set. Defaults to npm. `); } -const options = { packageNames: [] }; -const args = process.argv.slice(2); -for (let i = 0; i < args.length; i++) { - const arg = args[i]; - if (arg === "--help") { - printUsage(); - process.exit(0); - } - if (arg !== "--manifest" && arg !== "--consumer" && arg !== "--package") throw new Error(`Unknown option: ${arg}`); - const value = args[i + 1]; - if (!value || value.startsWith("--")) throw new Error(`${arg} requires a value`); - i++; - if (arg === "--manifest") options.manifest = value; - else if (arg === "--consumer") options.consumer = value; - else options.packageNames.push(value); +const { values } = parseArgs({ + options: { + consumer: { type: "string" }, + help: { type: "boolean", default: false }, + manifest: { type: "string" }, + package: { type: "string", multiple: true, default: [] }, + "package-manager": { type: "string", default: "npm" }, + }, +}); +if (values.help) { + printUsage(); + process.exit(0); +} +if (!values.manifest) throw new Error("--manifest is required"); +if (!values.consumer) throw new Error("--consumer is required"); +if (values.package.length === 0) throw new Error("At least one --package is required"); +if (values["package-manager"] !== "npm" && values["package-manager"] !== "pnpm") { + throw new Error(`Unsupported package manager: ${values["package-manager"]}`); } -if (!options.manifest) throw new Error("--manifest is required"); -if (!options.consumer) throw new Error("--consumer is required"); -if (options.packageNames.length === 0) throw new Error("At least one --package is required"); -const artifactSet = readArtifactSet(resolve(options.manifest)); -const consumerDirectory = resolve(options.consumer); -wireConsumer({ artifactSet, consumerDirectory, packageNames: options.packageNames }); +const artifactSet = readArtifactSet(resolve(values.manifest)); +const consumerDirectory = resolve(values.consumer); +wireConsumer({ artifactSet, consumerDirectory, packageManager: values["package-manager"], packageNames: values.package }); console.log(`Updated ${consumerDirectory}/package.json from ${artifactSet.manifestPath}`); -console.log("Run npm install --ignore-scripts in the consumer project."); +console.log(`Run ${values["package-manager"]} install --ignore-scripts in the consumer project.`);