Compare commits

...
13 Commits
Author SHA1 Message Date
Tabish Bidiwale 4cd0720e9b fix: restore lean PR workflow with async main branch matrix
- PRs run only essential tests on ubuntu-latest (fast feedback)
- Main branch runs full cross-platform matrix asynchronously
- Separate required-checks for each workflow type
- Different timeouts: 10min for PR, 15min for matrix
2025-09-29 22:19:37 +10:00
Tabish Bidiwale af7b6e3ed1 refactor: simplify CI workflow with unified matrix strategy
- Consolidate test_pr and test_matrix into single test job
- Add proper shell configuration with defaults
- Add timeout protection (15 minutes)
- Simplify required-checks to single job
- Maintain cross-platform testing (bash on Linux/macOS, pwsh on Windows)
2025-09-29 22:17:28 +10:00
Tabish Bidiwale 63089e01df chore(ci): split pr and main workflows 2025-09-29 22:15:19 +10:00
Tabish Bidiwale 287c1f3654 Merge branch 'main' into e2e-tests-setup 2025-09-29 20:37:19 +10:00
Tabish Bidiwale 7359b4846a docs(archive): document non-interactive flag (#90) 2025-09-29 15:59:51 +10:00
Tabish Bidiwale 88526e6b93 docs(readme): replace discord badge (#87) 2025-09-26 18:23:37 +10:00
Tabish Bidiwale e5b321c80c Merge branch 'main' into e2e-tests-setup 2025-09-26 00:49:50 +10:00
Tabish Bidiwale c8b839f3b4 fix: update core validation and json converter 2025-09-26 00:07:00 +10:00
Tabish Bidiwale 3a0992020b fix: ci 2025-09-25 19:25:59 +10:00
Tabish Bidiwale 4ec34c744b fix ci 2025-09-25 19:22:19 +10:00
Tabish Bidiwale 5534779843 fix: use multiline YAML for diagnostics command 2025-09-25 19:18:57 +10:00
Tabish Bidiwale 54467f2e1c fix: correct YAML syntax in CI workflow diagnostics command 2025-09-25 18:57:07 +10:00
Tabish Bidiwale 745845dd3e feat: implement Phase 1 E2E testing with cross-platform CI matrix
- Add shared runCLI helper in test/helpers/run-cli.ts for spawn testing
- Create test/cli-e2e/basic.test.ts covering help, version, validate flows
- Migrate existing CLI exec tests to use runCLI helper
- Extend CI matrix to bash (Linux/macOS) and pwsh (Windows)
- Update Phase 1 tasks and proposal with implementation status
2025-09-25 17:20:41 +10:00
16 changed files with 428 additions and 148 deletions
+89 -8
View File
@@ -15,10 +15,12 @@ concurrency:
cancel-in-progress: true
jobs:
test:
test_pr:
name: Test
runs-on: ubuntu-latest
timeout-minutes: 10
if: github.event_name == 'pull_request'
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -48,7 +50,68 @@ jobs:
- name: Upload test coverage
uses: actions/upload-artifact@v4
with:
name: coverage-report
name: coverage-report-pr
path: coverage/
retention-days: 7
test_matrix:
name: Test (${{ matrix.label }})
runs-on: ${{ matrix.os }}
timeout-minutes: 15
if: github.event_name != 'pull_request'
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
shell: bash
label: linux-bash
- os: macos-latest
shell: bash
label: macos-bash
- os: windows-latest
shell: pwsh
label: windows-pwsh
defaults:
run:
shell: ${{ matrix.shell }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 9
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'pnpm'
- name: Print environment diagnostics
run: |
node -p "JSON.stringify({ platform: process.platform, arch: process.arch, shell: process.env.SHELL || process.env.ComSpec || '' })"
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Build project
run: pnpm run build
- name: Run tests
run: pnpm test
- name: Upload test coverage
if: matrix.os == 'ubuntu-latest'
uses: actions/upload-artifact@v4
with:
name: coverage-report-main
path: coverage/
retention-days: 7
@@ -122,15 +185,15 @@ jobs:
echo "Changesets not configured, skipping validation"
fi
required-checks:
required-checks-pr:
name: All checks passed
runs-on: ubuntu-latest
needs: [test, lint]
if: always()
needs: [test_pr, lint]
if: always() && github.event_name == 'pull_request'
steps:
- name: Verify all checks passed
run: |
if [[ "${{ needs.test.result }}" != "success" ]]; then
if [[ "${{ needs.test_pr.result }}" != "success" ]]; then
echo "Test job failed"
exit 1
fi
@@ -138,4 +201,22 @@ jobs:
echo "Lint job failed"
exit 1
fi
echo "All required checks passed!"
echo "All required checks passed!"
required-checks-main:
name: All checks passed
runs-on: ubuntu-latest
needs: [test_matrix, lint]
if: always() && github.event_name != 'pull_request'
steps:
- name: Verify all checks passed
run: |
if [[ "${{ needs.test_matrix.result }}" != "success" ]]; then
echo "Matrix test job failed"
exit 1
fi
if [[ "${{ needs.lint.result }}" != "success" ]]; then
echo "Lint job failed"
exit 1
fi
echo "All required checks passed!"
+4 -4
View File
@@ -15,7 +15,7 @@
<a href="https://nodejs.org/"><img alt="node version" src="https://img.shields.io/node/v/@fission-ai/openspec?style=flat-square" /></a>
<a href="./LICENSE"><img alt="License: MIT" src="https://img.shields.io/badge/License-MIT-blue.svg?style=flat-square" /></a>
<a href="https://conventionalcommits.org"><img alt="Conventional Commits" src="https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg?style=flat-square" /></a>
<a href="https://discord.gg/saTQQGQZ"><img alt="Discord" src="https://img.shields.io/discord/1411657095639601154?logo=discord&logoColor=white&style=flat-square" /></a>
<a href="https://discord.gg/saTQQGQZ"><img alt="Discord" src="https://img.shields.io/badge/Discord-Join%20the%20community-5865F2?logo=discord&logoColor=white&style=flat-square" /></a>
</p>
<p align="center">
@@ -185,13 +185,13 @@ You: Please archive the change
(Shortcut for tools with slash commands: /openspec:archive add-profile-filters)
AI: I'll archive the add-profile-filters change.
*Runs: openspec archive add-profile-filters*
*Runs: openspec archive add-profile-filters --yes*
✓ Change archived successfully. Specs updated. Ready for the next feature!
```
Or run the command yourself in terminal:
```bash
$ openspec archive add-profile-filters # Archive the completed change
$ openspec archive add-profile-filters --yes # Archive the completed change without prompts
```
**Note:** Tools with native slash commands (Claude Code, Cursor) can use the shortcuts shown. All other tools work with natural language requests to "create an OpenSpec proposal", "apply the OpenSpec change", or "archive the change".
@@ -203,7 +203,7 @@ openspec list # View active change folders
openspec view # Interactive dashboard of specs and changes
openspec show <change> # Display change details (proposal, tasks, spec updates)
openspec validate <change> # Check spec formatting and structure
openspec archive <change> # Move a completed change into archive/
openspec archive <change> [--yes|-y] # Move a completed change into archive/ (non-interactive with --yes)
```
## Example: How AI Creates OpenSpec Files
+12 -4
View File
@@ -1,7 +1,15 @@
#!/usr/bin/env node
import { execSync } from 'child_process';
import { execFileSync } from 'child_process';
import { existsSync, rmSync } from 'fs';
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
const runTsc = (args = []) => {
const tscPath = require.resolve('typescript/bin/tsc');
execFileSync(process.execPath, [tscPath, ...args], { stdio: 'inherit' });
};
console.log('🔨 Building OpenSpec...\n');
@@ -14,10 +22,10 @@ if (existsSync('dist')) {
// Run TypeScript compiler (use local version explicitly)
console.log('Compiling TypeScript...');
try {
execSync('./node_modules/.bin/tsc -v', { stdio: 'inherit' });
execSync('./node_modules/.bin/tsc', { stdio: 'inherit' });
runTsc(['--version']);
runTsc();
console.log('\n✅ Build completed successfully!');
} catch (error) {
console.error('\n❌ Build failed!');
process.exit(1);
}
}
+4 -3
View File
@@ -58,7 +58,7 @@ Skip proposal for:
After deployment, create separate PR to:
- Move `changes/[name]/` → `changes/archive/YYYY-MM-DD-[name]/`
- Update `specs/` if capabilities changed
- Use `openspec archive [change] --skip-specs` for tooling-only changes
- Use `openspec archive [change] --skip-specs --yes` for tooling-only changes
- Run `openspec validate --strict` to confirm the archived change passes checks
## Before Any Task
@@ -95,7 +95,7 @@ openspec list --specs # List specifications
openspec show [item] # Display change or spec
openspec diff [change] # Show spec differences
openspec validate [item] # Validate changes or specs
openspec archive [change] # Archive after deployment
openspec archive [change] [--yes|-y] # Archive after deployment (add --yes for non-interactive runs)
# Project management
openspec init [path] # Initialize OpenSpec
@@ -117,6 +117,7 @@ openspec validate [change] --strict
- `--strict` - Comprehensive validation
- `--no-interactive` - Disable prompts
- `--skip-specs` - Archive without spec updates
- `--yes`/`-y` - Skip confirmation prompts (non-interactive archive)
## Directory Structure
@@ -447,7 +448,7 @@ openspec list # What's in progress?
openspec show [item] # View details
openspec diff [change] # What's changing?
openspec validate --strict # Is it correct?
openspec archive [change] # Mark complete
openspec archive [change] [--yes|-y] # Mark complete (add --yes for automation)
```
Remember: Specs are truth. Changes are proposals. Keep them in sync.
@@ -2,11 +2,18 @@
Recent cross-shell regressions for `openspec` commands revealed that our existing unit/integration tests do not exercise the packaged CLI or shell-specific behavior. The prior attempt at Vitest spawn tests stalled because it coupled e2e coverage with `pnpm pack` installs, which fail in network-restricted environments. With those findings incorporated, we now need an approved plan to realign the work.
## What Changes
- Adopt a phased strategy that first stabilizes direct spawn testing of the built CLI (`node dist/cli/index.js`) using lightweight fixtures and shared helpers.
- Expand coverage to cross-shell/OS matrices once the spawn harness is stable, ensuring both the direct `node dist/cli/index.js` invocation and the bin shim are exercised with non-TTY defaults and captured diagnostics.
- Adopt a phased strategy that first stabilizes direct spawn testing of the built CLI (`node dist/cli/index.js`) using lightweight fixtures and a shared `runCLI` helper.
- Expand coverage once the spawn harness is stable, keeping the initial matrix focused on bash jobs for Linux/macOS and `pwsh` on Windows while exercising both the direct `node dist/cli/index.js` invocation and the bin shim with non-TTY defaults and captured diagnostics.
- Treat packaging/install validation as an optional CI safeguard: when a runner has registry access, run a simple pnpm-based pack→install→smoke-test flow; otherwise document it as out of scope while closing remaining hardening items.
- Close out the remaining cross-shell hardening items: ensure `.gitattributes` covers packaged assets, enforce executable bits for CLI shims during CI, and finish the pending SIGINT handling improvements.
## Impact
- Tests: add `test/cli-e2e` spawn suite, helpers, and fixture usage updates; adjust `vitest.setup.ts` as needed.
- Tooling: update GitHub Actions workflows to add shell/OS matrices and (optionally) a packaging install check where network is available.
- Docs: keep `CROSS-SHELL-PLAN.md` aligned with the phased rollout and record any limitations called out during execution.
- Tests: add `test/cli-e2e` spawn suite, create the shared `runCLI` helper, and adjust `vitest.setup.ts` as needed.
- Tooling: update GitHub Actions workflows with the lightweight matrix above and (optionally) a packaging install check where network is available.
- Docs: note phase progress and any limitations inline in this proposal (or the relevant spec) so future phases have clear context.
### Phase 1 Status
- Shared `test/helpers/run-cli.ts` guarantees the CLI bundle exists before spawning and enforces non-TTY defaults for every invocation.
- New `test/cli-e2e/basic.test.ts` covers `--help`, `--version`, a successful `validate --all --json`, and an unknown-item error path against the `tmp-init` fixture copy.
- Legacy top-level `validate` exec tests now rely on `runCLI`, avoiding manual `execSync` usage while keeping their fixture authoring intact.
- CI matrix groundwork is in place (bash on Linux/macOS, pwsh on Windows) so the spawn suite runs the same way the helper does across supported shells.
@@ -1,13 +1,13 @@
## 1. Phase 1 – Stabilize Local Spawn Coverage
- [ ] 1.1 Update `vitest.setup.ts` and helpers so the CLI build runs once and `runCLI` executes `node dist/cli.js` with non-TTY defaults.
- [ ] 1.2 Reuse the minimal fixture set (`tmp-init` or copy) to seed initial spawn tests for help/version, a happy-path `validate`, and a representative error flow.
- [ ] 1.3 Document the Phase 1 coverage details in `CROSS-SHELL-PLAN.md`, noting any outstanding gaps.
- [x] 1.1 Add `test/helpers/run-cli.ts` that ensures the build runs once and executes `node dist/cli/index.js` with non-TTY defaults; update `vitest.setup.ts` to reuse the shared build step.
- [x] 1.2 Seed `test/cli-e2e` using the minimal fixture set (`tmp-init` or copy) to cover help/version, a happy-path `validate`, and a representative error flow via the new helper.
- [x] 1.3 Migrate the highest-value existing CLI exec tests (e.g., validate) onto `runCLI` and summarize Phase 1 coverage in this proposal for the next phase.
## 2. Phase 2 – Expand Cross-Shell Validation
- [ ] 2.1 Exercise both entry points (`node dist/cli.js`, `bin/openspec.js`) in the spawn suite and add diagnostics for shell/OS context.
- [ ] 2.2 Extend GitHub Actions to run the spawn suite across a matrix of shells (bash, zsh, fish, pwsh, cmd) on macOS, Linux, and Windows runners.
- [ ] 2.1 Exercise both entry points (`node dist/cli/index.js`, `bin/openspec.js`) in the spawn suite and add diagnostics for shell/OS context.
- [x] 2.2 Extend GitHub Actions to run the spawn suite on bash jobs for Linux/macOS and a `pwsh` job on Windows; capture shell/OS diagnostics and note follow-ups for additional shells.
## 3. Phase 3 – Package Validation (Optional)
- [ ] 3.1 Add a simple CI job on runners with registry access that runs `pnpm pack`, installs the tarball into a temp workspace (e.g., `pnpm add --no-save`), and executes `pnpm exec openspec --version`.
- [ ] 3.2 If network-restricted environments can’t exercise installs, document the limitation in `CROSS-SHELL-PLAN.md` and skip the job there.
- [ ] 3.3 Close out remaining hardening items from the original cross-shell plan (e.g., `.gitattributes`, chmod enforcement, SIGINT follow-ups) and update the plan accordingly.
- [ ] 3.2 If network-restricted environments can’t exercise installs, skip the job and note the limitation in this proposal’s rollout log.
- [ ] 3.3 Close out the enumerated hardening items: extend `.gitattributes` to cover packaged assets, enforce executable bits for CLI shims during CI, and finish the outstanding SIGINT handling work; update this proposal once they land.
+6 -4
View File
@@ -43,7 +43,8 @@ export class JsonConverter {
}
private extractNameFromPath(filePath: string): string {
const parts = filePath.split('/');
const normalizedPath = filePath.replaceAll('\\', '/');
const parts = normalizedPath.split('/');
for (let i = parts.length - 1; i >= 0; i--) {
if (parts[i] === 'specs' || parts[i] === 'changes') {
@@ -53,7 +54,8 @@ export class JsonConverter {
}
}
const fileName = parts[parts.length - 1];
return fileName.replace('.md', '');
const fileName = parts[parts.length - 1] ?? '';
const dotIndex = fileName.lastIndexOf('.');
return dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
}
}
}
+4 -3
View File
@@ -58,7 +58,7 @@ Skip proposal for:
After deployment, create separate PR to:
- Move \`changes/[name]/\` → \`changes/archive/YYYY-MM-DD-[name]/\`
- Update \`specs/\` if capabilities changed
- Use \`openspec archive [change] --skip-specs\` for tooling-only changes
- Use \`openspec archive [change] --skip-specs --yes\` for tooling-only changes
- Run \`openspec validate --strict\` to confirm the archived change passes checks
## Before Any Task
@@ -95,7 +95,7 @@ openspec list --specs # List specifications
openspec show [item] # Display change or spec
openspec diff [change] # Show spec differences
openspec validate [item] # Validate changes or specs
openspec archive [change] # Archive after deployment
openspec archive [change] [--yes|-y] # Archive after deployment (add --yes for non-interactive runs)
# Project management
openspec init [path] # Initialize OpenSpec
@@ -117,6 +117,7 @@ openspec validate [change] --strict
- \`--strict\` - Comprehensive validation
- \`--no-interactive\` - Disable prompts
- \`--skip-specs\` - Archive without spec updates
- \`--yes\`/\`-y\` - Skip confirmation prompts (non-interactive archive)
## Directory Structure
@@ -447,7 +448,7 @@ openspec list # What's in progress?
openspec show [item] # View details
openspec diff [change] # What's changing?
openspec validate --strict # Is it correct?
openspec archive [change] # Mark complete
openspec archive [change] [--yes|-y] # Mark complete (add --yes for automation)
\`\`\`
Remember: Specs are truth. Changes are proposals. Keep them in sync.
+6 -4
View File
@@ -331,7 +331,8 @@ export class Validator {
}
private extractNameFromPath(filePath: string): string {
const parts = filePath.split('/');
const normalizedPath = filePath.replaceAll('\\', '/');
const parts = normalizedPath.split('/');
// Look for the directory name after 'specs' or 'changes'
for (let i = parts.length - 1; i >= 0; i--) {
@@ -343,8 +344,9 @@ export class Validator {
}
// Fallback to filename without extension if not in expected structure
const fileName = parts[parts.length - 1];
return fileName.replace('.md', '');
const fileName = parts[parts.length - 1] ?? '';
const dotIndex = fileName.lastIndexOf('.');
return dotIndex > 0 ? fileName.slice(0, dotIndex) : fileName;
}
private createReport(issues: ValidationIssue[]): ValidationReport {
@@ -393,4 +395,4 @@ export class Validator {
const matches = blockRaw.match(/^####\s+/gm);
return matches ? matches.length : 0;
}
}
}
+56
View File
@@ -0,0 +1,56 @@
import { afterAll, describe, it, expect } from 'vitest';
import { promises as fs } from 'fs';
import path from 'path';
import { tmpdir } from 'os';
import { runCLI, cliProjectRoot } from '../helpers/run-cli.js';
const tempRoots: string[] = [];
async function prepareFixture(fixtureName: string): Promise<string> {
const base = await fs.mkdtemp(path.join(tmpdir(), 'openspec-cli-e2e-'));
tempRoots.push(base);
const projectDir = path.join(base, 'project');
await fs.mkdir(projectDir, { recursive: true });
const fixtureDir = path.join(cliProjectRoot, 'test', 'fixtures', fixtureName);
await fs.cp(fixtureDir, projectDir, { recursive: true });
return projectDir;
}
afterAll(async () => {
await Promise.all(tempRoots.map((dir) => fs.rm(dir, { recursive: true, force: true })));
});
describe('openspec CLI e2e basics', () => {
it('shows help output', async () => {
const result = await runCLI(['--help']);
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain('Usage: openspec');
expect(result.stderr).toBe('');
});
it('reports the package version', async () => {
const pkgRaw = await fs.readFile(path.join(cliProjectRoot, 'package.json'), 'utf-8');
const pkg = JSON.parse(pkgRaw);
const result = await runCLI(['--version']);
expect(result.exitCode).toBe(0);
expect(result.stdout.trim()).toBe(pkg.version);
});
it('validates the tmp-init fixture with --all --json', async () => {
const projectDir = await prepareFixture('tmp-init');
const result = await runCLI(['validate', '--all', '--json'], { cwd: projectDir });
expect(result.exitCode).toBe(0);
const output = result.stdout.trim();
expect(output).not.toBe('');
const json = JSON.parse(output);
expect(json.summary?.totals?.failed).toBe(0);
expect(json.items.some((item: any) => item.id === 'c1' && item.type === 'change')).toBe(true);
});
it('returns an error for unknown items in the fixture', async () => {
const projectDir = await prepareFixture('tmp-init');
const result = await runCLI(['validate', 'does-not-exist'], { cwd: projectDir });
expect(result.exitCode).toBe(1);
expect(result.stderr).toContain("Unknown item 'does-not-exist'");
});
});
+58 -87
View File
@@ -1,31 +1,33 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { promises as fs } from 'fs';
import path from 'path';
import { execSync } from 'child_process';
import { runCLI } from '../helpers/run-cli.js';
describe('top-level validate command', () => {
const projectRoot = process.cwd();
const testDir = path.join(projectRoot, 'test-validate-command-tmp');
const changesDir = path.join(testDir, 'openspec', 'changes');
const specsDir = path.join(testDir, 'openspec', 'specs');
const bin = path.join(projectRoot, 'bin', 'openspec.js');
beforeEach(async () => {
await fs.mkdir(changesDir, { recursive: true });
await fs.mkdir(specsDir, { recursive: true });
// Create a valid spec
const specContent = `## Purpose
Valid spec for testing.
## Requirements
### Requirement: Foo
Text
#### Scenario: Bar
Given A\nWhen B\nThen C`;
const specContent = [
'## Purpose',
'This spec ensures the validation harness exercises a deterministic alpha module for automated tests.',
'',
'## Requirements',
'',
'### Requirement: Alpha module SHALL produce deterministic output',
'The alpha module SHALL produce a deterministic response for validation.',
'',
'#### Scenario: Deterministic alpha run',
'- **GIVEN** a configured alpha module',
'- **WHEN** the module runs the default flow',
'- **THEN** the output matches the expected fixture result',
].join('\n');
await fs.mkdir(path.join(specsDir, 'alpha'), { recursive: true });
await fs.writeFile(path.join(specsDir, 'alpha', 'spec.md'), specContent, 'utf-8');
@@ -33,10 +35,26 @@ Given A\nWhen B\nThen C`;
const changeContent = `# Test Change\n\n## Why\nBecause reasons that are sufficiently long for validation.\n\n## What Changes\n- **alpha:** Add something`;
await fs.mkdir(path.join(changesDir, 'c1'), { recursive: true });
await fs.writeFile(path.join(changesDir, 'c1', 'proposal.md'), changeContent, 'utf-8');
const deltaContent = [
'## ADDED Requirements',
'### Requirement: Validator SHALL support alpha change deltas',
'The validator SHALL accept deltas provided by the test harness.',
'',
'#### Scenario: Apply alpha delta',
'- **GIVEN** the test change delta',
'- **WHEN** openspec validate runs',
'- **THEN** the validator reports the change as valid',
].join('\n');
const c1DeltaDir = path.join(changesDir, 'c1', 'specs', 'alpha');
await fs.mkdir(c1DeltaDir, { recursive: true });
await fs.writeFile(path.join(c1DeltaDir, 'spec.md'), deltaContent, 'utf-8');
// Duplicate name for ambiguity test
await fs.mkdir(path.join(changesDir, 'dup'), { recursive: true });
await fs.writeFile(path.join(changesDir, 'dup', 'proposal.md'), changeContent, 'utf-8');
const dupDeltaDir = path.join(changesDir, 'dup', 'specs', 'dup');
await fs.mkdir(dupDeltaDir, { recursive: true });
await fs.writeFile(path.join(dupDeltaDir, 'spec.md'), deltaContent, 'utf-8');
await fs.mkdir(path.join(specsDir, 'dup'), { recursive: true });
await fs.writeFile(path.join(specsDir, 'dup', 'spec.md'), specContent, 'utf-8');
});
@@ -45,77 +63,36 @@ Given A\nWhen B\nThen C`;
await fs.rm(testDir, { recursive: true, force: true });
});
it('prints a helpful hint when no args in non-interactive mode', () => {
const originalCwd = process.cwd();
const originalEnv = { ...process.env };
try {
process.chdir(testDir);
process.env.OPEN_SPEC_INTERACTIVE = '0';
let err: any;
try {
execSync(`node ${bin} validate`, { encoding: 'utf-8' });
} catch (e) { err = e; }
expect(err).toBeDefined();
expect(err.status).not.toBe(0);
expect(err.stderr.toString()).toContain('Nothing to validate. Try one of:');
} finally {
process.chdir(originalCwd);
process.env = originalEnv;
}
it('prints a helpful hint when no args in non-interactive mode', async () => {
const result = await runCLI(['validate'], { cwd: testDir });
expect(result.exitCode).toBe(1);
expect(result.stderr).toContain('Nothing to validate. Try one of:');
});
it('validates all with --all and outputs JSON summary', () => {
const originalCwd = process.cwd();
try {
process.chdir(testDir);
let outStr = '';
try {
outStr = execSync(`node ${bin} validate --all --json`, { encoding: 'utf-8' });
} catch (e: any) {
// If exit code is non-zero (e.g., on failures), still parse stdout JSON
outStr = e.stdout?.toString?.() ?? '';
}
const json = JSON.parse(outStr);
expect(Array.isArray(json.items)).toBe(true);
expect(json.summary?.totals?.items).toBeDefined();
expect(json.version).toBe('1.0');
} finally {
process.chdir(originalCwd);
}
it('validates all with --all and outputs JSON summary', async () => {
const result = await runCLI(['validate', '--all', '--json'], { cwd: testDir });
expect(result.exitCode).toBe(0);
const output = result.stdout.trim();
expect(output).not.toBe('');
const json = JSON.parse(output);
expect(Array.isArray(json.items)).toBe(true);
expect(json.summary?.totals?.items).toBeDefined();
expect(json.version).toBe('1.0');
});
it('validates only specs with --specs and respects --concurrency', () => {
const originalCwd = process.cwd();
try {
process.chdir(testDir);
let outStr = '';
try {
outStr = execSync(`node ${bin} validate --specs --json --concurrency 1`, { encoding: 'utf-8' });
} catch (e: any) {
outStr = e.stdout?.toString?.() ?? '';
}
const json = JSON.parse(outStr);
// All items should be specs
expect(json.items.every((i: any) => i.type === 'spec')).toBe(true);
} finally {
process.chdir(originalCwd);
}
it('validates only specs with --specs and respects --concurrency', async () => {
const result = await runCLI(['validate', '--specs', '--json', '--concurrency', '1'], { cwd: testDir });
expect(result.exitCode).toBe(0);
const output = result.stdout.trim();
expect(output).not.toBe('');
const json = JSON.parse(output);
expect(json.items.every((i: any) => i.type === 'spec')).toBe(true);
});
it('errors on ambiguous item names and suggests type override', () => {
const originalCwd = process.cwd();
try {
process.chdir(testDir);
let err: any;
try {
execSync(`node ${bin} validate dup`, { encoding: 'utf-8' });
} catch (e) { err = e; }
expect(err).toBeDefined();
expect(err.stderr.toString()).toContain('Ambiguous item');
expect(err.status).not.toBe(0);
} finally {
process.chdir(originalCwd);
}
it('errors on ambiguous item names and suggests type override', async () => {
const result = await runCLI(['validate', 'dup'], { cwd: testDir });
expect(result.exitCode).toBe(1);
expect(result.stderr).toContain('Ambiguous item');
});
it('accepts change proposals saved with CRLF line endings', async () => {
@@ -141,6 +118,7 @@ Given A\nWhen B\nThen C`;
'The parser SHALL accept CRLF change proposals without manual edits.',
'',
'#### Scenario: Validate CRLF change',
'- **GIVEN** a change proposal saved with CRLF line endings',
'- **WHEN** a developer runs openspec validate on the proposal',
'- **THEN** validation succeeds without section errors',
]);
@@ -149,14 +127,7 @@ Given A\nWhen B\nThen C`;
await fs.mkdir(deltaDir, { recursive: true });
await fs.writeFile(path.join(deltaDir, 'spec.md'), deltaContent, 'utf-8');
const originalCwd = process.cwd();
try {
process.chdir(testDir);
expect(() => execSync(`node ${bin} validate ${changeId}`, { encoding: 'utf-8' })).not.toThrow();
} finally {
process.chdir(originalCwd);
}
const result = await runCLI(['validate', changeId], { cwd: testDir });
expect(result.exitCode).toBe(0);
});
});
@@ -0,0 +1,7 @@
# Test Change
## Why
Because reasons that are sufficiently long for validation.
## What Changes
- **alpha:** Add something
@@ -0,0 +1,8 @@
## ADDED Requirements
### Requirement: Parser SHALL accept CRLF change proposals
The parser SHALL accept CRLF change proposals without manual edits.
#### Scenario: Validate CRLF change
- **GIVEN** a change proposal saved with CRLF line endings
- **WHEN** a developer runs openspec validate on the proposal
- **THEN** validation succeeds without section errors
+12
View File
@@ -0,0 +1,12 @@
## Purpose
This spec ensures the validation harness exercises a deterministic alpha module for automated tests.
## Requirements
### Requirement: Alpha module SHALL produce deterministic output
The alpha module SHALL produce a deterministic response for validation.
#### Scenario: Deterministic alpha run
- **GIVEN** a configured alpha module
- **WHEN** the module runs the default flow
- **THEN** the output matches the expected fixture result
+139
View File
@@ -0,0 +1,139 @@
import { spawn } from 'child_process';
import { existsSync } from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const projectRoot = path.resolve(__dirname, '..', '..');
const cliEntry = path.join(projectRoot, 'dist', 'cli', 'index.js');
let buildPromise: Promise<void> | undefined;
interface RunCommandOptions {
cwd?: string;
env?: NodeJS.ProcessEnv;
}
interface RunCLIOptions {
cwd?: string;
env?: NodeJS.ProcessEnv;
input?: string;
timeoutMs?: number;
}
export interface RunCLIResult {
exitCode: number | null;
signal: NodeJS.Signals | null;
stdout: string;
stderr: string;
timedOut: boolean;
command: string;
}
function runCommand(command: string, args: string[], options: RunCommandOptions = {}) {
return new Promise<void>((resolve, reject) => {
const child = spawn(command, args, {
cwd: options.cwd ?? projectRoot,
env: { ...process.env, ...options.env },
stdio: 'inherit',
shell: process.platform === 'win32',
});
child.on('error', (error) => reject(error));
child.on('close', (code, signal) => {
if (code === 0) {
resolve();
} else {
const reason = signal ? `signal ${signal}` : `exit code ${code}`;
reject(new Error(`Command failed (${reason}): ${command} ${args.join(' ')}`));
}
});
});
}
export async function ensureCliBuilt() {
if (existsSync(cliEntry)) {
return;
}
if (!buildPromise) {
buildPromise = runCommand('pnpm', ['run', 'build']).catch((error) => {
buildPromise = undefined;
throw error;
});
}
await buildPromise;
if (!existsSync(cliEntry)) {
throw new Error('CLI entry point missing after build. Expected dist/cli/index.js');
}
}
export async function runCLI(args: string[] = [], options: RunCLIOptions = {}): Promise<RunCLIResult> {
await ensureCliBuilt();
const finalArgs = Array.isArray(args) ? args : [args];
const invocation = [cliEntry, ...finalArgs].join(' ');
return new Promise<RunCLIResult>((resolve, reject) => {
const child = spawn(process.execPath, [cliEntry, ...finalArgs], {
cwd: options.cwd ?? projectRoot,
env: {
...process.env,
OPEN_SPEC_INTERACTIVE: '0',
...options.env,
},
stdio: ['pipe', 'pipe', 'pipe'],
windowsHide: true,
});
let stdout = '';
let stderr = '';
let timedOut = false;
const timeout = options.timeoutMs
? setTimeout(() => {
timedOut = true;
child.kill('SIGKILL');
}, options.timeoutMs)
: undefined;
child.stdout?.setEncoding('utf-8');
child.stdout?.on('data', (chunk) => {
stdout += chunk;
});
child.stderr?.setEncoding('utf-8');
child.stderr?.on('data', (chunk) => {
stderr += chunk;
});
child.on('error', (error) => {
if (timeout) clearTimeout(timeout);
reject(error);
});
child.on('close', (code, signal) => {
if (timeout) clearTimeout(timeout);
resolve({
exitCode: code,
signal,
stdout,
stderr,
timedOut,
command: `node ${invocation}`,
});
});
if (options.input && child.stdin) {
child.stdin.end(options.input);
} else if (child.stdin) {
child.stdin.end();
}
});
}
export const cliProjectRoot = projectRoot;
+4 -19
View File
@@ -1,21 +1,6 @@
import { execSync } from 'child_process';
import { existsSync } from 'fs';
import path from 'path';
import { ensureCliBuilt } from './test/helpers/run-cli.js';
// Run once before all tests
// Ensure the CLI bundle exists before tests execute
export async function setup() {
const distPath = path.join(process.cwd(), 'dist', 'cli', 'index.js');
if (!existsSync(distPath)) {
console.log('Building project before tests...');
try {
execSync('pnpm run build', {
stdio: 'inherit',
cwd: process.cwd()
});
} catch (error) {
console.error('Failed to build project:', error);
process.exit(1);
}
}
}
await ensureCliBuilt();
}