mirror of
https://github.com/THU-MAIC/OpenMAIC.git
synced 2026-10-02 01:15:18 +08:00
test: add PBL v2 planner eval harness (#803)
Co-authored-by: zlnn23 <zlnn23@mails.tsinghua.edu.cn>
This commit is contained in:
杨慎
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Build (or rebuild) `compare.html` for a harness run directory.
|
||||
*
|
||||
* Usage:
|
||||
* # explicit run dir
|
||||
* pnpm tsx eval/pbl-v2-planner/build-compare.ts eval/pbl-v2-planner/results/<model>/<ts>
|
||||
* # latest run under results/ (auto-pick)
|
||||
* pnpm tsx eval/pbl-v2-planner/build-compare.ts
|
||||
*
|
||||
* Then open the printed compare.html path in a browser.
|
||||
*/
|
||||
import { readdirSync, statSync, existsSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
import { buildCompareHtml } from './compare-html';
|
||||
|
||||
function here(): string {
|
||||
return typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url));
|
||||
}
|
||||
|
||||
/** Find the most recently modified run dir (the leaf containing projects/). */
|
||||
function latestRunDir(): string | undefined {
|
||||
const resultsRoot = join(here(), 'results');
|
||||
if (!existsSync(resultsRoot)) return undefined;
|
||||
let best: { dir: string; mtime: number } | undefined;
|
||||
for (const model of readdirSync(resultsRoot)) {
|
||||
const modelDir = join(resultsRoot, model);
|
||||
if (!statSync(modelDir).isDirectory()) continue;
|
||||
for (const ts of readdirSync(modelDir)) {
|
||||
const dir = join(modelDir, ts);
|
||||
if (!statSync(dir).isDirectory()) continue;
|
||||
if (!existsSync(join(dir, 'projects'))) continue;
|
||||
const mtime = statSync(dir).mtimeMs;
|
||||
if (!best || mtime > best.mtime) best = { dir, mtime };
|
||||
}
|
||||
}
|
||||
return best?.dir;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const arg = process.argv[2];
|
||||
const runDir = arg || latestRunDir();
|
||||
if (!runDir) {
|
||||
console.error('No run dir given and none found under results/. Run the harness first.');
|
||||
process.exit(1);
|
||||
}
|
||||
if (!existsSync(join(runDir, 'projects'))) {
|
||||
console.error(`"${runDir}" has no projects/ subdir — not a harness run dir.`);
|
||||
process.exit(1);
|
||||
}
|
||||
const out = buildCompareHtml(runDir);
|
||||
console.log(`Wrote ${out}`);
|
||||
console.log(`Open it: file://${out}`);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,248 @@
|
||||
/**
|
||||
* Shared rendering for the PBL v2 A/B comparison viewer.
|
||||
*
|
||||
* Two consumers:
|
||||
* - `build-compare.ts` / the runner → `buildCompareHtml(runDir)` writes a
|
||||
* self-contained static `compare.html` (data embedded).
|
||||
* - `serve.ts` → a small dev server that lists every run under `results/`
|
||||
* and loads each run's JSON on demand (data fetched).
|
||||
*
|
||||
* Both share `VIEWER_CSS` + `CLIENT_RENDER_JS` so the rendering stays in one
|
||||
* place. `collectRunData(runDir)` reads `projects/<case>__<variant>.json`
|
||||
* plus the optional `results.json` (judge scores) into one blob.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, readdirSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
export interface EmbeddedVariant {
|
||||
project?: unknown;
|
||||
result?: unknown;
|
||||
}
|
||||
export interface EmbeddedCase {
|
||||
id: string;
|
||||
variants: Record<string, EmbeddedVariant>;
|
||||
}
|
||||
export interface EmbeddedData {
|
||||
runDir: string;
|
||||
cases: EmbeddedCase[];
|
||||
}
|
||||
|
||||
function safeReadJson(path: string): unknown | undefined {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, 'utf-8'));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Collect `<case>__<variant>.json` + results.json for a run into one blob. */
|
||||
export function collectRunData(runDir: string): EmbeddedData {
|
||||
const projectsDir = join(runDir, 'projects');
|
||||
const cases = new Map<string, EmbeddedCase>();
|
||||
|
||||
if (existsSync(projectsDir)) {
|
||||
for (const file of readdirSync(projectsDir)) {
|
||||
if (!file.endsWith('.json')) continue;
|
||||
const base = file.slice(0, -'.json'.length);
|
||||
const sep = base.lastIndexOf('__');
|
||||
if (sep === -1) continue;
|
||||
const caseId = base.slice(0, sep);
|
||||
const variant = base.slice(sep + 2);
|
||||
if (!cases.has(caseId)) cases.set(caseId, { id: caseId, variants: {} });
|
||||
cases.get(caseId)!.variants[variant] = {
|
||||
...cases.get(caseId)!.variants[variant],
|
||||
project: safeReadJson(join(projectsDir, file)),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const results = safeReadJson(join(runDir, 'results.json'));
|
||||
if (Array.isArray(results)) {
|
||||
for (const r of results as Array<Record<string, unknown>>) {
|
||||
const caseId = String(r.caseId ?? '');
|
||||
const variant = String(r.variant ?? '');
|
||||
if (!caseId || !variant) continue;
|
||||
if (!cases.has(caseId)) cases.set(caseId, { id: caseId, variants: {} });
|
||||
const v = cases.get(caseId)!.variants[variant] ?? {};
|
||||
v.result = r;
|
||||
cases.get(caseId)!.variants[variant] = v;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
runDir,
|
||||
cases: [...cases.values()].sort((a, b) => a.id.localeCompare(b.id)),
|
||||
};
|
||||
}
|
||||
|
||||
/** Embed JSON safely inside a <script> tag. */
|
||||
function embed(data: unknown): string {
|
||||
return JSON.stringify(data).replace(/</g, '\\u003c');
|
||||
}
|
||||
|
||||
export const VIEWER_CSS = `
|
||||
:root { --bg:#0f1115; --panel:#171a21; --border:#2a2f3a; --fg:#e6e9ef; --muted:#9aa3b2;
|
||||
--accent:#6ea8fe; --good:#3fb950; --warn:#d29922; --bad:#f85149; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin:0; font:14px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"PingFang SC","Microsoft YaHei",sans-serif;
|
||||
background:var(--bg); color:var(--fg); }
|
||||
header { padding:10px 16px; border-bottom:1px solid var(--border); display:flex; gap:14px; align-items:center; flex-wrap:wrap; }
|
||||
header h1 { font-size:15px; margin:0; font-weight:600; }
|
||||
header .meta { color:var(--muted); font-size:12px; }
|
||||
select { background:var(--panel); color:var(--fg); border:1px solid var(--border); border-radius:6px; padding:5px 8px; font-size:12px; max-width:60vw; }
|
||||
.layout { display:flex; height:calc(100vh - 50px); }
|
||||
nav { width:240px; border-right:1px solid var(--border); overflow:auto; flex:0 0 auto; }
|
||||
nav button { display:block; width:100%; text-align:left; background:none; border:none; color:var(--fg);
|
||||
padding:10px 14px; cursor:pointer; border-bottom:1px solid var(--border); font-size:13px; }
|
||||
nav button:hover { background:var(--panel); }
|
||||
nav button.active { background:#1f2530; border-left:3px solid var(--accent); }
|
||||
nav .ov { float:right; color:var(--muted); font-variant-numeric:tabular-nums; }
|
||||
main { flex:1; overflow:auto; padding:16px; }
|
||||
.cols { display:grid; grid-template-columns:1fr 1fr; gap:16px; align-items:start; }
|
||||
.col { background:var(--panel); border:1px solid var(--border); border-radius:8px; padding:14px; }
|
||||
.col h2 { margin:0 0 4px; font-size:14px; }
|
||||
.judge { font-size:12px; color:var(--muted); margin:6px 0 10px; border:1px solid var(--border);
|
||||
border-radius:6px; padding:8px; background:#10131a; }
|
||||
.judge .dims { display:grid; grid-template-columns:1fr auto 1fr auto; gap:2px 10px; margin-top:6px; }
|
||||
.pill { display:inline-block; padding:1px 7px; border-radius:10px; font-size:11px; font-weight:600; }
|
||||
.pill.good { background:rgba(63,185,80,.15); color:var(--good); }
|
||||
.pill.warn { background:rgba(210,153,34,.15); color:var(--warn); }
|
||||
.pill.bad { background:rgba(248,81,73,.15); color:var(--bad); }
|
||||
.redlines { margin-top:6px; }
|
||||
.redlines .pill { margin:2px 4px 0 0; background:rgba(248,81,73,.15); color:var(--bad); }
|
||||
.proj-title { font-size:15px; font-weight:600; margin:2px 0; }
|
||||
.desc { color:var(--muted); margin:2px 0 8px; }
|
||||
.kv { font-size:12px; color:var(--muted); margin:2px 0; }
|
||||
.ms { border:1px solid var(--border); border-radius:6px; margin:8px 0; }
|
||||
.ms > summary { cursor:pointer; padding:8px 10px; font-weight:600; list-style:none; }
|
||||
.ms > summary::-webkit-details-marker { display:none; }
|
||||
.ms[open] > summary { border-bottom:1px solid var(--border); }
|
||||
.ms .body { padding:8px 10px; }
|
||||
.script { font-size:12px; margin:3px 0; }
|
||||
.script b { color:var(--accent); }
|
||||
.core { font-size:12px; color:var(--warn); margin:4px 0; }
|
||||
.mt { border-left:2px solid var(--border); padding:4px 0 4px 10px; margin:8px 0; }
|
||||
.mt .t { font-weight:600; }
|
||||
.mt .d { color:var(--muted); font-size:13px; }
|
||||
.mt ul { margin:4px 0 0; padding-left:18px; }
|
||||
.mt li { color:#c8cfdb; font-size:12.5px; }
|
||||
.doc { font-size:12px; color:var(--muted); margin-top:6px; }
|
||||
.missing { color:var(--bad); }
|
||||
`;
|
||||
|
||||
/** Client-side render functions. Exposes `renderRun(data)` which fills
|
||||
* #meta, #nav and #main. Pure DOM, no framework. */
|
||||
export const CLIENT_RENDER_JS = `
|
||||
const VARIANTS = ['loop','single-call'];
|
||||
const DIMS = ['projectNotLecture','taskEvaluability','typeFit','granularity','coherence','topicFidelity',
|
||||
'singleConcreteOutcome','difficultyProgressionAndFit','learnerAgency','authenticWorkflow','stageIntegrity','closureAndConsolidation'];
|
||||
const esc = s => String(s ?? '').replace(/[&<>]/g, c => ({'&':'&','<':'<','>':'>'}[c]));
|
||||
|
||||
function overallPill(o){
|
||||
if (o == null) return '';
|
||||
const cls = o >= 4 ? 'good' : o >= 3 ? 'warn' : 'bad';
|
||||
return '<span class="pill '+cls+'">overall '+o+'</span>';
|
||||
}
|
||||
function completabilityBlock(result){
|
||||
const c = result && result.completability;
|
||||
if (!c) return '<div class="judge">可完成性未评分（生成失败、gate 未过或旧 run）</div>';
|
||||
const cls = c.pass ? 'good' : 'bad';
|
||||
const blockers = c.blockers || [];
|
||||
const blockerHtml = blockers.length
|
||||
? '<div class="redlines">blockers '+blockers.map(x=>'<span class="pill">'+esc(x)+'</span>').join('')+'</div>'
|
||||
: '<div class="redlines"><span class="pill good">无 blocker</span></div>';
|
||||
return '<div class="judge"><span class="pill '+cls+'">complete '+(c.pass?'PASS':'FAIL')+' '+c.score+'</span>'
|
||||
+ '<span class="pill '+(c.riskLevel==='low'?'good':c.riskLevel==='medium'?'warn':'bad')+'" style="margin-left:4px">risk '+esc(c.riskLevel||'?')+'</span>'
|
||||
+ blockerHtml
|
||||
+ '<div style="margin-top:6px">'+esc(c.rationale||'')+'</div></div>';
|
||||
}
|
||||
function judgeBlock(result){
|
||||
const j = result && result.judge;
|
||||
if (!j) return '<div class="judge">未评分（生成失败或 gate 未过）</div>';
|
||||
let dims = '<div class="dims">';
|
||||
for (const d of DIMS){ const v=j.scores?.[d]; dims += '<span>'+d+'</span><span>'+(v??'-')+'</span>'; }
|
||||
dims += '</div>';
|
||||
const rl = (j.redLines||[]);
|
||||
const rlHtml = rl.length
|
||||
? '<div class="redlines">红线 '+rl.map(c=>'<span class="pill">'+esc(c)+'</span>').join('')+'</div>'
|
||||
: '<div class="redlines"><span class="pill good">无红线</span></div>';
|
||||
const dur = result.durationMs!=null ? ' · '+(result.durationMs/1000).toFixed(1)+'s' : '';
|
||||
return '<div class="judge">'+overallPill(j.overall)+rlHtml
|
||||
+ '<div style="margin-top:6px">'+esc(j.rationale||'')+'</div>'+dims
|
||||
+ '<div class="kv" style="margin-top:6px">'+(result.milestoneCount??'?')+' milestones · '
|
||||
+ (result.microtaskCount??'?')+' microtasks'+dur+'</div></div>';
|
||||
}
|
||||
function renderProject(p){
|
||||
if (!p) return '<div class="missing">（无此 variant 的生成结果）</div>';
|
||||
const inst = p.roles && p.roles.find(r=>r.type==='instructor');
|
||||
let h = '<div class="proj-title">'+esc(p.title)+'</div>';
|
||||
h += '<div class="desc">'+esc(p.description)+'</div>';
|
||||
if (p.learningObjective) h += '<div class="kv">🎯 '+esc(p.learningObjective)+'</div>';
|
||||
h += '<div class="kv">tier: '+esc(p.proficiency||'?')+'</div>';
|
||||
if (inst) h += '<div class="kv">👤 '+esc(inst.name)+(inst.description?' — '+esc(inst.description):'')+'</div>';
|
||||
for (const m of (p.milestones||[])){
|
||||
h += '<details class="ms"><summary>'+esc(m.title)+' <span class="kv">('+(m.microtasks||[]).length+' 任务)</span></summary><div class="body">';
|
||||
if (m.description) h += '<div class="desc">'+esc(m.description)+'</div>';
|
||||
if (m.briefing) h += '<div class="script"><b>briefing</b> '+esc(m.briefing)+'</div>';
|
||||
if (m.completionCriteria) h += '<div class="script"><b>done</b> '+esc(m.completionCriteria)+'</div>';
|
||||
if (m.debrief) h += '<div class="script"><b>debrief</b> '+esc(m.debrief)+'</div>';
|
||||
if (m.synthesisCheck?.coreConcept) h += '<div class="core">★ coreConcept: '+esc(m.synthesisCheck.coreConcept)+'</div>';
|
||||
for (const t of (m.microtasks||[])){
|
||||
h += '<div class="mt"><div class="t">'+esc(t.title)+'</div>';
|
||||
if (t.description) h += '<div class="d">'+esc(t.description)+'</div>';
|
||||
if (t.hints && t.hints.length){ h += '<ul>'+t.hints.map(x=>'<li>'+esc(x)+'</li>').join('')+'</ul>'; }
|
||||
h += '</div>';
|
||||
}
|
||||
for (const d of (m.documents||[])){ h += '<div class="doc">📄 '+esc(d.title)+'</div>'; }
|
||||
h += '</div></details>';
|
||||
}
|
||||
return h;
|
||||
}
|
||||
function renderCase(c){
|
||||
document.getElementById('main').innerHTML = '<div class="cols">' + VARIANTS.map(v => {
|
||||
const e = c.variants[v] || {};
|
||||
return '<div class="col"><h2>'+v+'</h2>'+completabilityBlock(e.result)+judgeBlock(e.result)+renderProject(e.project)+'</div>';
|
||||
}).join('') + '</div>';
|
||||
}
|
||||
function navLabel(c){
|
||||
const o = VARIANTS.map(v => {
|
||||
const cpl=c.variants[v]?.result?.completability;
|
||||
if (cpl) return (cpl.pass?'P':'F')+cpl.score;
|
||||
const j=c.variants[v]?.result?.judge;
|
||||
return j? j.overall : null;
|
||||
});
|
||||
return esc(c.id) + '<span class="ov">'+o.map(x=>x==null?'-':x).join('/')+'</span>';
|
||||
}
|
||||
function renderRun(data){
|
||||
const metaEl = document.getElementById('meta'); if (metaEl) metaEl.textContent = data.runDir || '';
|
||||
const nav = document.getElementById('nav'); nav.innerHTML='';
|
||||
document.getElementById('main').innerHTML='';
|
||||
if (!data.cases || !data.cases.length){ document.getElementById('main').innerHTML='<p class="missing">此 run 无数据。</p>'; return; }
|
||||
data.cases.forEach((c,i)=>{
|
||||
const b=document.createElement('button');
|
||||
b.innerHTML = navLabel(c);
|
||||
b.onclick=()=>{ [...nav.children].forEach(x=>x.classList.remove('active')); b.classList.add('active'); renderCase(c); };
|
||||
nav.appendChild(b);
|
||||
if (i===0){ b.classList.add('active'); renderCase(c); }
|
||||
});
|
||||
}
|
||||
`;
|
||||
|
||||
function staticTemplate(data: EmbeddedData): string {
|
||||
return `<!doctype html>
|
||||
<html lang="zh"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>PBL v2 — loop vs single-call</title><style>${VIEWER_CSS}</style></head>
|
||||
<body>
|
||||
<header><h1>PBL v2 — loop vs single-call</h1><span class="meta" id="meta"></span></header>
|
||||
<div class="layout"><nav id="nav"></nav><main id="main"></main></div>
|
||||
<script>${CLIENT_RENDER_JS}\nrenderRun(${embed(data)});</script>
|
||||
</body></html>`;
|
||||
}
|
||||
|
||||
export function buildCompareHtml(runDir: string): string {
|
||||
const data = collectRunData(runDir);
|
||||
const outPath = join(runDir, 'compare.html');
|
||||
writeFileSync(outPath, staticTemplate(data), 'utf-8');
|
||||
return outPath;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
You are an expert reviewer for PBL v2 runtime feasibility. Evaluate whether an auto-generated PBL project is **completable by a real learner in the actual OpenMAIC Live PBL v2 runtime**. This is a feasibility judge, not a pedagogy/style judge.
|
||||
|
||||
## Requested project
|
||||
|
||||
- **Topic**: {{topic}}
|
||||
- **Description**: {{description}}
|
||||
- **Target skills**: {{targetSkills}}
|
||||
- **Proficiency tier**: {{proficiency}}
|
||||
|
||||
## Generated project JSON
|
||||
|
||||
{{project}}
|
||||
|
||||
## Actual runtime capabilities
|
||||
|
||||
### Ordinary PBL runtime
|
||||
|
||||
An ordinary PBL project has only:
|
||||
|
||||
- A left roadmap of milestones and microtasks.
|
||||
- A center Instructor chat.
|
||||
- A right submission panel where the learner can paste text, upload their own work, or submit a link.
|
||||
|
||||
Ordinary PBL does **NOT** have a right-side briefing tab, resource panel, reference tab, preloaded image, preloaded screenshot, attached PDF, downloadable starter file, provided dataset, or hidden document viewer. The learner can use their own external tools, editor, browser, or files, but every required project-specific material must be present in visible milestone/task/instructor text.
|
||||
|
||||
If an ordinary project says "see the right-side briefing", "look at the provided image", "use the attached PDF", "open the starter file", "read the dataset", "参考右侧资料", "查看右侧图片", or similar, it is blocked unless that material is fully reproduced as visible text in the project.
|
||||
|
||||
Judge this semantically across languages, not by matching those example phrases. If a task tells the learner to read/inspect/analyze a brief, case note, material, image, dataset, excerpt, map, table, or scenario facts, verify that the actual content needed for the task is present in visible project text. A label like "read the brief below" is not enough when the brief itself is absent.
|
||||
|
||||
### Scenario PBL runtime
|
||||
|
||||
A scenario PBL project has a top-level `scenario` block and runs as:
|
||||
|
||||
- prep: Instructor explains the premise and rules.
|
||||
- one or more roleplay stages: Simulator characters interact with the learner.
|
||||
- wrapup: Instructor consolidates what happened.
|
||||
|
||||
Scenario projects may use the scenario briefing panel after prep, because that is part of the scenario runtime. Roleplay beats must still be advanceable: each roleplay microtask needs a concrete observable `successWhen` unless it is a prep/wrapup task. Hidden character facts may live in private `characterObjective`, but the learner must receive enough visible context to take the next action.
|
||||
|
||||
## What "completable" means
|
||||
|
||||
A project is completable only if a learner can progress from the first milestone to the final outcome using:
|
||||
|
||||
- visible project text,
|
||||
- Instructor/Simulator interaction provided by the runtime,
|
||||
- the learner's own external tools and own created artifacts,
|
||||
- paste/upload/link submission when evidence is needed.
|
||||
|
||||
Do not require the learner to know private facts, inspect nonexistent assets, open unavailable panels, or use a platform capability that the runtime does not provide.
|
||||
|
||||
## Blocker codes
|
||||
|
||||
List every blocker code that applies:
|
||||
|
||||
- **C1 hidden-unavailable-resource**: completion depends on a right-side briefing/resource/reference panel, preloaded image/screenshot, attached PDF, starter file, provided dataset, hidden document, or other material not available in the actual runtime. Also use C1 when the project refers to a brief/material/dataset as if it exists elsewhere but the actual content is not visible in the project text.
|
||||
- **C2 missing-prerequisite-material**: the task assumes domain context, example data, source text, case facts, API keys, account access, or setup that the project never gives and a learner could not reasonably create themselves. Also use C2 when the task asks the learner to extract facts from a brief/case/material, but those facts are not included in visible text.
|
||||
- **C3 unclear-done-evidence-path**: a learner cannot tell what to produce, how to show it, or what observable evidence lets the task/stage complete.
|
||||
- **C4 unavailable-platform-capability**: the project requires runtime behavior the platform does not provide, such as automatic code execution, built-in spreadsheet/database tools, browsing, grading private files, or branch-changing scenario logic.
|
||||
- **C5 impossible-ordering**: a task needs the output of a later task, or the ordering prevents progress.
|
||||
- **C6 scope-too-large**: the project asks for too much to complete in one focused sitting of roughly 15-45 minutes for the requested proficiency tier.
|
||||
- **C7 scenario-cannot-advance**: scenario skeleton is not prep -> roleplay(s) -> wrapup, or a roleplay beat lacks a concrete observable `successWhen` / action needed to advance.
|
||||
- **C8 private-unseen-info-required**: the learner is asked to use or answer with information that is private, hidden, or never surfaced through visible text or scenario interaction.
|
||||
|
||||
## Scoring
|
||||
|
||||
- **5**: Clearly completable. All required materials are visible or learner-created; done/evidence paths are clear.
|
||||
- **4**: Completable with minor ambiguity or friction, but no blocker.
|
||||
- **3**: Possibly completable, but enough ambiguity or missing context that many learners may stall. Usually `pass=false` unless you can explain why no blocker applies.
|
||||
- **2**: Likely blocked by at least one concrete runtime/material/evidence issue.
|
||||
- **1**: Impossible in the actual runtime.
|
||||
|
||||
Set `pass=true` only when the score is 4 or 5 and `blockers` is empty. Any blocker code must make `pass=false`.
|
||||
|
||||
Set `riskLevel`:
|
||||
|
||||
- `low`: score 4-5, no blockers.
|
||||
- `medium`: score 3 or minor uncertainty.
|
||||
- `high`: score 1-2 or any blocker.
|
||||
|
||||
## Output
|
||||
|
||||
Output **exactly one JSON object** and nothing else (no prose, no code fences):
|
||||
|
||||
{
|
||||
"score": <1-5>,
|
||||
"pass": <true|false>,
|
||||
"blockers": ["C1 hidden-unavailable-resource"],
|
||||
"riskLevel": "low" | "medium" | "high",
|
||||
"rationale": "<2-3 sentences explaining whether the learner can complete the project in the real runtime, naming the biggest blocker if any>"
|
||||
}
|
||||
|
||||
Use an empty array for `blockers` when there are no blockers.
|
||||
@@ -0,0 +1,97 @@
|
||||
You are an expert reviewer of **role-play scenario** PBL (Project-Based Learning) designs. Evaluate the design quality of an auto-generated role-play scenario — the cast, premise, and the staged beats (microtasks) a learner will actually live through in-character. Judge the design itself, independent of how it was generated.
|
||||
|
||||
## Core principle
|
||||
|
||||
A role-play scenario is something the learner **performs** — they step into a concrete situation and interact in-character with character(s) played at runtime by a separate Simulator. It is NOT a lecture and NOT a written worksheet. The premise is GIVEN (introduced by the Instructor in prep); the learner never guesses it. Quality lives in the beats: each is a meaningful unit of doing, with a concrete observable "done", building a dramatic arc toward a nameable endpoint, with a debrief that reflects real performance.
|
||||
|
||||
## How to read "done" for a scenario — two axes, never literally
|
||||
|
||||
A beat's "deliverable" is almost never a file. Judge it on two axes:
|
||||
- **Task nature** — most beats are **gradable-open**: a graceful performance or a defended decision with a clear better/worse by the scenario's rules / domain criteria (a poker decision's +EV, an interview answer's structure, an empathetic response's quality). A beat ADVANCES when the action is genuinely done; HOW WELL it was done is judged against those criteria — never reduced to "they said something". Some scenarios also have **convergent** rule-checks (a legal poker action) or **open-reflective** moments (how the learner felt). Never treat a skill beat as "any response passes".
|
||||
- **Delivery form** — the dominant form is **performance** (doing the target action well inside the interaction): empathise then ask, state a boundary, make and defend a decision, negotiate, answer an interview probe. A beat may instead be an **artifact** (the learner hands in something written — e.g. "write them a letter") or an explicit **decision**. Forcing a performance beat into a written form or a quiz is a defect.
|
||||
|
||||
## Requested scenario (source of truth for topic fidelity)
|
||||
|
||||
- **Topic**: {{topic}}
|
||||
- **Description**: {{description}}
|
||||
- **Target skills**: {{targetSkills}}
|
||||
- **Proficiency tier**: {{proficiency}}
|
||||
|
||||
## Generated scenario (JSON)
|
||||
|
||||
The project carries a top-level `scenario` block (setting / rules / learnerRole / characters) and milestones tagged `scenarioStage` (`prep` → `roleplay`×1..N → `wrapup`). Roleplay microtasks are beats carrying `successWhen` / `characterObjective` / `skillFocus` / `narration`.
|
||||
|
||||
{{project}}
|
||||
|
||||
## What is learner-visible vs private (read before judging spoilers / channels)
|
||||
|
||||
- **Learner-visible** (the learner reads these — spoilers here are S4): `setting`, `rules`, `learnerRole`, each character's `name` / `persona` / `situation` / `openingLine`, the prep `briefing`, and each roleplay beat's `description` / `narration`.
|
||||
- **Private by design** (NEVER shown to the learner, never narrated, never spoken): a beat's `characterObjective`. This is the **intended hiding place** for a fact the learner must uncover. A hidden cause / secret / opponent's cards living in `characterObjective` is CORRECT design, NOT a spoiler — do not flag S4 for it.
|
||||
- `successWhen` is the **advance gate** — the observable in-scene action that lets the beat progress. It is NOT required to embed the full grading rubric; HOW WELL the action was done is judged separately at runtime against the scenario's criteria. Do not flag S3 merely because `successWhen` names the action without spelling out a quality bar.
|
||||
- The authored `briefing` / `debrief` are design-time scripts; at runtime the debrief is grounded in the learner's actual performance. A pre-written debrief that reads as if the learner did well is a normal placeholder, not a defect — judge closure on whether the wrapup is SHAPED to deliver specific performance-based feedback, not on the placeholder wording.
|
||||
|
||||
## Quality standards — score each 1-5 (1 = poor, 3 = acceptable, 5 = excellent)
|
||||
|
||||
1. **projectNotLecture** — Is the scenario LIVED, not lectured? Prep teaches the premise; the roleplay stages are genuine in-character doing, not a disguised Q&A about the topic. (low: "beats" that are really quiz questions or the character explaining concepts)
|
||||
2. **taskEvaluability** — Does every roleplay beat carry a concrete, OBSERVABLE `successWhen` — a real in-scene action/decision the learner must say or do — judged by the scenario's criteria? (low: missing `successWhen`, or one that amounts to "they chatted")
|
||||
3. **typeFit** — Does each beat use the right delivery form for the situation (performance / decision / artifact), matching how the real situation actually plays out? (low: a conversation flattened into a form or quiz; a written deliverable demanded where a spoken exchange is the point)
|
||||
4. **granularity** — 2-4 meaningful beats per roleplay stage; each a substantive unit, not a trivial step or a bloated mega-beat. (low: one-line filler beats, or a single giant stage that should be split by round/phase)
|
||||
5. **coherence (dramatic arc)** — Do the beats interlock into an arc (hook → rising stakes → turning point/decision → resolution) and accumulate, rather than a flat reorderable checklist? (low: floating, order-independent beats)
|
||||
6. **topicFidelity** — Does it stay strictly on the requested scenario, no drift/substitution? (low: swapped for a generic "common" roleplay)
|
||||
7. **singleConcreteOutcome** — Does the scenario resolve to ONE nameable endpoint (a decision made and defended, a negotiation closed, an interview completed, a friend supported) that the wrapup reflects on? (low: it just stops mid-scene)
|
||||
8. **difficultyProgressionAndFit** — Do stakes/complexity rise across beats and match the proficiency tier (how much prep/hints scaffold)? (low: flat tension, tier mismatch, or a brutal opening beat)
|
||||
9. **learnerAgency** — Is the scenario FREE-FIRST (the learner always types their own response), never a planted "correct line" that overrules the learner? (low: rigid branching, or a single scripted right answer)
|
||||
10. **authenticWorkflow** — Does the flow resemble how this real situation actually unfolds, so the skill transfers beyond the exercise? (low: an artificial school-only sequence)
|
||||
11. **stageIntegrity** — Is the skeleton exactly prep → roleplay(s) → wrapup with each stage's briefing/debrief matching its beats? Prep is understanding-only (one task, gates nothing); learner-visible text has NO spoilers; channels stay separate (scene facts → narration, rule-teaching → prep Instructor, coaching → beat `hints`, character speaks only in-world). (low: gating prep, spoilers up front, character written as a coach, contradictory scripts)
|
||||
12. **closureAndConsolidation** — Does wrapup land the arc with light, specific feedback grounded in the learner's actual performance (highlights + one improvement)? (low: an empty congratulation, or the scene cut off with no wrapup)
|
||||
|
||||
## Red lines — list every code that is VIOLATED (a single violation means the design fails and must be fixed)
|
||||
|
||||
Shared design red lines:
|
||||
- **B1** forward dependency: a beat needs the result of a later beat.
|
||||
- **B2** prerequisite gap: a beat assumes context no prior stage/prep established.
|
||||
- **B3** floating beat: beats can be reordered freely, no arc, no accumulation.
|
||||
- **B5** mega-beat: one beat bundles several unrelated in-scene goals.
|
||||
- **B6** trivial fragmentation: a single exchange split into too many micro-beats.
|
||||
- **B7** redundant stage: roleplay stages that do the same thing or are pure filler.
|
||||
- **B8** no terminal outcome: the scenario never converges on any nameable endpoint.
|
||||
- **B9** invisible lecture: "beats" are really a Q&A / concept review, not in-character doing.
|
||||
- **B11** topic substitution: requested scenario replaced by a generic teaching scenario.
|
||||
- **B16** scope explosion: too many stages/beats to finish in one focused sitting (≈15-45 min).
|
||||
|
||||
Scenario-specific red lines (the ones that matter most here):
|
||||
- **S1** wrong skeleton: not exactly prep → roleplay(s) → wrapup, or `coreConcept` set on any scenario stage.
|
||||
- **S2** prep gates or guesses: prep has a do-before-advance task, has more than one microtask, or asks the learner to guess/invent the premise instead of being told it. (A prep `completionCriteria` that just says "you've read the background" is NOT a gate — prep is allowed its briefing/completionCriteria text.)
|
||||
- **S3** missing/empty beat success — **ROLEPLAY beats only**: a roleplay beat lacks a `successWhen`, or its `successWhen` names no observable in-scene action (it is literally "they chatted / discussed"). A `successWhen` that names a concrete action without spelling out the quality bar is FINE (quality is judged separately). Prep and wrapup correctly have NO `successWhen` — never flag S3 for them.
|
||||
- **S4** spoiler — **learner-visible fields only** (`setting` / `rules` / `learnerRole` / a character's `persona` / `situation` / `openingLine` / prep `briefing` / a beat's `description` / `narration`): one of these reveals a fact meant to be uncovered later, or pre-states a later beat's situation. A hidden fact placed in the private `characterObjective` is CORRECT and is NOT S4.
|
||||
- **S5** character-as-coach / channel bleed: a character is written to coach the LEARNER — grade them, ask them to justify their reasoning, give strategy/meta hints, narrate the scene, or say "your turn". An in-world evaluative motive (an interviewer privately assessing the candidate, an opponent reading the table) is the character's legitimate drive and is NOT S5; the violation is meta-talk aimed at the learner. A character implying it can see hidden info it shouldn't (e.g. the learner's hole cards) is S5.
|
||||
- **S6** missing rules: a rule-based scenario (game / interview / debate / structured negotiation) omits the concrete `rules` the Instructor needs to teach the premise in prep.
|
||||
- **S7** flattened performance: a beat that should be a live spoken exchange is forced into a written artifact or a quiz with no in-scene reason (delivery-form mismatch).
|
||||
- **S8** false branching / overruled agency: a planted "correct" line or rigid branch overrides the learner's own free response.
|
||||
|
||||
## Output
|
||||
|
||||
Output **exactly one JSON object** and nothing else (no prose, no code fences):
|
||||
|
||||
{
|
||||
"scores": {
|
||||
"projectNotLecture": <1-5>,
|
||||
"taskEvaluability": <1-5>,
|
||||
"typeFit": <1-5>,
|
||||
"granularity": <1-5>,
|
||||
"coherence": <1-5>,
|
||||
"topicFidelity": <1-5>,
|
||||
"singleConcreteOutcome": <1-5>,
|
||||
"difficultyProgressionAndFit": <1-5>,
|
||||
"learnerAgency": <1-5>,
|
||||
"authenticWorkflow": <1-5>,
|
||||
"stageIntegrity": <1-5>,
|
||||
"closureAndConsolidation": <1-5>
|
||||
},
|
||||
"redLines": ["S3", "S4"],
|
||||
"overall": <1-5>,
|
||||
"rationale": "<2-3 sentences: the overall judgement, the single biggest weakness, and any red line and why>"
|
||||
}
|
||||
|
||||
`redLines` may contain B-codes and S-codes; set it to [] when none are violated. "overall" is your holistic ship/no-ship judgement; any red line should pull it down hard.
|
||||
</content>
|
||||
@@ -0,0 +1,108 @@
|
||||
You are an expert PBL (Project-Based Learning) curriculum reviewer. Evaluate the design quality of an auto-generated PBL project — the stages (milestones) and microtasks a learner will actually experience. Judge the design itself, independent of how it was generated.
|
||||
|
||||
## Core principle
|
||||
|
||||
A PBL scene must be a project the learner **executes** (investigates, decides, builds, tests, performs, reflects…), NOT a restated lecture. Every stage and microtask must take the shape of real work in the actual domain.
|
||||
|
||||
## How to read "deliverable / done / outcome" — TWO AXES, never literally
|
||||
|
||||
Do NOT read "done" as "produced a tangible file" or "matched an expected answer". Judge every task on two axes. The design should make the right reading obvious; when it forces the wrong one, that is a defect.
|
||||
|
||||
**Axis A — task nature (how you decide "done well"):**
|
||||
- **Convergent** — there is a checkable right/wrong (code runs, calculation correct, fact right). Done = correct / works.
|
||||
- **Gradable-open** — no single answer, but the domain has a clear better/worse (a poker decision's +EV, a debate argument's strength, a negotiation / analysis / decision's quality). Done = **quality of reasoning + meeting domain criteria**. This is NOT "one correct answer" AND NOT "any stance passes" — most skill / analysis / decision tasks live here. A gradable-open task is only well-designed if the design **states the criteria that separate a strong response from a weak one**.
|
||||
- **Open-reflective** — genuinely no right/wrong; the value is in the thinking (an ethical stance, a personal interpretation, a creative piece, a reflection). Done = depth / honesty of thinking + a clearly stated position or reflection. NEVER "matched the expected answer".
|
||||
|
||||
**Axis B — delivery form (what the evidence looks like):**
|
||||
- **Artifact** — a checkable product (code, a file, a configured environment).
|
||||
- **Argument** — a written trace of thinking (a position + reasons, a decision + its basis, a plan, an analysis, a draft, a reflection).
|
||||
- **Performance** — doing the target action gracefully inside a situated interaction (empathise then ask, state a boundary, make and defend a decision, negotiate, interview).
|
||||
|
||||
For topics with no natural tangible product, the correct design is a gradable-open / open-reflective task delivered as argument or performance — NOT a manufactured fake artifact (a forced 500-word report, a quiz bolted onto a discussion). Forcing a fake artifact is the reddest red line (B17). Forcing a convergent shell onto open work, or labelling open work as having one right answer, is a mislabel (B15).
|
||||
|
||||
## Requested project (source of truth for topic fidelity)
|
||||
|
||||
- **Topic**: {{topic}}
|
||||
- **Description**: {{description}}
|
||||
- **Target skills**: {{targetSkills}}
|
||||
- **Proficiency tier**: {{proficiency}}
|
||||
|
||||
## Generated project (JSON)
|
||||
|
||||
{{project}}
|
||||
|
||||
## Quality standards — score each 1-5 (1 = poor, 3 = acceptable, 5 = excellent)
|
||||
|
||||
1. **projectNotLecture** — Do stages feel like *doing a project*, not restating a course? (low: invisible lecture outline "learn concept A → operation B")
|
||||
2. **taskEvaluability** — Does each microtask carry a clear, judgeable "done" definition appropriate to its task nature (Axis A)? Convergent → a checkable result; gradable-open → the stated criteria that separate strong from weak; open-reflective → a clearly demanded position / reflection. (low: vague "understand X" with no observable done-state; OR a gradable-open task with no criteria, so "done" means nothing more than "said something")
|
||||
3. **typeFit** — Do stages/tasks match the real shape of the domain on BOTH axes — correct task nature AND a fitting delivery form (artifact / argument / performance)? (low: a writing project forced into coding or a quiz; a performance task — a conversation, a negotiation — flattened into a written form; a skill judgement treated as "any answer goes")
|
||||
4. **granularity** — Is the count/grain of stages & tasks sensible? (low: trivial micro-steps, or huge vague mega-tasks)
|
||||
5. **coherence** — Do tasks interlock and point at one named outcome? (low: floating, order-independent tasks with no accumulation)
|
||||
6. **topicFidelity** — Does it stay strictly on the requested topic, no drift/substitution? (low: topic drift or swapped for a generic teaching project)
|
||||
7. **singleConcreteOutcome** — Is there one nameable final destination the learner can say out loud — a product, a defended position, a decision + rationale, a refined question, a reflection, or a graceful performance? (low: aimless, or a forced fake outcome)
|
||||
8. **difficultyProgressionAndFit** — Does difficulty rise gradually and match the proficiency tier? (low: flat difficulty, tier mismatch, or step-1-is-brutal)
|
||||
9. **learnerAgency** — Is there room for the learner to think and decide? (low: thinks for the learner — fill-in-the-blank, copy — stripping choice; OR an open task with a planted "standard answer" that overrules the learner)
|
||||
10. **authenticWorkflow** — Does the stage flow resemble a real practitioner's workflow? (low: artificial school-only sequence with no transfer value)
|
||||
11. **stageIntegrity** — Do stages have meaningful checkpoints, and do briefing/completionCriteria/debrief match the actual tasks inside? (low: arbitrarily split stages, or scripts contradicting task content)
|
||||
12. **closureAndConsolidation** — Does the project end on consolidating work (demo / test / reflection / a landed position)? (low: abruptly stops after the last build step, no wrap-up)
|
||||
|
||||
## Red lines — list every code that is VIOLATED (a single violation means the design fails and must be fixed)
|
||||
|
||||
Sequence & dependency:
|
||||
- **B1** forward dependency: a task needs the output of a later task.
|
||||
- **B2** prerequisite gap: a task assumes knowledge/material no prior task provided.
|
||||
- **B3** floating task: tasks can be reordered freely, no accumulation.
|
||||
|
||||
Decomposition:
|
||||
- **B4** task containment/nesting: one task already contains another's work.
|
||||
- **B5** mega-task: one task bundles several unrelated sub-goals.
|
||||
- **B6** trivial fragmentation: a small action split into too many micro-steps.
|
||||
- **B7** redundant stage: multiple stages do the same thing, or exist only as filler.
|
||||
|
||||
Project integrity:
|
||||
- **B8** no terminal outcome: stages never converge on any nameable endpoint.
|
||||
- **B9** invisible lecture: pure "learn → review" with no real doing.
|
||||
- **B10** stage-script inconsistency: stage briefing/completionCriteria contradicts its inner tasks.
|
||||
- **B11** topic substitution: requested topic replaced by a generic teaching project.
|
||||
|
||||
Fidelity & pedagogy:
|
||||
- **B12** shape mismatch: a non-coding project forced into coding or a quiz; OR a performance task (dialogue / negotiation) flattened into a written form.
|
||||
- **B13** answer leak: the task statement or a hint hands the full answer (the exact line / method / operator / control-flow), leaving no thinking.
|
||||
- **B14** false binary for open work: a multi-solution task forced into one choice then overruled.
|
||||
- **B15** task-shape mislabel (either axis): a convergent task treated as open, a gradable-open / reflective task treated as single-solution, OR a gradable-open task with no stated criteria so "done" collapses to "any stance passes".
|
||||
|
||||
Scope & overfitting:
|
||||
- **B16** scope explosion: too many stages to finish in one focused sitting (≈15-45 min).
|
||||
- **B17** manufactured fake outcome: forcing a fake tangible deliverable onto open-ended work (e.g. a mandatory 500-word report for a discussion). THE reddest red line.
|
||||
|
||||
## If this is a role-play scenario project
|
||||
|
||||
Role-play scenario projects are graded by a **separate scenario rubric**, not this one. If the project carries a top-level `scenario` block, it should not have reached this prompt — judge it on what you see, but the scenario-specific checks live elsewhere.
|
||||
|
||||
## Output
|
||||
|
||||
Output **exactly one JSON object** and nothing else (no prose, no code fences):
|
||||
|
||||
{
|
||||
"scores": {
|
||||
"projectNotLecture": <1-5>,
|
||||
"taskEvaluability": <1-5>,
|
||||
"typeFit": <1-5>,
|
||||
"granularity": <1-5>,
|
||||
"coherence": <1-5>,
|
||||
"topicFidelity": <1-5>,
|
||||
"singleConcreteOutcome": <1-5>,
|
||||
"difficultyProgressionAndFit": <1-5>,
|
||||
"learnerAgency": <1-5>,
|
||||
"authenticWorkflow": <1-5>,
|
||||
"stageIntegrity": <1-5>,
|
||||
"closureAndConsolidation": <1-5>
|
||||
},
|
||||
"redLines": ["B9", "B16"],
|
||||
"overall": <1-5>,
|
||||
"rationale": "<2-3 sentences: the overall judgement, the single biggest weakness, and any red line and why>"
|
||||
}
|
||||
|
||||
`redLines` lists the violated B-codes; set it to [] when none are violated. "overall" is your holistic ship/no-ship judgement; any red line should pull it down hard.
|
||||
</content>
|
||||
</invoke>
|
||||
@@ -0,0 +1,916 @@
|
||||
/**
|
||||
* PBL v2 Planner — Isolation Harness (A/B + LLM-judge)
|
||||
*
|
||||
* Bypasses the UI pipeline and calls the planner(s) directly with a
|
||||
* configured language model. Answers two questions:
|
||||
*
|
||||
* 1. Success rate — what fraction of runs produce a structurally
|
||||
* complete PBL project? (per variant)
|
||||
* 2. Completability — can a real learner finish it in the PBL v2 runtime?
|
||||
* (independent LLM-judge rubric)
|
||||
* 3. Output quality — how good is the project? (LLM-judge rubric)
|
||||
*
|
||||
* It runs each test case through one or more VARIANTS:
|
||||
* - `loop` — the legacy agentic tool-calling planner
|
||||
* (`generatePBLV2Project`)
|
||||
* - `single-call` — the single structured-output planner
|
||||
* (`generatePBLV2ProjectSingleCall`)
|
||||
*
|
||||
* Usage:
|
||||
* EVAL_PBL_MODEL=anthropic:claude-sonnet-4-6 \
|
||||
* EVAL_PBL_API_KEY=<key> \
|
||||
* EVAL_PBL_THINKING=true \
|
||||
* pnpm tsx eval/pbl-v2-planner/runner.ts
|
||||
*
|
||||
* # Only one variant:
|
||||
* EVAL_PBL_VARIANTS=single-call ... pnpm tsx eval/pbl-v2-planner/runner.ts
|
||||
* # Disable the LLM-judge (success-rate only):
|
||||
* EVAL_PBL_JUDGE=false ... pnpm tsx eval/pbl-v2-planner/runner.ts
|
||||
* # First N cases only:
|
||||
* EVAL_PBL_RUNS=4 ... pnpm tsx eval/pbl-v2-planner/runner.ts
|
||||
*
|
||||
* Output: prints tables to stdout and writes a markdown report under
|
||||
* eval/pbl-v2-planner/results/<model>/<timestamp>/.
|
||||
*/
|
||||
|
||||
import { readFileSync, mkdirSync, writeFileSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
import { createGoogleGenerativeAI } from '@ai-sdk/google';
|
||||
import { createAnthropic } from '@ai-sdk/anthropic';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { generateText, type LanguageModel } from 'ai';
|
||||
|
||||
import { generatePBLV2Project, PlannerV2Error } from '@/lib/pbl/v2/agents/planner';
|
||||
import { generatePBLV2ProjectSingleCall } from '@/lib/pbl/v2/agents/planner-single-call';
|
||||
import { parseJsonResponse } from '@/lib/generation/json-repair';
|
||||
import { buildCompareHtml } from './compare-html';
|
||||
import type { PBLPlannerV2Input, PBLProjectV2 } from '@/lib/pbl/v2/types';
|
||||
import type { SceneOutline } from '@/lib/types/generation';
|
||||
import type { ThinkingConfig } from '@/lib/types/provider';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type Variant = 'loop' | 'single-call';
|
||||
|
||||
interface TestCase {
|
||||
id: string;
|
||||
requirement: string;
|
||||
pblConfig: NonNullable<SceneOutline['pblConfig']>;
|
||||
languageDirective: string;
|
||||
}
|
||||
|
||||
interface JudgeScores {
|
||||
scores: {
|
||||
projectNotLecture: number;
|
||||
taskEvaluability: number;
|
||||
typeFit: number;
|
||||
granularity: number;
|
||||
coherence: number;
|
||||
topicFidelity: number;
|
||||
singleConcreteOutcome: number;
|
||||
difficultyProgressionAndFit: number;
|
||||
learnerAgency: number;
|
||||
authenticWorkflow: number;
|
||||
stageIntegrity: number;
|
||||
closureAndConsolidation: number;
|
||||
};
|
||||
redLines: string[];
|
||||
overall: number;
|
||||
rationale?: string;
|
||||
}
|
||||
|
||||
interface CompletabilityJudge {
|
||||
score: number;
|
||||
pass: boolean;
|
||||
blockers: string[];
|
||||
riskLevel: 'low' | 'medium' | 'high';
|
||||
rationale?: string;
|
||||
}
|
||||
|
||||
const JUDGE_DIMENSIONS: Array<keyof JudgeScores['scores']> = [
|
||||
'projectNotLecture',
|
||||
'taskEvaluability',
|
||||
'typeFit',
|
||||
'granularity',
|
||||
'coherence',
|
||||
'topicFidelity',
|
||||
'singleConcreteOutcome',
|
||||
'difficultyProgressionAndFit',
|
||||
'learnerAgency',
|
||||
'authenticWorkflow',
|
||||
'stageIntegrity',
|
||||
'closureAndConsolidation',
|
||||
];
|
||||
|
||||
interface RunResult {
|
||||
caseId: string;
|
||||
variant: Variant;
|
||||
ok: boolean;
|
||||
milestoneCount: number;
|
||||
microtaskCount: number;
|
||||
roleCount: number;
|
||||
durationMs: number;
|
||||
error?: string;
|
||||
/** True if the project passed the completion gate (all milestones have microtasks). */
|
||||
passesCompletionGate: boolean;
|
||||
/** True for role-play scenario cases (graded by the scenario rubric). */
|
||||
isScenario: boolean;
|
||||
/** Runtime feasibility judge: can the learner actually complete it? */
|
||||
completability?: CompletabilityJudge;
|
||||
judge?: JudgeScores;
|
||||
/** Full generated project, dumped to disk for inspection. */
|
||||
project?: PBLProjectV2;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getCurrentDir(): string {
|
||||
return typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url));
|
||||
}
|
||||
|
||||
function parseModelString(raw: string): { provider: string; modelId: string } {
|
||||
const colon = raw.indexOf(':');
|
||||
if (colon === -1) throw new Error(`Invalid model string "${raw}" — expected provider:modelId`);
|
||||
return { provider: raw.slice(0, colon), modelId: raw.slice(colon + 1) };
|
||||
}
|
||||
|
||||
function makeProvider(
|
||||
provider: string,
|
||||
apiKey: string,
|
||||
baseURL?: string,
|
||||
): (id: string) => LanguageModel {
|
||||
switch (provider) {
|
||||
case 'google': {
|
||||
const google = createGoogleGenerativeAI({ apiKey, ...(baseURL ? { baseURL } : {}) });
|
||||
return (id) => google(id);
|
||||
}
|
||||
case 'anthropic': {
|
||||
const anthropic = createAnthropic({ apiKey, ...(baseURL ? { baseURL } : {}) });
|
||||
return (id) => anthropic(id);
|
||||
}
|
||||
case 'openai': {
|
||||
// Use chat-completions (not the Responses API): OpenAI-compatible
|
||||
// gateways (DeepSeek, Qwen, etc.) only speak /v1/chat/completions.
|
||||
const openai = createOpenAI({ apiKey, ...(baseURL ? { baseURL } : {}) });
|
||||
return (id) => openai.chat(id);
|
||||
}
|
||||
default:
|
||||
console.error(
|
||||
`Error: unsupported provider "${provider}". Supported: google, anthropic, openai.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a model from a triple of env vars (model / api key / base url).
|
||||
* Returns null when the model var is unset (used for the optional judge
|
||||
* override). `required` exits the process on a missing model/key. */
|
||||
function modelFromEnv(
|
||||
modelVar: string,
|
||||
keyVar: string,
|
||||
baseVar: string,
|
||||
fallbackKeyVar: string,
|
||||
fallbackBaseVar: string,
|
||||
required: boolean,
|
||||
): LanguageModel | null {
|
||||
const raw = process.env[modelVar];
|
||||
if (!raw) {
|
||||
if (required) {
|
||||
console.error(
|
||||
`Error: ${modelVar} must be set. Example: ${modelVar}=google:gemini-3-flash-preview`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const apiKey = process.env[keyVar] || process.env[fallbackKeyVar];
|
||||
if (!apiKey) {
|
||||
console.error(`Error: ${keyVar} (or ${fallbackKeyVar}) must be set.`);
|
||||
process.exit(1);
|
||||
}
|
||||
const baseURL = process.env[baseVar] || process.env[fallbackBaseVar] || undefined;
|
||||
const { provider, modelId } = parseModelString(raw);
|
||||
return makeProvider(provider, apiKey, baseURL)(modelId);
|
||||
}
|
||||
|
||||
function createModel(): LanguageModel {
|
||||
return modelFromEnv(
|
||||
'EVAL_PBL_MODEL',
|
||||
'EVAL_PBL_API_KEY',
|
||||
'EVAL_PBL_BASE_URL',
|
||||
'EVAL_PBL_API_KEY',
|
||||
'EVAL_PBL_BASE_URL',
|
||||
true,
|
||||
)!;
|
||||
}
|
||||
|
||||
/** Judge model. Defaults to the generation model unless EVAL_PBL_JUDGE_MODEL
|
||||
* is set (recommended: a strong, independent model so a weak generator does
|
||||
* not grade its own homework). */
|
||||
function createJudgeModel(genModel: LanguageModel): LanguageModel {
|
||||
return (
|
||||
modelFromEnv(
|
||||
'EVAL_PBL_JUDGE_MODEL',
|
||||
'EVAL_PBL_JUDGE_API_KEY',
|
||||
'EVAL_PBL_JUDGE_BASE_URL',
|
||||
'EVAL_PBL_API_KEY',
|
||||
'EVAL_PBL_BASE_URL',
|
||||
false,
|
||||
) ?? genModel
|
||||
);
|
||||
}
|
||||
|
||||
function createThinkingConfig(): ThinkingConfig | undefined {
|
||||
const thinking = process.env.EVAL_PBL_THINKING;
|
||||
if (!thinking || thinking === 'false') return undefined;
|
||||
|
||||
const budget = parseInt(process.env.EVAL_PBL_THINKING_BUDGET || '1024', 10);
|
||||
return {
|
||||
enabled: true,
|
||||
mode: 'enabled',
|
||||
budgetTokens: budget,
|
||||
};
|
||||
}
|
||||
|
||||
function selectedVariants(): Variant[] {
|
||||
const raw = process.env.EVAL_PBL_VARIANTS;
|
||||
if (!raw) return ['loop', 'single-call'];
|
||||
const parsed = raw
|
||||
.split(',')
|
||||
.map((v) => v.trim())
|
||||
.filter((v): v is Variant => v === 'loop' || v === 'single-call');
|
||||
return parsed.length > 0 ? parsed : ['loop', 'single-call'];
|
||||
}
|
||||
|
||||
function judgeEnabled(): boolean {
|
||||
return process.env.EVAL_PBL_JUDGE !== 'false';
|
||||
}
|
||||
|
||||
function loadTestCases(): TestCase[] {
|
||||
const path = join(getCurrentDir(), 'scenarios', 'test-cases.json');
|
||||
return JSON.parse(readFileSync(path, 'utf-8')) as TestCase[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Outline builder
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let outlineCounter = 0;
|
||||
|
||||
function buildOutline(tc: TestCase): SceneOutline {
|
||||
outlineCounter += 1;
|
||||
return {
|
||||
id: `eval-pbl-${tc.id}`,
|
||||
type: 'pbl',
|
||||
title: tc.pblConfig.projectTopic,
|
||||
description: tc.pblConfig.projectDescription,
|
||||
keyPoints: tc.pblConfig.targetSkills.map((s) => `Learn ${s}`),
|
||||
teachingObjective: `完成 ${tc.pblConfig.projectTopic} 项目`,
|
||||
order: outlineCounter,
|
||||
pblConfig: tc.pblConfig,
|
||||
languageNote: tc.languageDirective,
|
||||
};
|
||||
}
|
||||
|
||||
function buildInput(tc: TestCase, outline: SceneOutline): PBLPlannerV2Input {
|
||||
return {
|
||||
outline,
|
||||
courseContext: {
|
||||
allOutlines: [outline],
|
||||
languageDirective: tc.languageDirective,
|
||||
},
|
||||
user: {
|
||||
requirement: tc.requirement,
|
||||
},
|
||||
targetLanguage: 'zh-CN',
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Validation helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function checkCompletionGate(project: PBLProjectV2): boolean {
|
||||
if (!project.title || !project.description) return false;
|
||||
if (!project.roles.some((r) => r.type === 'instructor')) return false;
|
||||
if (project.milestones.length === 0) return false;
|
||||
return project.milestones.every((m) => m.microtasks.length > 0);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LLM judge
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Compact, judge-facing view of a project (drops ids/timestamps/runtime).
|
||||
* Surfaces the scenario block + per-beat fields when the project is a
|
||||
* role-play scenario, so the judge can apply its scenario-specific rules. */
|
||||
function projectForJudge(project: PBLProjectV2): unknown {
|
||||
const instructor = project.roles.find((r) => r.type === 'instructor');
|
||||
const scenario = project.scenario
|
||||
? {
|
||||
setting: project.scenario.setting,
|
||||
goal: project.scenario.goal,
|
||||
rules: project.scenario.rules,
|
||||
learnerRole: project.scenario.learnerRole,
|
||||
characters: project.scenario.characters.map((c) => ({
|
||||
name: c.name,
|
||||
persona: c.persona,
|
||||
situation: c.situation,
|
||||
boundaries: c.boundaries,
|
||||
openingLine: c.openingLine,
|
||||
})),
|
||||
}
|
||||
: undefined;
|
||||
return {
|
||||
title: project.title,
|
||||
description: project.description,
|
||||
learningObjective: project.learningObjective,
|
||||
proficiency: project.proficiency,
|
||||
...(scenario ? { scenario } : {}),
|
||||
instructor: instructor ? { name: instructor.name, description: instructor.description } : null,
|
||||
milestones: project.milestones.map((m) => ({
|
||||
title: m.title,
|
||||
description: m.description,
|
||||
briefing: m.briefing,
|
||||
completionCriteria: m.completionCriteria,
|
||||
debrief: m.debrief,
|
||||
coreConcept: m.synthesisCheck?.coreConcept,
|
||||
...(m.scenarioStage ? { scenarioStage: m.scenarioStage } : {}),
|
||||
microtasks: m.microtasks.map((t) => ({
|
||||
title: t.title,
|
||||
description: t.description,
|
||||
hints: t.hints,
|
||||
...(t.successWhen ? { successWhen: t.successWhen } : {}),
|
||||
...(t.characterObjective ? { characterObjective: t.characterObjective } : {}),
|
||||
...(t.skillFocus ? { skillFocus: t.skillFocus } : {}),
|
||||
...(t.narration ? { narration: t.narration } : {}),
|
||||
})),
|
||||
documents: (m.documents ?? []).map((d) => ({ title: d.title })),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
let _judgeTemplate: string | undefined;
|
||||
let _judgeTemplateScenario: string | undefined;
|
||||
let _completabilityJudgeTemplate: string | undefined;
|
||||
/** Role-play scenario projects are graded by a separate rubric
|
||||
* (`judge-prompt-scenario.md`); everything else uses `judge-prompt.md`. */
|
||||
function judgeTemplate(isScenario: boolean): string {
|
||||
if (isScenario) {
|
||||
if (_judgeTemplateScenario === undefined) {
|
||||
_judgeTemplateScenario = readFileSync(
|
||||
join(getCurrentDir(), 'judge-prompt-scenario.md'),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
return _judgeTemplateScenario;
|
||||
}
|
||||
if (_judgeTemplate === undefined) {
|
||||
_judgeTemplate = readFileSync(join(getCurrentDir(), 'judge-prompt.md'), 'utf-8');
|
||||
}
|
||||
return _judgeTemplate;
|
||||
}
|
||||
|
||||
function completabilityJudgeTemplate(): string {
|
||||
if (_completabilityJudgeTemplate === undefined) {
|
||||
_completabilityJudgeTemplate = readFileSync(
|
||||
join(getCurrentDir(), 'judge-prompt-completability.md'),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
return _completabilityJudgeTemplate;
|
||||
}
|
||||
|
||||
async function judgeProject(
|
||||
project: PBLProjectV2,
|
||||
tc: TestCase,
|
||||
model: LanguageModel,
|
||||
): Promise<JudgeScores | undefined> {
|
||||
try {
|
||||
const prompt = judgeTemplate(!!project.scenario)
|
||||
.replace('{{topic}}', tc.pblConfig.projectTopic)
|
||||
.replace('{{description}}', tc.pblConfig.projectDescription)
|
||||
.replace('{{targetSkills}}', tc.pblConfig.targetSkills.join(', '))
|
||||
.replace('{{proficiency}}', project.proficiency || 'intermediate')
|
||||
.replace('{{project}}', JSON.stringify(projectForJudge(project), null, 2));
|
||||
|
||||
const { text } = await generateText({ model, prompt });
|
||||
const scores = parseJsonResponse<JudgeScores>(text);
|
||||
if (!scores || typeof scores.overall !== 'number' || typeof scores.scores !== 'object') {
|
||||
return undefined;
|
||||
}
|
||||
if (!Array.isArray(scores.redLines)) scores.redLines = [];
|
||||
return scores;
|
||||
} catch (err) {
|
||||
console.log(` judge failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function judgeCompletability(
|
||||
project: PBLProjectV2,
|
||||
tc: TestCase,
|
||||
model: LanguageModel,
|
||||
): Promise<CompletabilityJudge | undefined> {
|
||||
try {
|
||||
const prompt = completabilityJudgeTemplate()
|
||||
.replace('{{topic}}', tc.pblConfig.projectTopic)
|
||||
.replace('{{description}}', tc.pblConfig.projectDescription)
|
||||
.replace('{{targetSkills}}', tc.pblConfig.targetSkills.join(', '))
|
||||
.replace('{{proficiency}}', project.proficiency || 'intermediate')
|
||||
.replace('{{project}}', JSON.stringify(projectForJudge(project), null, 2));
|
||||
|
||||
const { text } = await generateText({ model, prompt });
|
||||
const result = parseJsonResponse<CompletabilityJudge>(text);
|
||||
if (
|
||||
!result ||
|
||||
typeof result.score !== 'number' ||
|
||||
typeof result.pass !== 'boolean' ||
|
||||
!['low', 'medium', 'high'].includes(result.riskLevel)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
if (!Array.isArray(result.blockers)) result.blockers = [];
|
||||
return result;
|
||||
} catch (err) {
|
||||
console.log(
|
||||
` completability judge failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Runner
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function runVariant(
|
||||
variant: Variant,
|
||||
input: PBLPlannerV2Input,
|
||||
model: LanguageModel,
|
||||
thinkingConfig?: ThinkingConfig,
|
||||
): Promise<PBLProjectV2> {
|
||||
return variant === 'single-call'
|
||||
? generatePBLV2ProjectSingleCall(input, model, undefined, thinkingConfig)
|
||||
: generatePBLV2Project(input, model, undefined, thinkingConfig);
|
||||
}
|
||||
|
||||
async function runOne(
|
||||
tc: TestCase,
|
||||
variant: Variant,
|
||||
model: LanguageModel,
|
||||
judgeModel: LanguageModel,
|
||||
thinkingConfig?: ThinkingConfig,
|
||||
): Promise<RunResult> {
|
||||
const outline = buildOutline(tc);
|
||||
const input = buildInput(tc, outline);
|
||||
const isScenario = tc.pblConfig.scenarioRoleplay === true;
|
||||
const startedAt = performance.now();
|
||||
|
||||
try {
|
||||
const project = await runVariant(variant, input, model, thinkingConfig);
|
||||
const durationMs = Math.round(performance.now() - startedAt);
|
||||
const microtaskCount = project.milestones.reduce((sum, m) => sum + m.microtasks.length, 0);
|
||||
const passesCompletionGate = checkCompletionGate(project);
|
||||
|
||||
const [completability, judge] =
|
||||
judgeEnabled() && passesCompletionGate
|
||||
? await Promise.all([
|
||||
judgeCompletability(project, tc, judgeModel),
|
||||
judgeProject(project, tc, judgeModel),
|
||||
])
|
||||
: [undefined, undefined];
|
||||
|
||||
return {
|
||||
caseId: tc.id,
|
||||
variant,
|
||||
ok: true,
|
||||
milestoneCount: project.milestones.length,
|
||||
microtaskCount,
|
||||
roleCount: project.roles.length,
|
||||
durationMs,
|
||||
passesCompletionGate,
|
||||
isScenario,
|
||||
completability,
|
||||
judge,
|
||||
project,
|
||||
};
|
||||
} catch (err) {
|
||||
const durationMs = Math.round(performance.now() - startedAt);
|
||||
const msg =
|
||||
err instanceof PlannerV2Error
|
||||
? `PlannerV2Error: ${err.message}`
|
||||
: err instanceof Error
|
||||
? err.message
|
||||
: String(err);
|
||||
return {
|
||||
caseId: tc.id,
|
||||
variant,
|
||||
ok: false,
|
||||
milestoneCount: 0,
|
||||
microtaskCount: 0,
|
||||
roleCount: 0,
|
||||
durationMs,
|
||||
error: msg,
|
||||
passesCompletionGate: false,
|
||||
isScenario,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Report
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function formatDuration(ms: number): string {
|
||||
if (ms < 1000) return `${ms}ms`;
|
||||
return `${(ms / 1000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
function avg(nums: number[]): number {
|
||||
return nums.length === 0 ? 0 : nums.reduce((a, b) => a + b, 0) / nums.length;
|
||||
}
|
||||
|
||||
function meanDims(j: JudgeScores): number {
|
||||
return avg(JUDGE_DIMENSIONS.map((d) => j.scores[d] ?? 0));
|
||||
}
|
||||
|
||||
function completabilityBlockerList(j: CompletabilityJudge): string {
|
||||
return j.blockers.length > 0 ? j.blockers.join(',') : 'none';
|
||||
}
|
||||
|
||||
function hasCompletabilityFailure(r: RunResult): boolean {
|
||||
return r.completability?.pass === false;
|
||||
}
|
||||
|
||||
/** Split results into the two grading categories (normal / scenario),
|
||||
* dropping any category with no cases so reports stay clean. */
|
||||
function splitByCategory(results: RunResult[]): Array<{ label: string; results: RunResult[] }> {
|
||||
const normal = results.filter((r) => !r.isScenario);
|
||||
const scenario = results.filter((r) => r.isScenario);
|
||||
const out: Array<{ label: string; results: RunResult[] }> = [];
|
||||
if (normal.length) out.push({ label: 'normal', results: normal });
|
||||
if (scenario.length) out.push({ label: 'scenario', results: scenario });
|
||||
return out;
|
||||
}
|
||||
|
||||
function variantSummary(results: RunResult[], variant: Variant): string {
|
||||
const rs = results.filter((r) => r.variant === variant);
|
||||
const total = rs.length;
|
||||
const ok = rs.filter((r) => r.ok).length;
|
||||
const gate = rs.filter((r) => r.passesCompletionGate).length;
|
||||
const completability = rs
|
||||
.map((r) => r.completability)
|
||||
.filter((j): j is CompletabilityJudge => !!j);
|
||||
const completable = completability.filter((j) => j.pass).length;
|
||||
const blockerRuns = completability.filter((j) => j.blockers.length > 0).length;
|
||||
const judged = rs.map((r) => r.judge).filter((j): j is JudgeScores => !!j);
|
||||
const redLineRuns = judged.filter((j) => j.redLines.length > 0).length;
|
||||
const completionLine =
|
||||
completability.length > 0
|
||||
? `complete(pass=${completable}/${completability.length}, score=${avg(
|
||||
completability.map((j) => j.score),
|
||||
).toFixed(2)}, blocker-runs=${blockerRuns}/${completability.length})`
|
||||
: 'complete(n/a)';
|
||||
const judgeLine =
|
||||
judged.length > 0
|
||||
? `judge(overall=${avg(judged.map((j) => j.overall)).toFixed(2)}, dims=${avg(
|
||||
judged.map(meanDims),
|
||||
).toFixed(2)}, redline-runs=${redLineRuns}/${judged.length})`
|
||||
: 'judge(n/a)';
|
||||
return ` ${variant.padEnd(12)} success ${ok}/${total} | gate ${gate}/${total} | avg ${formatDuration(
|
||||
Math.round(avg(rs.map((r) => r.durationMs))),
|
||||
)} | ${completionLine} | ${judgeLine}`;
|
||||
}
|
||||
|
||||
function renderRows(results: RunResult[], variants: Variant[]): string {
|
||||
const header = [
|
||||
'Case',
|
||||
'Variant',
|
||||
'Status',
|
||||
'MS',
|
||||
'MT',
|
||||
'Dur',
|
||||
'Comp',
|
||||
'Blockers',
|
||||
'Overall',
|
||||
'RedLines',
|
||||
].join(' | ');
|
||||
const sep = '------|---------|--------|----|----|------|------|----------|--------|--------';
|
||||
const caseIds = [...new Set(results.map((r) => r.caseId))];
|
||||
const rows: string[] = [];
|
||||
for (const caseId of caseIds) {
|
||||
for (const variant of variants) {
|
||||
const r = results.find((x) => x.caseId === caseId && x.variant === variant);
|
||||
if (!r) continue;
|
||||
const status = r.ok ? (r.passesCompletionGate ? '✓ OK' : '⚠ gate') : '✗ FAIL';
|
||||
rows.push(
|
||||
[
|
||||
r.caseId.padEnd(26),
|
||||
variant.padEnd(11),
|
||||
status.padEnd(8),
|
||||
String(r.milestoneCount).padEnd(4),
|
||||
String(r.microtaskCount).padEnd(4),
|
||||
formatDuration(r.durationMs).padEnd(6),
|
||||
r.completability
|
||||
? `${r.completability.pass ? 'PASS' : 'FAIL'} ${r.completability.score.toFixed(1)}`
|
||||
: '-',
|
||||
r.completability ? completabilityBlockerList(r.completability) : '-',
|
||||
r.judge ? r.judge.overall.toFixed(1) : '-',
|
||||
r.judge ? r.judge.redLines.join(',') || '—' : '-',
|
||||
].join(' | '),
|
||||
);
|
||||
}
|
||||
}
|
||||
return [header, sep, ...rows].join('\n');
|
||||
}
|
||||
|
||||
function printReport(results: RunResult[], variants: Variant[], modelStr: string): void {
|
||||
console.log('');
|
||||
console.log('═'.repeat(96));
|
||||
console.log(' PBL v2 Planner — A/B Harness Report');
|
||||
console.log('═'.repeat(96));
|
||||
console.log(` Model: ${modelStr}`);
|
||||
console.log('─'.repeat(96));
|
||||
for (const cat of splitByCategory(results)) {
|
||||
console.log(
|
||||
` ── ${cat.label} (${[...new Set(cat.results.map((r) => r.caseId))].length} case(s)) ──`,
|
||||
);
|
||||
for (const variant of variants) console.log(variantSummary(cat.results, variant));
|
||||
}
|
||||
console.log('─'.repeat(96));
|
||||
console.log(renderRows(results, variants));
|
||||
console.log('─'.repeat(96));
|
||||
|
||||
const failures = results.filter(
|
||||
(r) => !r.ok || !r.passesCompletionGate || hasCompletabilityFailure(r),
|
||||
);
|
||||
if (failures.length > 0) {
|
||||
console.log(' Failures:');
|
||||
for (const f of failures) {
|
||||
const reason = f.error
|
||||
? f.error
|
||||
: !f.passesCompletionGate
|
||||
? 'gate fail (incomplete project)'
|
||||
: f.completability
|
||||
? `completability fail (${completabilityBlockerList(f.completability)})`
|
||||
: 'completability fail';
|
||||
console.log(` ${f.caseId} [${f.variant}]: ${reason}`);
|
||||
}
|
||||
}
|
||||
console.log('═'.repeat(96));
|
||||
console.log('');
|
||||
}
|
||||
|
||||
/** Summary table rows (one per variant) for a result subset. */
|
||||
function summaryTableLines(results: RunResult[], variants: Variant[]): string[] {
|
||||
const lines: string[] = [
|
||||
'| Variant | Success | Gate | Completable | Comp score | Blocker runs | Avg dur | Overall | Dims avg | Red-line runs |',
|
||||
'|---------|---------|------|-------------|------------|--------------|---------|---------|----------|---------------|',
|
||||
];
|
||||
for (const variant of variants) {
|
||||
const rs = results.filter((r) => r.variant === variant);
|
||||
const total = rs.length;
|
||||
if (total === 0) continue;
|
||||
const ok = rs.filter((r) => r.ok).length;
|
||||
const gate = rs.filter((r) => r.passesCompletionGate).length;
|
||||
const completability = rs
|
||||
.map((r) => r.completability)
|
||||
.filter((j): j is CompletabilityJudge => !!j);
|
||||
const completable = completability.filter((j) => j.pass).length;
|
||||
const blockerRuns = completability.filter((j) => j.blockers.length > 0).length;
|
||||
const judged = rs.map((r) => r.judge).filter((j): j is JudgeScores => !!j);
|
||||
const redLineRuns = judged.filter((j) => j.redLines.length > 0).length;
|
||||
lines.push(
|
||||
`| ${variant} | ${ok}/${total} | ${gate}/${total} | ${
|
||||
completability.length ? `${completable}/${completability.length}` : '-'
|
||||
} | ${completability.length ? avg(completability.map((j) => j.score)).toFixed(2) : '-'} | ${
|
||||
completability.length ? `${blockerRuns}/${completability.length}` : '-'
|
||||
} | ${formatDuration(
|
||||
Math.round(avg(rs.map((r) => r.durationMs))),
|
||||
)} | ${judged.length ? avg(judged.map((j) => j.overall)).toFixed(2) : '-'} | ${
|
||||
judged.length ? avg(judged.map(meanDims)).toFixed(2) : '-'
|
||||
} | ${judged.length ? `${redLineRuns}/${judged.length}` : '-'} |`,
|
||||
);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Per-dimension averages table for a result subset. */
|
||||
function perDimensionLines(results: RunResult[], variants: Variant[]): string[] {
|
||||
const lines: string[] = [
|
||||
`| Dimension | ${variants.join(' | ')} |`,
|
||||
`|-----------|${variants.map(() => '---').join('|')}|`,
|
||||
];
|
||||
for (const dim of JUDGE_DIMENSIONS) {
|
||||
const cells = variants.map((variant) => {
|
||||
const judged = results
|
||||
.filter((r) => r.variant === variant)
|
||||
.map((r) => r.judge)
|
||||
.filter((j): j is JudgeScores => !!j);
|
||||
return judged.length ? avg(judged.map((j) => j.scores[dim] ?? 0)).toFixed(2) : '-';
|
||||
});
|
||||
lines.push(`| ${dim} | ${cells.join(' | ')} |`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function writeMarkdownReport(
|
||||
results: RunResult[],
|
||||
variants: Variant[],
|
||||
modelStr: string,
|
||||
judgeStr: string,
|
||||
thinkingConfig: ThinkingConfig | undefined,
|
||||
runDir: string,
|
||||
): void {
|
||||
const categories = splitByCategory(results);
|
||||
const lines: string[] = [
|
||||
`# PBL v2 Planner — A/B Harness Report`,
|
||||
'',
|
||||
`- **Generation model**: ${modelStr}`,
|
||||
`- **Judge model**: ${judgeStr}`,
|
||||
`- **Thinking**: ${thinkingConfig?.enabled ? `on (budget: ${thinkingConfig.budgetTokens ?? 'default'})` : 'off'}`,
|
||||
`- **Variants**: ${variants.join(', ')}`,
|
||||
`- **Categories**: ${categories.map((c) => `${c.label} (${[...new Set(c.results.map((r) => r.caseId))].length})`).join(', ')}`,
|
||||
'',
|
||||
`> Runtime completability is graded first by \`judge-prompt-completability.md\`. Normal project quality is graded by \`judge-prompt.md\`; role-play scenario quality by \`judge-prompt-scenario.md\`. Quality scores share the same 12 dimension keys but are NOT directly comparable across categories.`,
|
||||
];
|
||||
|
||||
for (const cat of categories) {
|
||||
lines.push(
|
||||
'',
|
||||
`## ${cat.label === 'scenario' ? 'Scenario' : 'Normal'} projects`,
|
||||
'',
|
||||
'### Summary',
|
||||
'',
|
||||
...summaryTableLines(cat.results, variants),
|
||||
'',
|
||||
'### Per-dimension averages (1-5)',
|
||||
'',
|
||||
...perDimensionLines(cat.results, variants),
|
||||
);
|
||||
}
|
||||
|
||||
lines.push('', '## Per-case', '');
|
||||
lines.push(
|
||||
'| Case | Cat | Variant | Status | MS | MT | Dur | Completability | Blockers | Overall | RedLines | Rationale / Error |',
|
||||
);
|
||||
lines.push(
|
||||
'|------|-----|---------|--------|----|----|-----|---------------|----------|---------|----------|-------------------|',
|
||||
);
|
||||
const caseIds = [...new Set(results.map((r) => r.caseId))];
|
||||
for (const caseId of caseIds) {
|
||||
for (const variant of variants) {
|
||||
const r = results.find((x) => x.caseId === caseId && x.variant === variant);
|
||||
if (!r) continue;
|
||||
const status = r.ok ? (r.passesCompletionGate ? '✓' : '⚠ gate') : '✗';
|
||||
const comp = r.completability
|
||||
? `${r.completability.pass ? 'PASS' : 'FAIL'} ${r.completability.score.toFixed(1)}`
|
||||
: '-';
|
||||
const note = (r.error ? r.error : (r.completability?.rationale ?? r.judge?.rationale ?? ''))
|
||||
.slice(0, 120)
|
||||
.replace(/\|/g, '/');
|
||||
lines.push(
|
||||
`| ${r.caseId} | ${r.isScenario ? 'S' : 'N'} | ${variant} | ${status} | ${r.milestoneCount} | ${r.microtaskCount} | ${formatDuration(
|
||||
r.durationMs,
|
||||
)} | ${comp} | ${r.completability ? completabilityBlockerList(r.completability) : '-'} | ${
|
||||
r.judge ? r.judge.overall.toFixed(1) : '-'
|
||||
} | ${r.judge ? r.judge.redLines.join(',') || '—' : '-'} | ${note} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
writeFileSync(join(runDir, 'report.md'), lines.join('\n'), 'utf-8');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const modelStr = process.env.EVAL_PBL_MODEL!;
|
||||
const judgeStr = process.env.EVAL_PBL_JUDGE_MODEL || `${modelStr} (self)`;
|
||||
const model = createModel();
|
||||
const judgeModel = createJudgeModel(model);
|
||||
const thinkingConfig = createThinkingConfig();
|
||||
const variants = selectedVariants();
|
||||
|
||||
const allCases = loadTestCases();
|
||||
const filter = process.env.EVAL_PBL_FILTER;
|
||||
const filtered = filter ? allCases.filter((c) => c.id.includes(filter)) : allCases;
|
||||
const maxRuns = parseInt(process.env.EVAL_PBL_RUNS || String(filtered.length), 10);
|
||||
const testCases = filtered.slice(0, maxRuns);
|
||||
|
||||
// Compute the run dir up front so projects can be dumped as they finish
|
||||
// (partial results survive a crash on a long run).
|
||||
const sanitizedModel = modelStr.replace(/[:/]/g, '-');
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
||||
const runDir = join(getCurrentDir(), 'results', sanitizedModel, timestamp);
|
||||
const projectsDir = join(runDir, 'projects');
|
||||
mkdirSync(projectsDir, { recursive: true });
|
||||
|
||||
const concurrency = parseInt(process.env.EVAL_PBL_CONCURRENCY || '10', 10);
|
||||
const staggerMs = parseInt(process.env.EVAL_PBL_STAGGER_MS || '1000', 10);
|
||||
console.log(
|
||||
`\nHarness: ${testCases.length} case(s) × ${variants.length} variant(s) [${variants.join(', ')}]`,
|
||||
);
|
||||
console.log(`Generation model: ${modelStr}`);
|
||||
console.log(`Judge model: ${judgeEnabled() ? judgeStr : 'off'}`);
|
||||
console.log(
|
||||
`Thinking: ${thinkingConfig?.enabled ? `on (budget: ${thinkingConfig.budgetTokens ?? 'default'})` : 'off'}`,
|
||||
);
|
||||
console.log(`Concurrency: ${concurrency} | stagger: ${staggerMs}ms`);
|
||||
console.log(`Output dir: ${runDir}`);
|
||||
console.log('');
|
||||
|
||||
// Flat job list (case-major: both variants of a case adjacent).
|
||||
const jobs: Array<{ tc: TestCase; variant: Variant; n: number }> = [];
|
||||
for (let i = 0; i < testCases.length; i++) {
|
||||
for (const variant of variants) jobs.push({ tc: testCases[i], variant, n: i + 1 });
|
||||
}
|
||||
const total = jobs.length;
|
||||
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
|
||||
|
||||
// Run with a concurrency cap of `concurrency` and a `staggerMs` gap between
|
||||
// successive launches (so calls don't all hit the gateway at once).
|
||||
const results: RunResult[] = new Array(total);
|
||||
const executing = new Set<Promise<void>>();
|
||||
let done = 0;
|
||||
for (let j = 0; j < jobs.length; j++) {
|
||||
const { tc, variant } = jobs[j];
|
||||
if (j > 0) await sleep(staggerMs);
|
||||
const p = (async () => {
|
||||
const result = await runOne(tc, variant, model, judgeModel, thinkingConfig);
|
||||
results[j] = result;
|
||||
|
||||
if (result.project) {
|
||||
writeFileSync(
|
||||
join(projectsDir, `${tc.id}__${variant}.json`),
|
||||
JSON.stringify(result.project, null, 2),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
done += 1;
|
||||
const tag = `[${done}/${total}] ${tc.id} [${variant}]`;
|
||||
if (result.ok && result.passesCompletionGate) {
|
||||
const completion = result.completability
|
||||
? ` complete=${result.completability.pass ? 'PASS' : 'FAIL'}:${result.completability.score.toFixed(
|
||||
1,
|
||||
)}${
|
||||
result.completability.blockers.length > 0
|
||||
? `:${result.completability.blockers.join(',')}`
|
||||
: ''
|
||||
}`
|
||||
: '';
|
||||
const redline =
|
||||
result.judge && result.judge.redLines.length > 0
|
||||
? ` ⛔${result.judge.redLines.join(',')}`
|
||||
: '';
|
||||
process.stdout.write(
|
||||
` ${tag} ✓ ${result.milestoneCount}ms ${result.microtaskCount}mt${completion}${result.judge ? ` judge=${result.judge.overall.toFixed(1)}${redline}` : ''} (${formatDuration(result.durationMs)})\n`,
|
||||
);
|
||||
} else if (result.ok) {
|
||||
process.stdout.write(` ${tag} ⚠ gate fail (${formatDuration(result.durationMs)})\n`);
|
||||
} else {
|
||||
process.stdout.write(` ${tag} ✗ FAIL (${formatDuration(result.durationMs)})\n`);
|
||||
if (result.error) console.log(` ${result.error.slice(0, 120)}`);
|
||||
}
|
||||
})();
|
||||
const tracked = p.finally(() => executing.delete(tracked));
|
||||
executing.add(tracked);
|
||||
if (executing.size >= concurrency) await Promise.race(executing);
|
||||
}
|
||||
await Promise.all(executing);
|
||||
|
||||
printReport(results, variants, modelStr);
|
||||
writeMarkdownReport(results, variants, modelStr, judgeStr, thinkingConfig, runDir);
|
||||
|
||||
// Dump metrics (judge scores etc, minus the full project) for the
|
||||
// compare page, then build the self-contained side-by-side HTML.
|
||||
const slim = results.map(({ project: _project, ...rest }) => rest);
|
||||
writeFileSync(join(runDir, 'results.json'), JSON.stringify(slim, null, 2), 'utf-8');
|
||||
const compareHtml = buildCompareHtml(runDir);
|
||||
|
||||
console.log(`Report saved: ${runDir}/report.md`);
|
||||
console.log(`Projects dumped: ${projectsDir}/<case>__<variant>.json`);
|
||||
console.log(`Compare page: file://${compareHtml}\n`);
|
||||
|
||||
const allPassed = results.every(
|
||||
(r) =>
|
||||
r.ok &&
|
||||
r.passesCompletionGate &&
|
||||
(!judgeEnabled() || (r.completability !== undefined && r.completability.pass)),
|
||||
);
|
||||
process.exit(allPassed ? 0 : 1);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Harness crashed:', err);
|
||||
process.exit(2);
|
||||
});
|
||||
@@ -0,0 +1,296 @@
|
||||
[
|
||||
{
|
||||
"id": "python-list-beginner",
|
||||
"requirement": "我要用PBL项目制学习来学python中的列表，最最最简单的版本，课程短一点，最多只要8个场景",
|
||||
"pblConfig": {
|
||||
"projectTopic": "Python列表入门：水果清单小程序",
|
||||
"projectDescription": "通过制作一个水果清单小程序来学习Python列表的基本操作",
|
||||
"targetSkills": ["Python list", "index", "append", "基本数据操作"],
|
||||
"issueCount": 3
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "python-set-beginner",
|
||||
"requirement": "我要用PBL项目制学习来学python中的set，最最最简单的版本，课程短一点，最多只要8个场景",
|
||||
"pblConfig": {
|
||||
"projectTopic": "Python set 水果清单去重助手",
|
||||
"projectDescription": "用set来帮水果店老板去掉重复的水果订单",
|
||||
"targetSkills": ["Python set", "去重", "add", "discard"],
|
||||
"issueCount": 4
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "python-dict-beginner",
|
||||
"requirement": "我要用PBL项目制学习来学python中的字典，最最最简单的版本，课程短一点，最多只要8个场景",
|
||||
"pblConfig": {
|
||||
"projectTopic": "用Python字典制作学生信息卡",
|
||||
"projectDescription": "用字典来存储和查询学生信息",
|
||||
"targetSkills": ["Python dict", "key-value", "读取和修改"],
|
||||
"issueCount": 3
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "python-boolean-beginner",
|
||||
"requirement": "我要用PBL项目制学习来学python中的布尔值，最最最简单的版本，课程短一点，最多只要8个场景",
|
||||
"pblConfig": {
|
||||
"projectTopic": "Python 布尔值门禁判断器",
|
||||
"projectDescription": "用布尔值做一个简单的门禁判断程序",
|
||||
"targetSkills": ["Python bool", "True/False", "比较运算", "if判断"],
|
||||
"issueCount": 3
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "python-number-beginner",
|
||||
"requirement": "我要用PBL项目制学习来学python中的数字和运算，最最最简单的版本，课程短一点，最多只要8个场景",
|
||||
"pblConfig": {
|
||||
"projectTopic": "Python 数字小助手",
|
||||
"projectDescription": "用Python做加减乘除计算器",
|
||||
"targetSkills": ["Python number", "加减乘除", "变量", "基本运算"],
|
||||
"issueCount": 3
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "algorithm-two-pointer",
|
||||
"requirement": "我要用PBL项目制学习来学快慢指针算法，最最最简单的版本，课程短一点，最多只要8个场景",
|
||||
"pblConfig": {
|
||||
"projectTopic": "用快慢指针判断链表是否有环",
|
||||
"projectDescription": "学习快慢指针算法来判断链表中的环",
|
||||
"targetSkills": ["快慢指针", "链表", "算法思维"],
|
||||
"issueCount": 3
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "python-string-beginner",
|
||||
"requirement": "我要用PBL项目制学习来学python中的字符串处理，最最最简单的版本，课程短一点",
|
||||
"pblConfig": {
|
||||
"projectTopic": "Python 字符串姓名处理器",
|
||||
"projectDescription": "用Python字符串方法处理姓名格式",
|
||||
"targetSkills": ["Python string", "split", "join", "upper/lower"],
|
||||
"issueCount": 3
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "python-loop-beginner",
|
||||
"requirement": "我要用PBL项目制学习来学python中的for循环，最最最简单的版本，课程短一点",
|
||||
"pblConfig": {
|
||||
"projectTopic": "Python for循环成绩统计器",
|
||||
"projectDescription": "用for循环来统计学生成绩",
|
||||
"targetSkills": ["Python for loop", "遍历", "累加", "条件判断"],
|
||||
"issueCount": 4
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "python-string-processing",
|
||||
"requirement": "我要用PBL项目制学习来学python中的字符串处理",
|
||||
"pblConfig": {
|
||||
"projectTopic": "Python 字符串处理：评论文本清洗小工具",
|
||||
"projectDescription": "用 Python 字符串方法做一个把杂乱评论文本清洗成规整格式的小工具",
|
||||
"targetSkills": [
|
||||
"Python string",
|
||||
"split/join",
|
||||
"strip/replace",
|
||||
"大小写与查找",
|
||||
"格式化输出"
|
||||
],
|
||||
"issueCount": 3
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "rag-qa-assistant",
|
||||
"requirement": "我要用PBL项目制学习来学RAG",
|
||||
"pblConfig": {
|
||||
"projectTopic": "搭建一个基于自有文档的 RAG 问答助手",
|
||||
"projectDescription": "从零搭建一个检索增强生成（RAG）问答系统，能基于自己的文档检索片段并生成有依据的回答",
|
||||
"targetSkills": [
|
||||
"文档切分与向量化",
|
||||
"向量检索",
|
||||
"上下文拼接",
|
||||
"提示词构造",
|
||||
"调用大模型生成答案",
|
||||
"效果评估"
|
||||
],
|
||||
"issueCount": 5
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "dynamic-programming-knapsack",
|
||||
"requirement": "我要用PBL项目制学习来学动态规划算法",
|
||||
"pblConfig": {
|
||||
"projectTopic": "用动态规划求解背包问题的最优装包方案",
|
||||
"projectDescription": "通过解决经典 0/1 背包问题，掌握动态规划的状态定义、转移方程与最优子结构",
|
||||
"targetSkills": [
|
||||
"识别最优子结构",
|
||||
"状态定义",
|
||||
"状态转移方程",
|
||||
"自底向上求解",
|
||||
"时间空间复杂度分析"
|
||||
],
|
||||
"issueCount": 4
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "business-startup-proposal",
|
||||
"requirement": "我想通过做项目的方式，学会写一份能拿去参加创业大赛的项目申报方案",
|
||||
"pblConfig": {
|
||||
"projectTopic": "校园二手交易平台创业申报方案",
|
||||
"projectDescription": "面向大学生市场，完成一份可提交创业大赛的项目申报方案，含问题、用户、解决方案、商业模式与可行性",
|
||||
"targetSkills": ["问题定义", "用户与市场分析", "商业模式设计", "可行性论证", "方案表达"],
|
||||
"issueCount": 4
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "writing-argumentative-essay",
|
||||
"requirement": "我想用项目制的方式练习写一篇有说服力的议论文",
|
||||
"pblConfig": {
|
||||
"projectTopic": "围绕\"AI 是否应进入中小学课堂\"写一篇议论文",
|
||||
"projectDescription": "完成一篇结构完整、论证有力的议论文，包含明确立场、分论点、证据与回应反方",
|
||||
"targetSkills": ["确立论点", "搭建论证结构", "选取与运用证据", "回应反驳", "修改润色"],
|
||||
"issueCount": 3
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "analysis-user-churn-case",
|
||||
"requirement": "我想通过分析一个真实业务案例，学会做用户流失的归因分析（不写代码，重在分析思路）",
|
||||
"pblConfig": {
|
||||
"projectTopic": "某在线课程 App 用户流失归因分析",
|
||||
"projectDescription": "面对一份用户流失现象的描述材料，产出一份有依据的归因分析与改进建议，重在分析框架与推理，而非编程",
|
||||
"targetSkills": ["拆解问题", "提出假设", "用证据验证", "归因推理", "给出可执行建议"],
|
||||
"issueCount": 4
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "research-literature-question",
|
||||
"requirement": "我是研究新手，想通过项目学会怎么把一个模糊的兴趣收敛成一个能做的研究问题，并做小型文献梳理",
|
||||
"pblConfig": {
|
||||
"projectTopic": "把\"短视频与青少年注意力\"收敛成一个可研究的问题",
|
||||
"projectDescription": "从一个宽泛兴趣出发，完成一次小型文献梳理并界定出一个具体、可回答的研究问题与初步研究思路",
|
||||
"targetSkills": [
|
||||
"界定研究问题",
|
||||
"检索与筛选文献",
|
||||
"归纳已有结论",
|
||||
"找出研究空白",
|
||||
"形成研究思路"
|
||||
],
|
||||
"issueCount": 3
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "scenario-comfort-friend",
|
||||
"requirement": "我想练习怎么安慰一个压力很大的朋友",
|
||||
"pblConfig": {
|
||||
"projectTopic": "安慰考前压力很大的朋友",
|
||||
"projectDescription": "在一场咖啡馆对话里练习倾听与共情，支持一位临近期末、嘴上说没事其实很累的朋友",
|
||||
"targetSkills": ["积极倾听", "共情回应", "开放式提问"],
|
||||
"issueCount": 3,
|
||||
"scenarioRoleplay": true,
|
||||
"scenarioBrief": "好友临近期末崩溃边缘却强撑说没事"
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "scenario-mock-interview",
|
||||
"requirement": "我想用角色扮演练习产品经理面试",
|
||||
"pblConfig": {
|
||||
"projectTopic": "模拟产品经理行为面试",
|
||||
"projectDescription": "扮演候选人，在一场行为面试里回答面试官的追问，练习用 STAR 结构讲清自己的经历",
|
||||
"targetSkills": ["STAR 结构表达", "应对追问", "突出影响力"],
|
||||
"issueCount": 4,
|
||||
"scenarioRoleplay": true,
|
||||
"scenarioBrief": "资深面试官，会就模糊回答持续追问细节"
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "roleplay-tcm-diagnosis",
|
||||
"requirement": "我是一名小学生，我的梦想是当中医，我想要用PBL情景模拟，需要一个模拟的病人来锻炼",
|
||||
"pblConfig": {
|
||||
"projectTopic": "中医诊疗情景模拟：望闻问切接诊练习",
|
||||
"projectDescription": "通过模拟接诊真实病人，练习中医四诊（望闻问切）的基本流程，从询问症状到开具基础调理方案",
|
||||
"targetSkills": ["望闻问切", "症状询问", "病情分析", "体质辨识", "调理建议"],
|
||||
"issueCount": 4,
|
||||
"scenarioRoleplay": true,
|
||||
"scenarioBrief": "一位主诉疲劳乏力、睡眠不好的成年病人前来就诊，嘴上说只是有点累，真正的诱因要靠四诊一步步问出来"
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "roleplay-texas-holdem",
|
||||
"requirement": "我要用PBL项目制学习（情景模拟项目）来学习打德州扑克",
|
||||
"pblConfig": {
|
||||
"projectTopic": "德州扑克实战情景：从规则到策略",
|
||||
"projectDescription": "在真实牌局情景中，从基础规则学起，逐步掌握位置策略、赔率计算和对手解读",
|
||||
"targetSkills": ["德州扑克规则", "手牌强度判断", "位置策略", "底池赔率", "对手行为解读"],
|
||||
"issueCount": 5,
|
||||
"scenarioRoleplay": true,
|
||||
"scenarioBrief": "一桌现金局，对手风格各异（紧凶老手、松弱新手），从发牌到摊牌逐手做决策"
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "roleplay-job-interview",
|
||||
"requirement": "我是应届毕业生，想用情景模拟来练习求职面试，特别是技术岗位的面试",
|
||||
"pblConfig": {
|
||||
"projectTopic": "互联网公司技术岗面试情景实战",
|
||||
"projectDescription": "模拟真实的技术岗面试全流程，从自我介绍到技术问答、项目讲解和HR面，全方位提升面试表现",
|
||||
"targetSkills": ["自我介绍", "项目经验阐述", "技术问题应对", "行为面试", "反向提问"],
|
||||
"issueCount": 4,
|
||||
"scenarioRoleplay": true,
|
||||
"scenarioBrief": "一位资深技术面试官，会就项目细节和模糊回答层层追问，期望结构化、清晰、有影响力的回答"
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "roleplay-customer-service",
|
||||
"requirement": "我在准备客服岗位，想用情景模拟来练习处理各种棘手的客户投诉",
|
||||
"pblConfig": {
|
||||
"projectTopic": "客服情景模拟：处理难缠客户的艺术",
|
||||
"projectDescription": "面对愤怒、纠缠、无理取闹等各类棘手客户，练习情绪管理、需求挖掘和问题解决的综合技巧",
|
||||
"targetSkills": ["倾听与共情", "情绪安抚", "需求识别", "解决方案提供", "冲突化解"],
|
||||
"issueCount": 4,
|
||||
"scenarioRoleplay": true,
|
||||
"scenarioBrief": "一位因物流延误而愤怒的客户，言辞激烈、要求赔偿，需先安抚情绪、问出真实诉求再给方案"
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "roleplay-negotiation-business",
|
||||
"requirement": "我想学习商务谈判技巧，希望通过情景模拟来练习谈判的各个环节",
|
||||
"pblConfig": {
|
||||
"projectTopic": "商务合作谈判情景实战：从开价到成交",
|
||||
"projectDescription": "模拟一次完整的商务谈判，练习如何开价、让步、打破僵局和达成双赢协议",
|
||||
"targetSkills": ["谈判准备", "开价策略", "让步技巧", "僵局破解", "协议达成"],
|
||||
"issueCount": 5,
|
||||
"scenarioRoleplay": true,
|
||||
"scenarioBrief": "对方是预算有限、压价强硬的采购代表，谈判中会制造僵局，需在让步里守住底线达成双赢"
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
},
|
||||
{
|
||||
"id": "roleplay-parent-teacher-conference",
|
||||
"requirement": "我是新手老师，想通过情景模拟来练习和家长沟通孩子的学习问题",
|
||||
"pblConfig": {
|
||||
"projectTopic": "家校沟通情景模拟：棘手家长会谈",
|
||||
"projectDescription": "模拟与不同类型家长的沟通场景，练习如何有效反馈学生问题、安抚焦虑家长并达成教育共识",
|
||||
"targetSkills": ["问题反馈技巧", "家长情绪管理", "教育理念沟通", "共识达成", "后续跟进"],
|
||||
"issueCount": 4,
|
||||
"scenarioRoleplay": true,
|
||||
"scenarioBrief": "一位对孩子成绩下滑既焦虑又护短的家长，听到问题容易激动，需要先稳住情绪再客观反馈并达成共识"
|
||||
},
|
||||
"languageDirective": "Reply in Simplified Chinese"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* PBL v2 A/B viewer — tiny dev server.
|
||||
*
|
||||
* Loads eval-result JSON on demand (NOT a baked-in HTML): lists every run
|
||||
* under `results/`, and serves each run's `projects/*.json` + `results.json`
|
||||
* through a small JSON API the page fetches. New runs show up on refresh —
|
||||
* no rebuild.
|
||||
*
|
||||
* Usage:
|
||||
* pnpm tsx eval/pbl-v2-planner/serve.ts # http://localhost:5179
|
||||
* EVAL_PBL_PORT=8080 pnpm tsx eval/pbl-v2-planner/serve.ts
|
||||
*
|
||||
* Routes:
|
||||
* GET / → viewer page (run selector + side-by-side compare)
|
||||
* GET /api/runs → [{ dir, label, caseCount, mtime }]
|
||||
* GET /api/run?dir=<abs> → { runDir, cases } for one run (dir must be under results/)
|
||||
*/
|
||||
import { createServer } from 'node:http';
|
||||
import { readdirSync, statSync, existsSync } from 'node:fs';
|
||||
import { join, dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { collectRunData, VIEWER_CSS, CLIENT_RENDER_JS } from './compare-html';
|
||||
|
||||
function here(): string {
|
||||
return typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url));
|
||||
}
|
||||
|
||||
const RESULTS_ROOT = resolve(join(here(), 'results'));
|
||||
const PORT = parseInt(process.env.EVAL_PBL_PORT || '5179', 10);
|
||||
|
||||
interface RunEntry {
|
||||
dir: string;
|
||||
label: string;
|
||||
caseCount: number;
|
||||
mtime: number;
|
||||
}
|
||||
|
||||
/** All run dirs (model/ts leaves containing a projects/ subdir), newest first. */
|
||||
function listRuns(): RunEntry[] {
|
||||
if (!existsSync(RESULTS_ROOT)) return [];
|
||||
const runs: RunEntry[] = [];
|
||||
for (const model of readdirSync(RESULTS_ROOT)) {
|
||||
const modelDir = join(RESULTS_ROOT, model);
|
||||
if (!statSync(modelDir).isDirectory()) continue;
|
||||
for (const ts of readdirSync(modelDir)) {
|
||||
const dir = join(modelDir, ts);
|
||||
if (!statSync(dir).isDirectory()) continue;
|
||||
const projectsDir = join(dir, 'projects');
|
||||
if (!existsSync(projectsDir)) continue;
|
||||
const caseCount = new Set(
|
||||
readdirSync(projectsDir)
|
||||
.filter((f) => f.endsWith('.json'))
|
||||
.map((f) => f.slice(0, f.lastIndexOf('__'))),
|
||||
).size;
|
||||
runs.push({
|
||||
dir,
|
||||
label: `${model} / ${ts} (${caseCount} cases)`,
|
||||
caseCount,
|
||||
mtime: statSync(dir).mtimeMs,
|
||||
});
|
||||
}
|
||||
}
|
||||
return runs.sort((a, b) => b.mtime - a.mtime);
|
||||
}
|
||||
|
||||
function pageHtml(): string {
|
||||
return `<!doctype html>
|
||||
<html lang="zh"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>PBL v2 — A/B viewer</title><style>${VIEWER_CSS}</style></head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>PBL v2 — loop vs single-call</h1>
|
||||
<select id="runSel" title="选择一次 eval run"></select>
|
||||
<span class="meta" id="meta"></span>
|
||||
</header>
|
||||
<div class="layout"><nav id="nav"></nav><main id="main"></main></div>
|
||||
<script>${CLIENT_RENDER_JS}
|
||||
async function loadRun(){
|
||||
const dir = document.getElementById('runSel').value;
|
||||
if (!dir) return;
|
||||
try {
|
||||
const data = await (await fetch('/api/run?dir='+encodeURIComponent(dir))).json();
|
||||
renderRun(data);
|
||||
} catch (e) { document.getElementById('main').innerHTML = '<p class="missing">加载失败: '+esc(e.message||e)+'</p>'; }
|
||||
}
|
||||
async function boot(){
|
||||
const sel = document.getElementById('runSel');
|
||||
let runs = [];
|
||||
try { runs = await (await fetch('/api/runs')).json(); } catch (e) {}
|
||||
if (!runs.length){ document.getElementById('main').innerHTML='<p class="missing">results/ 下还没有 run。先跑 runner。</p>'; return; }
|
||||
sel.innerHTML = runs.map(r => '<option value="'+esc(r.dir)+'">'+esc(r.label)+'</option>').join('');
|
||||
sel.onchange = loadRun;
|
||||
loadRun();
|
||||
}
|
||||
boot();
|
||||
</script>
|
||||
</body></html>`;
|
||||
}
|
||||
|
||||
function send(
|
||||
res: import('node:http').ServerResponse,
|
||||
code: number,
|
||||
type: string,
|
||||
body: string,
|
||||
): void {
|
||||
res.writeHead(code, { 'content-type': type, 'cache-control': 'no-store' });
|
||||
res.end(body);
|
||||
}
|
||||
|
||||
const server = createServer((req, res) => {
|
||||
try {
|
||||
const url = new URL(req.url ?? '/', `http://localhost:${PORT}`);
|
||||
if (url.pathname === '/') {
|
||||
return send(res, 200, 'text/html; charset=utf-8', pageHtml());
|
||||
}
|
||||
if (url.pathname === '/api/runs') {
|
||||
return send(res, 200, 'application/json', JSON.stringify(listRuns()));
|
||||
}
|
||||
if (url.pathname === '/api/run') {
|
||||
const dir = resolve(url.searchParams.get('dir') ?? '');
|
||||
// Path-traversal guard: only serve dirs under results/.
|
||||
if (!dir.startsWith(RESULTS_ROOT + '/') && dir !== RESULTS_ROOT) {
|
||||
return send(res, 403, 'application/json', JSON.stringify({ error: 'forbidden' }));
|
||||
}
|
||||
if (!existsSync(join(dir, 'projects'))) {
|
||||
return send(res, 404, 'application/json', JSON.stringify({ error: 'not a run dir' }));
|
||||
}
|
||||
return send(res, 200, 'application/json', JSON.stringify(collectRunData(dir)));
|
||||
}
|
||||
send(res, 404, 'text/plain', 'not found');
|
||||
} catch (err) {
|
||||
send(res, 500, 'application/json', JSON.stringify({ error: String(err) }));
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(PORT, () => {
|
||||
console.log(`PBL v2 A/B viewer → http://localhost:${PORT}`);
|
||||
console.log(`Serving runs from ${RESULTS_ROOT}`);
|
||||
});
|
||||
Reference in New Issue
Block a user