feat(eval): attach prior-turn whiteboard screenshot as user message image part

Gated behind EVAL_ATTACH_PRIOR_SCREENSHOT=1 env. When on, captures a
screenshot after every turn (not just checkpoints) and attaches it as a
file part with mediaType:'image/png' on the next user message. Teacher
prompt gets a new 'Prior-state image' section telling the agent to
treat it as direct visual feedback.

eval/whiteboard-layout/runner.ts

@@ -35,6 +35,9 @@ const { values: args } = parseArgs({
 const BASE_URL = args['base-url']!;
 const CHAT_MODEL_RAW = process.env.EVAL_CHAT_MODEL || process.env.DEFAULT_MODEL;
 const SCORER_MODEL_RAW = process.env.EVAL_SCORER_MODEL;
+const ATTACH_PRIOR_SCREENSHOT =
+  process.env.EVAL_ATTACH_PRIOR_SCREENSHOT === '1' ||
+  process.env.EVAL_ATTACH_PRIOR_SCREENSHOT === 'true';
 if (!CHAT_MODEL_RAW) {
   console.error(
     'Error: EVAL_CHAT_MODEL (or DEFAULT_MODEL) must be set. Example: EVAL_CHAT_MODEL=openai:gpt-4.1',
@@ -98,16 +101,39 @@ async function runScenario(
     metadata?: unknown;
   }> = [];
 
+  // Tracks the most recent whiteboard screenshot file so we can attach it to
+  // the NEXT user message as `prior state` feedback for the VLM.
+  let priorScreenshotPath: string | null = null;
+
   try {
     for (let turnIdx = 0; turnIdx < scenario.turns.length; turnIdx++) {
       const turn = scenario.turns[turnIdx];
       console.log(`    Turn ${turnIdx + 1}: "${turn.userMessage.slice(0, 50)}..."`);
 
-      // Add user message
+      // Build user message parts, optionally including the prior turn's
+      // whiteboard screenshot so the VLM can *see* what it produced last.
+      const parts: unknown[] = [{ type: 'text', text: turn.userMessage }];
+      if (ATTACH_PRIOR_SCREENSHOT && priorScreenshotPath) {
+        try {
+          const base64 = readFileSync(priorScreenshotPath).toString('base64');
+          parts.push({
+            type: 'file',
+            mediaType: 'image/png',
+            url: `data:image/png;base64,${base64}`,
+          });
+          console.log(
+            `      [screenshot] attached prior turn's board (${Math.round(base64.length / 1024)}kb base64)`,
+          );
+        } catch (err) {
+          const msg = err instanceof Error ? err.message : String(err);
+          console.warn(`      [screenshot] failed to attach: ${msg.slice(0, 120)}`);
+        }
+      }
+
       messages.push({
         role: 'user',
         content: turn.userMessage,
-        parts: [{ type: 'text', text: turn.userMessage }],
+        parts,
         metadata: { createdAt: Date.now() },
       });
 
@@ -243,22 +269,35 @@ async function runScenario(
 
       // Checkpoint: capture + score
       const isLastTurn = turnIdx === scenario.turns.length - 1;
-      if (turn.checkpoint || isLastTurn) {
+      const isCheckpoint = turn.checkpoint || isLastTurn;
+
+      // When prior-screenshot attachment is on, capture after EVERY turn so
+      // the next turn can see what was produced. Score only on checkpoint
+      // turns to preserve baseline-compatible report shape.
+      if (isCheckpoint || ATTACH_PRIOR_SCREENSHOT) {
         const elements = stateManager.getWhiteboardElements();
         const screenshotFilename = `run${runIndex}_turn${turnIdx}.png`;
         const screenshotPath = await captureWhiteboard(elements, scenarioDir, screenshotFilename);
 
-        console.log(`    Captured: ${screenshotFilename} (${elements.length} elements)`);
+        // Remember for the next turn's message attachment
+        priorScreenshotPath = screenshotPath;
 
-        try {
-          const score = await scoreScreenshot(screenshotPath, SCORER_MODEL);
-          console.log(`    Score: overall=${score.overall}, overlap=${score.overlap.score}`);
-          checkpoints.push({ turnIndex: turnIdx, screenshotPath, score, elements });
-        } catch (scoreErr) {
-          const msg = scoreErr instanceof Error ? scoreErr.message : String(scoreErr);
-          console.error(`    Score error (continuing): ${msg.slice(0, 120)}`);
-          // Preserve screenshot with null score so the report can still include it
-          checkpoints.push({ turnIndex: turnIdx, screenshotPath, score: null, elements });
+        if (isCheckpoint) {
+          console.log(`    Captured: ${screenshotFilename} (${elements.length} elements)`);
+
+          try {
+            const score = await scoreScreenshot(screenshotPath, SCORER_MODEL);
+            console.log(`    Score: overall=${score.overall}, overlap=${score.overlap.score}`);
+            checkpoints.push({ turnIndex: turnIdx, screenshotPath, score, elements });
+          } catch (scoreErr) {
+            const msg = scoreErr instanceof Error ? scoreErr.message : String(scoreErr);
+            console.error(`    Score error (continuing): ${msg.slice(0, 120)}`);
+            checkpoints.push({ turnIndex: turnIdx, screenshotPath, score: null, elements });
+          }
+        } else {
+          console.log(
+            `    [non-checkpoint] captured ${screenshotFilename} (${elements.length} elements) for next-turn feedback`,
+          );
         }
       }
     }

lib/prompts/templates/agent-system-wb-teacher/system.md

@@ -12,6 +12,14 @@ Before every response, look at "Current State" / "Whiteboard Changes This Round"
 - If the board is full of content from prior turns → call `wb_clear` first; a crowded board loses meaning.
 - If you cannot place a new element without overlapping existing elements by more than 30% → `wb_delete` the specific element you want to replace first, do not stack.
 
+## Prior-state image (if attached)
+
+The user message may include an image showing how the whiteboard actually looks right now. Use it as direct visual feedback:
+
+- If you see overlap, clutter, or garbage LaTeX (tokens like `ext`, `rac`, `heta`, `irc`) → the first thing in your response should be `wb_clear` or targeted `wb_delete` to fix the board.
+- If the board looks clean and useful → build on it; don't redraw.
+- Trust the image over the text state — "Current Whiteboard Elements (16)" is just a count; the image shows whether those 16 elements are actually readable.
+
 ## Animated step reveals
 
 Every `wb_draw_*` accepts `elementId`. To animate a multi-step explanation: draw step 1 with `elementId:"step1"`, narrate; next turn delete `step1` and draw step 2. This replaces drawing many elements with drawing few elements that evolve.