feat(orchestration): preserve image parts from UIMessage to AI SDK ModelMessage

lib/orchestration/ai-sdk-adapter.ts

@@ -59,20 +59,29 @@ export class AISdkLangGraphAdapter extends BaseChatModel {
   }
 
   /**
-   * Convert LangChain messages to AI SDK message format
+   * Convert LangChain messages to AI SDK message format.
+   *
+   * Content may be a plain string or a multimodal content array (e.g., a text
+   * part alongside image parts); we pass it through unchanged so downstream
+   * `streamText` / `generateText` calls receive multimodal input natively.
    */
   private convertMessages(
     messages: BaseMessage[],
-  ): { role: 'system' | 'user' | 'assistant'; content: string }[] {
+  ): {
+    role: 'system' | 'user' | 'assistant';
+    content: string | Array<Record<string, unknown>>;
+  }[] {
     return messages.map((msg) => {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any -- LangChain content is MessageContent which widens to string | complex[]
+      const content = msg.content as any;
       if (msg instanceof HumanMessage) {
-        return { role: 'user' as const, content: msg.content as string };
+        return { role: 'user' as const, content };
       } else if (msg instanceof AIMessage) {
-        return { role: 'assistant' as const, content: msg.content as string };
+        return { role: 'assistant' as const, content };
       } else if (msg instanceof SystemMessage) {
-        return { role: 'system' as const, content: msg.content as string };
+        return { role: 'system' as const, content };
       } else {
-        return { role: 'user' as const, content: msg.content as string };
+        return { role: 'user' as const, content };
       }
     });
   }
@@ -88,7 +97,10 @@ export class AISdkLangGraphAdapter extends BaseChatModel {
       const result = await callLLM(
         {
           model: this.languageModel,
-          messages: aiMessages,
+          // AI SDK's ModelMessage type accepts the `{type:'image'}` parts on
+          // user messages — our adapter just needs to pass them through.
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any -- see above
+          messages: aiMessages as any,
         },
         'chat-adapter',
         undefined,
@@ -134,7 +146,8 @@ export class AISdkLangGraphAdapter extends BaseChatModel {
     const result = streamLLM(
       {
         model: this.languageModel,
-        messages: aiMessages,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any -- see convertMessages comment
+        messages: aiMessages as any,
         abortSignal: options?.signal,
       },
       'chat-adapter-stream',

lib/orchestration/director-graph.ts

@@ -292,9 +292,14 @@ async function runAgentGeneration(
 
   const lcMessages = [
     new SystemMessage(systemPrompt),
-    ...openaiMessages.map((m) =>
-      m.role === 'user' ? new HumanMessage(m.content) : new AIMessage(m.content),
-    ),
+    ...openaiMessages.map((m) => {
+      // LangChain's MessageContent type doesn't model AI SDK's `{type:'image'}`
+      // parts; we stash them in `content` anyway and the adapter reads them
+      // back out without LangChain touching them.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any -- see above
+      const content = m.content as any;
+      return m.role === 'user' ? new HumanMessage({ content }) : new AIMessage({ content });
+    }),
   ];
 
   // Ensure the message list ends with a HumanMessage.

lib/orchestration/summarizers/conversation-summary.ts

@@ -1,11 +1,26 @@
 // ==================== Conversation Summary ====================
 
 /**
- * OpenAI message format (used by director)
+ * OpenAI-style message used by the director.
+ * Content may be a string or a multimodal content array; only the text
+ * portion is summarized (images are denoted as `[image]`).
  */
 export interface OpenAIMessage {
   role: 'system' | 'user' | 'assistant';
-  content: string;
+  content:
+    | string
+    | Array<{ type: 'text'; text: string } | { type: 'image'; image: string; mediaType?: string }>;
+}
+
+/**
+ * Flatten a multimodal content to plain text for summarization.
+ */
+function contentToSummaryText(content: OpenAIMessage['content']): string {
+  if (typeof content === 'string') return content;
+  return content
+    .map((part) => (part.type === 'text' ? part.text : '[image]'))
+    .filter(Boolean)
+    .join(' ');
 }
 
 /**
@@ -31,10 +46,8 @@ export function summarizeConversation(
   const lines = recent.map((msg) => {
     const roleLabel =
       msg.role === 'user' ? 'User' : msg.role === 'assistant' ? 'Assistant' : 'System';
-    const content =
-      msg.content.length > maxContentLength
-        ? msg.content.slice(0, maxContentLength) + '...'
-        : msg.content;
+    const text = contentToSummaryText(msg.content);
+    const content = text.length > maxContentLength ? text.slice(0, maxContentLength) + '...' : text;
     return `[${roleLabel}] ${content}`;
   });
 

lib/orchestration/summarizers/message-converter.ts

@@ -3,16 +3,52 @@ import type { StatelessChatRequest } from '@/lib/types/chat';
 // ==================== Message Conversion ====================
 
 /**
- * Convert UI messages to OpenAI format
- * Includes tool call information so the model knows what actions were taken
+ * Content part shapes compatible with Vercel AI SDK's `ModelMessage`.
+ * When a message has no attachments this stays a string; when there are
+ * image attachments it becomes an array.
+ */
+type ImagePart = { type: 'image'; image: string; mediaType?: string };
+type TextPart = { type: 'text'; text: string };
+export type ConvertedContent = string | Array<TextPart | ImagePart>;
+
+export interface ConvertedMessage {
+  role: 'system' | 'user' | 'assistant';
+  content: ConvertedContent;
+}
+
+/**
+ * Extract image URLs from UIMessage parts.
+ * Returns data-URLs or http(s) URLs where `mediaType` starts with `image/`.
+ */
+function extractImageParts(parts: unknown[] | undefined): ImagePart[] {
+  if (!parts) return [];
+  const out: ImagePart[] = [];
+  for (const part of parts) {
+    const p = part as Record<string, unknown>;
+    if (
+      p.type === 'file' &&
+      typeof p.mediaType === 'string' &&
+      p.mediaType.startsWith('image/') &&
+      typeof p.url === 'string'
+    ) {
+      out.push({ type: 'image', image: p.url, mediaType: p.mediaType });
+    }
+  }
+  return out;
+}
+
+/**
+ * Convert UI messages to OpenAI-compatible format.
+ * Includes tool call information so the model knows what actions were taken.
+ * Preserves image attachments on user messages as multimodal content parts.
  */
 export function convertMessagesToOpenAI(
   messages: StatelessChatRequest['messages'],
   currentAgentId?: string,
-): Array<{ role: 'system' | 'user' | 'assistant'; content: string }> {
+): ConvertedMessage[] {
   return messages
     .filter((msg) => msg.role === 'user' || msg.role === 'assistant')
-    .map((msg) => {
+    .map((msg): ConvertedMessage => {
       if (msg.role === 'assistant') {
         // Assistant messages use JSON array format to serve as few-shot examples
         // that match the expected output format from the system prompt
@@ -51,18 +87,18 @@ export function convertMessagesToOpenAI(
         if (currentAgentId && msgAgentId && msgAgentId !== currentAgentId) {
           const agentName = msg.metadata?.senderName || msgAgentId;
           return {
-            role: 'user' as const,
+            role: 'user',
             content: content ? `[${agentName}]: ${content}` : '',
           };
         }
 
         return {
-          role: 'assistant' as const,
+          role: 'assistant',
           content,
         };
       }
 
-      // User messages: keep plain text concatenation
+      // User messages: keep plain text concatenation + preserve image attachments
       const contentParts: string[] = [];
 
       if (msg.parts) {
@@ -88,9 +124,9 @@ export function convertMessagesToOpenAI(
 
       // Extract speaker name from metadata (e.g. other agents' messages in discussion)
       const senderName = msg.metadata?.senderName;
-      let content = contentParts.join('\n');
+      let textContent = contentParts.join('\n');
       if (senderName) {
-        content = `[${senderName}]: ${content}`;
+        textContent = `[${senderName}]: ${textContent}`;
       }
 
       // Annotate interrupted messages so the LLM knows context was cut short
@@ -98,16 +134,26 @@ export function convertMessagesToOpenAI(
         (msg as unknown as Record<string, unknown>).metadata &&
         ((msg as unknown as Record<string, unknown>).metadata as Record<string, unknown>)
           ?.interrupted;
-      return {
-        role: 'user' as const,
-        content: isInterrupted
-          ? `${content}\n[This response was interrupted — do NOT continue it. Start a new JSON array response.]`
-          : content,
-      };
+      if (isInterrupted) {
+        textContent = `${textContent}\n[This response was interrupted — do NOT continue it. Start a new JSON array response.]`;
+      }
+
+      const images = extractImageParts(msg.parts);
+      if (images.length > 0) {
+        // Multimodal content: text (possibly empty) followed by image parts.
+        const parts: Array<TextPart | ImagePart> = [];
+        if (textContent) parts.push({ type: 'text', text: textContent });
+        parts.push(...images);
+        return { role: 'user', content: parts };
+      }
+
+      return { role: 'user', content: textContent };
     })
     .filter((msg) => {
       // Drop empty messages and messages with only dots/ellipsis/whitespace
-      // (produced by failed agent streams)
+      // (produced by failed agent streams). Messages with image attachments are
+      // always kept, even if the text is empty.
+      if (typeof msg.content !== 'string') return true;
       const stripped = msg.content.replace(/[.\s…]+/g, '');
       return stripped.length > 0;
     });

tests/orchestration/message-converter.test.ts

@@ -0,0 +1,143 @@
+import { describe, test, expect } from 'vitest';
+import { convertMessagesToOpenAI } from '@/lib/orchestration/summarizers/message-converter';
+import type { StatelessChatRequest } from '@/lib/types/chat';
+
+type UIMsg = StatelessChatRequest['messages'][number];
+
+function userMsg(parts: unknown[], metadata: Record<string, unknown> = {}): UIMsg {
+  return {
+    role: 'user',
+    content: '',
+    parts,
+    metadata,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- test helper
+  } as any;
+}
+
+function assistantMsg(
+  parts: unknown[],
+  metadata: Record<string, unknown> = {},
+): UIMsg {
+  return {
+    role: 'assistant',
+    content: '',
+    parts,
+    metadata,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- test helper
+  } as any;
+}
+
+describe('convertMessagesToOpenAI — text-only (backwards compatibility)', () => {
+  test('user text message becomes a plain string content', () => {
+    const msgs: UIMsg[] = [userMsg([{ type: 'text', text: 'hello world' }])];
+    const out = convertMessagesToOpenAI(msgs);
+    expect(out).toEqual([{ role: 'user', content: 'hello world' }]);
+  });
+
+  test('assistant with text + action parts stays a string (JSON array serialization)', () => {
+    const msgs: UIMsg[] = [
+      assistantMsg([
+        { type: 'text', text: 'Here you go.' },
+        {
+          type: 'action-wb_draw_text',
+          actionName: 'wb_draw_text',
+          state: 'result',
+          output: { success: true, data: { id: 'abc' } },
+        },
+      ]),
+    ];
+    const out = convertMessagesToOpenAI(msgs);
+    expect(out).toHaveLength(1);
+    expect(out[0].role).toBe('assistant');
+    expect(typeof out[0].content).toBe('string');
+    expect(out[0].content).toContain('Here you go.');
+    expect(out[0].content).toContain('wb_draw_text');
+  });
+
+  test('empty / whitespace-only user message is dropped', () => {
+    const msgs: UIMsg[] = [userMsg([{ type: 'text', text: '...' }])];
+    expect(convertMessagesToOpenAI(msgs)).toEqual([]);
+  });
+
+  test('interrupted assistant is passed through as user with interruption suffix', () => {
+    const msgs: UIMsg[] = [
+      userMsg([{ type: 'text', text: 'ok' }], { interrupted: true }),
+    ];
+    const out = convertMessagesToOpenAI(msgs);
+    expect(out).toHaveLength(1);
+    expect(out[0].content).toContain('interrupted');
+  });
+});
+
+describe('convertMessagesToOpenAI — image attachments', () => {
+  test('user message with a single image/png file part → multimodal content array', () => {
+    const msgs: UIMsg[] = [
+      userMsg([
+        { type: 'text', text: 'Please look at this.' },
+        {
+          type: 'file',
+          mediaType: 'image/png',
+          url: 'data:image/png;base64,iVBORw0KGgo=',
+        },
+      ]),
+    ];
+    const out = convertMessagesToOpenAI(msgs);
+    expect(out).toHaveLength(1);
+    expect(out[0].role).toBe('user');
+    expect(Array.isArray(out[0].content)).toBe(true);
+    const parts = out[0].content as Array<Record<string, unknown>>;
+    expect(parts).toHaveLength(2);
+    expect(parts[0]).toEqual({ type: 'text', text: 'Please look at this.' });
+    expect(parts[1]).toEqual({
+      type: 'image',
+      image: 'data:image/png;base64,iVBORw0KGgo=',
+      mediaType: 'image/png',
+    });
+  });
+
+  test('image-only user message (no text) still emits content array with just the image', () => {
+    const msgs: UIMsg[] = [
+      userMsg([
+        {
+          type: 'file',
+          mediaType: 'image/png',
+          url: 'https://example.com/board.png',
+        },
+      ]),
+    ];
+    const out = convertMessagesToOpenAI(msgs);
+    expect(out).toHaveLength(1);
+    const parts = out[0].content as Array<Record<string, unknown>>;
+    expect(parts).toHaveLength(1);
+    expect(parts[0]).toMatchObject({ type: 'image', image: 'https://example.com/board.png' });
+  });
+
+  test('non-image file parts (pdf) are ignored (image-only experiment scope)', () => {
+    const msgs: UIMsg[] = [
+      userMsg([
+        { type: 'text', text: 'summary?' },
+        {
+          type: 'file',
+          mediaType: 'application/pdf',
+          url: 'data:application/pdf;base64,XXX',
+        },
+      ]),
+    ];
+    const out = convertMessagesToOpenAI(msgs);
+    expect(out[0].content).toBe('summary?');
+  });
+
+  test('image attachment on a message with empty text is retained (not filtered as empty)', () => {
+    const msgs: UIMsg[] = [
+      userMsg([
+        {
+          type: 'file',
+          mediaType: 'image/png',
+          url: 'data:image/png;base64,AA',
+        },
+      ]),
+    ];
+    const out = convertMessagesToOpenAI(msgs);
+    expect(out).toHaveLength(1);
+  });
+});