From 0e07e9e1319767a07e32f08300872d55c19f5f78 Mon Sep 17 00:00:00 2001
From: wyuc <wang-yc24@mails.tsinghua.edu.cn>
Date: Mon, 20 Apr 2026 23:13:32 +0800
Subject: [PATCH] fix(orchestration): strip data: prefix before passing image
 to AI SDK
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Vercel AI SDK's streamText/generateText treats ImagePart.image as a URL
to fetch when it's a string. data: URLs fail the http/https scheme check
and throw AI_DownloadError. Strip the data URL prefix and pass raw base64
with mediaType separately — AI SDK treats base64 strings as data content.
---
 .../summarizers/message-converter.ts          | 28 +++++++++++++++----
 tests/orchestration/message-converter.test.ts | 14 +++++++---
 2 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/lib/orchestration/summarizers/message-converter.ts b/lib/orchestration/summarizers/message-converter.ts
index 4a24638e0..e97c95ee6 100644
--- a/lib/orchestration/summarizers/message-converter.ts
+++ b/lib/orchestration/summarizers/message-converter.ts
@@ -18,7 +18,12 @@ export interface ConvertedMessage {
 
 /**
  * Extract image URLs from UIMessage parts.
- * Returns data-URLs or http(s) URLs where `mediaType` starts with `image/`.
+ * Returns image parts in AI SDK's `ImagePart` shape.
+ *
+ * AI SDK's `streamText`/`generateText` treats strings as URLs to download.
+ * data URLs fail that download check, so we unwrap `data:image/...;base64,<payload>`
+ * into a raw base64 string and carry the media type separately.
+ * http(s) URLs pass through unchanged (AI SDK will fetch them).
  */
 function extractImageParts(parts: unknown[] | undefined): ImagePart[] {
   if (!parts) return [];
@@ -26,12 +31,23 @@ function extractImageParts(parts: unknown[] | undefined): ImagePart[] {
   for (const part of parts) {
     const p = part as Record<string, unknown>;
     if (
-      p.type === 'file' &&
-      typeof p.mediaType === 'string' &&
-      p.mediaType.startsWith('image/') &&
-      typeof p.url === 'string'
+      p.type !== 'file' ||
+      typeof p.mediaType !== 'string' ||
+      !p.mediaType.startsWith('image/') ||
+      typeof p.url !== 'string'
     ) {
-      out.push({ type: 'image', image: p.url, mediaType: p.mediaType });
+      continue;
+    }
+
+    const url = p.url;
+    // Match data URLs: data:<mime>[;base64],<payload>
+    const dataUrlMatch = /^data:([^;,]+)(?:;base64)?,(.*)$/.exec(url);
+    if (dataUrlMatch) {
+      const payload = dataUrlMatch[2];
+      out.push({ type: 'image', image: payload, mediaType: p.mediaType });
+    } else {
+      // http / https URL — AI SDK will fetch it
+      out.push({ type: 'image', image: url, mediaType: p.mediaType });
     }
   }
   return out;
diff --git a/tests/orchestration/message-converter.test.ts b/tests/orchestration/message-converter.test.ts
index 2e29572e2..7174099c8 100644
--- a/tests/orchestration/message-converter.test.ts
+++ b/tests/orchestration/message-converter.test.ts
@@ -70,7 +70,7 @@ describe('convertMessagesToOpenAI — text-only (backwards compatibility)', () =
 });
 
 describe('convertMessagesToOpenAI — image attachments', () => {
-  test('user message with a single image/png file part → multimodal content array', () => {
+  test('user message with a data: URL image → base64 payload + mediaType (data prefix stripped)', () => {
     const msgs: UIMsg[] = [
       userMsg([
         { type: 'text', text: 'Please look at this.' },
@@ -88,14 +88,16 @@ describe('convertMessagesToOpenAI — image attachments', () => {
     const parts = out[0].content as Array<Record<string, unknown>>;
     expect(parts).toHaveLength(2);
     expect(parts[0]).toEqual({ type: 'text', text: 'Please look at this.' });
+    // The data URL prefix must be stripped — AI SDK rejects `data:` strings
+    // as invalid download URLs.
     expect(parts[1]).toEqual({
       type: 'image',
-      image: 'data:image/png;base64,iVBORw0KGgo=',
+      image: 'iVBORw0KGgo=',
       mediaType: 'image/png',
     });
   });
 
-  test('image-only user message (no text) still emits content array with just the image', () => {
+  test('http URL image passes through unchanged', () => {
     const msgs: UIMsg[] = [
       userMsg([
         {
@@ -109,7 +111,11 @@ describe('convertMessagesToOpenAI — image attachments', () => {
     expect(out).toHaveLength(1);
     const parts = out[0].content as Array<Record<string, unknown>>;
     expect(parts).toHaveLength(1);
-    expect(parts[0]).toMatchObject({ type: 'image', image: 'https://example.com/board.png' });
+    expect(parts[0]).toMatchObject({
+      type: 'image',
+      image: 'https://example.com/board.png',
+      mediaType: 'image/png',
+    });
   });
 
   test('non-image file parts (pdf) are ignored (image-only experiment scope)', () => {