fix(core): make V2 reads media-aware and binary-safe (#31038)

packages/llm/src/protocols/openai-chat.ts

@@ -9,10 +9,12 @@ import {
   Usage,
   type FinishReason,
   type LLMRequest,
+  type MediaPart,
   type ReasoningPart,
   type TextPart,
   type ToolCallPart,
   type ToolDefinition,
+  type ToolResultContentPart,
 } from "../schema"
 import { isRecord, JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared"
 import { OpenAIOptions } from "./utils/openai-options"
@@ -20,6 +22,7 @@ import { Lifecycle } from "./utils/lifecycle"
 import { ToolStream } from "./utils/tool-stream"
 
 const ADAPTER = "openai-chat"
+const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
 export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
 export const PATH = "/chat/completions"
 
@@ -51,9 +54,20 @@ const OpenAIChatAssistantToolCall = Schema.Struct({
 })
 type OpenAIChatAssistantToolCall = Schema.Schema.Type<typeof OpenAIChatAssistantToolCall>
 
+const OpenAIChatUserContent = Schema.Union([
+  Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }),
+  Schema.Struct({
+    type: Schema.Literal("image_url"),
+    image_url: Schema.Struct({ url: Schema.String }),
+  }),
+])
+
 const OpenAIChatMessage = Schema.Union([
   Schema.Struct({ role: Schema.Literal("system"), content: Schema.String }),
-  Schema.Struct({ role: Schema.Literal("user"), content: Schema.String }),
+  Schema.Struct({
+    role: Schema.Literal("user"),
+    content: Schema.Union([Schema.String, Schema.Array(OpenAIChatUserContent)]),
+  }),
   Schema.Struct({
     role: Schema.Literal("assistant"),
     content: Schema.NullOr(Schema.String),
@@ -186,17 +200,32 @@ const lowerToolCall = (part: ToolCallPart): OpenAIChatAssistantToolCall => ({
   },
 })
 
+const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (
+  part: Extract<MediaPart | ToolResultContentPart, { type: "media" }>,
+) {
+  const media = yield* ProviderShared.validateMedia("OpenAI Chat", part, IMAGE_MIMES)
+  return { type: "image_url" as const, image_url: { url: media.dataUrl } }
+})
+
 const openAICompatibleReasoningContent = (native: unknown) =>
   isRecord(native) && typeof native.reasoning_content === "string" ? native.reasoning_content : undefined
 
 const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (message: OpenAIChatRequestMessage) {
-  const content: TextPart[] = []
+  const content: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
   for (const part of message.content) {
-    if (!ProviderShared.supportsContent(part, ["text"]))
-      return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text"])
-    content.push(part)
+    if (part.type === "text") {
+      content.push({ type: "text", text: part.text })
+      continue
+    }
+    if (part.type === "media") {
+      content.push(yield* lowerMedia(part))
+      continue
+    }
+    return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text", "media"])
   }
-  return { role: "user" as const, content: ProviderShared.joinText(content) }
+  if (content.every((part) => part.type === "text"))
+    return { role: "user" as const, content: content.map((part) => part.text).join("") }
+  return { role: "user" as const, content }
 })
 
 const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(function* (
@@ -234,35 +263,68 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
 
 const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (message: OpenAIChatRequestMessage) {
   const messages: OpenAIChatMessage[] = []
+  const images: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
   for (const part of message.content) {
     if (!ProviderShared.supportsContent(part, ["tool-result"]))
       return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"])
-    messages.push({ role: "tool", tool_call_id: part.id, content: ProviderShared.toolResultText(part) })
+    if (part.result.type !== "content") {
+      messages.push({ role: "tool", tool_call_id: part.id, content: ProviderShared.toolResultText(part) })
+      continue
+    }
+    const content: ReadonlyArray<ToolResultContentPart> = part.result.value
+    const text = content.filter((item) => item.type === "text").map((item) => item.text)
+    messages.push({ role: "tool", tool_call_id: part.id, content: text.join("\n") })
+    const media = content.filter(
+      (item): item is Extract<ToolResultContentPart, { type: "media" }> => item.type === "media",
+    )
+    images.push(...(yield* Effect.forEach(media, lowerMedia)))
   }
-  return messages
+  return { messages, images }
 })
 
 const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (message: OpenAIChatRequestMessage) {
   if (message.role === "user") return [yield* lowerUserMessage(message)]
   if (message.role === "assistant") return [yield* lowerAssistantMessage(message)]
-  return yield* lowerToolMessages(message)
+  return (yield* lowerToolMessages(message)).messages
 })
 
 const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: LLMRequest) {
   const system: OpenAIChatMessage[] =
     request.system.length === 0 ? [] : [{ role: "system", content: ProviderShared.joinText(request.system) }]
   const messages = [...system]
+  const pendingImages: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
+  const flushImages = () => {
+    if (pendingImages.length === 0) return
+    messages.push({ role: "user", content: pendingImages.splice(0) })
+  }
   for (const message of request.messages) {
     if (message.role === "system") {
       const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message)
+      if (pendingImages.length > 0) {
+        messages.push({ role: "user", content: [...pendingImages.splice(0), { type: "text", text: part.text }] })
+        continue
+      }
       const previous = messages.at(-1)
-      if (previous?.role === "user")
+      if (previous?.role === "user" && typeof previous.content === "string")
         messages[messages.length - 1] = { role: "user", content: `${previous.content}\n${part.text}` }
+      else if (previous?.role === "user" && Array.isArray(previous.content))
+        messages[messages.length - 1] = {
+          role: "user",
+          content: [...previous.content, { type: "text", text: part.text }],
+        }
       else messages.push({ role: "user", content: part.text })
       continue
     }
+    if (message.role === "tool") {
+      const lowered = yield* lowerToolMessages(message)
+      messages.push(...lowered.messages)
+      pendingImages.push(...lowered.images)
+      continue
+    }
+    flushImages()
     messages.push(...(yield* lowerMessage(message)))
   }
+  flushImages()
   return messages
 })