fix(core): make V2 reads media-aware and binary-safe (#31038)

packages/llm/src/protocols/gemini.ts

@@ -14,12 +14,14 @@ import {
   type TextPart,
   type ToolCallPart,
   type ToolDefinition,
+  type ToolResultContentPart,
 } from "../schema"
 import { JsonObject, optionalArray, ProviderShared } from "./shared"
 import { GeminiToolSchema } from "./utils/gemini-tool-schema"
 import { Lifecycle } from "./utils/lifecycle"
 
 const ADAPTER = "gemini"
+const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
 export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
 
 // =============================================================================
@@ -140,8 +142,6 @@ interface ParserState {
   readonly reasoningSignature?: string
 }
 
-const mediaData = ProviderShared.mediaBytes
-
 // =============================================================================
 // Tool Schema Conversion
 // =============================================================================
@@ -180,8 +180,11 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
     tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
   })
 
-const lowerUserPart = (part: TextPart | MediaPart) =>
-  part.type === "text" ? { text: part.text } : { inlineData: { mimeType: part.mediaType, data: mediaData(part) } }
+const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPart | MediaPart) {
+  if (part.type === "text") return { text: part.text }
+  const media = yield* ProviderShared.validateMedia("Gemini", part, IMAGE_MIMES)
+  return { inlineData: { mimeType: media.mime, data: media.base64 } }
+})
 
 const googleMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ google: metadata })
 
@@ -215,7 +218,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
       for (const part of message.content) {
         if (!ProviderShared.supportsContent(part, ["text", "media"]))
           return yield* ProviderShared.unsupportedContent("Gemini", "user", ["text", "media"])
-        parts.push(lowerUserPart(part))
+        parts.push(yield* lowerUserPart(part))
       }
       contents.push({ role: "user", parts })
       continue
@@ -247,15 +250,34 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
     for (const part of message.content) {
       if (!ProviderShared.supportsContent(part, ["tool-result"]))
         return yield* ProviderShared.unsupportedContent("Gemini", "tool", ["tool-result"])
+      if (part.result.type !== "content") {
+        parts.push({
+          functionResponse: {
+            name: part.name,
+            response: {
+              name: part.name,
+              content: ProviderShared.toolResultText(part),
+            },
+          },
+        })
+        continue
+      }
+      const content: ReadonlyArray<ToolResultContentPart> = part.result.value
+      const text = content.filter((item) => item.type === "text").map((item) => item.text)
       parts.push({
         functionResponse: {
           name: part.name,
           response: {
             name: part.name,
-            content: ProviderShared.toolResultText(part),
+            content: text.join("\n"),
           },
         },
       })
+      for (const item of content) {
+        if (item.type === "text") continue
+        const media = yield* ProviderShared.validateMedia("Gemini", item, IMAGE_MIMES)
+        parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } })
+      }
     }
     contents.push({ role: "user", parts })
   }