feat(core): normalize tool and attachment images at settlement (#37141)
This commit is contained in:
@@ -37,6 +37,7 @@ import { Snapshot } from "./snapshot"
|
|||||||
import { SessionRevert } from "./session/revert"
|
import { SessionRevert } from "./session/revert"
|
||||||
import { Session } from "@opencode-ai/schema/session"
|
import { Session } from "@opencode-ai/schema/session"
|
||||||
import { FSUtil } from "./fs-util"
|
import { FSUtil } from "./fs-util"
|
||||||
|
import { Image } from "./image"
|
||||||
import { Mime } from "./mime"
|
import { Mime } from "./mime"
|
||||||
import type { EventLog } from "@opencode-ai/schema/event-log"
|
import type { EventLog } from "@opencode-ai/schema/event-log"
|
||||||
import { SkillV2 } from "./skill"
|
import { SkillV2 } from "./skill"
|
||||||
@@ -531,9 +532,13 @@ const layer = Layer.effect(
|
|||||||
// continues from the reverted boundary rather than stale post-boundary history.
|
// continues from the reverted boundary rather than stale post-boundary history.
|
||||||
if (session.revert)
|
if (session.revert)
|
||||||
yield* SessionRevert.commit(session).pipe(Effect.provideService(EventV2.Service, events))
|
yield* SessionRevert.commit(session).pipe(Effect.provideService(EventV2.Service, events))
|
||||||
const prompt = yield* resolvePrompt({ text: input.text, files: input.files, agents: input.agents }).pipe(
|
// Resolved lazily so prompt admission only boots location services when an
|
||||||
Effect.provideService(FSUtil.Service, fs),
|
// image attachment actually needs the resizer.
|
||||||
)
|
const image = Image.Service.pipe(Effect.provide(locations.get(session.location)))
|
||||||
|
const prompt = yield* resolvePrompt(
|
||||||
|
{ text: input.text, files: input.files, agents: input.agents },
|
||||||
|
image,
|
||||||
|
).pipe(Effect.provideService(FSUtil.Service, fs))
|
||||||
const messageID = input.id ?? SessionMessage.ID.create()
|
const messageID = input.id ?? SessionMessage.ID.create()
|
||||||
const admittedInput = SessionPending.Message.make({
|
const admittedInput = SessionPending.Message.make({
|
||||||
type: "user",
|
type: "user",
|
||||||
@@ -859,10 +864,13 @@ function synthesizeTerminalShellInfo(started: ShellSchema.Info): ShellSchema.Inf
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (input: PromptInput.Prompt) {
|
const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (
|
||||||
|
input: PromptInput.Prompt,
|
||||||
|
image: Effect.Effect<Image.Interface>,
|
||||||
|
) {
|
||||||
const fs = yield* FSUtil.Service
|
const fs = yield* FSUtil.Service
|
||||||
const files = input.files
|
const files = input.files
|
||||||
? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file), { concurrency: 8 })
|
? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file, image), { concurrency: 8 })
|
||||||
: undefined
|
: undefined
|
||||||
return Prompt.make({ text: input.text, agents: input.agents, files })
|
return Prompt.make({ text: input.text, agents: input.agents, files })
|
||||||
})
|
})
|
||||||
@@ -872,6 +880,7 @@ const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024
|
|||||||
const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(function* (
|
const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(function* (
|
||||||
fs: FSUtil.Interface,
|
fs: FSUtil.Interface,
|
||||||
input: PromptInput.FileAttachment,
|
input: PromptInput.FileAttachment,
|
||||||
|
image: Effect.Effect<Image.Interface>,
|
||||||
) {
|
) {
|
||||||
const resolved = input.uri.startsWith("data:")
|
const resolved = input.uri.startsWith("data:")
|
||||||
? {
|
? {
|
||||||
@@ -900,9 +909,15 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct
|
|||||||
.join("\n"),
|
.join("\n"),
|
||||||
)
|
)
|
||||||
: resolved.bytes
|
: resolved.bytes
|
||||||
return FileAttachment.create({
|
const normalized = yield* normalizeImageAttachment(
|
||||||
data: Base64.make(Buffer.from(content).toString("base64")),
|
input,
|
||||||
|
Base64.make(Buffer.from(content).toString("base64")),
|
||||||
mime,
|
mime,
|
||||||
|
image,
|
||||||
|
)
|
||||||
|
return FileAttachment.create({
|
||||||
|
data: normalized.data,
|
||||||
|
mime: normalized.mime,
|
||||||
source: resolved.source,
|
source: resolved.source,
|
||||||
name: input.name ?? resolved.name,
|
name: input.name ?? resolved.name,
|
||||||
description: input.description,
|
description: input.description,
|
||||||
@@ -910,6 +925,23 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const normalizeImageAttachment = Effect.fn("V2Session.normalizeImageAttachment")(function* (
|
||||||
|
input: PromptInput.FileAttachment,
|
||||||
|
data: Base64,
|
||||||
|
mime: string,
|
||||||
|
image: Effect.Effect<Image.Interface>,
|
||||||
|
) {
|
||||||
|
if (!mime.startsWith("image/")) return { data, mime }
|
||||||
|
const service = yield* image
|
||||||
|
const label = input.name ?? (input.uri.startsWith("data:") ? "inline attachment" : input.uri)
|
||||||
|
const content = { uri: label, content: data, encoding: "base64" as const, mime }
|
||||||
|
const normalized = yield* service.normalize(label, content).pipe(
|
||||||
|
Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)),
|
||||||
|
Effect.mapError((error) => new AttachmentError({ uri: label, message: error.message })),
|
||||||
|
)
|
||||||
|
return { data: Base64.make(normalized.content), mime: normalized.mime }
|
||||||
|
})
|
||||||
|
|
||||||
const readFileAttachment = Effect.fn("V2Session.readFileAttachment")(function* (fs: FSUtil.Interface, uri: string) {
|
const readFileAttachment = Effect.fn("V2Session.readFileAttachment")(function* (fs: FSUtil.Interface, uri: string) {
|
||||||
const url = yield* Effect.try({
|
const url = yield* Effect.try({
|
||||||
try: () => new URL(uri),
|
try: () => new URL(uri),
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import { ToolFailure } from "@opencode-ai/ai"
|
|||||||
import { Effect, Schema } from "effect"
|
import { Effect, Schema } from "effect"
|
||||||
import { FileSystem } from "../filesystem"
|
import { FileSystem } from "../filesystem"
|
||||||
import { FSUtil } from "../fs-util"
|
import { FSUtil } from "../fs-util"
|
||||||
import { Image } from "../image"
|
|
||||||
import { Location } from "../location"
|
import { Location } from "../location"
|
||||||
import { LocationMutation } from "../location-mutation"
|
import { LocationMutation } from "../location-mutation"
|
||||||
import { PermissionV2 } from "../permission"
|
import { PermissionV2 } from "../permission"
|
||||||
@@ -35,7 +34,6 @@ export const Plugin = {
|
|||||||
effect: Effect.fn("ReadTool.Plugin")(function* (ctx: PluginContext) {
|
effect: Effect.fn("ReadTool.Plugin")(function* (ctx: PluginContext) {
|
||||||
const reader = yield* ReadToolFileSystem.Service
|
const reader = yield* ReadToolFileSystem.Service
|
||||||
const mutation = yield* LocationMutation.Service
|
const mutation = yield* LocationMutation.Service
|
||||||
const image = yield* Image.Service
|
|
||||||
const permission = yield* PermissionV2.Service
|
const permission = yield* PermissionV2.Service
|
||||||
const sessionInstructions = yield* SessionInstructions.Service
|
const sessionInstructions = yield* SessionInstructions.Service
|
||||||
const fs = yield* FSUtil.Service
|
const fs = yield* FSUtil.Service
|
||||||
@@ -50,6 +48,12 @@ export const Plugin = {
|
|||||||
"Read a text file or supported image, page through a large UTF-8 text file by line offset, or list a directory page. Relative paths resolve from the current location; absolute paths inside it are accepted, while external absolute paths require external_directory approval.",
|
"Read a text file or supported image, page through a large UTF-8 text file by line offset, or list a directory page. Relative paths resolve from the current location; absolute paths inside it are accepted, while external absolute paths require external_directory approval.",
|
||||||
input: Input,
|
input: Input,
|
||||||
output: Output,
|
output: Output,
|
||||||
|
structured: Schema.toEncoded(Output),
|
||||||
|
// Image base64 reaches the model through content items (normalized generically
|
||||||
|
// at tool settlement); persisting a second copy in structured would store the
|
||||||
|
// original unresized bytes in the message row.
|
||||||
|
toStructuredOutput: ({ output }) =>
|
||||||
|
"encoding" in output && output.encoding === "base64" ? { ...output, content: "" } : output,
|
||||||
toModelOutput: ({ input, output }) => {
|
toModelOutput: ({ input, output }) => {
|
||||||
if (!("encoding" in output) || output.encoding !== "base64" || !SUPPORTED_IMAGE_MIMES.has(output.mime))
|
if (!("encoding" in output) || output.encoding !== "base64" || !SUPPORTED_IMAGE_MIMES.has(output.mime))
|
||||||
return []
|
return []
|
||||||
@@ -117,21 +121,14 @@ export const Plugin = {
|
|||||||
Effect.catch(() => Effect.void),
|
Effect.catch(() => Effect.void),
|
||||||
Effect.catchDefect(() => Effect.void),
|
Effect.catchDefect(() => Effect.void),
|
||||||
)
|
)
|
||||||
if ("encoding" in content && content.encoding === "base64" && SUPPORTED_IMAGE_MIMES.has(content.mime)) {
|
if ("encoding" in content && content.encoding === "base64" && !SUPPORTED_IMAGE_MIMES.has(content.mime))
|
||||||
return yield* image
|
|
||||||
.normalize(resource, { ...content, encoding: "base64" })
|
|
||||||
.pipe(Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)))
|
|
||||||
}
|
|
||||||
if ("encoding" in content && content.encoding === "base64")
|
|
||||||
return yield* Effect.fail(new ReadToolFileSystem.BinaryFileError({ resource }))
|
return yield* Effect.fail(new ReadToolFileSystem.BinaryFileError({ resource }))
|
||||||
return content
|
return content
|
||||||
}).pipe(
|
}).pipe(
|
||||||
Effect.mapError((error) => {
|
Effect.mapError((error) => {
|
||||||
const message =
|
const message =
|
||||||
error instanceof ReadToolFileSystem.BinaryFileError ||
|
error instanceof ReadToolFileSystem.BinaryFileError ||
|
||||||
error instanceof ReadToolFileSystem.MediaIngestLimitError ||
|
error instanceof ReadToolFileSystem.MediaIngestLimitError
|
||||||
error instanceof Image.DecodeError ||
|
|
||||||
error instanceof Image.SizeError
|
|
||||||
? error.message
|
? error.message
|
||||||
: `Unable to read ${input.path}`
|
: `Unable to read ${input.path}`
|
||||||
return new ToolFailure({ message, error })
|
return new ToolFailure({ message, error })
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ export * as ToolRegistry from "./registry"
|
|||||||
import { ToolOutput, type ToolCall, type ToolDefinition, type ToolResultValue } from "@opencode-ai/ai"
|
import { ToolOutput, type ToolCall, type ToolDefinition, type ToolResultValue } from "@opencode-ai/ai"
|
||||||
import { Context, Effect, Layer, Scope } from "effect"
|
import { Context, Effect, Layer, Scope } from "effect"
|
||||||
import type { AgentV2 } from "../agent"
|
import type { AgentV2 } from "../agent"
|
||||||
|
import { Image } from "../image"
|
||||||
import { PermissionV2 } from "../permission"
|
import { PermissionV2 } from "../permission"
|
||||||
import { SessionMessage } from "../session/message"
|
import { SessionMessage } from "../session/message"
|
||||||
import { SessionSchema } from "../session/schema"
|
import { SessionSchema } from "../session/schema"
|
||||||
@@ -57,6 +58,40 @@ const registryLayer = Layer.effect(
|
|||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const resources = yield* ToolOutputStore.Service
|
const resources = yield* ToolOutputStore.Service
|
||||||
const toolHooks = yield* ToolHooks.Service
|
const toolHooks = yield* ToolHooks.Service
|
||||||
|
const image = yield* Image.Service
|
||||||
|
|
||||||
|
type NormalizedItem = ToolOutput["content"][number] | "decode" | "size"
|
||||||
|
const normalizeImages = Effect.fn("ToolRegistry.normalizeImages")(function* (content: ToolOutput["content"]) {
|
||||||
|
const normalized = yield* Effect.forEach(content, (item): Effect.Effect<NormalizedItem> => {
|
||||||
|
if (item.type !== "file" || !item.mime.startsWith("image/")) return Effect.succeed(item)
|
||||||
|
// RFC 2397 permits parameters between the mime and ";base64".
|
||||||
|
const base64 = /^data:[^,]*;base64,(.*)$/s.exec(item.uri)?.[1]
|
||||||
|
if (base64 === undefined) return Effect.succeed(item)
|
||||||
|
const resource = item.name ?? `${item.mime} tool output`
|
||||||
|
return image
|
||||||
|
.normalize(resource, { uri: resource, content: base64, encoding: "base64", mime: item.mime })
|
||||||
|
.pipe(
|
||||||
|
Effect.map((result) => ({
|
||||||
|
...item,
|
||||||
|
uri: `data:${result.mime};base64,${result.content}`,
|
||||||
|
mime: result.mime,
|
||||||
|
})),
|
||||||
|
Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(item)),
|
||||||
|
Effect.catchTag("Image.DecodeError", () => Effect.succeed("decode" as const)),
|
||||||
|
Effect.catchTag("Image.SizeError", () => Effect.succeed("size" as const)),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
const note = (reason: "decode" | "size", text: string) => {
|
||||||
|
const count = normalized.filter((item) => item === reason).length
|
||||||
|
if (count === 0) return []
|
||||||
|
return [{ type: "text" as const, text: `[${count} image${count === 1 ? "" : "s"} omitted: ${text}]` }]
|
||||||
|
}
|
||||||
|
return [
|
||||||
|
...normalized.filter((item) => typeof item !== "string"),
|
||||||
|
...note("decode", "could not be decoded."),
|
||||||
|
...note("size", "could not be resized below the image size limit."),
|
||||||
|
]
|
||||||
|
})
|
||||||
type Registration = {
|
type Registration = {
|
||||||
readonly tool: AnyTool
|
readonly tool: AnyTool
|
||||||
readonly name: string
|
readonly name: string
|
||||||
@@ -84,10 +119,11 @@ const registryLayer = Layer.effect(
|
|||||||
agent: input.agent,
|
agent: input.agent,
|
||||||
messageID: input.messageID,
|
messageID: input.messageID,
|
||||||
callID: input.call.id,
|
callID: input.call.id,
|
||||||
progress: (update) =>
|
progress: (update) => {
|
||||||
input.progress?.({
|
const progress = input.progress
|
||||||
structured: update.structured,
|
if (!progress) return Effect.void
|
||||||
content: (update.content ?? []).map((part) =>
|
return normalizeImages(
|
||||||
|
(update.content ?? []).map((part) =>
|
||||||
part.type === "text"
|
part.type === "text"
|
||||||
? { type: "text" as const, text: part.text }
|
? { type: "text" as const, text: part.text }
|
||||||
: {
|
: {
|
||||||
@@ -97,7 +133,8 @@ const registryLayer = Layer.effect(
|
|||||||
name: part.name,
|
name: part.name,
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
}) ?? Effect.void,
|
).pipe(Effect.flatMap((content) => progress({ structured: update.structured, content })))
|
||||||
|
},
|
||||||
},
|
},
|
||||||
).pipe(
|
).pipe(
|
||||||
Effect.map((output) => ({ output })),
|
Effect.map((output) => ({ output })),
|
||||||
@@ -115,7 +152,7 @@ const registryLayer = Layer.effect(
|
|||||||
const bounded = yield* resources.bound({
|
const bounded = yield* resources.bound({
|
||||||
sessionID: input.sessionID,
|
sessionID: input.sessionID,
|
||||||
callID: input.call.id,
|
callID: input.call.id,
|
||||||
output: pending.output,
|
output: { structured: pending.output.structured, content: yield* normalizeImages(pending.output.content) },
|
||||||
})
|
})
|
||||||
const result = ToolOutput.toResultValue(bounded.output)
|
const result = ToolOutput.toResultValue(bounded.output)
|
||||||
settlement =
|
settlement =
|
||||||
@@ -232,11 +269,11 @@ function whollyDisabled(action: string, rules: PermissionV2.Ruleset) {
|
|||||||
export const node = makeLocationNode({
|
export const node = makeLocationNode({
|
||||||
service: Service,
|
service: Service,
|
||||||
layer,
|
layer,
|
||||||
deps: [ToolOutputStore.node, ToolHooks.node],
|
deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
|
||||||
})
|
})
|
||||||
|
|
||||||
export const toolsNode = makeLocationNode({
|
export const toolsNode = makeLocationNode({
|
||||||
service: Tools.Service,
|
service: Tools.Service,
|
||||||
layer,
|
layer,
|
||||||
deps: [ToolOutputStore.node, ToolHooks.node],
|
deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
import { Image } from "@opencode-ai/core/image"
|
||||||
|
import { Effect, Layer } from "effect"
|
||||||
|
|
||||||
|
/** Passthrough resizer for tests that build ToolRegistry.node without a Location. */
|
||||||
|
export const imagePassthrough = Layer.mock(Image.Service, {
|
||||||
|
normalize: (_resource, content) => Effect.succeed(content),
|
||||||
|
})
|
||||||
@@ -29,7 +29,9 @@ import { McpTool } from "@opencode-ai/core/tool/mcp"
|
|||||||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||||
import { Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect"
|
import { Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect"
|
||||||
|
import { Image } from "@opencode-ai/core/image"
|
||||||
import { testEffect } from "./lib/effect"
|
import { testEffect } from "./lib/effect"
|
||||||
|
import { imagePassthrough } from "./lib/image"
|
||||||
import { location } from "./fixture/location"
|
import { location } from "./fixture/location"
|
||||||
import { settleTool, toolDefinitions, toolIdentity, waitForTool } from "./lib/tool"
|
import { settleTool, toolDefinitions, toolIdentity, waitForTool } from "./lib/tool"
|
||||||
|
|
||||||
@@ -250,6 +252,7 @@ const it = testEffect(
|
|||||||
[PermissionV2.node, permissions],
|
[PermissionV2.node, permissions],
|
||||||
[EventV2.node, events],
|
[EventV2.node, events],
|
||||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||||
|
[Image.node, imagePassthrough],
|
||||||
]),
|
]),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect } from "bun:test"
|
import { describe, expect } from "bun:test"
|
||||||
import { DateTime, Effect, Fiber, Layer, Schema, Stream } from "effect"
|
import { DateTime, Effect, Fiber, Layer, LayerMap, Schema, Stream } from "effect"
|
||||||
import { mkdtemp, rm } from "fs/promises"
|
import { mkdtemp, rm } from "fs/promises"
|
||||||
import { tmpdir } from "os"
|
import { tmpdir } from "os"
|
||||||
import path from "path"
|
import path from "path"
|
||||||
@@ -24,7 +24,10 @@ import { SessionExecution } from "@opencode-ai/core/session/execution"
|
|||||||
import { SessionPending } from "@opencode-ai/core/session/pending"
|
import { SessionPending } from "@opencode-ai/core/session/pending"
|
||||||
import { SessionPendingTable, SessionMessageTable, SessionTable } from "@opencode-ai/core/session/sql"
|
import { SessionPendingTable, SessionMessageTable, SessionTable } from "@opencode-ai/core/session/sql"
|
||||||
import { SessionStore } from "@opencode-ai/core/session/store"
|
import { SessionStore } from "@opencode-ai/core/session/store"
|
||||||
|
import { LocationServiceMap } from "@opencode-ai/core/location-service-map"
|
||||||
|
import type { LocationServices } from "@opencode-ai/core/location-services"
|
||||||
import { testEffect } from "./lib/effect"
|
import { testEffect } from "./lib/effect"
|
||||||
|
import { imagePassthrough } from "./lib/image"
|
||||||
|
|
||||||
const executionCalls: SessionV2.ID[] = []
|
const executionCalls: SessionV2.ID[] = []
|
||||||
const interruptCalls: SessionV2.ID[] = []
|
const interruptCalls: SessionV2.ID[] = []
|
||||||
@@ -49,10 +52,22 @@ const execution = Layer.succeed(
|
|||||||
awaitIdle: () => Effect.void,
|
awaitIdle: () => Effect.void,
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
const locations = Layer.effect(
|
||||||
|
LocationServiceMap.Service,
|
||||||
|
LayerMap.make(
|
||||||
|
() =>
|
||||||
|
// Attachment admission only needs the location-scoped Image service.
|
||||||
|
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion
|
||||||
|
imagePassthrough as unknown as Layer.Layer<LocationServices>,
|
||||||
|
),
|
||||||
|
)
|
||||||
const it = testEffect(
|
const it = testEffect(
|
||||||
AppNodeBuilder.build(
|
AppNodeBuilder.build(
|
||||||
LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node, SessionV2.node]),
|
LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node, SessionV2.node]),
|
||||||
[[SessionExecution.node, execution]],
|
[
|
||||||
|
[SessionExecution.node, execution],
|
||||||
|
[LocationServiceMap.node, locations],
|
||||||
|
],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
const sessionID = SessionV2.ID.make("ses_prompt_test")
|
const sessionID = SessionV2.ID.make("ses_prompt_test")
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { Tool } from "@opencode-ai/core/tool/tool"
|
|||||||
import { AgentV2 } from "@opencode-ai/core/agent"
|
import { AgentV2 } from "@opencode-ai/core/agent"
|
||||||
import type { PermissionV2 } from "@opencode-ai/core/permission"
|
import type { PermissionV2 } from "@opencode-ai/core/permission"
|
||||||
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
|
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
|
||||||
|
import { Image } from "@opencode-ai/core/image"
|
||||||
import { SessionV2 } from "@opencode-ai/core/session"
|
import { SessionV2 } from "@opencode-ai/core/session"
|
||||||
import { SessionMessage } from "@opencode-ai/core/session/message"
|
import { SessionMessage } from "@opencode-ai/core/session/message"
|
||||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||||
@@ -28,7 +29,28 @@ const outputStore = Layer.mock(ToolOutputStore.Service, {
|
|||||||
)
|
)
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [[ToolOutputStore.node, outputStore]])
|
const imageStore = Layer.mock(Image.Service, {
|
||||||
|
normalize: (resource, content) => {
|
||||||
|
if (resource === "corrupt.png") return Effect.fail(new Image.DecodeError({ resource }))
|
||||||
|
if (resource === "too-large.png")
|
||||||
|
return Effect.fail(
|
||||||
|
new Image.SizeError({
|
||||||
|
resource,
|
||||||
|
width: 9_000,
|
||||||
|
height: 9_000,
|
||||||
|
bytes: content.content.length,
|
||||||
|
maxWidth: 2_000,
|
||||||
|
maxHeight: 2_000,
|
||||||
|
maxBytes: 5,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
return Effect.succeed({ ...content, content: "bm9ybWFsaXplZA==", mime: "image/jpeg" })
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [
|
||||||
|
[ToolOutputStore.node, outputStore],
|
||||||
|
[Image.node, imageStore],
|
||||||
|
])
|
||||||
const it = testEffect(registryLayer)
|
const it = testEffect(registryLayer)
|
||||||
const identity = {
|
const identity = {
|
||||||
agent: AgentV2.ID.make("build"),
|
agent: AgentV2.ID.make("build"),
|
||||||
@@ -255,6 +277,75 @@ describe("ToolRegistry", () => {
|
|||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
it.effect("normalizes image tool output at settlement and drops unresizable images", () =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const service = yield* ToolRegistry.Service
|
||||||
|
yield* service.register({
|
||||||
|
snapshot: Tool.make({
|
||||||
|
description: "Return images",
|
||||||
|
input: Schema.Struct({ text: Schema.String }),
|
||||||
|
output: Schema.Struct({ text: Schema.String }),
|
||||||
|
execute: ({ text }) => Effect.succeed({ text }),
|
||||||
|
toModelOutput: ({ output }) => [
|
||||||
|
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "frame.png" },
|
||||||
|
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "too-large.png" },
|
||||||
|
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "corrupt.png" },
|
||||||
|
{ type: "text", text: output.text },
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
}, { codemode: false })
|
||||||
|
|
||||||
|
const settlement = yield* settleTool(service, call("snapshot"))
|
||||||
|
expect(settlement.output?.content).toEqual([
|
||||||
|
{ type: "file", uri: "data:image/jpeg;base64,bm9ybWFsaXplZA==", mime: "image/jpeg", name: "frame.png" },
|
||||||
|
{ type: "text", text: "snapshot" },
|
||||||
|
{ type: "text", text: "[1 image omitted: could not be decoded.]" },
|
||||||
|
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
|
||||||
|
])
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
it.effect("normalizes image progress content before it is published", () =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const service = yield* ToolRegistry.Service
|
||||||
|
yield* service.register({
|
||||||
|
progressive: Tool.make({
|
||||||
|
description: "Emit image progress",
|
||||||
|
input: Schema.Struct({ text: Schema.String }),
|
||||||
|
output: Schema.Struct({ text: Schema.String }),
|
||||||
|
execute: ({ text }, context) =>
|
||||||
|
context
|
||||||
|
.progress({
|
||||||
|
structured: { stage: "capture" },
|
||||||
|
content: [
|
||||||
|
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "frame.png" },
|
||||||
|
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "too-large.png" },
|
||||||
|
],
|
||||||
|
})
|
||||||
|
.pipe(Effect.as({ text })),
|
||||||
|
}),
|
||||||
|
}, { codemode: false })
|
||||||
|
|
||||||
|
const updates: ToolRegistry.Progress[] = []
|
||||||
|
yield* settleTool(service, {
|
||||||
|
...call("progressive"),
|
||||||
|
progress: (update) =>
|
||||||
|
Effect.sync(() => {
|
||||||
|
updates.push(update)
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
expect(updates).toEqual([
|
||||||
|
{
|
||||||
|
structured: { stage: "capture" },
|
||||||
|
content: [
|
||||||
|
{ type: "file", uri: "data:image/jpeg;base64,bm9ybWFsaXplZA==", mime: "image/jpeg", name: "frame.png" },
|
||||||
|
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
])
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
it.effect("enforces transformed codecs at execution and projection boundaries", () =>
|
it.effect("enforces transformed codecs at execution and projection boundaries", () =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const service = yield* ToolRegistry.Service
|
const service = yield* ToolRegistry.Service
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ import { SessionV2 } from "@opencode-ai/core/session"
|
|||||||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||||
import { QuestionTool } from "@opencode-ai/core/tool/question"
|
import { QuestionTool } from "@opencode-ai/core/tool/question"
|
||||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||||
|
import { Image } from "@opencode-ai/core/image"
|
||||||
import { testEffect } from "./lib/effect"
|
import { testEffect } from "./lib/effect"
|
||||||
|
import { imagePassthrough } from "./lib/image"
|
||||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||||
|
|
||||||
@@ -84,6 +86,7 @@ const it = testEffect(
|
|||||||
[PermissionV2.node, permission],
|
[PermissionV2.node, permission],
|
||||||
[Form.node, form],
|
[Form.node, form],
|
||||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||||
|
[Image.node, imagePassthrough],
|
||||||
]),
|
]),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -291,6 +291,9 @@ describe("ReadTool", () => {
|
|||||||
name: "pixel.png",
|
name: "pixel.png",
|
||||||
mime: "image/png",
|
mime: "image/png",
|
||||||
encoding: "base64",
|
encoding: "base64",
|
||||||
|
// Image base64 is carried by the content file item only; structured is slimmed
|
||||||
|
// so the original bytes are never persisted twice.
|
||||||
|
content: "",
|
||||||
})
|
})
|
||||||
expect(settled.output?.content).toMatchObject([
|
expect(settled.output?.content).toMatchObject([
|
||||||
{ type: "text", text: "Image read successfully" },
|
{ type: "text", text: "Image read successfully" },
|
||||||
@@ -364,7 +367,7 @@ describe("ReadTool", () => {
|
|||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("rejects invalid image data returned by the filesystem", () =>
|
it.effect("drops undecodable image data at settlement", () =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
readResult = {
|
readResult = {
|
||||||
uri: "file:///truncated.png",
|
uri: "file:///truncated.png",
|
||||||
@@ -381,11 +384,17 @@ describe("ReadTool", () => {
|
|||||||
...toolIdentity,
|
...toolIdentity,
|
||||||
call: { type: "tool-call", id: "call-truncated-image", name: "read", input: { path: "truncated.png" } },
|
call: { type: "tool-call", id: "call-truncated-image", name: "read", input: { path: "truncated.png" } },
|
||||||
}),
|
}),
|
||||||
).toEqual({ type: "error", value: "Image could not be decoded: truncated.png" })
|
).toEqual({
|
||||||
|
type: "content",
|
||||||
|
value: [
|
||||||
|
{ type: "text", text: "Image read successfully" },
|
||||||
|
{ type: "text", text: "[1 image omitted: could not be decoded.]" },
|
||||||
|
],
|
||||||
|
})
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("rejects oversized images when resizing is disabled", () =>
|
it.effect("drops oversized images at settlement when resizing is disabled", () =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
|
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
|
||||||
const source = new photon.PhotonImage(new Uint8Array(Array.from({ length: 16 * 4 }, () => 255)), 16, 1)
|
const source = new photon.PhotonImage(new Uint8Array(Array.from({ length: 16 * 4 }, () => 255)), 16, 1)
|
||||||
@@ -409,14 +418,20 @@ describe("ReadTool", () => {
|
|||||||
}),
|
}),
|
||||||
]
|
]
|
||||||
const registry = yield* ToolRegistry.Service
|
const registry = yield* ToolRegistry.Service
|
||||||
const result = yield* executeTool(registry, {
|
|
||||||
|
expect(
|
||||||
|
yield* executeTool(registry, {
|
||||||
sessionID,
|
sessionID,
|
||||||
...toolIdentity,
|
...toolIdentity,
|
||||||
call: { type: "tool-call", id: "call-wide-image", name: "read", input: { path: "wide.png" } },
|
call: { type: "tool-call", id: "call-wide-image", name: "read", input: { path: "wide.png" } },
|
||||||
|
}),
|
||||||
|
).toEqual({
|
||||||
|
type: "content",
|
||||||
|
value: [
|
||||||
|
{ type: "text", text: "Image read successfully" },
|
||||||
|
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
|
||||||
|
],
|
||||||
})
|
})
|
||||||
|
|
||||||
expect(result.type).toBe("error")
|
|
||||||
if (result.type === "error") expect(result.value).toContain("exceeding configured limits 4x2000")
|
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -460,7 +475,7 @@ describe("ReadTool", () => {
|
|||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
it.effect("enforces max base64 bytes after resize attempts", () =>
|
it.effect("drops images that cannot fit max base64 bytes after resize attempts", () =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
||||||
readResult = {
|
readResult = {
|
||||||
@@ -481,14 +496,20 @@ describe("ReadTool", () => {
|
|||||||
}),
|
}),
|
||||||
]
|
]
|
||||||
const registry = yield* ToolRegistry.Service
|
const registry = yield* ToolRegistry.Service
|
||||||
const result = yield* executeTool(registry, {
|
|
||||||
|
expect(
|
||||||
|
yield* executeTool(registry, {
|
||||||
sessionID,
|
sessionID,
|
||||||
...toolIdentity,
|
...toolIdentity,
|
||||||
call: { type: "tool-call", id: "call-max-bytes", name: "read", input: { path: "pixel.png" } },
|
call: { type: "tool-call", id: "call-max-bytes", name: "read", input: { path: "pixel.png" } },
|
||||||
|
}),
|
||||||
|
).toEqual({
|
||||||
|
type: "content",
|
||||||
|
value: [
|
||||||
|
{ type: "text", text: "Image read successfully" },
|
||||||
|
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
|
||||||
|
],
|
||||||
})
|
})
|
||||||
|
|
||||||
expect(result.type).toBe("error")
|
|
||||||
if (result.type === "error") expect(result.value).toContain("/1 bytes")
|
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,9 @@ import { SkillTool } from "@opencode-ai/core/tool/skill"
|
|||||||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||||
import { tmpdir } from "./fixture/tmpdir"
|
import { tmpdir } from "./fixture/tmpdir"
|
||||||
|
import { Image } from "@opencode-ai/core/image"
|
||||||
import { it } from "./lib/effect"
|
import { it } from "./lib/effect"
|
||||||
|
import { imagePassthrough } from "./lib/image"
|
||||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||||
@@ -90,6 +92,7 @@ describe("SkillTool", () => {
|
|||||||
[PermissionV2.node, permission],
|
[PermissionV2.node, permission],
|
||||||
[SkillV2.node, skills],
|
[SkillV2.node, skills],
|
||||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||||
|
[Image.node, imagePassthrough],
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
|||||||
import { WebFetchTool } from "@opencode-ai/core/tool/webfetch"
|
import { WebFetchTool } from "@opencode-ai/core/tool/webfetch"
|
||||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||||
|
import { Image } from "@opencode-ai/core/image"
|
||||||
import { testEffect } from "./lib/effect"
|
import { testEffect } from "./lib/effect"
|
||||||
|
import { imagePassthrough } from "./lib/image"
|
||||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||||
|
|
||||||
const webFetchToolNode = makeLocationNode({
|
const webFetchToolNode = makeLocationNode({
|
||||||
@@ -50,6 +52,7 @@ const toolLayer = (replacements: LayerNode.Replacements = []) =>
|
|||||||
AppNodeBuilder.build(LayerNode.group([ToolRegistry.node, ToolRegistry.toolsNode, webFetchToolNode]), [
|
AppNodeBuilder.build(LayerNode.group([ToolRegistry.node, ToolRegistry.toolsNode, webFetchToolNode]), [
|
||||||
[PermissionV2.node, permission],
|
[PermissionV2.node, permission],
|
||||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||||
|
[Image.node, imagePassthrough],
|
||||||
...replacements,
|
...replacements,
|
||||||
])
|
])
|
||||||
const it = testEffect(toolLayer([[LayerNodePlatform.httpClient, http]]))
|
const it = testEffect(toolLayer([[LayerNodePlatform.httpClient, http]]))
|
||||||
|
|||||||
@@ -10,7 +10,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
|||||||
import { WebSearchTool } from "@opencode-ai/core/tool/websearch"
|
import { WebSearchTool } from "@opencode-ai/core/tool/websearch"
|
||||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||||
|
import { Image } from "@opencode-ai/core/image"
|
||||||
import { testEffect } from "./lib/effect"
|
import { testEffect } from "./lib/effect"
|
||||||
|
import { imagePassthrough } from "./lib/image"
|
||||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||||
|
|
||||||
const webSearchToolNode = makeLocationNode({
|
const webSearchToolNode = makeLocationNode({
|
||||||
@@ -138,6 +140,7 @@ const it = testEffect(
|
|||||||
[LayerNodePlatform.httpClient, http],
|
[LayerNodePlatform.httpClient, http],
|
||||||
[WebSearchTool.configNode, websearchConfig],
|
[WebSearchTool.configNode, websearchConfig],
|
||||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||||
|
[Image.node, imagePassthrough],
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user