EnterpriseAI/ClosedCode
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

feat(ai): add OpenAI image generation (#37714)

Browse Source 
Co-authored-by: Aiden Cline <rekram1-node@users.noreply.github.com>
This commit is contained in:
opencode-agent[bot]
2026-07-19 10:17:49 -05:00
committed by GitHub
co-authored by Aiden Cline
parent c50554d907
commit 04f0a771a3
17 changed files with 984 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/test/fixtures/recordings/openai-images/generates-an-image.json
Vendored
+32
View File
File diff suppressed because one or more lines are too long
packages/ai/test/fixtures/recordings/openai-responses-images/generates-and-edits-an-image-with-the-hosted-tool.json
Vendored
+50
View File
File diff suppressed because one or more lines are too long
packages/ai/test/image.test.ts
+95
View File
@@ -0,0 +1,95 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { Image, ImageClient } from "../src"
import { OpenAI } from "../src/providers"
import { it } from "./lib/effect"
import { dynamicResponse } from "./lib/http"
describe("Image", () => {
it.effect("generates images through the OpenAI Images API", () =>
Effect.gen(function* () {
const response = yield* Image.generate({
model: OpenAI.configure({
apiKey: "test",
baseURL: "https://api.openai.test/v1",
queryParams: { "api-version": "v1" },
http: { body: { deployment: "test" }, headers: { "x-default": "yes" } },
}).image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
count: 2,
size: { width: 1024, height: 1024 },
providerOptions: {
openai: { quality: "high", outputFormat: "webp" },
},
http: {
body: { request_metadata: "value" },
headers: { "x-request": "yes" },
query: { trace: "1" },
},
})
expect(response.images).toHaveLength(2)
expect(response.image?.mediaType).toBe("image/webp")
expect(response.image?.data).toEqual(Uint8Array.from([1, 2, 3]))
expect(response.image?.providerMetadata).toEqual({ openai: { revisedPrompt: "A precise robot" } })
expect(response.usage?.totalTokens).toBe(12)
}).pipe(
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(request.url).toBe("https://api.openai.test/v1/images/generations?api-version=v1&trace=1")
expect(request.headers.get("authorization")).toBe("Bearer test")
expect(request.headers.get("x-default")).toBe("yes")
expect(request.headers.get("x-request")).toBe("yes")
expect(JSON.parse(input.text)).toEqual({
model: "gpt-image-2",
prompt: "A robot tending a rooftop garden",
n: 2,
size: "1024x1024",
quality: "high",
output_format: "webp",
deployment: "test",
request_metadata: "value",
})
return input.respond(
JSON.stringify({
data: [{ b64_json: "AQID", revised_prompt: "A precise robot" }, { b64_json: "BAUG" }],
output_format: "webp",
usage: { input_tokens: 4, output_tokens: 8, total_tokens: 12 },
}),
{ headers: { "content-type": "application/json" } },
)
}),
),
),
),
),
),
)
it.effect("rejects invalid common and OpenAI image options locally", () =>
Image.generate({
model: OpenAI.configure({ apiKey: "test", baseURL: "https://api.openai.test/v1" }).image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
count: -1,
size: { width: -1, height: 0.5 },
providerOptions: { openai: { outputCompression: 101 } },
}).pipe(
Effect.flip,
Effect.tap((error) =>
Effect.sync(() => {
expect(error.reason._tag).toBe("InvalidRequest")
}),
),
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(dynamicResponse(() => Effect.die("invalid request should not reach the provider"))),
),
),
),
)
})
packages/ai/test/provider/openai-images.recorded.test.ts
+40
View File
@@ -0,0 +1,40 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Image } from "../../src"
import { OpenAI } from "../../src/providers"
import { recordedTests } from "../recorded-test"
const model = OpenAI.configure({
apiKey: process.env.OPENAI_API_KEY ?? "fixture",
image: {
providerOptions: {
quality: "low",
outputFormat: "jpeg",
outputCompression: 10,
},
},
}).image("gpt-image-1-mini")
const recorded = recordedTests({
prefix: "openai-images",
provider: "openai",
protocol: "openai-images",
requires: ["OPENAI_API_KEY"],
})
describe("OpenAI Images recorded", () => {
recorded.effect("generates an image", () =>
Effect.gen(function* () {
const response = yield* Image.generate({
model,
prompt: "A simple flat black circle centered on a plain white background.",
size: { width: 1024, height: 1024 },
})
expect(response.images).toHaveLength(1)
expect(response.image?.mediaType).toBe("image/jpeg")
expect(response.image?.data).toBeInstanceOf(Uint8Array)
expect(response.image?.data.length).toBeGreaterThan(0)
}),
)
})
packages/ai/test/provider/openai-responses-images.recorded.test.ts
+66
View File
@@ -0,0 +1,66 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMEvent, Message } from "../../src"
import { OpenAI } from "../../src/providers"
import { recordedTests } from "../recorded-test"
const openai = OpenAI.configure({
apiKey: process.env.OPENAI_API_KEY ?? "fixture",
})
const recorded = recordedTests({
prefix: "openai-responses-images",
provider: "openai",
protocol: "openai-responses",
requires: ["OPENAI_API_KEY"],
})
describe("OpenAI Responses image generation recorded", () => {
recorded.effect("generates and edits an image with the hosted tool", () =>
Effect.gen(function* () {
const initial = Message.user("Generate a simple flat black triangle centered on a plain white background.")
const tools = [
OpenAI.imageGeneration({
action: "auto",
quality: "low",
size: "1024x1024",
outputFormat: "jpeg",
outputCompression: 10,
partialImages: 0,
}),
]
const response = yield* LLM.generate(
LLM.request({
model: openai.responses("gpt-5-mini"),
messages: [initial],
tools,
toolChoice: "image_generation",
}),
)
const result = response.events.find(LLMEvent.is.toolResult)
expect(result).toBeDefined()
expect(result?.providerExecuted).toBe(true)
expect(result?.result.type).toBe("content")
if (result?.result.type !== "content") return
expect(result.result.value).toHaveLength(1)
expect(result.result.value[0]?.type).toBe("file")
if (result.result.value[0]?.type !== "file") return
expect(result.result.value[0].mime).toBe("image/jpeg")
expect(result.result.value[0].uri.startsWith("data:image/jpeg;base64,")).toBe(true)
const edited = yield* LLM.generate(
LLM.request({
model: openai.responses("gpt-5-mini"),
messages: [initial, response.message, Message.user("Now make the triangle blue.")],
tools,
toolChoice: "image_generation",
}),
)
const editedResult = edited.events.find(LLMEvent.is.toolResult)
expect(editedResult?.result.type).toBe("content")
if (editedResult?.result.type !== "content") return
expect(editedResult.result.value[0]?.type).toBe("file")
}),
)
})
packages/ai/test/provider/openai-responses.test.ts
+129 -1
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { ConfigProvider, Effect, Layer, Stream } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import { LLM, LLMError, LLMEvent, Message, Model, ToolCallPart, Usage } from "../../src"
import { LLM, LLMError, LLMEvent, Message, Model, ToolCallPart, ToolResultPart, Usage } from "../../src"
import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route"
import * as Azure from "../../src/providers/azure"
import * as OpenAI from "../../src/providers/openai"
@@ -58,6 +58,39 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("lowers the hosted OpenAI image generation tool", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
prompt: "Show me a rooftop garden.",
tools: [OpenAI.imageGeneration({ action: "generate", quality: "high", size: "1024x1024" })],
toolChoice: "image_generation",
}),
)
expect(prepared.body.tools).toEqual([
{ type: "image_generation", action: "generate", quality: "high", size: "1024x1024" },
])
expect(prepared.body.tool_choice).toEqual({ type: "image_generation" })
}),
)
it.effect("rejects invalid hosted image generation options locally", () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
LLM.request({
model,
prompt: "Show me a rooftop garden.",
tools: [OpenAI.imageGeneration({ outputCompression: -1, partialImages: 4, size: "bogus" })],
}),
).pipe(Effect.flip)
expect(error.reason._tag).toBe("InvalidRequest")
expect(error.message).toContain("image generation tool options are invalid")
}),
)
it.effect("lowers semantic service tier options", () =>
Effect.gen(function* () {
const input = LLM.updateRequest(request, { providerOptions: { openai: { serviceTier: "priority" } } })
@@ -1103,6 +1136,48 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("continues stateless hosted image generation with the generated image", () =>
Effect.gen(function* () {
const imageTool = OpenAI.imageGeneration({ action: "edit" })
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
messages: [
Message.user("Generate a black triangle."),
Message.assistant([
ToolCallPart.make({
id: "ig_1",
name: "image_generation",
input: {},
providerExecuted: true,
providerMetadata: { openai: { itemId: "ig_1" } },
}),
ToolResultPart.make({
id: "ig_1",
name: "image_generation",
result: {
type: "content",
value: [{ type: "file", uri: "data:image/png;base64,AQID", mime: "image/png" }],
},
providerExecuted: true,
providerMetadata: { openai: { itemId: "ig_1" } },
}),
]),
Message.user("Make it blue."),
],
tools: [imageTool],
}),
)
expect(prepared.body.store).toBe(false)
expect(prepared.body.input).toEqual([
{ role: "user", content: [{ type: "input_text", text: "Generate a black triangle." }] },
{ role: "user", content: [{ type: "input_image", image_url: "data:image/png;base64,AQID" }] },
{ role: "user", content: [{ type: "input_text", text: "Make it blue." }] },
])
}),
)
it.effect("joins streamed summary blocks into one continuation reasoning item", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
@@ -1361,6 +1436,59 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("decodes image generation output as image content", () =>
Effect.gen(function* () {
const item = {
type: "image_generation_call",
id: "ig_1",
status: "completed",
result: "AQID",
}
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ type: "response.output_item.done", item },
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
),
),
),
)
expect(response.events.find(LLMEvent.is.toolResult)).toMatchObject({
id: "ig_1",
name: "image_generation",
providerExecuted: true,
result: {
type: "content",
value: [{ type: "file", uri: "data:image/png;base64,AQID", mime: "image/png" }],
},
})
}),
)
it.effect("rejects malformed image generation base64", () =>
Effect.gen(function* () {
const error = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{
type: "response.output_item.done",
item: { type: "image_generation_call", id: "ig_bad", status: "completed", result: "%%%" },
},
{ type: "response.completed", response: {} },
),
),
),
Effect.flip,
)
expect(error.reason._tag).toBe("InvalidProviderOutput")
expect(error.message).toContain("invalid image base64")
}),
)
it.effect("decodes code_interpreter_call as provider-executed events with code input", () =>
Effect.gen(function* () {
const item = {
packages/ai/test/recorded-test.ts
+8 -2
View File
@@ -3,6 +3,8 @@ import { Layer } from "effect"
import * as path from "node:path"
import { fileURLToPath } from "node:url"
import { LLMClient, RequestExecutor, WebSocketExecutor } from "../src/route"
import { ImageClient } from "../src/image-client"
import type { Service as ImageClientService } from "../src/image-client"
import type { Service as LLMClientService } from "../src/route/client"
import type { Service as RequestExecutorService } from "../src/route/executor"
import type { Service as WebSocketExecutorService } from "../src/route/transport/websocket"
@@ -15,7 +17,7 @@ import {
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const FIXTURES_DIR = path.resolve(__dirname, "fixtures", "recordings")
type RecordedEnv = RequestExecutorService | WebSocketExecutorService | LLMClientService
type RecordedEnv = RequestExecutorService | WebSocketExecutorService | LLMClientService | ImageClientService
type RecordedTestsOptions = RecordedGroupOptions & {
readonly options?: HttpRecorder.RecorderOptions
@@ -81,6 +83,10 @@ export const recordedTests = (options: RecordedTestsOptions) =>
),
)
const deps = Layer.mergeAll(requestExecutor, WebSocketExecutor.layer)
return Layer.mergeAll(deps, LLMClient.layer.pipe(Layer.provide(deps)))
return Layer.mergeAll(
deps,
LLMClient.layer.pipe(Layer.provide(deps)),
ImageClient.layer.pipe(Layer.provide(deps)),
)
},
})