Add native LLM core foundation (#24712)
This commit is contained in:
Kit Langton
@@ -0,0 +1,46 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMError } from "../../src"
|
||||
import { LLMClient } from "../../src/route"
|
||||
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
|
||||
import { weatherToolName } from "../recorded-scenarios"
|
||||
import { recordedTests } from "../recorded-test"
|
||||
|
||||
const model = AnthropicMessages.model({
|
||||
id: "claude-haiku-4-5-20251001",
|
||||
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
|
||||
})
|
||||
|
||||
const malformedToolOrderRequest = LLM.request({
|
||||
id: "recorded_anthropic_malformed_tool_order",
|
||||
model,
|
||||
messages: [
|
||||
LLM.assistant([
|
||||
LLM.toolCall({ id: "call_1", name: weatherToolName, input: { city: "Paris" } }),
|
||||
{ type: "text", text: "I will check the weather." },
|
||||
]),
|
||||
LLM.toolMessage({ id: "call_1", name: weatherToolName, result: { temperature: "72F" } }),
|
||||
LLM.user("Use that result to answer briefly."),
|
||||
],
|
||||
tools: [{ name: weatherToolName, description: "Get weather", inputSchema: { type: "object", properties: {} } }],
|
||||
})
|
||||
|
||||
const recorded = recordedTests({
|
||||
prefix: "anthropic-messages",
|
||||
provider: "anthropic",
|
||||
protocol: "anthropic-messages",
|
||||
requires: ["ANTHROPIC_API_KEY"],
|
||||
options: { requestHeaders: ["content-type", "anthropic-version"] },
|
||||
})
|
||||
|
||||
describe("Anthropic Messages sad-path recorded", () => {
|
||||
recorded.effect.with("rejects malformed assistant tool order", { tags: ["tool", "sad-path"] }, () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(malformedToolOrderRequest).pipe(Effect.flip)
|
||||
|
||||
expect(error).toBeInstanceOf(LLMError)
|
||||
expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
|
||||
expect(error.message).toContain("HTTP 400")
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,377 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { CacheHint, LLM, LLMError } from "../../src"
|
||||
import { LLMClient } from "../../src/route"
|
||||
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
|
||||
import { it } from "../lib/effect"
|
||||
import { fixedResponse } from "../lib/http"
|
||||
import { sseEvents } from "../lib/sse"
|
||||
|
||||
const model = AnthropicMessages.model({
|
||||
id: "claude-sonnet-4-5",
|
||||
baseURL: "https://api.anthropic.test/v1/",
|
||||
headers: { "x-api-key": "test" },
|
||||
})
|
||||
|
||||
const request = LLM.request({
|
||||
id: "req_1",
|
||||
model,
|
||||
system: { type: "text", text: "You are concise.", cache: new CacheHint({ type: "ephemeral" }) },
|
||||
prompt: "Say hello.",
|
||||
generation: { maxTokens: 20, temperature: 0 },
|
||||
})
|
||||
|
||||
describe("Anthropic Messages route", () => {
|
||||
it.effect("prepares Anthropic Messages target", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(request)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "claude-sonnet-4-5",
|
||||
system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
|
||||
messages: [{ role: "user", content: [{ type: "text", text: "Say hello." }] }],
|
||||
stream: true,
|
||||
max_tokens: 20,
|
||||
temperature: 0,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("prepares tool call and tool result messages", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_tool_result",
|
||||
model,
|
||||
messages: [
|
||||
LLM.user("What is the weather?"),
|
||||
LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
|
||||
LLM.toolMessage({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "claude-sonnet-4-5",
|
||||
messages: [
|
||||
{ role: "user", content: [{ type: "text", text: "What is the weather?" }] },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "tool_use", id: "call_1", name: "lookup", input: { query: "weather" } }],
|
||||
},
|
||||
{ role: "user", content: [{ type: "tool_result", tool_use_id: "call_1", content: '{"forecast":"sunny"}' }] },
|
||||
],
|
||||
stream: true,
|
||||
max_tokens: 4096,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers preserved Anthropic reasoning signature metadata", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
model,
|
||||
messages: [
|
||||
LLM.assistant([
|
||||
{ type: "reasoning", text: "thinking", providerMetadata: { anthropic: { signature: "sig_1" } } },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
messages: [{ role: "assistant", content: [{ type: "thinking", thinking: "thinking", signature: "sig_1" }] }],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses text, reasoning, and usage stream fixtures", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{ type: "message_start", message: { usage: { input_tokens: 5, cache_read_input_tokens: 1 } } },
|
||||
{ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Hello" } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "!" } },
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{ type: "content_block_start", index: 1, content_block: { type: "thinking", thinking: "" } },
|
||||
{ type: "content_block_delta", index: 1, delta: { type: "thinking_delta", thinking: "thinking" } },
|
||||
{ type: "content_block_delta", index: 1, delta: { type: "signature_delta", signature: "sig_1" } },
|
||||
{ type: "content_block_stop", index: 1 },
|
||||
{
|
||||
type: "message_delta",
|
||||
delta: { stop_reason: "end_turn", stop_sequence: "\n\nHuman:" },
|
||||
usage: { output_tokens: 2 },
|
||||
},
|
||||
{ type: "message_stop" },
|
||||
)
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.text).toBe("Hello!")
|
||||
expect(response.reasoning).toBe("thinking")
|
||||
expect(response.usage).toMatchObject({
|
||||
inputTokens: 5,
|
||||
outputTokens: 2,
|
||||
cacheReadInputTokens: 1,
|
||||
totalTokens: 7,
|
||||
})
|
||||
expect(response.events.find((event) => event.type === "reasoning-delta" && event.text === "")).toMatchObject({
|
||||
providerMetadata: { anthropic: { signature: "sig_1" } },
|
||||
})
|
||||
expect(response.events.at(-1)).toMatchObject({
|
||||
type: "request-finish",
|
||||
reason: "stop",
|
||||
providerMetadata: { anthropic: { stopSequence: "\n\nHuman:" } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("assembles streamed tool call input", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
|
||||
{ type: "content_block_start", index: 0, content_block: { type: "tool_use", id: "call_1", name: "lookup" } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: '{"query"' } },
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: ':"weather"}' } },
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: { output_tokens: 1 } },
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.toolCalls).toEqual([
|
||||
{ type: "tool-call", id: "call_1", name: "lookup", input: { query: "weather" } },
|
||||
])
|
||||
expect(response.events).toEqual([
|
||||
{ type: "tool-input-delta", id: "call_1", name: "lookup", text: '{"query"' },
|
||||
{ type: "tool-input-delta", id: "call_1", name: "lookup", text: ':"weather"}' },
|
||||
{ type: "tool-call", id: "call_1", name: "lookup", input: { query: "weather" } },
|
||||
{
|
||||
type: "request-finish",
|
||||
reason: "tool-calls",
|
||||
usage: { inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { input_tokens: 5, output_tokens: 1 } },
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("emits provider-error events for mid-stream provider errors", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(sseEvents({ type: "error", error: { type: "overloaded_error", message: "Overloaded" } })),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.events).toEqual([{ type: "provider-error", message: "Overloaded" }])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails HTTP provider errors before stream parsing", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse('{"type":"error","error":{"type":"invalid_request_error","message":"Bad request"}}', {
|
||||
status: 400,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error).toBeInstanceOf(LLMError)
|
||||
expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
|
||||
expect(error.message).toContain("HTTP 400")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes server_tool_use + web_search_tool_result as provider-executed events", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 0,
|
||||
content_block: { type: "server_tool_use", id: "srvtoolu_abc", name: "web_search" },
|
||||
},
|
||||
{
|
||||
type: "content_block_delta",
|
||||
index: 0,
|
||||
delta: { type: "input_json_delta", partial_json: '{"query":"effect 4"}' },
|
||||
},
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 1,
|
||||
content_block: {
|
||||
type: "web_search_tool_result",
|
||||
tool_use_id: "srvtoolu_abc",
|
||||
content: [{ type: "web_search_result", url: "https://example.com", title: "Example" }],
|
||||
},
|
||||
},
|
||||
{ type: "content_block_stop", index: 1 },
|
||||
{ type: "content_block_start", index: 2, content_block: { type: "text", text: "" } },
|
||||
{ type: "content_block_delta", index: 2, delta: { type: "text_delta", text: "Found it." } },
|
||||
{ type: "content_block_stop", index: 2 },
|
||||
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 8 } },
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "web_search", description: "Web search", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
const toolCall = response.events.find((event) => event.type === "tool-call")
|
||||
expect(toolCall).toEqual({
|
||||
type: "tool-call",
|
||||
id: "srvtoolu_abc",
|
||||
name: "web_search",
|
||||
input: { query: "effect 4" },
|
||||
providerExecuted: true,
|
||||
})
|
||||
const toolResult = response.events.find((event) => event.type === "tool-result")
|
||||
expect(toolResult).toEqual({
|
||||
type: "tool-result",
|
||||
id: "srvtoolu_abc",
|
||||
name: "web_search",
|
||||
result: { type: "json", value: [{ type: "web_search_result", url: "https://example.com", title: "Example" }] },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { anthropic: { blockType: "web_search_tool_result" } },
|
||||
})
|
||||
expect(response.text).toBe("Found it.")
|
||||
expect(response.events.at(-1)).toMatchObject({ type: "request-finish", reason: "stop" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes web_search_tool_result_error as provider-executed error result", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{ type: "message_start", message: { usage: { input_tokens: 5 } } },
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 0,
|
||||
content_block: { type: "server_tool_use", id: "srvtoolu_x", name: "web_search" },
|
||||
},
|
||||
{ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: '{"query":"q"}' } },
|
||||
{ type: "content_block_stop", index: 0 },
|
||||
{
|
||||
type: "content_block_start",
|
||||
index: 1,
|
||||
content_block: {
|
||||
type: "web_search_tool_result",
|
||||
tool_use_id: "srvtoolu_x",
|
||||
content: { type: "web_search_tool_result_error", error_code: "max_uses_exceeded" },
|
||||
},
|
||||
},
|
||||
{ type: "content_block_stop", index: 1 },
|
||||
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } },
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "web_search", description: "Web search", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
const toolResult = response.events.find((event) => event.type === "tool-result")
|
||||
expect(toolResult).toMatchObject({
|
||||
type: "tool-result",
|
||||
id: "srvtoolu_x",
|
||||
name: "web_search",
|
||||
result: { type: "error" },
|
||||
providerExecuted: true,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("round-trips provider-executed assistant content into server tool blocks", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_round_trip",
|
||||
model,
|
||||
messages: [
|
||||
LLM.user("Search for something."),
|
||||
LLM.assistant([
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "srvtoolu_abc",
|
||||
name: "web_search",
|
||||
input: { query: "effect 4" },
|
||||
providerExecuted: true,
|
||||
},
|
||||
{
|
||||
type: "tool-result",
|
||||
id: "srvtoolu_abc",
|
||||
name: "web_search",
|
||||
result: { type: "json", value: [{ url: "https://example.com" }] },
|
||||
providerExecuted: true,
|
||||
},
|
||||
{ type: "text", text: "Found it." },
|
||||
]),
|
||||
LLM.user("Thanks."),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
messages: [
|
||||
{ role: "user", content: [{ type: "text", text: "Search for something." }] },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "server_tool_use", id: "srvtoolu_abc", name: "web_search", input: { query: "effect 4" } },
|
||||
{
|
||||
type: "web_search_tool_result",
|
||||
tool_use_id: "srvtoolu_abc",
|
||||
content: [{ url: "https://example.com" }],
|
||||
},
|
||||
{ type: "text", text: "Found it." },
|
||||
],
|
||||
},
|
||||
{ role: "user", content: [{ type: "text", text: "Thanks." }] },
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects round-trip for unknown server tool names", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_unknown_server_tool",
|
||||
model,
|
||||
messages: [
|
||||
LLM.assistant([
|
||||
{
|
||||
type: "tool-result",
|
||||
id: "srvtoolu_abc",
|
||||
name: "future_server_tool",
|
||||
result: { type: "json", value: {} },
|
||||
providerExecuted: true,
|
||||
},
|
||||
]),
|
||||
],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("future_server_tool")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported user media content", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_media",
|
||||
model,
|
||||
messages: [LLM.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("Anthropic Messages user messages only support text content for now")
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,533 @@
|
||||
import { EventStreamCodec } from "@smithy/eventstream-codec"
|
||||
import { fromUtf8, toUtf8 } from "@smithy/util-utf8"
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { CacheHint, LLM } from "../../src"
|
||||
import { LLMClient } from "../../src/route"
|
||||
import * as BedrockConverse from "../../src/protocols/bedrock-converse"
|
||||
import { it } from "../lib/effect"
|
||||
import { fixedResponse } from "../lib/http"
|
||||
import {
|
||||
eventSummary,
|
||||
expectWeatherToolLoop,
|
||||
runWeatherToolLoop,
|
||||
weatherTool,
|
||||
weatherToolLoopRequest,
|
||||
weatherToolName,
|
||||
} from "../recorded-scenarios"
|
||||
import { recordedTests } from "../recorded-test"
|
||||
|
||||
const codec = new EventStreamCodec(toUtf8, fromUtf8)
|
||||
const utf8Encoder = new TextEncoder()
|
||||
|
||||
// Build a single AWS event-stream frame for a Converse stream event. Each
|
||||
// frame carries `:message-type=event` + `:event-type=<name>` headers and a
|
||||
// JSON payload body.
|
||||
const eventFrame = (type: string, payload: object) =>
|
||||
codec.encode({
|
||||
headers: {
|
||||
":message-type": { type: "string", value: "event" },
|
||||
":event-type": { type: "string", value: type },
|
||||
":content-type": { type: "string", value: "application/json" },
|
||||
},
|
||||
body: utf8Encoder.encode(JSON.stringify(payload)),
|
||||
})
|
||||
|
||||
const concat = (frames: ReadonlyArray<Uint8Array>) => {
|
||||
const total = frames.reduce((sum, frame) => sum + frame.length, 0)
|
||||
const out = new Uint8Array(total)
|
||||
let offset = 0
|
||||
for (const frame of frames) {
|
||||
out.set(frame, offset)
|
||||
offset += frame.length
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const eventStreamBody = (...payloads: ReadonlyArray<readonly [string, object]>) =>
|
||||
concat(payloads.map(([type, payload]) => eventFrame(type, payload)))
|
||||
|
||||
// Override the default SSE content-type with the binary event-stream type so
|
||||
// the cassette layer treats the body as bytes when recording.
|
||||
const fixedBytes = (bytes: Uint8Array) =>
|
||||
fixedResponse(bytes.slice().buffer, { headers: { "content-type": "application/vnd.amazon.eventstream" } })
|
||||
|
||||
const model = BedrockConverse.model({
|
||||
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
baseURL: "https://bedrock-runtime.test",
|
||||
apiKey: "test-bearer",
|
||||
})
|
||||
|
||||
const baseRequest = LLM.request({
|
||||
id: "req_1",
|
||||
model,
|
||||
system: "You are concise.",
|
||||
prompt: "Say hello.",
|
||||
generation: { maxTokens: 64, temperature: 0 },
|
||||
})
|
||||
|
||||
describe("Bedrock Converse route", () => {
|
||||
it.effect("prepares Converse target with system, inference config, and messages", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(baseRequest)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
modelId: "anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
system: [{ text: "You are concise." }],
|
||||
messages: [{ role: "user", content: [{ text: "Say hello." }] }],
|
||||
inferenceConfig: { maxTokens: 64, temperature: 0 },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("prepares tool config with toolSpec and toolChoice", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.updateRequest(baseRequest, {
|
||||
tools: [
|
||||
{
|
||||
name: "lookup",
|
||||
description: "Lookup data",
|
||||
inputSchema: { type: "object", properties: { query: { type: "string" } }, required: ["query"] },
|
||||
},
|
||||
],
|
||||
toolChoice: LLM.toolChoice({ type: "required" }),
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
toolConfig: {
|
||||
tools: [
|
||||
{
|
||||
toolSpec: {
|
||||
name: "lookup",
|
||||
description: "Lookup data",
|
||||
inputSchema: {
|
||||
json: { type: "object", properties: { query: { type: "string" } }, required: ["query"] },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
toolChoice: { any: {} },
|
||||
},
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers assistant tool-call + tool-result message history", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_history",
|
||||
model,
|
||||
messages: [
|
||||
LLM.user("What is the weather?"),
|
||||
LLM.assistant([LLM.toolCall({ id: "tool_1", name: "lookup", input: { query: "weather" } })]),
|
||||
LLM.toolMessage({ id: "tool_1", name: "lookup", result: { forecast: "sunny" } }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
messages: [
|
||||
{ role: "user", content: [{ text: "What is the weather?" }] },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ toolUse: { toolUseId: "tool_1", name: "lookup", input: { query: "weather" } } }],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
toolResult: {
|
||||
toolUseId: "tool_1",
|
||||
content: [{ json: { forecast: "sunny" } }],
|
||||
status: "success",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes text-delta + messageStop + metadata usage from binary event stream", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = eventStreamBody(
|
||||
["messageStart", { role: "assistant" }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { text: "Hello" } }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { text: "!" } }],
|
||||
["contentBlockStop", { contentBlockIndex: 0 }],
|
||||
["messageStop", { stopReason: "end_turn" }],
|
||||
["metadata", { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } }],
|
||||
)
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
|
||||
|
||||
expect(response.text).toBe("Hello!")
|
||||
const finishes = response.events.filter((event) => event.type === "request-finish")
|
||||
// Bedrock splits the finish across `messageStop` (carries reason) and
|
||||
// `metadata` (carries usage). We consolidate them into a single
|
||||
// terminal `request-finish` event with both.
|
||||
expect(finishes).toHaveLength(1)
|
||||
expect(finishes[0]).toMatchObject({ type: "request-finish", reason: "stop" })
|
||||
expect(response.usage).toMatchObject({
|
||||
inputTokens: 5,
|
||||
outputTokens: 2,
|
||||
totalTokens: 7,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("assembles streamed tool call input", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = eventStreamBody(
|
||||
["messageStart", { role: "assistant" }],
|
||||
[
|
||||
"contentBlockStart",
|
||||
{
|
||||
contentBlockIndex: 0,
|
||||
start: { toolUse: { toolUseId: "tool_1", name: "lookup" } },
|
||||
},
|
||||
],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { toolUse: { input: '{"query"' } } }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { toolUse: { input: ':"weather"}' } } }],
|
||||
["contentBlockStop", { contentBlockIndex: 0 }],
|
||||
["messageStop", { stopReason: "tool_use" }],
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(baseRequest, {
|
||||
tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedBytes(body)))
|
||||
|
||||
expect(response.toolCalls).toEqual([
|
||||
{ type: "tool-call", id: "tool_1", name: "lookup", input: { query: "weather" } },
|
||||
])
|
||||
const events = response.events.filter((event) => event.type === "tool-input-delta")
|
||||
expect(events).toEqual([
|
||||
{ type: "tool-input-delta", id: "tool_1", name: "lookup", text: '{"query"' },
|
||||
{ type: "tool-input-delta", id: "tool_1", name: "lookup", text: ':"weather"}' },
|
||||
])
|
||||
expect(response.events.at(-1)).toMatchObject({ type: "request-finish", reason: "tool-calls" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes reasoning deltas", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = eventStreamBody(
|
||||
["messageStart", { role: "assistant" }],
|
||||
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { text: "Let me think." } } }],
|
||||
["contentBlockStop", { contentBlockIndex: 0 }],
|
||||
["messageStop", { stopReason: "end_turn" }],
|
||||
)
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
|
||||
|
||||
expect(response.reasoning).toBe("Let me think.")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("emits provider-error for throttlingException", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = eventStreamBody(
|
||||
["messageStart", { role: "assistant" }],
|
||||
["throttlingException", { message: "Slow down" }],
|
||||
)
|
||||
const response = yield* LLMClient.generate(baseRequest).pipe(Effect.provide(fixedBytes(body)))
|
||||
|
||||
expect(response.events.find((event) => event.type === "provider-error")).toEqual({
|
||||
type: "provider-error",
|
||||
message: "Slow down",
|
||||
retryable: true,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects requests with no auth path", () =>
|
||||
Effect.gen(function* () {
|
||||
const unsignedModel = BedrockConverse.model({
|
||||
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
baseURL: "https://bedrock-runtime.test",
|
||||
})
|
||||
const error = yield* LLMClient.generate(LLM.updateRequest(baseRequest, { model: unsignedModel })).pipe(
|
||||
Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: "end_turn" }]))),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error.message).toContain("Bedrock Converse requires either model.apiKey")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("signs requests with SigV4 when AWS credentials are provided (deterministic plumbing check)", () =>
|
||||
Effect.gen(function* () {
|
||||
const signed = BedrockConverse.model({
|
||||
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
baseURL: "https://bedrock-runtime.test",
|
||||
credentials: {
|
||||
region: "us-east-1",
|
||||
accessKeyId: "AKIAIOSFODNN7EXAMPLE",
|
||||
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
},
|
||||
})
|
||||
const prepared = yield* LLMClient.prepare(LLM.updateRequest(baseRequest, { model: signed }))
|
||||
|
||||
expect(prepared.route).toBe("bedrock-converse")
|
||||
// The prepare phase doesn't sign — toHttp does. We assert the credential
|
||||
// is plumbed onto the model native field for the signer to find.
|
||||
expect(prepared.model.native).toMatchObject({
|
||||
aws_credentials: { region: "us-east-1", accessKeyId: "AKIAIOSFODNN7EXAMPLE" },
|
||||
aws_region: "us-east-1",
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("emits cachePoint markers after system, user-text, and assistant-text with cache hints", () =>
|
||||
Effect.gen(function* () {
|
||||
const cache = new CacheHint({ type: "ephemeral" })
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_cache",
|
||||
model,
|
||||
system: [{ type: "text", text: "System prefix.", cache }],
|
||||
messages: [
|
||||
LLM.user([{ type: "text", text: "User prefix.", cache }]),
|
||||
LLM.assistant([{ type: "text", text: "Assistant prefix.", cache }]),
|
||||
],
|
||||
generation: { maxTokens: 16, temperature: 0 },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
// System: text block followed by cachePoint marker.
|
||||
system: [{ text: "System prefix." }, { cachePoint: { type: "default" } }],
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ text: "User prefix." }, { cachePoint: { type: "default" } }],
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ text: "Assistant prefix." }, { cachePoint: { type: "default" } }],
|
||||
},
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("does not emit cachePoint when no cache hint is set", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(baseRequest)
|
||||
expect(prepared.body).toMatchObject({
|
||||
system: [{ text: "You are concise." }],
|
||||
messages: [{ role: "user", content: [{ text: "Say hello." }] }],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers image media into Bedrock image blocks", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_image",
|
||||
model,
|
||||
messages: [
|
||||
LLM.user([
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "media", mediaType: "image/png", data: "AAAA" },
|
||||
{ type: "media", mediaType: "image/jpeg", data: "BBBB" },
|
||||
{ type: "media", mediaType: "image/jpg", data: "CCCC" },
|
||||
{ type: "media", mediaType: "image/webp", data: "DDDD" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ text: "What is in this image?" },
|
||||
{ image: { format: "png", source: { bytes: "AAAA" } } },
|
||||
{ image: { format: "jpeg", source: { bytes: "BBBB" } } },
|
||||
// image/jpg is a non-standard alias; we map it to jpeg.
|
||||
{ image: { format: "jpeg", source: { bytes: "CCCC" } } },
|
||||
{ image: { format: "webp", source: { bytes: "DDDD" } } },
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("base64-encodes Uint8Array image bytes", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_image_bytes",
|
||||
model,
|
||||
messages: [LLM.user([{ type: "media", mediaType: "image/png", data: new Uint8Array([1, 2, 3, 4, 5]) }])],
|
||||
}),
|
||||
)
|
||||
|
||||
// Buffer.from([1,2,3,4,5]).toString("base64") === "AQIDBAU="
|
||||
expect(prepared.body).toMatchObject({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ image: { format: "png", source: { bytes: "AQIDBAU=" } } }],
|
||||
},
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers document media into Bedrock document blocks with format and name", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_doc",
|
||||
model,
|
||||
messages: [
|
||||
LLM.user([
|
||||
{ type: "media", mediaType: "application/pdf", data: "PDFDATA", filename: "report.pdf" },
|
||||
{ type: "media", mediaType: "text/csv", data: "CSVDATA" },
|
||||
]),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
// Filename round-trips when supplied.
|
||||
{ document: { format: "pdf", name: "report.pdf", source: { bytes: "PDFDATA" } } },
|
||||
// Falls back to a stable placeholder when filename is missing.
|
||||
{ document: { format: "csv", name: "document.csv", source: { bytes: "CSVDATA" } } },
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported image media types", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_bad_image",
|
||||
model,
|
||||
messages: [LLM.user([{ type: "media", mediaType: "image/svg+xml", data: "x" }])],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("Bedrock Converse does not support image media type image/svg+xml")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported document media types", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_bad_doc",
|
||||
model,
|
||||
messages: [LLM.user([{ type: "media", mediaType: "application/x-tar", data: "x", filename: "a.tar" }])],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("Bedrock Converse does not support media type application/x-tar")
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
// Live recorded integration tests. Run with `RECORD=true AWS_ACCESS_KEY_ID=...
|
||||
// AWS_SECRET_ACCESS_KEY=... [AWS_SESSION_TOKEN=...] bun run test ...` to refresh
|
||||
// cassettes; replay is the default and works without credentials.
|
||||
//
|
||||
// Region is pinned to us-east-1 in tests so the request URL is stable across
|
||||
// machines on replay. If you need to record from a different region (e.g. your
|
||||
// account has access elsewhere), pass `BEDROCK_RECORDING_REGION=eu-west-1` —
|
||||
// but then commit the resulting cassette and others should record from the
|
||||
// same region too.
|
||||
const RECORDING_REGION = process.env.BEDROCK_RECORDING_REGION ?? "us-east-1"
|
||||
|
||||
const recordedModel = () =>
|
||||
BedrockConverse.model({
|
||||
// Most newer Anthropic models on Bedrock require a cross-region inference
|
||||
// profile (`us.` prefix). Nova does not require an Anthropic use-case form
|
||||
// and is on-demand-throughput accessible by default for most accounts.
|
||||
id: process.env.BEDROCK_MODEL_ID ?? "us.amazon.nova-micro-v1:0",
|
||||
credentials: {
|
||||
region: RECORDING_REGION,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID ?? "fixture",
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? "fixture",
|
||||
sessionToken: process.env.AWS_SESSION_TOKEN,
|
||||
},
|
||||
})
|
||||
|
||||
const recorded = recordedTests({
|
||||
prefix: "bedrock-converse",
|
||||
provider: "amazon-bedrock",
|
||||
protocol: "bedrock-converse",
|
||||
requires: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"],
|
||||
})
|
||||
|
||||
describe("Bedrock Converse recorded", () => {
|
||||
recorded.effect("streams text", () =>
|
||||
Effect.gen(function* () {
|
||||
const llm = yield* LLMClient.Service
|
||||
const response = yield* llm.generate(
|
||||
LLM.request({
|
||||
id: "recorded_bedrock_text",
|
||||
model: recordedModel(),
|
||||
system: "Reply with the single word 'Hello'.",
|
||||
prompt: "Say hello.",
|
||||
generation: { maxTokens: 16, temperature: 0 },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(eventSummary(response.events)).toEqual([
|
||||
{ type: "text", value: "Hello" },
|
||||
{ type: "finish", reason: "stop", usage: { inputTokens: 12, outputTokens: 2, totalTokens: 14 } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
recorded.effect.with("streams a tool call", { tags: ["tool"] }, () =>
|
||||
Effect.gen(function* () {
|
||||
const llm = yield* LLMClient.Service
|
||||
const response = yield* llm.generate(
|
||||
LLM.request({
|
||||
id: "recorded_bedrock_tool_call",
|
||||
model: recordedModel(),
|
||||
system: "Call tools exactly as requested.",
|
||||
prompt: "Call get_weather with city exactly Paris.",
|
||||
tools: [weatherTool],
|
||||
toolChoice: LLM.toolChoice(weatherTool),
|
||||
generation: { maxTokens: 80, temperature: 0 },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(eventSummary(response.events)).toEqual([
|
||||
{ type: "tool-call", name: weatherToolName, input: { city: "Paris" } },
|
||||
{ type: "finish", reason: "tool-calls", usage: { inputTokens: 419, outputTokens: 16, totalTokens: 435 } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
recorded.effect.with("drives a tool loop", { tags: ["tool", "tool-loop", "golden"] }, () =>
|
||||
Effect.gen(function* () {
|
||||
const llm = yield* LLMClient.Service
|
||||
expectWeatherToolLoop(
|
||||
yield* runWeatherToolLoop(
|
||||
weatherToolLoopRequest({
|
||||
id: "recorded_bedrock_tool_loop",
|
||||
model: recordedModel(),
|
||||
}),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,232 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { ConfigProvider, Effect, Schema } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM } from "../../src"
|
||||
import * as Cloudflare from "../../src/providers/cloudflare"
|
||||
import { LLMClient } from "../../src/route"
|
||||
import { it } from "../lib/effect"
|
||||
import { dynamicResponse } from "../lib/http"
|
||||
import { sseEvents } from "../lib/sse"
|
||||
|
||||
const Json = Schema.fromJsonString(Schema.Unknown)
|
||||
const decodeJson = Schema.decodeUnknownSync(Json)
|
||||
const withEnv = (env: Record<string, string>) => Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env })))
|
||||
|
||||
const deltaChunk = (delta: object, finishReason: string | null = null) => ({
|
||||
id: "chatcmpl_fixture",
|
||||
choices: [{ delta, finish_reason: finishReason }],
|
||||
usage: null,
|
||||
})
|
||||
|
||||
describe("Cloudflare", () => {
|
||||
it.effect("prepares AI Gateway models through the OpenAI-compatible Chat protocol", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = Cloudflare.aiGateway("workers-ai/@cf/meta/llama-3.3-70b-instruct", {
|
||||
accountId: "test-account",
|
||||
gatewayId: "test-gateway",
|
||||
apiKey: "test-token",
|
||||
})
|
||||
|
||||
expect(model).toMatchObject({
|
||||
id: "workers-ai/@cf/meta/llama-3.3-70b-instruct",
|
||||
provider: "cloudflare-ai-gateway",
|
||||
route: "cloudflare-ai-gateway",
|
||||
baseURL: "https://gateway.ai.cloudflare.com/v1/test-account/test-gateway/compat",
|
||||
})
|
||||
|
||||
const prepared = yield* LLMClient.prepare(LLM.request({ model, prompt: "Say hello." }))
|
||||
|
||||
expect(prepared.route).toBe("cloudflare-ai-gateway")
|
||||
expect(prepared.body).toMatchObject({
|
||||
model: "workers-ai/@cf/meta/llama-3.3-70b-instruct",
|
||||
messages: [{ role: "user", content: "Say hello." }],
|
||||
stream: true,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("posts to the derived gateway endpoint with bearer auth", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLM.generate(
|
||||
LLM.request({
|
||||
model: Cloudflare.aiGateway("openai/gpt-4o-mini", {
|
||||
accountId: "test-account",
|
||||
gatewayId: "test-gateway",
|
||||
apiKey: "test-token",
|
||||
}),
|
||||
prompt: "Say hello.",
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.url).toBe(
|
||||
"https://gateway.ai.cloudflare.com/v1/test-account/test-gateway/compat/chat/completions",
|
||||
)
|
||||
expect(web.headers.get("authorization")).toBe("Bearer test-token")
|
||||
expect(decodeJson(input.text)).toMatchObject({
|
||||
model: "openai/gpt-4o-mini",
|
||||
stream: true,
|
||||
messages: [{ role: "user", content: "Say hello." }],
|
||||
})
|
||||
return input.respond(
|
||||
sseEvents(deltaChunk({ role: "assistant", content: "Hello" }), deltaChunk({}, "stop")),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.text).toBe("Hello")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("defaults AI Gateway id to default when omitted or blank", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(
|
||||
Cloudflare.aiGateway("workers-ai/@cf/meta/llama-3.3-70b-instruct", {
|
||||
accountId: "test-account",
|
||||
gatewayId: "",
|
||||
gatewayApiKey: "test-token",
|
||||
}).baseURL,
|
||||
).toBe("https://gateway.ai.cloudflare.com/v1/test-account/default/compat")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("supports authenticated AI Gateway plus upstream provider auth", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* LLM.generate(
|
||||
LLM.request({
|
||||
model: Cloudflare.aiGateway("openai/gpt-4o-mini", {
|
||||
accountId: "test-account",
|
||||
gatewayApiKey: "gateway-token",
|
||||
apiKey: "provider-token",
|
||||
}),
|
||||
prompt: "Say hello.",
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.url).toBe("https://gateway.ai.cloudflare.com/v1/test-account/default/compat/chat/completions")
|
||||
expect(web.headers.get("cf-aig-authorization")).toBe("Bearer gateway-token")
|
||||
expect(web.headers.get("authorization")).toBe("Bearer provider-token")
|
||||
return input.respond(
|
||||
sseEvents(deltaChunk({ role: "assistant", content: "Hello" }), deltaChunk({}, "stop")),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("allows a fully configured baseURL override", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
model: Cloudflare.aiGateway("openai/gpt-4o-mini", {
|
||||
baseURL: "https://gateway.proxy.test/v1/custom/compat",
|
||||
apiKey: "test-token",
|
||||
}),
|
||||
prompt: "Say hello.",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.model.baseURL).toBe("https://gateway.proxy.test/v1/custom/compat")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("prepares direct Workers AI models through the OpenAI-compatible Chat protocol", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
|
||||
accountId: "test-account",
|
||||
apiKey: "test-token",
|
||||
})
|
||||
|
||||
expect(model).toMatchObject({
|
||||
id: "@cf/meta/llama-3.1-8b-instruct",
|
||||
provider: "cloudflare-workers-ai",
|
||||
route: "cloudflare-workers-ai",
|
||||
baseURL: "https://api.cloudflare.com/client/v4/accounts/test-account/ai/v1",
|
||||
})
|
||||
|
||||
const prepared = yield* LLMClient.prepare(LLM.request({ model, prompt: "Say hello." }))
|
||||
|
||||
expect(prepared.route).toBe("cloudflare-workers-ai")
|
||||
expect(prepared.body).toMatchObject({
|
||||
model: "@cf/meta/llama-3.1-8b-instruct",
|
||||
messages: [{ role: "user", content: "Say hello." }],
|
||||
stream: true,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("posts direct Workers AI requests to the account endpoint with bearer auth", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLM.generate(
|
||||
LLM.request({
|
||||
model: Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
|
||||
accountId: "test-account",
|
||||
apiKey: "test-token",
|
||||
}),
|
||||
prompt: "Say hello.",
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.url).toBe(
|
||||
"https://api.cloudflare.com/client/v4/accounts/test-account/ai/v1/chat/completions",
|
||||
)
|
||||
expect(web.headers.get("authorization")).toBe("Bearer test-token")
|
||||
expect(decodeJson(input.text)).toMatchObject({
|
||||
model: "@cf/meta/llama-3.1-8b-instruct",
|
||||
stream: true,
|
||||
messages: [{ role: "user", content: "Say hello." }],
|
||||
})
|
||||
return input.respond(
|
||||
sseEvents(deltaChunk({ role: "assistant", content: "Hello" }), deltaChunk({}, "stop")),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.text).toBe("Hello")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("supports direct Workers AI token aliases through auth config", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* LLM.generate(
|
||||
LLM.request({
|
||||
model: Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
|
||||
accountId: "test-account",
|
||||
}),
|
||||
prompt: "Say hello.",
|
||||
}),
|
||||
).pipe(
|
||||
withEnv({ CLOUDFLARE_WORKERS_AI_TOKEN: "test-token" }),
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.headers.get("authorization")).toBe("Bearer test-token")
|
||||
return input.respond(
|
||||
sseEvents(deltaChunk({ role: "assistant", content: "Hello" }), deltaChunk({}, "stop")),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,360 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMError } from "../../src"
|
||||
import { LLMClient } from "../../src/route"
|
||||
import * as Gemini from "../../src/protocols/gemini"
|
||||
import { it } from "../lib/effect"
|
||||
import { fixedResponse } from "../lib/http"
|
||||
import { sseEvents, sseRaw } from "../lib/sse"
|
||||
|
||||
const model = Gemini.model({
|
||||
id: "gemini-2.5-flash",
|
||||
baseURL: "https://generativelanguage.test/v1beta/",
|
||||
headers: { "x-goog-api-key": "test" },
|
||||
})
|
||||
|
||||
const request = LLM.request({
|
||||
id: "req_1",
|
||||
model,
|
||||
system: "You are concise.",
|
||||
prompt: "Say hello.",
|
||||
generation: { maxTokens: 20, temperature: 0 },
|
||||
})
|
||||
|
||||
describe("Gemini route", () => {
|
||||
it.effect("prepares Gemini target", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(request)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
contents: [{ role: "user", parts: [{ text: "Say hello." }] }],
|
||||
systemInstruction: { parts: [{ text: "You are concise." }] },
|
||||
generationConfig: { maxOutputTokens: 20, temperature: 0 },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("prepares multimodal user input and tool history", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_tool_result",
|
||||
model,
|
||||
tools: [
|
||||
{
|
||||
name: "lookup",
|
||||
description: "Lookup data",
|
||||
inputSchema: { type: "object", properties: { query: { type: "string" } } },
|
||||
},
|
||||
],
|
||||
toolChoice: { type: "tool", name: "lookup" },
|
||||
messages: [
|
||||
LLM.user([
|
||||
{ type: "text", text: "What is in this image?" },
|
||||
{ type: "media", mediaType: "image/png", data: "AAECAw==" },
|
||||
]),
|
||||
LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
|
||||
LLM.toolMessage({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [{ text: "What is in this image?" }, { inlineData: { mimeType: "image/png", data: "AAECAw==" } }],
|
||||
},
|
||||
{
|
||||
role: "model",
|
||||
parts: [{ functionCall: { name: "lookup", args: { query: "weather" } } }],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{ functionResponse: { name: "lookup", response: { name: "lookup", content: '{"forecast":"sunny"}' } } },
|
||||
],
|
||||
},
|
||||
],
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: [
|
||||
{
|
||||
name: "lookup",
|
||||
description: "Lookup data",
|
||||
parameters: { type: "object", properties: { query: { type: "string" } } },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
toolConfig: { functionCallingConfig: { mode: "ANY", allowedFunctionNames: ["lookup"] } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("omits tools when tool choice is none", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_no_tools",
|
||||
model,
|
||||
prompt: "Say hello.",
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
toolChoice: { type: "none" },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
contents: [{ role: "user", parts: [{ text: "Say hello." }] }],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("sanitizes integer enums, dangling required, untyped arrays, and scalar object keys", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_schema_patch",
|
||||
model,
|
||||
prompt: "Use the tool.",
|
||||
tools: [
|
||||
{
|
||||
name: "lookup",
|
||||
description: "Lookup data",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
required: ["status", "missing"],
|
||||
properties: {
|
||||
status: { type: "integer", enum: [1, 2] },
|
||||
tags: { type: "array" },
|
||||
name: { type: "string", properties: { ignored: { type: "string" } }, required: ["ignored"] },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: [
|
||||
{
|
||||
parameters: {
|
||||
type: "object",
|
||||
required: ["status"],
|
||||
properties: {
|
||||
status: { type: "string", enum: ["1", "2"] },
|
||||
tags: { type: "array", items: { type: "string" } },
|
||||
name: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses text, reasoning, and usage stream fixtures", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: { role: "model", parts: [{ text: "thinking", thought: true }] },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: { role: "model", parts: [{ text: "Hello" }] },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
candidates: [
|
||||
{
|
||||
content: { role: "model", parts: [{ text: "!" }] },
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
usageMetadata: {
|
||||
promptTokenCount: 5,
|
||||
candidatesTokenCount: 2,
|
||||
totalTokenCount: 7,
|
||||
thoughtsTokenCount: 1,
|
||||
cachedContentTokenCount: 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.text).toBe("Hello!")
|
||||
expect(response.reasoning).toBe("thinking")
|
||||
expect(response.usage).toMatchObject({
|
||||
inputTokens: 5,
|
||||
outputTokens: 2,
|
||||
reasoningTokens: 1,
|
||||
cacheReadInputTokens: 1,
|
||||
totalTokens: 7,
|
||||
})
|
||||
expect(response.events).toEqual([
|
||||
{ type: "reasoning-delta", text: "thinking" },
|
||||
{ type: "text-delta", text: "Hello" },
|
||||
{ type: "text-delta", text: "!" },
|
||||
{
|
||||
type: "request-finish",
|
||||
reason: "stop",
|
||||
usage: {
|
||||
inputTokens: 5,
|
||||
outputTokens: 2,
|
||||
reasoningTokens: 1,
|
||||
cacheReadInputTokens: 1,
|
||||
totalTokens: 7,
|
||||
native: {
|
||||
promptTokenCount: 5,
|
||||
candidatesTokenCount: 2,
|
||||
totalTokenCount: 7,
|
||||
thoughtsTokenCount: 1,
|
||||
cachedContentTokenCount: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("emits streamed tool calls and maps finish reason", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: "model",
|
||||
parts: [{ functionCall: { name: "lookup", args: { query: "weather" } } }],
|
||||
},
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 },
|
||||
})
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.toolCalls).toEqual([
|
||||
{ type: "tool-call", id: "tool_0", name: "lookup", input: { query: "weather" } },
|
||||
])
|
||||
expect(response.events).toEqual([
|
||||
{ type: "tool-call", id: "tool_0", name: "lookup", input: { query: "weather" } },
|
||||
{
|
||||
type: "request-finish",
|
||||
reason: "tool-calls",
|
||||
usage: {
|
||||
inputTokens: 5,
|
||||
outputTokens: 1,
|
||||
totalTokens: 6,
|
||||
native: { promptTokenCount: 5, candidatesTokenCount: 1 },
|
||||
},
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("assigns unique ids to multiple streamed tool calls", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents({
|
||||
candidates: [
|
||||
{
|
||||
content: {
|
||||
role: "model",
|
||||
parts: [
|
||||
{ functionCall: { name: "lookup", args: { query: "weather" } } },
|
||||
{ functionCall: { name: "lookup", args: { query: "news" } } },
|
||||
],
|
||||
},
|
||||
finishReason: "STOP",
|
||||
},
|
||||
],
|
||||
})
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.toolCalls).toEqual([
|
||||
{ type: "tool-call", id: "tool_0", name: "lookup", input: { query: "weather" } },
|
||||
{ type: "tool-call", id: "tool_1", name: "lookup", input: { query: "news" } },
|
||||
])
|
||||
expect(response.events.at(-1)).toMatchObject({ type: "request-finish", reason: "tool-calls" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("maps length and content-filter finish reasons", () =>
|
||||
Effect.gen(function* () {
|
||||
const length = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents({ candidates: [{ content: { role: "model", parts: [] }, finishReason: "MAX_TOKENS" }] }),
|
||||
),
|
||||
),
|
||||
)
|
||||
const filtered = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(sseEvents({ candidates: [{ content: { role: "model", parts: [] }, finishReason: "SAFETY" }] })),
|
||||
),
|
||||
)
|
||||
|
||||
expect(length.events).toEqual([{ type: "request-finish", reason: "length" }])
|
||||
expect(filtered.events).toEqual([{ type: "request-finish", reason: "content-filter" }])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("leaves total usage undefined when component counts are missing", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(fixedResponse(sseEvents({ usageMetadata: { thoughtsTokenCount: 1 } }))),
|
||||
)
|
||||
|
||||
expect(response.usage).toMatchObject({ reasoningTokens: 1 })
|
||||
expect(response.usage?.totalTokens).toBeUndefined()
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails invalid stream events", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(fixedResponse(sseRaw("data: {not json}"))),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error).toBeInstanceOf(LLMError)
|
||||
expect(error.reason).toMatchObject({ _tag: "InvalidProviderOutput" })
|
||||
expect(error.message).toContain("Invalid google/gemini stream event")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported assistant media content", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_media",
|
||||
model,
|
||||
messages: [LLM.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain(
|
||||
"Gemini assistant messages only support text, reasoning, and tool-call content for now",
|
||||
)
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,215 @@
|
||||
import * as AnthropicMessages from "../../src/protocols/anthropic-messages"
|
||||
import * as Gemini from "../../src/protocols/gemini"
|
||||
import * as OpenAIChat from "../../src/protocols/openai-chat"
|
||||
import * as OpenAIResponses from "../../src/protocols/openai-responses"
|
||||
import * as Cloudflare from "../../src/providers/cloudflare"
|
||||
import * as OpenAI from "../../src/providers/openai"
|
||||
import * as OpenAICompatible from "../../src/providers/openai-compatible"
|
||||
import * as OpenRouter from "../../src/providers/openrouter"
|
||||
import * as XAI from "../../src/providers/xai"
|
||||
import { describeRecordedGoldenScenarios } from "../recorded-golden"
|
||||
|
||||
const openAIChat = OpenAIChat.model({ id: "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY ?? "fixture" })
|
||||
const openAIResponses = OpenAIResponses.model({ id: "gpt-5.5", apiKey: process.env.OPENAI_API_KEY ?? "fixture" })
|
||||
const openAIResponsesWebSocket = OpenAI.responsesWebSocket("gpt-4.1-mini", {
|
||||
apiKey: process.env.OPENAI_API_KEY ?? "fixture",
|
||||
})
|
||||
const anthropicHaiku = AnthropicMessages.model({
|
||||
id: "claude-haiku-4-5-20251001",
|
||||
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
|
||||
})
|
||||
const anthropicOpus = AnthropicMessages.model({
|
||||
id: "claude-opus-4-7",
|
||||
apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture",
|
||||
})
|
||||
const gemini = Gemini.model({ id: "gemini-2.5-flash", apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture" })
|
||||
const xaiBasic = XAI.model("grok-3-mini", { apiKey: process.env.XAI_API_KEY ?? "fixture" })
|
||||
const xaiFlagship = XAI.model("grok-4.3", { apiKey: process.env.XAI_API_KEY ?? "fixture" })
|
||||
const cloudflareAIGatewayWorkers = Cloudflare.aiGateway("workers-ai/@cf/meta/llama-3.1-8b-instruct", {
|
||||
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
|
||||
gatewayId:
|
||||
process.env.CLOUDFLARE_GATEWAY_ID && process.env.CLOUDFLARE_GATEWAY_ID !== process.env.CLOUDFLARE_ACCOUNT_ID
|
||||
? process.env.CLOUDFLARE_GATEWAY_ID
|
||||
: undefined,
|
||||
gatewayApiKey: process.env.CLOUDFLARE_API_TOKEN ?? "fixture",
|
||||
})
|
||||
const cloudflareAIGatewayWorkersTools = Cloudflare.aiGateway("workers-ai/@cf/openai/gpt-oss-20b", {
|
||||
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
|
||||
gatewayId:
|
||||
process.env.CLOUDFLARE_GATEWAY_ID && process.env.CLOUDFLARE_GATEWAY_ID !== process.env.CLOUDFLARE_ACCOUNT_ID
|
||||
? process.env.CLOUDFLARE_GATEWAY_ID
|
||||
: undefined,
|
||||
gatewayApiKey: process.env.CLOUDFLARE_API_TOKEN ?? "fixture",
|
||||
})
|
||||
const cloudflareWorkersAI = Cloudflare.workersAI("@cf/meta/llama-3.1-8b-instruct", {
|
||||
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
|
||||
apiKey: process.env.CLOUDFLARE_API_KEY ?? "fixture",
|
||||
})
|
||||
const cloudflareWorkersAITools = Cloudflare.workersAI("@cf/openai/gpt-oss-20b", {
|
||||
accountId: process.env.CLOUDFLARE_ACCOUNT_ID ?? "fixture-account",
|
||||
apiKey: process.env.CLOUDFLARE_API_KEY ?? "fixture",
|
||||
})
|
||||
const deepseek = OpenAICompatible.deepseek.model("deepseek-chat", { apiKey: process.env.DEEPSEEK_API_KEY ?? "fixture" })
|
||||
const together = OpenAICompatible.togetherai.model("meta-llama/Llama-3.3-70B-Instruct-Turbo", {
|
||||
apiKey: process.env.TOGETHER_AI_API_KEY ?? "fixture",
|
||||
})
|
||||
const groq = OpenAICompatible.groq.model("llama-3.3-70b-versatile", { apiKey: process.env.GROQ_API_KEY ?? "fixture" })
|
||||
const openrouter = OpenRouter.model("openai/gpt-4o-mini", { apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
|
||||
const openrouterGpt55 = OpenRouter.model("openai/gpt-5.5", { apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
|
||||
const openrouterOpus = OpenRouter.model("anthropic/claude-opus-4.7", {
|
||||
apiKey: process.env.OPENROUTER_API_KEY ?? "fixture",
|
||||
})
|
||||
|
||||
const redactCloudflareURL = (url: string) =>
|
||||
url
|
||||
.replace(/\/client\/v4\/accounts\/[^/]+\/ai\/v1\//, "/client/v4/accounts/{account}/ai/v1/")
|
||||
.replace(/\/v1\/[^/]+\/[^/]+\/compat\//, "/v1/{account}/{gateway}/compat/")
|
||||
|
||||
const cloudflareOptions = {
|
||||
redact: { url: redactCloudflareURL },
|
||||
}
|
||||
|
||||
describeRecordedGoldenScenarios([
|
||||
{
|
||||
name: "OpenAI Chat gpt-4o-mini",
|
||||
prefix: "openai-chat",
|
||||
model: openAIChat,
|
||||
requires: ["OPENAI_API_KEY"],
|
||||
scenarios: ["text", "tool-call", "tool-loop"],
|
||||
},
|
||||
{
|
||||
name: "OpenAI Responses gpt-5.5",
|
||||
prefix: "openai-responses",
|
||||
model: openAIResponses,
|
||||
requires: ["OPENAI_API_KEY"],
|
||||
tags: ["flagship"],
|
||||
scenarios: [
|
||||
{ id: "text", temperature: false },
|
||||
{ id: "tool-call", temperature: false },
|
||||
{ id: "tool-loop", temperature: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "OpenAI Responses WebSocket gpt-4.1-mini",
|
||||
prefix: "openai-responses-websocket",
|
||||
model: openAIResponsesWebSocket,
|
||||
transport: "websocket",
|
||||
requires: ["OPENAI_API_KEY"],
|
||||
scenarios: ["tool-loop"],
|
||||
},
|
||||
{
|
||||
name: "Anthropic Haiku 4.5",
|
||||
prefix: "anthropic-messages",
|
||||
model: anthropicHaiku,
|
||||
requires: ["ANTHROPIC_API_KEY"],
|
||||
options: { requestHeaders: ["content-type", "anthropic-version"] },
|
||||
scenarios: ["text", "tool-call"],
|
||||
},
|
||||
{
|
||||
name: "Anthropic Opus 4.7",
|
||||
prefix: "anthropic-messages",
|
||||
model: anthropicOpus,
|
||||
requires: ["ANTHROPIC_API_KEY"],
|
||||
tags: ["flagship"],
|
||||
options: { requestHeaders: ["content-type", "anthropic-version"] },
|
||||
scenarios: [{ id: "tool-loop", temperature: false }],
|
||||
},
|
||||
{
|
||||
name: "Gemini 2.5 Flash",
|
||||
prefix: "gemini",
|
||||
model: gemini,
|
||||
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
|
||||
scenarios: [{ id: "text", maxTokens: 80 }, "tool-call"],
|
||||
},
|
||||
{
|
||||
name: "xAI Grok 3 Mini",
|
||||
prefix: "xai",
|
||||
model: xaiBasic,
|
||||
requires: ["XAI_API_KEY"],
|
||||
scenarios: ["text", "tool-call"],
|
||||
},
|
||||
{
|
||||
name: "xAI Grok 4.3",
|
||||
prefix: "xai",
|
||||
model: xaiFlagship,
|
||||
requires: ["XAI_API_KEY"],
|
||||
tags: ["flagship"],
|
||||
scenarios: [{ id: "tool-loop", timeout: 30_000 }],
|
||||
},
|
||||
{
|
||||
name: "Cloudflare AI Gateway Workers AI Llama 3.1 8B",
|
||||
prefix: "cloudflare-ai-gateway",
|
||||
model: cloudflareAIGatewayWorkers,
|
||||
requires: ["CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_TOKEN"],
|
||||
options: cloudflareOptions,
|
||||
scenarios: ["text"],
|
||||
},
|
||||
{
|
||||
name: "Cloudflare AI Gateway Workers AI GPT OSS 20B Tools",
|
||||
prefix: "cloudflare-ai-gateway",
|
||||
model: cloudflareAIGatewayWorkersTools,
|
||||
requires: ["CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_TOKEN"],
|
||||
options: cloudflareOptions,
|
||||
scenarios: [{ id: "tool-call", maxTokens: 120 }],
|
||||
},
|
||||
{
|
||||
name: "Cloudflare Workers AI Llama 3.1 8B",
|
||||
prefix: "cloudflare-workers-ai",
|
||||
model: cloudflareWorkersAI,
|
||||
requires: ["CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_KEY"],
|
||||
options: cloudflareOptions,
|
||||
scenarios: ["text"],
|
||||
},
|
||||
{
|
||||
name: "Cloudflare Workers AI GPT OSS 20B Tools",
|
||||
prefix: "cloudflare-workers-ai",
|
||||
model: cloudflareWorkersAITools,
|
||||
requires: ["CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_KEY"],
|
||||
options: cloudflareOptions,
|
||||
scenarios: [{ id: "tool-call", maxTokens: 120 }],
|
||||
},
|
||||
{
|
||||
name: "DeepSeek Chat",
|
||||
prefix: "openai-compatible-chat",
|
||||
model: deepseek,
|
||||
requires: ["DEEPSEEK_API_KEY"],
|
||||
scenarios: ["text"],
|
||||
},
|
||||
{
|
||||
name: "TogetherAI Llama 3.3 70B",
|
||||
prefix: "openai-compatible-chat",
|
||||
model: together,
|
||||
requires: ["TOGETHER_AI_API_KEY"],
|
||||
scenarios: ["text", "tool-call"],
|
||||
},
|
||||
{
|
||||
name: "Groq Llama 3.3 70B",
|
||||
prefix: "openai-compatible-chat",
|
||||
model: groq,
|
||||
requires: ["GROQ_API_KEY"],
|
||||
scenarios: ["text", "tool-call", { id: "tool-loop", timeout: 30_000 }],
|
||||
},
|
||||
{
|
||||
name: "OpenRouter gpt-4o-mini",
|
||||
prefix: "openai-compatible-chat",
|
||||
model: openrouter,
|
||||
requires: ["OPENROUTER_API_KEY"],
|
||||
scenarios: ["text", "tool-call", "tool-loop"],
|
||||
},
|
||||
{
|
||||
name: "OpenRouter gpt-5.5",
|
||||
prefix: "openai-compatible-chat",
|
||||
model: openrouterGpt55,
|
||||
requires: ["OPENROUTER_API_KEY"],
|
||||
tags: ["flagship"],
|
||||
scenarios: ["tool-loop"],
|
||||
},
|
||||
{
|
||||
name: "OpenRouter Claude Opus 4.7",
|
||||
prefix: "openai-compatible-chat",
|
||||
model: openrouterOpus,
|
||||
requires: ["OPENROUTER_API_KEY"],
|
||||
tags: ["flagship"],
|
||||
scenarios: ["tool-loop"],
|
||||
},
|
||||
])
|
||||
@@ -0,0 +1,355 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM, LLMError } from "../../src"
|
||||
import * as Azure from "../../src/providers/azure"
|
||||
import * as OpenAI from "../../src/providers/openai"
|
||||
import * as OpenAIChat from "../../src/protocols/openai-chat"
|
||||
import { LLMClient } from "../../src/route"
|
||||
import { it } from "../lib/effect"
|
||||
import { dynamicResponse, fixedResponse, truncatedStream } from "../lib/http"
|
||||
import { deltaChunk, usageChunk } from "../lib/openai-chunks"
|
||||
import { sseEvents } from "../lib/sse"
|
||||
|
||||
const TargetJson = Schema.fromJsonString(Schema.Unknown)
|
||||
const encodeJson = Schema.encodeSync(TargetJson)
|
||||
const decodeJson = Schema.decodeUnknownSync(TargetJson)
|
||||
|
||||
const model = OpenAIChat.model({
|
||||
id: "gpt-4o-mini",
|
||||
baseURL: "https://api.openai.test/v1/",
|
||||
headers: { authorization: "Bearer test" },
|
||||
})
|
||||
|
||||
const request = LLM.request({
|
||||
id: "req_1",
|
||||
model,
|
||||
system: "You are concise.",
|
||||
prompt: "Say hello.",
|
||||
generation: { maxTokens: 20, temperature: 0 },
|
||||
})
|
||||
|
||||
describe("OpenAI Chat route", () => {
|
||||
it.effect("prepares OpenAI Chat payload", () =>
|
||||
Effect.gen(function* () {
|
||||
// Pass the OpenAIChat payload type so `prepared.body` is statically
|
||||
// typed to the route's native shape — the assertions below read field
|
||||
// names without `unknown` casts.
|
||||
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(request)
|
||||
const _typed: { readonly model: string; readonly stream: true } = prepared.body
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "gpt-4o-mini",
|
||||
messages: [
|
||||
{ role: "system", content: "You are concise." },
|
||||
{ role: "user", content: "Say hello." },
|
||||
],
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
max_tokens: 20,
|
||||
temperature: 0,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("maps OpenAI provider options to Chat options", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIChat.OpenAIChatBody>(
|
||||
LLM.request({
|
||||
model: OpenAI.chat("gpt-4o-mini", { baseURL: "https://api.openai.test/v1/" }),
|
||||
prompt: "think",
|
||||
providerOptions: { openai: { reasoningEffort: "low" } },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.store).toBe(false)
|
||||
expect(prepared.body.reasoning_effort).toBe("low")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("adds native query params to the Chat Completions URL", () =>
|
||||
LLMClient.generate(
|
||||
LLM.updateRequest(request, { model: OpenAIChat.model({ ...model, queryParams: { "api-version": "v1" } }) }),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.url).toBe("https://api.openai.test/v1/chat/completions?api-version=v1")
|
||||
return input.respond(sseEvents(deltaChunk({}, "stop")), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("uses Azure api-key header for static OpenAI Chat keys", () =>
|
||||
LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: Azure.chat("gpt-4o-mini", {
|
||||
baseURL: "https://opencode-test.openai.azure.com/openai/v1/",
|
||||
apiKey: "azure-key",
|
||||
headers: { authorization: "Bearer stale" },
|
||||
}),
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.headers.get("api-key")).toBe("azure-key")
|
||||
expect(web.headers.get("authorization")).toBeNull()
|
||||
return input.respond(sseEvents(deltaChunk({}, "stop")), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("applies serializable HTTP overlays after payload lowering", () =>
|
||||
LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: OpenAIChat.model({ ...model, apiKey: "fresh-key", headers: { authorization: "Bearer stale" } }),
|
||||
http: {
|
||||
body: { metadata: { source: "test" } },
|
||||
headers: { authorization: "Bearer request", "x-custom": "yes" },
|
||||
query: { debug: "1" },
|
||||
},
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.url).toBe("https://api.openai.test/v1/chat/completions?debug=1")
|
||||
expect(web.headers.get("authorization")).toBe("Bearer fresh-key")
|
||||
expect(web.headers.get("x-custom")).toBe("yes")
|
||||
expect(decodeJson(input.text)).toMatchObject({
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
metadata: { source: "test" },
|
||||
})
|
||||
return input.respond(sseEvents(deltaChunk({}, "stop")), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("prepares assistant tool-call and tool-result messages", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_tool_result",
|
||||
model,
|
||||
messages: [
|
||||
LLM.user("What is the weather?"),
|
||||
LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
|
||||
LLM.toolMessage({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "gpt-4o-mini",
|
||||
messages: [
|
||||
{ role: "user", content: "What is the weather?" },
|
||||
{
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: "call_1",
|
||||
type: "function",
|
||||
function: { name: "lookup", arguments: encodeJson({ query: "weather" }) },
|
||||
},
|
||||
],
|
||||
},
|
||||
{ role: "tool", tool_call_id: "call_1", content: encodeJson({ forecast: "sunny" }) },
|
||||
],
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported user media content", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_media",
|
||||
model,
|
||||
messages: [LLM.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("OpenAI Chat user messages only support text content for now")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported assistant reasoning content", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_reasoning",
|
||||
model,
|
||||
messages: [LLM.assistant({ type: "reasoning", text: "hidden" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("OpenAI Chat assistant messages only support text and tool-call content for now")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses text and usage stream fixtures", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
deltaChunk({ role: "assistant", content: "Hello" }),
|
||||
deltaChunk({ content: "!" }),
|
||||
deltaChunk({}, "stop"),
|
||||
usageChunk({
|
||||
prompt_tokens: 5,
|
||||
completion_tokens: 2,
|
||||
total_tokens: 7,
|
||||
prompt_tokens_details: { cached_tokens: 1 },
|
||||
completion_tokens_details: { reasoning_tokens: 0 },
|
||||
}),
|
||||
)
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.text).toBe("Hello!")
|
||||
expect(response.events).toEqual([
|
||||
{ type: "text-delta", text: "Hello" },
|
||||
{ type: "text-delta", text: "!" },
|
||||
{
|
||||
type: "request-finish",
|
||||
reason: "stop",
|
||||
usage: {
|
||||
inputTokens: 5,
|
||||
outputTokens: 2,
|
||||
reasoningTokens: 0,
|
||||
cacheReadInputTokens: 1,
|
||||
totalTokens: 7,
|
||||
native: {
|
||||
prompt_tokens: 5,
|
||||
completion_tokens: 2,
|
||||
total_tokens: 7,
|
||||
prompt_tokens_details: { cached_tokens: 1 },
|
||||
completion_tokens_details: { reasoning_tokens: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("assembles streamed tool call input", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
deltaChunk({
|
||||
role: "assistant",
|
||||
tool_calls: [{ index: 0, id: "call_1", function: { name: "lookup", arguments: '{"query"' } }],
|
||||
}),
|
||||
deltaChunk({ tool_calls: [{ index: 0, function: { arguments: ':"weather"}' } }] }),
|
||||
deltaChunk({}, "tool_calls"),
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.events).toEqual([
|
||||
{ type: "tool-input-delta", id: "call_1", name: "lookup", text: '{"query"' },
|
||||
{ type: "tool-input-delta", id: "call_1", name: "lookup", text: ':"weather"}' },
|
||||
{ type: "tool-call", id: "call_1", name: "lookup", input: { query: "weather" } },
|
||||
{ type: "request-finish", reason: "tool-calls", usage: undefined },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("does not finalize streamed tool calls without a finish reason", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
deltaChunk({
|
||||
role: "assistant",
|
||||
tool_calls: [{ index: 0, id: "call_1", function: { name: "lookup", arguments: '{"query"' } }],
|
||||
}),
|
||||
deltaChunk({ tool_calls: [{ index: 0, function: { arguments: ':"weather"}' } }] }),
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.events).toEqual([
|
||||
{ type: "tool-input-delta", id: "call_1", name: "lookup", text: '{"query"' },
|
||||
{ type: "tool-input-delta", id: "call_1", name: "lookup", text: ':"weather"}' },
|
||||
])
|
||||
expect(response.toolCalls).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails on malformed stream events", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(deltaChunk({ content: 123 }))
|
||||
const error = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)), Effect.flip)
|
||||
|
||||
expect(error.message).toContain("Invalid openai/openai-chat stream event")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("surfaces transport errors that occur mid-stream", () =>
|
||||
Effect.gen(function* () {
|
||||
const layer = truncatedStream([
|
||||
`data: ${JSON.stringify(deltaChunk({ role: "assistant", content: "Hello" }))}\n\n`,
|
||||
])
|
||||
const error = yield* LLMClient.generate(request).pipe(Effect.provide(layer), Effect.flip)
|
||||
|
||||
expect(error.message).toContain("Failed to read openai/openai-chat stream")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails HTTP provider errors before stream parsing", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse('{"error":{"message":"Bad request","type":"invalid_request_error"}}', {
|
||||
status: 400,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error).toBeInstanceOf(LLMError)
|
||||
expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
|
||||
expect(error.message).toContain("HTTP 400")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("short-circuits the upstream stream when the consumer takes a prefix", () =>
|
||||
Effect.gen(function* () {
|
||||
// The body has more chunks than we'll consume. If `Stream.take(1)` did
|
||||
// not interrupt the upstream HTTP body the test would hang waiting for
|
||||
// the rest of the stream to drain.
|
||||
const body = sseEvents(
|
||||
deltaChunk({ role: "assistant", content: "Hello" }),
|
||||
deltaChunk({ content: " world" }),
|
||||
deltaChunk({}, "stop"),
|
||||
)
|
||||
|
||||
const events = Array.from(
|
||||
yield* LLMClient.stream(request).pipe(Stream.take(1), Stream.runCollect, Effect.provide(fixedResponse(body))),
|
||||
)
|
||||
expect(events.map((event) => event.type)).toEqual(["text-delta"])
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,237 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM } from "../../src"
|
||||
import { LLMClient } from "../../src/route"
|
||||
import * as OpenAICompatible from "../../src/providers/openai-compatible"
|
||||
import * as OpenAICompatibleChat from "../../src/protocols/openai-compatible-chat"
|
||||
import { it } from "../lib/effect"
|
||||
import { dynamicResponse } from "../lib/http"
|
||||
import { sseEvents } from "../lib/sse"
|
||||
|
||||
const Json = Schema.fromJsonString(Schema.Unknown)
|
||||
const decodeJson = Schema.decodeUnknownSync(Json)
|
||||
|
||||
const model = OpenAICompatibleChat.model({
|
||||
id: "deepseek-chat",
|
||||
provider: "deepseek",
|
||||
baseURL: "https://api.deepseek.test/v1/",
|
||||
apiKey: "test-key",
|
||||
queryParams: { "api-version": "2026-01-01" },
|
||||
})
|
||||
|
||||
const request = LLM.request({
|
||||
id: "req_1",
|
||||
model,
|
||||
system: "You are concise.",
|
||||
prompt: "Say hello.",
|
||||
generation: { maxTokens: 20, temperature: 0 },
|
||||
})
|
||||
|
||||
const deltaChunk = (delta: object, finishReason: string | null = null) => ({
|
||||
id: "chatcmpl_fixture",
|
||||
choices: [{ delta, finish_reason: finishReason }],
|
||||
usage: null,
|
||||
})
|
||||
|
||||
const usageChunk = (usage: object) => ({
|
||||
id: "chatcmpl_fixture",
|
||||
choices: [],
|
||||
usage,
|
||||
})
|
||||
|
||||
const providerFamilies = [
|
||||
["baseten", OpenAICompatible.baseten, "https://inference.baseten.co/v1"],
|
||||
["cerebras", OpenAICompatible.cerebras, "https://api.cerebras.ai/v1"],
|
||||
["deepinfra", OpenAICompatible.deepinfra, "https://api.deepinfra.com/v1/openai"],
|
||||
["deepseek", OpenAICompatible.deepseek, "https://api.deepseek.com/v1"],
|
||||
["fireworks", OpenAICompatible.fireworks, "https://api.fireworks.ai/inference/v1"],
|
||||
["togetherai", OpenAICompatible.togetherai, "https://api.together.xyz/v1"],
|
||||
] as const
|
||||
|
||||
describe("OpenAI-compatible Chat route", () => {
|
||||
it.effect("prepares generic Chat target", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
toolChoice: { type: "required" },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.route).toBe("openai-compatible-chat")
|
||||
expect(prepared.model).toMatchObject({
|
||||
id: "deepseek-chat",
|
||||
provider: "deepseek",
|
||||
route: "openai-compatible-chat",
|
||||
baseURL: "https://api.deepseek.test/v1/",
|
||||
apiKey: "test-key",
|
||||
queryParams: { "api-version": "2026-01-01" },
|
||||
})
|
||||
expect(prepared.body).toEqual({
|
||||
model: "deepseek-chat",
|
||||
messages: [
|
||||
{ role: "system", content: "You are concise." },
|
||||
{ role: "user", content: "Say hello." },
|
||||
],
|
||||
tools: [
|
||||
{
|
||||
type: "function",
|
||||
function: { name: "lookup", description: "Lookup data", parameters: { type: "object" } },
|
||||
},
|
||||
],
|
||||
tool_choice: "required",
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
max_tokens: 20,
|
||||
temperature: 0,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("provides model helpers for compatible provider families", () =>
|
||||
Effect.gen(function* () {
|
||||
expect(
|
||||
providerFamilies.map(([provider, family]) => {
|
||||
const model = family.model(`${provider}-model`, { apiKey: "test-key" })
|
||||
return {
|
||||
id: String(model.id),
|
||||
provider: String(model.provider),
|
||||
route: model.route,
|
||||
baseURL: model.baseURL,
|
||||
apiKey: model.apiKey,
|
||||
}
|
||||
}),
|
||||
).toEqual(
|
||||
providerFamilies.map(([provider, _, baseURL]) => ({
|
||||
id: `${provider}-model`,
|
||||
provider,
|
||||
route: "openai-compatible-chat",
|
||||
baseURL,
|
||||
apiKey: "test-key",
|
||||
})),
|
||||
)
|
||||
|
||||
const custom = OpenAICompatible.deepseek.model("deepseek-chat", {
|
||||
apiKey: "test-key",
|
||||
baseURL: "https://custom.deepseek.test/v1",
|
||||
})
|
||||
expect(custom).toMatchObject({
|
||||
provider: "deepseek",
|
||||
route: "openai-compatible-chat",
|
||||
baseURL: "https://custom.deepseek.test/v1",
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("matches AI SDK compatible basic request body fixture", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(request)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "deepseek-chat",
|
||||
messages: [
|
||||
{ role: "system", content: "You are concise." },
|
||||
{ role: "user", content: "Say hello." },
|
||||
],
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
max_tokens: 20,
|
||||
temperature: 0,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("matches AI SDK compatible tool request body fixture", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_tool_parity",
|
||||
model,
|
||||
tools: [
|
||||
{
|
||||
name: "lookup",
|
||||
description: "Lookup data",
|
||||
inputSchema: { type: "object", properties: { query: { type: "string" } }, required: ["query"] },
|
||||
},
|
||||
],
|
||||
toolChoice: "lookup",
|
||||
messages: [
|
||||
LLM.user("What is the weather?"),
|
||||
LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
|
||||
LLM.toolMessage({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "deepseek-chat",
|
||||
messages: [
|
||||
{ role: "user", content: "What is the weather?" },
|
||||
{
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: "call_1",
|
||||
type: "function",
|
||||
function: { name: "lookup", arguments: '{"query":"weather"}' },
|
||||
},
|
||||
],
|
||||
},
|
||||
{ role: "tool", tool_call_id: "call_1", content: '{"forecast":"sunny"}' },
|
||||
],
|
||||
tools: [
|
||||
{
|
||||
type: "function",
|
||||
function: {
|
||||
name: "lookup",
|
||||
description: "Lookup data",
|
||||
parameters: { type: "object", properties: { query: { type: "string" } }, required: ["query"] },
|
||||
},
|
||||
},
|
||||
],
|
||||
tool_choice: { type: "function", function: { name: "lookup" } },
|
||||
stream: true,
|
||||
stream_options: { include_usage: true },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("posts to the configured compatible endpoint and parses text usage", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.url).toBe("https://api.deepseek.test/v1/chat/completions?api-version=2026-01-01")
|
||||
expect(web.headers.get("authorization")).toBe("Bearer test-key")
|
||||
expect(decodeJson(input.text)).toMatchObject({
|
||||
model: "deepseek-chat",
|
||||
stream: true,
|
||||
messages: [
|
||||
{ role: "system", content: "You are concise." },
|
||||
{ role: "user", content: "Say hello." },
|
||||
],
|
||||
})
|
||||
return input.respond(
|
||||
sseEvents(
|
||||
deltaChunk({ role: "assistant", content: "Hello" }),
|
||||
deltaChunk({ content: "!" }),
|
||||
deltaChunk({}, "stop"),
|
||||
usageChunk({ prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }),
|
||||
),
|
||||
{ headers: { "content-type": "text/event-stream" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.text).toBe("Hello!")
|
||||
expect(response.usage).toMatchObject({ inputTokens: 5, outputTokens: 2, totalTokens: 7 })
|
||||
expect(response.events.at(-1)).toMatchObject({ type: "request-finish", reason: "stop" })
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,549 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { ConfigProvider, Effect, Layer, Stream } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { LLM, LLMError } from "../../src"
|
||||
import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route"
|
||||
import * as Azure from "../../src/providers/azure"
|
||||
import * as OpenAI from "../../src/providers/openai"
|
||||
import * as OpenAIResponses from "../../src/protocols/openai-responses"
|
||||
import * as ProviderShared from "../../src/protocols/shared"
|
||||
import { it } from "../lib/effect"
|
||||
import { dynamicResponse, fixedResponse } from "../lib/http"
|
||||
import { sseEvents } from "../lib/sse"
|
||||
|
||||
const model = OpenAIResponses.model({
|
||||
id: "gpt-4.1-mini",
|
||||
baseURL: "https://api.openai.test/v1/",
|
||||
headers: { authorization: "Bearer test" },
|
||||
})
|
||||
|
||||
const request = LLM.request({
|
||||
id: "req_1",
|
||||
model,
|
||||
system: "You are concise.",
|
||||
prompt: "Say hello.",
|
||||
generation: { maxTokens: 20, temperature: 0 },
|
||||
})
|
||||
|
||||
const configEnv = (env: Record<string, string>) => Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env })))
|
||||
|
||||
describe("OpenAI Responses route", () => {
|
||||
it.effect("prepares OpenAI Responses target", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(request)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "gpt-4.1-mini",
|
||||
input: [
|
||||
{ role: "system", content: "You are concise." },
|
||||
{ role: "user", content: [{ type: "input_text", text: "Say hello." }] },
|
||||
],
|
||||
stream: true,
|
||||
max_output_tokens: 20,
|
||||
temperature: 0,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("prepares OpenAI Responses WebSocket target", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.updateRequest(request, {
|
||||
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.route).toBe("openai-responses-websocket")
|
||||
expect(prepared.protocol).toBe("openai-responses")
|
||||
expect(prepared.metadata).toEqual({ transport: "websocket-json" })
|
||||
expect(prepared.body).toMatchObject({ model: "gpt-4.1-mini", stream: true })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("streams OpenAI Responses over WebSocket", () =>
|
||||
Effect.gen(function* () {
|
||||
const sent: string[] = []
|
||||
const opened: Array<{ readonly url: string; readonly authorization: string | undefined }> = []
|
||||
let closed = false
|
||||
const deps = Layer.mergeAll(
|
||||
Layer.succeed(
|
||||
RequestExecutor.Service,
|
||||
RequestExecutor.Service.of({
|
||||
execute: () => Effect.die("unexpected HTTP request"),
|
||||
}),
|
||||
),
|
||||
Layer.succeed(
|
||||
WebSocketExecutor.Service,
|
||||
WebSocketExecutor.Service.of({
|
||||
open: (input) =>
|
||||
Effect.succeed({
|
||||
sendText: (message) =>
|
||||
Effect.sync(() => {
|
||||
opened.push({ url: input.url, authorization: input.headers.authorization })
|
||||
sent.push(message)
|
||||
}),
|
||||
messages: Stream.fromArray([
|
||||
ProviderShared.encodeJson({ type: "response.output_text.delta", item_id: "msg_1", delta: "Hi" }),
|
||||
ProviderShared.encodeJson({ type: "response.completed", response: { id: "resp_ws" } }),
|
||||
]),
|
||||
close: Effect.sync(() => {
|
||||
closed = true
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
|
||||
prompt: "Say hello.",
|
||||
}),
|
||||
).pipe(Effect.provide(LLMClient.layerWithWebSocket.pipe(Layer.provide(deps))))
|
||||
|
||||
expect(response.text).toBe("Hi")
|
||||
expect(opened).toEqual([{ url: "wss://api.openai.test/v1/responses", authorization: "Bearer test" }])
|
||||
expect(closed).toBe(true)
|
||||
expect(sent).toHaveLength(1)
|
||||
expect(JSON.parse(sent[0])).toEqual({
|
||||
type: "response.create",
|
||||
model: "gpt-4.1-mini",
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: "Say hello." }] }],
|
||||
store: false,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("requires WebSocket runtime for OpenAI Responses WebSocket", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(
|
||||
LLM.request({
|
||||
model: OpenAI.responsesWebSocket("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", apiKey: "test" }),
|
||||
prompt: "Say hello.",
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
LLMClient.layer.pipe(
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
RequestExecutor.Service,
|
||||
RequestExecutor.Service.of({
|
||||
execute: () => Effect.die("unexpected HTTP request"),
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error.message).toContain("requires WebSocketExecutor.Service")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails immediately when WebSocket is already closed", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* WebSocketExecutor.fromWebSocket(
|
||||
{ readyState: globalThis.WebSocket.CLOSED } as globalThis.WebSocket,
|
||||
{ url: "wss://api.openai.test/v1/responses", headers: Headers.empty },
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("closed before opening")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("adds native query params to the Responses URL", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: OpenAIResponses.model({ ...model, queryParams: { "api-version": "v1" } }),
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.url).toBe("https://api.openai.test/v1/responses?api-version=v1")
|
||||
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("uses Azure api-key header for static OpenAI Responses keys", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: Azure.responses("gpt-4.1-mini", {
|
||||
baseURL: "https://opencode-test.openai.azure.com/openai/v1/",
|
||||
apiKey: "azure-key",
|
||||
headers: { authorization: "Bearer stale" },
|
||||
}),
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.headers.get("api-key")).toBe("azure-key")
|
||||
expect(web.headers.get("authorization")).toBeNull()
|
||||
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("loads OpenAI default auth from Effect Config", () =>
|
||||
LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: OpenAI.responses("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/" }),
|
||||
}),
|
||||
).pipe(
|
||||
configEnv({ OPENAI_API_KEY: "env-key" }),
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.headers.get("authorization")).toBe("Bearer env-key")
|
||||
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("lets explicit auth override OpenAI default API key auth", () =>
|
||||
LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
model: OpenAI.responses("gpt-4.1-mini", {
|
||||
baseURL: "https://api.openai.test/v1/",
|
||||
auth: Auth.bearer("oauth-token"),
|
||||
}),
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(web.headers.get("authorization")).toBe("Bearer oauth-token")
|
||||
return input.respond(sseEvents({ type: "response.completed", response: {} }), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("prepares function call and function output input items", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_tool_result",
|
||||
model,
|
||||
messages: [
|
||||
LLM.user("What is the weather?"),
|
||||
LLM.assistant([LLM.toolCall({ id: "call_1", name: "lookup", input: { query: "weather" } })]),
|
||||
LLM.toolMessage({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }),
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toEqual({
|
||||
model: "gpt-4.1-mini",
|
||||
input: [
|
||||
{ role: "user", content: [{ type: "input_text", text: "What is the weather?" }] },
|
||||
{ type: "function_call", call_id: "call_1", name: "lookup", arguments: '{"query":"weather"}' },
|
||||
{ type: "function_call_output", call_id: "call_1", output: '{"forecast":"sunny"}' },
|
||||
],
|
||||
stream: true,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("maps OpenAI provider options to Responses options", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
model: OpenAI.model("gpt-5.2", { baseURL: "https://api.openai.test/v1/" }),
|
||||
prompt: "think",
|
||||
providerOptions: {
|
||||
openai: {
|
||||
promptCacheKey: "session_123",
|
||||
reasoningEffort: "high",
|
||||
reasoningSummary: "auto",
|
||||
includeEncryptedReasoning: true,
|
||||
},
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.store).toBe(false)
|
||||
expect(prepared.body.prompt_cache_key).toBe("session_123")
|
||||
expect(prepared.body.include).toEqual(["reasoning.encrypted_content"])
|
||||
expect(prepared.body.reasoning).toEqual({ effort: "high", summary: "auto" })
|
||||
expect(prepared.body.text).toEqual({ verbosity: "low" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("request OpenAI provider options override model defaults", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
model: OpenAI.model("gpt-4.1-mini", {
|
||||
baseURL: "https://api.openai.test/v1/",
|
||||
providerOptions: { openai: { promptCacheKey: "model_cache" } },
|
||||
}),
|
||||
prompt: "no cache",
|
||||
providerOptions: { openai: { promptCacheKey: "request_cache" } },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.prompt_cache_key).toBe("request_cache")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses text and usage stream fixtures", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{ type: "response.output_text.delta", item_id: "msg_1", delta: "Hello" },
|
||||
{ type: "response.output_text.delta", item_id: "msg_1", delta: "!" },
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: "resp_1",
|
||||
service_tier: "default",
|
||||
usage: {
|
||||
input_tokens: 5,
|
||||
output_tokens: 2,
|
||||
total_tokens: 7,
|
||||
input_tokens_details: { cached_tokens: 1 },
|
||||
output_tokens_details: { reasoning_tokens: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.text).toBe("Hello!")
|
||||
expect(response.events).toEqual([
|
||||
{ type: "text-delta", id: "msg_1", text: "Hello", providerMetadata: { openai: { itemId: "msg_1" } } },
|
||||
{ type: "text-delta", id: "msg_1", text: "!", providerMetadata: { openai: { itemId: "msg_1" } } },
|
||||
{
|
||||
type: "request-finish",
|
||||
reason: "stop",
|
||||
providerMetadata: { openai: { responseId: "resp_1", serviceTier: "default" } },
|
||||
usage: {
|
||||
inputTokens: 5,
|
||||
outputTokens: 2,
|
||||
reasoningTokens: 0,
|
||||
cacheReadInputTokens: 1,
|
||||
totalTokens: 7,
|
||||
native: {
|
||||
input_tokens: 5,
|
||||
output_tokens: 2,
|
||||
total_tokens: 7,
|
||||
input_tokens_details: { cached_tokens: 1 },
|
||||
output_tokens_details: { reasoning_tokens: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("assembles streamed function call input", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: { type: "function_call", id: "item_1", call_id: "call_1", name: "lookup", arguments: "" },
|
||||
},
|
||||
{ type: "response.function_call_arguments.delta", item_id: "item_1", delta: '{"query"' },
|
||||
{ type: "response.function_call_arguments.delta", item_id: "item_1", delta: ':"weather"}' },
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
type: "function_call",
|
||||
id: "item_1",
|
||||
call_id: "call_1",
|
||||
name: "lookup",
|
||||
arguments: '{"query":"weather"}',
|
||||
},
|
||||
},
|
||||
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
|
||||
)
|
||||
const response = yield* LLMClient.generate(
|
||||
LLM.updateRequest(request, {
|
||||
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
|
||||
}),
|
||||
).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.events).toEqual([
|
||||
{
|
||||
type: "tool-input-delta",
|
||||
id: "call_1",
|
||||
name: "lookup",
|
||||
text: '{"query"',
|
||||
providerMetadata: { openai: { itemId: "item_1" } },
|
||||
},
|
||||
{
|
||||
type: "tool-input-delta",
|
||||
id: "call_1",
|
||||
name: "lookup",
|
||||
text: ':"weather"}',
|
||||
providerMetadata: { openai: { itemId: "item_1" } },
|
||||
},
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "call_1",
|
||||
name: "lookup",
|
||||
input: { query: "weather" },
|
||||
providerMetadata: { openai: { itemId: "item_1" } },
|
||||
},
|
||||
{
|
||||
type: "request-finish",
|
||||
reason: "tool-calls",
|
||||
usage: { inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { input_tokens: 5, output_tokens: 1 } },
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes web_search_call as provider-executed tool-call + tool-result", () =>
|
||||
Effect.gen(function* () {
|
||||
const item = {
|
||||
type: "web_search_call",
|
||||
id: "ws_1",
|
||||
status: "completed",
|
||||
action: { type: "search", query: "effect 4" },
|
||||
}
|
||||
const body = sseEvents(
|
||||
{ type: "response.output_item.added", item },
|
||||
{ type: "response.output_item.done", item },
|
||||
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
|
||||
)
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
const callsAndResults = response.events.filter(
|
||||
(event) => event.type === "tool-call" || event.type === "tool-result",
|
||||
)
|
||||
expect(callsAndResults).toEqual([
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "ws_1",
|
||||
name: "web_search",
|
||||
input: { type: "search", query: "effect 4" },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ws_1" } },
|
||||
},
|
||||
{
|
||||
type: "tool-result",
|
||||
id: "ws_1",
|
||||
name: "web_search",
|
||||
result: { type: "json", value: item },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ws_1" } },
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes code_interpreter_call as provider-executed events with code input", () =>
|
||||
Effect.gen(function* () {
|
||||
const item = {
|
||||
type: "code_interpreter_call",
|
||||
id: "ci_1",
|
||||
status: "completed",
|
||||
code: "print(1+1)",
|
||||
container_id: "cnt_xyz",
|
||||
outputs: [{ type: "logs", logs: "2\n" }],
|
||||
}
|
||||
const body = sseEvents(
|
||||
{ type: "response.output_item.done", item },
|
||||
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
|
||||
)
|
||||
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
const toolCall = response.events.find((event) => event.type === "tool-call")
|
||||
expect(toolCall).toEqual({
|
||||
type: "tool-call",
|
||||
id: "ci_1",
|
||||
name: "code_interpreter",
|
||||
input: { code: "print(1+1)", container_id: "cnt_xyz" },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ci_1" } },
|
||||
})
|
||||
const toolResult = response.events.find((event) => event.type === "tool-result")
|
||||
expect(toolResult).toEqual({
|
||||
type: "tool-result",
|
||||
id: "ci_1",
|
||||
name: "code_interpreter",
|
||||
result: { type: "json", value: item },
|
||||
providerExecuted: true,
|
||||
providerMetadata: { openai: { itemId: "ci_1" } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects unsupported user media content", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
id: "req_media",
|
||||
model,
|
||||
messages: [LLM.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
|
||||
}),
|
||||
).pipe(Effect.flip)
|
||||
|
||||
expect(error.message).toContain("OpenAI Responses user messages only support text content for now")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("emits provider-error events for mid-stream provider errors", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(fixedResponse(sseEvents({ type: "error", code: "rate_limit_exceeded", message: "Slow down" }))),
|
||||
)
|
||||
|
||||
expect(response.events).toEqual([{ type: "provider-error", message: "Slow down" }])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("falls back to error code when no message is present", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(fixedResponse(sseEvents({ type: "error", code: "internal_error" }))),
|
||||
)
|
||||
|
||||
expect(response.events).toEqual([{ type: "provider-error", message: "internal_error" }])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails HTTP provider errors before stream parsing", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse('{"error":{"type":"invalid_request_error","message":"Bad request"}}', {
|
||||
status: 400,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
|
||||
expect(error).toBeInstanceOf(LLMError)
|
||||
expect(error.reason).toMatchObject({ _tag: "InvalidRequest" })
|
||||
expect(error.message).toContain("HTTP 400")
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM } from "../../src"
|
||||
import { LLMClient } from "../../src/route"
|
||||
import * as OpenRouter from "../../src/providers/openrouter"
|
||||
import { it } from "../lib/effect"
|
||||
|
||||
describe("OpenRouter", () => {
|
||||
it.effect("prepares OpenRouter models through the OpenAI-compatible Chat route", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = OpenRouter.model("openai/gpt-4o-mini", { apiKey: "test-key" })
|
||||
|
||||
expect(model).toMatchObject({
|
||||
id: "openai/gpt-4o-mini",
|
||||
provider: "openrouter",
|
||||
route: "openrouter",
|
||||
baseURL: "https://openrouter.ai/api/v1",
|
||||
apiKey: "test-key",
|
||||
})
|
||||
|
||||
const prepared = yield* LLMClient.prepare(LLM.request({ model, prompt: "Say hello." }))
|
||||
|
||||
expect(prepared.route).toBe("openrouter")
|
||||
expect(prepared.body).toMatchObject({
|
||||
model: "openai/gpt-4o-mini",
|
||||
messages: [{ role: "user", content: "Say hello." }],
|
||||
stream: true,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("applies OpenRouter payload options from the model helper", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
model: OpenRouter.model("anthropic/claude-3.7-sonnet:thinking", {
|
||||
providerOptions: {
|
||||
openrouter: {
|
||||
usage: true,
|
||||
reasoning: { effort: "high" },
|
||||
promptCacheKey: "session_123",
|
||||
},
|
||||
},
|
||||
}),
|
||||
prompt: "Think briefly.",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
usage: { include: true },
|
||||
reasoning: { effort: "high" },
|
||||
prompt_cache_key: "session_123",
|
||||
})
|
||||
}),
|
||||
)
|
||||
})
|
||||
Reference in New Issue
Block a user