Add native LLM core foundation (#24712)
This commit is contained in:
Kit Langton
@@ -0,0 +1,57 @@
|
||||
import type { Config, Redacted } from "effect"
|
||||
import { Auth } from "./auth"
|
||||
|
||||
export type ApiKeyMode = "optional" | "required"
|
||||
|
||||
export type AuthOverride = {
|
||||
readonly auth: Auth
|
||||
readonly apiKey?: never
|
||||
}
|
||||
|
||||
export type OptionalApiKeyAuth = {
|
||||
readonly apiKey?: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>>
|
||||
readonly auth?: never
|
||||
}
|
||||
|
||||
export type RequiredApiKeyAuth = {
|
||||
readonly apiKey: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>>
|
||||
readonly auth?: never
|
||||
}
|
||||
|
||||
export type ProviderAuthOption<Mode extends ApiKeyMode> =
|
||||
| AuthOverride
|
||||
| (Mode extends "optional" ? OptionalApiKeyAuth : RequiredApiKeyAuth)
|
||||
|
||||
export type ModelOptions<Base, Mode extends ApiKeyMode> = Omit<Base, "apiKey" | "auth"> & ProviderAuthOption<Mode>
|
||||
|
||||
export type ModelArgs<Base, Mode extends ApiKeyMode> = Mode extends "optional"
|
||||
? readonly [options?: ModelOptions<Base, Mode>]
|
||||
: readonly [options: ModelOptions<Base, Mode>]
|
||||
|
||||
export type ModelFactory<Base, Mode extends ApiKeyMode, Model> = (id: string, ...args: ModelArgs<Base, Mode>) => Model
|
||||
|
||||
/**
|
||||
* Require at least one of the keys in `T`. Use for option shapes where any
|
||||
* subset of fields is acceptable but at least one must be present (e.g. Azure
|
||||
* accepts `resourceName` or `baseURL`).
|
||||
*/
|
||||
export type AtLeastOne<T> = {
|
||||
[K in keyof T]: Required<Pick<T, K>> & Partial<Omit<T, K>>
|
||||
}[keyof T]
|
||||
|
||||
/**
|
||||
* Standard bearer-auth resolution for providers: honor an explicit `auth`
|
||||
* override, otherwise resolve `apiKey` (option > config var) and apply it as
|
||||
* a bearer token.
|
||||
*/
|
||||
export const bearer = (options: ProviderAuthOption<"optional">, envVar: string | ReadonlyArray<string>): Auth => {
|
||||
if ("auth" in options && options.auth) return options.auth
|
||||
return (Array.isArray(envVar) ? envVar : [envVar])
|
||||
.reduce(
|
||||
(auth, name) => auth.orElse(Auth.config(name)),
|
||||
Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey"),
|
||||
)
|
||||
.bearer()
|
||||
}
|
||||
|
||||
export * as AuthOptions from "./auth-options"
|
||||
@@ -0,0 +1,196 @@
|
||||
import { Config, Effect, Redacted } from "effect"
|
||||
import { Headers } from "effect/unstable/http"
|
||||
import { AuthenticationReason, InvalidRequestReason, LLMError, type LLMRequest } from "../schema"
|
||||
|
||||
export class MissingCredentialError extends Error {
|
||||
readonly _tag = "MissingCredentialError"
|
||||
|
||||
constructor(readonly source: string) {
|
||||
super(`Missing auth credential: ${source}`)
|
||||
}
|
||||
}
|
||||
|
||||
export type CredentialError = MissingCredentialError | Config.ConfigError
|
||||
export type AuthError = CredentialError | LLMError
|
||||
|
||||
export interface AuthInput {
|
||||
readonly request: LLMRequest
|
||||
readonly method: "POST" | "GET"
|
||||
readonly url: string
|
||||
readonly body: string
|
||||
readonly headers: Headers.Headers
|
||||
}
|
||||
|
||||
export interface Credential {
|
||||
readonly load: Effect.Effect<Redacted.Redacted<string>, CredentialError>
|
||||
readonly orElse: (that: Credential) => Credential
|
||||
readonly bearer: () => Auth
|
||||
readonly header: (name: string) => Auth
|
||||
readonly pipe: <A>(f: (self: Credential) => A) => A
|
||||
}
|
||||
|
||||
export interface Auth {
|
||||
readonly apply: (input: AuthInput) => Effect.Effect<Headers.Headers, AuthError>
|
||||
readonly andThen: (that: Auth) => Auth
|
||||
readonly orElse: (that: Auth) => Auth
|
||||
readonly pipe: <A>(f: (self: Auth) => A) => A
|
||||
}
|
||||
|
||||
export const isAuth = (input: unknown): input is Auth =>
|
||||
typeof input === "object" && input !== null && "apply" in input && typeof input.apply === "function"
|
||||
|
||||
const credential = (load: Effect.Effect<Redacted.Redacted<string>, CredentialError>): Credential => {
|
||||
const self: Credential = {
|
||||
load,
|
||||
orElse: (that) => credential(load.pipe(Effect.catch(() => that.load))),
|
||||
bearer: () => fromCredential(self, (secret) => ({ authorization: `Bearer ${secret}` })),
|
||||
header: (name) => fromCredential(self, (secret) => ({ [name]: secret })),
|
||||
pipe: (f) => f(self),
|
||||
}
|
||||
return self
|
||||
}
|
||||
|
||||
const auth = (apply: Auth["apply"]): Auth => {
|
||||
const self: Auth = {
|
||||
apply,
|
||||
andThen: (that) =>
|
||||
auth((input) => apply(input).pipe(Effect.flatMap((headers) => that.apply({ ...input, headers })))),
|
||||
orElse: (that) => auth((input) => apply(input).pipe(Effect.catch(() => that.apply(input)))),
|
||||
pipe: (f) => f(self),
|
||||
}
|
||||
return self
|
||||
}
|
||||
|
||||
const fromCredential = (source: Credential, render: (secret: string) => Headers.Input) =>
|
||||
auth((input) =>
|
||||
source.load.pipe(Effect.map((secret) => Headers.setAll(input.headers, render(Redacted.value(secret))))),
|
||||
)
|
||||
|
||||
const secretEffect = (secret: string | Redacted.Redacted<string>, source: string) => {
|
||||
const redacted = typeof secret === "string" ? Redacted.make(secret) : secret
|
||||
if (Redacted.value(redacted) === "") return Effect.fail(new MissingCredentialError(source))
|
||||
return Effect.succeed(redacted)
|
||||
}
|
||||
|
||||
const credentialFromSecret = (
|
||||
secret: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>>,
|
||||
source: string,
|
||||
) => {
|
||||
if (typeof secret === "string" || Redacted.isRedacted(secret)) return credential(secretEffect(secret, source))
|
||||
return credential(
|
||||
Effect.gen(function* () {
|
||||
return yield* secretEffect(yield* secret, source)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
export const value = (secret: string, source = "value") => credentialFromSecret(secret, source)
|
||||
|
||||
export const optional = (
|
||||
secret: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | undefined,
|
||||
source = "optional value",
|
||||
) =>
|
||||
secret === undefined
|
||||
? credential(Effect.fail(new MissingCredentialError(source)))
|
||||
: credentialFromSecret(secret, source)
|
||||
|
||||
export const config = (name: string) => credentialFromSecret(Config.redacted(name), name)
|
||||
|
||||
export const effect = (load: Effect.Effect<Redacted.Redacted<string>, CredentialError>) => credential(load)
|
||||
|
||||
export const none = auth((input) => Effect.succeed(input.headers))
|
||||
|
||||
export const headers = (input: Headers.Input) =>
|
||||
auth((inputAuth) => Effect.succeed(Headers.setAll(inputAuth.headers, input)))
|
||||
|
||||
export const remove = (name: string) => auth((input) => Effect.succeed(Headers.remove(input.headers, name)))
|
||||
|
||||
export const custom = (apply: (input: AuthInput) => Effect.Effect<Headers.Headers, LLMError>) => auth(apply)
|
||||
|
||||
export const passthrough = none
|
||||
|
||||
const fromModelApiKey = (from: (apiKey: string) => Headers.Input) =>
|
||||
auth(({ request, headers }) => {
|
||||
const key = request.model.apiKey
|
||||
if (!key) return Effect.succeed(headers)
|
||||
return Effect.succeed(Headers.setAll(headers, from(key)))
|
||||
})
|
||||
|
||||
const credentialInput = (
|
||||
source: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential,
|
||||
) =>
|
||||
typeof source === "string" || Redacted.isRedacted(source) || Config.isConfig(source)
|
||||
? credentialFromSecret(source, "value")
|
||||
: source
|
||||
|
||||
export function bearer(): Auth
|
||||
export function bearer(
|
||||
source: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential,
|
||||
): Auth
|
||||
export function bearer(
|
||||
source?: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential,
|
||||
) {
|
||||
if (source === undefined) return fromModelApiKey((key) => ({ authorization: `Bearer ${key}` }))
|
||||
return credentialInput(source).bearer()
|
||||
}
|
||||
|
||||
export const apiKey = bearer
|
||||
|
||||
export const apiKeyHeader = (name: string) => fromModelApiKey((key) => ({ [name]: key }))
|
||||
|
||||
export function header(
|
||||
name: string,
|
||||
): (source: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential) => Auth
|
||||
export function header(
|
||||
name: string,
|
||||
source: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential,
|
||||
): Auth
|
||||
export function header(
|
||||
name: string,
|
||||
source?: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential,
|
||||
) {
|
||||
if (source === undefined) {
|
||||
return (
|
||||
next: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential,
|
||||
) => credentialInput(next).header(name)
|
||||
}
|
||||
return credentialInput(source).header(name)
|
||||
}
|
||||
|
||||
export function bearerHeader(
|
||||
name: string,
|
||||
): (source: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential) => Auth
|
||||
export function bearerHeader(
|
||||
name: string,
|
||||
source: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential,
|
||||
): Auth
|
||||
export function bearerHeader(
|
||||
name: string,
|
||||
source?: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential,
|
||||
) {
|
||||
const render = (input: string | Redacted.Redacted<string> | Config.Config<string | Redacted.Redacted<string>> | Credential) =>
|
||||
fromCredential(credentialInput(input), (secret) => ({ [name]: `Bearer ${secret}` }))
|
||||
if (source === undefined) return render
|
||||
return render(source)
|
||||
}
|
||||
|
||||
const toLLMError = (error: AuthError): LLMError => {
|
||||
if (error instanceof MissingCredentialError || error instanceof Config.ConfigError) {
|
||||
return new LLMError({
|
||||
module: "Auth",
|
||||
method: "apply",
|
||||
reason:
|
||||
error instanceof MissingCredentialError
|
||||
? new AuthenticationReason({ message: error.message, kind: "missing" })
|
||||
: new InvalidRequestReason({ message: `Failed to resolve auth config: ${error.message}` }),
|
||||
})
|
||||
}
|
||||
return error
|
||||
}
|
||||
|
||||
export const toEffect =
|
||||
(input: Auth) =>
|
||||
(authInput: AuthInput): Effect.Effect<Headers.Headers, LLMError> =>
|
||||
input.apply(authInput).pipe(Effect.mapError(toLLMError))
|
||||
|
||||
export * as Auth from "./auth"
|
||||
@@ -0,0 +1,528 @@
|
||||
import { Cause, Context, Effect, Layer, Schema, Stream } from "effect"
|
||||
import type { Auth as AuthDef } from "./auth"
|
||||
import type { Endpoint } from "./endpoint"
|
||||
import { RequestExecutor } from "./executor"
|
||||
import type { Framing } from "./framing"
|
||||
import { HttpTransport } from "./transport"
|
||||
import type { Transport, TransportRuntime } from "./transport"
|
||||
import { WebSocketExecutor } from "./transport"
|
||||
import type { Service as WebSocketExecutorService } from "./transport/websocket"
|
||||
import type { Protocol } from "./protocol"
|
||||
import * as ProviderShared from "../protocols/shared"
|
||||
import * as ToolRuntime from "../tool-runtime"
|
||||
import type { Tools } from "../tool"
|
||||
import type { LLMError, LLMEvent, PreparedRequestOf, ProtocolID } from "../schema"
|
||||
import {
|
||||
GenerationOptions,
|
||||
HttpOptions,
|
||||
LLMRequest,
|
||||
LLMResponse,
|
||||
ModelID,
|
||||
ModelLimits,
|
||||
ModelRef,
|
||||
LLMError as LLMErrorClass,
|
||||
NoRouteReason,
|
||||
PreparedRequest,
|
||||
ProviderID,
|
||||
RouteID,
|
||||
mergeGenerationOptions,
|
||||
mergeHttpOptions,
|
||||
mergeProviderOptions,
|
||||
} from "../schema"
|
||||
|
||||
export interface RouteBody<Body> {
|
||||
/** Schema for the validated provider-native body sent as the JSON request. */
|
||||
readonly schema: Schema.Codec<Body, unknown>
|
||||
/** Build the provider-native body from a common `LLMRequest`. */
|
||||
readonly from: (request: LLMRequest) => Effect.Effect<Body, LLMError>
|
||||
}
|
||||
|
||||
export interface Route<Body, Prepared = unknown> {
|
||||
readonly id: string
|
||||
readonly provider?: ProviderID
|
||||
readonly protocol: ProtocolID
|
||||
readonly transport: Transport<Body, Prepared, unknown>
|
||||
readonly defaults: RouteDefaults
|
||||
readonly body: RouteBody<Body>
|
||||
readonly with: (patch: RoutePatch<Body, Prepared>) => Route<Body, Prepared>
|
||||
readonly model: <Input extends RouteModelInput = RouteModelInput>(input: Input) => ModelRef
|
||||
readonly prepareTransport: (body: Body, request: LLMRequest) => Effect.Effect<Prepared, LLMError>
|
||||
readonly streamPrepared: (
|
||||
prepared: Prepared,
|
||||
request: LLMRequest,
|
||||
runtime: TransportRuntime,
|
||||
) => Stream.Stream<LLMEvent, LLMError>
|
||||
}
|
||||
|
||||
// Route registries intentionally erase body generics after construction.
|
||||
// Normal call sites use `OpenAIChat.route`; callers only need body types
|
||||
// when preparing a request with a protocol-specific type assertion.
|
||||
// oxlint-disable-next-line typescript-eslint/no-explicit-any
|
||||
export type AnyRoute = Route<any, any>
|
||||
|
||||
const routeRegistry = new Map<string, AnyRoute>()
|
||||
|
||||
// Route lookup is intentionally global: model refs name a route id, and
|
||||
// importing the provider/protocol/custom-route module registers the runnable
|
||||
// implementation. Duplicate ids are bugs because model refs cannot disambiguate
|
||||
// them.
|
||||
const register = <R extends AnyRoute>(route: R): R => {
|
||||
const existing = routeRegistry.get(route.id)
|
||||
if (existing && existing !== route) throw new Error(`Duplicate LLM route id "${route.id}"`)
|
||||
routeRegistry.set(route.id, route)
|
||||
return route
|
||||
}
|
||||
|
||||
const registeredRoute = (id: string) => routeRegistry.get(id)
|
||||
|
||||
export type HttpOptionsInput = HttpOptions.Input
|
||||
|
||||
export type ModelRefInput = Omit<
|
||||
ConstructorParameters<typeof ModelRef>[0],
|
||||
"id" | "provider" | "route" | "limits" | "generation" | "http" | "auth"
|
||||
> & {
|
||||
readonly id: string | ModelID
|
||||
readonly provider: string | ProviderID
|
||||
readonly route: string | RouteID
|
||||
readonly auth?: AuthDef
|
||||
readonly limits?: ModelLimits.Input
|
||||
readonly generation?: GenerationOptions.Input
|
||||
readonly http?: HttpOptionsInput
|
||||
}
|
||||
|
||||
// `baseURL` is required on `ModelRefInput` (every materialized `ModelRef` has
|
||||
// a host) but optional at the route-input layers below. The route's `defaults`
|
||||
// can supply a canonical URL (e.g. OpenAI/Anthropic) so the user's input may
|
||||
// omit it. Routes without a canonical URL (OpenAI-compatible, GitHub Copilot)
|
||||
// re-tighten this in their own input type.
|
||||
export type RouteModelInput = Omit<ModelRefInput, "provider" | "route" | "baseURL"> & {
|
||||
readonly baseURL?: string
|
||||
}
|
||||
|
||||
export type RouteModelDefaults = Omit<ModelRefInput, "id" | "route" | "baseURL"> & {
|
||||
readonly baseURL?: string
|
||||
}
|
||||
|
||||
export type RouteRoutedModelInput = Omit<ModelRefInput, "route" | "baseURL"> & {
|
||||
readonly baseURL?: string
|
||||
}
|
||||
|
||||
export type RouteRoutedModelDefaults = Partial<Omit<ModelRefInput, "id" | "provider" | "route">>
|
||||
|
||||
export type RouteDefaults = Partial<Omit<ModelRefInput, "id" | "provider" | "route">>
|
||||
|
||||
export interface RoutePatch<Body, Prepared> extends RouteDefaults {
|
||||
readonly id: string
|
||||
readonly provider?: string | ProviderID
|
||||
readonly transport?: Transport<Body, Prepared, unknown>
|
||||
}
|
||||
|
||||
type RouteMappedModelInput = RouteModelInput | RouteRoutedModelInput
|
||||
|
||||
export interface RouteModelOptions<
|
||||
Input extends RouteMappedModelInput,
|
||||
Output extends RouteMappedModelInput = RouteMappedModelInput,
|
||||
> {
|
||||
readonly mapInput?: (input: Input) => Output
|
||||
}
|
||||
|
||||
export interface RouteMappedModelOptions<Input, Output extends RouteMappedModelInput = RouteMappedModelInput> {
|
||||
readonly mapInput: (input: Input) => Output
|
||||
}
|
||||
|
||||
const modelWithDefaults =
|
||||
<Input>(
|
||||
route: AnyRoute,
|
||||
defaults: Partial<Omit<ModelRefInput, "id" | "route">>,
|
||||
options: { readonly mapInput?: (input: Input) => RouteMappedModelInput },
|
||||
) =>
|
||||
(input: Input) => {
|
||||
const mapped = options.mapInput === undefined ? (input as RouteMappedModelInput) : options.mapInput(input)
|
||||
const provider = defaults.provider ?? route.provider ?? ("provider" in mapped ? mapped.provider : undefined)
|
||||
if (!provider) throw new Error(`Route.model(${route.id}) requires a provider`)
|
||||
const baseURL = mapped.baseURL ?? defaults.baseURL ?? route.defaults.baseURL
|
||||
if (!baseURL)
|
||||
throw new Error(
|
||||
`Route.model(${route.id}) requires a baseURL — supply it via input, defaults, or route defaults`,
|
||||
)
|
||||
const generation = mergeGenerationOptions(route.defaults.generation, defaults.generation)
|
||||
const providerOptions = mergeProviderOptions(route.defaults.providerOptions, defaults.providerOptions)
|
||||
const http = mergeHttpOptions(httpOptions(route.defaults.http), httpOptions(defaults.http))
|
||||
return modelRef({
|
||||
...route.defaults,
|
||||
...defaults,
|
||||
...mapped,
|
||||
baseURL,
|
||||
provider,
|
||||
route: route.id,
|
||||
limits: mapped.limits ?? defaults.limits ?? route.defaults.limits,
|
||||
generation: mergeGenerationOptions(generation, mapped.generation),
|
||||
providerOptions: mergeProviderOptions(providerOptions, mapped.providerOptions),
|
||||
http: mergeHttpOptions(http, httpOptions(mapped.http)),
|
||||
})
|
||||
}
|
||||
|
||||
const mergeRouteDefaults = (base: RouteDefaults | undefined, patch: RouteDefaults): RouteDefaults => ({
|
||||
...base,
|
||||
...patch,
|
||||
limits: patch.limits ?? base?.limits,
|
||||
generation: mergeGenerationOptions(generationOptions(base?.generation), generationOptions(patch.generation)),
|
||||
providerOptions: mergeProviderOptions(base?.providerOptions, patch.providerOptions),
|
||||
http: mergeHttpOptions(httpOptions(base?.http), httpOptions(patch.http)),
|
||||
})
|
||||
|
||||
export const modelLimits = ModelLimits.make
|
||||
|
||||
export const generationOptions = (input: GenerationOptions.Input | undefined) =>
|
||||
input === undefined ? undefined : GenerationOptions.make(input)
|
||||
|
||||
export const httpOptions = (input: HttpOptionsInput | undefined) => {
|
||||
if (input === undefined) return input
|
||||
return HttpOptions.make(input)
|
||||
}
|
||||
|
||||
export const modelRef = (input: ModelRefInput) =>
|
||||
new ModelRef({
|
||||
...input,
|
||||
id: ModelID.make(input.id),
|
||||
provider: ProviderID.make(input.provider),
|
||||
route: RouteID.make(input.route),
|
||||
limits: modelLimits(input.limits),
|
||||
generation: generationOptions(input.generation),
|
||||
http: httpOptions(input.http),
|
||||
})
|
||||
|
||||
function model<Input extends RouteModelInput = RouteModelInput>(
|
||||
route: AnyRoute,
|
||||
defaults: RouteModelDefaults,
|
||||
options?: RouteModelOptions<Input, RouteModelInput>,
|
||||
): (input: Input) => ModelRef
|
||||
function model<Input extends RouteRoutedModelInput = RouteRoutedModelInput>(
|
||||
route: AnyRoute,
|
||||
defaults?: RouteRoutedModelDefaults,
|
||||
options?: RouteModelOptions<Input, RouteRoutedModelInput>,
|
||||
): (input: Input) => ModelRef
|
||||
function model<Input, Output extends RouteMappedModelInput = RouteMappedModelInput>(
|
||||
route: AnyRoute,
|
||||
defaults: Partial<Omit<ModelRefInput, "id" | "route">>,
|
||||
options: RouteMappedModelOptions<Input, Output>,
|
||||
): (input: Input) => ModelRef
|
||||
function model<Input>(
|
||||
route: AnyRoute,
|
||||
defaults: Partial<Omit<ModelRefInput, "id" | "route">> = {},
|
||||
options: { readonly mapInput?: (input: Input) => RouteMappedModelInput } = {},
|
||||
) {
|
||||
return modelWithDefaults(route, defaults, options)
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
/**
|
||||
* Compile a request through protocol body construction, validation, and HTTP
|
||||
* preparation without sending it. Returns the prepared request including the
|
||||
* provider-native body.
|
||||
*
|
||||
* Pass a `Body` type argument to statically expose the route's body
|
||||
* shape (e.g. `prepare<OpenAIChatBody>(...)`) — the runtime body is
|
||||
* identical, so this is a type-level assertion the caller makes about which
|
||||
* route the request will resolve to.
|
||||
*/
|
||||
readonly prepare: <Body = unknown>(request: LLMRequest) => Effect.Effect<PreparedRequestOf<Body>, LLMError>
|
||||
readonly stream: StreamMethod
|
||||
readonly generate: GenerateMethod
|
||||
}
|
||||
|
||||
export interface StreamMethod {
|
||||
(request: LLMRequest): Stream.Stream<LLMEvent, LLMError>
|
||||
<T extends Tools>(options: ToolRuntime.RunOptions<T>): Stream.Stream<LLMEvent, LLMError>
|
||||
}
|
||||
|
||||
export interface GenerateMethod {
|
||||
(request: LLMRequest): Effect.Effect<LLMResponse, LLMError>
|
||||
<T extends Tools>(options: ToolRuntime.RunOptions<T>): Effect.Effect<LLMResponse, LLMError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/LLMClient") {}
|
||||
|
||||
const noRoute = (model: ModelRef) =>
|
||||
new LLMErrorClass({
|
||||
module: "LLMClient",
|
||||
method: "resolveRoute",
|
||||
reason: new NoRouteReason({ route: model.route, provider: model.provider, model: model.id }),
|
||||
})
|
||||
|
||||
const resolveRequestOptions = (request: LLMRequest) =>
|
||||
LLMRequest.update(request, {
|
||||
generation: mergeGenerationOptions(request.model.generation, request.generation) ?? new GenerationOptions({}),
|
||||
providerOptions: mergeProviderOptions(request.model.providerOptions, request.providerOptions),
|
||||
http: mergeHttpOptions(request.model.http, request.http),
|
||||
})
|
||||
|
||||
export interface MakeInput<Body, Frame, Event, State> {
|
||||
/** Route id used in registry lookup and error messages. */
|
||||
readonly id: string
|
||||
/** Provider identity for route-owned model construction. */
|
||||
readonly provider?: string | ProviderID
|
||||
/** Semantic API contract — owns body construction, body schema, and parsing. */
|
||||
readonly protocol: Protocol<Body, Frame, Event, State>
|
||||
/** Where the request is sent. */
|
||||
readonly endpoint: Endpoint<Body>
|
||||
/** Per-request transport auth. Model-level `Auth` overrides this. */
|
||||
readonly auth?: AuthDef
|
||||
/** Stream framing — bytes -> frames before `protocol.stream.event` decoding. */
|
||||
readonly framing: Framing<Frame>
|
||||
/** Static / per-request headers added before `auth` runs. */
|
||||
readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
|
||||
/** Model defaults used by the route's `.model(...)` helper. */
|
||||
readonly defaults?: RouteDefaults
|
||||
}
|
||||
|
||||
export interface MakeTransportInput<Body, Prepared, Frame, Event, State> {
|
||||
/** Route id used in registry lookup and error messages. */
|
||||
readonly id: string
|
||||
/** Provider identity for route-owned model construction. */
|
||||
readonly provider?: string | ProviderID
|
||||
/** Semantic API contract — owns body construction, body schema, and parsing. */
|
||||
readonly protocol: Protocol<Body, Frame, Event, State>
|
||||
/** Runnable transport route. */
|
||||
readonly transport: Transport<Body, Prepared, Frame>
|
||||
/** Provider/model defaults used by the route's `.model(...)` helper. */
|
||||
readonly defaults?: RouteDefaults
|
||||
}
|
||||
|
||||
const streamError = (route: string, message: string, cause: Cause.Cause<unknown>) => {
|
||||
const failed = cause.reasons.find(Cause.isFailReason)?.error
|
||||
if (failed instanceof LLMErrorClass) return failed
|
||||
return ProviderShared.eventError(route, message, Cause.pretty(cause))
|
||||
}
|
||||
|
||||
function makeFromTransport<Body, Prepared, Frame, Event, State>(
|
||||
input: MakeTransportInput<Body, Prepared, Frame, Event, State>,
|
||||
): Route<Body, Prepared> {
|
||||
const protocol = input.protocol
|
||||
const decodeEventEffect = Schema.decodeUnknownEffect(protocol.stream.event)
|
||||
const decodeEvent = (route: string) => (frame: Frame) =>
|
||||
decodeEventEffect(frame).pipe(
|
||||
Effect.mapError(() =>
|
||||
ProviderShared.eventError(
|
||||
input.id,
|
||||
`Invalid ${route} stream event`,
|
||||
typeof frame === "string" ? frame : ProviderShared.encodeJson(frame),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
const build = (routeInput: MakeTransportInput<Body, Prepared, Frame, Event, State>): Route<Body, Prepared> => {
|
||||
const route: Route<Body, Prepared> = {
|
||||
id: routeInput.id,
|
||||
provider: routeInput.provider === undefined ? undefined : ProviderID.make(routeInput.provider),
|
||||
protocol: protocol.id,
|
||||
transport: routeInput.transport,
|
||||
defaults: routeInput.defaults ?? {},
|
||||
body: protocol.body,
|
||||
with: (patch: RoutePatch<Body, Prepared>) => {
|
||||
const { id, provider, transport, ...defaults } = patch
|
||||
if (!id || id === routeInput.id) throw new Error(`Route.with(${routeInput.id}) requires a new route id`)
|
||||
return build({
|
||||
...routeInput,
|
||||
id,
|
||||
provider: provider ?? routeInput.provider,
|
||||
transport: (transport as Transport<Body, Prepared, Frame> | undefined) ?? routeInput.transport,
|
||||
defaults: mergeRouteDefaults(routeInput.defaults, defaults),
|
||||
})
|
||||
},
|
||||
model: (input: RouteModelInput): ModelRef => modelWithDefaults<RouteModelInput>(route, {}, {})(input),
|
||||
prepareTransport: routeInput.transport.prepare,
|
||||
streamPrepared: (prepared: Prepared, request: LLMRequest, runtime: TransportRuntime) => {
|
||||
const route = `${request.model.provider}/${request.model.route}`
|
||||
const events = routeInput.transport
|
||||
.frames(prepared, request, runtime)
|
||||
.pipe(
|
||||
Stream.mapEffect(decodeEvent(route)),
|
||||
protocol.stream.terminal ? Stream.takeUntil(protocol.stream.terminal) : (stream) => stream,
|
||||
)
|
||||
return events.pipe(
|
||||
Stream.mapAccumEffect(
|
||||
protocol.stream.initial,
|
||||
protocol.stream.step,
|
||||
protocol.stream.onHalt ? { onHalt: protocol.stream.onHalt } : undefined,
|
||||
),
|
||||
Stream.catchCause((cause) => Stream.fail(streamError(route, `Failed to read ${route} stream`, cause))),
|
||||
)
|
||||
},
|
||||
} satisfies Route<Body, Prepared>
|
||||
return register(route)
|
||||
}
|
||||
|
||||
return build(input)
|
||||
}
|
||||
|
||||
export function make<Body, Prepared, Frame, Event, State>(
|
||||
input: MakeTransportInput<Body, Prepared, Frame, Event, State>,
|
||||
): Route<Body, Prepared>
|
||||
/**
|
||||
* Build a `Route` by composing the four orthogonal pieces of a deployment:
|
||||
*
|
||||
* - `Protocol` — what is the API I'm speaking?
|
||||
* - `Endpoint` — where do I send the request?
|
||||
* - `Auth` — how do I authenticate it?
|
||||
* - `Framing` — how do I cut the response stream into protocol frames?
|
||||
*
|
||||
* Plus optional `headers` for cross-cutting deployment concerns (provider
|
||||
* version pins, per-deployment quirks).
|
||||
*
|
||||
* This is the canonical route constructor. If a new route does not fit
|
||||
* this four-axis model, add a purpose-built constructor rather than widening
|
||||
* the public surface preemptively.
|
||||
*/
|
||||
export function make<Body, Frame, Event, State>(
|
||||
input: MakeInput<Body, Frame, Event, State>,
|
||||
): Route<Body, HttpTransport.HttpPrepared<Frame>>
|
||||
export function make<Body, Prepared, Frame, Event, State>(
|
||||
input: MakeInput<Body, Frame, Event, State> | MakeTransportInput<Body, Prepared, Frame, Event, State>,
|
||||
): Route<Body, Prepared> | Route<Body, HttpTransport.HttpPrepared<Frame>> {
|
||||
if ("transport" in input) return makeFromTransport(input)
|
||||
const protocol = input.protocol
|
||||
const encodeBody = Schema.encodeSync(Schema.fromJsonString(protocol.body.schema))
|
||||
return makeFromTransport({
|
||||
id: input.id,
|
||||
provider: input.provider,
|
||||
protocol,
|
||||
transport: HttpTransport.httpJson({
|
||||
endpoint: input.endpoint,
|
||||
auth: input.auth,
|
||||
framing: input.framing,
|
||||
encodeBody,
|
||||
headers: input.headers,
|
||||
}),
|
||||
defaults: input.defaults,
|
||||
})
|
||||
}
|
||||
|
||||
// `compile` is the important boundary: it turns a common `LLMRequest` into a
|
||||
// validated provider body plus transport-private prepared data, but does not
|
||||
// execute transport.
|
||||
const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest) {
|
||||
const resolved = resolveRequestOptions(request)
|
||||
const route = registeredRoute(resolved.model.route)
|
||||
if (!route) return yield* noRoute(resolved.model)
|
||||
|
||||
const body = yield* route.body
|
||||
.from(resolved)
|
||||
.pipe(Effect.flatMap(ProviderShared.validateWith(Schema.decodeUnknownEffect(route.body.schema))))
|
||||
const prepared = yield* route.prepareTransport(body, resolved)
|
||||
|
||||
return {
|
||||
request: resolved,
|
||||
route,
|
||||
body,
|
||||
prepared,
|
||||
}
|
||||
})
|
||||
|
||||
const prepareWith = Effect.fn("LLMClient.prepare")(function* (request: LLMRequest) {
|
||||
const compiled = yield* compile(request)
|
||||
|
||||
return new PreparedRequest({
|
||||
id: compiled.request.id ?? "request",
|
||||
route: compiled.route.id,
|
||||
protocol: compiled.route.protocol,
|
||||
model: compiled.request.model,
|
||||
body: compiled.body,
|
||||
metadata: { transport: compiled.route.transport.id },
|
||||
})
|
||||
})
|
||||
|
||||
const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest) =>
|
||||
Stream.unwrap(
|
||||
Effect.gen(function* () {
|
||||
const compiled = yield* compile(request)
|
||||
return compiled.route.streamPrepared(compiled.prepared, compiled.request, runtime)
|
||||
}),
|
||||
)
|
||||
|
||||
const isToolRunOptions = (input: LLMRequest | ToolRuntime.RunOptions<Tools>): input is ToolRuntime.RunOptions<Tools> =>
|
||||
"request" in input && "tools" in input
|
||||
|
||||
const streamWith = (streamRequest: (request: LLMRequest) => Stream.Stream<LLMEvent, LLMError>): StreamMethod =>
|
||||
((input: LLMRequest | ToolRuntime.RunOptions<Tools>) => {
|
||||
if (isToolRunOptions(input)) return ToolRuntime.stream({ ...input, stream: streamRequest })
|
||||
return streamRequest(input)
|
||||
}) as StreamMethod
|
||||
|
||||
const generateWith = (stream: Interface["stream"]) =>
|
||||
Effect.fn("LLM.generate")(function* (input: LLMRequest | ToolRuntime.RunOptions<Tools>) {
|
||||
return new LLMResponse(
|
||||
yield* stream(input as never).pipe(
|
||||
Stream.runFold(
|
||||
() => ({ events: [] as LLMEvent[], usage: undefined as LLMResponse["usage"] }),
|
||||
(acc, event) => {
|
||||
acc.events.push(event)
|
||||
if ("usage" in event && event.usage !== undefined) acc.usage = event.usage
|
||||
return acc
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
export const prepare = <Body = unknown>(request: LLMRequest) =>
|
||||
prepareWith(request) as Effect.Effect<PreparedRequestOf<Body>, LLMError>
|
||||
|
||||
export function stream(request: LLMRequest): Stream.Stream<LLMEvent, LLMError>
|
||||
export function stream<T extends Tools>(options: ToolRuntime.RunOptions<T>): Stream.Stream<LLMEvent, LLMError>
|
||||
export function stream(input: LLMRequest | ToolRuntime.RunOptions<Tools>) {
|
||||
return Stream.unwrap(
|
||||
Effect.gen(function* () {
|
||||
return (yield* Service).stream(input as never)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
export function generate(request: LLMRequest): Effect.Effect<LLMResponse, LLMError>
|
||||
export function generate<T extends Tools>(options: ToolRuntime.RunOptions<T>): Effect.Effect<LLMResponse, LLMError>
|
||||
export function generate(input: LLMRequest | ToolRuntime.RunOptions<Tools>) {
|
||||
return Effect.gen(function* () {
|
||||
return yield* (yield* Service).generate(input as never)
|
||||
})
|
||||
}
|
||||
|
||||
export const streamRequest = (request: LLMRequest) =>
|
||||
Stream.unwrap(
|
||||
Effect.gen(function* () {
|
||||
return (yield* Service).stream(request)
|
||||
}),
|
||||
)
|
||||
|
||||
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const stream = streamWith(streamRequestWith({ http: yield* RequestExecutor.Service }))
|
||||
return Service.of({ prepare: prepareWith as Interface["prepare"], stream, generate: generateWith(stream) })
|
||||
}),
|
||||
)
|
||||
|
||||
export const layerWithWebSocket: Layer.Layer<Service, never, RequestExecutor.Service | WebSocketExecutorService> =
|
||||
Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const stream = streamWith(
|
||||
streamRequestWith({
|
||||
http: yield* RequestExecutor.Service,
|
||||
webSocket: yield* WebSocketExecutor.Service,
|
||||
}),
|
||||
)
|
||||
return Service.of({ prepare: prepareWith as Interface["prepare"], stream, generate: generateWith(stream) })
|
||||
}),
|
||||
)
|
||||
|
||||
export const Route = { make, model } as const
|
||||
|
||||
export const LLMClient = {
|
||||
Service,
|
||||
layer,
|
||||
layerWithWebSocket,
|
||||
prepare,
|
||||
stream,
|
||||
generate,
|
||||
stepCountIs: ToolRuntime.stepCountIs,
|
||||
} as const
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { LLMRequest } from "../schema"
|
||||
import * as ProviderShared from "../protocols/shared"
|
||||
|
||||
export interface EndpointInput<Body> {
|
||||
readonly request: LLMRequest
|
||||
readonly body: Body
|
||||
}
|
||||
|
||||
export type EndpointPart<Body> = string | ((input: EndpointInput<Body>) => string)
|
||||
|
||||
/**
|
||||
* Declarative URL construction for one route.
|
||||
*
|
||||
* `Endpoint` carries only the path. The host always lives on `model.baseURL`,
|
||||
* supplied by the provider helper that constructs the model. `render(...)`
|
||||
* just appends the path (and any `model.queryParams`) to that host.
|
||||
*
|
||||
* `path` may be a string or a function of `EndpointInput`, for routes whose
|
||||
* URL embeds the model id, region, or another body field (e.g. Bedrock,
|
||||
* Gemini).
|
||||
*/
|
||||
export interface Endpoint<Body> {
|
||||
readonly path: EndpointPart<Body>
|
||||
}
|
||||
|
||||
/** Construct an `Endpoint` from a path string or path function. */
|
||||
export const path = <Body>(value: EndpointPart<Body>): Endpoint<Body> => ({ path: value })
|
||||
|
||||
const renderPart = <Body>(part: EndpointPart<Body>, input: EndpointInput<Body>) =>
|
||||
typeof part === "function" ? part(input) : part
|
||||
|
||||
export const render = <Body>(endpoint: Endpoint<Body>, input: EndpointInput<Body>) => {
|
||||
const url = new URL(
|
||||
`${ProviderShared.trimBaseUrl(input.request.model.baseURL)}${renderPart(endpoint.path, input)}`,
|
||||
)
|
||||
const params = input.request.model.queryParams
|
||||
if (params) for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value)
|
||||
return url
|
||||
}
|
||||
|
||||
export * as Endpoint from "./endpoint"
|
||||
@@ -0,0 +1,374 @@
|
||||
import { Cause, Context, Effect, Layer, Random } from "effect"
|
||||
import {
|
||||
FetchHttpClient,
|
||||
Headers,
|
||||
HttpClient,
|
||||
HttpClientError,
|
||||
HttpClientRequest,
|
||||
HttpClientResponse,
|
||||
} from "effect/unstable/http"
|
||||
import {
|
||||
AuthenticationReason,
|
||||
ContentPolicyReason,
|
||||
HttpContext,
|
||||
HttpRateLimitDetails,
|
||||
HttpRequestDetails,
|
||||
HttpResponseDetails,
|
||||
InvalidRequestReason,
|
||||
LLMError,
|
||||
ProviderInternalReason,
|
||||
QuotaExceededReason,
|
||||
RateLimitReason,
|
||||
TransportReason,
|
||||
UnknownProviderReason,
|
||||
} from "../schema"
|
||||
|
||||
export interface Interface {
|
||||
readonly execute: (
|
||||
request: HttpClientRequest.HttpClientRequest,
|
||||
) => Effect.Effect<HttpClientResponse.HttpClientResponse, LLMError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/LLM/RequestExecutor") {}
|
||||
|
||||
const BODY_LIMIT = 16_384
|
||||
const MAX_RETRIES = 2
|
||||
const BASE_DELAY_MS = 500
|
||||
const MAX_DELAY_MS = 10_000
|
||||
const REDACTED = "<redacted>"
|
||||
|
||||
// One source of truth for what counts as a sensitive name across headers,
|
||||
// URL query keys, and field names embedded inside request/response bodies.
|
||||
//
|
||||
// `SENSITIVE_NAME` is used as both a substring matcher (for free-form header
|
||||
// names like `Authorization` / `X-API-Key`) and as the body-field alternation
|
||||
// list. `SHORT_QUERY_NAME` covers anchored short keys like `?key=…` / `?sig=…`
|
||||
// that are too generic to redact substring-style without false positives.
|
||||
const SENSITIVE_NAME_SOURCE =
|
||||
"authorization|api[-_]?key|access[-_]?token|refresh[-_]?token|id[-_]?token|token|secret|credential|signature|x-amz-signature"
|
||||
const SENSITIVE_NAME = new RegExp(SENSITIVE_NAME_SOURCE, "i")
|
||||
const SHORT_QUERY_NAME = /^(key|sig)$/i
|
||||
const SENSITIVE_BODY_FIELD = new RegExp(`(?:${SENSITIVE_NAME_SOURCE}|key)`, "i")
|
||||
const REDACT_JSON_FIELD = new RegExp(`("(?:${SENSITIVE_BODY_FIELD.source})"\\s*:\\s*)"[^"]*"`, "gi")
|
||||
const REDACT_QUERY_FIELD = new RegExp(`((?:${SENSITIVE_BODY_FIELD.source})=)[^&\\s"]+`, "gi")
|
||||
|
||||
const isSensitiveHeaderName = (name: string) => SENSITIVE_NAME.test(name)
|
||||
|
||||
const isSensitiveQueryName = (name: string) => isSensitiveHeaderName(name) || SHORT_QUERY_NAME.test(name)
|
||||
|
||||
const redactHeaders = (headers: Headers.Headers, redactedNames: ReadonlyArray<string | RegExp>) =>
|
||||
Object.fromEntries(
|
||||
Object.entries(Headers.redact(headers, [...redactedNames, SENSITIVE_NAME])).map(([name, value]) => [
|
||||
name,
|
||||
String(value),
|
||||
]),
|
||||
)
|
||||
|
||||
const redactUrl = (value: string) => {
|
||||
if (!URL.canParse(value)) return REDACTED
|
||||
const url = new URL(value)
|
||||
url.searchParams.forEach((_, key) => {
|
||||
if (isSensitiveQueryName(key)) url.searchParams.set(key, REDACTED)
|
||||
})
|
||||
return url.toString()
|
||||
}
|
||||
|
||||
const normalizedHeaders = (headers: Headers.Headers) =>
|
||||
Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]))
|
||||
|
||||
const requestId = (headers: Record<string, string>) => {
|
||||
return (
|
||||
headers["x-request-id"] ??
|
||||
headers["request-id"] ??
|
||||
headers["x-amzn-requestid"] ??
|
||||
headers["x-amz-request-id"] ??
|
||||
headers["x-goog-request-id"] ??
|
||||
headers["cf-ray"]
|
||||
)
|
||||
}
|
||||
|
||||
const retryableStatus = (status: number) => status === 429 || status === 503 || status === 504 || status === 529
|
||||
|
||||
const retryAfterMs = (headers: Record<string, string>) => {
|
||||
const millis = Number(headers["retry-after-ms"])
|
||||
if (Number.isFinite(millis)) return Math.max(0, millis)
|
||||
|
||||
const value = headers["retry-after"]
|
||||
if (!value) return undefined
|
||||
|
||||
const seconds = Number(value)
|
||||
if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000)
|
||||
|
||||
const date = Date.parse(value)
|
||||
if (!Number.isNaN(date)) return Math.max(0, date - Date.now())
|
||||
return undefined
|
||||
}
|
||||
|
||||
const addRateLimitValue = (target: Record<string, string>, key: string, value: string) => {
|
||||
if (key.length > 0) target[key] = value
|
||||
}
|
||||
|
||||
const rateLimitDetails = (headers: Record<string, string>, retryAfter: number | undefined) => {
|
||||
const limit: Record<string, string> = {}
|
||||
const remaining: Record<string, string> = {}
|
||||
const reset: Record<string, string> = {}
|
||||
|
||||
Object.entries(headers).forEach(([name, value]) => {
|
||||
const openaiLimit = /^x-ratelimit-limit-(.+)$/.exec(name)?.[1]
|
||||
if (openaiLimit) return addRateLimitValue(limit, openaiLimit, value)
|
||||
|
||||
const openaiRemaining = /^x-ratelimit-remaining-(.+)$/.exec(name)?.[1]
|
||||
if (openaiRemaining) return addRateLimitValue(remaining, openaiRemaining, value)
|
||||
|
||||
const openaiReset = /^x-ratelimit-reset-(.+)$/.exec(name)?.[1]
|
||||
if (openaiReset) return addRateLimitValue(reset, openaiReset, value)
|
||||
|
||||
const anthropic = /^anthropic-ratelimit-(.+)-(limit|remaining|reset)$/.exec(name)
|
||||
if (!anthropic) return
|
||||
if (anthropic[2] === "limit") return addRateLimitValue(limit, anthropic[1], value)
|
||||
if (anthropic[2] === "remaining") return addRateLimitValue(remaining, anthropic[1], value)
|
||||
return addRateLimitValue(reset, anthropic[1], value)
|
||||
})
|
||||
|
||||
if (
|
||||
retryAfter === undefined &&
|
||||
Object.keys(limit).length === 0 &&
|
||||
Object.keys(remaining).length === 0 &&
|
||||
Object.keys(reset).length === 0
|
||||
)
|
||||
return undefined
|
||||
|
||||
return new HttpRateLimitDetails({
|
||||
retryAfterMs: retryAfter,
|
||||
limit: Object.keys(limit).length === 0 ? undefined : limit,
|
||||
remaining: Object.keys(remaining).length === 0 ? undefined : remaining,
|
||||
reset: Object.keys(reset).length === 0 ? undefined : reset,
|
||||
})
|
||||
}
|
||||
|
||||
const requestDetails = (request: HttpClientRequest.HttpClientRequest, redactedNames: ReadonlyArray<string | RegExp>) =>
|
||||
new HttpRequestDetails({
|
||||
method: request.method,
|
||||
url: redactUrl(request.url),
|
||||
headers: redactHeaders(request.headers, redactedNames),
|
||||
})
|
||||
|
||||
const responseDetails = (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
redactedNames: ReadonlyArray<string | RegExp>,
|
||||
) =>
|
||||
new HttpResponseDetails({
|
||||
status: response.status,
|
||||
headers: redactHeaders(response.headers, redactedNames),
|
||||
})
|
||||
|
||||
const secretValues = (request: HttpClientRequest.HttpClientRequest) => {
|
||||
const values = new Set<string>()
|
||||
const add = (value: string) => {
|
||||
if (value.length < 4) return
|
||||
values.add(value)
|
||||
values.add(encodeURIComponent(value))
|
||||
}
|
||||
|
||||
Object.entries(request.headers).forEach(([name, value]) => {
|
||||
if (!isSensitiveHeaderName(name)) return
|
||||
add(value)
|
||||
const bearer = /^Bearer\s+(.+)$/i.exec(value)?.[1]
|
||||
if (bearer) add(bearer)
|
||||
})
|
||||
|
||||
if (!URL.canParse(request.url)) return values
|
||||
new URL(request.url).searchParams.forEach((value, key) => {
|
||||
if (isSensitiveQueryName(key)) add(value)
|
||||
})
|
||||
return values
|
||||
}
|
||||
|
||||
// Two passes: structural (redact `"name": "value"` and `name=value` patterns
|
||||
// for any field name that looks sensitive) plus literal (replace any actual
|
||||
// secret values we sent in the request, in case the response echoes one back).
|
||||
const redactBody = (body: string, request: HttpClientRequest.HttpClientRequest) =>
|
||||
Array.from(secretValues(request)).reduce(
|
||||
(text, secret) => text.split(secret).join(REDACTED),
|
||||
body.replace(REDACT_JSON_FIELD, `$1"${REDACTED}"`).replace(REDACT_QUERY_FIELD, `$1${REDACTED}`),
|
||||
)
|
||||
|
||||
const responseBody = (body: string | void, request: HttpClientRequest.HttpClientRequest) => {
|
||||
if (body === undefined) return {}
|
||||
const redacted = redactBody(body, request)
|
||||
if (redacted.length <= BODY_LIMIT) return { body: redacted }
|
||||
return { body: redacted.slice(0, BODY_LIMIT), bodyTruncated: true }
|
||||
}
|
||||
|
||||
const providerMessage = (status: number, body: { readonly body?: string }) => {
|
||||
if (body.body && body.body.length <= 500) return `Provider request failed with HTTP ${status}: ${body.body}`
|
||||
return `Provider request failed with HTTP ${status}`
|
||||
}
|
||||
|
||||
const responseHttp = (input: {
|
||||
readonly request: HttpClientRequest.HttpClientRequest
|
||||
readonly response: HttpClientResponse.HttpClientResponse
|
||||
readonly redactedNames: ReadonlyArray<string | RegExp>
|
||||
readonly body: ReturnType<typeof responseBody>
|
||||
readonly requestId?: string | undefined
|
||||
readonly rateLimit?: HttpRateLimitDetails | undefined
|
||||
}) =>
|
||||
new HttpContext({
|
||||
request: requestDetails(input.request, input.redactedNames),
|
||||
response: responseDetails(input.response, input.redactedNames),
|
||||
...input.body,
|
||||
requestId: input.requestId,
|
||||
rateLimit: input.rateLimit,
|
||||
})
|
||||
|
||||
const statusReason = (input: {
|
||||
readonly status: number
|
||||
readonly message: string
|
||||
readonly retryAfterMs?: number | undefined
|
||||
readonly rateLimit?: HttpRateLimitDetails | undefined
|
||||
readonly http: HttpContext
|
||||
}) => {
|
||||
const body = input.http.body ?? ""
|
||||
if (/content[-_\s]?policy|content_filter|safety/i.test(body)) {
|
||||
return new ContentPolicyReason({ message: input.message, http: input.http })
|
||||
}
|
||||
if (input.status === 401) {
|
||||
return new AuthenticationReason({ message: input.message, kind: "invalid", http: input.http })
|
||||
}
|
||||
if (input.status === 403) {
|
||||
return new AuthenticationReason({ message: input.message, kind: "insufficient-permissions", http: input.http })
|
||||
}
|
||||
if (input.status === 429) {
|
||||
if (/insufficient[-_\s]?quota|quota[-_\s]?exceeded/i.test(body)) {
|
||||
return new QuotaExceededReason({ message: input.message, http: input.http })
|
||||
}
|
||||
return new RateLimitReason({
|
||||
message: input.message,
|
||||
retryAfterMs: input.retryAfterMs,
|
||||
rateLimit: input.rateLimit,
|
||||
http: input.http,
|
||||
})
|
||||
}
|
||||
if (input.status === 400 || input.status === 404 || input.status === 409 || input.status === 422) {
|
||||
return new InvalidRequestReason({ message: input.message, http: input.http })
|
||||
}
|
||||
if (input.status >= 500 || retryableStatus(input.status)) {
|
||||
return new ProviderInternalReason({
|
||||
message: input.message,
|
||||
status: input.status,
|
||||
retryAfterMs: input.retryAfterMs,
|
||||
http: input.http,
|
||||
})
|
||||
}
|
||||
return new UnknownProviderReason({ message: input.message, status: input.status, http: input.http })
|
||||
}
|
||||
|
||||
const statusError =
|
||||
(request: HttpClientRequest.HttpClientRequest, redactedNames: ReadonlyArray<string | RegExp>) =>
|
||||
(response: HttpClientResponse.HttpClientResponse) =>
|
||||
Effect.gen(function* () {
|
||||
if (response.status < 400) return response
|
||||
const body = yield* response.text.pipe(Effect.catch(() => Effect.void))
|
||||
const headers = normalizedHeaders(response.headers)
|
||||
const retryAfter = retryAfterMs(headers)
|
||||
const rateLimit = rateLimitDetails(headers, retryAfter)
|
||||
const details = responseBody(body, request)
|
||||
return yield* new LLMError({
|
||||
module: "RequestExecutor",
|
||||
method: "execute",
|
||||
reason: statusReason({
|
||||
status: response.status,
|
||||
message: providerMessage(response.status, details),
|
||||
retryAfterMs: retryAfter,
|
||||
rateLimit,
|
||||
http: responseHttp({
|
||||
request,
|
||||
response,
|
||||
redactedNames,
|
||||
body: details,
|
||||
requestId: requestId(headers),
|
||||
rateLimit,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
})
|
||||
|
||||
const toHttpError = (redactedNames: ReadonlyArray<string | RegExp>) => (error: unknown) => {
|
||||
const transportError = (input: {
|
||||
readonly message: string
|
||||
readonly kind?: string | undefined
|
||||
readonly request?: HttpClientRequest.HttpClientRequest | undefined
|
||||
}) =>
|
||||
new LLMError({
|
||||
module: "RequestExecutor",
|
||||
method: "execute",
|
||||
reason: new TransportReason({
|
||||
message: input.message,
|
||||
kind: input.kind,
|
||||
url: input.request ? redactUrl(input.request.url) : undefined,
|
||||
http: input.request ? new HttpContext({ request: requestDetails(input.request, redactedNames) }) : undefined,
|
||||
}),
|
||||
})
|
||||
|
||||
if (Cause.isTimeoutError(error)) {
|
||||
return transportError({ message: error.message, kind: "Timeout" })
|
||||
}
|
||||
if (!HttpClientError.isHttpClientError(error)) {
|
||||
return transportError({ message: "HTTP transport failed" })
|
||||
}
|
||||
const request = "request" in error ? error.request : undefined
|
||||
if (error.reason._tag === "TransportError") {
|
||||
return transportError({
|
||||
message: error.reason.description ?? "HTTP transport failed",
|
||||
kind: error.reason._tag,
|
||||
request,
|
||||
})
|
||||
}
|
||||
return transportError({
|
||||
message: `HTTP transport failed: ${error.reason._tag}`,
|
||||
kind: error.reason._tag,
|
||||
request,
|
||||
})
|
||||
}
|
||||
|
||||
const retryDelay = (error: LLMError, attempt: number) => {
|
||||
if (error.retryAfterMs !== undefined) return Effect.succeed(Math.min(error.retryAfterMs, MAX_DELAY_MS))
|
||||
return Random.nextBetween(
|
||||
Math.min(BASE_DELAY_MS * 2 ** attempt * 0.8, MAX_DELAY_MS),
|
||||
Math.min(BASE_DELAY_MS * 2 ** attempt * 1.2, MAX_DELAY_MS),
|
||||
).pipe(Effect.map((delay) => Math.round(delay)))
|
||||
}
|
||||
|
||||
const retryStatusFailures = <A, R>(
|
||||
effect: Effect.Effect<A, LLMError, R>,
|
||||
retries = MAX_RETRIES,
|
||||
attempt = 0,
|
||||
): Effect.Effect<A, LLMError, R> =>
|
||||
Effect.catchTag(effect, "LLM.Error", (error): Effect.Effect<A, LLMError, R> => {
|
||||
if (!error.retryable || retries <= 0) return Effect.fail(error)
|
||||
return retryDelay(error, attempt).pipe(
|
||||
Effect.flatMap((delay) => Effect.sleep(delay)),
|
||||
Effect.flatMap(() => retryStatusFailures(effect, retries - 1, attempt + 1)),
|
||||
)
|
||||
})
|
||||
|
||||
export const layer: Layer.Layer<Service, never, HttpClient.HttpClient> = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const http = yield* HttpClient.HttpClient
|
||||
const executeOnce = (request: HttpClientRequest.HttpClientRequest) =>
|
||||
Effect.gen(function* () {
|
||||
const redactedNames = yield* Headers.CurrentRedactedNames
|
||||
return yield* http
|
||||
.execute(request)
|
||||
.pipe(Effect.mapError(toHttpError(redactedNames)), Effect.flatMap(statusError(request, redactedNames)))
|
||||
})
|
||||
return Service.of({
|
||||
execute: (request) => retryStatusFailures(executeOnce(request)),
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const defaultLayer = layer.pipe(Layer.provide(FetchHttpClient.layer))
|
||||
|
||||
export * as RequestExecutor from "./executor"
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { Stream } from "effect"
|
||||
import * as ProviderShared from "../protocols/shared"
|
||||
import type { LLMError } from "../schema"
|
||||
|
||||
/**
|
||||
* Decode a streaming HTTP response body into provider-protocol frames.
|
||||
*
|
||||
* `Framing` is the byte-stream-shaped seam between transport and protocol:
|
||||
*
|
||||
* - SSE (`Framing.sse`) — UTF-8 decode the body, run the SSE channel decoder,
|
||||
* drop empty / `[DONE]` keep-alives. Each emitted frame is the JSON `data:`
|
||||
* payload of one event.
|
||||
* - AWS event stream — length-prefixed binary frames with CRC checksums.
|
||||
* Each emitted frame is one parsed binary event record.
|
||||
*
|
||||
* The frame type is opaque to this layer; the protocol's `decode` step turns
|
||||
* a frame into a typed chunk.
|
||||
*/
|
||||
export interface Framing<Frame> {
|
||||
readonly id: string
|
||||
readonly frame: (bytes: Stream.Stream<Uint8Array, LLMError>) => Stream.Stream<Frame, LLMError>
|
||||
}
|
||||
|
||||
/** Server-Sent Events framing. Used by every JSON-streaming HTTP provider. */
|
||||
export const sse: Framing<string> = { id: "sse", frame: ProviderShared.sseFraming }
|
||||
|
||||
export * as Framing from "./framing"
|
||||
@@ -0,0 +1,26 @@
|
||||
export { Route, LLMClient, modelLimits, modelRef } from "./client"
|
||||
export type {
|
||||
Route as RouteShape,
|
||||
RouteModelDefaults,
|
||||
RouteModelInput,
|
||||
RouteRoutedModelDefaults,
|
||||
RouteRoutedModelInput,
|
||||
AnyRoute,
|
||||
Interface as LLMClientShape,
|
||||
Service as LLMClientService,
|
||||
ModelRefInput,
|
||||
} from "./client"
|
||||
export * from "./executor"
|
||||
export { Auth } from "./auth"
|
||||
export { AuthOptions } from "./auth-options"
|
||||
export { Endpoint } from "./endpoint"
|
||||
export { Framing } from "./framing"
|
||||
export { Protocol } from "./protocol"
|
||||
export { HttpTransport, WebSocketExecutor, WebSocketTransport } from "./transport"
|
||||
export * as Transport from "./transport"
|
||||
export type { Auth as AuthShape, AuthInput, Credential, CredentialError } from "./auth"
|
||||
export type { ApiKeyMode, AuthOverride, ProviderAuthOption } from "./auth-options"
|
||||
export type { Endpoint as EndpointFn, EndpointInput } from "./endpoint"
|
||||
export type { Framing as FramingDef } from "./framing"
|
||||
export type { Protocol as ProtocolDef } from "./protocol"
|
||||
export type { Transport as TransportDef, TransportRuntime } from "./transport"
|
||||
@@ -0,0 +1,84 @@
|
||||
import { Schema, type Effect } from "effect"
|
||||
import type { LLMError, LLMEvent, LLMRequest, ProtocolID } from "../schema"
|
||||
|
||||
/**
|
||||
* The semantic API contract of one model server family.
|
||||
*
|
||||
* A `Protocol` owns the parts of a route that are intrinsic to "what does
|
||||
* this API look like": how a common `LLMRequest` becomes a provider-native
|
||||
* body, what schema that body must satisfy before it is JSON-encoded, and
|
||||
* how the streaming response decodes back into common `LLMEvent`s.
|
||||
*
|
||||
* Examples:
|
||||
*
|
||||
* - `OpenAIChat.protocol` — chat completions style
|
||||
* - `OpenAIResponses.protocol` — responses API
|
||||
* - `AnthropicMessages.protocol` — messages API with content blocks
|
||||
* - `Gemini.protocol` — generateContent
|
||||
* - `BedrockConverse.protocol` — Converse with binary event-stream framing
|
||||
*
|
||||
* A `Protocol` is **not** a deployment. It does not know which URL, which
|
||||
* headers, or which auth scheme to use. Those are deployment concerns owned
|
||||
* by `Route.make(...)` along with the chosen `Endpoint`, `Auth`,
|
||||
* and `Framing`. This separation is what lets DeepSeek, TogetherAI, Cerebras,
|
||||
* etc. all reuse `OpenAIChat.protocol` without forking 300 lines per provider.
|
||||
*
|
||||
* The four type parameters reflect the pipeline:
|
||||
*
|
||||
* - `Body` — provider-native request body candidate. `Route.make(...)`
|
||||
* validates and JSON-encodes it with `body.schema`.
|
||||
* - `Frame` — one unit of the framed response stream. SSE: a JSON data
|
||||
* string. AWS event stream: a parsed binary frame.
|
||||
* - `Event` — schema-decoded provider event produced from one frame.
|
||||
* - `State` — accumulator threaded through `stream.step` to translate event
|
||||
* sequences into `LLMEvent` sequences.
|
||||
*/
|
||||
export interface Protocol<Body, Frame, Event, State> {
|
||||
/** Stable id for the wire protocol implementation. */
|
||||
readonly id: ProtocolID
|
||||
/** Request side: schema for the provider-native body and how to build it. */
|
||||
readonly body: ProtocolBody<Body>
|
||||
/** Response side: streaming state machine. */
|
||||
readonly stream: ProtocolStream<Frame, Event, State>
|
||||
}
|
||||
|
||||
export interface ProtocolBody<Body> {
|
||||
/** Schema for the validated provider-native body sent as the JSON request. */
|
||||
readonly schema: Schema.Codec<Body, unknown>
|
||||
/** Build the provider-native body from a common `LLMRequest`. */
|
||||
readonly from: (request: LLMRequest) => Effect.Effect<Body, LLMError>
|
||||
}
|
||||
|
||||
export interface ProtocolStream<Frame, Event, State> {
|
||||
/** Schema for one decoded streaming event, decoded from a transport frame. */
|
||||
readonly event: Schema.Codec<Event, Frame>
|
||||
/** Initial parser state. Called once per response. */
|
||||
readonly initial: () => State
|
||||
/** Translate one event into emitted `LLMEvent`s plus the next state. */
|
||||
readonly step: (state: State, event: Event) => Effect.Effect<readonly [State, ReadonlyArray<LLMEvent>], LLMError>
|
||||
/** Optional request-completion signal for transports that do not end naturally. */
|
||||
readonly terminal?: (event: Event) => boolean
|
||||
/** Optional flush emitted when the framed stream ends. */
|
||||
readonly onHalt?: (state: State) => ReadonlyArray<LLMEvent>
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a `Protocol` from its body and stream pieces:
|
||||
*
|
||||
* - `body.schema` infers the provider-native request body shape.
|
||||
* - `body.from` ties the common `LLMRequest` to the provider body.
|
||||
* - `stream.event` infers the decoded streaming event and the wire frame.
|
||||
* - `stream.initial`, `stream.step`, and `stream.onHalt` infer the parser state.
|
||||
*
|
||||
* Provider implementations should usually call `Protocol.make({ ... })`
|
||||
* without explicit type arguments; the schemas and parser functions are the
|
||||
* source of truth. The constructor remains as the public seam for future
|
||||
* cross-cutting concerns such as tracing or instrumentation.
|
||||
*/
|
||||
export const make = <Body, Frame, Event, State>(
|
||||
input: Protocol<Body, Frame, Event, State>,
|
||||
): Protocol<Body, Frame, Event, State> => input
|
||||
|
||||
export const jsonEvent = <const S extends Schema.Top>(schema: S) => Schema.fromJsonString(schema)
|
||||
|
||||
export * as Protocol from "./protocol"
|
||||
@@ -0,0 +1,122 @@
|
||||
import { Effect, Stream } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import { Auth, type Auth as AuthDef } from "../auth"
|
||||
import { type Endpoint, render as renderEndpoint } from "../endpoint"
|
||||
import type { Framing } from "../framing"
|
||||
import type { Transport } from "./index"
|
||||
import * as ProviderShared from "../../protocols/shared"
|
||||
import { mergeJsonRecords, type LLMRequest } from "../../schema"
|
||||
|
||||
export interface JsonRequestInput<Body> {
|
||||
readonly body: Body
|
||||
readonly request: LLMRequest
|
||||
readonly endpoint: Endpoint<Body>
|
||||
readonly auth: AuthDef
|
||||
readonly encodeBody: (body: Body) => string
|
||||
readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
|
||||
}
|
||||
|
||||
export interface JsonRequestParts<Body = unknown> {
|
||||
readonly url: string
|
||||
readonly jsonBody: Body | Record<string, unknown>
|
||||
readonly bodyText: string
|
||||
readonly headers: Headers.Headers
|
||||
}
|
||||
|
||||
export interface HttpPrepared<Frame> {
|
||||
readonly request: HttpClientRequest.HttpClientRequest
|
||||
readonly framing: Framing<Frame>
|
||||
}
|
||||
|
||||
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
|
||||
if (!query) return url
|
||||
const next = new URL(url)
|
||||
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
|
||||
return next.toString()
|
||||
}
|
||||
|
||||
const bodyWithOverlay = <Body>(body: Body, request: LLMRequest, encodeBody: (body: Body) => string) =>
|
||||
Effect.gen(function* () {
|
||||
if (request.http?.body === undefined) return { jsonBody: body, bodyText: encodeBody(body) }
|
||||
if (ProviderShared.isRecord(body)) {
|
||||
const overlaid = mergeJsonRecords(body, request.http.body) ?? {}
|
||||
return { jsonBody: overlaid, bodyText: ProviderShared.encodeJson(overlaid) }
|
||||
}
|
||||
return yield* ProviderShared.invalidRequest("http.body can only overlay JSON object request bodies")
|
||||
})
|
||||
|
||||
export const jsonRequestParts = <Body>(input: JsonRequestInput<Body>) =>
|
||||
Effect.gen(function* () {
|
||||
const url = applyQuery(
|
||||
renderEndpoint(input.endpoint, { request: input.request, body: input.body }).toString(),
|
||||
input.request.http?.query,
|
||||
)
|
||||
const body = yield* bodyWithOverlay(input.body, input.request, input.encodeBody)
|
||||
const headers = yield* Auth.toEffect(Auth.isAuth(input.request.model.auth) ? input.request.model.auth : input.auth)(
|
||||
{
|
||||
request: input.request,
|
||||
method: "POST",
|
||||
url,
|
||||
body: body.bodyText,
|
||||
headers: Headers.fromInput({
|
||||
...(input.headers?.({ request: input.request }) ?? {}),
|
||||
...input.request.model.headers,
|
||||
...input.request.http?.headers,
|
||||
}),
|
||||
},
|
||||
)
|
||||
return { url, jsonBody: body.jsonBody, bodyText: body.bodyText, headers }
|
||||
})
|
||||
|
||||
export interface HttpJsonInput<Body, Frame> {
|
||||
readonly endpoint: Endpoint<Body>
|
||||
readonly auth?: AuthDef
|
||||
readonly framing: Framing<Frame>
|
||||
readonly encodeBody: (body: Body) => string
|
||||
readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
|
||||
}
|
||||
|
||||
export type HttpJsonPatch<Body, Frame> = Partial<HttpJsonInput<Body, Frame>>
|
||||
|
||||
export interface HttpJsonTransport<Body, Frame> extends Transport<Body, HttpPrepared<Frame>, Frame> {
|
||||
readonly with: (patch: HttpJsonPatch<Body, Frame>) => HttpJsonTransport<Body, Frame>
|
||||
}
|
||||
|
||||
export const httpJson = <Body, Frame>(input: HttpJsonInput<Body, Frame>): HttpJsonTransport<Body, Frame> => ({
|
||||
id: "http-json",
|
||||
with: (patch) => httpJson({ ...input, ...patch }),
|
||||
prepare: (body, request) =>
|
||||
jsonRequestParts({
|
||||
body,
|
||||
request,
|
||||
endpoint: input.endpoint,
|
||||
auth: input.auth ?? Auth.bearer(),
|
||||
encodeBody: input.encodeBody,
|
||||
headers: input.headers,
|
||||
}).pipe(
|
||||
Effect.map((parts) => ({
|
||||
request: ProviderShared.jsonPost({ url: parts.url, body: parts.bodyText, headers: parts.headers }),
|
||||
framing: input.framing,
|
||||
})),
|
||||
),
|
||||
frames: (prepared, request, runtime) =>
|
||||
Stream.unwrap(
|
||||
runtime.http
|
||||
.execute(prepared.request)
|
||||
.pipe(
|
||||
Effect.map((response) =>
|
||||
prepared.framing.frame(
|
||||
response.stream.pipe(
|
||||
Stream.mapError((error) =>
|
||||
ProviderShared.eventError(
|
||||
`${request.model.provider}/${request.model.route}`,
|
||||
`Failed to read ${request.model.provider}/${request.model.route} stream`,
|
||||
ProviderShared.errorText(error),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
})
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { Effect, Stream } from "effect"
|
||||
import type { Interface as RequestExecutorInterface } from "../executor"
|
||||
import type { Interface as WebSocketExecutorInterface } from "./websocket"
|
||||
import type { LLMError, LLMRequest } from "../../schema"
|
||||
|
||||
export interface TransportRuntime {
|
||||
readonly http: RequestExecutorInterface
|
||||
readonly webSocket?: WebSocketExecutorInterface
|
||||
}
|
||||
|
||||
export interface Transport<Body, Prepared, Frame> {
|
||||
readonly id: string
|
||||
readonly prepare: (body: Body, request: LLMRequest) => Effect.Effect<Prepared, LLMError>
|
||||
readonly frames: (
|
||||
prepared: Prepared,
|
||||
request: LLMRequest,
|
||||
runtime: TransportRuntime,
|
||||
) => Stream.Stream<Frame, LLMError>
|
||||
}
|
||||
|
||||
export * as HttpTransport from "./http"
|
||||
export { WebSocketExecutor, WebSocketTransport } from "./websocket"
|
||||
@@ -0,0 +1,282 @@
|
||||
import { Cause, Context, Effect, Queue, Stream } from "effect"
|
||||
import { Headers } from "effect/unstable/http"
|
||||
import { Auth, type Auth as AuthDef } from "../auth"
|
||||
import type { Endpoint } from "../endpoint"
|
||||
import { LLMError, TransportReason, type LLMRequest } from "../../schema"
|
||||
import * as HttpTransport from "./http"
|
||||
import type { Transport } from "./index"
|
||||
|
||||
export interface WebSocketRequest {
|
||||
readonly url: string
|
||||
readonly headers: Headers.Headers
|
||||
}
|
||||
|
||||
export interface WebSocketConnection {
|
||||
readonly sendText: (message: string) => Effect.Effect<void, LLMError>
|
||||
readonly messages: Stream.Stream<string | Uint8Array, LLMError>
|
||||
readonly close: Effect.Effect<void, never>
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
readonly open: (input: WebSocketRequest) => Effect.Effect<WebSocketConnection, LLMError>
|
||||
}
|
||||
|
||||
type WebSocketConstructorWithHeaders = new (
|
||||
url: string,
|
||||
options?: { readonly headers?: Headers.Headers },
|
||||
) => globalThis.WebSocket
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/LLM/WebSocketExecutor") {}
|
||||
|
||||
const transportError = (
|
||||
method: string,
|
||||
message: string,
|
||||
input: { readonly url?: string; readonly kind?: string } = {},
|
||||
) =>
|
||||
new LLMError({
|
||||
module: "WebSocketExecutor",
|
||||
method,
|
||||
reason: new TransportReason({ message, url: input.url, kind: input.kind }),
|
||||
})
|
||||
|
||||
const eventMessage = (event: Event) => {
|
||||
if ("message" in event && typeof event.message === "string") return event.message
|
||||
return event.type
|
||||
}
|
||||
|
||||
const binaryMessage = (data: unknown) => {
|
||||
if (data instanceof Uint8Array) return data
|
||||
if (data instanceof ArrayBuffer) return new Uint8Array(data)
|
||||
if (ArrayBuffer.isView(data)) return new Uint8Array(data.buffer, data.byteOffset, data.byteLength)
|
||||
return undefined
|
||||
}
|
||||
|
||||
const waitOpen = (ws: globalThis.WebSocket, input: WebSocketRequest) => {
|
||||
if (ws.readyState === globalThis.WebSocket.OPEN) return Effect.void
|
||||
if (ws.readyState === globalThis.WebSocket.CLOSING || ws.readyState === globalThis.WebSocket.CLOSED) {
|
||||
return Effect.fail(
|
||||
transportError("open", `WebSocket closed before opening (state ${ws.readyState})`, {
|
||||
url: input.url,
|
||||
kind: "open",
|
||||
}),
|
||||
)
|
||||
}
|
||||
return Effect.callback<void, LLMError>((resume, signal) => {
|
||||
const cleanup = () => {
|
||||
ws.removeEventListener("open", onOpen)
|
||||
ws.removeEventListener("error", onError)
|
||||
ws.removeEventListener("close", onClose)
|
||||
signal.removeEventListener("abort", onAbort)
|
||||
}
|
||||
const onAbort = () => {
|
||||
cleanup()
|
||||
if (ws.readyState !== globalThis.WebSocket.CLOSED && ws.readyState !== globalThis.WebSocket.CLOSING)
|
||||
ws.close(1000)
|
||||
}
|
||||
const onOpen = () => {
|
||||
cleanup()
|
||||
resume(Effect.void)
|
||||
}
|
||||
const onError = (event: Event) => {
|
||||
cleanup()
|
||||
resume(
|
||||
Effect.fail(
|
||||
transportError("open", `Failed to open WebSocket: ${eventMessage(event)}`, { url: input.url, kind: "open" }),
|
||||
),
|
||||
)
|
||||
}
|
||||
const onClose = (event: CloseEvent) => {
|
||||
cleanup()
|
||||
resume(
|
||||
Effect.fail(
|
||||
transportError("open", `WebSocket closed before opening with code ${event.code}`, {
|
||||
url: input.url,
|
||||
kind: "open",
|
||||
}),
|
||||
),
|
||||
)
|
||||
}
|
||||
ws.addEventListener("open", onOpen, { once: true })
|
||||
ws.addEventListener("error", onError, { once: true })
|
||||
ws.addEventListener("close", onClose, { once: true })
|
||||
signal.addEventListener("abort", onAbort, { once: true })
|
||||
})
|
||||
}
|
||||
|
||||
const webSocketUrl = (value: string) =>
|
||||
Effect.try({
|
||||
try: () => {
|
||||
const url = new URL(value)
|
||||
if (url.protocol === "https:") {
|
||||
url.protocol = "wss:"
|
||||
return url.toString()
|
||||
}
|
||||
if (url.protocol === "http:") {
|
||||
url.protocol = "ws:"
|
||||
return url.toString()
|
||||
}
|
||||
throw new Error(`Unsupported WebSocket URL protocol ${url.protocol}`)
|
||||
},
|
||||
catch: (error) =>
|
||||
transportError("prepare", error instanceof Error ? error.message : "Invalid WebSocket URL", {
|
||||
url: value,
|
||||
kind: "websocket",
|
||||
}),
|
||||
})
|
||||
|
||||
export const open = (input: WebSocketRequest) =>
|
||||
Effect.try({
|
||||
try: () =>
|
||||
new (globalThis.WebSocket as unknown as WebSocketConstructorWithHeaders)(input.url, { headers: input.headers }),
|
||||
catch: (error) =>
|
||||
transportError("open", error instanceof Error ? error.message : "Failed to construct WebSocket", {
|
||||
url: input.url,
|
||||
kind: "open",
|
||||
}),
|
||||
}).pipe(Effect.flatMap((ws) => fromWebSocket(ws, input)))
|
||||
|
||||
export const fromWebSocket = (
|
||||
ws: globalThis.WebSocket,
|
||||
input: WebSocketRequest,
|
||||
): Effect.Effect<WebSocketConnection, LLMError> =>
|
||||
Effect.gen(function* () {
|
||||
yield* waitOpen(ws, input)
|
||||
const messages = yield* Queue.bounded<string | Uint8Array, LLMError | Cause.Done<void>>(128)
|
||||
|
||||
const onMessage = (event: MessageEvent) => {
|
||||
if (typeof event.data === "string") return Queue.offerUnsafe(messages, event.data)
|
||||
const binary = binaryMessage(event.data)
|
||||
if (binary) return Queue.offerUnsafe(messages, binary)
|
||||
Queue.failCauseUnsafe(
|
||||
messages,
|
||||
Cause.fail(
|
||||
transportError("message", "Unsupported WebSocket message payload", { url: input.url, kind: "message" }),
|
||||
),
|
||||
)
|
||||
}
|
||||
const onError = (event: Event) => {
|
||||
Queue.failCauseUnsafe(
|
||||
messages,
|
||||
Cause.fail(
|
||||
transportError("message", `WebSocket error: ${eventMessage(event)}`, { url: input.url, kind: "message" }),
|
||||
),
|
||||
)
|
||||
}
|
||||
const onClose = (event: CloseEvent) => {
|
||||
if (event.code === 1000 || event.code === 1005) return Queue.endUnsafe(messages)
|
||||
Queue.failCauseUnsafe(
|
||||
messages,
|
||||
Cause.fail(
|
||||
transportError("message", `WebSocket closed with code ${event.code}`, { url: input.url, kind: "close" }),
|
||||
),
|
||||
)
|
||||
}
|
||||
const cleanup = Effect.sync(() => {
|
||||
ws.removeEventListener("message", onMessage)
|
||||
ws.removeEventListener("error", onError)
|
||||
ws.removeEventListener("close", onClose)
|
||||
}).pipe(Effect.andThen(Queue.shutdown(messages)))
|
||||
|
||||
ws.addEventListener("message", onMessage)
|
||||
ws.addEventListener("error", onError)
|
||||
ws.addEventListener("close", onClose)
|
||||
|
||||
return {
|
||||
sendText: (message) =>
|
||||
Effect.try({
|
||||
try: () => ws.send(message),
|
||||
catch: (error) =>
|
||||
transportError("sendText", error instanceof Error ? error.message : "Failed to send WebSocket message", {
|
||||
url: input.url,
|
||||
kind: "write",
|
||||
}),
|
||||
}),
|
||||
messages: Stream.fromQueue(messages),
|
||||
close: cleanup.pipe(
|
||||
Effect.andThen(
|
||||
Effect.sync(() => {
|
||||
if (ws.readyState === globalThis.WebSocket.CLOSED || ws.readyState === globalThis.WebSocket.CLOSING) return
|
||||
ws.close(1000)
|
||||
}),
|
||||
),
|
||||
),
|
||||
}
|
||||
})
|
||||
|
||||
export const messageText = (message: string | Uint8Array, decoder: TextDecoder) =>
|
||||
typeof message === "string" ? message : decoder.decode(message)
|
||||
|
||||
export interface JsonPrepared {
|
||||
readonly url: string
|
||||
readonly headers: Headers.Headers
|
||||
readonly message: string
|
||||
}
|
||||
|
||||
export interface JsonInput<Body, Message> {
|
||||
readonly endpoint: Endpoint<Body>
|
||||
readonly auth?: AuthDef
|
||||
readonly encodeBody: (body: Body) => string
|
||||
readonly toMessage: (body: Body | Record<string, unknown>) => Effect.Effect<Message, LLMError>
|
||||
readonly encodeMessage: (message: Message) => string
|
||||
readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
|
||||
}
|
||||
|
||||
export type JsonPatch<Body, Message> = Partial<JsonInput<Body, Message>>
|
||||
|
||||
export interface JsonTransport<Body, Message> extends Transport<Body, JsonPrepared, string> {
|
||||
readonly with: (patch: JsonPatch<Body, Message>) => JsonTransport<Body, Message>
|
||||
}
|
||||
|
||||
export const json = <Body, Message>(input: JsonInput<Body, Message>): JsonTransport<Body, Message> => ({
|
||||
id: "websocket-json",
|
||||
with: (patch) => json({ ...input, ...patch }),
|
||||
prepare: (body, request) =>
|
||||
Effect.gen(function* () {
|
||||
const parts = yield* HttpTransport.jsonRequestParts({
|
||||
body,
|
||||
request,
|
||||
endpoint: input.endpoint,
|
||||
auth: input.auth ?? Auth.bearer(),
|
||||
encodeBody: input.encodeBody,
|
||||
headers: input.headers,
|
||||
})
|
||||
return {
|
||||
url: yield* webSocketUrl(parts.url),
|
||||
headers: parts.headers,
|
||||
message: input.encodeMessage(yield* input.toMessage(parts.jsonBody)),
|
||||
}
|
||||
}),
|
||||
frames: (prepared, _request, runtime) => {
|
||||
const webSocket = runtime.webSocket
|
||||
if (!webSocket) {
|
||||
return Stream.fail(
|
||||
transportError("json", "WebSocket JSON transport requires WebSocketExecutor.Service", {
|
||||
url: prepared.url,
|
||||
kind: "websocket",
|
||||
}),
|
||||
)
|
||||
}
|
||||
const decoder = new TextDecoder()
|
||||
return Stream.unwrap(
|
||||
Effect.gen(function* () {
|
||||
const connection = yield* Effect.acquireRelease(
|
||||
webSocket.open({ url: prepared.url, headers: prepared.headers }),
|
||||
(connection) => connection.close,
|
||||
)
|
||||
yield* connection.sendText(prepared.message)
|
||||
return connection.messages.pipe(Stream.map((message) => messageText(message, decoder)))
|
||||
}),
|
||||
)
|
||||
},
|
||||
})
|
||||
|
||||
export const WebSocketExecutor = {
|
||||
Service,
|
||||
open,
|
||||
fromWebSocket,
|
||||
messageText,
|
||||
} as const
|
||||
|
||||
export const WebSocketTransport = {
|
||||
json,
|
||||
} as const
|
||||
Reference in New Issue
Block a user