EnterpriseAI/ClosedCode
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

test(e2e): isolate prompt tests with per-worker backend (#20464)

Browse Source
This commit is contained in:
Kit Langton
2026-04-01 15:58:11 +00:00
committed by GitHub
parent d58004a864
commit 38d2276592
8 changed files with 429 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/e2e/prompt/mock.ts
+46
View File
@@ -0,0 +1,46 @@
import { createSdk } from "../utils"
export const openaiModel = { providerID: "openai", modelID: "gpt-5.3-chat-latest" }
type Hit = { body: Record<string, unknown> }
export function bodyText(hit: Hit) {
return JSON.stringify(hit.body)
}
export function titleMatch(hit: Hit) {
return bodyText(hit).includes("Generate a title for this conversation")
}
export function promptMatch(token: string) {
return (hit: Hit) => bodyText(hit).includes(token)
}
export async function withMockOpenAI<T>(input: { serverUrl: string; llmUrl: string; fn: () => Promise<T> }) {
const sdk = createSdk(undefined, input.serverUrl)
const prev = await sdk.global.config.get().then((res) => res.data ?? {})
try {
await sdk.global.config.update({
config: {
...prev,
model: `${openaiModel.providerID}/${openaiModel.modelID}`,
enabled_providers: ["openai"],
provider: {
...prev.provider,
openai: {
...prev.provider?.openai,
options: {
...prev.provider?.openai?.options,
apiKey: "test-key",
baseURL: input.llmUrl,
},
},
},
},
})
return await input.fn()
} finally {
await sdk.global.config.update({ config: prev })
}
}
packages/app/e2e/prompt/prompt-async.spec.ts
+32 -26
View File
@@ -1,47 +1,53 @@
import { test, expect } from "../fixtures"
import { promptSelector } from "../selectors"
import { cleanupSession, sessionIDFromUrl, withSession } from "../actions"
import { assistantText, sessionIDFromUrl, withSession } from "../actions"
import { openaiModel, promptMatch, withMockOpenAI } from "./mock"
const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim()
// Regression test for Issue #12453: the synchronous POST /message endpoint holds
// the connection open while the agent works, causing "Failed to fetch" over
// VPN/Tailscale. The fix switches to POST /prompt_async which returns immediately.
test("prompt succeeds when sync message endpoint is unreachable", async ({ page, sdk, gotoSession }) => {
test("prompt succeeds when sync message endpoint is unreachable", async ({
page,
llm,
backend,
withBackendProject,
}) => {
test.setTimeout(120_000)
// Simulate Tailscale/VPN killing the long-lived sync connection
await page.route("**/session/*/message", (route) => route.abort("connectionfailed"))
await gotoSession()
await withMockOpenAI({
serverUrl: backend.url,
llmUrl: llm.url,
fn: async () => {
const token = `E2E_ASYNC_${Date.now()}`
await llm.textMatch(promptMatch(token), token)
const token = `E2E_ASYNC_${Date.now()}`
await page.locator(promptSelector).click()
await page.keyboard.type(`Reply with exactly: ${token}`)
await page.keyboard.press("Enter")
await withBackendProject(
async (project) => {
await page.locator(promptSelector).click()
await page.keyboard.type(`Reply with exactly: ${token}`)
await page.keyboard.press("Enter")
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
const sessionID = sessionIDFromUrl(page.url())!
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
const sessionID = sessionIDFromUrl(page.url())!
project.trackSession(sessionID)
try {
// Agent response arrives via SSE despite sync endpoint being dead
await expect
.poll(
async () => {
const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
return messages
.filter((m) => m.info.role === "assistant")
.flatMap((m) => m.parts)
.filter((p) => p.type === "text")
.map((p) => p.text)
.join("\n")
await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
await expect
.poll(() => assistantText(project.sdk, sessionID), { timeout: 90_000 })
.toContain(token)
},
{
model: openaiModel,
},
{ timeout: 90_000 },
)
.toContain(token)
} finally {
await cleanupSession({ sdk, sessionID })
}
},
})
})
test("failed prompt send restores the composer input", async ({ page, sdk, gotoSession }) => {
packages/app/e2e/prompt/prompt.spec.ts
+33 -73
View File
@@ -1,44 +1,9 @@
import fs from "node:fs/promises"
import path from "node:path"
import { test, expect } from "../fixtures"
import { promptSelector } from "../selectors"
import { sessionIDFromUrl } from "../actions"
import { createSdk } from "../utils"
import { assistantText, sessionIDFromUrl } from "../actions"
import { openaiModel, promptMatch, titleMatch, withMockOpenAI } from "./mock"
async function config(dir: string, url: string) {
await fs.writeFile(
path.join(dir, "opencode.json"),
JSON.stringify({
$schema: "https://opencode.ai/config.json",
enabled_providers: ["e2e-llm"],
provider: {
"e2e-llm": {
name: "E2E LLM",
npm: "@ai-sdk/openai-compatible",
env: [],
models: {
"test-model": {
name: "Test Model",
tool_call: true,
limit: { context: 128000, output: 32000 },
},
},
options: {
apiKey: "test-key",
baseURL: url,
},
},
},
agent: {
build: {
model: "e2e-llm/test-model",
},
},
}),
)
}
test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => {
test("can send a prompt and receive a reply", async ({ page, llm, backend, withBackendProject }) => {
test.setTimeout(120_000)
const pageErrors: string[] = []
@@ -48,48 +13,43 @@ test("can send a prompt and receive a reply", async ({ page, llm, withProject })
page.on("pageerror", onPageError)
try {
await withProject(
async (project) => {
const sdk = createSdk(project.directory)
await withMockOpenAI({
serverUrl: backend.url,
llmUrl: llm.url,
fn: async () => {
const token = `E2E_OK_${Date.now()}`
await llm.text(token)
await project.gotoSession()
await llm.textMatch(titleMatch, "E2E Title")
await llm.textMatch(promptMatch(token), token)
const prompt = page.locator(promptSelector)
await prompt.click()
await page.keyboard.type(`Reply with exactly: ${token}`)
await page.keyboard.press("Enter")
await withBackendProject(
async (project) => {
const prompt = page.locator(promptSelector)
await prompt.click()
await page.keyboard.type(`Reply with exactly: ${token}`)
await page.keyboard.press("Enter")
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
const sessionID = (() => {
const id = sessionIDFromUrl(page.url())
if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
return id
})()
project.trackSession(sessionID)
const sessionID = (() => {
const id = sessionIDFromUrl(page.url())
if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
return id
})()
project.trackSession(sessionID)
await expect
.poll(
async () => {
const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
return messages
.filter((m) => m.info.role === "assistant")
.flatMap((m) => m.parts)
.filter((p) => p.type === "text")
.map((p) => p.text)
.join("\n")
},
{ timeout: 30_000 },
)
.toContain(token)
await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
await expect
.poll(() => assistantText(project.sdk, sessionID), { timeout: 30_000 })
.toContain(token)
},
{
model: openaiModel,
},
)
},
{
model: { providerID: "e2e-llm", modelID: "test-model" },
setup: (dir) => config(dir, llm.url),
},
)
})
} finally {
page.off("pageerror", onPageError)
}