feat(simulation): expose semantic UI snapshots (#37802)

packages/simulation/src/frontend/actions.ts

@@ -9,9 +9,10 @@ import {
   type MockInput,
   type MockMouse,
 } from "@opentui/core/testing"
-import { Config, Effect, FileSystem } from "effect"
+import { Config, Effect, FileSystem, Schema } from "effect"
-import type { SimulationProtocol } from "../protocol"
+import { SimulationProtocol } from "../protocol"
 import { SimulationRenderer } from "./renderer"
+import { SimulationSemantics } from "./semantics"
 
 export type Action = SimulationProtocol.Frontend.Action
 export type Element = SimulationProtocol.Frontend.Element
@@ -60,7 +61,8 @@ function hit(renderer: CliRenderer, renderable: Renderable) {
   if (renderable.width <= 0 || renderable.height <= 0) return false
   const x = Math.floor(renderable.screenX + renderable.width / 2)
   const y = Math.floor(renderable.screenY + renderable.height / 2)
-  return renderer.hitTest(x, y) === renderable.num
+  const target = renderer.hitTest(x, y)
+  return all(renderable).some((item) => item.num === target)
 }
 
 /**
@@ -122,6 +124,25 @@ export function state(harness: Harness) {
   }
 }
 
+export function snapshot(harness: Harness): SimulationProtocol.Frontend.SemanticSnapshot {
+  const ids = new Set<string>()
+  const visit = (renderable: Renderable, parent?: string): SimulationProtocol.Frontend.SemanticNode[] => {
+    if (!renderable.visible || renderable.isDestroyed) return []
+    const definition = SimulationSemantics.read(renderable)?.()
+    if (definition && ids.has(renderable.id)) throw new Error(`duplicate semantic UI id: ${renderable.id}`)
+    if (definition) ids.add(renderable.id)
+    const node = definition
+      ? [{ id: renderable.id, ...definition, ...(parent === undefined ? {} : { parent }), element: renderable.num }]
+      : []
+    const ancestor = definition ? renderable.id : parent
+    return [...node, ...children(renderable).flatMap((child) => visit(child, ancestor))]
+  }
+  return Schema.decodeUnknownSync(SimulationProtocol.Frontend.SemanticSnapshot)({
+    format: "opencode-ui-snapshot-v1",
+    nodes: visit(harness.renderer.root),
+  })
+}
+
 export function matches(harness: Pick<Harness, "screen">, text: string) {
   return harness.screen().includes(text)
 }
@@ -183,9 +204,22 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness
         .find((item) => item.num === action.target)
         ?.focus()
       break
-    case "ui.click":
+    case "ui.click": {
-      yield* Effect.tryPromise(() => harness.mockMouse.click(action.x, action.y))
+      const target = all(harness.renderer.root).find((item) => item.num === action.target)
+      if (!target || !target.visible || target.isDestroyed)
+        return yield* Effect.fail(new Error(`click target is stale or unavailable: ${action.target}`))
+      if (
+        !Number.isFinite(action.x) ||
+        action.x < 0 ||
+        action.x >= target.width ||
+        !Number.isFinite(action.y) ||
+        action.y < 0 ||
+        action.y >= target.height
+      )
+        return yield* Effect.fail(new Error("click position must be within the target element"))
+      yield* Effect.tryPromise(() => harness.mockMouse.click(target.screenX + action.x, target.screenY + action.y))
       break
+    }
     case "ui.resize":
       if (
         !Number.isSafeInteger(action.cols) ||

packages/simulation/src/frontend/semantics.ts

@@ -0,0 +1,20 @@
+import type { Renderable } from "@opentui/core"
+import type { SimulationProtocol } from "../protocol"
+
+// Semantic renderables set an explicit stable OpenTUI id so ui.state and
+// ui.snapshot expose the same identity. Hierarchy and element handles come
+// from the live render tree.
+export type Definition = Omit<SimulationProtocol.Frontend.SemanticNode, "id" | "element" | "parent">
+
+const key = Symbol.for("opencode.simulation.semantics")
+
+const bind = (definition: () => Definition) => (renderable: Renderable) => {
+  Object.defineProperty(renderable, key, { value: definition, configurable: true })
+}
+
+export const read = (renderable: Renderable) => {
+  const definition: unknown = Reflect.get(renderable, key)
+  return typeof definition === "function" ? (definition as () => Definition) : undefined
+}
+
+export const SimulationSemantics = { bind, read }

packages/simulation/src/frontend/server.ts

@@ -22,6 +22,8 @@ function handle(harness: Harness, request: SimulationProtocol.Frontend.Request)
       return SimulationActions.screenshot(harness, request.params?.name)
     case "ui.state":
       return Effect.sync(() => SimulationActions.state(harness))
+    case "ui.snapshot":
+      return Effect.sync(() => SimulationActions.snapshot(harness))
     case "ui.matches":
       return Effect.sync(() => SimulationActions.matches(harness, request.params.text))
     case "ui.recording.finish":

packages/simulation/src/protocol/index.ts

@@ -67,9 +67,10 @@ export namespace Handshake {
   export const Params = Schema.Struct({
     client: Identity,
     expectedRole: EndpointRole,
-    offeredVersions: Schema.Array(
+    offeredVersions: Schema.Array(Schema.Int.check(Schema.isGreaterThan(0))).check(
-      Schema.Int.check(Schema.isGreaterThan(0)),
+      Schema.isMinLength(1),
-    ).check(Schema.isMinLength(1), Schema.isUnique()),
+      Schema.isUnique(),
+    ),
     requiredCapabilities: Schema.Array(Capability).check(Schema.isUnique()),
     optionalCapabilities: Schema.Array(Capability).check(Schema.isUnique()),
   })
@@ -174,6 +175,7 @@ export namespace Frontend {
     "ui.matches",
     "ui.screenshot",
     "ui.state",
+    "ui.snapshot",
     "ui.capture",
     "ui.recording.finish",
   ] as const satisfies ReadonlyArray<Handshake.Capability>
@@ -221,6 +223,46 @@ export namespace Frontend {
   })
   export interface State extends Schema.Schema.Type<typeof State> {}
 
+  export const SemanticNode = Schema.Struct({
+    id: Schema.NonEmptyString,
+    instance: Schema.optionalKey(Schema.NonEmptyString),
+    parent: Schema.optionalKey(Schema.NonEmptyString),
+    role: Schema.NonEmptyString,
+    label: Schema.optionalKey(Schema.NonEmptyString),
+    element: Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)),
+    focused: Schema.optionalKey(Schema.Boolean),
+    selected: Schema.optionalKey(Schema.Boolean),
+    expanded: Schema.optionalKey(Schema.Boolean),
+    disabled: Schema.optionalKey(Schema.Boolean),
+  })
+  export interface SemanticNode extends Schema.Schema.Type<typeof SemanticNode> {}
+
+  export const SemanticSnapshot = Schema.Struct({
+    format: Schema.Literal("opencode-ui-snapshot-v1"),
+    nodes: Schema.Array(SemanticNode).check(
+      Schema.makeFilter((nodes) => {
+        const ids = new Set(nodes.map((node) => node.id))
+        if (ids.size !== nodes.length) return "semantic node ids must be unique"
+        if (new Set(nodes.map((node) => node.element)).size !== nodes.length)
+          return "semantic node elements must be unique"
+        if (nodes.some((node) => node.parent !== undefined && !ids.has(node.parent)))
+          return "semantic node parents must reference another node"
+        const parents = new Map(nodes.map((node) => [node.id, node.parent]))
+        for (const node of nodes) {
+          const visited = new Set<string>()
+          let current: string | undefined = node.id
+          while (current !== undefined) {
+            if (visited.has(current)) return "semantic node hierarchy must be acyclic"
+            visited.add(current)
+            current = parents.get(current)
+          }
+        }
+        return undefined
+      }),
+    ),
+  })
+  export interface SemanticSnapshot extends Schema.Schema.Type<typeof SemanticSnapshot> {}
+
   export const Screenshot = Schema.String
   export type Screenshot = Schema.Schema.Type<typeof Screenshot>
 
@@ -293,7 +335,7 @@ export namespace Frontend {
     }),
     Schema.Struct({
       ...JsonRpc.RequestFields,
-      method: Schema.Literals(["ui.enter", "ui.state", "ui.recording.finish"]),
+      method: Schema.Literals(["ui.enter", "ui.state", "ui.snapshot", "ui.recording.finish"]),
     }),
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.capture") }),
   ])

packages/simulation/test/actions.test.ts

@@ -1,6 +1,9 @@
 import { expect, test } from "bun:test"
+import { BoxRenderable, TextRenderable } from "@opentui/core"
 import { Effect } from "effect"
-import { execute, type Harness, matches } from "../src/frontend/actions"
+import { createHarness, execute, type Harness, matches, snapshot, state } from "../src/frontend/actions"
+import { SimulationRenderer } from "../src/frontend/renderer"
+import { SimulationSemantics } from "../src/frontend/semantics"
 
 test("matches literal screen text", () => {
   const harness = { screen: () => "OpenCode [ready].*" }
@@ -39,3 +42,144 @@ test("normalizes named keys for OpenTUI", async () => {
     ["x", undefined],
   ])
 })
+
+test("clicks a target at relative coordinates through descendant text", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        let clicks = 0
+        const button = new BoxRenderable(renderer, {
+          id: "permission.action.once",
+          width: 12,
+          height: 1,
+          onMouseUp: () => clicks++,
+        })
+        button.add(new TextRenderable(renderer, { content: "Allow once" }))
+        renderer.root.add(button)
+        const harness = createHarness(renderer)
+        yield* Effect.promise(() => harness.renderOnce())
+
+        expect(state(harness).elements).toContainEqual(expect.objectContaining({ id: button.id, clickable: true }))
+        yield* execute(harness, { type: "ui.click", target: button.num, x: 1, y: 0 })
+        expect(clicks).toBe(1)
+
+        renderer.root.remove(button)
+        const error = yield* execute(harness, { type: "ui.click", target: button.num, x: 1, y: 0 }).pipe(Effect.flip)
+        expect(error.message).toContain("click target is stale or unavailable")
+      }),
+    ),
+  )
+})
+
+test("snapshots lazy semantic hierarchy and interaction state", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        let selected = "once"
+        const dialog = new BoxRenderable(renderer, { id: "session.permission" })
+        const actions = new BoxRenderable(renderer, { id: "session.permission.actions" })
+        const once = new BoxRenderable(renderer, { id: "session.permission.action.once" })
+        SimulationSemantics.bind(() => ({
+          instance: "permission-1",
+          role: "dialog",
+          label: "Permission required",
+          expanded: false,
+        }))(dialog)
+        SimulationSemantics.bind(() => ({
+          instance: "permission-1",
+          role: "listbox",
+          label: "Permission choices",
+        }))(actions)
+        SimulationSemantics.bind(() => ({
+          instance: "permission-1",
+          role: "option",
+          label: "Allow once",
+          focused: selected === "once",
+          selected: selected === "once",
+          disabled: false,
+        }))(once)
+        renderer.root.add(dialog)
+        dialog.add(actions)
+        actions.add(once)
+
+        expect(snapshot(createHarness(renderer))).toEqual({
+          format: "opencode-ui-snapshot-v1",
+          nodes: [
+            {
+              id: "session.permission",
+              instance: "permission-1",
+              role: "dialog",
+              label: "Permission required",
+              element: dialog.num,
+              expanded: false,
+            },
+            {
+              id: "session.permission.actions",
+              instance: "permission-1",
+              parent: "session.permission",
+              role: "listbox",
+              label: "Permission choices",
+              element: actions.num,
+            },
+            {
+              id: "session.permission.action.once",
+              instance: "permission-1",
+              parent: "session.permission.actions",
+              role: "option",
+              label: "Allow once",
+              element: once.num,
+              focused: true,
+              selected: true,
+              disabled: false,
+            },
+          ],
+        })
+
+        selected = "reject"
+        expect(snapshot(createHarness(renderer)).nodes.at(-1)).toMatchObject({
+          id: "session.permission.action.once",
+          focused: false,
+          selected: false,
+        })
+        dialog.visible = false
+        expect(snapshot(createHarness(renderer)).nodes).toEqual([])
+      }),
+    ),
+  )
+})
+
+test("rejects duplicate semantic identities", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        const first = new BoxRenderable(renderer, { id: "duplicate" })
+        const second = new BoxRenderable(renderer, { id: "duplicate" })
+        const definition = () => ({ role: "option" })
+        SimulationSemantics.bind(definition)(first)
+        SimulationSemantics.bind(definition)(second)
+        renderer.root.add(first)
+        renderer.root.add(second)
+
+        expect(() => snapshot(createHarness(renderer))).toThrow("duplicate semantic UI id: duplicate")
+      }),
+    ),
+  )
+})
+
+test("validates lazy semantic definitions before returning them", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        const invalid = new BoxRenderable(renderer, { id: "invalid" })
+        SimulationSemantics.bind(() => ({ role: "" }))(invalid)
+        renderer.root.add(invalid)
+
+        expect(() => snapshot(createHarness(renderer))).toThrow()
+      }),
+    ),
+  )
+})

packages/simulation/test/frontend-server.test.ts

@@ -39,7 +39,7 @@ test("scopes the frontend control server and reports malformed JSON", async () =
             protocolVersion: 1,
             role: "ui",
             server: { name: "opencode", version: expect.any(String) },
-            capabilities: expect.arrayContaining(["ui.state", "ui.capture"]),
+            capabilities: expect.arrayContaining(["ui.state", "ui.snapshot", "ui.capture"]),
           },
         })
 
@@ -60,6 +60,13 @@ test("scopes the frontend control server and reports malformed JSON", async () =
           },
         })
 
+        socket.send(JSON.stringify({ jsonrpc: "2.0", id: 3, method: "ui.snapshot" }))
+        expect(yield* Queue.take(messages)).toEqual({
+          jsonrpc: "2.0",
+          id: 3,
+          result: { format: "opencode-ui-snapshot-v1", nodes: [] },
+        })
+
         socket.send("{")
         expect(yield* Queue.take(messages)).toMatchObject({
           id: null,

packages/simulation/test/protocol.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test"
-import { Effect } from "effect"
+import { Effect, Schema } from "effect"
 import { Backend, Frontend, Handshake } from "../src/protocol"
 
 test("decodes ui.matches text params", () => {
@@ -21,6 +21,53 @@ test("decodes ui.matches text params", () => {
   ).toThrow()
 })
 
+test("decodes semantic UI snapshots", () => {
+  expect(
+    Frontend.decodeRequest({
+      jsonrpc: "2.0",
+      id: 1,
+      method: "ui.snapshot",
+    }),
+  ).toMatchObject({ method: "ui.snapshot" })
+  const decode = Schema.decodeUnknownSync(Frontend.SemanticSnapshot)
+  expect(
+    decode({
+      format: "opencode-ui-snapshot-v1",
+      nodes: [
+        {
+          id: "session.permission",
+          role: "dialog",
+          label: "Permission required",
+          element: 1,
+          expanded: false,
+        },
+      ],
+    }),
+  ).toMatchObject({ nodes: [{ role: "dialog", expanded: false }] })
+  expect(() =>
+    decode({
+      format: "opencode-ui-snapshot-v1",
+      nodes: [{ id: "", role: "dialog", element: 0 }],
+    }),
+  ).toThrow()
+  for (const nodes of [
+    [
+      { id: "duplicate", role: "dialog", element: 1 },
+      { id: "duplicate", role: "option", element: 2 },
+    ],
+    [
+      { id: "first", role: "dialog", element: 1 },
+      { id: "second", role: "option", element: 1 },
+    ],
+    [{ id: "orphan", parent: "missing", role: "option", element: 1 }],
+    [
+      { id: "first", parent: "second", role: "dialog", element: 1 },
+      { id: "second", parent: "first", role: "option", element: 2 },
+    ],
+  ])
+    expect(() => decode({ format: "opencode-ui-snapshot-v1", nodes })).toThrow()
+})
+
 const params: Handshake.Params = {
   client: { name: "opencode-drive", version: "test" },
   expectedRole: "ui",

packages/tui/src/routes/session/index.tsx

@@ -957,7 +957,14 @@ export function Session() {
               <Switch>
                 <Match when={composer.open || (!!session()?.parentID && forms().length === 0)}>{null}</Match>
                 <Match when={permissions().length > 0}>
-                  <PermissionPrompt request={permissions()[0]} directory={session()?.location.directory} />
+                  <Show when={permissions()[0]?.id} keyed>
+                    {(_) => {
+                      const request = permissions()[0]
+                      return request ? (
+                        <PermissionPrompt request={request} directory={session()?.location.directory} />
+                      ) : null
+                    }}
+                  </Show>
                 </Match>
                 <Match when={forms().length > 0}>
                   <Show when={forms()[0]?.id} keyed>
@@ -1460,7 +1467,8 @@ function CompactionMessage(props: { message: Extract<SessionMessageInfo, { type:
   const { themeV2, syntax } = useTheme()
   const status = () => props.message.status
   const cancelled = () => props.message.status === "failed" && props.message.error.type === "aborted"
-  const text = () => (props.message.status === "failed" ? (cancelled() ? "" : props.message.error.message) : props.message.summary)
+  const text = () =>
+    props.message.status === "failed" ? (cancelled() ? "" : props.message.error.message) : props.message.summary
   const content = createMemo(() => text().trim())
   const color = () => (status() === "failed" && !cancelled() ? themeV2.text.feedback.error() : themeV2.text.subdued())
   return (
@@ -1807,7 +1815,9 @@ function AssistantMessage(props: { message: SessionMessageAssistant; last: boole
         <Match when={props.last || final() || props.message.error}>
           <box paddingLeft={3}>
             <text>
-              <span style={{ fg: props.message.error ? themeV2.text.subdued() : local.agent.color(props.message.agent) }}>
+              <span
+                style={{ fg: props.message.error ? themeV2.text.subdued() : local.agent.color(props.message.agent) }}
+              >
                 {Locale.titlecase(props.message.agent)}
               </span>
               <span style={{ fg: themeV2.text.subdued() }}> · {model()}</span>
@@ -2360,9 +2370,7 @@ function BlockTool(props: {
       paddingBottom={1}
       paddingLeft={2}
       gap={1}
-      backgroundColor={
+      backgroundColor={hover() ? themeV2.raise(themeV2.background()) : themeV2.background()}
-        hover() ? themeV2.raise(themeV2.background()) : themeV2.background()
-      }
       customBorderChars={SplitBorder.customBorderChars}
       borderColor={themeV2.background()}
       onMouseOver={() => props.onClick && setHover(true)}
@@ -2379,9 +2387,13 @@ function BlockTool(props: {
             {(title) => (
               <Show
                 when={props.spinner}
-                fallback={<text fg={permission() ? themeV2.text.feedback.warning() : themeV2.text.subdued()}>{title()}</text>}
+                fallback={
+                  <text fg={permission() ? themeV2.text.feedback.warning() : themeV2.text.subdued()}>{title()}</text>
+                }
               >
-                <Spinner color={permission() ? themeV2.text.feedback.warning() : themeV2.text.subdued()}>{title().replace(/^# /, "")}</Spinner>
+                <Spinner color={permission() ? themeV2.text.feedback.warning() : themeV2.text.subdued()}>
+                  {title().replace(/^# /, "")}
+                </Spinner>
               </Show>
             )}
           </Show>

packages/tui/src/routes/session/permission.tsx

@@ -15,6 +15,7 @@ import { getScrollAcceleration } from "../../util/scroll"
 import { useConfig } from "../../config"
 import { Keymap } from "../../context/keymap"
 import { usePathFormatter } from "../../context/path-format"
+import { SimulationSemantics } from "../../simulation/semantics"
 
 type PermissionStage = "permission" | "always" | "reject"
 
@@ -160,6 +161,8 @@ export function PermissionPrompt(props: { request: PermissionV2Request; director
       <Match when={store.stage === "always"}>
         <Prompt
           title="Always allow"
+          semanticLabel={`Always allow ${props.request.action}`}
+          instance={props.request.id}
           body={
             <Switch>
               <Match when={props.request.save?.length === 1 && props.request.save[0] === "*"}>
@@ -167,7 +170,9 @@ export function PermissionPrompt(props: { request: PermissionV2Request; director
               </Match>
               <Match when={true}>
                 <box paddingLeft={1} gap={1}>
-                  <text fg={themeV2.text.subdued()}>This will allow the following patterns until OpenCode is restarted</text>
+                  <text fg={themeV2.text.subdued()}>
+                    This will allow the following patterns until OpenCode is restarted
+                  </text>
                   <box>
                     <For each={props.request.save ?? []}>
                       {(pattern) => (
@@ -197,6 +202,8 @@ export function PermissionPrompt(props: { request: PermissionV2Request; director
       </Match>
       <Match when={store.stage === "reject"}>
         <RejectPrompt
+          action={props.request.action}
+          instance={props.request.id}
           onConfirm={(message) => {
             void client.api.permission.reply({
               sessionID: props.request.sessionID,
@@ -425,6 +432,8 @@ export function PermissionPrompt(props: { request: PermissionV2Request; director
           const body = (
             <Prompt
               title="Permission required"
+              semanticLabel={permissionSemanticLabel(props.request.action, current.title)}
+              instance={props.request.id}
               header={header()}
               body={current.body}
               options={
@@ -467,7 +476,16 @@ export function PermissionPrompt(props: { request: PermissionV2Request; director
   )
 }
 
-function RejectPrompt(props: { onConfirm: (message: string) => void; onCancel: () => void }) {
+export function permissionSemanticLabel(action: string, title?: string) {
+  return `Permission required: ${title ?? action}`
+}
+
+function RejectPrompt(props: {
+  action: string
+  instance: string
+  onConfirm: (message: string) => void
+  onCancel: () => void
+}) {
   let input: TextareaRenderable
   const { themeV2 } = useTheme().contextual("elevated")
   const dimensions = useTerminalDimensions()
@@ -495,6 +513,12 @@ function RejectPrompt(props: { onConfirm: (message: string) => void; onCancel: (
 
   return (
     <box
+      id="session.permission.reject"
+      ref={SimulationSemantics.bind(() => ({
+        instance: props.instance,
+        role: "dialog",
+        label: `Reject permission: ${props.action}`,
+      }))}
       backgroundColor={themeV2.background()}
       border={["left"]}
       borderColor={themeV2.text.feedback.error()}
@@ -522,8 +546,16 @@ function RejectPrompt(props: { onConfirm: (message: string) => void; onCancel: (
         gap={1}
       >
         <textarea
+          id="session.permission.reject.message"
           ref={(val: TextareaRenderable) => {
             input = val
+            SimulationSemantics.bind(() => ({
+              instance: props.instance,
+              role: "textbox",
+              label: "Rejection reason",
+              focused: val.focused,
+              disabled: false,
+            }))(val)
             val.traits = { status: "REJECT" }
           }}
           focused
@@ -531,13 +563,45 @@ function RejectPrompt(props: { onConfirm: (message: string) => void; onCancel: (
           focusedTextColor={themeV2.text()}
           cursorColor={themeV2.text()}
         />
-        <box flexDirection="row" gap={2} flexShrink={0}>
+        <box
-          <text fg={themeV2.text()}>
+          id="session.permission.reject.actions"
-            enter <span style={{ fg: themeV2.text.subdued() }}>confirm</span>
+          ref={SimulationSemantics.bind(() => ({
-          </text>
+            instance: props.instance,
-          <text fg={themeV2.text()}>
+            role: "group",
-            esc <span style={{ fg: themeV2.text.subdued() }}>cancel</span>
+            label: "Rejection actions",
-          </text>
+          }))}
+          flexDirection="row"
+          gap={2}
+          flexShrink={0}
+        >
+          <box
+            id="session.permission.reject.confirm"
+            ref={SimulationSemantics.bind(() => ({
+              instance: props.instance,
+              role: "button",
+              label: "Confirm rejection",
+              disabled: false,
+            }))}
+            onMouseUp={() => props.onConfirm(input.plainText)}
+          >
+            <text fg={themeV2.text()}>
+              enter <span style={{ fg: themeV2.text.subdued() }}>confirm</span>
+            </text>
+          </box>
+          <box
+            id="session.permission.reject.cancel"
+            ref={SimulationSemantics.bind(() => ({
+              instance: props.instance,
+              role: "button",
+              label: "Cancel rejection",
+              disabled: false,
+            }))}
+            onMouseUp={props.onCancel}
+          >
+            <text fg={themeV2.text()}>
+              esc <span style={{ fg: themeV2.text.subdued() }}>cancel</span>
+            </text>
+          </box>
         </box>
       </box>
     </box>
@@ -546,6 +610,8 @@ function RejectPrompt(props: { onConfirm: (message: string) => void; onCancel: (
 
 function Prompt<const T extends Record<string, string>>(props: {
   title: string
+  semanticLabel?: string
+  instance: string
   header?: JSX.Element
   body: JSX.Element
   options: T
@@ -643,10 +709,7 @@ function Prompt<const T extends Record<string, string>>(props: {
           ]
         : []),
     ],
-    bindings: [
+    bindings: [...(props.escapeKey ? ["app.exit"] : []), ...(props.fullscreen ? ["permission.prompt.fullscreen"] : [])],
-      ...(props.escapeKey ? ["app.exit"] : []),
-      ...(props.fullscreen ? ["permission.prompt.fullscreen"] : []),
-    ],
   }))
 
   const hint = createMemo(() => (store.expanded ? "minimize" : "fullscreen"))
@@ -654,6 +717,13 @@ function Prompt<const T extends Record<string, string>>(props: {
 
   const content = () => (
     <box
+      id="session.permission"
+      ref={SimulationSemantics.bind(() => ({
+        instance: props.instance,
+        role: "dialog",
+        label: props.semanticLabel ?? props.title,
+        expanded: store.expanded,
+      }))}
       backgroundColor={themeV2.background()}
       border={["left"]}
       borderColor={themeV2.background.action("focused")}
@@ -697,26 +767,39 @@ function Prompt<const T extends Record<string, string>>(props: {
         justifyContent={narrow() ? "flex-start" : "space-between"}
         alignItems={narrow() ? "flex-start" : "center"}
       >
-        <box flexDirection="row" gap={1} flexShrink={0}>
+        <box
+          id="session.permission.actions"
+          ref={SimulationSemantics.bind(() => ({
+            instance: props.instance,
+            role: "listbox",
+            label: "Permission choices",
+          }))}
+          flexDirection="row"
+          gap={1}
+          flexShrink={0}
+        >
           <For each={keys}>
             {(option) => (
               <box
+                id={`session.permission.action.${String(option)}`}
+                ref={SimulationSemantics.bind(() => ({
+                  instance: props.instance,
+                  role: "option",
+                  label: props.options[option],
+                  focused: option === store.selected,
+                  selected: option === store.selected,
+                  disabled: false,
+                }))}
                 paddingLeft={1}
                 paddingRight={1}
-                backgroundColor={themeV2.background.action(
+                backgroundColor={themeV2.background.action(option === store.selected ? "focused" : "default")}
-                  option === store.selected ? "focused" : "default",
-                )}
                 onMouseOver={() => setStore("selected", option)}
                 onMouseUp={() => {
                   setStore("selected", option)
                   props.onSelect(option)
                 }}
               >
-                <text
+                <text fg={themeV2.text.action(option === store.selected ? "focused" : "default")}>
-                  fg={themeV2.text.action(
-                    option === store.selected ? "focused" : "default",
-                  )}
-                >
                   {props.options[option]}
                 </text>
               </box>
@@ -726,7 +809,8 @@ function Prompt<const T extends Record<string, string>>(props: {
         <box flexDirection="row" gap={2} flexShrink={0}>
           <Show when={props.fullscreen}>
             <text fg={themeV2.text()}>
-              {shortcuts.get("permission.prompt.fullscreen")} <span style={{ fg: themeV2.text.subdued() }}>{hint()}</span>
+              {shortcuts.get("permission.prompt.fullscreen")}{" "}
+              <span style={{ fg: themeV2.text.subdued() }}>{hint()}</span>
             </text>
           </Show>
           <text fg={themeV2.text()}>

packages/tui/src/simulation/semantics.ts

@@ -0,0 +1,12 @@
+import type { Renderable } from "@opentui/core"
+import type { SimulationProtocol } from "@opencode-ai/simulation/protocol"
+
+type Definition = Omit<SimulationProtocol.Frontend.SemanticNode, "id" | "element" | "parent">
+
+const key = Symbol.for("opencode.simulation.semantics")
+
+const bind = (definition: () => Definition) => (renderable: Renderable) => {
+  Object.defineProperty(renderable, key, { value: definition, configurable: true })
+}
+
+export const SimulationSemantics = { bind }

packages/tui/test/cli/tui/permission.test.ts

@@ -0,0 +1,7 @@
+import { expect, test } from "bun:test"
+import { permissionSemanticLabel } from "../../../src/routes/session/permission"
+
+test("uses the permission action when a surface has no display title", () => {
+  expect(permissionSemanticLabel("shell")).toBe("Permission required: shell")
+  expect(permissionSemanticLabel("edit", "Edit fixture.txt")).toBe("Permission required: Edit fixture.txt")
+})

packages/tui/test/simulation/semantics.test.ts

@@ -0,0 +1,23 @@
+import { expect, test } from "bun:test"
+import { BoxRenderable } from "@opentui/core"
+import { Effect } from "effect"
+import { SimulationSemantics as Reader } from "@opencode-ai/simulation/frontend/semantics"
+import { SimulationRenderer } from "@opencode-ai/simulation/frontend/renderer"
+import { SimulationSemantics } from "../../src/simulation/semantics"
+
+test("shares lazy semantic annotations with the simulation renderer", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        const renderable = new BoxRenderable(renderer, { id: "permission" })
+        let selected = false
+        SimulationSemantics.bind(() => ({ role: "option", selected }))(renderable)
+
+        expect(Reader.read(renderable)?.()).toEqual({ role: "option", selected: false })
+        selected = true
+        expect(Reader.read(renderable)?.()).toEqual({ role: "option", selected: true })
+      }),
+    ),
+  )
+})