EnterpriseAI/ClosedCode
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

feat(server): durable log reads, changes feed, and watermarked snapshots (#34962)

Browse Source
This commit is contained in:
Kit Langton
2026-07-02 16:42:30 -04:00
committed by GitHub
parent 33705e632a
commit bc2e270f82
28 changed files with 1352 additions and 1555 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/test/event.test.ts
+141 -14
View File
@@ -78,6 +78,12 @@ const durableData = (sessionID: Session.ID, text: string) => ({
messageID: SessionV1.MessageID.ascending(`msg_${text}`),
})
/** Followed log read without markers: the old `durable` stream shape. */
const tail = (events: EventV2.Interface, input: { aggregateID: string; after?: number }) =>
events
.log({ ...input, follow: true })
.pipe(Stream.filter((item): item is EventV2.Payload => !EventV2.isCaughtUp(item)))
const it = testEffect(
AppNodeBuilder.build(LayerNode.group([Database.node, EventV2.node, Location.node]), [[Location.node, locationLayer]]),
)
@@ -119,7 +125,7 @@ describe("EventV2", () => {
const event = yield* events.publish(VersionedMessage, { id: "one", text: "hello" })
expect(event.type).toBe("test.versioned")
expect(event.durable?.version).toBe(2)
expect(event.durable?.version).toBe(EventV2.Version.make(2))
}),
)
@@ -145,7 +151,7 @@ describe("EventV2", () => {
Effect.gen(function* () {
const events = yield* EventV2.Service
const typed = yield* events.subscribe(Message).pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped)
const wildcard = yield* events.all().pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped)
const wildcard = yield* events.live().pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped)
yield* Effect.yieldNow
const event = yield* events.publish(Message, { text: "hello" })
@@ -226,7 +232,7 @@ describe("EventV2", () => {
Effect.gen(function* () {
const events = yield* EventV2.Service
const received = new Array<string>()
const fiber = yield* events.all().pipe(
const fiber = yield* events.live().pipe(
Stream.take(1),
Stream.runForEach(() => Effect.sync(() => received.push("stream"))),
Effect.forkScoped,
@@ -325,8 +331,8 @@ describe("EventV2", () => {
const events = yield* EventV2.Service
const consuming = yield* Deferred.make<void>()
const release = yield* Deferred.make<void>()
const slowStream = yield* EventV2.allBounded(events, 1)
const fastStream = yield* EventV2.allBounded(events, 8)
const slowStream = yield* EventV2.liveBounded(events, 1)
const fastStream = yield* EventV2.liveBounded(events, 8)
const slow = yield* slowStream.pipe(
Stream.runForEach(() => Deferred.succeed(consuming, undefined).pipe(Effect.andThen(Deferred.await(release)))),
Effect.forkScoped,
@@ -425,9 +431,11 @@ describe("EventV2", () => {
const aggregateID = Session.ID.create()
yield* events.publish(DurableMessage, durableData(aggregateID, "zero"))
yield* events.publish(DurableMessage, durableData(aggregateID, "one"))
const fiber = yield* events
.durable({ aggregateID, after: 0 })
.pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped)
const fiber = yield* tail(events, { aggregateID, after: 0 }).pipe(
Stream.take(2),
Stream.runCollect,
Effect.forkScoped,
)
yield* Effect.yieldNow
yield* events.publish(DurableMessage, durableData(aggregateID, "two"))
@@ -444,7 +452,7 @@ describe("EventV2", () => {
const events = yield* EventV2.Service
const aggregateID = Session.ID.create()
yield* events.publish(DurableMessage, durableData(aggregateID, "zero"))
const fiber = yield* events.durable({ aggregateID }).pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped)
const fiber = yield* tail(events, { aggregateID }).pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped)
yield* events.publish(DurableMessage, durableData(aggregateID, "one"))
@@ -470,7 +478,7 @@ describe("EventV2", () => {
yield* Effect.gen(function* () {
const events = yield* EventV2.Service
const aggregateID = Session.ID.create()
const fiber = yield* events.durable({ aggregateID }).pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped)
const fiber = yield* tail(events, { aggregateID }).pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped)
yield* Deferred.await(readStarted)
pause = false
@@ -489,9 +497,7 @@ describe("EventV2", () => {
const events = yield* EventV2.Service
const aggregateID = Session.ID.create()
const count = 64
const fiber = yield* events
.durable({ aggregateID })
.pipe(Stream.take(count), Stream.runCollect, Effect.forkScoped)
const fiber = yield* tail(events, { aggregateID }).pipe(Stream.take(count), Stream.runCollect, Effect.forkScoped)
yield* Effect.yieldNow
for (let index = 0; index < count; index++) {
@@ -508,7 +514,7 @@ describe("EventV2", () => {
Effect.gen(function* () {
const events = yield* EventV2.Service
const aggregateID = Session.ID.create()
const fiber = yield* events.durable({ aggregateID }).pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped)
const fiber = yield* tail(events, { aggregateID }).pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped)
yield* Effect.yieldNow
yield* events.publish(Message, { text: "live only" })
@@ -1121,4 +1127,125 @@ describe("EventV2", () => {
expect(received[0]?.data).toEqual(durableData(aggregateID, "replayed"))
}),
)
it.effect("log without follow replays events and completes with a caught-up marker", () =>
Effect.gen(function* () {
const events = yield* EventV2.Service
const aggregateID = Session.ID.create()
yield* events.publish(DurableMessage, durableData(aggregateID, "zero"))
yield* events.publish(DurableMessage, durableData(aggregateID, "one"))
const items = Array.from(yield* Stream.runCollect(events.log({ aggregateID })))
expect(items.map((item) => (EventV2.isCaughtUp(item) ? item.type : item.durable?.seq))).toEqual([
EventV2.Seq.make(0),
EventV2.Seq.make(1),
"log.caught_up",
])
expect(items.at(-1)).toEqual({ type: "log.caught_up", aggregateID, seq: EventV2.Seq.make(1) })
}),
)
it.effect("log caught-up marker omits seq for an empty log and keeps the cursor otherwise", () =>
Effect.gen(function* () {
const events = yield* EventV2.Service
const aggregateID = Session.ID.create()
const empty = Array.from(yield* Stream.runCollect(events.log({ aggregateID })))
yield* events.publish(DurableMessage, durableData(aggregateID, "zero"))
const drained = Array.from(yield* Stream.runCollect(events.log({ aggregateID, after: 0 })))
expect(empty).toEqual([{ type: "log.caught_up", aggregateID }])
expect(empty[0]).not.toHaveProperty("seq")
expect(drained).toEqual([{ type: "log.caught_up", aggregateID, seq: EventV2.Seq.make(0) }])
}),
)
it.effect("log with follow emits the caught-up marker at the replay-to-live boundary", () =>
Effect.gen(function* () {
const events = yield* EventV2.Service
const aggregateID = Session.ID.create()
yield* events.publish(DurableMessage, durableData(aggregateID, "zero"))
const fiber = yield* events
.log({ aggregateID, follow: true })
.pipe(Stream.take(3), Stream.runCollect, Effect.forkScoped)
yield* Effect.yieldNow
yield* events.publish(DurableMessage, durableData(aggregateID, "one"))
const items = Array.from(yield* Fiber.join(fiber))
expect(items.map((item) => (EventV2.isCaughtUp(item) ? item : item.durable?.seq))).toEqual([
EventV2.Seq.make(0),
{ type: "log.caught_up", aggregateID, seq: EventV2.Seq.make(0) },
EventV2.Seq.make(1),
])
}),
)
it.effect("changes emits sweep-required on subscribe then coalesced hints per aggregate", () =>
Effect.gen(function* () {
const events = yield* EventV2.Service
const first = Session.ID.create()
const second = Session.ID.create()
const pull = yield* Stream.toPull(events.changes())
expect(Array.from(yield* pull)).toEqual([{ type: "log.sweep_required" }])
yield* events.publish(DurableMessage, durableData(first, "zero"))
yield* events.publish(DurableMessage, durableData(first, "one"))
yield* events.publish(DurableMessage, durableData(first, "two"))
yield* events.publish(DurableMessage, durableData(second, "zero"))
expect(Array.from(yield* pull)).toEqual([
{ type: "log.hint", aggregateID: first, seq: EventV2.Seq.make(2) },
{ type: "log.hint", aggregateID: second, seq: EventV2.Seq.make(0) },
])
}),
)
it.effect("changes abandons the hint buffer for a sweep when key retention is exceeded", () =>
Effect.gen(function* () {
const eventLayer = EventV2.layerWith({ changesKeyCapacity: 2 }).pipe(
Layer.provide(LayerNode.compile(Database.node)),
)
yield* Effect.gen(function* () {
const events = yield* EventV2.Service
const pull = yield* Stream.toPull(events.changes())
expect(Array.from(yield* pull)).toEqual([{ type: "log.sweep_required" }])
yield* events.publish(DurableMessage, durableData(Session.ID.create(), "a"))
yield* events.publish(DurableMessage, durableData(Session.ID.create(), "b"))
yield* events.publish(DurableMessage, durableData(Session.ID.create(), "c"))
expect(Array.from(yield* pull)).toEqual([{ type: "log.sweep_required" }])
const late = Session.ID.create()
yield* events.publish(DurableMessage, durableData(late, "d"))
expect(Array.from(yield* pull)).toEqual([{ type: "log.hint", aggregateID: late, seq: EventV2.Seq.make(0) }])
}).pipe(Effect.provide(Layer.merge(LayerNode.compile(Database.node), eventLayer)))
}),
)
it.effect("sequences returns the latest committed seq per aggregate and omits unknown aggregates", () =>
Effect.gen(function* () {
const events = yield* EventV2.Service
const first = Session.ID.create()
const second = Session.ID.create()
yield* events.publish(DurableMessage, durableData(first, "zero"))
yield* events.publish(DurableMessage, durableData(first, "one"))
yield* events.publish(DurableMessage, durableData(second, "zero"))
const sequences = yield* events.sequences([first, second, Session.ID.create()])
expect(sequences).toEqual(
new Map([
[first, EventV2.Seq.make(1)],
[second, EventV2.Seq.make(0)],
]),
)
expect(yield* events.sequences([])).toEqual(new Map())
}),
)
})