chore: generate
This commit is contained in:
@@ -20,11 +20,10 @@ const INCREMENTAL_LOOKBACK_MS = 2 * 3_600_000
|
|||||||
export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string }
|
export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string }
|
||||||
export type SyncStatsError = AthenaQueryError | AthenaQueryTimeoutError | DatabaseError
|
export type SyncStatsError = AthenaQueryError | AthenaQueryTimeoutError | DatabaseError
|
||||||
|
|
||||||
export const syncStats: (options?: { full?: boolean }) => Effect.Effect<
|
export const syncStats: (options?: {
|
||||||
SyncStatsResult,
|
full?: boolean
|
||||||
SyncStatsError,
|
}) => Effect.Effect<SyncStatsResult, SyncStatsError, Athena | ModelStatRepo | ProviderStatRepo | GeoStatRepo> =
|
||||||
Athena | ModelStatRepo | ProviderStatRepo | GeoStatRepo
|
Effect.fn("StatSync.sync")(function* (options?: { full?: boolean }) {
|
||||||
> = Effect.fn("StatSync.sync")(function* (options?: { full?: boolean }) {
|
|
||||||
const startedAt = yield* DateTime.nowAsDate
|
const startedAt = yield* DateTime.nowAsDate
|
||||||
const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000)
|
const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000)
|
||||||
const periodStart = options?.full ? fullPeriodStart(periodEnd) : incrementalPeriodStart(periodEnd)
|
const periodStart = options?.full ? fullPeriodStart(periodEnd) : incrementalPeriodStart(periodEnd)
|
||||||
@@ -36,9 +35,7 @@ export const syncStats: (options?: { full?: boolean }) => Effect.Effect<
|
|||||||
yield* logRuntimeCheck()
|
yield* logRuntimeCheck()
|
||||||
|
|
||||||
const rows = yield* athena.query(buildStatsQuery(periodStart, periodEnd))
|
const rows = yield* athena.query(buildStatsQuery(periodStart, periodEnd))
|
||||||
const modelRows = modelRowsFromAggregates(
|
const modelRows = modelRowsFromAggregates(rows.filter((row) => row.dimension === "model").flatMap(toModelAggregate))
|
||||||
rows.filter((row) => row.dimension === "model").flatMap(toModelAggregate),
|
|
||||||
)
|
|
||||||
const providerRows = providerRowsFromAggregates(
|
const providerRows = providerRowsFromAggregates(
|
||||||
rows.filter((row) => row.dimension === "provider").flatMap(toProviderAggregate),
|
rows.filter((row) => row.dimension === "provider").flatMap(toProviderAggregate),
|
||||||
)
|
)
|
||||||
@@ -78,7 +75,7 @@ export const syncStats: (options?: { full?: boolean }) => Effect.Effect<
|
|||||||
periodStart: periodStart.toISOString(),
|
periodStart: periodStart.toISOString(),
|
||||||
periodEnd: periodEnd.toISOString(),
|
periodEnd: periodEnd.toISOString(),
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// May 27 was partial, so keep Athena stats anchored at the first complete day.
|
// May 27 was partial, so keep Athena stats anchored at the first complete day.
|
||||||
function fullPeriodStart(periodEnd: Date) {
|
function fullPeriodStart(periodEnd: Date) {
|
||||||
@@ -96,10 +93,7 @@ function fullPeriodStart(periodEnd: Date) {
|
|||||||
// dimension cleanup).
|
// dimension cleanup).
|
||||||
function incrementalPeriodStart(periodEnd: Date) {
|
function incrementalPeriodStart(periodEnd: Date) {
|
||||||
return new Date(
|
return new Date(
|
||||||
Math.max(
|
Math.max(startOfIsoWeek(new Date(periodEnd.getTime() - INCREMENTAL_LOOKBACK_MS)).getTime(), STATS_DATA_START_MS),
|
||||||
startOfIsoWeek(new Date(periodEnd.getTime() - INCREMENTAL_LOOKBACK_MS)).getTime(),
|
|
||||||
STATS_DATA_START_MS,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user