fix(stats): tolerate pending user column
This commit is contained in:
@@ -8,6 +8,8 @@ import {
|
|||||||
chunks,
|
chunks,
|
||||||
collapseRows,
|
collapseRows,
|
||||||
inserted,
|
inserted,
|
||||||
|
isMissingUniqueUsersColumn,
|
||||||
|
omitUniqueUsers,
|
||||||
rankRowsWithMarketShare,
|
rankRowsWithMarketShare,
|
||||||
statPeriodKey,
|
statPeriodKey,
|
||||||
statRowScope,
|
statRowScope,
|
||||||
@@ -136,16 +138,30 @@ export class GeoStatRepo extends Context.Service<GeoStatRepo, GeoStatRepo.Servic
|
|||||||
chunks(rows, UPSERT_CHUNK_SIZE),
|
chunks(rows, UPSERT_CHUNK_SIZE),
|
||||||
(chunk) =>
|
(chunk) =>
|
||||||
Effect.tryPromise({
|
Effect.tryPromise({
|
||||||
try: () =>
|
try: async () => {
|
||||||
db
|
try {
|
||||||
|
return await upsertGeoChunk(chunk, true)
|
||||||
|
} catch (cause) {
|
||||||
|
if (!isMissingUniqueUsersColumn(cause)) throw cause
|
||||||
|
return upsertGeoChunk(chunk, false)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
catch: (cause) => DatabaseError.make({ cause }),
|
||||||
|
}),
|
||||||
|
{ discard: true },
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
function upsertGeoChunk(chunk: GeoStatRow[], includeUniqueUsers: boolean) {
|
||||||
|
return db
|
||||||
.insert(geoStat)
|
.insert(geoStat)
|
||||||
.values(chunk)
|
.values(includeUniqueUsers ? chunk : omitUniqueUsers(chunk))
|
||||||
.onDuplicateKeyUpdate({
|
.onDuplicateKeyUpdate({
|
||||||
set: {
|
set: {
|
||||||
continent: inserted("continent"),
|
continent: inserted("continent"),
|
||||||
sessions: inserted("sessions"),
|
sessions: inserted("sessions"),
|
||||||
requests: inserted("requests"),
|
requests: inserted("requests"),
|
||||||
unique_users: inserted("unique_users"),
|
...(includeUniqueUsers ? { unique_users: inserted("unique_users") } : {}),
|
||||||
input_tokens: inserted("input_tokens"),
|
input_tokens: inserted("input_tokens"),
|
||||||
output_tokens: inserted("output_tokens"),
|
output_tokens: inserted("output_tokens"),
|
||||||
reasoning_tokens: inserted("reasoning_tokens"),
|
reasoning_tokens: inserted("reasoning_tokens"),
|
||||||
@@ -172,12 +188,8 @@ export class GeoStatRepo extends Context.Service<GeoStatRepo, GeoStatRepo.Servic
|
|||||||
rank_by_sessions: inserted("rank_by_sessions"),
|
rank_by_sessions: inserted("rank_by_sessions"),
|
||||||
rank_by_cost: inserted("rank_by_cost"),
|
rank_by_cost: inserted("rank_by_cost"),
|
||||||
},
|
},
|
||||||
}),
|
|
||||||
catch: (cause) => DatabaseError.make({ cause }),
|
|
||||||
}),
|
|
||||||
{ discard: true },
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
}
|
||||||
|
|
||||||
const deleteRetiredDimensions = Effect.fn("GeoStatRepo.deleteRetiredDimensions")(function* (rows: GeoStatRow[]) {
|
const deleteRetiredDimensions = Effect.fn("GeoStatRepo.deleteRetiredDimensions")(function* (rows: GeoStatRow[]) {
|
||||||
const scope = statRowScope(rows)
|
const scope = statRowScope(rows)
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import {
|
|||||||
chunks,
|
chunks,
|
||||||
collapseRows,
|
collapseRows,
|
||||||
inserted,
|
inserted,
|
||||||
|
isMissingUniqueUsersColumn,
|
||||||
|
omitUniqueUsers,
|
||||||
rankBy,
|
rankBy,
|
||||||
statPeriodKey,
|
statPeriodKey,
|
||||||
statRowScope,
|
statRowScope,
|
||||||
@@ -56,8 +58,9 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
|
|||||||
|
|
||||||
const listDaily = Effect.fn("ModelStatRepo.listDaily")(function* () {
|
const listDaily = Effect.fn("ModelStatRepo.listDaily")(function* () {
|
||||||
return yield* Effect.tryPromise({
|
return yield* Effect.tryPromise({
|
||||||
try: () =>
|
try: async () => {
|
||||||
db
|
try {
|
||||||
|
return await db
|
||||||
.select({
|
.select({
|
||||||
periodKey: modelStat.period_key,
|
periodKey: modelStat.period_key,
|
||||||
updatedAt: modelStat.updated_at,
|
updatedAt: modelStat.updated_at,
|
||||||
@@ -76,15 +79,34 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
|
|||||||
totalCostMicrocents: modelStat.total_cost_microcents,
|
totalCostMicrocents: modelStat.total_cost_microcents,
|
||||||
})
|
})
|
||||||
.from(modelStat)
|
.from(modelStat)
|
||||||
.where(
|
.where(modelDailyScope())
|
||||||
and(
|
.orderBy(asc(modelStat.period_key))
|
||||||
eq(modelStat.grain, "day"),
|
} catch (cause) {
|
||||||
eq(modelStat.client, "all"),
|
if (!isMissingUniqueUsersColumn(cause)) throw cause
|
||||||
eq(modelStat.source, "all"),
|
return (
|
||||||
inArray(modelStat.tier, ["Go", "go"]),
|
await db
|
||||||
),
|
.select({
|
||||||
)
|
periodKey: modelStat.period_key,
|
||||||
.orderBy(asc(modelStat.period_key)),
|
updatedAt: modelStat.updated_at,
|
||||||
|
tier: modelStat.tier,
|
||||||
|
provider: modelStat.provider,
|
||||||
|
model: modelStat.model,
|
||||||
|
sessions: modelStat.sessions,
|
||||||
|
inputTokens: modelStat.input_tokens,
|
||||||
|
outputTokens: modelStat.output_tokens,
|
||||||
|
reasoningTokens: modelStat.reasoning_tokens,
|
||||||
|
cacheReadTokens: modelStat.cache_read_tokens,
|
||||||
|
totalTokens: modelStat.total_tokens,
|
||||||
|
inputCostMicrocents: modelStat.input_cost_microcents,
|
||||||
|
outputCostMicrocents: modelStat.output_cost_microcents,
|
||||||
|
totalCostMicrocents: modelStat.total_cost_microcents,
|
||||||
|
})
|
||||||
|
.from(modelStat)
|
||||||
|
.where(modelDailyScope())
|
||||||
|
.orderBy(asc(modelStat.period_key))
|
||||||
|
).map((row) => ({ ...row, uniqueUsers: 0 }))
|
||||||
|
}
|
||||||
|
},
|
||||||
catch: (cause) => DatabaseError.make({ cause }),
|
catch: (cause) => DatabaseError.make({ cause }),
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
@@ -94,16 +116,30 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
|
|||||||
chunks(rows, UPSERT_CHUNK_SIZE),
|
chunks(rows, UPSERT_CHUNK_SIZE),
|
||||||
(chunk) =>
|
(chunk) =>
|
||||||
Effect.tryPromise({
|
Effect.tryPromise({
|
||||||
try: () =>
|
try: async () => {
|
||||||
db
|
try {
|
||||||
|
return await upsertModelChunk(chunk, true)
|
||||||
|
} catch (cause) {
|
||||||
|
if (!isMissingUniqueUsersColumn(cause)) throw cause
|
||||||
|
return upsertModelChunk(chunk, false)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
catch: (cause) => DatabaseError.make({ cause }),
|
||||||
|
}),
|
||||||
|
{ discard: true },
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
function upsertModelChunk(chunk: ModelStatRow[], includeUniqueUsers: boolean) {
|
||||||
|
return db
|
||||||
.insert(modelStat)
|
.insert(modelStat)
|
||||||
.values(chunk)
|
.values(includeUniqueUsers ? chunk : omitUniqueUsers(chunk))
|
||||||
.onDuplicateKeyUpdate({
|
.onDuplicateKeyUpdate({
|
||||||
set: {
|
set: {
|
||||||
provider_model: inserted("provider_model"),
|
provider_model: inserted("provider_model"),
|
||||||
sessions: inserted("sessions"),
|
sessions: inserted("sessions"),
|
||||||
requests: inserted("requests"),
|
requests: inserted("requests"),
|
||||||
unique_users: inserted("unique_users"),
|
...(includeUniqueUsers ? { unique_users: inserted("unique_users") } : {}),
|
||||||
input_tokens: inserted("input_tokens"),
|
input_tokens: inserted("input_tokens"),
|
||||||
output_tokens: inserted("output_tokens"),
|
output_tokens: inserted("output_tokens"),
|
||||||
reasoning_tokens: inserted("reasoning_tokens"),
|
reasoning_tokens: inserted("reasoning_tokens"),
|
||||||
@@ -126,12 +162,8 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
|
|||||||
rank_by_requests: inserted("rank_by_requests"),
|
rank_by_requests: inserted("rank_by_requests"),
|
||||||
rank_by_cost: inserted("rank_by_cost"),
|
rank_by_cost: inserted("rank_by_cost"),
|
||||||
},
|
},
|
||||||
}),
|
|
||||||
catch: (cause) => DatabaseError.make({ cause }),
|
|
||||||
}),
|
|
||||||
{ discard: true },
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
}
|
||||||
|
|
||||||
const deleteRetiredDimensions = Effect.fn("ModelStatRepo.deleteRetiredDimensions")(function* (
|
const deleteRetiredDimensions = Effect.fn("ModelStatRepo.deleteRetiredDimensions")(function* (
|
||||||
rows: ModelStatRow[],
|
rows: ModelStatRow[],
|
||||||
@@ -165,6 +197,15 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function modelDailyScope() {
|
||||||
|
return and(
|
||||||
|
eq(modelStat.grain, "day"),
|
||||||
|
eq(modelStat.client, "all"),
|
||||||
|
eq(modelStat.source, "all"),
|
||||||
|
inArray(modelStat.tier, ["Go", "go"]),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
export function rowsFromAggregates(aggregates: ModelStatAggregate[]) {
|
export function rowsFromAggregates(aggregates: ModelStatAggregate[]) {
|
||||||
return rankRows([
|
return rankRows([
|
||||||
...synthesizeAllTierRows(
|
...synthesizeAllTierRows(
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import {
|
|||||||
chunks,
|
chunks,
|
||||||
collapseRows,
|
collapseRows,
|
||||||
inserted,
|
inserted,
|
||||||
|
isMissingUniqueUsersColumn,
|
||||||
|
omitUniqueUsers,
|
||||||
rankRowsWithMarketShare,
|
rankRowsWithMarketShare,
|
||||||
statRowScope,
|
statRowScope,
|
||||||
synthesizeAllTierRows,
|
synthesizeAllTierRows,
|
||||||
@@ -107,15 +109,29 @@ export class ProviderStatRepo extends Context.Service<ProviderStatRepo, Provider
|
|||||||
chunks(rows, UPSERT_CHUNK_SIZE),
|
chunks(rows, UPSERT_CHUNK_SIZE),
|
||||||
(chunk) =>
|
(chunk) =>
|
||||||
Effect.tryPromise({
|
Effect.tryPromise({
|
||||||
try: () =>
|
try: async () => {
|
||||||
db
|
try {
|
||||||
|
return await upsertProviderChunk(chunk, true)
|
||||||
|
} catch (cause) {
|
||||||
|
if (!isMissingUniqueUsersColumn(cause)) throw cause
|
||||||
|
return upsertProviderChunk(chunk, false)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
catch: (cause) => DatabaseError.make({ cause }),
|
||||||
|
}),
|
||||||
|
{ discard: true },
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
function upsertProviderChunk(chunk: ProviderStatRow[], includeUniqueUsers: boolean) {
|
||||||
|
return db
|
||||||
.insert(providerStat)
|
.insert(providerStat)
|
||||||
.values(chunk)
|
.values(includeUniqueUsers ? chunk : omitUniqueUsers(chunk))
|
||||||
.onDuplicateKeyUpdate({
|
.onDuplicateKeyUpdate({
|
||||||
set: {
|
set: {
|
||||||
sessions: inserted("sessions"),
|
sessions: inserted("sessions"),
|
||||||
requests: inserted("requests"),
|
requests: inserted("requests"),
|
||||||
unique_users: inserted("unique_users"),
|
...(includeUniqueUsers ? { unique_users: inserted("unique_users") } : {}),
|
||||||
input_tokens: inserted("input_tokens"),
|
input_tokens: inserted("input_tokens"),
|
||||||
output_tokens: inserted("output_tokens"),
|
output_tokens: inserted("output_tokens"),
|
||||||
reasoning_tokens: inserted("reasoning_tokens"),
|
reasoning_tokens: inserted("reasoning_tokens"),
|
||||||
@@ -142,12 +158,8 @@ export class ProviderStatRepo extends Context.Service<ProviderStatRepo, Provider
|
|||||||
rank_by_sessions: inserted("rank_by_sessions"),
|
rank_by_sessions: inserted("rank_by_sessions"),
|
||||||
rank_by_cost: inserted("rank_by_cost"),
|
rank_by_cost: inserted("rank_by_cost"),
|
||||||
},
|
},
|
||||||
}),
|
|
||||||
catch: (cause) => DatabaseError.make({ cause }),
|
|
||||||
}),
|
|
||||||
{ discard: true },
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
}
|
||||||
|
|
||||||
const deleteRetiredDimensions = Effect.fn("ProviderStatRepo.deleteRetiredDimensions")(function* (
|
const deleteRetiredDimensions = Effect.fn("ProviderStatRepo.deleteRetiredDimensions")(function* (
|
||||||
rows: ProviderStatRow[],
|
rows: ProviderStatRow[],
|
||||||
|
|||||||
@@ -147,6 +147,18 @@ export function combineRows<T extends StatBaseRow>(left: T, right: T): T {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isMissingUniqueUsersColumn(cause: unknown): boolean {
|
||||||
|
return errorText(cause).includes("Unknown column 'unique_users'")
|
||||||
|
}
|
||||||
|
|
||||||
|
export function omitUniqueUsers<T extends { unique_users?: number }>(rows: T[]) {
|
||||||
|
return rows.map((row) => {
|
||||||
|
const result = { ...row }
|
||||||
|
delete result.unique_users
|
||||||
|
return result
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
export function statPeriodKey(row: StatBaseRow) {
|
export function statPeriodKey(row: StatBaseRow) {
|
||||||
return [row.grain, row.period_key, row.dataset, row.tier, row.client, row.source].join("\u0000")
|
return [row.grain, row.period_key, row.dataset, row.tier, row.client, row.source].join("\u0000")
|
||||||
}
|
}
|
||||||
@@ -242,6 +254,15 @@ export function inserted(column: string) {
|
|||||||
return sql.raw(`values(\`${column}\`)`)
|
return sql.raw(`values(\`${column}\`)`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function errorText(cause: unknown): string {
|
||||||
|
if (cause instanceof Error) return `${cause.message} ${errorText((cause as { cause?: unknown }).cause)}`
|
||||||
|
if (typeof cause === "object" && cause)
|
||||||
|
return Object.values(cause as Record<string, unknown>)
|
||||||
|
.map(errorText)
|
||||||
|
.join(" ")
|
||||||
|
return String(cause)
|
||||||
|
}
|
||||||
|
|
||||||
export function weightedAverage(
|
export function weightedAverage(
|
||||||
left: number | null | undefined,
|
left: number | null | undefined,
|
||||||
leftWeight = 0,
|
leftWeight = 0,
|
||||||
|
|||||||
Reference in New Issue
Block a user