EnterpriseAI/ClosedCode
1
0
Fork 0
Code Issues Pull Requests Actions 31 Packages Projects Releases Wiki Activity

zen: tpm routing

Browse Source
This commit is contained in:
Frank
2026-04-20 22:21:06 -04:00
parent 3e8abac625
commit f74a255ca9
5 changed files with 2746 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/console/app/src/routes/zen/util/handler.ts
+16 -7
View File
@@ -448,31 +448,40 @@ export async function handler(
return modelInfo.providers.find((provider) => provider.id === modelInfo.byokProvider)
}
// Filter out TPM limited providers
const allProviders = modelInfo.providers.filter((provider) => {
if (!provider.tpmLimit) return true
const usage = modelTpmLimits?.[`${provider.id}/${provider.model}`] ?? 0
return usage < provider.tpmLimit * 1_000_000
})
// Always use the same provider for the same session
if (stickyProvider) {
const provider = modelInfo.providers.find((provider) => provider.id === stickyProvider)
const provider = allProviders.find((provider) => provider.id === stickyProvider)
if (provider) return provider
}
if (trialProviders) {
const trialProvider = trialProviders[Math.floor(Math.random() * trialProviders.length)]
const provider = modelInfo.providers.find((provider) => provider.id === trialProvider)
const provider = allProviders.find((provider) => provider.id === trialProvider)
if (provider) return provider
}
if (retry.retryCount !== MAX_FAILOVER_RETRIES) {
const allProviders = modelInfo.providers
let topPriority = Infinity
const providers = allProviders
.filter((provider) => !provider.disabled)
.filter((provider) => provider.weight !== 0)
.filter((provider) => !retry.excludeProviders.includes(provider.id))
.filter((provider) => {
if (!provider.tpmLimit) return true
const usage = modelTpmLimits?.[`${provider.id}/${provider.model}`] ?? 0
return usage < provider.tpmLimit * 1_000_000
return usage < provider.tpmLimit * 1_000_000 * 0.8
})
.map((provider) => {
topPriority = Math.min(topPriority, provider.priority)
return provider
})
const topPriority = Math.min(...allProviders.map((p) => p.priority))
const providers = allProviders
.filter((p) => p.priority <= topPriority)
.flatMap((provider) => Array<typeof provider>(provider.weight).fill(provider))