Spaces:
Paused
Paused
File size: 4,225 Bytes
93d826e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | package shared
import (
"log"
"github.com/davecgh/go-spew/spew"
"github.com/jinzhu/copier"
)
type ModelErrKind string
const (
ErrOverloaded ModelErrKind = "ErrOverloaded"
ErrContextTooLong ModelErrKind = "ErrContextTooLong"
ErrRateLimited ModelErrKind = "ErrRateLimited"
ErrSubscriptionQuotaExhausted ModelErrKind = "ErrSubscriptionQuotaExhausted"
ErrOther ModelErrKind = "ErrOther"
ErrCacheSupport ModelErrKind = "ErrCacheSupport"
)
type ModelError struct {
Kind ModelErrKind
Retriable bool
RetryAfterSeconds int
}
func (m ModelError) ShouldIncrementRetry() bool {
return m.Kind != ErrSubscriptionQuotaExhausted && m.Kind != ErrCacheSupport
}
// if fallback is defined, retry with main model, then remaining tries use error fallback
type FallbackType string
const (
FallbackTypeError FallbackType = "error"
FallbackTypeContext FallbackType = "context"
FallbackTypeProvider FallbackType = "provider"
)
type FallbackResult struct {
ModelRoleConfig *ModelRoleConfig
IsFallback bool
FallbackType FallbackType
BaseModelConfig *BaseModelConfig
}
const MAX_RETRIES_BEFORE_FALLBACK = 1
func (m *ModelRoleConfig) GetFallbackForModelError(
numTotalRetry int,
didProviderFallback bool,
modelErr *ModelError,
authVars map[string]string,
settings *PlanSettings,
orgUserConfig *OrgUserConfig,
) FallbackResult {
if m == nil || modelErr == nil {
return FallbackResult{
ModelRoleConfig: m,
BaseModelConfig: m.GetBaseModelConfig(authVars, settings, orgUserConfig),
IsFallback: false,
}
}
if modelErr.Kind == ErrContextTooLong {
if m.LargeContextFallback != nil {
return FallbackResult{
ModelRoleConfig: m.LargeContextFallback,
BaseModelConfig: m.LargeContextFallback.GetBaseModelConfig(authVars, settings, orgUserConfig),
FallbackType: FallbackTypeContext,
IsFallback: true,
}
}
} else if !modelErr.Retriable || numTotalRetry > MAX_RETRIES_BEFORE_FALLBACK {
if m.ErrorFallback != nil {
return FallbackResult{
ModelRoleConfig: m.ErrorFallback,
BaseModelConfig: m.ErrorFallback.GetBaseModelConfig(authVars, settings, orgUserConfig),
FallbackType: FallbackTypeError,
IsFallback: true,
}
} else if !didProviderFallback {
log.Println("no error fallback, trying provider fallback")
providerFallback := m.GetProviderFallback(authVars, settings, orgUserConfig)
log.Println(spew.Sdump(map[string]interface{}{
"providerFallback": providerFallback,
}))
if providerFallback != nil {
return FallbackResult{
ModelRoleConfig: providerFallback,
BaseModelConfig: providerFallback.GetBaseModelConfig(authVars, settings, orgUserConfig),
FallbackType: FallbackTypeProvider,
IsFallback: true,
}
}
}
}
return FallbackResult{
ModelRoleConfig: m,
IsFallback: false,
}
}
// we just try a single provider fallback if all defined fallbacks are exhausted
// if we've got openrouter credentials in the stack, we always use OpenRouter as the fallback since it has its own routing/fallback routing to maximize resilience
// otherwise we just use the second provider in the stack
// if we're using the claude subscription, we also go to second provider in the stack rather than openrouter
func (m ModelRoleConfig) GetProviderFallback(authVars map[string]string, settings *PlanSettings, orgUserConfig *OrgUserConfig) *ModelRoleConfig {
providers := m.GetProvidersForAuthVars(authVars, settings, orgUserConfig)
if len(providers) < 2 {
return nil
}
firstProvider := providers[0]
res := ModelRoleConfig{}
copier.Copy(&res, m)
var provider ModelProvider
if !firstProvider.HasClaudeMaxAuth {
for _, p := range providers {
if p.Provider == ModelProviderOpenRouter {
provider = p.Provider
break
}
}
}
if provider == "" {
provider = providers[1].Provider
}
availableModel := GetAvailableModel(provider, m.ModelId)
if availableModel != nil {
c := availableModel.BaseModelConfig
res.BaseModelConfig = &c
} else {
c := m.GetBaseModelConfig(authVars, settings, orgUserConfig)
res.BaseModelConfig = c
}
return &res
}
|