From 06af0b8e0000aac357e04b4d94dddbc1f170a5b5 Mon Sep 17 00:00:00 2001 From: zijiren233 Date: Mon, 7 Sep 2026 15:34:32 +0800 Subject: [PATCH 1/2] fix: bound cache follow lifetime and honor preferred backups --- core/controller/relay-backup_test.go | 62 ++++++------ core/controller/relay-budget_test.go | 1 + core/controller/relay-channel.go | 15 +-- core/relay/meta/meta.go | 2 + core/relay/plugin/cachefollow/README.md | 16 +++- core/relay/plugin/cachefollow/cachefollow.go | 8 +- .../plugin/cachefollow/cachefollow_test.go | 94 +++++++++++++++++-- core/relay/plugin/cachefollow/config.go | 3 +- web/public/locales/en/translation.json | 4 +- web/public/locales/zh/translation.json | 4 +- 10 files changed, 151 insertions(+), 58 deletions(-) diff --git a/core/controller/relay-backup_test.go b/core/controller/relay-backup_test.go index f2f36c366..474cb6862 100644 --- a/core/controller/relay-backup_test.go +++ b/core/controller/relay-backup_test.go @@ -38,7 +38,7 @@ func TestInitialChannelSelectionPrefersOrdinaryAndAllowsBackupOnlyPool(t *testin ignored map[int64]struct{} wantBackup bool }{ - {name: "preferred backup stays excluded while primary is eligible", channels: []*model.Channel{primary, backup}}, + {name: "preferred backup is eligible alongside primary", channels: []*model.Channel{primary, backup}, wantBackup: true}, { name: "high error primary unlocks backup", channels: []*model.Channel{primary, backup}, errorRates: map[int64]float64{1: 1}, wantBackup: true, @@ -50,7 +50,7 @@ func TestInitialChannelSelectionPrefersOrdinaryAndAllowsBackupOnlyPool(t *testin {name: "backup alone can start a request", channels: []*model.Channel{backup}, wantBackup: true}, { name: "primary at error threshold remains eligible", channels: []*model.Channel{primary, backup}, - errorRates: map[int64]float64{1: maxRetryErrorRate}, + errorRates: map[int64]float64{1: maxRetryErrorRate}, wantBackup: true, }, { name: "unhealthy backup does not replace primary fallback", channels: []*model.Channel{primary, backup}, @@ -92,7 +92,7 @@ func TestInitialChannelSelectionPrefersOrdinaryAndAllowsBackupOnlyPool(t *testin } } -func TestInitialBackupUsesPreferencesAndStaysUnlockedAfterPrimaryRecovery(t *testing.T) { +func TestInitialPreferredBackupKeepsGeneralBackupsLocked(t *testing.T) { t.Parallel() primary := &model.Channel{ @@ -135,7 +135,7 @@ func TestInitialBackupUsesPreferencesAndStaysUnlockedAfterPrimaryRecovery(t *tes }, model.Price{}, time.Now(), ) - assert.True(t, state.backupOnlyEnabled) + assert.False(t, state.backupOnlyEnabled) assert.ElementsMatch( t, []*model.Channel{primary, backup, generalPrimary, generalBackup}, @@ -154,10 +154,13 @@ func TestInitialBackupUsesPreferencesAndStaysUnlockedAfterPrimaryRecovery(t *tes []*model.Channel{generalPrimary, generalBackup}, getRetryCandidates(state, nil), ) - assert.True(t, state.backupOnlyEnabled) + assert.False(t, state.backupOnlyEnabled) + channel, err = getRetryChannel(context.Background(), state) + require.NoError(t, err) + assert.Equal(t, generalPrimary, channel) } -func TestOrdinarySelectionCompletesBeforeRestartingWithBackups(t *testing.T) { +func TestPreferencesIncludeBackupsBeforeRestartingRound(t *testing.T) { t.Parallel() channels := []*model.Channel{ @@ -185,7 +188,7 @@ func TestOrdinarySelectionCompletesBeforeRestartingWithBackups(t *testing.T) { mc, nil, "backup-test", mode.Responses, []int{2, 1, 3, 4}, nil, nil, ) require.NoError(t, err) - require.Equal(t, 1, initial.channel.ID) + require.Equal(t, 2, initial.channel.ID) assert.False(t, initial.backupOnlyEnabled) state := initRetryState( @@ -201,15 +204,11 @@ func TestOrdinarySelectionCompletesBeforeRestartingWithBackups(t *testing.T) { time.Now(), ) - for i, id := range []int{3, 2, 1, 3, 4} { + for _, id := range []int{1, 3, 4} { channel, err := getRetryChannel(context.Background(), state) require.NoError(t, err) require.Equal(t, id, channel.ID) - assert.Equal(t, i != 0, state.backupOnlyEnabled) - - if i == 1 { - assert.Empty(t, state.failedChannelIDs) - } + assert.False(t, state.backupOnlyEnabled) state.failedChannelIDs[int64(id)] = struct{}{} } @@ -232,14 +231,6 @@ func TestBackupUnlockRestartsRoundWithOrdinaryChannels(t *testing.T) { priorities [2]int32 wantIDs []int }{ - { - name: "preferred primary retries before backup", preferred: []int{1, 2}, - priorities: [2]int32{-1, 10}, wantIDs: []int{1, 1, 2}, - }, - { - name: "preferred backup retries before primary", preferred: []int{2, 1}, - priorities: [2]int32{10, -1}, wantIDs: []int{1, 2, 1}, - }, { name: "weighted primary retries before backup", priorities: [2]int32{10, -1}, wantIDs: []int{1, 1, 2}, @@ -356,14 +347,14 @@ func TestPreferredBackupEligibility(t *testing.T) { wantIDs []int wantUnlocked bool }{ - {name: "preferred backup waits for all ordinary channels", preferred: []int{2}, wantIDs: []int{1, 3}}, + {name: "preferred backup precedes ordinary channels", preferred: []int{2}, wantIDs: []int{2}}, { name: "preferred primary unhealthy", preferred: []int{1, 2}, - errorRates: map[int64]float64{1: 1}, wantIDs: []int{3}, + errorRates: map[int64]float64{1: 1}, wantIDs: []int{2}, }, { name: "preferred primary banned", preferred: []int{1, 2}, - ignored: map[int64]struct{}{1: {}}, wantIDs: []int{3}, + ignored: map[int64]struct{}{1: {}}, wantIDs: []int{2}, }, { name: "preferred backup unhealthy", preferred: []int{1, 2}, @@ -375,15 +366,15 @@ func TestPreferredBackupEligibility(t *testing.T) { }, { name: "preferred backup at error threshold", preferred: []int{1, 2}, - errorRates: map[int64]float64{1: 1, 2: maxRetryErrorRate, 3: 1}, wantIDs: []int{2}, wantUnlocked: true, + errorRates: map[int64]float64{1: 1, 2: maxRetryErrorRate, 3: 1}, wantIDs: []int{2}, }, { name: "all primaries unhealthy restores preferred backup", preferred: []int{2}, - errorRates: map[int64]float64{1: 1, 3: 1}, wantIDs: []int{2}, wantUnlocked: true, + errorRates: map[int64]float64{1: 1, 3: 1}, wantIDs: []int{2}, }, { name: "all primaries banned restores preferred backup", preferred: []int{2}, - ignored: map[int64]struct{}{1: {}, 3: {}}, wantIDs: []int{2}, wantUnlocked: true, + ignored: map[int64]struct{}{1: {}, 3: {}}, wantIDs: []int{2}, }, { name: "preferred backup banned after unlock", preferred: []int{2}, @@ -548,7 +539,7 @@ func TestBackupOnlyUnlocksAfterPrimaryFailuresAndSurvivesRounds(t *testing.T) { {ID: 4, Status: model.ChannelStatusEnabled, BackupOnly: true}, } initial := &initialChannel{ - channel: channels[0], migratedChannels: channels, preferChannelIDs: []int{3, 4, 2}, + channel: channels[0], migratedChannels: channels, preferChannelIDs: []int{2}, } state := initRetryState( 10, initial, meta.NewMeta(channels[0], mode.Responses, "backup-test", model.ModelConfig{}), @@ -563,17 +554,22 @@ func TestBackupOnlyUnlocksAfterPrimaryFailuresAndSurvivesRounds(t *testing.T) { require.NoError(t, err) assert.Equal(t, 2, channel.ID) assert.False(t, state.backupOnlyEnabled) - assert.Equal(t, []int{3, 4, 2}, state.preferChannelIDs) + assert.Equal(t, []int{2}, state.preferChannelIDs) state.failedChannelIDs[2] = struct{}{} - for _, id := range []int{3, 4, 2, 1} { + ids := make([]int, 0, 4) + for i := range 4 { channel, err = getRetryChannel(context.Background(), state) require.NoError(t, err) - assert.Equal(t, id, channel.ID) + if i == 0 { + assert.Equal(t, 2, channel.ID) + } + ids = append(ids, channel.ID) assert.True(t, state.backupOnlyEnabled) - assert.Equal(t, []int{3, 4, 2}, state.preferChannelIDs) - state.failedChannelIDs[int64(id)] = struct{}{} + assert.Equal(t, []int{2}, state.preferChannelIDs) + state.failedChannelIDs[int64(channel.ID)] = struct{}{} } + assert.ElementsMatch(t, []int{1, 2, 3, 4}, ids) channel, err = getRetryChannel(context.Background(), state) require.NoError(t, err) diff --git a/core/controller/relay-budget_test.go b/core/controller/relay-budget_test.go index 9a062bef0..1a3c4068c 100644 --- a/core/controller/relay-budget_test.go +++ b/core/controller/relay-budget_test.go @@ -118,6 +118,7 @@ func TestRetryLoopBudgetAndCount(t *testing.T) { } if tt.initialBackoff { + state.preferChannelIDs = []int{1} state.recordChannelFailure(1, time.Now()) } diff --git a/core/controller/relay-channel.go b/core/controller/relay-channel.go index 31c3b29ad..893eb1dab 100644 --- a/core/controller/relay-channel.go +++ b/core/controller/relay-channel.go @@ -357,23 +357,22 @@ type channelSelectionState struct { backupOnlyEnabled bool // Remains enabled across retry rounds for this request. } -// Channels have already passed health and failure filtering. Both selection -// passes reuse that snapshot and restart from the same preference order. +// Preferences bypass backup-only gating, while health and failure filters still apply. func (s *channelSelectionState) selectChannel( channels []*model.Channel, preferChannelIDs []int, errorRates map[int64]float64, ) (*model.Channel, error) { + if channel := pickPreferredChannel(channels, preferChannelIDs); channel != nil { + return channel, nil + } + for { candidates := channels if !s.backupOnlyEnabled { candidates = nonBackupChannels(channels) } - if channel := pickPreferredChannel(candidates, preferChannelIDs); channel != nil { - return channel, nil - } - channel, err := pickChannel(candidates, errorRates) if err == nil || s.backupOnlyEnabled { return channel, err @@ -697,6 +696,10 @@ func (s *retryState) selectRetryChannel( errorRates map[int64]float64, ) (*model.Channel, error) { candidates := getRetryCandidates(s, errorRates) + if channel := pickPreferredChannel(candidates, s.preferChannelIDs); channel != nil { + return channel, nil + } + if !s.backupOnlyEnabled && len(candidates) > 0 && len(nonBackupChannels(candidates)) == 0 { // Adding eligible backups starts a fresh round while preserving cache preferences. s.backupOnlyEnabled = true diff --git a/core/relay/meta/meta.go b/core/relay/meta/meta.go index 632ceb3a0..201649bbf 100644 --- a/core/relay/meta/meta.go +++ b/core/relay/meta/meta.go @@ -15,6 +15,7 @@ type ChannelMeta struct { Key string ID int Type model.ChannelType + BackupOnly bool ModelMapping map[string]string EnabledAutoBalanceCheck bool SkipTLSVerify bool @@ -196,6 +197,7 @@ func (m *Meta) SetChannel(channel *model.Channel) { m.Channel.Key = channel.Key m.Channel.ID = channel.ID m.Channel.Type = channel.Type + m.Channel.BackupOnly = channel.BackupOnly m.Channel.EnabledAutoBalanceCheck = channel.EnabledAutoBalanceCheck m.Channel.SkipTLSVerify = channel.SkipTLSVerify m.Channel.EnabledNoPermissionBan = channel.EnabledNoPermissionBan diff --git a/core/relay/plugin/cachefollow/README.md b/core/relay/plugin/cachefollow/README.md index 1ad81ffbf..cde55a224 100644 --- a/core/relay/plugin/cachefollow/README.md +++ b/core/relay/plugin/cachefollow/README.md @@ -25,7 +25,7 @@ Default behavior, with only `prompt_cache_key` and `user` scopes enabled: "cachefollow": { "enable": true, "followed_channel_ttl_seconds": 180, - "recent_channel_update_debounce_seconds": 30 + "recent_channel_update_debounce_seconds": 45 } } } @@ -42,7 +42,7 @@ If you also want generic cache-follow: "enable": true, "enable_generic_follow": true, "followed_channel_ttl_seconds": 180, - "recent_channel_update_debounce_seconds": 30 + "recent_channel_update_debounce_seconds": 45 } } } @@ -55,7 +55,9 @@ If you also want generic cache-follow: | `enable` | `bool` | `false` | Supports `prompt_cache_key`-based targeted channel follow and user-scoped cache-follow by default, helping improve upstream cache hit rate. Disabled by default. | | `enable_generic_follow` | `bool` | `false` | Enables generic cache-follow for the model-level scope. This is only used when no `prompt_cache_key` mapping or user-scoped mapping is available. Recommended when each user effectively has an isolated `group` and `token`; not recommended when many users share the same `group` and `token` scope. | | `followed_channel_ttl_seconds` | `integer` | `180` | Controls how long a remembered cache-effective channel stays valid. It always applies to user-scoped entries, applies to generic entries only when generic follow is enabled, and also applies to `prompt_cache_key` entries when the upstream does not return a more specific retention. | -| `recent_channel_update_debounce_seconds` | `integer` | `30` | Controls the minimum refresh interval for the `recent` channel mapping in the same scope, reducing noisy `recent` updates while still following recent upstream routing changes. | +| `recent_channel_update_debounce_seconds` | `integer` | `45` | Controls the minimum refresh interval for the `recent` channel mapping in the same scope, reducing noisy `recent` updates while still following recent upstream routing changes. | + +Every new mapping has a maximum TTL of `300s`, including configured TTLs and upstream retention such as `24h`. Further successful cache usage can refresh `recent` after the debounce interval, starting a new bounded TTL. ## Scopes and Priority @@ -102,6 +104,7 @@ They are written differently: - `stable` is only written when that scope does not already have a stable mapping - `recent` can be refreshed continuously, but is rate-limited by `recent_channel_update_debounce_seconds` +- backup-only channels only write `recent` mappings In practice: @@ -160,7 +163,7 @@ When the request includes `prompt_cache_key` and the request has cache-related u TTL rules for this scope: -- if the upstream returns a valid `prompt_cache_retention`, that value is used first +- if the upstream returns a valid `prompt_cache_retention`, it is capped at 5 minutes - otherwise `followed_channel_ttl_seconds` is used - if `followed_channel_ttl_seconds` is not configured, the built-in default `180s` is used @@ -248,13 +251,15 @@ This means: Remembered channels are only preferences. They still go through normal availability checks. +A preferred `backup_only` channel is eligible immediately, in preference order. Selecting it does not unlock other backup channels. Channels that failed in the current retry round remain excluded until the round resets. + A preferred channel is skipped if any of the following is true: - it no longer exists in the model's currently available channel set - it is disabled - it does not support the current request mode - it is currently banned by monitor -- its error rate is higher than `0.75` +- its error rate is higher than `0.85` If a preferred channel is skipped, the system moves to the next preferred channel. If no preferred channel survives filtering, selection falls back to the normal channel selection flow. @@ -271,3 +276,4 @@ This also means: - generic mappings are only read and written when `enable_generic_follow = true` - the plugin only affects channel preference; it does not modify the request body - `recent_channel_update_debounce_seconds` only affects `recent` refresh frequency and does not affect `stable` +- newly written cache mappings never exceed 5 minutes, regardless of upstream retention diff --git a/core/relay/plugin/cachefollow/cachefollow.go b/core/relay/plugin/cachefollow/cachefollow.go index 82628e0a3..dcd54c1eb 100644 --- a/core/relay/plugin/cachefollow/cachefollow.go +++ b/core/relay/plugin/cachefollow/cachefollow.go @@ -19,8 +19,9 @@ import ( ) const ( + maxFollowedChannelTTL = 5 * time.Minute defaultFollowedChannelTTL = 3 * time.Minute - defaultRecentChannelUpdateDebounce = 30 * time.Second + defaultRecentChannelUpdateDebounce = 45 * time.Second ) var _ plugin.Plugin = (*Plugin)(nil) @@ -44,6 +45,7 @@ func (p *Plugin) getConfig(meta *meta.Meta) (*Config, error) { } func getFollowedChannelTTL(retention string, defaultTTL time.Duration) time.Duration { + defaultTTL = min(defaultTTL, maxFollowedChannelTTL) retention = strings.TrimSpace(strings.ToLower(retention)) if retention == "" || retention == "in-memory" || retention == "in_memory" { return defaultTTL @@ -54,7 +56,7 @@ func getFollowedChannelTTL(retention string, defaultTTL time.Duration) time.Dura return defaultTTL } - return ttl + return min(ttl, maxFollowedChannelTTL) } func getNodeStringField(node *ast.Node, key string) (string, bool) { @@ -154,7 +156,7 @@ func saveStableStoreMapping( meta *meta.Meta, expiresAt time.Time, ) error { - if id == "" { + if id == "" || meta.Channel.BackupOnly { return nil } diff --git a/core/relay/plugin/cachefollow/cachefollow_test.go b/core/relay/plugin/cachefollow/cachefollow_test.go index efaea63b4..060121da3 100644 --- a/core/relay/plugin/cachefollow/cachefollow_test.go +++ b/core/relay/plugin/cachefollow/cachefollow_test.go @@ -2,6 +2,7 @@ package cachefollow import ( + "math" "net/http" "net/http/httptest" "testing" @@ -164,10 +165,10 @@ func TestDoResponseRecordsPromptAndGenericMappingsForResponses(t *testing.T) { model.CacheFollowStoreID("gpt-5", model.CacheKeyTypeRecent), store.saved[1].ID, ) - assert.True(t, store.savedIfNotExist[0].ExpiresAt.After(start.Add(24*time.Hour-time.Second))) - assert.True(t, store.savedIfNotExist[0].ExpiresAt.Before(end.Add(24*time.Hour+time.Second))) - assert.True(t, store.saved[0].ExpiresAt.After(start.Add(24*time.Hour-time.Second))) - assert.True(t, store.saved[0].ExpiresAt.Before(end.Add(24*time.Hour+time.Second))) + assert.True(t, store.savedIfNotExist[0].ExpiresAt.After(start.Add(5*time.Minute-time.Second))) + assert.True(t, store.savedIfNotExist[0].ExpiresAt.Before(end.Add(5*time.Minute+time.Second))) + assert.True(t, store.saved[0].ExpiresAt.After(start.Add(5*time.Minute-time.Second))) + assert.True(t, store.saved[0].ExpiresAt.Before(end.Add(5*time.Minute+time.Second))) } func TestDoResponseRecordsPromptAndGenericMappingsForResponsesCompact(t *testing.T) { @@ -864,10 +865,14 @@ func TestConfigFollowedChannelTiming(t *testing.T) { ) assert.Equal(t, defaultRecentChannelUpdateDebounce, Config{}.GetRecentChannelUpdateDebounce()) + assert.Equal(t, 45*time.Second, Config{}.GetRecentChannelUpdateDebounce()) + for _, seconds := range []int64{301, 86400, math.MaxInt64} { + assert.Equal(t, 5*time.Minute, Config{FollowedChannelTTLSeconds: seconds}.GetFollowedChannelTTL()) + } assert.Equal( t, - 45*time.Second, - Config{RecentChannelUpdateDebounceSeconds: 45}.GetRecentChannelUpdateDebounce(), + 30*time.Second, + Config{RecentChannelUpdateDebounceSeconds: 30}.GetRecentChannelUpdateDebounce(), ) assert.Equal( t, @@ -875,3 +880,80 @@ func TestConfigFollowedChannelTiming(t *testing.T) { Config{RecentChannelUpdateDebounceSeconds: -1}.GetRecentChannelUpdateDebounce(), ) } + +func TestFollowedChannelTTLCap(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + retention string + fallback time.Duration + want time.Duration + }{ + {"24h", 3 * time.Minute, 5 * time.Minute}, + {"5m", 3 * time.Minute, 5 * time.Minute}, + {"30s", 3 * time.Minute, 30 * time.Second}, + {"", 3 * time.Minute, 3 * time.Minute}, + {"in-memory", time.Hour, 5 * time.Minute}, + {"in_memory", time.Hour, 5 * time.Minute}, + {"invalid", time.Hour, 5 * time.Minute}, + {"0s", time.Hour, 5 * time.Minute}, + {"-1s", time.Hour, 5 * time.Minute}, + } { + assert.Equal(t, tt.want, getFollowedChannelTTL(tt.retention, tt.fallback), tt.retention) + } +} + +func TestRecentMappingDefaultDebounce(t *testing.T) { + t.Parallel() + requestMeta := meta.NewMeta(&model.Channel{ID: 9, BackupOnly: true}, mode.Responses, "gpt-5", model.ModelConfig{}) + for _, age := range []time.Duration{30 * time.Second, 46 * time.Second} { + store := &recordingStore{stores: map[string]adaptor.StoreCache{ + "recent": {ID: "recent", ChannelID: 1, UpdatedAt: time.Now().Add(-age)}, + }} + require.NoError(t, saveRecentStoreMapping(store, "recent", requestMeta, time.Now().Add(time.Minute), Config{}.GetRecentChannelUpdateDebounce())) + if age < 45*time.Second { + assert.Empty(t, store.saved) + assert.Equal(t, 1, store.stores["recent"].ChannelID) + } else { + require.Len(t, store.saved, 1) + assert.Equal(t, 9, store.stores["recent"].ChannelID) + } + } +} + +func TestBackupOnlyRecordsRecentMappings(t *testing.T) { + t.Parallel() + for _, backupOnly := range []bool{false, true} { + requestMeta := meta.NewMeta(&model.Channel{ID: 9, BackupOnly: backupOnly}, mode.Responses, "gpt-5", model.ModelConfig{ + Plugin: map[string]map[string]any{PluginName: { + "enable": true, "enable_generic_follow": true, "followed_channel_ttl_seconds": 86400, + }}, + }) + requestMeta.PromptCacheKey = "cache-key" + requestMeta.User = "user-1" + store := &recordingStore{} + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/v1/responses", nil) + start := time.Now() + _, relayErr := (&Plugin{}).DoResponse(requestMeta, store, c, &http.Response{StatusCode: http.StatusOK}, doResponseFunc{ + fn: func(_ *meta.Meta, _ adaptor.Store, c *gin.Context, _ *http.Response) (adaptor.DoResponseResult, adaptor.Error) { + c.Status(http.StatusOK) + _, _ = c.Writer.Write([]byte(`{"prompt_cache_retention":"24h"}`)) + return adaptor.DoResponseResult{Usage: model.Usage{CachedTokens: 4}}, nil + }, + }) + require.Nil(t, relayErr) + if backupOnly { + assert.Empty(t, store.savedIfNotExist) + } else { + require.Len(t, store.savedIfNotExist, 3) + } + require.Len(t, store.saved, 3) + for _, mapping := range store.stores { + assert.Equal(t, 9, mapping.ChannelID) + assert.False(t, mapping.ExpiresAt.Before(start.Add(5*time.Minute))) + assert.False(t, mapping.ExpiresAt.After(time.Now().Add(5*time.Minute))) + } + requestMeta.SetChannel(&model.Channel{ID: 1}) + assert.False(t, requestMeta.Channel.BackupOnly) + } +} diff --git a/core/relay/plugin/cachefollow/config.go b/core/relay/plugin/cachefollow/config.go index a30f65f2e..e652d3bfe 100644 --- a/core/relay/plugin/cachefollow/config.go +++ b/core/relay/plugin/cachefollow/config.go @@ -14,7 +14,8 @@ type Config struct { func (c Config) GetFollowedChannelTTL() time.Duration { if c.FollowedChannelTTLSeconds > 0 { - return time.Duration(c.FollowedChannelTTLSeconds) * time.Second + seconds := min(c.FollowedChannelTTLSeconds, int64(maxFollowedChannelTTL/time.Second)) + return time.Duration(seconds) * time.Second } return defaultFollowedChannelTTL diff --git a/web/public/locales/en/translation.json b/web/public/locales/en/translation.json index 26300a83f..cc2602444 100644 --- a/web/public/locales/en/translation.json +++ b/web/public/locales/en/translation.json @@ -553,9 +553,9 @@ "enableGenericFollowDescription": "Enables generic cache-follow for the group + token + model scope when no prompt_cache_key or user-scoped mapping is available. Recommended when each user or tenant has an isolated group/token. Do not enable it when many users share the same group/token or a global admin key, otherwise requests in that shared scope may converge on one remembered channel and reduce load balancing.", "followedChannelTTLSeconds": "Followed Channel TTL (seconds)", "followedChannelTTLSecondsPlaceholder": "Leave empty to use the default value 180", - "followedChannelTTLSecondsDescription": "Controls how long a remembered cache-effective channel stays valid. It always applies to user-scoped entries, applies to generic entries only when generic follow is enabled, and also applies to prompt_cache_key entries when the upstream does not return a more specific retention.", + "followedChannelTTLSecondsDescription": "Controls how long a remembered cache-effective channel stays valid. Applies to user entries, enabled generic entries, and prompt_cache_key entries without upstream retention. All new entries are capped at 300 seconds, including upstream retention.", "recentChannelUpdateDebounceSeconds": "Recent Channel Update Debounce (seconds)", - "recentChannelUpdateDebounceSecondsPlaceholder": "Leave empty to use the default value 30", + "recentChannelUpdateDebounceSecondsPlaceholder": "Leave empty to use the default value 45", "recentChannelUpdateDebounceSecondsDescription": "Controls the minimum refresh interval for the recent channel mapping in the same scope, reducing noisy recent writes while still following upstream routing changes." }, "webSearchPlugin": { diff --git a/web/public/locales/zh/translation.json b/web/public/locales/zh/translation.json index 485827e43..e7c1e6313 100644 --- a/web/public/locales/zh/translation.json +++ b/web/public/locales/zh/translation.json @@ -542,9 +542,9 @@ "enableGenericFollowDescription": "开启 group + token + model 这一层通用范围的缓存渠道跟随。只有在没有 prompt_cache_key 映射和 user 范围映射可用时，才会额外尝试这层通用跟随。适合每个用户或租户都有独立 group/token 的场景；如果很多人共用同一个 group/token，或通过全局 admin key 承载流量，则不建议开启，否则这部分请求可能都会跟随到同一个已记住渠道，削弱负载均衡。", "followedChannelTTLSeconds": "跟随渠道保留时长（秒）", "followedChannelTTLSecondsPlaceholder": "留空则使用默认值 180", - "followedChannelTTLSecondsDescription": "控制已记住的缓存有效渠道保留多久。对 user 范围始终生效；对通用范围则仅在启用了通用缓存渠道跟随后生效；对于 prompt_cache_key，请求若上游没有返回更具体的 retention，也会使用这个值。", + "followedChannelTTLSecondsDescription": "控制缓存有效渠道的保留时间，适用于 user、已启用的通用范围及上游未返回 retention 的 prompt_cache_key。所有新记录最长保留 300 秒，上游返回的 retention 同样受此限制。", "recentChannelUpdateDebounceSeconds": "最近渠道更新时间去抖窗口（秒）", - "recentChannelUpdateDebounceSecondsPlaceholder": "留空则使用默认值 30", + "recentChannelUpdateDebounceSecondsPlaceholder": "留空则使用默认值 45", "recentChannelUpdateDebounceSecondsDescription": "控制同一作用域下 recent 渠道映射的最小刷新间隔，用于限制频繁更新 recent 记录，同时保留对上游最近路由变化的跟随能力。" }, "webSearchPlugin": { From 9c73fdcba3e071e7e560a1588f02330b98c09687 Mon Sep 17 00:00:00 2001 From: zijiren233 Date: Mon, 7 Sep 2026 15:41:41 +0800 Subject: [PATCH 2/2] style: satisfy cache follow lint checks --- core/controller/relay-backup_test.go | 4 + core/relay/plugin/cachefollow/cachefollow.go | 1 + .../plugin/cachefollow/cachefollow_test.go | 76 +++++++++++++++---- 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/core/controller/relay-backup_test.go b/core/controller/relay-backup_test.go index 474cb6862..4985e0de8 100644 --- a/core/controller/relay-backup_test.go +++ b/core/controller/relay-backup_test.go @@ -561,14 +561,18 @@ func TestBackupOnlyUnlocksAfterPrimaryFailuresAndSurvivesRounds(t *testing.T) { for i := range 4 { channel, err = getRetryChannel(context.Background(), state) require.NoError(t, err) + if i == 0 { assert.Equal(t, 2, channel.ID) } + ids = append(ids, channel.ID) + assert.True(t, state.backupOnlyEnabled) assert.Equal(t, []int{2}, state.preferChannelIDs) state.failedChannelIDs[int64(channel.ID)] = struct{}{} } + assert.ElementsMatch(t, []int{1, 2, 3, 4}, ids) channel, err = getRetryChannel(context.Background(), state) diff --git a/core/relay/plugin/cachefollow/cachefollow.go b/core/relay/plugin/cachefollow/cachefollow.go index dcd54c1eb..5dd6a071b 100644 --- a/core/relay/plugin/cachefollow/cachefollow.go +++ b/core/relay/plugin/cachefollow/cachefollow.go @@ -46,6 +46,7 @@ func (p *Plugin) getConfig(meta *meta.Meta) (*Config, error) { func getFollowedChannelTTL(retention string, defaultTTL time.Duration) time.Duration { defaultTTL = min(defaultTTL, maxFollowedChannelTTL) + retention = strings.TrimSpace(strings.ToLower(retention)) if retention == "" || retention == "in-memory" || retention == "in_memory" { return defaultTTL diff --git a/core/relay/plugin/cachefollow/cachefollow_test.go b/core/relay/plugin/cachefollow/cachefollow_test.go index 060121da3..6e5028c2d 100644 --- a/core/relay/plugin/cachefollow/cachefollow_test.go +++ b/core/relay/plugin/cachefollow/cachefollow_test.go @@ -866,9 +866,15 @@ func TestConfigFollowedChannelTiming(t *testing.T) { assert.Equal(t, defaultRecentChannelUpdateDebounce, Config{}.GetRecentChannelUpdateDebounce()) assert.Equal(t, 45*time.Second, Config{}.GetRecentChannelUpdateDebounce()) + for _, seconds := range []int64{301, 86400, math.MaxInt64} { - assert.Equal(t, 5*time.Minute, Config{FollowedChannelTTLSeconds: seconds}.GetFollowedChannelTTL()) + assert.Equal( + t, + 5*time.Minute, + Config{FollowedChannelTTLSeconds: seconds}.GetFollowedChannelTTL(), + ) } + assert.Equal( t, 30*time.Second, @@ -883,6 +889,7 @@ func TestConfigFollowedChannelTiming(t *testing.T) { func TestFollowedChannelTTLCap(t *testing.T) { t.Parallel() + for _, tt := range []struct { retention string fallback time.Duration @@ -904,12 +911,28 @@ func TestFollowedChannelTTLCap(t *testing.T) { func TestRecentMappingDefaultDebounce(t *testing.T) { t.Parallel() - requestMeta := meta.NewMeta(&model.Channel{ID: 9, BackupOnly: true}, mode.Responses, "gpt-5", model.ModelConfig{}) + + requestMeta := meta.NewMeta( + &model.Channel{ID: 9, BackupOnly: true}, + mode.Responses, + "gpt-5", + model.ModelConfig{}, + ) for _, age := range []time.Duration{30 * time.Second, 46 * time.Second} { store := &recordingStore{stores: map[string]adaptor.StoreCache{ "recent": {ID: "recent", ChannelID: 1, UpdatedAt: time.Now().Add(-age)}, }} - require.NoError(t, saveRecentStoreMapping(store, "recent", requestMeta, time.Now().Add(time.Minute), Config{}.GetRecentChannelUpdateDebounce())) + require.NoError( + t, + saveRecentStoreMapping( + store, + "recent", + requestMeta, + time.Now().Add(time.Minute), + Config{}.GetRecentChannelUpdateDebounce(), + ), + ) + if age < 45*time.Second { assert.Empty(t, store.saved) assert.Equal(t, 1, store.stores["recent"].ChannelID) @@ -922,37 +945,60 @@ func TestRecentMappingDefaultDebounce(t *testing.T) { func TestBackupOnlyRecordsRecentMappings(t *testing.T) { t.Parallel() + for _, backupOnly := range []bool{false, true} { - requestMeta := meta.NewMeta(&model.Channel{ID: 9, BackupOnly: backupOnly}, mode.Responses, "gpt-5", model.ModelConfig{ - Plugin: map[string]map[string]any{PluginName: { - "enable": true, "enable_generic_follow": true, "followed_channel_ttl_seconds": 86400, - }}, - }) + requestMeta := meta.NewMeta( + &model.Channel{ID: 9, BackupOnly: backupOnly}, + mode.Responses, + "gpt-5", + model.ModelConfig{ + Plugin: map[string]map[string]any{PluginName: { + "enable": true, + "enable_generic_follow": true, + "followed_channel_ttl_seconds": 86400, + }}, + }, + ) requestMeta.PromptCacheKey = "cache-key" requestMeta.User = "user-1" store := &recordingStore{} c, _ := gin.CreateTestContext(httptest.NewRecorder()) - c.Request = httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/v1/responses", nil) + c.Request = httptest.NewRequestWithContext( + t.Context(), + http.MethodPost, + "/v1/responses", + nil, + ) start := time.Now() - _, relayErr := (&Plugin{}).DoResponse(requestMeta, store, c, &http.Response{StatusCode: http.StatusOK}, doResponseFunc{ - fn: func(_ *meta.Meta, _ adaptor.Store, c *gin.Context, _ *http.Response) (adaptor.DoResponseResult, adaptor.Error) { - c.Status(http.StatusOK) - _, _ = c.Writer.Write([]byte(`{"prompt_cache_retention":"24h"}`)) - return adaptor.DoResponseResult{Usage: model.Usage{CachedTokens: 4}}, nil + _, relayErr := (&Plugin{}).DoResponse( + requestMeta, + store, + c, + &http.Response{StatusCode: http.StatusOK}, + doResponseFunc{ + fn: func(_ *meta.Meta, _ adaptor.Store, c *gin.Context, _ *http.Response) (adaptor.DoResponseResult, adaptor.Error) { + c.Status(http.StatusOK) + _, _ = c.Writer.Write([]byte(`{"prompt_cache_retention":"24h"}`)) + return adaptor.DoResponseResult{Usage: model.Usage{CachedTokens: 4}}, nil + }, }, - }) + ) require.Nil(t, relayErr) + if backupOnly { assert.Empty(t, store.savedIfNotExist) } else { require.Len(t, store.savedIfNotExist, 3) } + require.Len(t, store.saved, 3) + for _, mapping := range store.stores { assert.Equal(t, 9, mapping.ChannelID) assert.False(t, mapping.ExpiresAt.Before(start.Add(5*time.Minute))) assert.False(t, mapping.ExpiresAt.After(time.Now().Add(5*time.Minute))) } + requestMeta.SetChannel(&model.Channel{ID: 1}) assert.False(t, requestMeta.Channel.BackupOnly) }