From 8bf23ba9c93d4e5f1a621018f9cb8973fcdf77f1 Mon Sep 17 00:00:00 2001 From: Pradeep Kintali Date: Fri, 20 Nov 2026 22:52:00 -0800 Subject: [PATCH] Add MySQL rejection and recovery telemetry Preserve existing admission behavior and counters while exposing configured-source rejection reasons, recovery transitions, and bounded INFO logs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fbc83df2-d1af-406f-b97b-51cf0aaa570d --- doc/mysql.md | 39 +++++ pkg/throttle/check.go | 5 + pkg/throttle/check_result.go | 13 +- pkg/throttle/safety_telemetry.go | 72 +++++++++ pkg/throttle/safety_telemetry_test.go | 206 ++++++++++++++++++++++++++ pkg/throttle/throttler.go | 25 +++- 6 files changed, 350 insertions(+), 10 deletions(-) create mode 100644 pkg/throttle/safety_telemetry.go create mode 100644 pkg/throttle/safety_telemetry_test.go diff --git a/doc/mysql.md b/doc/mysql.md index f123217..9c1381c 100644 --- a/doc/mysql.md +++ b/doc/mysql.md @@ -247,6 +247,45 @@ checks do not increment these counters. These are process-local cumulative request counters that reset on restart, not counts of overloaded probe readings or distinct overload episodes. They do not measure memcache client rejections. +#### Safety telemetry and logs + +The existing rejection counters above are unchanged. Additional counters record +non-internal MySQL checks returning `429` or `500`, with the configured requested +cluster, first blocking dependency, status, and reason: + +```text +check.rejection.mysql.prod4.dependency.prod4-primary.status.429.threshold.total +check.rejection.mysql.prod4.dependency.prod4-primary.status.500.no_hosts.total +``` + +Reasons are `threshold`, `recovery`, `shared_domain`, `no_hosts`, +`missing_metric`, `metric_error`, or `other`. `metric_error` includes probe +failures and measurements that have not been collected; it does not establish +the underlying cause. Fallback requests use the configured fallback cluster, +not arbitrary names supplied by callers. These counters do not include +application identities or memcache rejections. + +Recovery-enabled clusters also emit +`check.recovery.mysql...total`, where events are `initialized`, +`blocked`, `started`, `reset`, and `completed`. Initialization means a new +process or leader is requiring a healthy window, not that overload occurred. +Repeated readings in the same state do not increment transition counters. +The existing `recovery.mysql..active` gauge is unchanged. + +Both new counter families use `check.*` so the existing production Go Expvar +configuration collects them as rates. Local counters are cumulative since +startup; Datadog rates are events per second, not cumulative totals. + +At INFO level, recovery transitions emit `event=freno_recovery_` with +the cluster, reason, value, thresholds, and configured recovery duration. +Rejections emit `event=freno_admission_rejected` with the requested cluster, +blocking dependency, HTTP status, reason, value, and threshold. Rejection logs +are limited to one per minute per cluster/dependency/status/reason combination; +every eligible rejection still increments its counter. Logs omit raw error +messages, credentials, and request contents. Standard error follows the existing +log collection path; this change does not configure Splunk or repair Datadog +delivery gaps. + #### Configuration example A cluster can require additional cluster metrics to pass before `freno` permits diff --git a/pkg/throttle/check.go b/pkg/throttle/check.go index 13b31e8..b4c817a 100644 --- a/pkg/throttle/check.go +++ b/pkg/throttle/check.go @@ -82,6 +82,7 @@ func (check *ThrottlerCheck) checkAppMetricResult(appName string, storeType stri // shared-domain dependency check., checkResult = NewCheckResult(http.StatusTooManyRequests, value, threshold, base.ThresholdExceededError) + checkResult.rejectionReason = "shared_domain" } checkResult.MetricName = metricName return checkResult @@ -121,6 +122,7 @@ func (check *ThrottlerCheck) checkMySQLCluster( fmt.Errorf("required MySQL cluster metric %q is unavailable", requiredCluster), ) checkResult.MetricName = fmt.Sprintf("mysql/%s", requiredCluster) + checkResult.rejectionReason = "missing_metric" return checkResult } return requiredResult @@ -171,6 +173,9 @@ func (check *ThrottlerCheck) Check(appName string, storeType string, storeName s metricName := strings.ReplaceAll(checkResult.MetricName, "/", ".") metrics.GetOrRegisterCounter(fmt.Sprintf("check.rejected.%s.%s.total", metricName, reason), nil).Inc(1) } + if storeType == "mysql" { + check.recordMySQLRejection(appName, storeName, checkResult) + } go func(statusCode int) { metrics.GetOrRegisterCounter("check.any.total", nil).Inc(1) diff --git a/pkg/throttle/check_result.go b/pkg/throttle/check_result.go index a6730eb..fd6a9da 100644 --- a/pkg/throttle/check_result.go +++ b/pkg/throttle/check_result.go @@ -31,12 +31,13 @@ func newMetricCheckResult(value, threshold float64, err error, failOnNoHosts boo // CheckResult is the result for an app inquiring on a metric. It also exports as JSON via the API type CheckResult struct { - StatusCode int `json:"StatusCode"` - Value float64 `json:"Value"` - Threshold float64 `json:"Threshold"` - MetricName string `json:"MetricName,omitempty"` - Error error `json:"-"` - Message string `json:"Message"` + StatusCode int `json:"StatusCode"` + Value float64 `json:"Value"` + Threshold float64 `json:"Threshold"` + MetricName string `json:"MetricName,omitempty"` + Error error `json:"-"` + Message string `json:"Message"` + rejectionReason string } func NewCheckResult(statusCode int, value float64, threshold float64, err error) *CheckResult { diff --git a/pkg/throttle/safety_telemetry.go b/pkg/throttle/safety_telemetry.go new file mode 100644 index 0000000..5d91639 --- /dev/null +++ b/pkg/throttle/safety_telemetry.go @@ -0,0 +1,72 @@ +package throttle + +import ( + "errors" + "fmt" + "net/http" + "strings" + + "github.com/github/freno/pkg/base" + "github.com/github/freno/pkg/config" + "github.com/outbrain/golib/log" + "github.com/patrickmn/go-cache" + metrics "github.com/rcrowley/go-metrics" +) + +func safetyMetricErrorReason(err error) string { + switch { + case errors.Is(err, base.NoHostsError): + return "no_hosts" + case errors.Is(err, base.NoSuchMetricError): + return "missing_metric" + case errors.Is(err, base.RecoveryNotCompleteError): + return "recovery" + default: + return "metric_error" + } +} + +func (check *ThrottlerCheck) recordMySQLRejection(appName, clusterName string, result *CheckResult) { + if appName == frenoAppName || appName == frenoShareDmainAppName || + (result.StatusCode != http.StatusTooManyRequests && result.StatusCode != http.StatusInternalServerError) { + return + } + // Use configured names, including the resolved fallback, rather than unbounded caller input. + clusters := config.Settings().Stores.MySQL.Clusters + dependency := strings.TrimPrefix(result.MetricName, "mysql/") + if clusters[clusterName] == nil || clusters[dependency] == nil { + return + } + reason := result.rejectionReason + if reason == "" { + switch { + case result.StatusCode == http.StatusInternalServerError: + reason = safetyMetricErrorReason(result.Error) + case errors.Is(result.Error, base.RecoveryNotCompleteError): + reason = "recovery" + case errors.Is(result.Error, base.ThresholdExceededError) && result.Value > result.Threshold: + reason = "threshold" + default: + reason = "other" + } + } + name := fmt.Sprintf("check.rejection.mysql.%s.dependency.%s.status.%d.%s.total", + clusterName, dependency, result.StatusCode, reason) + metrics.GetOrRegisterCounter(name, nil).Inc(1) + if check.throttler.safetyRejectionLogs.Add(name, true, cache.DefaultExpiration) == nil { + log.Infof("event=freno_admission_rejected cluster=%q dependency=%q status=%d reason=%q value=%g threshold=%g", + clusterName, dependency, result.StatusCode, reason, result.Value, result.Threshold) + } +} + +func reportRecoveryTransition(clusterName, event, reason string, value float64, settings *config.MySQLClusterConfigurationSettings) { + // check.* counters are already collected as rates by the production Go Expvar integration. + name := fmt.Sprintf("check.recovery.mysql.%s.%s.total", clusterName, event) + metrics.GetOrRegisterCounter(name, nil).Inc(1) + recoveryThreshold := settings.ThrottleThreshold + if settings.RecoveryThreshold != nil { + recoveryThreshold = *settings.RecoveryThreshold + } + log.Infof("event=freno_recovery_%s cluster=%q reason=%q value=%g threshold=%g recovery_threshold=%g recovery_duration_ms=%d", + event, clusterName, reason, value, settings.ThrottleThreshold, recoveryThreshold, settings.RecoveryDurationMillis) +} diff --git a/pkg/throttle/safety_telemetry_test.go b/pkg/throttle/safety_telemetry_test.go new file mode 100644 index 0000000..adf220b --- /dev/null +++ b/pkg/throttle/safety_telemetry_test.go @@ -0,0 +1,206 @@ +package throttle + +import ( + "errors" + "fmt" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/github/freno/pkg/base" + "github.com/github/freno/pkg/config" + "github.com/outbrain/golib/log" + metrics "github.com/rcrowley/go-metrics" + "github.com/stretchr/testify/assert" +) + +func captureSafetyLogs(t *testing.T) func() string { + t.Helper() + file, err := os.CreateTemp(t.TempDir(), "safety-logs") + if err != nil { + t.Fatal(err) + } + stderr, level := os.Stderr, log.GetLevel() + os.Stderr = file + log.SetLevel(log.INFO) + t.Cleanup(func() { + os.Stderr = stderr + log.SetLevel(level) + file.Close() + }) + return func() string { + data, err := os.ReadFile(file.Name()) + if err != nil { + t.Fatal(err) + } + return string(data) + } +} + +func clearSafetyCounters(t *testing.T) { + t.Helper() + clear := func() { + var names []string + metrics.DefaultRegistry.Each(func(name string, _ interface{}) { + if strings.HasPrefix(name, "check.rejection.") || strings.HasPrefix(name, "check.recovery.") { + names = append(names, name) + } + }) + for _, name := range names { + metrics.DefaultRegistry.Unregister(name) + } + } + clear() + t.Cleanup(clear) +} + +func TestSafetyRejectionTelemetry(t *testing.T) { + original := config.Settings().Stores.MySQL + t.Cleanup(func() { config.Settings().Stores.MySQL = original }) + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + FallbackCluster: "telemetry-replicas", + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "telemetry-replicas": {RequiredClusters: []string{"telemetry-primary"}}, + "telemetry-primary": {FailOnNoHosts: true}, + }, + } + for _, test := range []struct { + name string + metric base.MetricResult + replica float64 + shared bool + app string + requested string + wantStatus int + wantReason string + dependency string + }{ + {name: "accepted", metric: base.NewSimpleMetricResult(10), wantStatus: http.StatusOK}, + {name: "primary threshold", metric: base.NewSimpleMetricResult(60), wantStatus: 429, wantReason: "threshold"}, + {name: "first blocker only", replica: 6, metric: base.NewSimpleMetricResult(60), wantStatus: 429, wantReason: "threshold", dependency: "telemetry-replicas"}, + {name: "recovery", metric: base.NewErrorMetricResult(35, base.RecoveryNotCompleteError), wantStatus: 429, wantReason: "recovery"}, + {name: "shared domain", shared: true, metric: base.NewSimpleMetricResult(10), wantStatus: 429, wantReason: "shared_domain"}, + {name: "missing measurement", metric: base.NoSuchMetric, wantStatus: 500, wantReason: "missing_metric"}, + {name: "no hosts", metric: base.NoHostsMetricResult, wantStatus: 500, wantReason: "no_hosts"}, + {name: "metric error", metric: base.NewErrorMetricResult(0, errors.New("private connection details")), wantStatus: 500, wantReason: "metric_error"}, + {name: "fallback is bounded", requested: "unbounded-caller-input", metric: base.NewSimpleMetricResult(60), wantStatus: 429, wantReason: "threshold"}, + {name: "internal check excluded", app: frenoAppName, metric: base.NewSimpleMetricResult(60), wantStatus: 429}, + {name: "shared internal check excluded", app: frenoShareDmainAppName, metric: base.NewSimpleMetricResult(60), wantStatus: 429}, + {name: "application denial excluded", app: "abusing-app", metric: base.NewSimpleMetricResult(10), wantStatus: 417}, + } { + t.Run(test.name, func(t *testing.T) { + clearSafetyCounters(t) + logs := captureSafetyLogs(t) + throttler := NewThrottler() + setTestMySQLClusterMetric(throttler, "telemetry-replicas", test.replica, 5) + setTestMySQLClusterMetric(throttler, "telemetry-primary", 10, 50) + throttler.aggregatedMetrics.SetDefault("mysql/telemetry-primary", test.metric) + if test.shared { + throttler.shareDomainMetricHealth.SetDefault("mysql/telemetry-primary", &base.MetricHealth{SecondsSinceLastHealthy: 1}) + } + app, requested := test.app, test.requested + if app == "" { + app = "test-app" + } + if requested == "" { + requested = "telemetry-replicas" + } + check := NewThrottlerCheck(throttler) + for attempt := 0; attempt < 2; attempt++ { + result := check.Check(app, "mysql", requested, "", StandardCheckFlags) + assert.Equal(t, test.wantStatus, result.StatusCode) + } + var counters []string + metrics.DefaultRegistry.Each(func(name string, _ interface{}) { + if strings.HasPrefix(name, "check.rejection.") { + counters = append(counters, name) + } + }) + if test.wantReason == "" { + assert.Empty(t, counters) + assert.NotContains(t, logs(), "event=freno_admission_rejected") + return + } + dependency := test.dependency + if dependency == "" { + dependency = "telemetry-primary" + } + name := fmt.Sprintf("check.rejection.mysql.telemetry-replicas.dependency.%s.status.%d.%s.total", dependency, test.wantStatus, test.wantReason) + if !assert.Equal(t, []string{name}, counters) { + return + } + assert.Equal(t, int64(2), metrics.Get(name).(metrics.Counter).Count()) + assert.Equal(t, 1, strings.Count(logs(), "event=freno_admission_rejected")) + assert.Contains(t, logs(), fmt.Sprintf("reason=%q", test.wantReason)) + assert.Contains(t, logs(), fmt.Sprintf("dependency=%q", dependency)) + assert.NotContains(t, logs(), "private connection details") + assert.NotContains(t, logs(), "unbounded-caller-input") + // The next interval logs again without suppressing any request counters. + throttler.safetyRejectionLogs.Delete(name) + check.Check(app, "mysql", requested, "", StandardCheckFlags) + assert.Equal(t, int64(3), metrics.Get(name).(metrics.Counter).Count()) + assert.Equal(t, 2, strings.Count(logs(), "event=freno_admission_rejected")) + }) + } +} + +func TestRecoveryTransitionTelemetry(t *testing.T) { + clearSafetyCounters(t) + logs := captureSafetyLogs(t) + original := config.Settings().Stores.MySQL + t.Cleanup(func() { config.Settings().Stores.MySQL = original }) + recoveryThreshold := 40.0 + config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{ + Clusters: map[string]*config.MySQLClusterConfigurationSettings{ + "telemetry-primary": { + ThrottleThreshold: 50, RecoveryThreshold: &recoveryThreshold, + RecoveryDurationMillis: 5000, FailOnNoHosts: true, + }, + }, + } + throttler := NewThrottler() + start := time.Date(2026, time.November, 20, 0, 0, 0, 0, time.UTC) + for _, step := range []struct { + second int + value float64 + err error + want error + }{ + {0, 30, nil, base.RecoveryNotCompleteError}, + {1, 30, nil, base.RecoveryNotCompleteError}, + {2, 45, nil, base.RecoveryNotCompleteError}, + {3, 30, nil, base.RecoveryNotCompleteError}, + {4, 0, base.NoHostsError, base.NoHostsError}, + {5, 0, base.NoHostsError, base.NoHostsError}, + {6, 30, nil, base.RecoveryNotCompleteError}, + {11, 30, nil, nil}, + {12, 60, nil, nil}, + {13, 60, nil, nil}, + {14, 30, nil, base.RecoveryNotCompleteError}, + {19, 30, nil, nil}, + } { + result := throttler.stabilizeMySQLMetric("telemetry-primary", base.NewErrorMetricResult(step.value, step.err), start.Add(time.Duration(step.second)*time.Second)) + value, err := result.Get() + assert.Equal(t, step.value, value) + assert.Equal(t, step.want, err) + } + throttler.isLeader = true + throttler.updateLeadership(false) + throttler.updateLeadership(true) + result := throttler.stabilizeMySQLMetric("telemetry-primary", base.NewSimpleMetricResult(30), start.Add(20*time.Second)) + _, err := result.Get() + assert.ErrorIs(t, err, base.RecoveryNotCompleteError) + for event, want := range map[string]int64{"initialized": 2, "started": 5, "reset": 2, "completed": 2, "blocked": 1} { + name := "check.recovery.mysql.telemetry-primary." + event + ".total" + if !assert.NotNil(t, metrics.Get(name), name) { + continue + } + assert.Equal(t, want, metrics.Get(name).(metrics.Counter).Count(), event) + assert.Equal(t, int(want), strings.Count(logs(), "event=freno_recovery_"+event+" "), event) + } + assert.Contains(t, logs(), `reason="above_recovery_threshold"`) + assert.Contains(t, logs(), `reason="no_hosts"`) + assert.Contains(t, logs(), "recovery_duration_ms=5000") +} diff --git a/pkg/throttle/throttler.go b/pkg/throttle/throttler.go index 89778e4..6499c1c 100644 --- a/pkg/throttle/throttler.go +++ b/pkg/throttle/throttler.go @@ -74,10 +74,11 @@ type Throttler struct { proxysqlClient *proxysql.Client - throttledAppsMutex sync.Mutex - skippedHostsMutex sync.Mutex - mysqlRecoveryMutex sync.Mutex - mysqlRecoveryState map[string]*mysqlClusterRecoveryState + throttledAppsMutex sync.Mutex + skippedHostsMutex sync.Mutex + mysqlRecoveryMutex sync.Mutex + mysqlRecoveryState map[string]*mysqlClusterRecoveryState + safetyRejectionLogs *cache.Cache nonLowPriorityAppRequestsThrottled *cache.Cache httpClient *http.Client @@ -107,6 +108,7 @@ func NewThrottler() *Throttler { metricsHealth: cache.New(cache.NoExpiration, 0), shareDomainMetricHealth: cache.New(5*sharedDomainCollectInterval, sharedDomainCollectInterval), mysqlRecoveryState: make(map[string]*mysqlClusterRecoveryState), + safetyRejectionLogs: cache.New(time.Minute, time.Minute), nonLowPriorityAppRequestsThrottled: cache.New(nonDeprioritizedAppMapExpiration, nonDeprioritizedAppMapInterval), @@ -548,9 +550,19 @@ func (throttler *Throttler) stabilizeMySQLMetric(clusterName string, metricResul // was healthy before this process began observing it. state = &mysqlClusterRecoveryState{throttled: true} throttler.mysqlRecoveryState[clusterName] = state + reportRecoveryTransition(clusterName, "initialized", "startup_or_leadership", value, clusterSettings) } if err != nil || value > clusterSettings.ThrottleThreshold { + reason := "threshold" + if err != nil { + reason = safetyMetricErrorReason(err) + } + if !state.throttled { + reportRecoveryTransition(clusterName, "blocked", reason, value, clusterSettings) + } else if !state.healthySince.IsZero() { + reportRecoveryTransition(clusterName, "reset", reason, value, clusterSettings) + } state.throttled = true state.healthySince = time.Time{} metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1) @@ -566,12 +578,16 @@ func (throttler *Throttler) stabilizeMySQLMetric(clusterName string, metricResul recoveryThreshold = *clusterSettings.RecoveryThreshold } if value > recoveryThreshold { + if !state.healthySince.IsZero() { + reportRecoveryTransition(clusterName, "reset", "above_recovery_threshold", value, clusterSettings) + } state.healthySince = time.Time{} metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1) return base.NewErrorMetricResult(value, base.RecoveryNotCompleteError) } if state.healthySince.IsZero() { state.healthySince = now + reportRecoveryTransition(clusterName, "started", "below_recovery_threshold", value, clusterSettings) metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(1) return base.NewErrorMetricResult(value, base.RecoveryNotCompleteError) } @@ -584,6 +600,7 @@ func (throttler *Throttler) stabilizeMySQLMetric(clusterName string, metricResul state.throttled = false state.healthySince = time.Time{} + reportRecoveryTransition(clusterName, "completed", "healthy_window_complete", value, clusterSettings) metrics.GetOrRegisterGauge(fmt.Sprintf("recovery.mysql.%s.active", clusterName), nil).Update(0) return metricResult }