Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions doc/mysql.md
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,45 @@ checks do not increment these counters. These are process-local cumulative
request counters that reset on restart, not counts of overloaded probe readings
or distinct overload episodes. They do not measure memcache client rejections.

#### Safety telemetry and logs

The existing rejection counters above are unchanged. Additional counters record
non-internal MySQL checks returning `429` or `500`, with the configured requested
cluster, first blocking dependency, status, and reason:

```text
check.rejection.mysql.prod4.dependency.prod4-primary.status.429.threshold.total
check.rejection.mysql.prod4.dependency.prod4-primary.status.500.no_hosts.total
```

Reasons are `threshold`, `recovery`, `shared_domain`, `no_hosts`,
`missing_metric`, `metric_error`, or `other`. `metric_error` includes probe
failures and measurements that have not been collected; it does not establish
the underlying cause. Fallback requests use the configured fallback cluster,
not arbitrary names supplied by callers. These counters do not include
application identities or memcache rejections.

Recovery-enabled clusters also emit
`check.recovery.mysql.<cluster>.<event>.total`, where events are `initialized`,
`blocked`, `started`, `reset`, and `completed`. Initialization means a new
process or leader is requiring a healthy window, not that overload occurred.
Repeated readings in the same state do not increment transition counters.
The existing `recovery.mysql.<cluster>.active` gauge is unchanged.

Both new counter families use `check.*` so the existing production Go Expvar
configuration collects them as rates. Local counters are cumulative since
startup; Datadog rates are events per second, not cumulative totals.

At INFO level, recovery transitions emit `event=freno_recovery_<event>` with
the cluster, reason, value, thresholds, and configured recovery duration.
Rejections emit `event=freno_admission_rejected` with the requested cluster,
blocking dependency, HTTP status, reason, value, and threshold. Rejection logs
are limited to one per minute per cluster/dependency/status/reason combination;
every eligible rejection still increments its counter. Logs omit raw error
messages, credentials, and request contents. Standard error follows the existing
log collection path; this change does not configure Splunk or repair Datadog
delivery gaps.

#### Configuration example

A cluster can require additional cluster metrics to pass before `freno` permits
Expand Down
5 changes: 5 additions & 0 deletions pkg/throttle/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ func (check *ThrottlerCheck) checkAppMetricResult(appName string, storeType stri
// shared-domain dependency check.,

checkResult = NewCheckResult(http.StatusTooManyRequests, value, threshold, base.ThresholdExceededError)
checkResult.rejectionReason = "shared_domain"
}
checkResult.MetricName = metricName
return checkResult
Expand Down Expand Up @@ -121,6 +122,7 @@ func (check *ThrottlerCheck) checkMySQLCluster(
fmt.Errorf("required MySQL cluster metric %q is unavailable", requiredCluster),
)
checkResult.MetricName = fmt.Sprintf("mysql/%s", requiredCluster)
checkResult.rejectionReason = "missing_metric"
return checkResult
}
return requiredResult
Expand Down Expand Up @@ -171,6 +173,9 @@ func (check *ThrottlerCheck) Check(appName string, storeType string, storeName s
metricName := strings.ReplaceAll(checkResult.MetricName, "/", ".")
metrics.GetOrRegisterCounter(fmt.Sprintf("check.rejected.%s.%s.total", metricName, reason), nil).Inc(1)
}
if storeType == "mysql" {
check.recordMySQLRejection(appName, storeName, checkResult)
}

go func(statusCode int) {
metrics.GetOrRegisterCounter("check.any.total", nil).Inc(1)
Expand Down
13 changes: 7 additions & 6 deletions pkg/throttle/check_result.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,13 @@ func newMetricCheckResult(value, threshold float64, err error, failOnNoHosts boo

// CheckResult is the result for an app inquiring on a metric. It also exports as JSON via the API
type CheckResult struct {
StatusCode int `json:"StatusCode"`
Value float64 `json:"Value"`
Threshold float64 `json:"Threshold"`
MetricName string `json:"MetricName,omitempty"`
Error error `json:"-"`
Message string `json:"Message"`
StatusCode int `json:"StatusCode"`
Value float64 `json:"Value"`
Threshold float64 `json:"Threshold"`
MetricName string `json:"MetricName,omitempty"`
Error error `json:"-"`
Message string `json:"Message"`
rejectionReason string
}

func NewCheckResult(statusCode int, value float64, threshold float64, err error) *CheckResult {
Expand Down
72 changes: 72 additions & 0 deletions pkg/throttle/safety_telemetry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package throttle

import (
"errors"
"fmt"
"net/http"
"strings"

"github.com/github/freno/pkg/base"
"github.com/github/freno/pkg/config"
"github.com/outbrain/golib/log"
"github.com/patrickmn/go-cache"
metrics "github.com/rcrowley/go-metrics"
)

func safetyMetricErrorReason(err error) string {
switch {
case errors.Is(err, base.NoHostsError):
return "no_hosts"
case errors.Is(err, base.NoSuchMetricError):
return "missing_metric"
case errors.Is(err, base.RecoveryNotCompleteError):
return "recovery"
default:
return "metric_error"
}
}

func (check *ThrottlerCheck) recordMySQLRejection(appName, clusterName string, result *CheckResult) {
if appName == frenoAppName || appName == frenoShareDmainAppName ||
(result.StatusCode != http.StatusTooManyRequests && result.StatusCode != http.StatusInternalServerError) {
return
}
// Use configured names, including the resolved fallback, rather than unbounded caller input.
clusters := config.Settings().Stores.MySQL.Clusters
dependency := strings.TrimPrefix(result.MetricName, "mysql/")
if clusters[clusterName] == nil || clusters[dependency] == nil {
return
}
reason := result.rejectionReason
if reason == "" {
switch {
case result.StatusCode == http.StatusInternalServerError:
reason = safetyMetricErrorReason(result.Error)
case errors.Is(result.Error, base.RecoveryNotCompleteError):
reason = "recovery"
case errors.Is(result.Error, base.ThresholdExceededError) && result.Value > result.Threshold:
reason = "threshold"
default:
reason = "other"
}
}
name := fmt.Sprintf("check.rejection.mysql.%s.dependency.%s.status.%d.%s.total",
clusterName, dependency, result.StatusCode, reason)
metrics.GetOrRegisterCounter(name, nil).Inc(1)
if check.throttler.safetyRejectionLogs.Add(name, true, cache.DefaultExpiration) == nil {
log.Infof("event=freno_admission_rejected cluster=%q dependency=%q status=%d reason=%q value=%g threshold=%g",
clusterName, dependency, result.StatusCode, reason, result.Value, result.Threshold)
}
}

func reportRecoveryTransition(clusterName, event, reason string, value float64, settings *config.MySQLClusterConfigurationSettings) {
// check.* counters are already collected as rates by the production Go Expvar integration.
name := fmt.Sprintf("check.recovery.mysql.%s.%s.total", clusterName, event)
metrics.GetOrRegisterCounter(name, nil).Inc(1)
recoveryThreshold := settings.ThrottleThreshold
if settings.RecoveryThreshold != nil {
recoveryThreshold = *settings.RecoveryThreshold
}
log.Infof("event=freno_recovery_%s cluster=%q reason=%q value=%g threshold=%g recovery_threshold=%g recovery_duration_ms=%d",
event, clusterName, reason, value, settings.ThrottleThreshold, recoveryThreshold, settings.RecoveryDurationMillis)
}
206 changes: 206 additions & 0 deletions pkg/throttle/safety_telemetry_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
package throttle

import (
"errors"
"fmt"
"net/http"
"os"
"strings"
"testing"
"time"

"github.com/github/freno/pkg/base"
"github.com/github/freno/pkg/config"
"github.com/outbrain/golib/log"
metrics "github.com/rcrowley/go-metrics"
"github.com/stretchr/testify/assert"
)

func captureSafetyLogs(t *testing.T) func() string {
t.Helper()
file, err := os.CreateTemp(t.TempDir(), "safety-logs")
if err != nil {
t.Fatal(err)
}
stderr, level := os.Stderr, log.GetLevel()
os.Stderr = file
log.SetLevel(log.INFO)
t.Cleanup(func() {
os.Stderr = stderr
log.SetLevel(level)
file.Close()
})
return func() string {
data, err := os.ReadFile(file.Name())
if err != nil {
t.Fatal(err)
}
return string(data)
}
}

func clearSafetyCounters(t *testing.T) {
t.Helper()
clear := func() {
var names []string
metrics.DefaultRegistry.Each(func(name string, _ interface{}) {
if strings.HasPrefix(name, "check.rejection.") || strings.HasPrefix(name, "check.recovery.") {
names = append(names, name)
}
})
for _, name := range names {
metrics.DefaultRegistry.Unregister(name)
}
}
clear()
t.Cleanup(clear)
}

func TestSafetyRejectionTelemetry(t *testing.T) {
original := config.Settings().Stores.MySQL
t.Cleanup(func() { config.Settings().Stores.MySQL = original })
config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{
FallbackCluster: "telemetry-replicas",
Clusters: map[string]*config.MySQLClusterConfigurationSettings{
"telemetry-replicas": {RequiredClusters: []string{"telemetry-primary"}},
"telemetry-primary": {FailOnNoHosts: true},
},
}
for _, test := range []struct {
name string
metric base.MetricResult
replica float64
shared bool
app string
requested string
wantStatus int
wantReason string
dependency string
}{
{name: "accepted", metric: base.NewSimpleMetricResult(10), wantStatus: http.StatusOK},
{name: "primary threshold", metric: base.NewSimpleMetricResult(60), wantStatus: 429, wantReason: "threshold"},
{name: "first blocker only", replica: 6, metric: base.NewSimpleMetricResult(60), wantStatus: 429, wantReason: "threshold", dependency: "telemetry-replicas"},
{name: "recovery", metric: base.NewErrorMetricResult(35, base.RecoveryNotCompleteError), wantStatus: 429, wantReason: "recovery"},
{name: "shared domain", shared: true, metric: base.NewSimpleMetricResult(10), wantStatus: 429, wantReason: "shared_domain"},
{name: "missing measurement", metric: base.NoSuchMetric, wantStatus: 500, wantReason: "missing_metric"},
{name: "no hosts", metric: base.NoHostsMetricResult, wantStatus: 500, wantReason: "no_hosts"},
{name: "metric error", metric: base.NewErrorMetricResult(0, errors.New("private connection details")), wantStatus: 500, wantReason: "metric_error"},
{name: "fallback is bounded", requested: "unbounded-caller-input", metric: base.NewSimpleMetricResult(60), wantStatus: 429, wantReason: "threshold"},
{name: "internal check excluded", app: frenoAppName, metric: base.NewSimpleMetricResult(60), wantStatus: 429},
{name: "shared internal check excluded", app: frenoShareDmainAppName, metric: base.NewSimpleMetricResult(60), wantStatus: 429},
{name: "application denial excluded", app: "abusing-app", metric: base.NewSimpleMetricResult(10), wantStatus: 417},
} {
t.Run(test.name, func(t *testing.T) {
clearSafetyCounters(t)
logs := captureSafetyLogs(t)
throttler := NewThrottler()
setTestMySQLClusterMetric(throttler, "telemetry-replicas", test.replica, 5)
setTestMySQLClusterMetric(throttler, "telemetry-primary", 10, 50)
throttler.aggregatedMetrics.SetDefault("mysql/telemetry-primary", test.metric)
if test.shared {
throttler.shareDomainMetricHealth.SetDefault("mysql/telemetry-primary", &base.MetricHealth{SecondsSinceLastHealthy: 1})
}
app, requested := test.app, test.requested
if app == "" {
app = "test-app"
}
if requested == "" {
requested = "telemetry-replicas"
}
check := NewThrottlerCheck(throttler)
for attempt := 0; attempt < 2; attempt++ {
result := check.Check(app, "mysql", requested, "", StandardCheckFlags)
assert.Equal(t, test.wantStatus, result.StatusCode)
}
var counters []string
metrics.DefaultRegistry.Each(func(name string, _ interface{}) {
if strings.HasPrefix(name, "check.rejection.") {
counters = append(counters, name)
}
})
if test.wantReason == "" {
assert.Empty(t, counters)
assert.NotContains(t, logs(), "event=freno_admission_rejected")
return
}
dependency := test.dependency
if dependency == "" {
dependency = "telemetry-primary"
}
name := fmt.Sprintf("check.rejection.mysql.telemetry-replicas.dependency.%s.status.%d.%s.total", dependency, test.wantStatus, test.wantReason)
if !assert.Equal(t, []string{name}, counters) {
return
}
assert.Equal(t, int64(2), metrics.Get(name).(metrics.Counter).Count())
assert.Equal(t, 1, strings.Count(logs(), "event=freno_admission_rejected"))
assert.Contains(t, logs(), fmt.Sprintf("reason=%q", test.wantReason))
assert.Contains(t, logs(), fmt.Sprintf("dependency=%q", dependency))
assert.NotContains(t, logs(), "private connection details")
assert.NotContains(t, logs(), "unbounded-caller-input")
// The next interval logs again without suppressing any request counters.
throttler.safetyRejectionLogs.Delete(name)
check.Check(app, "mysql", requested, "", StandardCheckFlags)
assert.Equal(t, int64(3), metrics.Get(name).(metrics.Counter).Count())
assert.Equal(t, 2, strings.Count(logs(), "event=freno_admission_rejected"))
})
}
}

func TestRecoveryTransitionTelemetry(t *testing.T) {
clearSafetyCounters(t)
logs := captureSafetyLogs(t)
original := config.Settings().Stores.MySQL
t.Cleanup(func() { config.Settings().Stores.MySQL = original })
recoveryThreshold := 40.0
config.Settings().Stores.MySQL = config.MySQLConfigurationSettings{
Clusters: map[string]*config.MySQLClusterConfigurationSettings{
"telemetry-primary": {
ThrottleThreshold: 50, RecoveryThreshold: &recoveryThreshold,
RecoveryDurationMillis: 5000, FailOnNoHosts: true,
},
},
}
throttler := NewThrottler()
start := time.Date(2026, time.November, 20, 0, 0, 0, 0, time.UTC)
for _, step := range []struct {
second int
value float64
err error
want error
}{
{0, 30, nil, base.RecoveryNotCompleteError},
{1, 30, nil, base.RecoveryNotCompleteError},
{2, 45, nil, base.RecoveryNotCompleteError},
{3, 30, nil, base.RecoveryNotCompleteError},
{4, 0, base.NoHostsError, base.NoHostsError},
{5, 0, base.NoHostsError, base.NoHostsError},
{6, 30, nil, base.RecoveryNotCompleteError},
{11, 30, nil, nil},
{12, 60, nil, nil},
{13, 60, nil, nil},
{14, 30, nil, base.RecoveryNotCompleteError},
{19, 30, nil, nil},
} {
result := throttler.stabilizeMySQLMetric("telemetry-primary", base.NewErrorMetricResult(step.value, step.err), start.Add(time.Duration(step.second)*time.Second))
value, err := result.Get()
assert.Equal(t, step.value, value)
assert.Equal(t, step.want, err)
}
throttler.isLeader = true
throttler.updateLeadership(false)
throttler.updateLeadership(true)
result := throttler.stabilizeMySQLMetric("telemetry-primary", base.NewSimpleMetricResult(30), start.Add(20*time.Second))
_, err := result.Get()
assert.ErrorIs(t, err, base.RecoveryNotCompleteError)
for event, want := range map[string]int64{"initialized": 2, "started": 5, "reset": 2, "completed": 2, "blocked": 1} {
name := "check.recovery.mysql.telemetry-primary." + event + ".total"
if !assert.NotNil(t, metrics.Get(name), name) {
continue
}
assert.Equal(t, want, metrics.Get(name).(metrics.Counter).Count(), event)
assert.Equal(t, int(want), strings.Count(logs(), "event=freno_recovery_"+event+" "), event)
}
assert.Contains(t, logs(), `reason="above_recovery_threshold"`)
assert.Contains(t, logs(), `reason="no_hosts"`)
assert.Contains(t, logs(), "recovery_duration_ms=5000")
}
Loading