diff --git a/CHANGELOG.md b/CHANGELOG.md index 969e6c8f3b5..c926d497d86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master / unreleased +* [CHANGE] Distributor/Ingester: Add `type` label to `cortex_discarded_samples_total` metric to distinguish between float and native histogram samples. #7898 * [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828 * [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873 * [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539 diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index b938247ccee..3725a8f6238 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -689,7 +689,7 @@ func (d *Distributor) checkSample(ctx context.Context, userID, cluster, replica func (d *Distributor) validateSeries(ts cortexpb.PreallocTimeseries, userID string, skipLabelNameValidation bool, limits *validation.Limits) (cortexpb.PreallocTimeseries, validation.ValidationError) { d.labelsHistogram.Observe(float64(len(ts.Labels))) - if err := validation.ValidateLabels(d.validateMetrics, limits, userID, ts.Labels, skipLabelNameValidation, d.cfg.NameValidationScheme); err != nil { + if err := validation.ValidateLabels(d.validateMetrics, limits, userID, ts.Labels, skipLabelNameValidation, d.cfg.NameValidationScheme, len(ts.Samples), len(ts.Histograms)); err != nil { return emptyPreallocSeries, err } @@ -698,7 +698,7 @@ func (d *Distributor) validateSeries(ts cortexpb.PreallocTimeseries, userID stri // Only alloc when data present samples = make([]cortexpb.Sample, 0, len(ts.Samples)) for _, s := range ts.Samples { - if err := validation.ValidateSampleTimestamp(d.validateMetrics, limits, userID, ts.Labels, s.TimestampMs); err != nil { + if err := validation.ValidateSampleTimestamp(d.validateMetrics, limits, userID, ts.Labels, s.TimestampMs, validation.SampleTypeFloat); err != nil { return emptyPreallocSeries, err } samples = append(samples, s) @@ -727,7 +727,7 @@ func (d *Distributor) validateSeries(ts cortexpb.PreallocTimeseries, userID stri histograms = make([]cortexpb.WrappedHistogram, 0, len(ts.Histograms)) receivedBucketsObserver := d.receivedHistogramBuckets.WithLabelValues(userID) for i, h := range ts.Histograms { - if err := validation.ValidateSampleTimestamp(d.validateMetrics, limits, userID, ts.Labels, h.TimestampMs); err != nil { + if err := validation.ValidateSampleTimestamp(d.validateMetrics, limits, userID, ts.Labels, h.TimestampMs, validation.SampleTypeHistogram); err != nil { return emptyPreallocSeries, err } receivedBucketsObserver.Observe(float64(h.BucketCount())) @@ -833,7 +833,12 @@ func (d *Distributor) Push(ctx context.Context, req *cortexpb.WriteRequest) (*co } if errors.Is(err, ha.TooManyReplicaGroupsError{}) { - d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID).Add(float64(numFloatSamples + numHistogramSamples)) + if numFloatSamples > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID, validation.SampleTypeFloat).Add(float64(numFloatSamples)) + } + if numHistogramSamples > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID, validation.SampleTypeHistogram).Add(float64(numHistogramSamples)) + } return nil, httpgrpc.Errorf(http.StatusBadRequest, "%s", err.Error()) } return nil, err @@ -864,7 +869,12 @@ func (d *Distributor) Push(ctx context.Context, req *cortexpb.WriteRequest) (*co totalSamples := validatedFloatSamples + validatedHistogramSamples totalN := totalSamples + validatedExemplars + len(validatedMetadata) if !d.ingestionRateLimiter.AllowN(now, userID, totalN) { - d.validateMetrics.DiscardedSamples.WithLabelValues(validation.RateLimited, userID).Add(float64(totalSamples)) + if validatedFloatSamples > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.RateLimited, userID, validation.SampleTypeFloat).Add(float64(validatedFloatSamples)) + } + if validatedHistogramSamples > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.RateLimited, userID, validation.SampleTypeHistogram).Add(float64(validatedHistogramSamples)) + } d.validateMetrics.DiscardedExemplars.WithLabelValues(validation.RateLimited, userID).Add(float64(validatedExemplars)) d.validateMetrics.DiscardedMetadata.WithLabelValues(validation.RateLimited, userID).Add(float64(len(validatedMetadata))) // Return a 429 here to tell the client it is going too fast. @@ -879,7 +889,7 @@ func (d *Distributor) Push(ctx context.Context, req *cortexpb.WriteRequest) (*co var nativeHistogramErr error if !d.nativeHistogramIngestionRateLimiter.AllowN(now, userID, validatedHistogramSamples) { - d.validateMetrics.DiscardedSamples.WithLabelValues(validation.NativeHistogramRateLimited, userID).Add(float64(validatedHistogramSamples)) + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.NativeHistogramRateLimited, userID, validation.SampleTypeHistogram).Add(float64(validatedHistogramSamples)) nativeHistogramErr = httpgrpc.Errorf(http.StatusTooManyRequests, "native histogram ingestion rate limit (%v) exceeded while adding %d native histogram samples", d.nativeHistogramIngestionRateLimiter.Limit(now, userID), validatedHistogramSamples) validatedHistogramSamples = 0 } else { @@ -1121,7 +1131,12 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write d.dedupedSamples.WithLabelValues(userID, cluster).Add(float64(len(ts.Samples) + len(ts.Histograms))) } if errors.Is(err, ha.TooManyReplicaGroupsError{}) { - d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID).Add(float64(len(ts.Samples) + len(ts.Histograms))) + if len(ts.Samples) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID, validation.SampleTypeFloat).Add(float64(len(ts.Samples))) + } + if len(ts.Histograms) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID, validation.SampleTypeHistogram).Add(float64(len(ts.Histograms))) + } } continue @@ -1145,10 +1160,20 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write l, _ := relabel.Process(cortexpb.FromLabelAdaptersToLabels(ts.Labels), mrc...) if l.Len() == 0 { // all labels are gone, samples will be discarded - d.validateMetrics.DiscardedSamples.WithLabelValues( - validation.DroppedByRelabelConfiguration, - userID, - ).Add(float64(len(ts.Samples) + len(ts.Histograms))) + if len(ts.Samples) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues( + validation.DroppedByRelabelConfiguration, + userID, + validation.SampleTypeFloat, + ).Add(float64(len(ts.Samples))) + } + if len(ts.Histograms) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues( + validation.DroppedByRelabelConfiguration, + userID, + validation.SampleTypeHistogram, + ).Add(float64(len(ts.Histograms))) + } // all labels are gone, exemplars will be discarded d.validateMetrics.DiscardedExemplars.WithLabelValues( @@ -1173,12 +1198,16 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write // Reject series with missing or empty metric name before removeEmptyLabels (which would strip __name__=""). if validationErr, reason := validation.ValidateMetricName(limits, ts.Labels, d.cfg.NameValidationScheme); reason != "" { - samplesCount := float64(len(ts.Samples) + len(ts.Histograms)) exemplarsCount := float64(len(ts.Exemplars)) if firstPartialErr == nil { firstPartialErr = httpgrpc.Errorf(http.StatusBadRequest, "%s", validationErr.Error()) } - d.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID).Add(samplesCount) + if len(ts.Samples) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, validation.SampleTypeFloat).Add(float64(len(ts.Samples))) + } + if len(ts.Histograms) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, validation.SampleTypeHistogram).Add(float64(len(ts.Histograms))) + } d.validateMetrics.DiscardedExemplars.WithLabelValues(reason, userID).Add(exemplarsCount) continue } @@ -1187,10 +1216,20 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write removeEmptyLabels(&ts.Labels) if len(ts.Labels) == 0 { - d.validateMetrics.DiscardedSamples.WithLabelValues( - validation.DroppedByUserConfigurationOverride, - userID, - ).Add(float64(len(ts.Samples) + len(ts.Histograms))) + if len(ts.Samples) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues( + validation.DroppedByUserConfigurationOverride, + userID, + validation.SampleTypeFloat, + ).Add(float64(len(ts.Samples))) + } + if len(ts.Histograms) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues( + validation.DroppedByUserConfigurationOverride, + userID, + validation.SampleTypeHistogram, + ).Add(float64(len(ts.Histograms))) + } d.validateMetrics.DiscardedExemplars.WithLabelValues( validation.DroppedByUserConfigurationOverride, diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 79c95bc71a1..a0b5e3714a9 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -981,7 +981,7 @@ func TestDistributor_PushIngestionRateLimiter_Histograms(t *testing.T) { assert.Nil(t, response) } } - assert.Equal(t, float64(push.expectedNHDiscardedSampleMetricValue), testutil.ToFloat64(distributors[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.NativeHistogramRateLimited, "user"))) + assert.Equal(t, float64(push.expectedNHDiscardedSampleMetricValue), testutil.ToFloat64(distributors[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.NativeHistogramRateLimited, "user", validation.SampleTypeHistogram))) } }) } @@ -2142,7 +2142,11 @@ func TestDistributor_Push_LabelRemoval(t *testing.T) { _, err = ds[0].Push(ctx, req) require.NoError(t, err) - actualDiscardedSamples := testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByUserConfigurationOverride, "user")) + sampleType := validation.SampleTypeFloat + if histogram { + sampleType = validation.SampleTypeHistogram + } + actualDiscardedSamples := testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByUserConfigurationOverride, "user", sampleType)) actualDiscardedExemplars := testutil.ToFloat64(ds[0].validateMetrics.DiscardedExemplars.WithLabelValues(validation.DroppedByUserConfigurationOverride, "user")) require.Equal(t, float64(expectedDiscardedSamples), actualDiscardedSamples) require.Equal(t, float64(expectedDiscardedExemplars), actualDiscardedExemplars) @@ -4613,7 +4617,11 @@ func TestDistributor_Push_RelabelDropWillExportMetricOfDroppedSamples(t *testing assert.Equal(t, 1, len(timeseries)) } - require.Equal(t, testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByRelabelConfiguration, id)), float64(1)) + sampleType := validation.SampleTypeFloat + if histogramEnabled { + sampleType = validation.SampleTypeHistogram + } + require.Equal(t, testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByRelabelConfiguration, id, sampleType)), float64(1)) require.Equal(t, testutil.ToFloat64(ds[0].validateMetrics.DiscardedExemplars.WithLabelValues(validation.DroppedByRelabelConfiguration, id)), float64(2)) receivedFloatSamples := testutil.ToFloat64(ds[0].receivedSamples.WithLabelValues(id, "float")) receivedHistogramSamples := testutil.ToFloat64(ds[0].receivedSamples.WithLabelValues(id, "histogram")) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index d877b90a12f..5c883ac810b 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1415,6 +1415,30 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte reasonCounter := newLabelSetReasonCounters() + // discardedCounts tracks discarded samples for one reason, split by + // sample type. The ingester's append-failure handler is shared by the + // float and native histogram loops, so a single counter per reason + // would misattribute histogram discards to the float series. + type discardedCounts struct { + float int + histogram int + } + incDiscarded := func(c *discardedCounts, sampleType string) { + if sampleType == validation.SampleTypeHistogram { + c.histogram++ + } else { + c.float++ + } + } + emitDiscarded := func(reason string, c discardedCounts) { + if c.float > 0 { + i.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, validation.SampleTypeFloat).Add(float64(c.float)) + } + if c.histogram > 0 { + i.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, validation.SampleTypeHistogram).Add(float64(c.histogram)) + } + } + // Keep track of some stats which are tracked only if the samples will be // successfully committed var ( @@ -1427,14 +1451,14 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte startTimestampSampleAppendFailCount = 0 startTimestampHistogramAppendFailCount = 0 startAppend = time.Now() - sampleOutOfBoundsCount = 0 - sampleOutOfOrderCount = 0 - sampleTooOldCount = 0 - newValueForTimestampCount = 0 - perUserSeriesLimitCount = 0 + sampleOutOfBoundsCount discardedCounts + sampleOutOfOrderCount discardedCounts + sampleTooOldCount discardedCounts + newValueForTimestampCount discardedCounts + perUserSeriesLimitCount discardedCounts perUserNativeHistogramSeriesLimitCount = 0 - perLabelSetSeriesLimitCount = 0 - perMetricSeriesLimitCount = 0 + perLabelSetSeriesLimitCount discardedCounts + perMetricSeriesLimitCount discardedCounts discardedNativeHistogramCount = 0 // Only the ruler sets the write request source to RULE. @@ -1456,7 +1480,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte // isStaleNaN reports whether the failed sample carried the reserved stale NaN // payload. Callers appending native histograms always pass false, because a // staleness marker is only ever appended as a float sample. - handleAppendFailure = func(err error, timestampMs int64, lbls []cortexpb.LabelAdapter, copiedLabels labels.Labels, matchedLabelSetLimits []validation.LimitsPerLabelSet, isStaleNaN bool) (rollback bool) { + handleAppendFailure = func(err error, timestampMs int64, lbls []cortexpb.LabelAdapter, copiedLabels labels.Labels, matchedLabelSetLimits []validation.LimitsPerLabelSet, isStaleNaN bool, sampleType string) (rollback bool) { // A staleness marker from the ruler is internal bookkeeping rather than // tenant data. Prometheus' rules.Group.cleanupStaleSeries appends one for // every series a rule stopped producing, and treats out-of-order, too-old @@ -1474,7 +1498,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte // we actually ingested all samples which haven't failed. switch cause := errors.Cause(err); { case errors.Is(cause, storage.ErrOutOfBounds): - sampleOutOfBoundsCount++ + incDiscarded(&sampleOutOfBoundsCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(sampleOutOfBounds, userID, copiedLabels.Hash()) updateFirstPartial(func() error { return wrappedTSDBIngestErrWithHeadMaxTime(err, model.Time(timestampMs), model.Time(headMaxTime), lbls) @@ -1482,21 +1506,21 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte case errors.Is(cause, storage.ErrOutOfOrderSample): if !ruleStaleMarker { - sampleOutOfOrderCount++ + incDiscarded(&sampleOutOfOrderCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(sampleOutOfOrder, userID, copiedLabels.Hash()) } updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(timestampMs), lbls) }) case errors.Is(cause, storage.ErrDuplicateSampleForTimestamp): if !ruleStaleMarker { - newValueForTimestampCount++ + incDiscarded(&newValueForTimestampCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(newValueForTimestamp, userID, copiedLabels.Hash()) } updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(timestampMs), lbls) }) case errors.Is(cause, storage.ErrTooOldSample): if !ruleStaleMarker { - sampleTooOldCount++ + incDiscarded(&sampleTooOldCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(sampleTooOld, userID, copiedLabels.Hash()) } updateFirstPartial(func() error { @@ -1504,7 +1528,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte }) case errors.Is(cause, errMaxSeriesPerUserLimitExceeded): - perUserSeriesLimitCount++ + incDiscarded(&perUserSeriesLimitCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(perUserSeriesLimit, userID, copiedLabels.Hash()) updateFirstPartial(func() error { return makeLimitError(perUserSeriesLimit, i.limiter.FormatError(userID, cause, copiedLabels)) @@ -1517,14 +1541,14 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte }) case errors.Is(cause, errMaxSeriesPerMetricLimitExceeded): - perMetricSeriesLimitCount++ + incDiscarded(&perMetricSeriesLimitCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(perMetricSeriesLimit, userID, copiedLabels.Hash()) updateFirstPartial(func() error { return makeMetricLimitError(perMetricSeriesLimit, copiedLabels, i.limiter.FormatError(userID, cause, copiedLabels)) }) case errors.As(cause, &errMaxSeriesPerLabelSetLimitExceeded{}): - perLabelSetSeriesLimitCount++ + incDiscarded(&perLabelSetSeriesLimitCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(perLabelsetSeriesLimit, userID, copiedLabels.Hash()) for _, matchedLabelset := range matchedLabelSetLimits { i.validateMetrics.DiscardedSeriesPerLabelsetTracker.Track(userID, copiedLabels.Hash(), matchedLabelset.Hash, matchedLabelset.Id) @@ -1634,7 +1658,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte failedSamplesCount++ - if rollback := handleAppendFailure(err, s.TimestampMs, ts.Labels, copiedLabels, matchedLabelSetLimits, value.IsStaleNaN(s.Value)); !rollback { + if rollback := handleAppendFailure(err, s.TimestampMs, ts.Labels, copiedLabels, matchedLabelSetLimits, value.IsStaleNaN(s.Value), validation.SampleTypeFloat); !rollback { continue } // The error looks an issue on our side, so we should rollback. @@ -1692,7 +1716,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte failedHistogramsCount++ - if rollback := handleAppendFailure(err, hp.TimestampMs, ts.Labels, copiedLabels, matchedLabelSetLimits, false); !rollback { + if rollback := handleAppendFailure(err, hp.TimestampMs, ts.Labels, copiedLabels, matchedLabelSetLimits, false, validation.SampleTypeHistogram); !rollback { continue } // The error looks an issue on our side, so we should rollback. @@ -1784,33 +1808,19 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte i.metrics.ingestedExemplars.Add(float64(succeededExemplarsCount)) i.metrics.ingestedExemplarsFail.Add(float64(failedExemplarsCount)) - if sampleOutOfBoundsCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(sampleOutOfBounds, userID).Add(float64(sampleOutOfBoundsCount)) - } - if sampleOutOfOrderCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(sampleOutOfOrder, userID).Add(float64(sampleOutOfOrderCount)) - } - if sampleTooOldCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(sampleTooOld, userID).Add(float64(sampleTooOldCount)) - } - if newValueForTimestampCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(newValueForTimestamp, userID).Add(float64(newValueForTimestampCount)) - } - if perUserSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perUserSeriesLimit, userID).Add(float64(perUserSeriesLimitCount)) - } + emitDiscarded(sampleOutOfBounds, sampleOutOfBoundsCount) + emitDiscarded(sampleOutOfOrder, sampleOutOfOrderCount) + emitDiscarded(sampleTooOld, sampleTooOldCount) + emitDiscarded(newValueForTimestamp, newValueForTimestampCount) + emitDiscarded(perUserSeriesLimit, perUserSeriesLimitCount) if perUserNativeHistogramSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perUserNativeHistogramSeriesLimit, userID).Add(float64(perUserNativeHistogramSeriesLimitCount)) - } - if perMetricSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perMetricSeriesLimit, userID).Add(float64(perMetricSeriesLimitCount)) - } - if perLabelSetSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perLabelsetSeriesLimit, userID).Add(float64(perLabelSetSeriesLimitCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(perUserNativeHistogramSeriesLimit, userID, validation.SampleTypeHistogram).Add(float64(perUserNativeHistogramSeriesLimitCount)) } + emitDiscarded(perMetricSeriesLimit, perMetricSeriesLimitCount) + emitDiscarded(perLabelsetSeriesLimit, perLabelSetSeriesLimitCount) if !i.limits.EnableNativeHistograms(userID) && discardedNativeHistogramCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSample, userID).Add(float64(discardedNativeHistogramCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSample, userID, validation.SampleTypeHistogram).Add(float64(discardedNativeHistogramCount)) } for h, counter := range reasonCounter.counters { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index d9a5299e0aa..2a6f73f9e2e 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -351,7 +351,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{label2=\"value2\"}",reason="per_labelset_series_limit",user="1"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 2 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 2 # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. # TYPE cortex_ingester_limits_per_labelset gauge cortex_ingester_limits_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 @@ -402,7 +402,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{label2=\"value2\"}",reason="per_labelset_series_limit",user="1"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 2 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 2 # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. # TYPE cortex_ingester_limits_per_labelset gauge cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 2 @@ -454,7 +454,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{label2=\"value2\"}",reason="per_labelset_series_limit",user="1"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 3 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 3 # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. # TYPE cortex_ingester_limits_per_labelset gauge cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 2 @@ -852,7 +852,7 @@ func TestIngesterUserLimitExceeded(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_user_series_limit",user="1"} 1 + cortex_discarded_samples_total{reason="per_user_series_limit",type="float",user="1"} 1 `), "cortex_discarded_samples_total")) } @@ -1078,7 +1078,7 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_metric_series_limit",user="1"} 1 + cortex_discarded_samples_total{reason="per_metric_series_limit",type="float",user="1"} 1 `), "cortex_discarded_samples_total")) } @@ -1179,7 +1179,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="native-histogram-sample",user="test"} 1 + cortex_discarded_samples_total{reason="native-histogram-sample",type="histogram",user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -1512,7 +1512,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_tsdb_out_of_order_samples_total{type="histogram",user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-order",user="test"} 1 + cortex_discarded_samples_total{reason="sample-out-of-order",type="float",user="test"} 1 `, }, "ooo disabled, tenant staleness marker out of order is counted as a discarded sample": { @@ -1564,7 +1564,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_tsdb_out_of_order_samples_total{type="histogram",user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-order",user="test"} 1 + cortex_discarded_samples_total{reason="sample-out-of-order",type="float",user="test"} 1 `, }, "ooo disabled, should soft fail on sample out of order": { @@ -1630,7 +1630,8 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-order",user="test"} 2 + cortex_discarded_samples_total{reason="sample-out-of-order",type="float",user="test"} 1 + cortex_discarded_samples_total{reason="sample-out-of-order",type="histogram",user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -1688,7 +1689,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-bounds",user="test"} 2 + cortex_discarded_samples_total{reason="sample-out-of-bounds",type="float",user="test"} 2 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -1743,7 +1744,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-too-old",user="test"} 1 + cortex_discarded_samples_total{reason="sample-too-old",type="float",user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -1859,7 +1860,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_active_series{user="test"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-too-old",user="test"} 1 + cortex_discarded_samples_total{reason="sample-too-old",type="histogram",user="test"} 1 # HELP cortex_ingester_active_native_histogram_series Number of currently active native histogram series per user. # TYPE cortex_ingester_active_native_histogram_series gauge cortex_ingester_active_native_histogram_series{user="test"} 1 @@ -1967,7 +1968,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="new-value-for-timestamp",user="test"} 1 + cortex_discarded_samples_total{reason="new-value-for-timestamp",type="float",user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -7980,7 +7981,7 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"bar\"}",reason="per_labelset_series_limit",user="1"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 1 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 1 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Expect per labelset validate metrics cleaned up. @@ -7989,7 +7990,7 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 1 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 1 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) } diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index 31c4b1f34cd..01a0a4bb14c 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -25,6 +25,11 @@ import ( const ( discardReasonLabel = "reason" + // SampleTypeFloat indicates a float sample. + SampleTypeFloat = "float" + // SampleTypeHistogram indicates a native histogram sample. + SampleTypeHistogram = "histogram" + errMetadataMissingMetricName = "metadata missing metric name" errMetadataTooLong = "metadata '%s' value too long: %.200q metric %.200q" @@ -109,7 +114,7 @@ func NewValidateMetrics(r prometheus.Registerer) *ValidateMetrics { Name: "cortex_discarded_samples_total", Help: "The total number of samples that were discarded.", }, - []string{discardReasonLabel, "user"}, + []string{discardReasonLabel, "user", "type"}, ) registerCollector(r, discardedSamples) discardedSamplesPerLabelSet := prometheus.NewCounterVec( @@ -192,14 +197,14 @@ func NewValidateMetrics(r prometheus.Registerer) *ValidateMetrics { // Used in test only for now. func (m *ValidateMetrics) updateSamplesDiscardedForSeries(userID, reason string, labelSetLimits []LimitsPerLabelSet, lbls labels.Labels, count int) { matchedLimits := LimitsPerLabelSetsForSeries(labelSetLimits, lbls) - m.updateSamplesDiscarded(userID, reason, matchedLimits, count) + m.updateSamplesDiscarded(userID, reason, SampleTypeFloat, matchedLimits, count) } // updateSamplesDiscarded updates discarded samples and discarded samples per labelset for the provided reason. // The provided label set needs to be pre-filtered to match the series if applicable. // Used in test only for now. -func (m *ValidateMetrics) updateSamplesDiscarded(userID, reason string, labelSetLimits []LimitsPerLabelSet, count int) { - m.DiscardedSamples.WithLabelValues(reason, userID).Add(float64(count)) +func (m *ValidateMetrics) updateSamplesDiscarded(userID, reason, sampleType string, labelSetLimits []LimitsPerLabelSet, count int) { + m.DiscardedSamples.WithLabelValues(reason, userID, sampleType).Add(float64(count)) for _, limit := range labelSetLimits { m.LabelSetTracker.Track(userID, limit.Hash, limit.LabelSet) m.DiscardedSamplesPerLabelSet.WithLabelValues(reason, userID, limit.LabelSet.String()).Add(float64(count)) @@ -221,18 +226,31 @@ func (m *ValidateMetrics) UpdateLabelSet(userSet map[string]map[uint64]struct{}, }) } +// discardSamplesForSeries records one discarded-series event, attributed to +// each sample type present in the series. When the caller has no sample +// counts (e.g. direct unit tests), the event is attributed to float samples, +// matching historical behaviour. +func discardSamplesForSeries(validateMetrics *ValidateMetrics, reason, userID string, numFloatSamples, numHistogramSamples int) { + if numFloatSamples > 0 || numHistogramSamples == 0 { + validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, SampleTypeFloat).Inc() + } + if numHistogramSamples > 0 { + validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, SampleTypeHistogram).Inc() + } +} + // ValidateSampleTimestamp returns an err if the sample timestamp is invalid. // The returned error may retain the provided series labels. -func ValidateSampleTimestamp(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, timestampMs int64) ValidationError { +func ValidateSampleTimestamp(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, timestampMs int64, sampleType string) ValidationError { unsafeMetricName, _ := extract.UnsafeMetricNameFromLabelAdapters(ls) if limits.RejectOldSamples && model.Time(timestampMs) < model.Now().Add(-time.Duration(limits.RejectOldSamplesMaxAge)) { - validateMetrics.DiscardedSamples.WithLabelValues(greaterThanMaxSampleAge, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(greaterThanMaxSampleAge, userID, sampleType).Inc() return newSampleTimestampTooOldError(unsafeMetricName, timestampMs) } if model.Time(timestampMs) > model.Now().Add(time.Duration(limits.CreationGracePeriod)) { - validateMetrics.DiscardedSamples.WithLabelValues(tooFarInFuture, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(tooFarInFuture, userID, sampleType).Inc() return newSampleTimestampTooNewError(unsafeMetricName, timestampMs) } @@ -296,10 +314,10 @@ func ValidateMetricName(limits *Limits, ls []cortexpb.LabelAdapter, nameValidati // ValidateLabels returns an err if the labels are invalid. // The returned error may retain the provided series labels. // Callers must validate metric name (e.g. via ValidateMetricName) before calling this when EnforceMetricName is true. -func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, skipLabelNameValidation bool, nameValidationScheme model.ValidationScheme) ValidationError { +func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, skipLabelNameValidation bool, nameValidationScheme model.ValidationScheme, numFloatSamples, numHistogramSamples int) ValidationError { numLabelNames := len(ls) if numLabelNames > limits.MaxLabelNamesPerSeries { - validateMetrics.DiscardedSamples.WithLabelValues(maxLabelNamesPerSeries, userID).Inc() + discardSamplesForSeries(validateMetrics, maxLabelNamesPerSeries, userID, numFloatSamples, numHistogramSamples) return newTooManyLabelsError(ls, limits.MaxLabelNamesPerSeries) } @@ -311,21 +329,21 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str for _, l := range ls { if !skipLabelNameValidation && !nameValidationScheme.IsValidLabelName(l.Name) { - validateMetrics.DiscardedSamples.WithLabelValues(invalidLabel, userID).Inc() + discardSamplesForSeries(validateMetrics, invalidLabel, userID, numFloatSamples, numHistogramSamples) return newInvalidLabelError(ls, l.Name) } else if len(l.Name) > maxLabelNameLength { - validateMetrics.DiscardedSamples.WithLabelValues(labelNameTooLong, userID).Inc() + discardSamplesForSeries(validateMetrics, labelNameTooLong, userID, numFloatSamples, numHistogramSamples) return newLabelNameTooLongError(ls, l.Name, maxLabelNameLength) } else if len(l.Value) > maxLabelValueLength { - validateMetrics.DiscardedSamples.WithLabelValues(labelValueTooLong, userID).Inc() + discardSamplesForSeries(validateMetrics, labelValueTooLong, userID, numFloatSamples, numHistogramSamples) return newLabelValueTooLongError(ls, l.Name, l.Value, maxLabelValueLength) } else if cmp := strings.Compare(lastLabelName, l.Name); cmp >= 0 { if cmp == 0 { - validateMetrics.DiscardedSamples.WithLabelValues(duplicateLabelNames, userID).Inc() + discardSamplesForSeries(validateMetrics, duplicateLabelNames, userID, numFloatSamples, numHistogramSamples) return newDuplicatedLabelError(ls, l.Name) } - validateMetrics.DiscardedSamples.WithLabelValues(labelsNotSorted, userID).Inc() + discardSamplesForSeries(validateMetrics, labelsNotSorted, userID, numFloatSamples, numHistogramSamples) return newLabelsNotSortedError(ls, l.Name) } @@ -334,7 +352,7 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str } validateMetrics.LabelSizeBytes.WithLabelValues(userID).Observe(float64(labelsSizeBytes)) if maxLabelsSizeBytes > 0 && labelsSizeBytes > maxLabelsSizeBytes { - validateMetrics.DiscardedSamples.WithLabelValues(labelsSizeBytesExceeded, userID).Inc() + discardSamplesForSeries(validateMetrics, labelsSizeBytesExceeded, userID, numFloatSamples, numHistogramSamples) return labelSizeBytesExceededError(ls, labelsSizeBytes, maxLabelsSizeBytes) } return nil @@ -376,7 +394,7 @@ func ValidateMetadata(validateMetrics *ValidateMetrics, cfg *Limits, userID stri func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, histogramSample cortexpb.Histogram) (cortexpb.Histogram, error) { // sample size validation for native histogram if limits.MaxNativeHistogramSampleSizeBytes > 0 && histogramSample.Size() > limits.MaxNativeHistogramSampleSizeBytes { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSampleSizeBytesExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSampleSizeBytesExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramSampleSizeBytesExceededError(ls, histogramSample.Size(), limits.MaxNativeHistogramSampleSizeBytes) } @@ -386,7 +404,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u isExponentialSchema := schema >= histogram.ExponentialSchemaMin && schema <= histogram.ExponentialSchemaMax isValidSchema := isCustomBucketsSchema || isExponentialSchema if !isValidSchema { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalidSchema, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalidSchema, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramSchemaInvalidError(ls, int(schema)) } @@ -397,7 +415,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u if histogramSample.IsFloatHistogram() { fh := cortexpb.FloatHistogramProtoToFloatHistogram(histogramSample) if err := fh.Validate(); err != nil { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramInvalidError(ls, err) } @@ -409,7 +427,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u // Custom bucket cannot reduce resolution if isCustomBucketsSchema { if len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > limits.MaxNativeHistogramBuckets { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } return histogramSample, nil @@ -422,18 +440,18 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u // Exceed limit. if schema <= histogram.ExponentialSchemaMin { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } oBuckets := len(fh.PositiveBuckets) + len(fh.NegativeBuckets) for len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > limits.MaxNativeHistogramBuckets { if fh.Schema <= histogram.ExponentialSchemaMin { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } if err := fh.ReduceResolution(fh.Schema - 1); err != nil { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramInvalidError(ls, err) } } @@ -446,7 +464,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u h := cortexpb.HistogramProtoToHistogram(histogramSample) if err := h.Validate(); err != nil { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramInvalidError(ls, err) } @@ -458,7 +476,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u // Custom bucket cannot reduce resolution if isCustomBucketsSchema { if len(h.PositiveBuckets)+len(h.NegativeBuckets) > limits.MaxNativeHistogramBuckets { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } return histogramSample, nil @@ -470,17 +488,17 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u } // Exceed limit. if schema <= histogram.ExponentialSchemaMin { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } oBuckets := len(h.PositiveBuckets) + len(h.NegativeBuckets) for len(h.PositiveBuckets)+len(h.NegativeBuckets) > limits.MaxNativeHistogramBuckets { if h.Schema <= histogram.ExponentialSchemaMin { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } if err := h.ReduceResolution(h.Schema - 1); err != nil { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramInvalidError(ls, err) } } diff --git a/pkg/util/validation/validate_test.go b/pkg/util/validation/validate_test.go index 6a55e735a9a..66e547735e9 100644 --- a/pkg/util/validation/validate_test.go +++ b/pkg/util/validation/validate_test.go @@ -66,7 +66,7 @@ func TestValidateLabels_UTF8(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { - err := ValidateLabels(validateMetrics, cfg, userID, cortexpb.FromMetricsToLabelAdapters(test.metric), test.skipLabelNameValidation, model.UTF8Validation) + err := ValidateLabels(validateMetrics, cfg, userID, cortexpb.FromMetricsToLabelAdapters(test.metric), test.skipLabelNameValidation, model.UTF8Validation, 0, 0) assert.Equal(t, test.expectedErr, err, "wrong error") }) } @@ -187,22 +187,22 @@ func TestValidateLabels(t *testing.T) { nil, }, } { - err := ValidateLabels(validateMetrics, cfg, userID, cortexpb.FromMetricsToLabelAdapters(c.metric), c.skipLabelNameValidation, model.LegacyValidation) + err := ValidateLabels(validateMetrics, cfg, userID, cortexpb.FromMetricsToLabelAdapters(c.metric), c.skipLabelNameValidation, model.LegacyValidation, 0, 0) assert.Equal(t, c.err, err, "wrong error") } - validateMetrics.DiscardedSamples.WithLabelValues("random reason", "different user").Inc() + validateMetrics.DiscardedSamples.WithLabelValues("random reason", "different user", SampleTypeFloat).Inc() require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="label_invalid",user="testUser"} 1 - cortex_discarded_samples_total{reason="label_name_too_long",user="testUser"} 1 - cortex_discarded_samples_total{reason="label_value_too_long",user="testUser"} 1 - cortex_discarded_samples_total{reason="max_label_names_per_series",user="testUser"} 1 - cortex_discarded_samples_total{reason="labels_size_bytes_exceeded",user="testUser"} 1 + cortex_discarded_samples_total{reason="label_invalid",type="float",user="testUser"} 1 + cortex_discarded_samples_total{reason="label_name_too_long",type="float",user="testUser"} 1 + cortex_discarded_samples_total{reason="label_value_too_long",type="float",user="testUser"} 1 + cortex_discarded_samples_total{reason="max_label_names_per_series",type="float",user="testUser"} 1 + cortex_discarded_samples_total{reason="labels_size_bytes_exceeded",type="float",user="testUser"} 1 - cortex_discarded_samples_total{reason="random reason",user="different user"} 1 + cortex_discarded_samples_total{reason="random reason",type="float",user="different user"} 1 `), "cortex_discarded_samples_total")) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` @@ -218,7 +218,7 @@ func TestValidateLabels(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="random reason",user="different user"} 1 + cortex_discarded_samples_total{reason="random reason",type="float",user="different user"} 1 `), "cortex_discarded_samples_total")) } @@ -348,7 +348,7 @@ func TestValidateLabelOrder(t *testing.T) { {Name: model.MetricNameLabel, Value: "m"}, {Name: "b", Value: "b"}, {Name: "a", Value: "a"}, - }, false, model.LegacyValidation) + }, false, model.LegacyValidation, 0, 0) expected := newLabelsNotSortedError([]cortexpb.LabelAdapter{ {Name: model.MetricNameLabel, Value: "m"}, {Name: "b", Value: "b"}, @@ -359,7 +359,7 @@ func TestValidateLabelOrder(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="labels_not_sorted",user="testUser"} 1 + cortex_discarded_samples_total{reason="labels_not_sorted",type="float",user="testUser"} 1 `), "cortex_discarded_samples_total")) } @@ -375,7 +375,7 @@ func TestValidateLabelDuplication(t *testing.T) { actual := ValidateLabels(validateMetrics, cfg, userID, []cortexpb.LabelAdapter{ {Name: model.MetricNameLabel, Value: "a"}, {Name: model.MetricNameLabel, Value: "b"}, - }, false, model.LegacyValidation) + }, false, model.LegacyValidation, 0, 0) expected := newDuplicatedLabelError([]cortexpb.LabelAdapter{ {Name: model.MetricNameLabel, Value: "a"}, {Name: model.MetricNameLabel, Value: "b"}, @@ -386,7 +386,7 @@ func TestValidateLabelDuplication(t *testing.T) { {Name: model.MetricNameLabel, Value: "a"}, {Name: "a", Value: "a"}, {Name: "a", Value: "a"}, - }, false, model.LegacyValidation) + }, false, model.LegacyValidation, 0, 0) expected = newDuplicatedLabelError([]cortexpb.LabelAdapter{ {Name: model.MetricNameLabel, Value: "a"}, {Name: "a", Value: "a"}, @@ -397,7 +397,7 @@ func TestValidateLabelDuplication(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="duplicate_label_names",user="testUser"} 2 + cortex_discarded_samples_total{reason="duplicate_label_names",type="float",user="testUser"} 2 `), "cortex_discarded_samples_total")) } @@ -721,7 +721,7 @@ func TestValidateNativeHistogram(t *testing.T) { actualHistogram, actualErr := ValidateNativeHistogram(validateMetrics, limits, userID, lbls, tc.histogram) if tc.expectedErr != nil { require.Equal(t, tc.expectedErr.Error(), actualErr.Error()) - require.Equal(t, float64(1), testutil.ToFloat64(validateMetrics.DiscardedSamples.WithLabelValues(tc.discardedSampleLabelValue, userID))) + require.Equal(t, float64(1), testutil.ToFloat64(validateMetrics.DiscardedSamples.WithLabelValues(tc.discardedSampleLabelValue, userID, SampleTypeHistogram))) // Should never increment if error was returned require.Equal(t, float64(0), testutil.ToFloat64(validateMetrics.HistogramSamplesReducedResolution.WithLabelValues(userID))) @@ -761,7 +761,7 @@ func TestValidateMetrics_UpdateSamplesDiscardedForSeries(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"bar\"}",reason="dummy",user="user"} 100 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) v.updateSamplesDiscardedForSeries(userID, "out-of-order", limits, labels.FromMap(map[string]string{"foo": "baz"}), 1) @@ -772,8 +772,8 @@ func TestValidateMetrics_UpdateSamplesDiscardedForSeries(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"baz\"}",reason="out-of-order",user="user"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 - cortex_discarded_samples_total{reason="out-of-order",user="user"} 1 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 + cortex_discarded_samples_total{reason="out-of-order",type="float",user="user"} 1 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Match default partition. @@ -786,9 +786,9 @@ func TestValidateMetrics_UpdateSamplesDiscardedForSeries(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{}",reason="too-old",user="user"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 - cortex_discarded_samples_total{reason="out-of-order",user="user"} 1 - cortex_discarded_samples_total{reason="too-old",user="user"} 1 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 + cortex_discarded_samples_total{reason="out-of-order",type="float",user="user"} 1 + cortex_discarded_samples_total{reason="too-old",type="float",user="user"} 1 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) } @@ -812,7 +812,7 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { }, } - v.updateSamplesDiscarded(userID, "dummy", limits, 100) + v.updateSamplesDiscarded(userID, "dummy", SampleTypeFloat, limits, 100) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_per_labelset_total The total number of samples that were discarded for each labelset. # TYPE cortex_discarded_samples_per_labelset_total counter @@ -821,7 +821,7 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{}",reason="dummy",user="user"} 100 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Remove default partition. @@ -837,7 +837,7 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"baz\"}",reason="dummy",user="user"} 100 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Remove limit 1. @@ -852,7 +852,7 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"bar\"}",reason="dummy",user="user"} 100 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Remove user. @@ -861,6 +861,6 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) }