From 087f1f44c6911fba6419efca40e1c30f1fd27deb Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:15:19 -0400 Subject: [PATCH 01/10] Add type label to cortex_discarded_samples_total Distinguish discarded float samples from discarded native histogram samples in the cortex_discarded_samples_total metric. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> --- pkg/distributor/distributor.go | 67 +++++++++++++++++++++++------ pkg/distributor/distributor_test.go | 14 ++++-- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index b938247ccee..834234e601a 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -833,7 +833,12 @@ func (d *Distributor) Push(ctx context.Context, req *cortexpb.WriteRequest) (*co } if errors.Is(err, ha.TooManyReplicaGroupsError{}) { - d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID).Add(float64(numFloatSamples + numHistogramSamples)) + if numFloatSamples > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID, validation.SampleTypeFloat).Add(float64(numFloatSamples)) + } + if numHistogramSamples > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID, validation.SampleTypeHistogram).Add(float64(numHistogramSamples)) + } return nil, httpgrpc.Errorf(http.StatusBadRequest, "%s", err.Error()) } return nil, err @@ -864,7 +869,12 @@ func (d *Distributor) Push(ctx context.Context, req *cortexpb.WriteRequest) (*co totalSamples := validatedFloatSamples + validatedHistogramSamples totalN := totalSamples + validatedExemplars + len(validatedMetadata) if !d.ingestionRateLimiter.AllowN(now, userID, totalN) { - d.validateMetrics.DiscardedSamples.WithLabelValues(validation.RateLimited, userID).Add(float64(totalSamples)) + if validatedFloatSamples > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.RateLimited, userID, validation.SampleTypeFloat).Add(float64(validatedFloatSamples)) + } + if validatedHistogramSamples > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.RateLimited, userID, validation.SampleTypeHistogram).Add(float64(validatedHistogramSamples)) + } d.validateMetrics.DiscardedExemplars.WithLabelValues(validation.RateLimited, userID).Add(float64(validatedExemplars)) d.validateMetrics.DiscardedMetadata.WithLabelValues(validation.RateLimited, userID).Add(float64(len(validatedMetadata))) // Return a 429 here to tell the client it is going too fast. @@ -879,7 +889,7 @@ func (d *Distributor) Push(ctx context.Context, req *cortexpb.WriteRequest) (*co var nativeHistogramErr error if !d.nativeHistogramIngestionRateLimiter.AllowN(now, userID, validatedHistogramSamples) { - d.validateMetrics.DiscardedSamples.WithLabelValues(validation.NativeHistogramRateLimited, userID).Add(float64(validatedHistogramSamples)) + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.NativeHistogramRateLimited, userID, validation.SampleTypeHistogram).Add(float64(validatedHistogramSamples)) nativeHistogramErr = httpgrpc.Errorf(http.StatusTooManyRequests, "native histogram ingestion rate limit (%v) exceeded while adding %d native histogram samples", d.nativeHistogramIngestionRateLimiter.Limit(now, userID), validatedHistogramSamples) validatedHistogramSamples = 0 } else { @@ -1121,7 +1131,12 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write d.dedupedSamples.WithLabelValues(userID, cluster).Add(float64(len(ts.Samples) + len(ts.Histograms))) } if errors.Is(err, ha.TooManyReplicaGroupsError{}) { - d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID).Add(float64(len(ts.Samples) + len(ts.Histograms))) + if len(ts.Samples) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID, validation.SampleTypeFloat).Add(float64(len(ts.Samples))) + } + if len(ts.Histograms) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(validation.TooManyHAClusters, userID, validation.SampleTypeHistogram).Add(float64(len(ts.Histograms))) + } } continue @@ -1145,10 +1160,20 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write l, _ := relabel.Process(cortexpb.FromLabelAdaptersToLabels(ts.Labels), mrc...) if l.Len() == 0 { // all labels are gone, samples will be discarded - d.validateMetrics.DiscardedSamples.WithLabelValues( - validation.DroppedByRelabelConfiguration, - userID, - ).Add(float64(len(ts.Samples) + len(ts.Histograms))) + if len(ts.Samples) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues( + validation.DroppedByRelabelConfiguration, + userID, + validation.SampleTypeFloat, + ).Add(float64(len(ts.Samples))) + } + if len(ts.Histograms) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues( + validation.DroppedByRelabelConfiguration, + userID, + validation.SampleTypeHistogram, + ).Add(float64(len(ts.Histograms))) + } // all labels are gone, exemplars will be discarded d.validateMetrics.DiscardedExemplars.WithLabelValues( @@ -1173,12 +1198,16 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write // Reject series with missing or empty metric name before removeEmptyLabels (which would strip __name__=""). if validationErr, reason := validation.ValidateMetricName(limits, ts.Labels, d.cfg.NameValidationScheme); reason != "" { - samplesCount := float64(len(ts.Samples) + len(ts.Histograms)) exemplarsCount := float64(len(ts.Exemplars)) if firstPartialErr == nil { firstPartialErr = httpgrpc.Errorf(http.StatusBadRequest, "%s", validationErr.Error()) } - d.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID).Add(samplesCount) + if len(ts.Samples) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, validation.SampleTypeFloat).Add(float64(len(ts.Samples))) + } + if len(ts.Histograms) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, validation.SampleTypeHistogram).Add(float64(len(ts.Histograms))) + } d.validateMetrics.DiscardedExemplars.WithLabelValues(reason, userID).Add(exemplarsCount) continue } @@ -1187,10 +1216,20 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write removeEmptyLabels(&ts.Labels) if len(ts.Labels) == 0 { - d.validateMetrics.DiscardedSamples.WithLabelValues( - validation.DroppedByUserConfigurationOverride, - userID, - ).Add(float64(len(ts.Samples) + len(ts.Histograms))) + if len(ts.Samples) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues( + validation.DroppedByUserConfigurationOverride, + userID, + validation.SampleTypeFloat, + ).Add(float64(len(ts.Samples))) + } + if len(ts.Histograms) > 0 { + d.validateMetrics.DiscardedSamples.WithLabelValues( + validation.DroppedByUserConfigurationOverride, + userID, + validation.SampleTypeHistogram, + ).Add(float64(len(ts.Histograms))) + } d.validateMetrics.DiscardedExemplars.WithLabelValues( validation.DroppedByUserConfigurationOverride, diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 79c95bc71a1..a0b5e3714a9 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -981,7 +981,7 @@ func TestDistributor_PushIngestionRateLimiter_Histograms(t *testing.T) { assert.Nil(t, response) } } - assert.Equal(t, float64(push.expectedNHDiscardedSampleMetricValue), testutil.ToFloat64(distributors[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.NativeHistogramRateLimited, "user"))) + assert.Equal(t, float64(push.expectedNHDiscardedSampleMetricValue), testutil.ToFloat64(distributors[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.NativeHistogramRateLimited, "user", validation.SampleTypeHistogram))) } }) } @@ -2142,7 +2142,11 @@ func TestDistributor_Push_LabelRemoval(t *testing.T) { _, err = ds[0].Push(ctx, req) require.NoError(t, err) - actualDiscardedSamples := testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByUserConfigurationOverride, "user")) + sampleType := validation.SampleTypeFloat + if histogram { + sampleType = validation.SampleTypeHistogram + } + actualDiscardedSamples := testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByUserConfigurationOverride, "user", sampleType)) actualDiscardedExemplars := testutil.ToFloat64(ds[0].validateMetrics.DiscardedExemplars.WithLabelValues(validation.DroppedByUserConfigurationOverride, "user")) require.Equal(t, float64(expectedDiscardedSamples), actualDiscardedSamples) require.Equal(t, float64(expectedDiscardedExemplars), actualDiscardedExemplars) @@ -4613,7 +4617,11 @@ func TestDistributor_Push_RelabelDropWillExportMetricOfDroppedSamples(t *testing assert.Equal(t, 1, len(timeseries)) } - require.Equal(t, testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByRelabelConfiguration, id)), float64(1)) + sampleType := validation.SampleTypeFloat + if histogramEnabled { + sampleType = validation.SampleTypeHistogram + } + require.Equal(t, testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByRelabelConfiguration, id, sampleType)), float64(1)) require.Equal(t, testutil.ToFloat64(ds[0].validateMetrics.DiscardedExemplars.WithLabelValues(validation.DroppedByRelabelConfiguration, id)), float64(2)) receivedFloatSamples := testutil.ToFloat64(ds[0].receivedSamples.WithLabelValues(id, "float")) receivedHistogramSamples := testutil.ToFloat64(ds[0].receivedSamples.WithLabelValues(id, "histogram")) From cfa083d60491309bf3fe83474786e4aa31f821ab Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:28:37 -0400 Subject: [PATCH 02/10] Add type label to cortex_discarded_samples_total Distinguish discarded float samples from discarded native histogram samples in the cortex_discarded_samples_total metric. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> --- pkg/ingester/ingester.go | 18 +++++++++--------- pkg/ingester/ingester_test.go | 26 +++++++++++++------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index d877b90a12f..3dfaeb26b08 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1785,32 +1785,32 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte i.metrics.ingestedExemplarsFail.Add(float64(failedExemplarsCount)) if sampleOutOfBoundsCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(sampleOutOfBounds, userID).Add(float64(sampleOutOfBoundsCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(sampleOutOfBounds, userID, validation.SampleTypeFloat).Add(float64(sampleOutOfBoundsCount)) } if sampleOutOfOrderCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(sampleOutOfOrder, userID).Add(float64(sampleOutOfOrderCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(sampleOutOfOrder, userID, validation.SampleTypeFloat).Add(float64(sampleOutOfOrderCount)) } if sampleTooOldCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(sampleTooOld, userID).Add(float64(sampleTooOldCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(sampleTooOld, userID, validation.SampleTypeFloat).Add(float64(sampleTooOldCount)) } if newValueForTimestampCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(newValueForTimestamp, userID).Add(float64(newValueForTimestampCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(newValueForTimestamp, userID, validation.SampleTypeFloat).Add(float64(newValueForTimestampCount)) } if perUserSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perUserSeriesLimit, userID).Add(float64(perUserSeriesLimitCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(perUserSeriesLimit, userID, validation.SampleTypeFloat).Add(float64(perUserSeriesLimitCount)) } if perUserNativeHistogramSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perUserNativeHistogramSeriesLimit, userID).Add(float64(perUserNativeHistogramSeriesLimitCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(perUserNativeHistogramSeriesLimit, userID, validation.SampleTypeHistogram).Add(float64(perUserNativeHistogramSeriesLimitCount)) } if perMetricSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perMetricSeriesLimit, userID).Add(float64(perMetricSeriesLimitCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(perMetricSeriesLimit, userID, validation.SampleTypeFloat).Add(float64(perMetricSeriesLimitCount)) } if perLabelSetSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perLabelsetSeriesLimit, userID).Add(float64(perLabelSetSeriesLimitCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(perLabelsetSeriesLimit, userID, validation.SampleTypeFloat).Add(float64(perLabelSetSeriesLimitCount)) } if !i.limits.EnableNativeHistograms(userID) && discardedNativeHistogramCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSample, userID).Add(float64(discardedNativeHistogramCount)) + i.validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSample, userID, validation.SampleTypeHistogram).Add(float64(discardedNativeHistogramCount)) } for h, counter := range reasonCounter.counters { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index d9a5299e0aa..b4d64de3d87 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -351,7 +351,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{label2=\"value2\"}",reason="per_labelset_series_limit",user="1"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 2 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 2 # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. # TYPE cortex_ingester_limits_per_labelset gauge cortex_ingester_limits_per_labelset{labelset="{label1=\"value1\"}",limit="max_series",user="1"} 3 @@ -402,7 +402,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{label2=\"value2\"}",reason="per_labelset_series_limit",user="1"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 2 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 2 # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. # TYPE cortex_ingester_limits_per_labelset gauge cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 2 @@ -454,7 +454,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{label2=\"value2\"}",reason="per_labelset_series_limit",user="1"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 3 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 3 # HELP cortex_ingester_limits_per_labelset Limits per user and labelset. # TYPE cortex_ingester_limits_per_labelset gauge cortex_ingester_limits_per_labelset{labelset="{comp1=\"compValue1\", comp2=\"compValue2\"}",limit="max_series",user="1"} 2 @@ -852,7 +852,7 @@ func TestIngesterUserLimitExceeded(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_user_series_limit",user="1"} 1 + cortex_discarded_samples_total{reason="per_user_series_limit",type="float",user="1"} 1 `), "cortex_discarded_samples_total")) } @@ -1078,7 +1078,7 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_metric_series_limit",user="1"} 1 + cortex_discarded_samples_total{reason="per_metric_series_limit",type="float",user="1"} 1 `), "cortex_discarded_samples_total")) } @@ -1179,7 +1179,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="native-histogram-sample",user="test"} 1 + cortex_discarded_samples_total{reason="native-histogram-sample",type="histogram",user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -1630,7 +1630,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-order",user="test"} 2 + cortex_discarded_samples_total{reason="sample-out-of-order",type="float",user="test"} 2 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -1688,7 +1688,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-bounds",user="test"} 2 + cortex_discarded_samples_total{reason="sample-out-of-bounds",type="float",user="test"} 2 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -1743,7 +1743,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-too-old",user="test"} 1 + cortex_discarded_samples_total{reason="sample-too-old",type="float",user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -1859,7 +1859,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_active_series{user="test"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-too-old",user="test"} 1 + cortex_discarded_samples_total{reason="sample-too-old",type="float",user="test"} 1 # HELP cortex_ingester_active_native_histogram_series Number of currently active native histogram series per user. # TYPE cortex_ingester_active_native_histogram_series gauge cortex_ingester_active_native_histogram_series{user="test"} 1 @@ -1967,7 +1967,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="new-value-for-timestamp",user="test"} 1 + cortex_discarded_samples_total{reason="new-value-for-timestamp",type="float",user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -7980,7 +7980,7 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"bar\"}",reason="per_labelset_series_limit",user="1"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 1 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 1 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Expect per labelset validate metrics cleaned up. @@ -7989,7 +7989,7 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="per_labelset_series_limit",user="1"} 1 + cortex_discarded_samples_total{reason="per_labelset_series_limit",type="float",user="1"} 1 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) } From 54e61e029310dbe9ff095e0758e26952a552b6a8 Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:28:48 -0400 Subject: [PATCH 03/10] Add type label to cortex_discarded_samples_total Distinguish discarded float samples from discarded native histogram samples in the cortex_discarded_samples_total metric. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 969e6c8f3b5..a5b9a9621a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master / unreleased +* [ENHANCEMENT] Distributor/Ingester: Add `type` label to `cortex_discarded_samples_total` metric to distinguish between float and native histogram samples. #6221 * [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828 * [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873 * [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539 From b38690bd5cbfe616b92e10aad8a540d211572dfd Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:29:30 -0400 Subject: [PATCH 04/10] Add type label to cortex_discarded_samples_total Distinguish discarded float samples from discarded native histogram samples in the cortex_discarded_samples_total metric. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> From b818306bc781f460bf966fc30ea0cb5b3ee2c525 Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:31:53 -0400 Subject: [PATCH 05/10] Add type label to cortex_discarded_samples_total Distinguish discarded float samples from discarded native histogram samples in the cortex_discarded_samples_total metric. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> --- pkg/util/validation/validate.go | 55 +++++++++++++++------------- pkg/util/validation/validate_test.go | 44 +++++++++++----------- 2 files changed, 52 insertions(+), 47 deletions(-) diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index 31c4b1f34cd..7d7a50983e5 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -25,6 +25,11 @@ import ( const ( discardReasonLabel = "reason" + // SampleTypeFloat indicates a float sample. + SampleTypeFloat = "float" + // SampleTypeHistogram indicates a native histogram sample. + SampleTypeHistogram = "histogram" + errMetadataMissingMetricName = "metadata missing metric name" errMetadataTooLong = "metadata '%s' value too long: %.200q metric %.200q" @@ -109,7 +114,7 @@ func NewValidateMetrics(r prometheus.Registerer) *ValidateMetrics { Name: "cortex_discarded_samples_total", Help: "The total number of samples that were discarded.", }, - []string{discardReasonLabel, "user"}, + []string{discardReasonLabel, "user", "type"}, ) registerCollector(r, discardedSamples) discardedSamplesPerLabelSet := prometheus.NewCounterVec( @@ -192,14 +197,14 @@ func NewValidateMetrics(r prometheus.Registerer) *ValidateMetrics { // Used in test only for now. func (m *ValidateMetrics) updateSamplesDiscardedForSeries(userID, reason string, labelSetLimits []LimitsPerLabelSet, lbls labels.Labels, count int) { matchedLimits := LimitsPerLabelSetsForSeries(labelSetLimits, lbls) - m.updateSamplesDiscarded(userID, reason, matchedLimits, count) + m.updateSamplesDiscarded(userID, reason, SampleTypeFloat, matchedLimits, count) } // updateSamplesDiscarded updates discarded samples and discarded samples per labelset for the provided reason. // The provided label set needs to be pre-filtered to match the series if applicable. // Used in test only for now. -func (m *ValidateMetrics) updateSamplesDiscarded(userID, reason string, labelSetLimits []LimitsPerLabelSet, count int) { - m.DiscardedSamples.WithLabelValues(reason, userID).Add(float64(count)) +func (m *ValidateMetrics) updateSamplesDiscarded(userID, reason, sampleType string, labelSetLimits []LimitsPerLabelSet, count int) { + m.DiscardedSamples.WithLabelValues(reason, userID, sampleType).Add(float64(count)) for _, limit := range labelSetLimits { m.LabelSetTracker.Track(userID, limit.Hash, limit.LabelSet) m.DiscardedSamplesPerLabelSet.WithLabelValues(reason, userID, limit.LabelSet.String()).Add(float64(count)) @@ -227,12 +232,12 @@ func ValidateSampleTimestamp(validateMetrics *ValidateMetrics, limits *Limits, u unsafeMetricName, _ := extract.UnsafeMetricNameFromLabelAdapters(ls) if limits.RejectOldSamples && model.Time(timestampMs) < model.Now().Add(-time.Duration(limits.RejectOldSamplesMaxAge)) { - validateMetrics.DiscardedSamples.WithLabelValues(greaterThanMaxSampleAge, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(greaterThanMaxSampleAge, userID, SampleTypeFloat).Inc() return newSampleTimestampTooOldError(unsafeMetricName, timestampMs) } if model.Time(timestampMs) > model.Now().Add(time.Duration(limits.CreationGracePeriod)) { - validateMetrics.DiscardedSamples.WithLabelValues(tooFarInFuture, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(tooFarInFuture, userID, SampleTypeFloat).Inc() return newSampleTimestampTooNewError(unsafeMetricName, timestampMs) } @@ -299,7 +304,7 @@ func ValidateMetricName(limits *Limits, ls []cortexpb.LabelAdapter, nameValidati func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, skipLabelNameValidation bool, nameValidationScheme model.ValidationScheme) ValidationError { numLabelNames := len(ls) if numLabelNames > limits.MaxLabelNamesPerSeries { - validateMetrics.DiscardedSamples.WithLabelValues(maxLabelNamesPerSeries, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(maxLabelNamesPerSeries, userID, SampleTypeFloat).Inc() return newTooManyLabelsError(ls, limits.MaxLabelNamesPerSeries) } @@ -311,21 +316,21 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str for _, l := range ls { if !skipLabelNameValidation && !nameValidationScheme.IsValidLabelName(l.Name) { - validateMetrics.DiscardedSamples.WithLabelValues(invalidLabel, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(invalidLabel, userID, SampleTypeFloat).Inc() return newInvalidLabelError(ls, l.Name) } else if len(l.Name) > maxLabelNameLength { - validateMetrics.DiscardedSamples.WithLabelValues(labelNameTooLong, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(labelNameTooLong, userID, SampleTypeFloat).Inc() return newLabelNameTooLongError(ls, l.Name, maxLabelNameLength) } else if len(l.Value) > maxLabelValueLength { - validateMetrics.DiscardedSamples.WithLabelValues(labelValueTooLong, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(labelValueTooLong, userID, SampleTypeFloat).Inc() return newLabelValueTooLongError(ls, l.Name, l.Value, maxLabelValueLength) } else if cmp := strings.Compare(lastLabelName, l.Name); cmp >= 0 { if cmp == 0 { - validateMetrics.DiscardedSamples.WithLabelValues(duplicateLabelNames, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(duplicateLabelNames, userID, SampleTypeFloat).Inc() return newDuplicatedLabelError(ls, l.Name) } - validateMetrics.DiscardedSamples.WithLabelValues(labelsNotSorted, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(labelsNotSorted, userID, SampleTypeFloat).Inc() return newLabelsNotSortedError(ls, l.Name) } @@ -334,7 +339,7 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str } validateMetrics.LabelSizeBytes.WithLabelValues(userID).Observe(float64(labelsSizeBytes)) if maxLabelsSizeBytes > 0 && labelsSizeBytes > maxLabelsSizeBytes { - validateMetrics.DiscardedSamples.WithLabelValues(labelsSizeBytesExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(labelsSizeBytesExceeded, userID, SampleTypeFloat).Inc() return labelSizeBytesExceededError(ls, labelsSizeBytes, maxLabelsSizeBytes) } return nil @@ -376,7 +381,7 @@ func ValidateMetadata(validateMetrics *ValidateMetrics, cfg *Limits, userID stri func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, histogramSample cortexpb.Histogram) (cortexpb.Histogram, error) { // sample size validation for native histogram if limits.MaxNativeHistogramSampleSizeBytes > 0 && histogramSample.Size() > limits.MaxNativeHistogramSampleSizeBytes { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSampleSizeBytesExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSampleSizeBytesExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramSampleSizeBytesExceededError(ls, histogramSample.Size(), limits.MaxNativeHistogramSampleSizeBytes) } @@ -386,7 +391,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u isExponentialSchema := schema >= histogram.ExponentialSchemaMin && schema <= histogram.ExponentialSchemaMax isValidSchema := isCustomBucketsSchema || isExponentialSchema if !isValidSchema { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalidSchema, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalidSchema, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramSchemaInvalidError(ls, int(schema)) } @@ -397,7 +402,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u if histogramSample.IsFloatHistogram() { fh := cortexpb.FloatHistogramProtoToFloatHistogram(histogramSample) if err := fh.Validate(); err != nil { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramInvalidError(ls, err) } @@ -409,7 +414,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u // Custom bucket cannot reduce resolution if isCustomBucketsSchema { if len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > limits.MaxNativeHistogramBuckets { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } return histogramSample, nil @@ -422,18 +427,18 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u // Exceed limit. if schema <= histogram.ExponentialSchemaMin { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } oBuckets := len(fh.PositiveBuckets) + len(fh.NegativeBuckets) for len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > limits.MaxNativeHistogramBuckets { if fh.Schema <= histogram.ExponentialSchemaMin { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } if err := fh.ReduceResolution(fh.Schema - 1); err != nil { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramInvalidError(ls, err) } } @@ -446,7 +451,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u h := cortexpb.HistogramProtoToHistogram(histogramSample) if err := h.Validate(); err != nil { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramInvalidError(ls, err) } @@ -458,7 +463,7 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u // Custom bucket cannot reduce resolution if isCustomBucketsSchema { if len(h.PositiveBuckets)+len(h.NegativeBuckets) > limits.MaxNativeHistogramBuckets { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } return histogramSample, nil @@ -470,17 +475,17 @@ func ValidateNativeHistogram(validateMetrics *ValidateMetrics, limits *Limits, u } // Exceed limit. if schema <= histogram.ExponentialSchemaMin { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } oBuckets := len(h.PositiveBuckets) + len(h.NegativeBuckets) for len(h.PositiveBuckets)+len(h.NegativeBuckets) > limits.MaxNativeHistogramBuckets { if h.Schema <= histogram.ExponentialSchemaMin { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramBucketCountLimitExceeded, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newHistogramBucketLimitExceededError(ls, limits.MaxNativeHistogramBuckets) } if err := h.ReduceResolution(h.Schema - 1); err != nil { - validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramInvalid, userID, SampleTypeHistogram).Inc() return cortexpb.Histogram{}, newNativeHistogramInvalidError(ls, err) } } diff --git a/pkg/util/validation/validate_test.go b/pkg/util/validation/validate_test.go index 6a55e735a9a..beb0186d930 100644 --- a/pkg/util/validation/validate_test.go +++ b/pkg/util/validation/validate_test.go @@ -191,18 +191,18 @@ func TestValidateLabels(t *testing.T) { assert.Equal(t, c.err, err, "wrong error") } - validateMetrics.DiscardedSamples.WithLabelValues("random reason", "different user").Inc() + validateMetrics.DiscardedSamples.WithLabelValues("random reason", "different user", SampleTypeFloat).Inc() require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="label_invalid",user="testUser"} 1 - cortex_discarded_samples_total{reason="label_name_too_long",user="testUser"} 1 - cortex_discarded_samples_total{reason="label_value_too_long",user="testUser"} 1 - cortex_discarded_samples_total{reason="max_label_names_per_series",user="testUser"} 1 - cortex_discarded_samples_total{reason="labels_size_bytes_exceeded",user="testUser"} 1 + cortex_discarded_samples_total{reason="label_invalid",type="float",user="testUser"} 1 + cortex_discarded_samples_total{reason="label_name_too_long",type="float",user="testUser"} 1 + cortex_discarded_samples_total{reason="label_value_too_long",type="float",user="testUser"} 1 + cortex_discarded_samples_total{reason="max_label_names_per_series",type="float",user="testUser"} 1 + cortex_discarded_samples_total{reason="labels_size_bytes_exceeded",type="float",user="testUser"} 1 - cortex_discarded_samples_total{reason="random reason",user="different user"} 1 + cortex_discarded_samples_total{reason="random reason",type="float",user="different user"} 1 `), "cortex_discarded_samples_total")) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` @@ -218,7 +218,7 @@ func TestValidateLabels(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="random reason",user="different user"} 1 + cortex_discarded_samples_total{reason="random reason",type="float",user="different user"} 1 `), "cortex_discarded_samples_total")) } @@ -359,7 +359,7 @@ func TestValidateLabelOrder(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="labels_not_sorted",user="testUser"} 1 + cortex_discarded_samples_total{reason="labels_not_sorted",type="float",user="testUser"} 1 `), "cortex_discarded_samples_total")) } @@ -397,7 +397,7 @@ func TestValidateLabelDuplication(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="duplicate_label_names",user="testUser"} 2 + cortex_discarded_samples_total{reason="duplicate_label_names",type="float",user="testUser"} 2 `), "cortex_discarded_samples_total")) } @@ -721,7 +721,7 @@ func TestValidateNativeHistogram(t *testing.T) { actualHistogram, actualErr := ValidateNativeHistogram(validateMetrics, limits, userID, lbls, tc.histogram) if tc.expectedErr != nil { require.Equal(t, tc.expectedErr.Error(), actualErr.Error()) - require.Equal(t, float64(1), testutil.ToFloat64(validateMetrics.DiscardedSamples.WithLabelValues(tc.discardedSampleLabelValue, userID))) + require.Equal(t, float64(1), testutil.ToFloat64(validateMetrics.DiscardedSamples.WithLabelValues(tc.discardedSampleLabelValue, userID, SampleTypeHistogram))) // Should never increment if error was returned require.Equal(t, float64(0), testutil.ToFloat64(validateMetrics.HistogramSamplesReducedResolution.WithLabelValues(userID))) @@ -761,7 +761,7 @@ func TestValidateMetrics_UpdateSamplesDiscardedForSeries(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"bar\"}",reason="dummy",user="user"} 100 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) v.updateSamplesDiscardedForSeries(userID, "out-of-order", limits, labels.FromMap(map[string]string{"foo": "baz"}), 1) @@ -772,8 +772,8 @@ func TestValidateMetrics_UpdateSamplesDiscardedForSeries(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"baz\"}",reason="out-of-order",user="user"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 - cortex_discarded_samples_total{reason="out-of-order",user="user"} 1 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 + cortex_discarded_samples_total{reason="out-of-order",type="float",user="user"} 1 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Match default partition. @@ -786,9 +786,9 @@ func TestValidateMetrics_UpdateSamplesDiscardedForSeries(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{}",reason="too-old",user="user"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 - cortex_discarded_samples_total{reason="out-of-order",user="user"} 1 - cortex_discarded_samples_total{reason="too-old",user="user"} 1 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 + cortex_discarded_samples_total{reason="out-of-order",type="float",user="user"} 1 + cortex_discarded_samples_total{reason="too-old",type="float",user="user"} 1 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) } @@ -812,7 +812,7 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { }, } - v.updateSamplesDiscarded(userID, "dummy", limits, 100) + v.updateSamplesDiscarded(userID, "dummy", SampleTypeFloat, limits, 100) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_per_labelset_total The total number of samples that were discarded for each labelset. # TYPE cortex_discarded_samples_per_labelset_total counter @@ -821,7 +821,7 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{}",reason="dummy",user="user"} 100 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Remove default partition. @@ -837,7 +837,7 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"baz\"}",reason="dummy",user="user"} 100 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Remove limit 1. @@ -852,7 +852,7 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { cortex_discarded_samples_per_labelset_total{labelset="{foo=\"bar\"}",reason="dummy",user="user"} 100 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) // Remove user. @@ -861,6 +861,6 @@ func TestValidateMetrics_UpdateLabelSet(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="dummy",user="user"} 100 + cortex_discarded_samples_total{reason="dummy",type="float",user="user"} 100 `), "cortex_discarded_samples_total", "cortex_discarded_samples_per_labelset_total")) } From f1c2404d6741a0884fc5f298e19bd84d47810ecb Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:32:01 -0400 Subject: [PATCH 06/10] Add type label to cortex_discarded_samples_total Distinguish discarded float samples from discarded native histogram samples in the cortex_discarded_samples_total metric. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> From 799405553b5ac069e2b4e3b86a0b4c14a3ac1e65 Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:16:25 -0400 Subject: [PATCH 07/10] Address review: attribute discards by sample type in shared paths Thread the sample type through the paths shared by float and native histogram samples so histogram discards are no longer recorded as float: the ingester append-failure handler now splits its per-reason counters by type, ValidateSampleTimestamp takes the sample type from its caller, and ValidateLabels attributes rejections to each sample type present in the series. Also correct the TestIngester_Push expectations that locked in the mislabel, and record the change as [CHANGE] citing #7898 in the CHANGELOG. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5b9a9621a0..c926d497d86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## master / unreleased -* [ENHANCEMENT] Distributor/Ingester: Add `type` label to `cortex_discarded_samples_total` metric to distinguish between float and native histogram samples. #6221 +* [CHANGE] Distributor/Ingester: Add `type` label to `cortex_discarded_samples_total` metric to distinguish between float and native histogram samples. #7898 * [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828 * [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873 * [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539 From 09da17a19ab33eede72038ad4d52173940e186d9 Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:39:05 -0400 Subject: [PATCH 08/10] Address review: attribute discards by sample type in shared paths Thread the sample type through the paths shared by float and native histogram samples so histogram discards are no longer recorded as float: the ingester append-failure handler now splits its per-reason counters by type, ValidateSampleTimestamp takes the sample type from its caller, and ValidateLabels attributes rejections to each sample type present in the series. Also correct the TestIngester_Push expectations that locked in the mislabel, and record the change as [CHANGE] citing #7898 in the CHANGELOG. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> --- pkg/distributor/distributor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 834234e601a..3725a8f6238 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -689,7 +689,7 @@ func (d *Distributor) checkSample(ctx context.Context, userID, cluster, replica func (d *Distributor) validateSeries(ts cortexpb.PreallocTimeseries, userID string, skipLabelNameValidation bool, limits *validation.Limits) (cortexpb.PreallocTimeseries, validation.ValidationError) { d.labelsHistogram.Observe(float64(len(ts.Labels))) - if err := validation.ValidateLabels(d.validateMetrics, limits, userID, ts.Labels, skipLabelNameValidation, d.cfg.NameValidationScheme); err != nil { + if err := validation.ValidateLabels(d.validateMetrics, limits, userID, ts.Labels, skipLabelNameValidation, d.cfg.NameValidationScheme, len(ts.Samples), len(ts.Histograms)); err != nil { return emptyPreallocSeries, err } @@ -698,7 +698,7 @@ func (d *Distributor) validateSeries(ts cortexpb.PreallocTimeseries, userID stri // Only alloc when data present samples = make([]cortexpb.Sample, 0, len(ts.Samples)) for _, s := range ts.Samples { - if err := validation.ValidateSampleTimestamp(d.validateMetrics, limits, userID, ts.Labels, s.TimestampMs); err != nil { + if err := validation.ValidateSampleTimestamp(d.validateMetrics, limits, userID, ts.Labels, s.TimestampMs, validation.SampleTypeFloat); err != nil { return emptyPreallocSeries, err } samples = append(samples, s) @@ -727,7 +727,7 @@ func (d *Distributor) validateSeries(ts cortexpb.PreallocTimeseries, userID stri histograms = make([]cortexpb.WrappedHistogram, 0, len(ts.Histograms)) receivedBucketsObserver := d.receivedHistogramBuckets.WithLabelValues(userID) for i, h := range ts.Histograms { - if err := validation.ValidateSampleTimestamp(d.validateMetrics, limits, userID, ts.Labels, h.TimestampMs); err != nil { + if err := validation.ValidateSampleTimestamp(d.validateMetrics, limits, userID, ts.Labels, h.TimestampMs, validation.SampleTypeHistogram); err != nil { return emptyPreallocSeries, err } receivedBucketsObserver.Observe(float64(h.BucketCount())) From 94b812889da12635f27e32b997854fca013a2112 Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:41:07 -0400 Subject: [PATCH 09/10] Address review: attribute discards by sample type in shared paths Thread the sample type through the paths shared by float and native histogram samples so histogram discards are no longer recorded as float: the ingester append-failure handler now splits its per-reason counters by type, ValidateSampleTimestamp takes the sample type from its caller, and ValidateLabels attributes rejections to each sample type present in the series. Also correct the TestIngester_Push expectations that locked in the mislabel, and record the change as [CHANGE] citing #7898 in the CHANGELOG. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> --- pkg/ingester/ingester.go | 86 +++++++++++++++++++---------------- pkg/ingester/ingester_test.go | 9 ++-- 2 files changed, 53 insertions(+), 42 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 3dfaeb26b08..5c883ac810b 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1415,6 +1415,30 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte reasonCounter := newLabelSetReasonCounters() + // discardedCounts tracks discarded samples for one reason, split by + // sample type. The ingester's append-failure handler is shared by the + // float and native histogram loops, so a single counter per reason + // would misattribute histogram discards to the float series. + type discardedCounts struct { + float int + histogram int + } + incDiscarded := func(c *discardedCounts, sampleType string) { + if sampleType == validation.SampleTypeHistogram { + c.histogram++ + } else { + c.float++ + } + } + emitDiscarded := func(reason string, c discardedCounts) { + if c.float > 0 { + i.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, validation.SampleTypeFloat).Add(float64(c.float)) + } + if c.histogram > 0 { + i.validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, validation.SampleTypeHistogram).Add(float64(c.histogram)) + } + } + // Keep track of some stats which are tracked only if the samples will be // successfully committed var ( @@ -1427,14 +1451,14 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte startTimestampSampleAppendFailCount = 0 startTimestampHistogramAppendFailCount = 0 startAppend = time.Now() - sampleOutOfBoundsCount = 0 - sampleOutOfOrderCount = 0 - sampleTooOldCount = 0 - newValueForTimestampCount = 0 - perUserSeriesLimitCount = 0 + sampleOutOfBoundsCount discardedCounts + sampleOutOfOrderCount discardedCounts + sampleTooOldCount discardedCounts + newValueForTimestampCount discardedCounts + perUserSeriesLimitCount discardedCounts perUserNativeHistogramSeriesLimitCount = 0 - perLabelSetSeriesLimitCount = 0 - perMetricSeriesLimitCount = 0 + perLabelSetSeriesLimitCount discardedCounts + perMetricSeriesLimitCount discardedCounts discardedNativeHistogramCount = 0 // Only the ruler sets the write request source to RULE. @@ -1456,7 +1480,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte // isStaleNaN reports whether the failed sample carried the reserved stale NaN // payload. Callers appending native histograms always pass false, because a // staleness marker is only ever appended as a float sample. - handleAppendFailure = func(err error, timestampMs int64, lbls []cortexpb.LabelAdapter, copiedLabels labels.Labels, matchedLabelSetLimits []validation.LimitsPerLabelSet, isStaleNaN bool) (rollback bool) { + handleAppendFailure = func(err error, timestampMs int64, lbls []cortexpb.LabelAdapter, copiedLabels labels.Labels, matchedLabelSetLimits []validation.LimitsPerLabelSet, isStaleNaN bool, sampleType string) (rollback bool) { // A staleness marker from the ruler is internal bookkeeping rather than // tenant data. Prometheus' rules.Group.cleanupStaleSeries appends one for // every series a rule stopped producing, and treats out-of-order, too-old @@ -1474,7 +1498,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte // we actually ingested all samples which haven't failed. switch cause := errors.Cause(err); { case errors.Is(cause, storage.ErrOutOfBounds): - sampleOutOfBoundsCount++ + incDiscarded(&sampleOutOfBoundsCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(sampleOutOfBounds, userID, copiedLabels.Hash()) updateFirstPartial(func() error { return wrappedTSDBIngestErrWithHeadMaxTime(err, model.Time(timestampMs), model.Time(headMaxTime), lbls) @@ -1482,21 +1506,21 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte case errors.Is(cause, storage.ErrOutOfOrderSample): if !ruleStaleMarker { - sampleOutOfOrderCount++ + incDiscarded(&sampleOutOfOrderCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(sampleOutOfOrder, userID, copiedLabels.Hash()) } updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(timestampMs), lbls) }) case errors.Is(cause, storage.ErrDuplicateSampleForTimestamp): if !ruleStaleMarker { - newValueForTimestampCount++ + incDiscarded(&newValueForTimestampCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(newValueForTimestamp, userID, copiedLabels.Hash()) } updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(timestampMs), lbls) }) case errors.Is(cause, storage.ErrTooOldSample): if !ruleStaleMarker { - sampleTooOldCount++ + incDiscarded(&sampleTooOldCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(sampleTooOld, userID, copiedLabels.Hash()) } updateFirstPartial(func() error { @@ -1504,7 +1528,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte }) case errors.Is(cause, errMaxSeriesPerUserLimitExceeded): - perUserSeriesLimitCount++ + incDiscarded(&perUserSeriesLimitCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(perUserSeriesLimit, userID, copiedLabels.Hash()) updateFirstPartial(func() error { return makeLimitError(perUserSeriesLimit, i.limiter.FormatError(userID, cause, copiedLabels)) @@ -1517,14 +1541,14 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte }) case errors.Is(cause, errMaxSeriesPerMetricLimitExceeded): - perMetricSeriesLimitCount++ + incDiscarded(&perMetricSeriesLimitCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(perMetricSeriesLimit, userID, copiedLabels.Hash()) updateFirstPartial(func() error { return makeMetricLimitError(perMetricSeriesLimit, copiedLabels, i.limiter.FormatError(userID, cause, copiedLabels)) }) case errors.As(cause, &errMaxSeriesPerLabelSetLimitExceeded{}): - perLabelSetSeriesLimitCount++ + incDiscarded(&perLabelSetSeriesLimitCount, sampleType) i.validateMetrics.DiscardedSeriesTracker.Track(perLabelsetSeriesLimit, userID, copiedLabels.Hash()) for _, matchedLabelset := range matchedLabelSetLimits { i.validateMetrics.DiscardedSeriesPerLabelsetTracker.Track(userID, copiedLabels.Hash(), matchedLabelset.Hash, matchedLabelset.Id) @@ -1634,7 +1658,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte failedSamplesCount++ - if rollback := handleAppendFailure(err, s.TimestampMs, ts.Labels, copiedLabels, matchedLabelSetLimits, value.IsStaleNaN(s.Value)); !rollback { + if rollback := handleAppendFailure(err, s.TimestampMs, ts.Labels, copiedLabels, matchedLabelSetLimits, value.IsStaleNaN(s.Value), validation.SampleTypeFloat); !rollback { continue } // The error looks an issue on our side, so we should rollback. @@ -1692,7 +1716,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte failedHistogramsCount++ - if rollback := handleAppendFailure(err, hp.TimestampMs, ts.Labels, copiedLabels, matchedLabelSetLimits, false); !rollback { + if rollback := handleAppendFailure(err, hp.TimestampMs, ts.Labels, copiedLabels, matchedLabelSetLimits, false, validation.SampleTypeHistogram); !rollback { continue } // The error looks an issue on our side, so we should rollback. @@ -1784,30 +1808,16 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte i.metrics.ingestedExemplars.Add(float64(succeededExemplarsCount)) i.metrics.ingestedExemplarsFail.Add(float64(failedExemplarsCount)) - if sampleOutOfBoundsCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(sampleOutOfBounds, userID, validation.SampleTypeFloat).Add(float64(sampleOutOfBoundsCount)) - } - if sampleOutOfOrderCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(sampleOutOfOrder, userID, validation.SampleTypeFloat).Add(float64(sampleOutOfOrderCount)) - } - if sampleTooOldCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(sampleTooOld, userID, validation.SampleTypeFloat).Add(float64(sampleTooOldCount)) - } - if newValueForTimestampCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(newValueForTimestamp, userID, validation.SampleTypeFloat).Add(float64(newValueForTimestampCount)) - } - if perUserSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perUserSeriesLimit, userID, validation.SampleTypeFloat).Add(float64(perUserSeriesLimitCount)) - } + emitDiscarded(sampleOutOfBounds, sampleOutOfBoundsCount) + emitDiscarded(sampleOutOfOrder, sampleOutOfOrderCount) + emitDiscarded(sampleTooOld, sampleTooOldCount) + emitDiscarded(newValueForTimestamp, newValueForTimestampCount) + emitDiscarded(perUserSeriesLimit, perUserSeriesLimitCount) if perUserNativeHistogramSeriesLimitCount > 0 { i.validateMetrics.DiscardedSamples.WithLabelValues(perUserNativeHistogramSeriesLimit, userID, validation.SampleTypeHistogram).Add(float64(perUserNativeHistogramSeriesLimitCount)) } - if perMetricSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perMetricSeriesLimit, userID, validation.SampleTypeFloat).Add(float64(perMetricSeriesLimitCount)) - } - if perLabelSetSeriesLimitCount > 0 { - i.validateMetrics.DiscardedSamples.WithLabelValues(perLabelsetSeriesLimit, userID, validation.SampleTypeFloat).Add(float64(perLabelSetSeriesLimitCount)) - } + emitDiscarded(perMetricSeriesLimit, perMetricSeriesLimitCount) + emitDiscarded(perLabelsetSeriesLimit, perLabelSetSeriesLimitCount) if !i.limits.EnableNativeHistograms(userID) && discardedNativeHistogramCount > 0 { i.validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSample, userID, validation.SampleTypeHistogram).Add(float64(discardedNativeHistogramCount)) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index b4d64de3d87..2a6f73f9e2e 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -1512,7 +1512,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_tsdb_out_of_order_samples_total{type="histogram",user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-order",user="test"} 1 + cortex_discarded_samples_total{reason="sample-out-of-order",type="float",user="test"} 1 `, }, "ooo disabled, tenant staleness marker out of order is counted as a discarded sample": { @@ -1564,7 +1564,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_tsdb_out_of_order_samples_total{type="histogram",user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-order",user="test"} 1 + cortex_discarded_samples_total{reason="sample-out-of-order",type="float",user="test"} 1 `, }, "ooo disabled, should soft fail on sample out of order": { @@ -1630,7 +1630,8 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_memory_series_removed_total{user="test"} 0 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-order",type="float",user="test"} 2 + cortex_discarded_samples_total{reason="sample-out-of-order",type="float",user="test"} 1 + cortex_discarded_samples_total{reason="sample-out-of-order",type="histogram",user="test"} 1 # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="test"} 1 @@ -1859,7 +1860,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_active_series{user="test"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-too-old",type="float",user="test"} 1 + cortex_discarded_samples_total{reason="sample-too-old",type="histogram",user="test"} 1 # HELP cortex_ingester_active_native_histogram_series Number of currently active native histogram series per user. # TYPE cortex_ingester_active_native_histogram_series gauge cortex_ingester_active_native_histogram_series{user="test"} 1 From d45cd89b39eba112497f407c1c41b2dc9c423536 Mon Sep 17 00:00:00 2001 From: "Navid Khan (Ali)" <60997541+Sun-Wukhan@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:42:56 -0400 Subject: [PATCH 10/10] Address review: attribute discards by sample type in shared paths Thread the sample type through the paths shared by float and native histogram samples so histogram discards are no longer recorded as float: the ingester append-failure handler now splits its per-reason counters by type, ValidateSampleTimestamp takes the sample type from its caller, and ValidateLabels attributes rejections to each sample type present in the series. Also correct the TestIngester_Push expectations that locked in the mislabel, and record the change as [CHANGE] citing #7898 in the CHANGELOG. Signed-off-by: Navid Khan (Ali) <60997541+Sun-Wukhan@users.noreply.github.com> --- pkg/util/validation/validate.go | 35 +++++++++++++++++++--------- pkg/util/validation/validate_test.go | 10 ++++---- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index 7d7a50983e5..01a0a4bb14c 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -226,18 +226,31 @@ func (m *ValidateMetrics) UpdateLabelSet(userSet map[string]map[uint64]struct{}, }) } +// discardSamplesForSeries records one discarded-series event, attributed to +// each sample type present in the series. When the caller has no sample +// counts (e.g. direct unit tests), the event is attributed to float samples, +// matching historical behaviour. +func discardSamplesForSeries(validateMetrics *ValidateMetrics, reason, userID string, numFloatSamples, numHistogramSamples int) { + if numFloatSamples > 0 || numHistogramSamples == 0 { + validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, SampleTypeFloat).Inc() + } + if numHistogramSamples > 0 { + validateMetrics.DiscardedSamples.WithLabelValues(reason, userID, SampleTypeHistogram).Inc() + } +} + // ValidateSampleTimestamp returns an err if the sample timestamp is invalid. // The returned error may retain the provided series labels. -func ValidateSampleTimestamp(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, timestampMs int64) ValidationError { +func ValidateSampleTimestamp(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, timestampMs int64, sampleType string) ValidationError { unsafeMetricName, _ := extract.UnsafeMetricNameFromLabelAdapters(ls) if limits.RejectOldSamples && model.Time(timestampMs) < model.Now().Add(-time.Duration(limits.RejectOldSamplesMaxAge)) { - validateMetrics.DiscardedSamples.WithLabelValues(greaterThanMaxSampleAge, userID, SampleTypeFloat).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(greaterThanMaxSampleAge, userID, sampleType).Inc() return newSampleTimestampTooOldError(unsafeMetricName, timestampMs) } if model.Time(timestampMs) > model.Now().Add(time.Duration(limits.CreationGracePeriod)) { - validateMetrics.DiscardedSamples.WithLabelValues(tooFarInFuture, userID, SampleTypeFloat).Inc() + validateMetrics.DiscardedSamples.WithLabelValues(tooFarInFuture, userID, sampleType).Inc() return newSampleTimestampTooNewError(unsafeMetricName, timestampMs) } @@ -301,10 +314,10 @@ func ValidateMetricName(limits *Limits, ls []cortexpb.LabelAdapter, nameValidati // ValidateLabels returns an err if the labels are invalid. // The returned error may retain the provided series labels. // Callers must validate metric name (e.g. via ValidateMetricName) before calling this when EnforceMetricName is true. -func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, skipLabelNameValidation bool, nameValidationScheme model.ValidationScheme) ValidationError { +func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID string, ls []cortexpb.LabelAdapter, skipLabelNameValidation bool, nameValidationScheme model.ValidationScheme, numFloatSamples, numHistogramSamples int) ValidationError { numLabelNames := len(ls) if numLabelNames > limits.MaxLabelNamesPerSeries { - validateMetrics.DiscardedSamples.WithLabelValues(maxLabelNamesPerSeries, userID, SampleTypeFloat).Inc() + discardSamplesForSeries(validateMetrics, maxLabelNamesPerSeries, userID, numFloatSamples, numHistogramSamples) return newTooManyLabelsError(ls, limits.MaxLabelNamesPerSeries) } @@ -316,21 +329,21 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str for _, l := range ls { if !skipLabelNameValidation && !nameValidationScheme.IsValidLabelName(l.Name) { - validateMetrics.DiscardedSamples.WithLabelValues(invalidLabel, userID, SampleTypeFloat).Inc() + discardSamplesForSeries(validateMetrics, invalidLabel, userID, numFloatSamples, numHistogramSamples) return newInvalidLabelError(ls, l.Name) } else if len(l.Name) > maxLabelNameLength { - validateMetrics.DiscardedSamples.WithLabelValues(labelNameTooLong, userID, SampleTypeFloat).Inc() + discardSamplesForSeries(validateMetrics, labelNameTooLong, userID, numFloatSamples, numHistogramSamples) return newLabelNameTooLongError(ls, l.Name, maxLabelNameLength) } else if len(l.Value) > maxLabelValueLength { - validateMetrics.DiscardedSamples.WithLabelValues(labelValueTooLong, userID, SampleTypeFloat).Inc() + discardSamplesForSeries(validateMetrics, labelValueTooLong, userID, numFloatSamples, numHistogramSamples) return newLabelValueTooLongError(ls, l.Name, l.Value, maxLabelValueLength) } else if cmp := strings.Compare(lastLabelName, l.Name); cmp >= 0 { if cmp == 0 { - validateMetrics.DiscardedSamples.WithLabelValues(duplicateLabelNames, userID, SampleTypeFloat).Inc() + discardSamplesForSeries(validateMetrics, duplicateLabelNames, userID, numFloatSamples, numHistogramSamples) return newDuplicatedLabelError(ls, l.Name) } - validateMetrics.DiscardedSamples.WithLabelValues(labelsNotSorted, userID, SampleTypeFloat).Inc() + discardSamplesForSeries(validateMetrics, labelsNotSorted, userID, numFloatSamples, numHistogramSamples) return newLabelsNotSortedError(ls, l.Name) } @@ -339,7 +352,7 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str } validateMetrics.LabelSizeBytes.WithLabelValues(userID).Observe(float64(labelsSizeBytes)) if maxLabelsSizeBytes > 0 && labelsSizeBytes > maxLabelsSizeBytes { - validateMetrics.DiscardedSamples.WithLabelValues(labelsSizeBytesExceeded, userID, SampleTypeFloat).Inc() + discardSamplesForSeries(validateMetrics, labelsSizeBytesExceeded, userID, numFloatSamples, numHistogramSamples) return labelSizeBytesExceededError(ls, labelsSizeBytes, maxLabelsSizeBytes) } return nil diff --git a/pkg/util/validation/validate_test.go b/pkg/util/validation/validate_test.go index beb0186d930..66e547735e9 100644 --- a/pkg/util/validation/validate_test.go +++ b/pkg/util/validation/validate_test.go @@ -66,7 +66,7 @@ func TestValidateLabels_UTF8(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { - err := ValidateLabels(validateMetrics, cfg, userID, cortexpb.FromMetricsToLabelAdapters(test.metric), test.skipLabelNameValidation, model.UTF8Validation) + err := ValidateLabels(validateMetrics, cfg, userID, cortexpb.FromMetricsToLabelAdapters(test.metric), test.skipLabelNameValidation, model.UTF8Validation, 0, 0) assert.Equal(t, test.expectedErr, err, "wrong error") }) } @@ -187,7 +187,7 @@ func TestValidateLabels(t *testing.T) { nil, }, } { - err := ValidateLabels(validateMetrics, cfg, userID, cortexpb.FromMetricsToLabelAdapters(c.metric), c.skipLabelNameValidation, model.LegacyValidation) + err := ValidateLabels(validateMetrics, cfg, userID, cortexpb.FromMetricsToLabelAdapters(c.metric), c.skipLabelNameValidation, model.LegacyValidation, 0, 0) assert.Equal(t, c.err, err, "wrong error") } @@ -348,7 +348,7 @@ func TestValidateLabelOrder(t *testing.T) { {Name: model.MetricNameLabel, Value: "m"}, {Name: "b", Value: "b"}, {Name: "a", Value: "a"}, - }, false, model.LegacyValidation) + }, false, model.LegacyValidation, 0, 0) expected := newLabelsNotSortedError([]cortexpb.LabelAdapter{ {Name: model.MetricNameLabel, Value: "m"}, {Name: "b", Value: "b"}, @@ -375,7 +375,7 @@ func TestValidateLabelDuplication(t *testing.T) { actual := ValidateLabels(validateMetrics, cfg, userID, []cortexpb.LabelAdapter{ {Name: model.MetricNameLabel, Value: "a"}, {Name: model.MetricNameLabel, Value: "b"}, - }, false, model.LegacyValidation) + }, false, model.LegacyValidation, 0, 0) expected := newDuplicatedLabelError([]cortexpb.LabelAdapter{ {Name: model.MetricNameLabel, Value: "a"}, {Name: model.MetricNameLabel, Value: "b"}, @@ -386,7 +386,7 @@ func TestValidateLabelDuplication(t *testing.T) { {Name: model.MetricNameLabel, Value: "a"}, {Name: "a", Value: "a"}, {Name: "a", Value: "a"}, - }, false, model.LegacyValidation) + }, false, model.LegacyValidation, 0, 0) expected = newDuplicatedLabelError([]cortexpb.LabelAdapter{ {Name: model.MetricNameLabel, Value: "a"}, {Name: "a", Value: "a"},