diff --git a/expfmt/decode.go b/expfmt/decode.go index dbff6cb3..9ee251f6 100644 --- a/expfmt/decode.go +++ b/expfmt/decode.go @@ -72,6 +72,10 @@ func ResponseFormat(h http.Header) Format { return FmtOpenMetrics_0_0_1 case OpenMetricsVersion_1_0_0: return FmtOpenMetrics_1_0_0 + case OpenMetricsVersion_2_0_0_rc: + return FmtOpenMetrics_2_0_0_rc + case OpenMetricsVersion_2_0_0: + return FmtOpenMetrics_2_0_0 default: return FmtUnknown } @@ -84,9 +88,9 @@ func ResponseFormat(h http.Header) Format { // names are validated based on the provided Format -- if the format requires // escaping, traditional Prometheus validity checking is used. Otherwise, names // are checked for UTF-8 validity. Supported formats include delimited protobuf, -// Prometheus text format, and OpenMetrics 0.0.1 / 1.0.0. For historical +// Prometheus text format, and OpenMetrics 0.0.1 / 1.0.0 / 2.0.0-rc / 2.0.0. For historical // reasons, this decoder falls back to classic text decoding for other legacy -// formats, but returns an error for unsupported formats such as OpenMetrics 2.0. +// formats, and returns an error for unsupported formats such as text protobuf. // This decoder may not support the latest features of Prometheus text format // and is not intended for high-performance applications. // See: https://github.com/prometheus/common/issues/812 @@ -108,6 +112,15 @@ func NewDecoder(r io.Reader, format Format) Decoder { switch params["version"] { case "", OpenMetricsVersion_0_0_1, OpenMetricsVersion_1_0_0: return &openMetricsDecoder{p: newOpenMetricsParser(r, scheme)} + case OpenMetricsVersion_2_0_0_rc, OpenMetricsVersion_2_0_0: + // format.ToEscapingScheme() defaults to UnderscoreEscaping when no escaping + // parameter is set, whereas OpenMetrics 2.0 supports UTF-8 names by default + // unless a non-UTF-8 escaping scheme is explicitly requested. + om2Scheme := model.UTF8Validation + if _, hasEscaping := params[model.EscapingKey]; hasEscaping && format.ToEscapingScheme() != model.NoEscaping { + om2Scheme = model.LegacyValidation + } + return &openMetrics20Decoder{p: newOpenMetrics20Parser(r, om2Scheme)} default: return &errDecoder{err: fmt.Errorf("format %s not supported for decoding", format)} } @@ -178,6 +191,27 @@ func (d *openMetricsDecoder) Decode(v *dto.MetricFamily) error { return nil } +// openMetrics20Decoder implements the Decoder interface for the OpenMetrics 2.0 protocol. +// Like openMetricsDecoder, it uses goabnf.ParseForest per line without a line-length +// cap (which has superlinear parse time in line length) and is intended for +// reference/conformance decoding rather than high-performance or untrusted inputs. +type openMetrics20Decoder struct { + p *openMetrics20Parser + err error +} + +// Decode implements the Decoder interface. +func (d *openMetrics20Decoder) Decode(v *dto.MetricFamily) error { + if d.err != nil { + return d.err + } + if err := d.p.next(v); err != nil { + d.err = err + return err + } + return nil +} + // textDecoder implements the Decoder interface for the text protocol. type textDecoder struct { r io.Reader diff --git a/expfmt/decode_test.go b/expfmt/decode_test.go index 1b6022e7..83f458d7 100644 --- a/expfmt/decode_test.go +++ b/expfmt/decode_test.go @@ -467,6 +467,14 @@ func testDiscriminatorHTTPHeader(t testing.TB) { input: map[string]string{"Content-Type": `application/openmetrics-text; version=1.0.0`}, output: FmtOpenMetrics_1_0_0, }, + { + input: map[string]string{"Content-Type": `application/openmetrics-text; version=2.0.0-rc`}, + output: FmtOpenMetrics_2_0_0_rc, + }, + { + input: map[string]string{"Content-Type": `application/openmetrics-text; version=2.0.0`}, + output: FmtOpenMetrics_2_0_0, + }, { input: map[string]string{"Content-Type": `application/openmetrics-text; version=0.0.2`}, output: FmtUnknown, @@ -642,24 +650,26 @@ func TestNewDecoder(t *testing.T) { { name: "OpenMetrics 2.0.0-rc", format: om20RCFormat, - expectError: true, + input: "# EOF\n", + expectError: false, }, { name: "OpenMetrics 2.0.0-rc with escaping", format: om20RCFormat.WithEscapingScheme(model.ValueEncodingEscaping), - expectError: true, + input: "# EOF\n", + expectError: false, }, { name: "OpenMetrics 2.0.0", format: om20Format, input: "# EOF\n", - expectError: true, + expectError: false, }, { name: "OpenMetrics 2.0.0 with escaping", format: om20Format.WithEscapingScheme(model.ValueEncodingEscaping), input: "# EOF\n", - expectError: true, + expectError: false, }, } @@ -742,3 +752,68 @@ mf1{label="value2"} 42 sort.Sort(out) require.Equal(t, out, all) } + +func TestOpenMetrics20Decoder(t *testing.T) { + var ( + ts = model.Now() + in = `# TYPE mf2_total counter +mf2_total 3 +mf2_total 4 +mf1{label="value1"} -3.14 123.456 +mf1{label="value2"} 42 +# EOF +` + out = model.Vector{ + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "mf1", + "label": "value1", + }, + Value: -3.14, + Timestamp: 123456, + }, + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "mf1", + "label": "value2", + }, + Value: 42, + Timestamp: ts, + }, + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "mf2_total", + }, + Value: 3, + Timestamp: ts, + }, + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "mf2_total", + }, + Value: 4, + Timestamp: ts, + }, + } + ) + + dec := &SampleDecoder{ + Dec: NewDecoder(strings.NewReader(in), FmtOpenMetrics_2_0_0), + Opts: &DecodeOptions{ + Timestamp: ts, + }, + } + var all model.Vector + for { + var smpls model.Vector + err := dec.Decode(&smpls) + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + all = append(all, smpls...) + } + sort.Sort(all) + sort.Sort(out) + require.Equal(t, out, all) +} diff --git a/expfmt/expfmt.go b/expfmt/expfmt.go index e58c4b09..7f5fe7fa 100644 --- a/expfmt/expfmt.go +++ b/expfmt/expfmt.go @@ -110,7 +110,7 @@ func NewFormat(t FormatType) Format { // NewOpenMetricsFormat generates a new OpenMetrics format matching the // specified version number. // -// Note: OpenMetrics versions 2.0.0-rc and 2.0.0 are experimental and encode-only (supporting +// Note: OpenMetrics versions 2.0.0-rc and 2.0.0 are experimental (supporting // counter, gauge, summary, untyped, histogram, and gaugehistogram metric types). func NewOpenMetricsFormat(version string) (Format, error) { switch version { @@ -119,10 +119,10 @@ func NewOpenMetricsFormat(version string) (Format, error) { case OpenMetricsVersion_1_0_0: return FmtOpenMetrics_1_0_0, nil case OpenMetricsVersion_2_0_0_rc: - // OpenMetrics 2.0.0-rc is experimental and encode-only. + // OpenMetrics 2.0.0-rc is experimental. return FmtOpenMetrics_2_0_0_rc, nil case OpenMetricsVersion_2_0_0: - // OpenMetrics 2.0.0 is experimental and encode-only. + // OpenMetrics 2.0.0 is experimental. return FmtOpenMetrics_2_0_0, nil default: return FmtUnknown, errors.New("unknown open metrics version string") diff --git a/expfmt/openmetrics_2_0_create.go b/expfmt/openmetrics_2_0_create.go index 600ac32f..47a88f79 100644 --- a/expfmt/openmetrics_2_0_create.go +++ b/expfmt/openmetrics_2_0_create.go @@ -25,14 +25,16 @@ import ( dto "github.com/prometheus/client_model/go" "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/prometheus/common/model" ) // MetricFamilyToOpenMetrics20 converts a MetricFamily proto message into the // OpenMetrics text format version 2.0.0 and writes the resulting lines to 'out'. // It returns the number of bytes written and any error encountered. // -// NOTE: This method targets OpenMetrics 2.0.0 (currently aligned with 2.0-rc.0) which is experimental and -// encode-only. All metric types defined in client_model (counter, gauge, summary, untyped, histogram, +// NOTE: This method targets OpenMetrics 2.0.0 (currently aligned with 2.0-rc.0) which is experimental. +// All metric types defined in client_model (counter, gauge, summary, untyped, histogram, // and gaugehistogram) are supported; info and stateset are not supported as they are not present in // client_model. Breaking changes might happen in the future. // EncoderOptions are accepted for signature compatibility with @@ -383,9 +385,6 @@ func writeCompositeSummary(w enhancedWriter, name string, metric *dto.Metric) (i if math.IsNaN(sum) { return 0, fmt.Errorf("summary sum cannot be NaN in metric %s", name) } - if sum < 0 { - return 0, fmt.Errorf("summary sum cannot be negative (%g) in metric %s", sum, name) - } var prevQuantile float64 for i, q := range s.Quantile { @@ -403,11 +402,6 @@ func writeCompositeSummary(w enhancedWriter, name string, metric *dto.Metric) (i return 0, fmt.Errorf("summary quantiles must be strictly increasing: %g <= %g in metric %s", qv, prevQuantile, name) } prevQuantile = qv - - v := q.GetValue() - if !math.IsNaN(v) && v < 0 { - return 0, fmt.Errorf("summary quantile value cannot be negative (%g) in metric %s", v, name) - } } if s.CreatedTimestamp != nil { @@ -547,7 +541,10 @@ func writeCompositeHistogram(w enhancedWriter, name string, metric *dto.Metric, } return 0, fmt.Errorf("histogram count cannot be NaN in metric %s", name) } - if !isGauge && c < 0 { + if c < 0 { + if isGauge { + return 0, fmt.Errorf("gaugehistogram count cannot be negative (%g) in metric %s", c, name) + } return 0, fmt.Errorf("histogram count cannot be negative (%g) in metric %s", c, name) } } @@ -565,7 +562,7 @@ func writeCompositeHistogram(w enhancedWriter, name string, metric *dto.Metric, } if isNative { - if err := validateNativeHistogram(name, h, isGauge); err != nil { + if err := validateNativeHistogram(name, h); err != nil { return 0, err } } @@ -709,7 +706,7 @@ func writeCompositeHistogram(w enhancedWriter, name string, metric *dto.Metric, return written, nil } -func validateNativeHistogram(name string, h *dto.Histogram, isGauge bool) error { +func validateNativeHistogram(name string, h *dto.Histogram) error { schema := *h.Schema if schema < -4 || schema > 8 { return fmt.Errorf("native histogram schema %d is out of range [-4, 8] in metric %s", schema, name) @@ -725,15 +722,15 @@ func validateNativeHistogram(name string, h *dto.Histogram, isGauge bool) error if math.IsNaN(c) { return fmt.Errorf("native histogram zero_count cannot be NaN in metric %s", name) } - if !isGauge && c < 0 { + if c < 0 { return fmt.Errorf("native histogram zero_count cannot be negative (%g) in metric %s", c, name) } } - if err := validateSpansAndBuckets(name, "negative", h.NegativeSpan, h.NegativeDelta, h.NegativeCount, isGauge); err != nil { + if err := validateSpansAndBuckets(name, "negative", h.NegativeSpan, h.NegativeDelta, h.NegativeCount); err != nil { return err } - return validateSpansAndBuckets(name, "positive", h.PositiveSpan, h.PositiveDelta, h.PositiveCount, isGauge) + return validateSpansAndBuckets(name, "positive", h.PositiveSpan, h.PositiveDelta, h.PositiveCount) } func validateSpansAndBuckets( @@ -742,7 +739,6 @@ func validateSpansAndBuckets( spans []*dto.BucketSpan, deltas []int64, floatCounts []float64, - isGauge bool, ) error { isFloatBuckets := len(floatCounts) > 0 var numBuckets int @@ -779,7 +775,7 @@ func validateSpansAndBuckets( if math.IsNaN(v) { return fmt.Errorf("%s bucket count cannot be NaN in metric %s", spanName, name) } - if !isGauge && v < 0 { + if v < 0 { return fmt.Errorf("%s bucket count cannot be negative (%g) in metric %s", spanName, v, name) } } @@ -787,7 +783,7 @@ func validateSpansAndBuckets( var current int64 for _, d := range deltas { current += d - if !isGauge && current < 0 { + if current < 0 { return fmt.Errorf("%s bucket count cannot be negative (%d) in metric %s", spanName, current, name) } } @@ -996,7 +992,7 @@ func validateClassicBuckets( if math.IsNaN(c) { return fmt.Errorf("classic bucket count cannot be NaN in metric %s", name) } - if !isGauge && c < 0 { + if c < 0 { return fmt.Errorf("classic bucket count cannot be negative (%g) in metric %s", c, name) } } @@ -1009,7 +1005,7 @@ func validateClassicBuckets( bCount = float64(*b.CumulativeCount) } - if !isGauge && i > 0 && bCount < prevCount { + if i > 0 && bCount < prevCount { return fmt.Errorf("classic bucket counts must be monotonically increasing: %g < %g in metric %s", bCount, prevCount, name) } prevCount = bCount @@ -1037,7 +1033,7 @@ func validateClassicBuckets( } } - if !infSeen && len(h.Bucket) > 0 && !isGauge { + if !infSeen && len(h.Bucket) > 0 { if sampleCount < prevCount { return fmt.Errorf("sample count (%g) is less than highest bucket count (%g) in metric %s", sampleCount, prevCount, name) } @@ -1142,6 +1138,9 @@ func validateLabels20(labels []*dto.LabelPair) error { if lname == "" { return errors.New("label name cannot be empty") } + if lname == string(model.MetricNameLabel) { + return fmt.Errorf("label name %q is reserved", model.MetricNameLabel) + } if containsRawNewline(lname) { return fmt.Errorf("label name %q contains raw newlines", lname) } diff --git a/expfmt/openmetrics_2_0_create_test.go b/expfmt/openmetrics_2_0_create_test.go index 4c42f3b2..4d161975 100644 --- a/expfmt/openmetrics_2_0_create_test.go +++ b/expfmt/openmetrics_2_0_create_test.go @@ -302,6 +302,30 @@ http_requests_total 1027.0 }, out: `# TYPE http_requests_total counter http_requests_total 1027.0 +`, + }, + { + name: "CounterWithReservedNameExemplarLabel", + in: &dto.MetricFamily{ + Name: proto.String("http_requests_total"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(1027), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + {Name: proto.String("__name__"), Value: proto.String("foo")}, + }, + Value: proto.Float64(1), + Timestamp: ×tamppb.Timestamp{Seconds: 1234567890}, + }, + }, + }, + }, + }, + out: `# TYPE http_requests_total counter +http_requests_total 1027.0 `, }, { @@ -851,25 +875,25 @@ acme_http_request_seconds:rate5m{path="/api/v1",method="GET"} {gcount:0.01,gsum: `, }, { - name: "GaugeHistogram_NegativeValues", + name: "GaugeHistogram_NegativeSumAndThreshold", in: &dto.MetricFamily{ Name: proto.String("net_flow_rate"), Type: dto.MetricType_GAUGE_HISTOGRAM.Enum(), Metric: []*dto.Metric{ { Histogram: &dto.Histogram{ - SampleCountFloat: proto.Float64(-1.5), + SampleCountFloat: proto.Float64(1.5), SampleSum: proto.Float64(-3.2), Bucket: []*dto.Bucket{ - {UpperBound: proto.Float64(-0.5), CumulativeCountFloat: proto.Float64(-2.0)}, - {UpperBound: proto.Float64(math.Inf(+1)), CumulativeCountFloat: proto.Float64(-1.5)}, + {UpperBound: proto.Float64(-0.5), CumulativeCountFloat: proto.Float64(1.0)}, + {UpperBound: proto.Float64(math.Inf(+1)), CumulativeCountFloat: proto.Float64(1.5)}, }, }, }, }, }, out: `# TYPE net_flow_rate gaugehistogram -net_flow_rate {gcount:-1.5,gsum:-3.2,bucket:[-0.5:-2,+Inf:-1.5]} +net_flow_rate {gcount:1.5,gsum:-3.2,bucket:[-0.5:1,+Inf:1.5]} `, }, { @@ -1106,6 +1130,28 @@ foo {count:17,sum:324789.3,quantile:[]} 1234567891 st@1234567890 }, out: `# TYPE foo summary foo {count:5,sum:10,quantile:[0.5:NaN,0.99:+Inf]} +`, + }, + { + name: "Summary_NegativeSumAndQuantiles", + in: &dto.MetricFamily{ + Name: proto.String("temp_delta"), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Summary: &dto.Summary{ + SampleCount: proto.Uint64(5), + SampleSum: proto.Float64(-10.5), + Quantile: []*dto.Quantile{ + {Quantile: proto.Float64(0.5), Value: proto.Float64(-2.5)}, + {Quantile: proto.Float64(0.99), Value: proto.Float64(1.0)}, + }, + }, + }, + }, + }, + out: `# TYPE temp_delta summary +temp_delta {count:5,sum:-10.5,quantile:[0.5:-2.5,0.99:1]} `, }, { @@ -1335,21 +1381,6 @@ func TestCreateOpenMetrics20_Errors(t *testing.T) { }, expectedErr: "summary sum cannot be NaN", }, - { - name: "Summary_SumNegative", - in: &dto.MetricFamily{ - Name: proto.String("test_summary"), - Type: dto.MetricType_SUMMARY.Enum(), - Metric: []*dto.Metric{ - { - Summary: &dto.Summary{ - SampleSum: proto.Float64(-1.0), - }, - }, - }, - }, - expectedErr: "summary sum cannot be negative", - }, { name: "Summary_NilQuantile", in: &dto.MetricFamily{ @@ -1469,23 +1500,6 @@ func TestCreateOpenMetrics20_Errors(t *testing.T) { }, expectedErr: "summary quantiles must be strictly increasing", }, - { - name: "Summary_QuantileValueNegative", - in: &dto.MetricFamily{ - Name: proto.String("test_summary"), - Type: dto.MetricType_SUMMARY.Enum(), - Metric: []*dto.Metric{ - { - Summary: &dto.Summary{ - Quantile: []*dto.Quantile{ - {Quantile: proto.Float64(0.5), Value: proto.Float64(-1.0)}, - }, - }, - }, - }, - }, - expectedErr: "summary quantile value cannot be negative", - }, { name: "Summary_InvalidCreatedTimestamp", in: &dto.MetricFamily{ @@ -1583,6 +1597,22 @@ func TestCreateOpenMetrics20_Errors(t *testing.T) { }, expectedErr: "gaugehistogram count cannot be NaN", }, + { + name: "GaugeHistogramCountNegative", + in: &dto.MetricFamily{ + Name: proto.String("test_gauge_histogram"), + Type: dto.MetricType_GAUGE_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCountFloat: proto.Float64(-1.0), + SampleSum: proto.Float64(0.0), + }, + }, + }, + }, + expectedErr: "gaugehistogram count cannot be negative", + }, { name: "MetricHasLeLabel_ClassicHistogram", in: &dto.MetricFamily{ @@ -2405,7 +2435,7 @@ func TestCreateOpenMetrics20_SummaryError_NoPartialBytes(t *testing.T) { metric: &dto.Metric{ Summary: &dto.Summary{ SampleCount: proto.Uint64(1), - SampleSum: proto.Float64(-1.0), + SampleSum: proto.Float64(math.NaN()), }, }, }, diff --git a/expfmt/openmetrics_2_0_parse.go b/expfmt/openmetrics_2_0_parse.go new file mode 100644 index 00000000..52ec10de --- /dev/null +++ b/expfmt/openmetrics_2_0_parse.go @@ -0,0 +1,1341 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expfmt + +import ( + "bufio" + "errors" + "fmt" + "io" + "math" + "math/bits" + "strconv" + "strings" + "sync" + + goabnf "github.com/pandatix/go-abnf" + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/prometheus/common/model" +) + +// openMetrics20ABNF is the OpenMetrics 2.0 ABNF grammar (RFC 5234 / RFC 7405). +// +// NOTE: Every OpenMetrics 2.0 metric-descriptor, sample, and eof construct is +// strictly single-line (normal-char excludes LF %x0A). Adding the `line` rule +// allows streaming line-by-line from an io.Reader via goabnf.ParseForest per +// line while keeping the full exposition grammar intact. +const openMetrics20ABNF = `exposition = metricset eof-line + +metricset = *(metric-descriptor / sample) + +line = metric-descriptor / sample / eof-line +eof-line = HASH SP %s"EOF" [ LF ] + +metric-descriptor = HASH SP descriptor-type SP metric-descriptor-name SP metric-type LF +metric-descriptor =/ HASH SP descriptor-help SP metric-descriptor-name SP help-value LF +metric-descriptor =/ HASH SP descriptor-unit SP metric-descriptor-name SP unit-value LF + +descriptor-type = %s"TYPE" +descriptor-help = %s"HELP" +descriptor-unit = %s"UNIT" + +metric-descriptor-name = metricname / metricname-utf8 +help-value = escaped-string +unit-value = *metricname-char + +metric-type = %s"counter" / %s"gauge" / %s"histogram" / %s"gaugehistogram" / %s"stateset" +metric-type =/ %s"info" / %s"summary" / %s"unknown" + +sample = metricname-and-labels SP value [SP sample-timestamp] [SP start-timestamp] *exemplar LF + +value = sample-number / "{" composite-value "}" +sample-number = number + +sample-timestamp = timestamp +timestamp = realnumber + +; Lowercase st @ timestamp +start-timestamp = %s"st" "@" timestamp + +exemplar = SP HASH SP labels-in-braces SP exemplar-value SP exemplar-timestamp +exemplar-value = number +exemplar-timestamp = timestamp + +metricname-and-labels = metricname [labels-in-braces] / name-and-labels-in-braces +labels-in-braces = "{" [label *(COMMA label)] "}" +name-and-labels-in-braces = "{" metricname-utf8 *(COMMA label) "}" + +label = label-key "=" DQUOTE label-value DQUOTE +label-key = label-name / label-name-utf8 +label-name-utf8 = DQUOTE escaped-string-non-empty DQUOTE +label-value = escaped-string + +; Number value +number = realnumber +; Case insensitive +number =/ [SIGN] (%i"inf" / %i"infinity") +number =/ %i"nan" + +; Real floats +; Leading 0s explicitly okay +realnumber = [SIGN] 1*DIGIT ["." *DIGIT] [ "e" [SIGN] 1*DIGIT ] +realnumber =/ [SIGN] *DIGIT "." 1*DIGIT [ "e" [SIGN] 1*DIGIT ] + +; Integers +; Leading 0s explicitly okay +integer = [SIGN] 1*DIGIT +non-negative-integer = ["+"] 1*DIGIT + +BS = "\" +COMMA = "," +HASH = "#" +SIGN = "-" / "+" + +metricname = metricname-initial-char 0*metricname-char + +metricname-char = metricname-initial-char / DIGIT +metricname-initial-char = ALPHA / "_" / ":" +metricname-utf8 = DQUOTE escaped-string-non-empty DQUOTE + +label-name = label-name-initial-char *label-name-char + +label-name-char = label-name-initial-char / DIGIT +label-name-initial-char = ALPHA / "_" + +escaped-string = *escaped-char +escaped-string-non-empty = 1*escaped-char + +escaped-char = normal-char +escaped-char =/ BS (DQUOTE / BS / normal-char) + +; Any unicode character, except newline, double quote, and backslash +normal-char = %x00-09 / %x0B-21 / %x23-5B / %x5D-D7FF / %xE000-10FFFF + +; Composite values +composite-value = histogram-value / gauge-histogram-value / summary-value + +; Histograms +histogram-value = h-count "," h-sum "," histogram-buckets +gauge-histogram-value = %s"g" h-count "," %s"g" h-sum "," histogram-buckets + +; count:x +h-count = %s"count" ":" number +; sum:f allows real numbers and +-Inf and NaN +h-sum = %s"sum" ":" number + +histogram-buckets = classic-buckets / native-buckets [ "," classic-buckets ] + +; bucket:[...,+Inf:v] The +Inf bucket is required. +classic-buckets = %s"bucket" ":" "[" [ ch-le-counts "," ] ch-pos-inf-bucket "]" +ch-le-counts = (ch-neg-inf-bucket / ch-le-bucket) *("," ch-le-bucket) +ch-pos-inf-bucket = "+" %s"Inf" ":" number +ch-neg-inf-bucket = "-" %s"Inf" ":" number +ch-le-bucket = realnumber ":" number + +; schema:3,zero_threshold:1e-128,zero_count:2,negative_spans:[1:1],negative_buckets:[2],positive_spans:[-3:1,2:2],positive_buckets:[3,1,0] +native-buckets = nh-schema "," nh-zero-threshold "," nh-zero-count [ "," nh-negative-spans "," nh-negative-buckets ] [ "," nh-positive-spans "," nh-positive-buckets ] + +; schema:i +nh-schema = %s"schema" ":" integer +; zero_threshold:f +nh-zero-threshold = %s"zero_threshold" ":" realnumber +; zero_count:x +nh-zero-count = %s"zero_count" ":" number +; negative_spans:[1:2,3:4] and positive_spans:[-3:1,2:2] +nh-negative-spans = %s"negative_spans" ":" "[" [nh-spans] "]" +nh-positive-spans = %s"positive_spans" ":" "[" [nh-spans] "]" +; Spans hold offset and length. The offset can start from any index, even +; negative, however subsequent spans can only advance the index, not decrease it. +nh-spans = nh-start-span *("," nh-span) +nh-start-span = integer ":" non-negative-integer +nh-span = non-negative-integer ":" non-negative-integer + +; negative_buckets:[1,2,3] and positive_buckets:[1,2,3] +nh-negative-buckets = %s"negative_buckets" ":" "[" [nh-buckets] "]" +nh-positive-buckets = %s"positive_buckets" ":" "[" [nh-buckets] "]" + +nh-buckets = number *("," number) + +; Summary + +; count:12.0,sum:100.0,quantile:[0.9:2.0,0.95:3.0,0.99:20.0] +summary-value = cs-count "," cs-sum "," cs-quantile + +; count:x where x is a number +cs-count = %s"count" ":" number +; sum:x where x is a real number or +-Inf or NaN +cs-sum = %s"sum" ":" number +; quantile:[...] +cs-quantile = %s"quantile" ":" "[" [ cs-q-counts ] "]" +cs-q-counts = cs-q-count *("," cs-q-count) +cs-q-count = realnumber ":" number +` + +var ( + om20GrammarOnce sync.Once + om20Grammar *goabnf.Grammar + om20GrammarErr error +) + +func getOpenMetrics20Grammar() (*goabnf.Grammar, error) { + om20GrammarOnce.Do(func() { + raw := strings.ReplaceAll(openMetrics20ABNF, "\r\n", "\n") + raw = strings.ReplaceAll(raw, "\n", "\r\n") + om20Grammar, om20GrammarErr = goabnf.ParseABNF([]byte(raw)) + }) + return om20Grammar, om20GrammarErr +} + +// openMetrics20Parser streams OpenMetrics 2.0 text exposition format line-by-line +// using the OpenMetrics 2.0 ABNF grammar. +// +// NOTE: Like openMetricsParser, this parser invokes goabnf.ParseForest on each +// full line without a line-length cap, which has superlinear (roughly quadratic) +// parse time in line length. As documented on NewDecoder, it is intended for +// reference and conformance decoding rather than high-performance or untrusted inputs. +type openMetrics20Parser struct { + br *bufio.Reader + g *goabnf.Grammar + initErr error + scheme model.ValidationScheme + families map[string]*openMetricsFamilyState + curFamily *openMetricsFamilyState + lineNum int + pendingLine []byte + pendingNode *goabnf.ParseTree + done bool +} + +func newOpenMetrics20Parser(r io.Reader, scheme model.ValidationScheme) *openMetrics20Parser { + br, ok := r.(*bufio.Reader) + if !ok { + br = bufio.NewReader(r) + } + g, err := getOpenMetrics20Grammar() + if err != nil { + err = fmt.Errorf("failed to initialize OpenMetrics 2.0 ABNF grammar: %w", err) + } + return &openMetrics20Parser{ + br: br, + g: g, + initErr: err, + scheme: scheme, + families: make(map[string]*openMetricsFamilyState), + } +} + +func (p *openMetrics20Parser) validationScheme() model.ValidationScheme { + if p.scheme == model.UnsetValidation { + return model.UTF8Validation + } + return p.scheme +} + +func emitOpenMetrics20Family(st *openMetricsFamilyState, v *dto.MetricFamily) { + if st.mf.Type == nil { + st.mf.Type = dto.MetricType_UNTYPED.Enum() + } + v.Name = st.mf.Name + v.Help = st.mf.Help + v.Type = st.mf.Type + v.Metric = st.mf.Metric + v.Unit = st.mf.Unit + st.emitted = true + st.mf.Metric = nil +} + +// next reads from the input stream until the next MetricFamily is complete and +// populates v, or returns io.EOF when # EOF is reached. +func (p *openMetrics20Parser) next(v *dto.MetricFamily) error { + if p.initErr != nil { + return p.initErr + } + for { + if p.done { + return io.EOF + } + + var ( + line []byte + node *goabnf.ParseTree + ) + if p.pendingNode != nil { + line = p.pendingLine + node = p.pendingNode + p.pendingLine = nil + p.pendingNode = nil + } else { + rawLine, readErr := p.br.ReadBytes('\n') + if len(rawLine) == 0 { + if errors.Is(readErr, io.EOF) { + return ParseError{ + Line: max(1, p.lineNum+1), + Msg: "data does not end with # EOF", + } + } + return readErr + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return readErr + } + p.lineNum++ + line = rawLine + + forest, err := goabnf.ParseForest(line, p.g, "line") + if err != nil { + return err + } + if !forest.Valid() { + return formatABNFLineParseError(p.lineNum, forest.ParseError()) + } + tree := forest.Tree() + if tree == nil || len(tree.Children) == 0 { + return ParseError{Line: p.lineNum, Msg: "failed to extract parse tree"} + } + node = tree.Children[0] + } + + switch node.Rule { + case "eof-line": + if _, peekErr := p.br.Peek(1); !errors.Is(peekErr, io.EOF) { + if peekErr != nil { + return peekErr + } + return ParseError{ + Line: p.lineNum + 1, + Msg: "unexpected data after # EOF", + } + } + p.done = true + if p.curFamily != nil && p.curFamily.hasSamples && !p.curFamily.emitted { + st := p.curFamily + p.curFamily = nil + emitOpenMetrics20Family(st, v) + return nil + } + return io.EOF + + case "metric-descriptor": + if p.curFamily != nil && !p.curFamily.emitted { + mfName := extractDescriptorMetricName(line, node) + if mfName != p.curFamily.mf.GetName() { + if p.curFamily.hasSamples { + p.pendingLine = line + p.pendingNode = node + st := p.curFamily + p.curFamily = nil + emitOpenMetrics20Family(st, v) + return nil + } + p.curFamily.emitted = true + p.curFamily = nil + } + } + if err := p.handleDescriptor(line, node); err != nil { + return err + } + + case "sample": + if p.curFamily != nil && !p.curFamily.emitted { + sampleName := extractSampleMetricName(line, node) + if sampleName != p.curFamily.mf.GetName() { + if p.curFamily.hasSamples { + p.pendingLine = line + p.pendingNode = node + st := p.curFamily + p.curFamily = nil + emitOpenMetrics20Family(st, v) + return nil + } + p.curFamily.emitted = true + p.curFamily = nil + } + } + if err := p.handleSample(line, node); err != nil { + return err + } + } + } +} + +func extractDescriptorMetricName(input []byte, node *goabnf.ParseTree) string { + for _, c := range node.Children { + if c.Rule == "metric-descriptor-name" { + return extractMetricNameNode(input, c) + } + } + return "" +} + +func (p *openMetrics20Parser) getOrCreateDescriptorFamily(name string) (*openMetricsFamilyState, error) { + if !p.validationScheme().IsValidMetricName(name) { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid metric name %q", name)} + } + if p.curFamily != nil && !p.curFamily.emitted && p.curFamily.mf.GetName() == name { + if p.curFamily.hasSamples { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("metadata for metric %q after samples", name)} + } + return p.curFamily, nil + } + if prev, exists := p.families[name]; exists { + if prev.hasSamples { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("metadata for metric %q after samples", name)} + } + if prev.emitted { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("interleaved metric family %q", name)} + } + return prev, nil + } + st := &openMetricsFamilyState{ + mf: &dto.MetricFamily{ + Name: proto.String(name), + }, + } + p.families[name] = st + if p.curFamily == nil || !p.curFamily.hasSamples { + p.curFamily = st + } + return st, nil +} + +func (p *openMetrics20Parser) handleDescriptor(input []byte, node *goabnf.ParseTree) error { + var ( + descKind string + mfName string + valNode *goabnf.ParseTree + ) + for _, c := range node.Children { + switch c.Rule { + case "descriptor-type", "descriptor-help", "descriptor-unit": + descKind = c.Rule + case "metric-descriptor-name": + mfName = extractMetricNameNode(input, c) + case "metric-type", "help-value", "unit-value": + valNode = c + } + } + + st, err := p.getOrCreateDescriptorFamily(mfName) + if err != nil { + return err + } + + switch descKind { + case "descriptor-type": + if st.hasType { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("second TYPE line for metric name %q", mfName)} + } + st.hasType = true + rawType := string(input[valNode.Start:valNode.End]) + switch rawType { + case "counter": + st.mf.Type = dto.MetricType_COUNTER.Enum() + case "gauge": + st.mf.Type = dto.MetricType_GAUGE.Enum() + case "histogram": + st.mf.Type = dto.MetricType_HISTOGRAM.Enum() + case "gaugehistogram": + st.mf.Type = dto.MetricType_GAUGE_HISTOGRAM.Enum() + case "summary": + st.mf.Type = dto.MetricType_SUMMARY.Enum() + case "unknown": + st.mf.Type = dto.MetricType_UNTYPED.Enum() + case "info", "stateset": + // TODO(bwplotka): Expand client_model (dto.MetricType) to support INFO and STATESET metric types. + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("unsupported metric type %q", rawType)} + default: + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("unknown metric type %q", rawType)} + } + case "descriptor-help": + if st.hasHelp { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("second HELP line for metric name %q", mfName)} + } + st.hasHelp = true + rawHelp := string(input[valNode.Start:valNode.End]) + if strings.HasSuffix(rawHelp, "\r") { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("unexpected carriage return in HELP for metric %q", mfName)} + } + st.mf.Help = proto.String(unescapeOpenMetricsString(rawHelp)) + case "descriptor-unit": + if st.hasUnit { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("second UNIT line for metric name %q", mfName)} + } + st.hasUnit = true + unit := string(input[valNode.Start:valNode.End]) + if unit != "" { + st.mf.Unit = proto.String(unit) + } + } + return nil +} + +func (p *openMetrics20Parser) resolveSampleFamily(sampleName string) (*openMetricsFamilyState, error) { + if !p.validationScheme().IsValidMetricName(sampleName) { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid metric name %q", sampleName)} + } + + if p.curFamily != nil && !p.curFamily.emitted && p.curFamily.mf.GetName() == sampleName { + if p.curFamily.mf.Type == nil { + p.curFamily.mf.Type = dto.MetricType_UNTYPED.Enum() + } + p.curFamily.hasSamples = true + return p.curFamily, nil + } + + if st, exists := p.families[sampleName]; exists { + if st.emitted { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("interleaved metric family %q", st.mf.GetName())} + } + if st.mf.Type == nil { + st.mf.Type = dto.MetricType_UNTYPED.Enum() + } + st.hasSamples = true + p.curFamily = st + return st, nil + } + + st := &openMetricsFamilyState{ + mf: &dto.MetricFamily{ + Name: proto.String(sampleName), + Type: dto.MetricType_UNTYPED.Enum(), + }, + hasSamples: true, + } + p.families[sampleName] = st + p.curFamily = st + return st, nil +} + +func (p *openMetrics20Parser) handleSample(input []byte, node *goabnf.ParseTree) error { + var ( + nameAndLabelsNode *goabnf.ParseTree + valueNode *goabnf.ParseTree + hasTS bool + tsStr string + hasST bool + stStr string + exemplarNodes []*goabnf.ParseTree + ) + for _, c := range node.Children { + switch c.Rule { + case "metricname-and-labels": + nameAndLabelsNode = c + case "value": + valueNode = c + case "sample-timestamp": + hasTS = true + tsStr = string(input[c.Start:c.End]) + case "start-timestamp": + for _, sc := range c.Children { + if sc.Rule == "timestamp" { + hasST = true + stStr = string(input[sc.Start:sc.End]) + } + } + case "exemplar": + exemplarNodes = append(exemplarNodes, c) + } + } + + sampleName, rawLabels := extractNameAndLabels(input, nameAndLabelsNode) + st, err := p.resolveSampleFamily(sampleName) + if err != nil { + return err + } + + labelPairs, err := p.validateAndBuildLabels(st.mf.GetName(), rawLabels) + if err != nil { + return err + } + + var tsMs *int64 + if hasTS { + parsedTS, err := parseSampleTimestampMs(tsStr) + if err != nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid timestamp %q: %v", tsStr, err)} + } + tsMs = proto.Int64(parsedTS) + } + + var startTS *timestamppb.Timestamp + if hasST { + startTS, err = parseTimestampToProto(stStr) + if err != nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid start timestamp %q: %v", stStr, err)} + } + } + + var exemplars []*dto.Exemplar + if len(exemplarNodes) > 0 { + exemplars = make([]*dto.Exemplar, 0, len(exemplarNodes)) + for _, en := range exemplarNodes { + ex, err := p.parseExemplar(input, en) + if err != nil { + return err + } + exemplars = append(exemplars, ex) + } + } + + var ( + scalarNode *goabnf.ParseTree + compositeNode *goabnf.ParseTree + ) + for _, vc := range valueNode.Children { + switch vc.Rule { + case "sample-number": + scalarNode = vc + case "composite-value": + compositeNode = vc + } + } + + mtype := st.mf.GetType() + switch mtype { + case dto.MetricType_COUNTER: + if scalarNode == nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("composite value not supported for counter %q", sampleName)} + } + valStr := string(input[scalarNode.Start:scalarNode.End]) + val, err := parseOpenMetricsFloat(valStr) + if err != nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid counter value %q: %v", valStr, err)} + } + if math.IsNaN(val) { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("counter sample value must not be NaN for %q", sampleName)} + } + if val < 0 { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("counter sample value must not be negative for %q", sampleName)} + } + // NOTE: dto.Counter only supports a single Exemplar field in client_model; + // if multiple exemplars are present on the wire, keep the first one. + var ex *dto.Exemplar + if len(exemplars) > 0 { + ex = exemplars[0] + } + st.mf.Metric = append(st.mf.Metric, &dto.Metric{ + Label: labelPairs, + Counter: &dto.Counter{ + Value: proto.Float64(val), + CreatedTimestamp: startTS, + Exemplar: ex, + }, + TimestampMs: tsMs, + }) + + case dto.MetricType_GAUGE: + if scalarNode == nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("composite value not supported for gauge %q", sampleName)} + } + if startTS != nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("start timestamp not allowed on gauge metric %q", sampleName)} + } + if len(exemplars) > 0 { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("exemplar not allowed on gauge metric %q", sampleName)} + } + valStr := string(input[scalarNode.Start:scalarNode.End]) + val, err := parseOpenMetricsFloat(valStr) + if err != nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid gauge value %q: %v", valStr, err)} + } + st.mf.Metric = append(st.mf.Metric, &dto.Metric{ + Label: labelPairs, + Gauge: &dto.Gauge{Value: proto.Float64(val)}, + TimestampMs: tsMs, + }) + + case dto.MetricType_UNTYPED: + if scalarNode == nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("composite value not supported for untyped %q", sampleName)} + } + if startTS != nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("start timestamp not allowed on untyped metric %q", sampleName)} + } + if len(exemplars) > 0 { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("exemplar not allowed on untyped metric %q", sampleName)} + } + valStr := string(input[scalarNode.Start:scalarNode.End]) + val, err := parseOpenMetricsFloat(valStr) + if err != nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid untyped value %q: %v", valStr, err)} + } + st.mf.Metric = append(st.mf.Metric, &dto.Metric{ + Label: labelPairs, + Untyped: &dto.Untyped{Value: proto.Float64(val)}, + TimestampMs: tsMs, + }) + + case dto.MetricType_SUMMARY: + if compositeNode == nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("composite value required for metric type summary %q", sampleName)} + } + if len(exemplars) > 0 { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("exemplar not allowed on summary metric %q", sampleName)} + } + summaryProto, err := parseCompositeSummary(p.lineNum, input, compositeNode, sampleName, labelPairs, startTS) + if err != nil { + return err + } + st.mf.Metric = append(st.mf.Metric, &dto.Metric{ + Label: labelPairs, + Summary: summaryProto, + TimestampMs: tsMs, + }) + + case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM: + if compositeNode == nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("composite value required for metric type %s %q", mtype, sampleName)} + } + isGauge := mtype == dto.MetricType_GAUGE_HISTOGRAM + if isGauge && startTS != nil { + return ParseError{Line: p.lineNum, Msg: fmt.Sprintf("start timestamp not allowed on gaugehistogram metric %q", sampleName)} + } + histProto, err := parseCompositeHistogram(p.lineNum, input, compositeNode, sampleName, labelPairs, isGauge, startTS, exemplars) + if err != nil { + return err + } + st.mf.Metric = append(st.mf.Metric, &dto.Metric{ + Label: labelPairs, + Histogram: histProto, + TimestampMs: tsMs, + }) + } + + return nil +} + +func (p *openMetrics20Parser) validateAndBuildLabels(mfName string, rawLabels []parsedLabel) ([]*dto.LabelPair, error) { + if len(rawLabels) == 0 { + return nil, nil + } + pairs := make([]*dto.LabelPair, 0, len(rawLabels)) + seen := make(map[string]struct{}, len(rawLabels)) + for _, rl := range rawLabels { + if rl.name == "" { + return nil, ParseError{Line: p.lineNum, Msg: "label name must not be empty"} + } + if rl.name == string(model.MetricNameLabel) { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("label name %q is reserved", model.MetricNameLabel)} + } + if !p.validationScheme().IsValidLabelName(rl.name) { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid label name %q", rl.name)} + } + if !model.LabelValue(rl.value).IsValid() { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid label value %q", rl.value)} + } + if _, dup := seen[rl.name]; dup { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("duplicate label %q for metric %q", rl.name, mfName)} + } + seen[rl.name] = struct{}{} + pairs = append(pairs, &dto.LabelPair{ + Name: proto.String(rl.name), + Value: proto.String(rl.value), + }) + } + return pairs, nil +} + +func (p *openMetrics20Parser) parseExemplar(input []byte, node *goabnf.ParseTree) (*dto.Exemplar, error) { + om1 := openMetricsParser{scheme: p.validationScheme(), lineNum: p.lineNum} + return om1.parseExemplar(input, node, false) +} + +func parseCompositeSummary( + lineNum int, + input []byte, + compNode *goabnf.ParseTree, + mfName string, + labels []*dto.LabelPair, + startTS *timestamppb.Timestamp, +) (*dto.Summary, error) { + for _, lp := range labels { + if lp.GetName() == model.QuantileLabel { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("summary %q label set must not contain %q label", mfName, model.QuantileLabel)} + } + } + + var sumNode *goabnf.ParseTree + for _, c := range compNode.Children { + if c.Rule == "summary-value" { + sumNode = c + break + } + } + if sumNode == nil { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("expected summary composite value for %q", mfName)} + } + + var ( + countStr string + sumStr string + quantiles []*dto.Quantile + ) + for _, c := range sumNode.Children { + switch c.Rule { + case "cs-count": + countStr = extractFirstRuleText(input, c, "number") + case "cs-sum": + sumStr = extractFirstRuleText(input, c, "number") + case "cs-quantile": + var prevQ float64 + for _, qc := range c.Children { + if qc.Rule != "cs-q-counts" { + continue + } + for i, qpair := range qc.Children { + if qpair.Rule != "cs-q-count" { + continue + } + qStr := extractFirstRuleText(input, qpair, "realnumber") + vStr := extractFirstRuleText(input, qpair, "number") + q, err := parseOpenMetricsFloat(qStr) + if err != nil || math.IsNaN(q) { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid quantile %q", qStr)} + } + if q < 0 || q > 1 { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("quantile must be between 0.0 and 1.0, got %g", q)} + } + if i > 0 && q <= prevQ { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("quantiles must be sorted in increasing order: %g <= %g", q, prevQ)} + } + prevQ = q + + v, err := parseOpenMetricsFloat(vStr) + if err != nil { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid quantile value %q: %v", vStr, err)} + } + quantiles = append(quantiles, &dto.Quantile{ + Quantile: proto.Float64(q), + Value: proto.Float64(v), + }) + } + } + } + } + + countUint, ok := parsePlainUint64(countStr) + if !ok { + f, err := parseOpenMetricsFloat(countStr) + if err != nil || math.IsNaN(f) || math.IsInf(f, 0) || f < 0 || f >= math.Exp2(64) || float64(uint64(f)) != f { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid summary count %q", countStr)} + } + countUint = uint64(f) + } + + // Per the OpenMetrics 2.0 spec, summary sum must not be NaN (unlike + // histogram sum, which can be NaN). + sumVal, err := parseOpenMetricsFloat(sumStr) + if err != nil || math.IsNaN(sumVal) { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid summary sum %q", sumStr)} + } + + return &dto.Summary{ + SampleCount: proto.Uint64(countUint), + SampleSum: proto.Float64(sumVal), + Quantile: quantiles, + CreatedTimestamp: startTS, + }, nil +} + +func parseCompositeHistogram( + lineNum int, + input []byte, + compNode *goabnf.ParseTree, + mfName string, + labels []*dto.LabelPair, + isGauge bool, + startTS *timestamppb.Timestamp, + exemplars []*dto.Exemplar, +) (*dto.Histogram, error) { + expectedRule := "histogram-value" + if isGauge { + expectedRule = "gauge-histogram-value" + } + + var hNode *goabnf.ParseTree + for _, c := range compNode.Children { + if c.Rule == expectedRule { + hNode = c + break + } + } + if hNode == nil { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("expected %s for %q", expectedRule, mfName)} + } + + var ( + countStr string + sumStr string + bucketsNode *goabnf.ParseTree + ) + for _, c := range hNode.Children { + switch c.Rule { + case "h-count": + countStr = extractFirstRuleText(input, c, "number") + case "h-sum": + sumStr = extractFirstRuleText(input, c, "number") + case "histogram-buckets": + bucketsNode = c + } + } + + countFloat, err := parseOpenMetricsFloat(countStr) + if err != nil || math.IsNaN(countFloat) { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid histogram count %q", countStr)} + } + if countFloat < 0 { + if isGauge { + return nil, ParseError{Line: lineNum, Msg: "gcount must not be negative"} + } + return nil, ParseError{Line: lineNum, Msg: "count must not be negative"} + } + countUint, countIsInt := parsePlainUint64(countStr) + + sumFloat, err := parseOpenMetricsFloat(sumStr) + if err != nil { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid histogram sum %q: %v", sumStr, err)} + } + + var ( + classicNode *goabnf.ParseTree + nativeNode *goabnf.ParseTree + ) + for _, c := range bucketsNode.Children { + switch c.Rule { + case "classic-buckets": + classicNode = c + case "native-buckets": + nativeNode = c + } + } + + // NOTE: OpenMetrics 2.0 has no distinct wire tag for integer vs float histograms, + // and writeFloat encodes whole-number floats without a trailing ".0". We classify + // a histogram as a FloatHistogram iff any count, zero_count, or bucket count token + // is not a plain integer (e.g. contains a decimal point, exponent, or "+Inf"). + // Consequently, a FloatHistogram whose counts and buckets are all whole integers + // on the wire decodes as an integer Histogram. + isFloatHistogram := !countIsInt + h := &dto.Histogram{ + SampleSum: proto.Float64(sumFloat), + CreatedTimestamp: startTS, + Exemplars: exemplars, + } + + type parsedClassicBucket struct { + ub float64 + fVal float64 + uVal uint64 + isInt bool + } + var classicBuckets []parsedClassicBucket + + if classicNode != nil { + for _, lp := range labels { + if lp.GetName() == model.BucketLabel { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("histogram %q with classic buckets must not contain %q label", mfName, model.BucketLabel)} + } + } + + var extractBucket func(n *goabnf.ParseTree) error + extractBucket = func(n *goabnf.ParseTree) error { + for _, c := range n.Children { + switch c.Rule { + case "ch-le-counts": + if err := extractBucket(c); err != nil { + return err + } + case "ch-neg-inf-bucket": + vStr := extractFirstRuleText(input, c, "number") + fv, err := parseOpenMetricsFloat(vStr) + if err != nil || math.IsNaN(fv) { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid bucket value %q", vStr)} + } + if fv < 0 { + return ParseError{Line: lineNum, Msg: "classic bucket value must not be negative"} + } + uv, isInt := parsePlainUint64(vStr) + if !isInt { + isFloatHistogram = true + } + classicBuckets = append(classicBuckets, parsedClassicBucket{ + ub: math.Inf(-1), fVal: fv, uVal: uv, isInt: isInt, + }) + case "ch-le-bucket": + ubStr := extractFirstRuleText(input, c, "realnumber") + vStr := extractFirstRuleText(input, c, "number") + ub, err := parseOpenMetricsFloat(ubStr) + if err != nil || math.IsNaN(ub) { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid bucket threshold %q", ubStr)} + } + fv, err := parseOpenMetricsFloat(vStr) + if err != nil || math.IsNaN(fv) { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid bucket value %q", vStr)} + } + if fv < 0 { + return ParseError{Line: lineNum, Msg: "classic bucket value must not be negative"} + } + uv, isInt := parsePlainUint64(vStr) + if !isInt { + isFloatHistogram = true + } + classicBuckets = append(classicBuckets, parsedClassicBucket{ + ub: ub, fVal: fv, uVal: uv, isInt: isInt, + }) + case "ch-pos-inf-bucket": + vStr := extractFirstRuleText(input, c, "number") + fv, err := parseOpenMetricsFloat(vStr) + if err != nil || math.IsNaN(fv) { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid bucket value %q", vStr)} + } + if fv < 0 { + return ParseError{Line: lineNum, Msg: "classic bucket value must not be negative"} + } + uv, isInt := parsePlainUint64(vStr) + if !isInt { + isFloatHistogram = true + } + classicBuckets = append(classicBuckets, parsedClassicBucket{ + ub: math.Inf(1), fVal: fv, uVal: uv, isInt: isInt, + }) + } + } + return nil + } + if err := extractBucket(classicNode); err != nil { + return nil, err + } + } + + var ( + zeroCountFloat float64 + zeroCountUint uint64 + negBucketStrs []string + posBucketStrs []string + ) + + if nativeNode != nil { + for _, c := range nativeNode.Children { + switch c.Rule { + case "nh-schema": + sStr := extractFirstRuleText(input, c, "integer") + schemaVal, err := strconv.ParseInt(sStr, 10, 64) + if err != nil || schemaVal < -4 || schemaVal > 8 { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("schema must be between -4 and 8, got %q", sStr)} + } + h.Schema = proto.Int32(int32(schemaVal)) + + case "nh-zero-threshold": + ztStr := extractFirstRuleText(input, c, "realnumber") + ztVal, err := parseOpenMetricsFloat(ztStr) + if err != nil || math.IsNaN(ztVal) || math.IsInf(ztVal, 0) || ztVal < 0 { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("zero_threshold must be a non-negative, finite number, got %q", ztStr)} + } + h.ZeroThreshold = proto.Float64(ztVal) + + case "nh-zero-count": + zcStr := extractFirstRuleText(input, c, "number") + zcVal, err := parseOpenMetricsFloat(zcStr) + if err != nil || math.IsNaN(zcVal) { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid zero_count %q", zcStr)} + } + if zcVal < 0 { + return nil, ParseError{Line: lineNum, Msg: "zero_count must not be negative"} + } + zeroCountFloat = zcVal + var zcIsInt bool + zeroCountUint, zcIsInt = parsePlainUint64(zcStr) + if !zcIsInt { + isFloatHistogram = true + } + + case "nh-negative-spans": + spans, err := parseNativeSpans(lineNum, input, c) + if err != nil { + return nil, err + } + h.NegativeSpan = spans + + case "nh-positive-spans": + spans, err := parseNativeSpans(lineNum, input, c) + if err != nil { + return nil, err + } + h.PositiveSpan = spans + + case "nh-negative-buckets": + negBucketStrs = extractNumberList(input, c) + for _, bs := range negBucketStrs { + if _, ok := parsePlainInt64(bs); !ok { + isFloatHistogram = true + } + } + + case "nh-positive-buckets": + posBucketStrs = extractNumberList(input, c) + for _, bs := range posBucketStrs { + if _, ok := parsePlainInt64(bs); !ok { + isFloatHistogram = true + } + } + } + } + } + + if isFloatHistogram { + h.SampleCountFloat = proto.Float64(countFloat) + } else { + h.SampleCount = proto.Uint64(countUint) + } + + if nativeNode != nil { + if isFloatHistogram { + h.ZeroCountFloat = proto.Float64(zeroCountFloat) + } else { + h.ZeroCount = proto.Uint64(zeroCountUint) + } + + if err := populateNativeBuckets(lineNum, h, isFloatHistogram, negBucketStrs, posBucketStrs); err != nil { + return nil, err + } + } + + if len(classicBuckets) > 0 { + h.Bucket = make([]*dto.Bucket, 0, len(classicBuckets)) + for i, cb := range classicBuckets { + if i > 0 { + if cb.ub <= classicBuckets[i-1].ub { + return nil, ParseError{Line: lineNum, Msg: "classic histogram buckets must be sorted in increasing order"} + } + if cb.fVal < classicBuckets[i-1].fVal { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("classic bucket counts must be monotonically increasing: %g < %g", cb.fVal, classicBuckets[i-1].fVal)} + } + } + if math.IsInf(cb.ub, +1) { + if isFloatHistogram { + if cb.fVal != countFloat { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("classic bucket +Inf count (%g) does not match sample count (%g)", cb.fVal, countFloat)} + } + } else if cb.uVal != countUint { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("classic bucket +Inf count (%d) does not match sample count (%d)", cb.uVal, countUint)} + } + } + b := &dto.Bucket{ + UpperBound: proto.Float64(cb.ub), + } + if isFloatHistogram { + b.CumulativeCountFloat = proto.Float64(cb.fVal) + } else { + b.CumulativeCount = proto.Uint64(cb.uVal) + } + h.Bucket = append(h.Bucket, b) + } + } + + return h, nil +} + +func parseNativeSpans(lineNum int, input []byte, node *goabnf.ParseTree) ([]*dto.BucketSpan, error) { + var spans []*dto.BucketSpan + for _, c := range node.Children { + if c.Rule != "nh-spans" { + continue + } + for _, sc := range c.Children { + if sc.Rule != "nh-start-span" && sc.Rule != "nh-span" { + continue + } + var offStr, lenStr string + for _, part := range sc.Children { + switch part.Rule { + case "integer": + offStr = string(input[part.Start:part.End]) + case "non-negative-integer": + if offStr == "" && sc.Rule == "nh-span" { + offStr = string(input[part.Start:part.End]) + } else { + lenStr = string(input[part.Start:part.End]) + } + } + } + off, err := strconv.ParseInt(offStr, 10, 32) + if err != nil { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid span offset %q: %v", offStr, err)} + } + length, err := strconv.ParseUint(strings.TrimPrefix(lenStr, "+"), 10, 32) + if err != nil { + return nil, ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid span length %q: %v", lenStr, err)} + } + spans = append(spans, &dto.BucketSpan{ + Offset: proto.Int32(int32(off)), + Length: proto.Uint32(uint32(length)), + }) + } + } + return spans, nil +} + +func populateNativeBuckets( + lineNum int, + h *dto.Histogram, + isFloat bool, + negStrs, posStrs []string, +) error { + var negSpanLen uint64 + for _, s := range h.NegativeSpan { + negSpanLen += uint64(s.GetLength()) + } + if negSpanLen != uint64(len(negStrs)) { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("negative_spans length (%d) does not match negative_buckets count (%d)", negSpanLen, len(negStrs))} + } + + var posSpanLen uint64 + for _, s := range h.PositiveSpan { + posSpanLen += uint64(s.GetLength()) + } + if posSpanLen != uint64(len(posStrs)) { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("positive_spans length (%d) does not match positive_buckets count (%d)", posSpanLen, len(posStrs))} + } + + if isFloat { + // NOTE: For float native histograms, we intentionally omit the exact + // bucket-observation-sum vs sample-count check to match Prometheus's + // histogram.FloatHistogram.Validate(). Summing many scaled IEEE-754 float64 + // bucket counts is subject to rounding drift and would cause false-positive + // rejections, whereas classic histogram "+Inf == count" compares two single + // cumulative numbers directly. + if len(negStrs) > 0 { + h.NegativeCount = make([]float64, 0, len(negStrs)) + for _, s := range negStrs { + fv, err := parseOpenMetricsFloat(s) + if err != nil || math.IsNaN(fv) || fv < 0 { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid negative_buckets value %q", s)} + } + h.NegativeCount = append(h.NegativeCount, fv) + } + } + if len(posStrs) > 0 { + h.PositiveCount = make([]float64, 0, len(posStrs)) + for _, s := range posStrs { + fv, err := parseOpenMetricsFloat(s) + if err != nil || math.IsNaN(fv) || fv < 0 { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid positive_buckets value %q", s)} + } + h.PositiveCount = append(h.PositiveCount, fv) + } + } + return nil + } + + var negTotal, posTotal uint64 + if len(negStrs) > 0 { + h.NegativeDelta = make([]int64, 0, len(negStrs)) + var prev int64 + for _, s := range negStrs { + iv, ok := parsePlainInt64(s) + if !ok || iv < 0 { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid negative_buckets value %q", s)} + } + var carry uint64 + negTotal, carry = bits.Add64(negTotal, uint64(iv), 0) + if carry != 0 { + return ParseError{Line: lineNum, Msg: "native histogram bucket observations overflow uint64"} + } + h.NegativeDelta = append(h.NegativeDelta, iv-prev) + prev = iv + } + } + if len(posStrs) > 0 { + h.PositiveDelta = make([]int64, 0, len(posStrs)) + var prev int64 + for _, s := range posStrs { + iv, ok := parsePlainInt64(s) + if !ok || iv < 0 { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("invalid positive_buckets value %q", s)} + } + var carry uint64 + posTotal, carry = bits.Add64(posTotal, uint64(iv), 0) + if carry != 0 { + return ParseError{Line: lineNum, Msg: "native histogram bucket observations overflow uint64"} + } + h.PositiveDelta = append(h.PositiveDelta, iv-prev) + prev = iv + } + } + if h.SampleCount != nil { + obs, c1 := bits.Add64(h.GetZeroCount(), negTotal, 0) + obs, c2 := bits.Add64(obs, posTotal, c1) + if c2 != 0 { + return ParseError{Line: lineNum, Msg: "native histogram bucket observations overflow uint64"} + } + if math.IsNaN(h.GetSampleSum()) { + if obs > h.GetSampleCount() { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("native histogram bucket observations (%d) exceed sample count (%d)", obs, h.GetSampleCount())} + } + } else if obs != h.GetSampleCount() { + return ParseError{Line: lineNum, Msg: fmt.Sprintf("native histogram bucket observations (%d) do not match sample count (%d)", obs, h.GetSampleCount())} + } + } + return nil +} + +func extractFirstRuleText(input []byte, node *goabnf.ParseTree, rule string) string { + for _, c := range node.Children { + if c.Rule == rule { + return string(input[c.Start:c.End]) + } + } + return "" +} + +func extractNumberList(input []byte, node *goabnf.ParseTree) []string { + var out []string + for _, c := range node.Children { + if c.Rule != "nh-buckets" { + continue + } + for _, nc := range c.Children { + if nc.Rule == "number" { + out = append(out, string(input[nc.Start:nc.End])) + } + } + } + return out +} + +func parsePlainUint64(s string) (uint64, bool) { + if s == "" || strings.ContainsAny(s, ".eE") { + return 0, false + } + s = strings.TrimPrefix(s, "+") + v, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, false + } + return v, true +} + +func parsePlainInt64(s string) (int64, bool) { + if s == "" || strings.ContainsAny(s, ".eE") { + return 0, false + } + v, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0, false + } + return v, true +} diff --git a/expfmt/openmetrics_2_0_parse_test.go b/expfmt/openmetrics_2_0_parse_test.go new file mode 100644 index 00000000..4ee85387 --- /dev/null +++ b/expfmt/openmetrics_2_0_parse_test.go @@ -0,0 +1,1813 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expfmt + +import ( + "bytes" + "errors" + "io" + "math" + "strings" + "testing" + + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/prometheus/common/model" +) + +func decodeOpenMetrics20ToMap(r io.Reader, format Format) (map[string]*dto.MetricFamily, error) { + dec := NewDecoder(r, format) + fams := make(map[string]*dto.MetricFamily) + for { + mf := &dto.MetricFamily{} + if err := dec.Decode(mf); err != nil { + if errors.Is(err, io.EOF) { + return fams, nil + } + return nil, err + } + if _, exists := fams[mf.GetName()]; exists { + return nil, errors.New("duplicate MetricFamily " + mf.GetName()) + } + fams[mf.GetName()] = mf + } +} + +func requireEqualFamilies(t *testing.T, expected, actual map[string]*dto.MetricFamily) { + t.Helper() + require.Len(t, actual, len(expected)) + for name, want := range expected { + got, ok := actual[name] + require.Truef(t, ok, "missing metric family %q", name) + assertProtoEqual(t, want, got, "metric family %q mismatch", name) + } +} + +func TestOpenMetrics2Parse(t *testing.T) { + longExemplarValue := strings.Repeat("x", 200) + + for _, tc := range []struct { + name string + input string + expected map[string]*dto.MetricFamily + }{ + { + name: "mixed exposition", + input: `# HELP go_gc_duration_seconds A summary of GC invocation durations. +# TYPE go_gc_duration_seconds summary +# UNIT go_gc_duration_seconds seconds +go_gc_duration_seconds {count:99,sum:0.004,quantile:[0:4.9351e-05,0.5:8.3835e-05]} +# TYPE go_goroutines gauge +go_goroutines 33 123456.789 +# TYPE hh histogram +hh {count:1,sum:1.0,bucket:[+Inf:1]} +# TYPE un unknown +un 1 +# TYPE foo_total counter +foo_total 17.0 1520879607.789 # {id="counter-test"} 5 1520879600.789 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "go_gc_duration_seconds": { + Name: proto.String("go_gc_duration_seconds"), + Help: proto.String("A summary of GC invocation durations."), + Type: dto.MetricType_SUMMARY.Enum(), + Unit: proto.String("seconds"), + Metric: []*dto.Metric{ + { + Summary: &dto.Summary{ + SampleCount: proto.Uint64(99), + SampleSum: proto.Float64(0.004), + Quantile: []*dto.Quantile{ + {Quantile: proto.Float64(0.0), Value: proto.Float64(4.9351e-05)}, + {Quantile: proto.Float64(0.5), Value: proto.Float64(8.3835e-05)}, + }, + }, + }, + }, + }, + "go_goroutines": { + Name: proto.String("go_goroutines"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Gauge: &dto.Gauge{Value: proto.Float64(33)}, + TimestampMs: proto.Int64(123456789), + }, + }, + }, + "hh": { + Name: proto.String("hh"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(1), + SampleSum: proto.Float64(1.0), + Bucket: []*dto.Bucket{ + { + CumulativeCount: proto.Uint64(1), + UpperBound: proto.Float64(math.Inf(1)), + }, + }, + }, + }, + }, + }, + "un": { + Name: proto.String("un"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Untyped: &dto.Untyped{Value: proto.Float64(1)}, + }, + }, + }, + "foo_total": { + Name: proto.String("foo_total"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(17.0), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + {Name: proto.String("id"), Value: proto.String("counter-test")}, + }, + Value: proto.Float64(5), + Timestamp: ×tamppb.Timestamp{Seconds: 1520879600, Nanos: 789000000}, + }, + }, + TimestampMs: proto.Int64(1520879607789), + }, + }, + }, + }, + }, + { + name: "start timestamp with main timestamp", + input: `# TYPE requests_total counter +requests_total 42.0 1000000.0 st@500000.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "requests_total": { + Name: proto.String("requests_total"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(42.0), + CreatedTimestamp: ×tamppb.Timestamp{Seconds: 500000, Nanos: 0}, + }, + TimestampMs: proto.Int64(1000000000), + }, + }, + }, + }, + }, + { + name: "start timestamp without main timestamp", + input: `# TYPE requests_total counter +requests_total 42.0 st@500000.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "requests_total": { + Name: proto.String("requests_total"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(42.0), + CreatedTimestamp: ×tamppb.Timestamp{Seconds: 500000, Nanos: 0}, + }, + }, + }, + }, + }, + }, + { + // dto.Counter has a single Exemplar field, so it retains the first exemplar; + // dto.Histogram has an Exemplars slice that holds all exemplars. + name: "multiple exemplars on counter", + input: `# TYPE foo_total counter +foo_total 17.0 # {id="a"} 1.0 1234566.0 # {id="b"} 2.0 1234567.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "foo_total": { + Name: proto.String("foo_total"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(17.0), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{{Name: proto.String("id"), Value: proto.String("a")}}, + Value: proto.Float64(1.0), + Timestamp: ×tamppb.Timestamp{Seconds: 1234566, Nanos: 0}, + }, + }, + }, + }, + }, + }, + }, + { + name: "exemplar with multiple labels", + input: `# TYPE http_requests_total counter +http_requests_total 1027 1710000000.0 # {trace_id="abc123",span_id="def456"} 1.0 1709999999.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "http_requests_total": { + Name: proto.String("http_requests_total"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(1027), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + {Name: proto.String("trace_id"), Value: proto.String("abc123")}, + {Name: proto.String("span_id"), Value: proto.String("def456")}, + }, + Value: proto.Float64(1.0), + Timestamp: ×tamppb.Timestamp{Seconds: 1709999999, Nanos: 0}, + }, + }, + TimestampMs: proto.Int64(1710000000000), + }, + }, + }, + }, + }, + { + name: "exemplar exceeding OM1 rune limit", + input: `# TYPE foo_total counter +foo_total 1.0 # {data="` + longExemplarValue + `"} 1.0 1234567.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "foo_total": { + Name: proto.String("foo_total"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(1.0), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + {Name: proto.String("data"), Value: proto.String(longExemplarValue)}, + }, + Value: proto.Float64(1.0), + Timestamp: ×tamppb.Timestamp{Seconds: 1234567, Nanos: 0}, + }, + }, + }, + }, + }, + }, + }, + { + name: "exemplar with quoted UTF-8 label key", + input: `# TYPE foo_total counter +foo_total 17.0 # {"my.id"="abc"} 5 1520879600.789 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "foo_total": { + Name: proto.String("foo_total"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(17.0), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + {Name: proto.String("my.id"), Value: proto.String("abc")}, + }, + Value: proto.Float64(5), + Timestamp: ×tamppb.Timestamp{Seconds: 1520879600, Nanos: 789000000}, + }, + }, + }, + }, + }, + }, + }, + { + name: "composite classic gauge histogram", + input: `# TYPE foo gaugehistogram +foo {gcount:42,gsum:3289.3,bucket:[0.01:20,0.1:25,1:34,+Inf:42]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "foo": { + Name: proto.String("foo"), + Type: dto.MetricType_GAUGE_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(42), + SampleSum: proto.Float64(3289.3), + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(20), UpperBound: proto.Float64(0.01)}, + {CumulativeCount: proto.Uint64(25), UpperBound: proto.Float64(0.1)}, + {CumulativeCount: proto.Uint64(34), UpperBound: proto.Float64(1)}, + {CumulativeCount: proto.Uint64(42), UpperBound: proto.Float64(math.Inf(1))}, + }, + }, + }, + }, + }, + }, + }, + { + name: "composite summary", + input: `# HELP rpc_duration_seconds RPC duration. +# TYPE rpc_duration_seconds summary +rpc_duration_seconds {count:100,sum:30000.0,quantile:[0.5:100.0,0.9:200.0,0.99:300.0]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "rpc_duration_seconds": { + Name: proto.String("rpc_duration_seconds"), + Help: proto.String("RPC duration."), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Summary: &dto.Summary{ + SampleCount: proto.Uint64(100), + SampleSum: proto.Float64(30000.0), + Quantile: []*dto.Quantile{ + {Quantile: proto.Float64(0.5), Value: proto.Float64(100.0)}, + {Quantile: proto.Float64(0.9), Value: proto.Float64(200.0)}, + {Quantile: proto.Float64(0.99), Value: proto.Float64(300.0)}, + }, + }, + }, + }, + }, + }, + }, + { + name: "composite summary with negative sum and quantile values", + input: `# TYPE temp_delta_celsius summary +temp_delta_celsius {count:10,sum:-15.5,quantile:[0.5:-2.5,0.9:1.0]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "temp_delta_celsius": { + Name: proto.String("temp_delta_celsius"), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Summary: &dto.Summary{ + SampleCount: proto.Uint64(10), + SampleSum: proto.Float64(-15.5), + Quantile: []*dto.Quantile{ + {Quantile: proto.Float64(0.5), Value: proto.Float64(-2.5)}, + {Quantile: proto.Float64(0.9), Value: proto.Float64(1.0)}, + }, + }, + }, + }, + }, + }, + }, + { + name: "composite classic histogram", + input: `# HELP req_duration Request duration histogram. +# TYPE req_duration histogram +req_duration {count:3,sum:6.0,bucket:[0.1:1,1.0:2,+Inf:3]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "req_duration": { + Name: proto.String("req_duration"), + Help: proto.String("Request duration histogram."), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(3), + SampleSum: proto.Float64(6.0), + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(1), UpperBound: proto.Float64(0.1)}, + {CumulativeCount: proto.Uint64(2), UpperBound: proto.Float64(1.0)}, + {CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(math.Inf(1))}, + }, + }, + }, + }, + }, + }, + }, + { + name: "classic histogram with -Inf bucket", + input: `# TYPE req_duration histogram +req_duration {count:3,sum:6.0,bucket:[-Inf:1,1.0:2,+Inf:3]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "req_duration": { + Name: proto.String("req_duration"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(3), + SampleSum: proto.Float64(6.0), + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(1), UpperBound: proto.Float64(math.Inf(-1))}, + {CumulativeCount: proto.Uint64(2), UpperBound: proto.Float64(1.0)}, + {CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(math.Inf(1))}, + }, + }, + }, + }, + }, + }, + }, + { + name: "composite native and classic histogram", + input: `# HELP req_duration Request duration histogram with both representations. +# TYPE req_duration histogram +req_duration {count:3,sum:6.0,schema:0,zero_threshold:0.001,zero_count:0,positive_spans:[0:2],positive_buckets:[1,2],bucket:[0.1:1,1.0:2,+Inf:3]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "req_duration": { + Name: proto.String("req_duration"), + Help: proto.String("Request duration histogram with both representations."), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(3), + SampleSum: proto.Float64(6.0), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(0), + PositiveSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(2)}, + }, + PositiveDelta: []int64{1, 1}, + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(1), UpperBound: proto.Float64(0.1)}, + {CumulativeCount: proto.Uint64(2), UpperBound: proto.Float64(1.0)}, + {CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(math.Inf(1))}, + }, + }, + }, + }, + }, + }, + }, + { + name: "native histogram with empty negative spans and buckets", + input: `# HELP test_histogram Native histogram. +# TYPE test_histogram histogram +test_histogram {count:6,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,negative_spans:[],negative_buckets:[],positive_spans:[0:3],positive_buckets:[1,2,1]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "test_histogram": { + Name: proto.String("test_histogram"), + Help: proto.String("Native histogram."), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(6), + SampleSum: proto.Float64(12.1), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(2), + PositiveSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(3)}, + }, + PositiveDelta: []int64{1, 1, -1}, + }, + }, + }, + }, + }, + }, + { + name: "native histogram with multiple positive and negative spans", + input: `# TYPE multi_span histogram +multi_span {count:8,sum:15.0,schema:0,zero_threshold:0.001,zero_count:1,negative_spans:[-2:1,1:1],negative_buckets:[1,2],positive_spans:[0:2,1:2],positive_buckets:[1,1,1,1]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "multi_span": { + Name: proto.String("multi_span"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(8), + SampleSum: proto.Float64(15.0), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(1), + NegativeSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(-2), Length: proto.Uint32(1)}, + {Offset: proto.Int32(1), Length: proto.Uint32(1)}, + }, + NegativeDelta: []int64{1, 1}, + PositiveSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(2)}, + {Offset: proto.Int32(1), Length: proto.Uint32(2)}, + }, + PositiveDelta: []int64{1, 0, 0, 0}, + }, + }, + }, + }, + }, + }, + { + name: "native histogram with omitted empty negative fields", + input: `# TYPE test_histogram histogram +test_histogram {count:6,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:3],positive_buckets:[1,2,1]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "test_histogram": { + Name: proto.String("test_histogram"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(6), + SampleSum: proto.Float64(12.1), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(2), + PositiveSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(3)}, + }, + PositiveDelta: []int64{1, 1, -1}, + }, + }, + }, + }, + }, + }, + { + name: "native histogram with omitted empty positive fields", + input: `# TYPE test_histogram histogram +test_histogram {count:6,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,negative_spans:[0:3],negative_buckets:[1,2,1]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "test_histogram": { + Name: proto.String("test_histogram"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(6), + SampleSum: proto.Float64(12.1), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(2), + NegativeSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(3)}, + }, + NegativeDelta: []int64{1, 1, -1}, + }, + }, + }, + }, + }, + }, + { + name: "UTF-8 metric name", + input: `# TYPE "metric.name" gauge +{"metric.name",label="value"} 1.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "metric.name": { + Name: proto.String("metric.name"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + {Name: proto.String("label"), Value: proto.String("value")}, + }, + Gauge: &dto.Gauge{Value: proto.Float64(1.0)}, + }, + }, + }, + }, + }, + { + name: "unknown metric with explicit help and type", + input: `# HELP un Some unknown metric. +# TYPE un unknown +un 1 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "un": { + Name: proto.String("un"), + Help: proto.String("Some unknown metric."), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + {Untyped: &dto.Untyped{Value: proto.Float64(1)}}, + }, + }, + }, + }, + { + name: "unknown metric with no type declaration", + input: `bare_metric 42 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "bare_metric": { + Name: proto.String("bare_metric"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + {Untyped: &dto.Untyped{Value: proto.Float64(42)}}, + }, + }, + }, + }, + { + name: "unknown metric with explicit type and unit", + input: `# TYPE un_seconds unknown +# UNIT un_seconds seconds +un_seconds 1.5 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "un_seconds": { + Name: proto.String("un_seconds"), + Type: dto.MetricType_UNTYPED.Enum(), + Unit: proto.String("seconds"), + Metric: []*dto.Metric{ + {Untyped: &dto.Untyped{Value: proto.Float64(1.5)}}, + }, + }, + }, + }, + { + name: "composite extra labels with UTF-8 metric name", + input: `# TYPE "req.duration" histogram +{"req.duration",job="api"} {count:3,sum:6.0,bucket:[0.1:1,+Inf:3]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "req.duration": { + Name: proto.String("req.duration"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + {Name: proto.String("job"), Value: proto.String("api")}, + }, + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(3), + SampleSum: proto.Float64(6.0), + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(1), UpperBound: proto.Float64(0.1)}, + {CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(math.Inf(1))}, + }, + }, + }, + }, + }, + }, + }, + { + name: "composite extra labels with multiple labels", + input: `# TYPE req_duration histogram +req_duration{job="api",env="prod"} {count:3,sum:6.0,bucket:[0.1:1,1.0:2,+Inf:3]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "req_duration": { + Name: proto.String("req_duration"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + {Name: proto.String("job"), Value: proto.String("api")}, + {Name: proto.String("env"), Value: proto.String("prod")}, + }, + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(3), + SampleSum: proto.Float64(6.0), + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(1), UpperBound: proto.Float64(0.1)}, + {CumulativeCount: proto.Uint64(2), UpperBound: proto.Float64(1.0)}, + {CumulativeCount: proto.Uint64(3), UpperBound: proto.Float64(math.Inf(1))}, + }, + }, + }, + }, + }, + }, + }, + { + name: "composite extra labels with quoted UTF-8 label containing equals", + input: `# TYPE foo summary +foo{"a=b"="v",job="api"} {count:1,sum:2.0,quantile:[0.5:1.0]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "foo": { + Name: proto.String("foo"), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + {Name: proto.String("a=b"), Value: proto.String("v")}, + {Name: proto.String("job"), Value: proto.String("api")}, + }, + Summary: &dto.Summary{ + SampleCount: proto.Uint64(1), + SampleSum: proto.Float64(2.0), + Quantile: []*dto.Quantile{ + {Quantile: proto.Float64(0.5), Value: proto.Float64(1.0)}, + }, + }, + }, + }, + }, + }, + }, + { + name: "composite histogram with timestamp and start timestamp", + input: `# TYPE req_duration histogram +req_duration {count:2,sum:4.0,bucket:[+Inf:2]} 1234567.0 st@1000.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "req_duration": { + Name: proto.String("req_duration"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(2), + SampleSum: proto.Float64(4.0), + CreatedTimestamp: ×tamppb.Timestamp{Seconds: 1000, Nanos: 0}, + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(2), UpperBound: proto.Float64(math.Inf(1))}, + }, + }, + TimestampMs: proto.Int64(1234567000), + }, + }, + }, + }, + }, + { + name: "float native histogram", + input: `# TYPE test_histogram histogram +test_histogram {count:5.5,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2.5,negative_spans:[],negative_buckets:[],positive_spans:[0:2],positive_buckets:[2.0,1.0]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "test_histogram": { + Name: proto.String("test_histogram"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCountFloat: proto.Float64(5.5), + SampleSum: proto.Float64(12.1), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(2.5), + PositiveSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(2)}, + }, + PositiveCount: []float64{2.0, 1.0}, + }, + }, + }, + }, + }, + }, + { + name: "composite native gauge histogram", + input: `# TYPE req_size gaugehistogram +req_size {gcount:6,gsum:100.0,schema:0,zero_threshold:0.001,zero_count:1,negative_spans:[],negative_buckets:[],positive_spans:[0:2],positive_buckets:[3,2]} +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "req_size": { + Name: proto.String("req_size"), + Type: dto.MetricType_GAUGE_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(6), + SampleSum: proto.Float64(100.0), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(1), + PositiveSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(2)}, + }, + PositiveDelta: []int64{3, -1}, + }, + }, + }, + }, + }, + }, + { + name: "exemplar on composite histogram line", + input: `# TYPE req_duration histogram +req_duration {count:2,sum:4.0,bucket:[1.0:1,+Inf:2]} # {id="req-1"} 3.8 9999999.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "req_duration": { + Name: proto.String("req_duration"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(2), + SampleSum: proto.Float64(4.0), + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(1), UpperBound: proto.Float64(1.0)}, + {CumulativeCount: proto.Uint64(2), UpperBound: proto.Float64(math.Inf(1))}, + }, + Exemplars: []*dto.Exemplar{ + { + Label: []*dto.LabelPair{ + {Name: proto.String("id"), Value: proto.String("req-1")}, + }, + Value: proto.Float64(3.8), + Timestamp: ×tamppb.Timestamp{Seconds: 9999999, Nanos: 0}, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "descriptor ordering: TYPE before HELP and UNIT", + input: `# TYPE open_fds gauge +# HELP open_fds Number of open file descriptors. +# UNIT open_fds fds +open_fds 8.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "open_fds": { + Name: proto.String("open_fds"), + Help: proto.String("Number of open file descriptors."), + Type: dto.MetricType_GAUGE.Enum(), + Unit: proto.String("fds"), + Metric: []*dto.Metric{{Gauge: &dto.Gauge{Value: proto.Float64(8.0)}}}, + }, + }, + }, + { + name: "descriptor ordering: UNIT before TYPE", + input: `# UNIT http_requests requests +# TYPE http_requests counter +http_requests 1.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "http_requests": { + Name: proto.String("http_requests"), + Type: dto.MetricType_COUNTER.Enum(), + Unit: proto.String("requests"), + Metric: []*dto.Metric{{Counter: &dto.Counter{Value: proto.Float64(1.0)}}}, + }, + }, + }, + { + name: "family reset: undeclared metric after declared metric", + input: `# TYPE http_requests counter +# UNIT http_requests requests +http_requests 1.0 +bare_metric 2.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "http_requests": { + Name: proto.String("http_requests"), + Type: dto.MetricType_COUNTER.Enum(), + Unit: proto.String("requests"), + Metric: []*dto.Metric{{Counter: &dto.Counter{Value: proto.Float64(1.0)}}}, + }, + "bare_metric": { + Name: proto.String("bare_metric"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{{Untyped: &dto.Untyped{Value: proto.Float64(2.0)}}}, + }, + }, + }, + { + // "foo" has no samples so it is omitted; "foo_total" is parsed as its own untyped family. + name: "family reset: suffix mismatch reverts to untyped", + input: `# TYPE foo counter +foo_total 1.0 +# EOF +`, + expected: map[string]*dto.MetricFamily{ + "foo_total": { + Name: proto.String("foo_total"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{{Untyped: &dto.Untyped{Value: proto.Float64(1.0)}}}, + }, + }, + }, + { + name: "exemplar with escaped label names and null bytes", + input: "# TYPE foo_total counter\nfoo_total 1.0 1700000000.0 # {\"trace\\\\id\\n\\\"1\\\"\"=\"a\x00b\\nc\"} 1.0 1700000000.5\n# EOF\n", + expected: map[string]*dto.MetricFamily{ + "foo_total": { + Name: proto.String("foo_total"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(1.0), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + {Name: proto.String("trace\\id\n\"1\""), Value: proto.String("a\x00b\nc")}, + }, + Value: proto.Float64(1.0), + Timestamp: ×tamppb.Timestamp{Seconds: 1700000000, Nanos: 500000000}, + }, + }, + TimestampMs: proto.Int64(1700000000000), + }, + }, + }, + }, + }, + { + name: "carriage return in HELP", + input: "# HELP foo first\rsecond\nfoo 1\n# EOF\n", + expected: map[string]*dto.MetricFamily{ + "foo": { + Name: proto.String("foo"), + Help: proto.String("first\rsecond"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{{Untyped: &dto.Untyped{Value: proto.Float64(1)}}}, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + fams, err := decodeOpenMetrics20ToMap(strings.NewReader(tc.input), FmtOpenMetrics_2_0_0) + require.NoError(t, err) + requireEqualFamilies(t, tc.expected, fams) + }) + } + + t.Run("NaN and Inf exemplar values", func(t *testing.T) { + nanInfInput := "# TYPE c_total counter\nc_total{k=\"nan\"} 1.0 # {id=\"1\"} NaN 1700000000.0\nc_total{k=\"inf\"} 2.0 # {id=\"2\"} +Inf 1700000000.0\n# EOF\n" + got, err := decodeOpenMetrics20ToMap(strings.NewReader(nanInfInput), FmtOpenMetrics_2_0_0) + require.NoError(t, err) + require.Len(t, got["c_total"].GetMetric(), 2) + require.True(t, math.IsNaN(got["c_total"].GetMetric()[0].GetCounter().GetExemplar().GetValue())) + require.True(t, math.IsInf(got["c_total"].GetMetric()[1].GetCounter().GetExemplar().GetValue(), 1)) + }) +} + +func TestOpenMetrics2ParseNativeHistogramIntVsFloatDiscriminator(t *testing.T) { + for _, tc := range []struct { + name string + input string + hist *dto.Histogram + }{ + { + name: "all integer fields -> integer Histogram", + input: `h {count:5,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:2],positive_buckets:[2,1]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(2), SampleCount: proto.Uint64(5), SampleSum: proto.Float64(12.1), + PositiveSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + PositiveDelta: []int64{2, -1}, + }, + }, + { + name: "fractional count -> FloatHistogram", + input: `h {count:5.5,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:2],positive_buckets:[2,1]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(2), SampleCountFloat: proto.Float64(5.5), SampleSum: proto.Float64(12.1), + PositiveSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + PositiveCount: []float64{2, 1}, + }, + }, + { + name: "fractional zero_count -> FloatHistogram", + input: `h {count:5,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2.5,positive_spans:[0:2],positive_buckets:[2,1]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(2.5), SampleCountFloat: proto.Float64(5), SampleSum: proto.Float64(12.1), + PositiveSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + PositiveCount: []float64{2, 1}, + }, + }, + { + name: "fractional positive bucket -> FloatHistogram", + input: `h {count:5,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:2],positive_buckets:[2.0,1.0]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(2), SampleCountFloat: proto.Float64(5), SampleSum: proto.Float64(12.1), + PositiveSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + PositiveCount: []float64{2.0, 1.0}, + }, + }, + { + name: "fractional negative bucket -> FloatHistogram", + input: `h {count:5,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,negative_spans:[0:2],negative_buckets:[2.0,1.0]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(2), SampleCountFloat: proto.Float64(5), SampleSum: proto.Float64(12.1), + NegativeSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + NegativeCount: []float64{2.0, 1.0}, + }, + }, + { + name: "exponent notation in bucket -> FloatHistogram", + input: `h {count:5,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:2],positive_buckets:[1e2,1]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(2), SampleCountFloat: proto.Float64(5), SampleSum: proto.Float64(12.1), + PositiveSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + PositiveCount: []float64{1e2, 1}, + }, + }, + { + name: "count with no uint64 representation -> FloatHistogram", + input: `h {count:1e20,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:2],positive_buckets:[2,1]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(2), SampleCountFloat: proto.Float64(1e20), SampleSum: proto.Float64(12.1), + PositiveSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + PositiveCount: []float64{2, 1}, + }, + }, + { + name: "infinite count -> FloatHistogram", + input: `h {count:+Inf,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:2],positive_buckets:[2,1]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(2), SampleCountFloat: proto.Float64(math.Inf(1)), SampleSum: proto.Float64(12.1), + PositiveSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + PositiveCount: []float64{2, 1}, + }, + }, + { + name: "zero_count with no uint64 representation -> FloatHistogram", + input: `h {count:5,sum:12.1,schema:0,zero_threshold:0.001,zero_count:1e20,positive_spans:[0:2],positive_buckets:[2,1]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(1e20), SampleCountFloat: proto.Float64(5), SampleSum: proto.Float64(12.1), + PositiveSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + PositiveCount: []float64{2, 1}, + }, + }, + { + name: "fractional sum only -> integer Histogram (sum does not discriminate)", + input: `h {count:5,sum:12.5,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:2],positive_buckets:[2,1]}`, + hist: &dto.Histogram{ + Schema: proto.Int32(0), ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(2), SampleCount: proto.Uint64(5), SampleSum: proto.Float64(12.5), + PositiveSpan: []*dto.BucketSpan{{Offset: proto.Int32(0), Length: proto.Uint32(2)}}, + PositiveDelta: []int64{2, -1}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + input := "# TYPE h histogram\n" + tc.input + "\n# EOF\n" + fams, err := decodeOpenMetrics20ToMap(strings.NewReader(input), FmtOpenMetrics_2_0_0) + require.NoError(t, err) + expected := map[string]*dto.MetricFamily{ + "h": { + Name: proto.String("h"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{{Histogram: tc.hist}}, + }, + } + requireEqualFamilies(t, expected, fams) + }) + } +} + +func TestOpenMetrics2ParseUnsupportedTypes(t *testing.T) { + tests := []struct { + name string + input string + err string + }{ + { + name: "info metric type", + input: `# TYPE ii_info info +ii_info{foo="bar"} 1 +# EOF +`, + err: `unsupported metric type "info"`, + }, + { + name: "stateset metric type", + input: `# TYPE ss stateset +ss{ss="foo"} 1 +# EOF +`, + err: `unsupported metric type "stateset"`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := decodeOpenMetrics20ToMap(strings.NewReader(tc.input), FmtOpenMetrics_2_0_0) + require.ErrorContains(t, err, tc.err) + }) + } +} + +func TestOpenMetrics2ParseErrors(t *testing.T) { + cases := []struct { + name string + input string + err string + }{ + { + name: "missing EOF", + input: "metric 1.0\n", + err: "data does not end with # EOF", + }, + { + name: "extra data after EOF", + input: "# EOF\nextra\n", + err: "unexpected data after # EOF", + }, + { + name: "composite value with unknown type", + input: `foo {count:1,sum:1.0,bucket:[+Inf:1]} +# EOF +`, + err: "composite value not supported", + }, + { + name: "exemplar missing timestamp", + input: `# TYPE foo_total counter +foo_total 1.0 # {id="x"} 1.0 +# EOF +`, + err: "parse error", + }, + { + name: "histogram missing count", + input: "# TYPE foo histogram\nfoo {sum:1.0,schema:0,zero_threshold:0,zero_count:0}\n# EOF\n", + err: "parse error", + }, + { + name: "histogram missing sum", + input: "# TYPE foo histogram\nfoo {count:1,schema:0,zero_threshold:0,zero_count:0}\n# EOF\n", + err: "parse error", + }, + { + name: "histogram missing zero_threshold", + input: "# TYPE foo histogram\nfoo {count:1,sum:1.0,schema:0,zero_count:0}\n# EOF\n", + err: "parse error", + }, + { + name: "histogram missing zero_count", + input: "# TYPE foo histogram\nfoo {count:1,sum:1.0,schema:0,zero_threshold:0}\n# EOF\n", + err: "parse error", + }, + { + name: "duplicate timestamp", + input: "# TYPE requests_total counter\nrequests_total 42.0 1000000.0 2000000.0\n# EOF\n", + err: "parse error", + }, + { + name: "duplicate start timestamp", + input: "# TYPE requests_total counter\nrequests_total 42.0 1000000.0 st@500000.0 st@600000.0\n# EOF\n", + err: "parse error", + }, + { + name: "start timestamp before timestamp", + input: "# TYPE requests_total counter\nrequests_total 42.0 st@500000.0 1000000.0\n# EOF\n", + err: "parse error", + }, + { + name: "unsorted quantiles", + input: "# TYPE foo summary\nfoo {count:3,sum:6.0,quantile:[0.99:1.0,0.5:2.0]}\n# EOF\n", + err: "quantiles must be sorted in increasing order", + }, + { + name: "invalid UTF-8 metric name in sample", + input: "{\"a\xffb\"} 1.0\n# EOF\n", + err: "parse error", + }, + { + name: "invalid UTF-8 metric name in TYPE", + input: "# TYPE \"a\xffb\" counter\n# EOF\n", + err: "parse error", + }, + { + name: "invalid UTF-8 label name", + input: "# TYPE foo counter\nfoo{\"a\xffb\"=\"v\"} 1.0\n# EOF\n", + err: "parse error", + }, + { + name: "empty quoted metric name in sample", + input: "{\"\",label=\"v\"} 1.0\n# EOF\n", + err: "parse error", + }, + { + name: "empty quoted metric name in TYPE", + input: "# TYPE \"\" counter\n# EOF\n", + err: "parse error", + }, + { + name: "metric name not first in braces", + input: "{label=\"v\",\"m.n\"} 1.0\n# EOF\n", + err: "parse error", + }, + { + name: "unsorted classic histogram buckets", + input: "# TYPE foo histogram\nfoo {count:3,sum:6.0,bucket:[+Inf:3,0.1:1,1.0:2]}\n# EOF\n", + err: "parse error", + }, + { + name: "unsorted finite classic histogram buckets", + input: "# TYPE foo histogram\nfoo {count:3,sum:6.0,bucket:[1.0:2,0.1:1,+Inf:3]}\n# EOF\n", + err: "classic histogram buckets must be sorted in increasing order", + }, + { + name: "duplicate composite field", + input: "# TYPE foo histogram\nfoo {count:1,count:99,sum:2.0,bucket:[+Inf:1]}\n# EOF\n", + err: "parse error", + }, + { + name: "unknown composite field on histogram", + input: "# TYPE foo histogram\nfoo {count:5,sum:2.0,buckets:[1:3,+Inf:5]}\n# EOF\n", + err: "parse error", + }, + { + name: "unknown composite field on summary", + input: "# TYPE foo summary\nfoo {count:1,sum:2.0,quantiles:[0.5:1.0]}\n# EOF\n", + err: "parse error", + }, + { + name: "schema > 8", + input: "# TYPE foo histogram\nfoo {count:1,sum:1.0,schema:9,zero_threshold:0,zero_count:0}\n# EOF\n", + err: "schema must be between -4 and 8", + }, + { + name: "schema < -4", + input: "# TYPE foo histogram\nfoo {count:1,sum:1.0,schema:-5,zero_threshold:0,zero_count:0}\n# EOF\n", + err: "schema must be between -4 and 8", + }, + { + name: "schema overflow", + input: "# TYPE foo histogram\nfoo {count:1,sum:1.0,schema:4294967296,zero_threshold:0,zero_count:0}\n# EOF\n", + err: "schema must be between -4 and 8", + }, + { + name: "negative zero_threshold", + input: "# TYPE foo histogram\nfoo {count:1,sum:1.0,schema:0,zero_threshold:-1,zero_count:0}\n# EOF\n", + err: "zero_threshold must be a non-negative, finite number", + }, + { + name: "NaN zero_threshold", + input: "# TYPE foo histogram\nfoo {count:1,sum:1.0,schema:0,zero_threshold:NaN,zero_count:0}\n# EOF\n", + err: "parse error", + }, + { + name: "+Inf zero_threshold", + input: "# TYPE foo histogram\nfoo {count:1,sum:1.0,schema:0,zero_threshold:+Inf,zero_count:0}\n# EOF\n", + err: "parse error", + }, + { + name: "scalar value for histogram", + input: "# TYPE foo histogram\nfoo 5\n# EOF\n", + err: "composite value required for metric type", + }, + { + name: "scalar value for gaugehistogram", + input: "# TYPE foo gaugehistogram\nfoo 5\n# EOF\n", + err: "composite value required for metric type", + }, + { + name: "scalar value for summary", + input: "# TYPE foo summary\nfoo 5\n# EOF\n", + err: "composite value required for metric type", + }, + { + name: "classic histogram missing +Inf bucket", + input: "# TYPE foo histogram\nfoo {count:1,sum:2.0,bucket:[0.1:1]}\n# EOF\n", + err: "parse error", + }, + { + name: "counter NaN value", + input: "# TYPE foo_total counter\nfoo_total NaN\n# EOF\n", + err: "counter sample value must not be NaN", + }, + { + name: "negative histogram count", + input: "# TYPE foo histogram\nfoo {count:-1,sum:NaN,schema:0,zero_threshold:0,zero_count:0,positive_spans:[0:2],positive_buckets:[2,1]}\n# EOF\n", + err: "count must not be negative", + }, + { + name: "negative gaugehistogram gcount", + input: "# TYPE foo gaugehistogram\nfoo {gcount:-1,gsum:NaN,schema:0,zero_threshold:0,zero_count:0,positive_spans:[0:2],positive_buckets:[2,1]}\n# EOF\n", + err: "gcount must not be negative", + }, + { + name: "negative zero_count", + input: "# TYPE foo histogram\nfoo {count:1,sum:1.0,schema:0,zero_threshold:0,zero_count:-1}\n# EOF\n", + err: "zero_count must not be negative", + }, + { + name: "negative classic bucket count", + input: "# TYPE foo histogram\nfoo {count:3,sum:6.0,bucket:[1.0:-1,+Inf:3]}\n# EOF\n", + err: "value must not be negative", + }, + { + name: "invalid classic bucket threshold", + input: "# TYPE foo histogram\nfoo {count:3,sum:6.0,bucket:[abc:1,+Inf:3]}\n# EOF\n", + err: "parse error", + }, + { + name: "NaN classic bucket threshold", + input: "# TYPE foo histogram\nfoo {count:3,sum:6.0,bucket:[NaN:1,+Inf:3]}\n# EOF\n", + err: "parse error", + }, + { + name: "-Inf not first bucket", + input: "# TYPE foo histogram\nfoo {count:3,sum:6.0,bucket:[1.0:1,-Inf:2,+Inf:3]}\n# EOF\n", + err: "parse error", + }, + { + name: "invalid summary quantile", + input: "# TYPE foo summary\nfoo {count:1,sum:2.0,quantile:[abc:1.0]}\n# EOF\n", + err: "parse error", + }, + { + name: "summary quantile > 1.0", + input: "# TYPE foo summary\nfoo {count:1,sum:2.0,quantile:[1.5:1.0]}\n# EOF\n", + err: "quantile must be between 0.0 and 1.0", + }, + { + name: "NaN summary quantile", + input: "# TYPE foo summary\nfoo {count:1,sum:2.0,quantile:[NaN:1.0]}\n# EOF\n", + err: "parse error", + }, + { + name: "NaN summary sum", + input: "# TYPE foo summary\nfoo {count:1,sum:NaN,quantile:[0.5:1.0]}\n# EOF\n", + err: `invalid summary sum "NaN"`, + }, + { + name: "empty quoted label name in sample", + input: "foo{\"\"=\"val\"} 1.0\n# EOF\n", + err: "parse error", + }, + { + name: "empty quoted label name in exemplar", + input: "# TYPE foo counter\nfoo 1.0 # {\"\"=\"val\"} 1.0 1700000000.0\n# EOF\n", + err: "parse error", + }, + { + name: "carriage return in HELP line", + input: "# HELP foo help text\r\nfoo 1.0\n# EOF\n", + err: "unexpected carriage return", + }, + { + name: "carriage return in TYPE line", + input: "# TYPE foo gauge\r\nfoo 1.0\n# EOF\n", + err: "parse error", + }, + { + name: "carriage return in UNIT line", + input: "# UNIT foo seconds\r\nfoo 1.0\n# EOF\n", + err: "parse error", + }, + { + name: "carriage return in scalar sample line", + input: "foo 1.0\r\n# EOF\n", + err: "parse error", + }, + { + name: "carriage return in composite sample line", + input: "# TYPE foo histogram\nfoo {count:1,sum:1,bucket:[+Inf:1]}\r\n# EOF\n", + err: "parse error", + }, + { + name: "carriage return in EOF line", + input: "foo 1.0\n# EOF\r\n", + err: "parse error", + }, + { + name: "UTF-8 metric name rejected under explicit legacy escaping", + input: "{\"metric.name\"} 1.0\n# EOF\n", + err: `invalid metric name "metric.name"`, + }, + { + name: "interleaved metric families rejected", + input: "# TYPE a gauge\na{k=\"1\"} 1\n# TYPE b gauge\nb 2\na{k=\"2\"} 3\n# EOF\n", + err: `interleaved metric family "a"`, + }, + { + name: "interleaved sample after new family descriptor rejected", + input: "# TYPE a gauge\na{k=\"1\"} 1\n# TYPE b gauge\na{k=\"2\"} 2\n# EOF\n", + err: `interleaved metric family "a"`, + }, + { + name: "interleaved sample after descriptor-only family rejected", + input: "# TYPE a counter\nb 1\na 2\n# EOF\n", + err: `interleaved metric family "a"`, + }, + { + name: "interleaved descriptors rejected", + input: "# TYPE a counter\n# TYPE b gauge\n# HELP a help\n# EOF\n", + err: `interleaved metric family "a"`, + }, + { + name: "summary float count overflowing uint64 rejected", + input: "# TYPE s summary\ns {count:18446744073709551616.0,sum:1.0,quantile:[]}\n# EOF\n", + err: `invalid summary count "18446744073709551616.0"`, + }, + { + name: "decreasing classic histogram bucket counts rejected", + input: "# TYPE foo histogram\nfoo {count:3,sum:6.0,bucket:[0.1:2,1.0:1,+Inf:3]}\n# EOF\n", + err: "classic bucket counts must be monotonically increasing: 1 < 2", + }, + { + name: "decreasing classic gaugehistogram bucket counts rejected", + input: "# TYPE foo gaugehistogram\nfoo {gcount:3,gsum:6.0,bucket:[0.1:2,1.0:1,+Inf:3]}\n# EOF\n", + err: "classic bucket counts must be monotonically increasing: 1 < 2", + }, + { + name: "classic histogram +Inf bucket count mismatch rejected", + input: "# TYPE foo histogram\nfoo {count:5,sum:6.0,bucket:[0.1:1,1.0:2,+Inf:3]}\n# EOF\n", + err: "classic bucket +Inf count (3) does not match sample count (5)", + }, + { + name: "native histogram bucket observations mismatch rejected", + input: "# TYPE foo histogram\nfoo {count:10,sum:12.1,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:2],positive_buckets:[2,1]}\n# EOF\n", + err: "native histogram bucket observations (5) do not match sample count (10)", + }, + { + name: "native gaugehistogram bucket observations mismatch rejected", + input: "# TYPE foo gaugehistogram\nfoo {gcount:10,gsum:12.1,schema:0,zero_threshold:0.001,zero_count:2,positive_spans:[0:2],positive_buckets:[2,1]}\n# EOF\n", + err: "native histogram bucket observations (5) do not match sample count (10)", + }, + { + name: "native histogram bucket observations uint64 overflow rejected", + input: "# TYPE h histogram\nh {count:2,sum:1,schema:0,zero_threshold:0,zero_count:18446744073709551615,positive_spans:[0:1],positive_buckets:[3]}\n# EOF\n", + err: "native histogram bucket observations overflow uint64", + }, + { + name: "negative native bucket count in gaugehistogram rejected", + input: "# TYPE foo gaugehistogram\nfoo {gcount:1,gsum:1.0,schema:0,zero_threshold:0,zero_count:0,positive_spans:[0:1],positive_buckets:[-1]}\n# EOF\n", + err: `invalid positive_buckets value "-1"`, + }, + { + name: "sample timestamp overflowing int64 ms rejected", + input: "# TYPE foo gauge\nfoo 1.0 1e20\n# EOF\n", + err: `invalid timestamp "1e20"`, + }, + { + name: "start timestamp out of protobuf range rejected", + input: "# TYPE foo_total counter\nfoo_total 1.0 st@1e20\n# EOF\n", + err: `invalid start timestamp "1e20"`, + }, + { + name: "negative_spans and negative_buckets length mismatch", + input: "# TYPE foo histogram\nfoo {count:2,sum:1.0,schema:0,zero_threshold:0,zero_count:1,negative_spans:[0:2],negative_buckets:[1]}\n# EOF\n", + err: "negative_spans length (2) does not match negative_buckets count (1)", + }, + { + name: "positive_spans and positive_buckets length mismatch", + input: "# TYPE foo histogram\nfoo {count:2,sum:1.0,schema:0,zero_threshold:0,zero_count:1,positive_spans:[0:2],positive_buckets:[1]}\n# EOF\n", + err: "positive_spans length (2) does not match positive_buckets count (1)", + }, + { + name: "duplicate TYPE metadata", + input: "# TYPE foo gauge\n# TYPE foo gauge\nfoo 1\n# EOF\n", + err: `second TYPE line for metric name "foo"`, + }, + { + name: "duplicate HELP metadata", + input: "# HELP foo first\n# HELP foo second\nfoo 1\n# EOF\n", + err: `second HELP line for metric name "foo"`, + }, + { + name: "duplicate UNIT metadata", + input: "# UNIT foo_seconds seconds\n# UNIT foo_seconds seconds\nfoo_seconds 1\n# EOF\n", + err: `second UNIT line for metric name "foo_seconds"`, + }, + { + name: "metadata after samples", + input: "foo 1\n# HELP foo help\n# EOF\n", + err: `metadata for metric "foo" after samples`, + }, + { + name: "reserved __name__ label", + input: "foo{__name__=\"bar\"} 1\n# EOF\n", + err: `label name "__name__" is reserved`, + }, + { + name: "reserved __name__ label in exemplar", + input: "# TYPE foo_total counter\nfoo_total 1.0 # {__name__=\"bar\"} 1.0 1000.0\n# EOF\n", + err: `exemplar label name "__name__" is reserved`, + }, + { + name: "duplicate label name", + input: "foo{k=\"1\",k=\"2\"} 1\n# EOF\n", + err: `duplicate label "k" for metric "foo"`, + }, + { + name: "negative counter value", + input: "# TYPE foo_total counter\nfoo_total -1\n# EOF\n", + err: "counter sample value must not be negative", + }, + { + name: "start timestamp on gauge rejected", + input: "# TYPE g gauge\ng 1.0 st@1000.0\n# EOF\n", + err: `start timestamp not allowed on gauge metric "g"`, + }, + { + name: "start timestamp on untyped rejected", + input: "u 1.0 st@1000.0\n# EOF\n", + err: `start timestamp not allowed on untyped metric "u"`, + }, + { + name: "start timestamp on gaugehistogram rejected", + input: "# TYPE gh gaugehistogram\ngh {gcount:1,gsum:1.0,bucket:[+Inf:1]} st@1000.0\n# EOF\n", + err: `start timestamp not allowed on gaugehistogram metric "gh"`, + }, + { + name: "exemplar on gauge rejected", + input: "# TYPE g gauge\ng 1.0 # {id=\"1\"} 1.0 1000.0\n# EOF\n", + err: `exemplar not allowed on gauge metric "g"`, + }, + { + name: "exemplar on untyped rejected", + input: "u 1.0 # {id=\"1\"} 1.0 1000.0\n# EOF\n", + err: `exemplar not allowed on untyped metric "u"`, + }, + { + name: "exemplar on summary rejected", + input: "# TYPE s summary\ns {count:1,sum:1.0,quantile:[0.5:1.0]} # {id=\"1\"} 1.0 1000.0\n# EOF\n", + err: `exemplar not allowed on summary metric "s"`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + format := FmtOpenMetrics_2_0_0 + if strings.Contains(tc.name, "explicit legacy escaping") { + format = FmtOpenMetrics_2_0_0.WithEscapingScheme(model.ValueEncodingEscaping) + } + _, err := decodeOpenMetrics20ToMap(strings.NewReader(tc.input), format) + require.Error(t, err) + require.ErrorContains(t, err, tc.err) + }) + } +} + +func TestOpenMetrics2RoundTrip(t *testing.T) { + families := []*dto.MetricFamily{ + { + Name: proto.String("http_requests_total"), + Help: proto.String("Total number of HTTP requests."), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + {Name: proto.String("method"), Value: proto.String("GET")}, + {Name: proto.String("code"), Value: proto.String("200")}, + }, + Counter: &dto.Counter{ + Value: proto.Float64(1027), + CreatedTimestamp: ×tamppb.Timestamp{Seconds: 1234567890, Nanos: 500000000}, + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + {Name: proto.String("trace_id"), Value: proto.String("abc")}, + }, + Value: proto.Float64(1), + Timestamp: ×tamppb.Timestamp{Seconds: 1234567891, Nanos: 0}, + }, + }, + TimestampMs: proto.Int64(1234567892000), + }, + }, + }, + { + Name: proto.String("node.memory.active_bytes"), + Help: proto.String("Active memory in bytes."), + Type: dto.MetricType_GAUGE.Enum(), + Unit: proto.String("bytes"), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + {Name: proto.String("host.name"), Value: proto.String("node-1")}, + }, + Gauge: &dto.Gauge{Value: proto.Float64(1.2345e+09)}, + }, + }, + }, + { + Name: proto.String("rpc_duration_seconds"), + Help: proto.String("RPC duration."), + Type: dto.MetricType_SUMMARY.Enum(), + Unit: proto.String("seconds"), + Metric: []*dto.Metric{ + { + Summary: &dto.Summary{ + SampleCount: proto.Uint64(100), + SampleSum: proto.Float64(300.5), + CreatedTimestamp: ×tamppb.Timestamp{Seconds: 1000, Nanos: 0}, + Quantile: []*dto.Quantile{ + {Quantile: proto.Float64(0.5), Value: proto.Float64(1.2)}, + {Quantile: proto.Float64(0.99), Value: proto.Float64(5.4)}, + }, + }, + }, + }, + }, + { + Name: proto.String("req_duration_seconds"), + Help: proto.String("Native and classic histogram."), + Type: dto.MetricType_HISTOGRAM.Enum(), + Unit: proto.String("seconds"), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(6), + SampleSum: proto.Float64(12.1), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(2), + CreatedTimestamp: ×tamppb.Timestamp{Seconds: 2000, Nanos: 0}, + PositiveSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(3)}, + }, + PositiveDelta: []int64{1, 1, -1}, + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(2), UpperBound: proto.Float64(0.1)}, + {CumulativeCount: proto.Uint64(6), UpperBound: proto.Float64(math.Inf(1))}, + }, + Exemplars: []*dto.Exemplar{ + { + Label: []*dto.LabelPair{{Name: proto.String("id"), Value: proto.String("1")}}, + Value: proto.Float64(0.5), + Timestamp: ×tamppb.Timestamp{Seconds: 2001, Nanos: 0}, + }, + { + Label: []*dto.LabelPair{{Name: proto.String("id"), Value: proto.String("2")}}, + Value: proto.Float64(1.5), + Timestamp: ×tamppb.Timestamp{Seconds: 2002, Nanos: 0}, + }, + }, + }, + }, + }, + }, + { + Name: proto.String("scaled_req_duration_seconds"), + Help: proto.String("Float native and classic histogram."), + Type: dto.MetricType_HISTOGRAM.Enum(), + Unit: proto.String("seconds"), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCountFloat: proto.Float64(5.5), + SampleSum: proto.Float64(12.1), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCountFloat: proto.Float64(2.5), + CreatedTimestamp: ×tamppb.Timestamp{Seconds: 2000, Nanos: 0}, + PositiveSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(2)}, + }, + PositiveCount: []float64{1.5, 1.5}, + Bucket: []*dto.Bucket{ + {CumulativeCountFloat: proto.Float64(2.5), UpperBound: proto.Float64(0.1)}, + {CumulativeCountFloat: proto.Float64(5.5), UpperBound: proto.Float64(math.Inf(1))}, + }, + }, + }, + }, + }, + { + Name: proto.String("queue_size"), + Help: proto.String("Gauge histogram."), + Type: dto.MetricType_GAUGE_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(6), + SampleSum: proto.Float64(100.0), + Schema: proto.Int32(0), + ZeroThreshold: proto.Float64(0.001), + ZeroCount: proto.Uint64(1), + PositiveSpan: []*dto.BucketSpan{ + {Offset: proto.Int32(0), Length: proto.Uint32(2)}, + }, + PositiveDelta: []int64{3, -1}, + Bucket: []*dto.Bucket{ + {CumulativeCount: proto.Uint64(4), UpperBound: proto.Float64(10)}, + {CumulativeCount: proto.Uint64(6), UpperBound: proto.Float64(math.Inf(1))}, + }, + }, + }, + }, + }, + } + + var buf bytes.Buffer + format := FmtOpenMetrics_2_0_0.WithEscapingScheme(model.NoEscaping) + enc := NewEncoder(&buf, format) + for _, mf := range families { + require.NoError(t, enc.Encode(mf)) + } + closer, ok := enc.(Closer) + require.True(t, ok) + require.NoError(t, closer.Close()) + + got, err := decodeOpenMetrics20ToMap(&buf, format) + require.NoError(t, err) + + expected := make(map[string]*dto.MetricFamily, len(families)) + for _, mf := range families { + expected[mf.GetName()] = mf + } + requireEqualFamilies(t, expected, got) +} + +func TestOpenMetrics2Streaming(t *testing.T) { + pr, pw := io.Pipe() + t.Cleanup(func() { _ = pr.Close() }) + + dec := NewDecoder(pr, FmtOpenMetrics_2_0_0) + + errCh := make(chan error, 1) + stepCh := make(chan struct{}) + go func() { + _, err := io.WriteString(pw, "# TYPE first_total counter\nfirst_total 1\n# TYPE second gauge\n") + if err != nil { + errCh <- err + return + } + <-stepCh + _, err = io.WriteString(pw, "second 2\n# EOF\n") + _ = pw.CloseWithError(err) + errCh <- nil + }() + + var mf1 dto.MetricFamily + require.NoError(t, dec.Decode(&mf1)) + require.Equal(t, "first_total", mf1.GetName()) + require.Equal(t, dto.MetricType_COUNTER, mf1.GetType()) + require.Len(t, mf1.GetMetric(), 1) + require.Equal(t, 1.0, mf1.GetMetric()[0].GetCounter().GetValue()) + + close(stepCh) + + var mf2 dto.MetricFamily + require.NoError(t, dec.Decode(&mf2)) + require.Equal(t, "second", mf2.GetName()) + require.Equal(t, dto.MetricType_GAUGE, mf2.GetType()) + require.Len(t, mf2.GetMetric(), 1) + require.Equal(t, 2.0, mf2.GetMetric()[0].GetGauge().GetValue()) + + var mf3 dto.MetricFamily + require.ErrorIs(t, dec.Decode(&mf3), io.EOF) + require.NoError(t, <-errCh) +} diff --git a/expfmt/openmetrics_parse.go b/expfmt/openmetrics_parse.go index 28b68a89..6eb424d0 100644 --- a/expfmt/openmetrics_parse.go +++ b/expfmt/openmetrics_parse.go @@ -854,7 +854,7 @@ func (p *openMetricsParser) handleSample(input []byte, node *goabnf.ParseTree) e return nil } -func (p *openMetricsParser) parseExemplar(input []byte, node *goabnf.ParseTree, enforceRuneLimit bool) (*dto.Exemplar, error) { +func (p *openMetricsParser) parseExemplar(input []byte, node *goabnf.ParseTree, isOM1 bool) (*dto.Exemplar, error) { var ( rawLabels []parsedLabel valStr string @@ -874,7 +874,7 @@ func (p *openMetricsParser) parseExemplar(input []byte, node *goabnf.ParseTree, } val, err := parseOpenMetricsFloat(valStr) - if err != nil || math.IsNaN(val) || math.IsInf(val, 0) { + if err != nil || (isOM1 && (math.IsNaN(val) || math.IsInf(val, 0))) { return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid exemplar value %q", valStr)} } @@ -884,6 +884,9 @@ func (p *openMetricsParser) parseExemplar(input []byte, node *goabnf.ParseTree, totalRune int ) for _, rl := range rawLabels { + if rl.name == string(model.MetricNameLabel) { + return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("exemplar label name %q is reserved", model.MetricNameLabel)} + } if !p.validationScheme().IsValidLabelName(rl.name) { return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("invalid exemplar label name %q", rl.name)} } @@ -900,7 +903,7 @@ func (p *openMetricsParser) parseExemplar(input []byte, node *goabnf.ParseTree, Value: proto.String(rl.value), }) } - if enforceRuneLimit && totalRune > openMetricsExemplarMaxRunes { + if isOM1 && totalRune > openMetricsExemplarMaxRunes { return nil, ParseError{Line: p.lineNum, Msg: fmt.Sprintf("exemplar label set exceeds %d runes (%d)", openMetricsExemplarMaxRunes, totalRune)} } @@ -1022,26 +1025,34 @@ func parseSampleTimestampMs(s string) (int64, error) { if err != nil { return 0, err } - if math.IsNaN(f) || math.IsInf(f, 0) { + ms := math.Round(f * 1000) + if math.IsNaN(ms) || math.IsInf(ms, 0) || ms < float64(math.MinInt64) || ms >= math.Exp2(63) { return 0, fmt.Errorf("invalid timestamp %v", f) } - return int64(math.Round(f * 1000)), nil + return int64(ms), nil } func parseTimestampToProto(s string) (*timestamppb.Timestamp, error) { if ts, ok := parseExactDecimalTimestamp(s); ok { + if err := ts.CheckValid(); err != nil { + return nil, fmt.Errorf("invalid timestamp %q: %w", s, err) + } return ts, nil } f, err := parseOpenMetricsFloat(s) if err != nil { return nil, err } - if math.IsNaN(f) || math.IsInf(f, 0) { + if math.IsNaN(f) || math.IsInf(f, 0) || f < float64(math.MinInt64) || f >= math.Exp2(63) { return nil, fmt.Errorf("invalid timestamp %v", f) } sec, frac := math.Modf(f) nanos := int64(math.Round(frac * 1e9)) - return timestamppb.New(time.Unix(int64(sec), nanos)), nil + ts := timestamppb.New(time.Unix(int64(sec), nanos)) + if err := ts.CheckValid(); err != nil { + return nil, fmt.Errorf("invalid timestamp %q: %w", s, err) + } + return ts, nil } func parseExactDecimalTimestamp(s string) (*timestamppb.Timestamp, bool) { diff --git a/expfmt/openmetrics_parse_test.go b/expfmt/openmetrics_parse_test.go index 7514200c..54482004 100644 --- a/expfmt/openmetrics_parse_test.go +++ b/expfmt/openmetrics_parse_test.go @@ -908,6 +908,11 @@ func TestOpenMetricsParseErrors(t *testing.T) { input: "# TYPE c counter\nc_total 1 # {b=\"1\",b=\"2\"} 1\n# EOF\n", wantErr: `duplicate exemplar label name "b"`, }, + { + name: "reserved __name__ label on exemplar", + input: "# TYPE c counter\nc_total 1 # {__name__=\"x\"} 1\n# EOF\n", + wantErr: `exemplar label name "__name__" is reserved`, + }, { name: "counter with _created only and no _total", input: "# TYPE c counter\nc_created 1520872607.123\n# EOF\n",