Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -244,12 +244,13 @@ init_worker_by_lua_block {

### prometheus:collect()

**syntax:** prometheus:collect()
**syntax:** prometheus:collect(*options*)

Presents all metrics in a text format compatible with Prometheus. This should be
called in
[content_by_lua_block](https://github.com/openresty/lua-nginx-module#content_by_lua_block)
to expose the metrics on a separate HTTP page.
to expose the metrics on a separate HTTP page. Accepts the same optional
argument as [metric_data()](#prometheusmetric_data).

Example:
```nginx
Expand All @@ -262,9 +263,12 @@ location /metrics {

### prometheus:metric_data()

**syntax:** prometheus:metric_data()
**syntax:** prometheus:metric_data(*options*)

Returns metric data as an array of strings.
Returns metric data as an array of strings. The optional `options.error_metric_labels`
table maps label names to string, number, or boolean values for the built-in error
metric on this call. An omitted or empty map emits the bare error metric. An
invalid label map is logged and the bare error metric is emitted.

### counter:inc()

Expand Down Expand Up @@ -411,6 +415,19 @@ The module increments an error metric called `nginx_metric_errors_total`
an error (for example, when `lua_shared_dict` becomes full). You might want
to configure an alert on that metric.

The error metric can carry labels, for example to tell instances apart when
their metrics are merged:

```lua
prometheus:collect({
error_metric_labels = {group = group_id, instance_id = instance_id},
})
```

The labels are only added on output: errors are still counted under the bare
metric name, so counting an error never needs a new key in the shared
dictionary.

## Caveats

### Usage in stream module
Expand Down
52 changes: 48 additions & 4 deletions prometheus.lua
Original file line number Diff line number Diff line change
Expand Up @@ -928,17 +928,57 @@ function Prometheus:histogram(name, help, label_names, buckets, exptime)
return register(self, name, help, label_names, buckets, TYPE_HISTOGRAM, exptime)
end

local function error_metric_full_name(name, labels)
if type(labels) ~= "table" then
return nil, "error_metric_labels must be a table"
end
local names = {}
for key, value in pairs(labels) do
if type(key) ~= "string" then
return nil, "error_metric_labels keys must be strings"
end
local typ = type(value)
if typ ~= "string" and typ ~= "number" and typ ~= "boolean" then
return nil, "invalid error metric label value for " .. key
end
names[#names + 1] = key
end
if #names == 0 then
return name
end
local err = check_metric_and_label_names(name, names)
if err then
return nil, err
end
table.sort(names)
local values = {}
for i, key in ipairs(names) do
values[i] = labels[key]
end
return full_metric_name(name, names, values)
end

-- Prometheus compatible metric data as an array of strings.
--
-- Returns:
-- Array of strings with all metrics in a text format compatible with
-- Prometheus.
function Prometheus:metric_data()
function Prometheus:metric_data(options)
if not self.initialized then
ngx.log(ngx.ERR, "Prometheus module has not been initialized")
return
end

local error_name = self.error_metric_name
if type(options) == "table" and options.error_metric_labels ~= nil then
local name, err = error_metric_full_name(error_name, options.error_metric_labels)
if err then
ngx.log(ngx.ERR, err)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this go through self:log_error(err) instead? Every other misuse the library catches (bad metric/label names at register time, wrong label count on inc, the Error getting path a few lines below) bumps the error counter, which is what people alert on. As written a bad label map only shows up in error.log on every scrape while nginx_metric_errors_total stays flat. Calling log_error here also makes the increment visible in the same exposition, since the dict is read after this point.

else
error_name = name
end
end

-- Force a manual sync of counter local state (mostly to make tests work).
self._counter:sync()

Expand Down Expand Up @@ -969,7 +1009,11 @@ function Prometheus:metric_data()
end
seen_metrics[short_name] = true
end
key = fix_histogram_bucket_labels(key)
if key == self.error_metric_name then
key = error_name
else
key = fix_histogram_bucket_labels(key)
end
table_insert_tail(output, string.format("%s%s %s\n", self.prefix, key, value))
else
if type(err) == "string" then
Expand All @@ -985,9 +1029,9 @@ end
-- This function should be used to expose the metrics on a separate HTTP page.
-- It will get the metrics from the dictionary, sort them, and expose them
-- aling with TYPE and HELP comments.
function Prometheus:collect()
function Prometheus:collect(options)
ngx.header.content_type = "text/plain"
ngx.print(self:metric_data())
ngx.print(self:metric_data(options))
end

-- Log an error, incrementing the error counter.
Expand Down
45 changes: 45 additions & 0 deletions prometheus_test.lua
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,51 @@ function TestPrometheus:testInitOptions()
luaunit.assertEquals(ngx.logs, nil)
end

function TestPrometheus:testErrorMetricLabels()
local p = require('prometheus').init("metrics", {
prefix = "pre_", error_metric_name = "errors_total",
})
p:counter("requests", "Requests"):inc(3)
p:log_error("boom")
ngx.logs = nil

local output = table.concat(p:metric_data({
error_metric_labels = {instance = 'i"1', group = 12},
}))
luaunit.assertStrContains(output, "# TYPE pre_errors_total counter\n")
luaunit.assertStrContains(output, 'pre_errors_total{group="12",instance="i\\"1"} 1\n')
luaunit.assertStrContains(output, "\npre_requests 3\n")
luaunit.assertNotStrContains(output, "pre_errors_total 1")

ngx.printed = nil
p:collect({error_metric_labels = {active = false}})
luaunit.assertNotNil(find_idx(ngx.printed, 'pre_errors_total{active="false"} 1'))
local bare = table.concat(p:metric_data())
luaunit.assertStrContains(bare, "\npre_errors_total 1\n")
luaunit.assertEquals(table.concat(p:metric_data({error_metric_labels = {}})), bare)
luaunit.assertEquals(self.dict:get("errors_total"), 1)
luaunit.assertEquals(ngx.logs, nil)
end

function TestPrometheus:testInvalidErrorMetricLabels()
self.counter1:inc(3)
local cases = {
{false, "must be a table"},
{{"value"}, "keys must be strings"},
{{["bad-name"] = "value"}, "label name 'bad-name' is invalid"},
{{le = "value"}, "Invalid label name 'le'"},
{{instance = {}}, "invalid error metric label value for instance"},
}
for _, case in ipairs(cases) do
ngx.logs = nil
local output = table.concat(self.p:metric_data({error_metric_labels = case[1]}))
luaunit.assertStrContains(output, "\nnginx_metric_errors_total 0\n")
luaunit.assertStrContains(output, "\nmetric1 3\n")
luaunit.assertEquals(#ngx.logs, 1)
luaunit.assertStrContains(ngx.logs[1], case[2])
end
end

function TestPrometheus:testInitWorker()
self.dict = setmetatable({}, SimpleDict)
ngx.shared.metrics = self.dict
Expand Down
Loading