From a39a5f6b76065833c8ff045cf8e219e5df1c9b29 Mon Sep 17 00:00:00 2001 From: Manohar Reddy Date: Thu, 3 Sep 2026 15:11:45 +0200 Subject: [PATCH] feat(alerting): cluster event log alert rules for docker clusters Adds `sbctl cluster event-alerts `, which provisions eight Grafana alert rules that read the cluster event log through /api/v2/clusters//logs instead of the Thanos metrics the rules in alerting/alert_rules.yaml read. Ports the rules from simplyblock-operator#474, which does the same for the Helm chart. The event log carries the transition an entity made, not only the state it ended in, so these rules can tell an operator's shutdown apart from a fault that ended in the same state, and can report three conditions no metric carries: a device removal and the two journal-compression conditions. Opt-in, because the rules need a REST data source and therefore a Grafana plugin the deployed image does not carry and downloads once (~74 MB). The command works on a running cluster, which on docker is every cluster there will be: it writes the two provisioning files into the directories Grafana already bind-mounts -- on every management node, since Grafana is constrained to managers and may be rescheduled to any of them -- and then recreates the Grafana task so it re-reads them. Paths are read off the running service rather than computed from this package's location. Not wired into `cluster create` or the compose file on purpose: nothing on the create path changes, so an upgrade cannot alter the behaviour of a cluster whose operator does not run the command. Five of the eight overlap a rule in alert_rules.yaml, three of which can only ever fire via noDataState today (the metrics exporter skips any node that is not ONLINE and any device that is not online/read_only/ cannot_allocate, so snode_status_code is always 0 and device_status_code never carries unavailable). Retiring those is a separate change. Co-Authored-By: Claude Opus 5 (1M context) --- simplyblock_cli/cli-reference.yaml | 47 +++ simplyblock_cli/cli.py | 13 + simplyblock_cli/clibase.py | 16 + simplyblock_core/cluster_ops.py | 172 +++++++++ simplyblock_core/constants.py | 14 + .../alerting/event_alert_rules.yaml.j2 | 363 ++++++++++++++++++ .../scripts/datasource-events.yml.j2 | 37 ++ simplyblock_core/utils/__init__.py | 51 ++- 8 files changed, 712 insertions(+), 1 deletion(-) create mode 100644 simplyblock_core/scripts/alerting/event_alert_rules.yaml.j2 create mode 100644 simplyblock_core/scripts/datasource-events.yml.j2 diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index a1ca25b31..9996f3005 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1586,6 +1586,53 @@ commands: dest: force type: bool action: store_true + - name: event-alerts + help: "Provisions the Grafana alert rules read from the cluster event log." + usage: > + Adds eight alert rules that read the cluster event log through /api/v2/clusters//logs + instead of the Thanos metrics the rules in alert_rules.yaml read. The event log carries + the transition an entity made, which is what lets these rules tell an operator's shutdown + apart from a fault that ended in the same state, and lets them report a device removal + and the two journal-compression conditions that no metric carries. + Opt-in: the rules need a REST data source, so Grafana downloads a plugin (~74 MB) on the + next restart. Works on a running cluster -- the provisioning files are written to every + management node and the Grafana task is recreated, so Grafana is unavailable for a few + seconds. Run it once, from any management node; re-run it to change a setting. + arguments: + - name: "cluster_id" + help: "The cluster id." + dest: cluster_id + type: str + completer: _completer_get_cluster_list + - name: "--disable" + help: "Remove the event log alert rules and the data source." + dest: disable + type: bool + action: store_true + - name: "--log-limit" + help: "How many of the newest event log records each rule evaluation reads. Default: `1000`." + description: > + Too low on a cluster that produces events quickly and an alert heals itself as soon + as the transition that opened it scrolls out of the window. + dest: log_limit + type: int + - name: "--interval" + help: "How often the rules run, as a Grafana duration. Default: `1m`." + dest: interval + type: str + - name: "--pending-period" + help: "How long a condition must hold before it notifies, as a Grafana duration. Default: `1m`." + dest: pending_period + type: str + - name: "--plugin-url" + help: "Where to fetch the Infinity data source plugin from." + dest: plugin_url + type: str + - name: "--plugin-preinstalled" + help: "The data source plugin is already in the Grafana image; do not download it." + dest: plugin_preinstalled + type: bool + action: store_true - name: switch-write-protection help: "Activate v2 distrib write protection cluster-wide." usage: "Needed once after upgrading a cluster from a release without v2; new clusters are created on v2 already. Sends the runtime distr_write_protection_v2 RPC to every ONLINE storage node -- the only way an already-created distrib gains v2, since a create parameter cannot retrofit an existing bdev. Only if every online node succeeds is the generation recorded, on the cluster row and on every node's lvstore_stack distrib entries, so later (re-)creations use the v2 parameter. Partial success records nothing, so a retry is a plain re-run. Offline nodes are not an error: they have no running bdev to migrate and come back on v2 at their next restart." diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 26e4ef960..ff779e6a6 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -387,6 +387,7 @@ def init_cluster(self): if self.developer_mode: self.init_cluster__set(subparser) self.init_cluster__set_shared_placement(subparser) + self.init_cluster__event_alerts(subparser) self.init_cluster__switch_write_protection(subparser) self.init_cluster__change_name(subparser) self.init_cluster__add_replication(subparser) @@ -621,6 +622,16 @@ def init_cluster__set_shared_placement(self, subparser): subcommand.add_argument('--disable', help='Reverse transition (per-chunk -> per-page). Debug only; only safe on a balanced or empty bdev. Requires --force.', dest='disable', action='store_true') subcommand.add_argument('--force', help='Bypass the rebalancing / non-online-node guards. Required when --disable is passed.', dest='force', action='store_true') + def init_cluster__event_alerts(self, subparser): + subcommand = self.add_sub_command(subparser, 'event-alerts', 'Provisions the Grafana alert rules read from the cluster event log.') + subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list + subcommand.add_argument('--disable', help='Remove the event log alert rules and the data source.', dest='disable', action='store_true') + subcommand.add_argument('--log-limit', help='How many of the newest event log records each rule evaluation reads. Default: `1000`.', type=int, dest='log_limit') + subcommand.add_argument('--interval', help='How often the rules run, as a Grafana duration. Default: `1m`.', type=str, dest='interval') + subcommand.add_argument('--pending-period', help='How long a condition must hold before it notifies, as a Grafana duration. Default: `1m`.', type=str, dest='pending_period') + subcommand.add_argument('--plugin-url', help='Where to fetch the Infinity data source plugin from.', type=str, dest='plugin_url') + subcommand.add_argument('--plugin-preinstalled', help='The data source plugin is already in the Grafana image; do not download it.', dest='plugin_preinstalled', action='store_true') + def init_cluster__switch_write_protection(self, subparser): subcommand = self.add_sub_command(subparser, 'switch-write-protection', 'Activate v2 distrib write protection cluster-wide.') subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list @@ -1489,6 +1500,8 @@ def run(self): ret = self.cluster__set(sub_command, args) elif sub_command in ['set-shared-placement']: ret = self.cluster__set_shared_placement(sub_command, args) + elif sub_command in ['event-alerts']: + ret = self.cluster__event_alerts(sub_command, args) elif sub_command in ['switch-write-protection']: ret = self.cluster__switch_write_protection(sub_command, args) elif sub_command in ['change-name']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index cc3c1970c..b7dd6dbd8 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -472,6 +472,22 @@ def cluster__activate(self, sub_command, args): return False return True + def cluster__event_alerts(self, sub_command, args): + cluster_ops.set_event_alerts( + args.cluster_id, + enabled=not args.disable, + log_limit=args.log_limit, + interval=args.interval, + pending_period=args.pending_period, + plugin_url=args.plugin_url, + plugin_preinstalled=args.plugin_preinstalled, + ) + if args.disable: + return "Event log alert rules removed from Grafana." + return ("Event log alert rules provisioned. Grafana was restarted; the rules start " + "evaluating once the data source plugin has loaded, which takes a minute or two " + "the first time.") + def cluster__op_stop(self, sub_command, args): return cluster_ops.set_object_ops(args.cluster_id, True) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index a0745c037..699ecdfd2 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -1,6 +1,9 @@ # coding=utf-8 +import base64 import json import os +import re +import shlex import socket import subprocess import threading @@ -2551,6 +2554,175 @@ def get_logs(cluster_id, limit=50, **kwargs) -> t.List[dict]: return out +# Grafana's provisioning paths inside the monitoring container, as mounted by +# docker-compose-swarm-monitoring.yml. +GRAFANA_DATASOURCE_TARGET = "/etc/grafana/provisioning/datasources/datasource.yaml" +GRAFANA_ALERTING_TARGET = "/etc/grafana/provisioning/alerting" +GRAFANA_EVENT_DATASOURCE_TARGET = "/etc/grafana/provisioning/datasources/datasource-events.yaml" + +_GRAFANA_DURATION = re.compile(r'^(?:\d+(?:ms|[smhdwy]))+$') + + +def _event_alert_settings(enabled, log_limit, interval, pending_period, + plugin_url, plugin_preinstalled) -> t.Dict[str, t.Any]: + """Validate the tuning knobs and shape them for the templates. + + Validated here because Grafana answers a malformed duration or limit by + refusing to start, long after the command that caused it. + """ + settings = { + 'enabled': enabled, + 'logLimit': 1000 if log_limit is None else log_limit, + 'interval': interval or '1m', + 'for': pending_period or '1m', + 'plugin': { + 'url': constants.GRAFANA_EVENT_ALERTS_PLUGIN_URL if plugin_url is None else plugin_url, + 'preinstalled': plugin_preinstalled, + }, + } + + if not enabled: + return settings + + if settings['logLimit'] < 1: + raise ValueError(f"--log-limit must be at least 1, got {settings['logLimit']}") + + for flag, key in (('--interval', 'interval'), ('--pending-period', 'for')): + if not _GRAFANA_DURATION.match(settings[key]): + raise ValueError( + f"{flag} must be a Grafana duration such as 30s, 1m or 1h, got {settings[key]!r}") + + if not settings['plugin']['preinstalled'] and not settings['plugin']['url']: + raise ValueError( + "--plugin-url is empty and --plugin-preinstalled was not given, so the data source " + "plugin the rules query would never be installed") + + return settings + + +def _grafana_provisioning_dirs(cluster_docker) -> t.Tuple[str, str]: + """The host directories the running Grafana reads its provisioning from. + + Read off the service rather than computed from this package's location, so + the files land where the cluster's own mounts point. + """ + try: + service = cluster_docker.services.get(constants.MONITORING_GRAFANA_SERVICE) + except docker.errors.NotFound as e: + raise ValueError( + f"Service {constants.MONITORING_GRAFANA_SERVICE} not found: this cluster has no " + f"monitoring stack") from e + + mounts = service.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Mounts') or [] + sources = {mount.get('Target'): mount.get('Source') for mount in mounts} + for target in (GRAFANA_DATASOURCE_TARGET, GRAFANA_ALERTING_TARGET): + if not sources.get(target): + raise ValueError( + f"Service {constants.MONITORING_GRAFANA_SERVICE} has no bind mount at {target}; " + f"redeploy the monitoring stack before provisioning event log alerts") + + return os.path.dirname(sources[GRAFANA_DATASOURCE_TARGET]), sources[GRAFANA_ALERTING_TARGET] + + +def _install_provisioning_on_all_managers(files: t.Dict[str, str]) -> None: + """Write Grafana's provisioning files onto every management node. + + Grafana is constrained to node.role == manager and may be rescheduled to + any of them, and these are host paths, so a manager that lacks them + provisions no rules the first time Grafana lands there. They cannot ship + with the package the way alert_rules.yaml does, carrying this cluster's id + and secret. Written through each node's docker API, the same way SNodeAPI + is started on a remote node; base64 keeps the YAML out of shell quoting, + and user=root because the image runs as USER simplyblock, which cannot + write into the package directory. + """ + script = "; ".join( + f"umask 022 && echo {base64.b64encode(content.encode('utf-8')).decode('ascii')} " + f"| base64 -d > {shlex.quote(path)}" + for path, content in files.items() + ) + volumes = sorted({os.path.dirname(path) for path in files}) + + failed = {} + for node in db_controller.get_mgmt_nodes(): + try: + node_docker = docker.DockerClient( + base_url=f"tcp://{node.docker_ip_port}", version="auto", timeout=60) + node_docker.containers.run( + constants.SIMPLY_BLOCK_DOCKER_IMAGE, ["sh", "-c", script], user="root", + volumes=[f"{directory}:{directory}" for directory in volumes], + remove=True, detach=False) + logger.info("Provisioning files written on %s", node.mgmt_ip) + except Exception as e: + failed[node.mgmt_ip] = str(e) + + if failed: + raise RuntimeError( + "Could not write the provisioning files on: " + + "; ".join(f"{ip} ({reason})" for ip, reason in failed.items()) + + ". Fix those nodes and re-run; Grafana provisions nothing on a node it cannot " + "read them from") + + +def set_event_alerts(cluster_id, enabled=True, log_limit=None, interval=None, pending_period=None, + plugin_url=None, plugin_preinstalled=False) -> None: + """Provision (or remove) the Grafana alert rules read from the cluster event log. + + Unlike the Thanos-backed rules in alerting/alert_rules.yaml, which ship + with the package and are provisioned unconditionally, these are opt-in: + they query the control plane's REST API, which needs a Grafana plugin the + deployed image does not carry and downloads once. + + Runs against the live stack -- files onto every management node, then the + Grafana task recreated so it re-reads them -- so it configures a cluster + that already exists. + """ + cluster = db_controller.get_cluster_by_id(cluster_id) + + if cluster.mode != "docker": + raise ValueError( + "Event log alerts are provisioned by the simplyblock-operator Helm chart on " + "kubernetes; this command configures the docker monitoring stack only") + + if cluster.disable_monitoring: + raise ValueError("This cluster was created with --disable-monitoring, so it has no Grafana") + + settings = _event_alert_settings(enabled, log_limit, interval, pending_period, + plugin_url, plugin_preinstalled) + + cluster_docker = utils.get_docker_client(cluster_id) + scripts_dir, alerting_dir = _grafana_provisioning_dirs(cluster_docker) + + # The secret is the bearer token the data source sends, and reaches + # plaintext only here, on its way into a file Grafana reads -- the same + # treatment prometheus.yml gets. + rendered = utils.render_event_alert_configs( + settings, [(cluster.get_id(), cluster.secret.get_secret_value())]) + datasource_path = os.path.join(scripts_dir, utils.EVENT_ALERT_DATASOURCE_FILE) + _install_provisioning_on_all_managers({ + os.path.join(alerting_dir, utils.EVENT_ALERT_RULES_FILE): + rendered[utils.EVENT_ALERT_RULES_FILE], + datasource_path: rendered[utils.EVENT_ALERT_DATASOURCE_FILE], + }) + + # --force recreates the task, which is what makes Grafana re-read + # provisioning; a single-file bind mount also tracks the inode it started + # with, so the rewritten data source is only picked up here. Mounts are + # keyed by target and env by name, so re-running updates both in place. + # Disabling leaves them: both files are empty now, and keeping the plugin + # makes re-enabling a restart instead of another download. + update = ["sudo", "docker", "service", "update", "--force"] + if enabled: + if not settings['plugin']['preinstalled']: + update += ["--env-add", f"GF_INSTALL_PLUGINS={settings['plugin']['url']};" + f"{constants.GRAFANA_EVENT_ALERTS_PLUGIN_ID}"] + update += ["--mount-add", f"type=bind,src={datasource_path}," + f"dst={GRAFANA_EVENT_DATASOURCE_TARGET},readonly"] + + logger.info("Restarting Grafana...") + subprocess.check_call(update + [constants.MONITORING_GRAFANA_SERVICE]) + + def get_cluster(cl_id) -> dict: return db_controller.get_cluster_by_id(cl_id).get_clean_dict() diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 918455dce..3fcc708b4 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -588,6 +588,20 @@ def get_config_var(name, default=None): CR_GROUP = "storage.simplyblock.io" CR_VERSION = "v1alpha1" +# Grafana alert rules read from the cluster event log rather than from Thanos, +# provisioned by `sbctl cluster event-alerts`. The plugin id is both the folder +# Grafana installs the plugin into and the data source `type` the rules use. +# 2.12.2 is the last Infinity release compatible with Grafana 10.0.12. +GRAFANA_EVENT_ALERTS_PLUGIN_ID = "yesoreyeram-infinity-datasource" +GRAFANA_EVENT_ALERTS_PLUGIN_URL = ( + "https://grafana.com/api/plugins/yesoreyeram-infinity-datasource/versions/2.12.2/download") + +# The control plane as reached from the monitoring stack. HAProxy's default +# backend is the web API, so /api/v2 needs no route of its own; the same host +# prometheus.yml.j2 scrapes /cluster/metrics from. +MONITORING_CONTROL_PLANE_ADDR = "http://HAProxy" +MONITORING_GRAFANA_SERVICE = "monitoring_grafana" + GRAFANA_K8S_ENDPOINT = "http://simplyblock-grafana:3000" GRAYLOG_K8S_ENDPOINT = "http://simplyblock-graylog:9000" OS_K8S_ENDPOINT = "http://opensearch-cluster-master:9200" diff --git a/simplyblock_core/scripts/alerting/event_alert_rules.yaml.j2 b/simplyblock_core/scripts/alerting/event_alert_rules.yaml.j2 new file mode 100644 index 000000000..131659b15 --- /dev/null +++ b/simplyblock_core/scripts/alerting/event_alert_rules.yaml.j2 @@ -0,0 +1,363 @@ +# Grafana alert rules derived from the control plane's cluster event log, +# rendered by `sbctl cluster event-alerts` into the alerting folder Grafana is +# given as a whole, next to the checked-in, Thanos-backed alert_rules.yaml. +# +# Each query folds the newest log records into the current state of every node, +# device and cluster and returns one row per entity that is currently wrong, so +# healing is structural: the reverting event removes the row, and noDataState: +# OK turns an empty result into the resolved state. The fold reads transitions +# because the cause is not in the payload -- get_logs drops caused_by, and +# set_node_status writes every transition under the same one -- so what +# separates an operator's shutdown from a fault is the state it passes through. +# +# Kept in step with the same rules in the simplyblock-operator Helm chart +# (helm-charts/charts/simplyblock-operator/templates/controlplane_configmap.yaml). +{% macro rule(cluster, ea, uid_suffix, title, columns, severity, summary, description) %} + - uid: sbev-{{ cluster.uid_prefix }}-{{ uid_suffix }} + # Titles must be unique within a group, and each cluster adds a copy. + title: {{ title }}_{{ cluster.id }} + condition: B + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: {{ cluster.ds_uid }} + model: + refId: A + type: json + source: url + format: table + # Only the backend parser runs server-side, which is what an + # alert rule needs. root_selector is tried as gjson first and + # falls back to JSONata; opening with a bracket forces that. + parser: backend + url: /api/v2/clusters/{{ cluster.id }}/logs + url_options: + method: GET + params: + - key: limit + value: "{{ ea.logLimit }}" + root_selector: |- +{{ caller() | trim | indent(18, first=True) }} + # One numeric field and the rest strings is what makes Grafana + # read the frame as a numeric table, turning each string field + # into a label of its own alert instance. + columns: +{% for name, type in columns %} + - selector: {{ name }} + text: {{ name }} + type: {{ type }} +{% endfor %} + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: threshold + datasource: + type: __expr__ + uid: __expr__ + expression: A + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: last + type: query + # An empty result is the healthy state; a failing query is not. + noDataState: OK + execErrState: Error + for: {{ ea['for'] }} + annotations: + # Single-quoted so the Go template expressions Grafana expands at + # notification time survive as written. + summary: '{{ summary | replace("'", "''") }}' + description: '{{ description | replace("'", "''") }}' + labels: + app: simplyblock + cluster: {{ cluster.id }} + severity: {{ severity }} + isPaused: false +{% endmacro %} +apiVersion: 1 +{% if EVENT_ALERT_CLUSTERS %} +groups: + - orgId: 1 + name: simplyblock_events + folder: grafana_folder + interval: {{ EVENT_ALERTS.interval }} + rules: +{% for cluster in EVENT_ALERT_CLUSTERS %} +{% call rule(cluster, EVENT_ALERTS, 'node-left-online', 'StorageNode_left_online', + [('node', 'string'), ('state', 'string'), ('since', 'string'), ('value', 'number')], + 'critical', + 'Storage node {{ $labels.node }} left the online state and is now {{ $labels.state }}.', + 'The rule folds the cluster event log into the current state of every storage node and' + ' fires while a node sits in anything other than online or in_creation. A shutdown or' + ' removal an operator asked for is exempt: those reach in_shutdown, pending_removal,' + ' in_removal, or removed, which no fault path does. The alert clears when the node' + ' returns to online.') %} +( + $planned := ['in_shutdown', 'in_removal', 'pending_removal', 'removed']; + $healthy := ['online', 'in_creation']; + $ev := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Storage node status changed from: ')]]; + $fold := function($evs) { + $reduce($evs, function($a, $e) { + ( + $to := $substringAfter($e.Message, ' to: '); + { + 'state': $to, + 'since': $e.Date, + 'planned': $to in $planned or ($a.planned and $to = 'offline') + } + ) + }, { 'state': 'online', 'since': '', 'planned': false }) + }; + $nodes := $count($ev) = 0 ? [] : [$each($ev{ NodeId: [$] }, function($evs, $id) { + $merge([{ 'node': $id }, $fold($evs)]) + })]; + [$nodes[$not(state in $healthy) and $not(planned)].{ + 'node': node, + 'state': state, + 'since': since, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'device-unavailable', 'Device_became_unavailable', + [('device', 'string'), ('order', 'string'), ('since', 'string'), ('value', 'number')], + 'warning', + 'Device {{ $labels.device }} (cluster device order {{ $labels.order }}) is unavailable.', + 'The rule fires for a device that is currently unavailable and did not become so as' + ' part of a whole node going down. A node leaving online cascades every one of its' + ' devices to unavailable within seconds, and that is reported once as the storage node' + ' alert rather than once per device. The alert clears when the device returns to' + ' online.') %} +( + $window := 120000; + $ms := function($d) { $toMillis($substring($d & '.000', 0, 23), '[Y0001]-[M01]-[D01] [H01]:[m01]:[s01].[f001]') }; + $healthy := ['online', 'in_creation']; + $nodeEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Storage node status changed from: ')]]; + $cascades := [$map($nodeEv[$not($substringAfter(Message, ' to: ') in $healthy)], function($e) { $ms($e.Date) })]; + $devEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Device status changed from: ')]]; + $fold := function($evs) { + $reduce($evs, function($a, $e) { + { + 'state': $substringAfter($e.Message, ' to: '), + 'since': $e.Date, + 'at': $ms($e.Date), + 'order': $e.Storage_ID + } + }, { 'state': 'online', 'since': '', 'at': 0, 'order': '' }) + }; + $devs := $count($devEv) = 0 ? [] : [$each($devEv{ NodeId: [$] }, function($evs, $id) { + ( + $f := $fold($evs); + $merge([ + { 'device': $id, 'cascade': $count($cascades[$ <= $f.at and $f.at - $ <= $window]) > 0 }, + $f + ]) + ) + })]; + [$devs[state = 'unavailable' and $not(cascade)].{ + 'device': device, + 'order': order, + 'since': since, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'device-removed', 'Device_removed', + [('device', 'string'), ('order', 'string'), ('since', 'string'), ('value', 'number')], + 'critical', + 'Device {{ $labels.device }} (cluster device order {{ $labels.order }}) was removed.', + 'The rule fires for every device whose latest state is removed, whatever the cause. It' + ' clears when the device is brought back online, and otherwise ages out a day after the' + ' removal, since a device that is genuinely gone has no reversion to wait for.') %} +( + $recent := 86400000; + $ms := function($d) { $toMillis($substring($d & '.000', 0, 23), '[Y0001]-[M01]-[D01] [H01]:[m01]:[s01].[f001]') }; + $now := $millis(); + $devEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Device status changed from: ')]]; + $fold := function($evs) { + $reduce($evs, function($a, $e) { + { + 'state': $substringAfter($e.Message, ' to: '), + 'since': $e.Date, + 'at': $ms($e.Date), + 'order': $e.Storage_ID + } + }, { 'state': 'online', 'since': '', 'at': 0, 'order': '' }) + }; + $devs := $count($devEv) = 0 ? [] : [$each($devEv{ NodeId: [$] }, function($evs, $id) { + $merge([{ 'device': $id }, $fold($evs)]) + })]; + [$devs[state = 'removed' and $now - at <= $recent].{ + 'device': device, + 'order': order, + 'since': since, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'cluster-degraded', 'Cluster_became_degraded', + [('state', 'string'), ('since', 'string'), ('value', 'number')], + 'warning', + 'The cluster is degraded.', + "The rule fires while the cluster's latest status is degraded. A degradation an operator" + ' caused by shutting a node down is exempt, decided by whether any node currently out of' + ' service got there through in_shutdown or a removal state. The alert clears when the' + ' cluster returns to active.') %} +( + $planned := ['in_shutdown', 'in_removal', 'pending_removal', 'removed']; + $healthy := ['online', 'in_creation']; + $nodeEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Storage node status changed from: ')]]; + $fold := function($evs) { + $reduce($evs, function($a, $e) { + ( + $to := $substringAfter($e.Message, ' to: '); + { + 'state': $to, + 'planned': $to in $planned or ($a.planned and $to = 'offline') + } + ) + }, { 'state': 'online', 'planned': false }) + }; + $nodes := $count($nodeEv) = 0 ? [] : [$each($nodeEv{ NodeId: [$] }, function($evs, $id) { $fold($evs) })]; + $operatorDown := $count($nodes[$not(state in $healthy) and planned]) > 0; + $cluEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Cluster status changed from ')]]; + $last := $cluEv[-1]; + $state := $exists($last) ? $substringAfter($last.Message, ' to ') : 'active'; + [$state = 'degraded' and $not($operatorDown) ? { + 'state': $state, + 'since': $last.Date, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'cluster-suspended', 'Cluster_became_suspended', + [('state', 'string'), ('since', 'string'), ('value', 'number')], + 'critical', + 'The cluster is suspended.', + "The rule fires while the cluster's latest status is suspended, whatever the cause," + " including a suspension that followed an operator's node shutdown. The alert clears" + ' when the cluster leaves suspended.') %} +( + $cluEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Cluster status changed from ')]]; + $last := $cluEv[-1]; + $state := $exists($last) ? $substringAfter($last.Message, ' to ') : 'active'; + [$state = 'suspended' ? { + 'state': $state, + 'since': $last.Date, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'cluster-capacity', 'Cluster_capacity_reached', + [('kind', 'string'), ('severity', 'string'), ('value', 'number')], + 'warning', + '{{ $labels.kind }} cluster capacity is at {{ $value }}% ({{ $labels.severity }}).', + 'The rule reports the newest capacity event per kind, absolute and provisioned, and' + ' carries the utilization as its value. The control plane emits no back-to-normal' + ' event, so the alert clears by the event ceasing: the capacity monitor re-emits every' + ' 30 seconds while over a threshold, except an absolute critical which it throttles to' + ' once every 15 minutes, and the rule allows for both.') %} +( + $ms := function($d) { $toMillis($substring($d & '.000', 0, 23), '[Y0001]-[M01]-[D01] [H01]:[m01]:[s01].[f001]') }; + $now := $millis(); + $ev := [$[Event = 'CAPACITY']]; + $latest := $count($ev) = 0 ? [] : [$each( + $ev{ ($contains(Message, 'provisioned capacity') ? 'provisioned' : 'absolute'): [$] }, + function($evs, $kind) { + ( + $e := $evs[-1]; + $window := ($kind = 'absolute' and $e.Level = 'Critical') ? 1200000 : 180000; + { + 'kind': $kind, + 'severity': $e.Level, + 'value': $substringBefore($substringAfter($e.Message, 'reached: '), '%'), + 'fresh': $now - $ms($e.Date) <= $window + } + ) + })]; + [$latest[fresh and (severity = 'Warning' or severity = 'Critical')].{ + 'kind': kind, + 'severity': severity, + 'value': value + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'jm-records-threshold', 'JM_records_threshold_exceeded', + [('node', 'string'), ('since', 'string'), ('value', 'number')], + 'critical', + 'The journal on storage node {{ $labels.node }} holds more records awaiting compression' + ' than the configured threshold.', + 'Compression is not keeping up with the journal, and journal replay on the next restart' + ' or failover grows with every record. The control plane latches this alert on the' + ' upward crossing and re-arms silently, emitting nothing when the backlog drains, so the' + ' alert clears an hour after the last crossing rather than on a recovery event.') %} +( + $recent := 3600000; + $ms := function($d) { $toMillis($substring($d & '.000', 0, 23), '[Y0001]-[M01]-[D01] [H01]:[m01]:[s01].[f001]') }; + $now := $millis(); + $ev := [$[Event = 'JM_COMPRESSION_BACKLOG']]; + $latest := $count($ev) = 0 ? [] : [$each($ev{ NodeId: [$] }, function($evs, $id) { + ( + $e := $evs[-1]; + { 'node': $id, 'since': $e.Date, 'at': $ms($e.Date) } + ) + })]; + [$latest[$now - at <= $recent].{ + 'node': node, + 'since': since, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'jm-compression-error', 'JM_compression_error', + [('node', 'string'), ('jm_vuid', 'string'), ('detail', 'string'), ('value', 'number')], + 'critical', + 'Journal compression failed on storage node {{ $labels.node }} for jm_vuid' + ' {{ $labels.jm_vuid }}: {{ $labels.detail }}.', + 'The rule tracks the newest compression event per node and journal and fires while that' + ' event is an error, which is either a compression_failed status or a non-zero error' + ' code. The alert clears when the same journal reports a clean compression run.') %} +( + $ev := [$[Event = 'jm_compression']]; + $latest := $count($ev) = 0 ? [] : [$each($ev{ (NodeId & '/' & VUID): [$] }, function($evs, $key) { + ( + $e := $evs[-1]; + { + 'node': $substringBefore($key, '/'), + 'jm_vuid': $substringAfter($key, '/'), + 'level': $e.Level, + 'detail': $e.Message + } + ) + })]; + [$latest[level = 'Error'].{ + 'node': node, + 'jm_vuid': jm_vuid, + 'detail': detail, + 'value': 1 + }] +) +{% endcall %} +{% endfor %} +{% else %} +# No cluster to read an event log from, or the event log alerts are off, so +# nothing to provision. Rendered rather than skipped, so that `--disable` +# overwrites the rules an earlier run left on this management node instead of +# leaving Grafana to provision them again on its next restart. +groups: [] +{% endif %} diff --git a/simplyblock_core/scripts/datasource-events.yml.j2 b/simplyblock_core/scripts/datasource-events.yml.j2 new file mode 100644 index 000000000..6dab0d0ba --- /dev/null +++ b/simplyblock_core/scripts/datasource-events.yml.j2 @@ -0,0 +1,37 @@ +# Grafana data sources for the cluster event log alert rules, rendered by +# `sbctl cluster event-alerts`. One per cluster: /api/v2 authenticates with the +# cluster secret as a bearer token, and Grafana binds credentials to a data +# source rather than to a query. The secret goes through secureJsonData, which +# Grafana stores encrypted and never returns to the browser. +apiVersion: 1 +{% if EVENT_ALERT_CLUSTERS %} +datasources: +{% for cluster in EVENT_ALERT_CLUSTERS %} + - name: Simplyblock Events {{ cluster.id }} + type: yesoreyeram-infinity-datasource + uid: {{ cluster.ds_uid }} + url: {{ CONTROL_PLANE_ADDR }} + access: proxy + editable: false + jsonData: + httpHeaderName1: Authorization + # Infinity refuses every URL query until the host is allowlisted, and + # reports the refusal as a per-rule evaluation error rather than as a + # configuration warning. Matched by prefix, so the base address covers + # every /api/v2 path the rules query. + allowedHosts: + - {{ CONTROL_PLANE_ADDR }} + # Otherwise the health check probes the bare base URL, which the control + # plane answers with a string Infinity cannot read as a table. + customHealthCheckEnabled: false + tlsSkipVerify: true + timeoutInSeconds: 30 + secureJsonData: + httpHeaderValue1: {{ ('Bearer ' ~ cluster.secret) | tojson }} +{% endfor %} +{% else %} +# Off, so nothing to provision. Written rather than deleted: the file stays +# bind-mounted, and a bind mount whose source is gone comes back as a +# directory, which Grafana refuses to start on. +datasources: [] +{% endif %} diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 50183c6ec..babc1aef2 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1,5 +1,6 @@ # coding=utf-8 import glob +import hashlib import json import logging import math @@ -28,7 +29,7 @@ from docker.errors import APIError, DockerException, ImageNotFound, NotFound import tempfile -from jinja2 import Environment, FileSystemLoader +from jinja2 import Environment, FileSystemLoader, StrictUndefined from simplyblock_core import constants from simplyblock_core import shell_utils @@ -61,6 +62,11 @@ ALERT_RESOURCES_FILE = "alert_resources.yaml" ALERTS_TEMPLATE_FOLDER = "simplyblock_core/scripts/alerting/" +SCRIPTS_FOLDER = "simplyblock_core/scripts/" + +# Provisioning files for the cluster event log alerts (`cluster event-alerts`). +EVENT_ALERT_RULES_FILE = "event_alert_rules.yaml" +EVENT_ALERT_DATASOURCE_FILE = "datasource-events.yml" def get_env_var(name, default=None, is_required=False): if not name: @@ -2567,6 +2573,10 @@ def _alerts_template_folder() -> str: return os.path.join(_top_dir(), ALERTS_TEMPLATE_FOLDER) +def _scripts_folder() -> str: + return os.path.join(_top_dir(), SCRIPTS_FOLDER) + + def _top_dir() -> str: return os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) @@ -2608,6 +2618,45 @@ def render_configfile_alerting(alert_config: Dict[str, Any]) -> str: return template.render(alert_config) +def render_event_alert_configs(settings: Dict[str, Any], + clusters: Iterable[Tuple[str, str]]) -> Dict[str, str]: + """Render the Grafana provisioning files for the cluster event log alerts. + + Returns {filename: content} for both files, always, so that disabling the + alerts overwrites what enabling them wrote. A cluster missing either its id + or its secret is skipped: the API checks that the two agree, so a + half-configured data source fails every evaluation with a 401. Both uids + are derived from the cluster id so the two files agree on them and a re-run + updates the same objects instead of adding a second set. + + StrictUndefined because a missing value would reach Grafana as empty YAML, + which it answers by refusing to start. + """ + entries = [] + if settings.get('enabled'): + for cluster_id, secret in clusters: + if not cluster_id or not secret: + continue + digest = hashlib.sha256(cluster_id.encode('utf-8')).hexdigest() + entries.append({'id': cluster_id, 'secret': secret, + 'ds_uid': f'sbev-{digest[:12]}', 'uid_prefix': digest[:8]}) + + values = { + 'EVENT_ALERTS': settings, + 'EVENT_ALERT_CLUSTERS': entries, + 'CONTROL_PLANE_ADDR': constants.MONITORING_CONTROL_PLANE_ADDR, + } + + rendered = {} + for folder, filename in ((_alerts_template_folder(), EVENT_ALERT_RULES_FILE), + (_scripts_folder(), EVENT_ALERT_DATASOURCE_FILE)): + env = Environment(loader=FileSystemLoader(folder), trim_blocks=True, lstrip_blocks=True, + undefined=StrictUndefined) + rendered[filename] = env.get_template(f'{filename}.j2').render(values) + + return rendered + + def render_and_deploy_alerting_configs(alert_config: Optional[Dict[str, Any]], contact_point: Optional[str], grafana_endpoint, cluster_uuid, cluster_secret): top_dir = _top_dir()