diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index a1ca25b31..9996f3005 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1586,6 +1586,53 @@ commands: dest: force type: bool action: store_true + - name: event-alerts + help: "Provisions the Grafana alert rules read from the cluster event log." + usage: > + Adds eight alert rules that read the cluster event log through /api/v2/clusters//logs + instead of the Thanos metrics the rules in alert_rules.yaml read. The event log carries + the transition an entity made, which is what lets these rules tell an operator's shutdown + apart from a fault that ended in the same state, and lets them report a device removal + and the two journal-compression conditions that no metric carries. + Opt-in: the rules need a REST data source, so Grafana downloads a plugin (~74 MB) on the + next restart. Works on a running cluster -- the provisioning files are written to every + management node and the Grafana task is recreated, so Grafana is unavailable for a few + seconds. Run it once, from any management node; re-run it to change a setting. + arguments: + - name: "cluster_id" + help: "The cluster id." + dest: cluster_id + type: str + completer: _completer_get_cluster_list + - name: "--disable" + help: "Remove the event log alert rules and the data source." + dest: disable + type: bool + action: store_true + - name: "--log-limit" + help: "How many of the newest event log records each rule evaluation reads. Default: `1000`." + description: > + Too low on a cluster that produces events quickly and an alert heals itself as soon + as the transition that opened it scrolls out of the window. + dest: log_limit + type: int + - name: "--interval" + help: "How often the rules run, as a Grafana duration. Default: `1m`." + dest: interval + type: str + - name: "--pending-period" + help: "How long a condition must hold before it notifies, as a Grafana duration. Default: `1m`." + dest: pending_period + type: str + - name: "--plugin-url" + help: "Where to fetch the Infinity data source plugin from." + dest: plugin_url + type: str + - name: "--plugin-preinstalled" + help: "The data source plugin is already in the Grafana image; do not download it." + dest: plugin_preinstalled + type: bool + action: store_true - name: switch-write-protection help: "Activate v2 distrib write protection cluster-wide." usage: "Needed once after upgrading a cluster from a release without v2; new clusters are created on v2 already. Sends the runtime distr_write_protection_v2 RPC to every ONLINE storage node -- the only way an already-created distrib gains v2, since a create parameter cannot retrofit an existing bdev. Only if every online node succeeds is the generation recorded, on the cluster row and on every node's lvstore_stack distrib entries, so later (re-)creations use the v2 parameter. Partial success records nothing, so a retry is a plain re-run. Offline nodes are not an error: they have no running bdev to migrate and come back on v2 at their next restart." diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 26e4ef960..ff779e6a6 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -387,6 +387,7 @@ def init_cluster(self): if self.developer_mode: self.init_cluster__set(subparser) self.init_cluster__set_shared_placement(subparser) + self.init_cluster__event_alerts(subparser) self.init_cluster__switch_write_protection(subparser) self.init_cluster__change_name(subparser) self.init_cluster__add_replication(subparser) @@ -621,6 +622,16 @@ def init_cluster__set_shared_placement(self, subparser): subcommand.add_argument('--disable', help='Reverse transition (per-chunk -> per-page). Debug only; only safe on a balanced or empty bdev. Requires --force.', dest='disable', action='store_true') subcommand.add_argument('--force', help='Bypass the rebalancing / non-online-node guards. Required when --disable is passed.', dest='force', action='store_true') + def init_cluster__event_alerts(self, subparser): + subcommand = self.add_sub_command(subparser, 'event-alerts', 'Provisions the Grafana alert rules read from the cluster event log.') + subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list + subcommand.add_argument('--disable', help='Remove the event log alert rules and the data source.', dest='disable', action='store_true') + subcommand.add_argument('--log-limit', help='How many of the newest event log records each rule evaluation reads. Default: `1000`.', type=int, dest='log_limit') + subcommand.add_argument('--interval', help='How often the rules run, as a Grafana duration. Default: `1m`.', type=str, dest='interval') + subcommand.add_argument('--pending-period', help='How long a condition must hold before it notifies, as a Grafana duration. Default: `1m`.', type=str, dest='pending_period') + subcommand.add_argument('--plugin-url', help='Where to fetch the Infinity data source plugin from.', type=str, dest='plugin_url') + subcommand.add_argument('--plugin-preinstalled', help='The data source plugin is already in the Grafana image; do not download it.', dest='plugin_preinstalled', action='store_true') + def init_cluster__switch_write_protection(self, subparser): subcommand = self.add_sub_command(subparser, 'switch-write-protection', 'Activate v2 distrib write protection cluster-wide.') subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list @@ -1489,6 +1500,8 @@ def run(self): ret = self.cluster__set(sub_command, args) elif sub_command in ['set-shared-placement']: ret = self.cluster__set_shared_placement(sub_command, args) + elif sub_command in ['event-alerts']: + ret = self.cluster__event_alerts(sub_command, args) elif sub_command in ['switch-write-protection']: ret = self.cluster__switch_write_protection(sub_command, args) elif sub_command in ['change-name']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index cc3c1970c..b7dd6dbd8 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -472,6 +472,22 @@ def cluster__activate(self, sub_command, args): return False return True + def cluster__event_alerts(self, sub_command, args): + cluster_ops.set_event_alerts( + args.cluster_id, + enabled=not args.disable, + log_limit=args.log_limit, + interval=args.interval, + pending_period=args.pending_period, + plugin_url=args.plugin_url, + plugin_preinstalled=args.plugin_preinstalled, + ) + if args.disable: + return "Event log alert rules removed from Grafana." + return ("Event log alert rules provisioned. Grafana was restarted; the rules start " + "evaluating once the data source plugin has loaded, which takes a minute or two " + "the first time.") + def cluster__op_stop(self, sub_command, args): return cluster_ops.set_object_ops(args.cluster_id, True) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index a0745c037..699ecdfd2 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -1,6 +1,9 @@ # coding=utf-8 +import base64 import json import os +import re +import shlex import socket import subprocess import threading @@ -2551,6 +2554,175 @@ def get_logs(cluster_id, limit=50, **kwargs) -> t.List[dict]: return out +# Grafana's provisioning paths inside the monitoring container, as mounted by +# docker-compose-swarm-monitoring.yml. +GRAFANA_DATASOURCE_TARGET = "/etc/grafana/provisioning/datasources/datasource.yaml" +GRAFANA_ALERTING_TARGET = "/etc/grafana/provisioning/alerting" +GRAFANA_EVENT_DATASOURCE_TARGET = "/etc/grafana/provisioning/datasources/datasource-events.yaml" + +_GRAFANA_DURATION = re.compile(r'^(?:\d+(?:ms|[smhdwy]))+$') + + +def _event_alert_settings(enabled, log_limit, interval, pending_period, + plugin_url, plugin_preinstalled) -> t.Dict[str, t.Any]: + """Validate the tuning knobs and shape them for the templates. + + Validated here because Grafana answers a malformed duration or limit by + refusing to start, long after the command that caused it. + """ + settings = { + 'enabled': enabled, + 'logLimit': 1000 if log_limit is None else log_limit, + 'interval': interval or '1m', + 'for': pending_period or '1m', + 'plugin': { + 'url': constants.GRAFANA_EVENT_ALERTS_PLUGIN_URL if plugin_url is None else plugin_url, + 'preinstalled': plugin_preinstalled, + }, + } + + if not enabled: + return settings + + if settings['logLimit'] < 1: + raise ValueError(f"--log-limit must be at least 1, got {settings['logLimit']}") + + for flag, key in (('--interval', 'interval'), ('--pending-period', 'for')): + if not _GRAFANA_DURATION.match(settings[key]): + raise ValueError( + f"{flag} must be a Grafana duration such as 30s, 1m or 1h, got {settings[key]!r}") + + if not settings['plugin']['preinstalled'] and not settings['plugin']['url']: + raise ValueError( + "--plugin-url is empty and --plugin-preinstalled was not given, so the data source " + "plugin the rules query would never be installed") + + return settings + + +def _grafana_provisioning_dirs(cluster_docker) -> t.Tuple[str, str]: + """The host directories the running Grafana reads its provisioning from. + + Read off the service rather than computed from this package's location, so + the files land where the cluster's own mounts point. + """ + try: + service = cluster_docker.services.get(constants.MONITORING_GRAFANA_SERVICE) + except docker.errors.NotFound as e: + raise ValueError( + f"Service {constants.MONITORING_GRAFANA_SERVICE} not found: this cluster has no " + f"monitoring stack") from e + + mounts = service.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Mounts') or [] + sources = {mount.get('Target'): mount.get('Source') for mount in mounts} + for target in (GRAFANA_DATASOURCE_TARGET, GRAFANA_ALERTING_TARGET): + if not sources.get(target): + raise ValueError( + f"Service {constants.MONITORING_GRAFANA_SERVICE} has no bind mount at {target}; " + f"redeploy the monitoring stack before provisioning event log alerts") + + return os.path.dirname(sources[GRAFANA_DATASOURCE_TARGET]), sources[GRAFANA_ALERTING_TARGET] + + +def _install_provisioning_on_all_managers(files: t.Dict[str, str]) -> None: + """Write Grafana's provisioning files onto every management node. + + Grafana is constrained to node.role == manager and may be rescheduled to + any of them, and these are host paths, so a manager that lacks them + provisions no rules the first time Grafana lands there. They cannot ship + with the package the way alert_rules.yaml does, carrying this cluster's id + and secret. Written through each node's docker API, the same way SNodeAPI + is started on a remote node; base64 keeps the YAML out of shell quoting, + and user=root because the image runs as USER simplyblock, which cannot + write into the package directory. + """ + script = "; ".join( + f"umask 022 && echo {base64.b64encode(content.encode('utf-8')).decode('ascii')} " + f"| base64 -d > {shlex.quote(path)}" + for path, content in files.items() + ) + volumes = sorted({os.path.dirname(path) for path in files}) + + failed = {} + for node in db_controller.get_mgmt_nodes(): + try: + node_docker = docker.DockerClient( + base_url=f"tcp://{node.docker_ip_port}", version="auto", timeout=60) + node_docker.containers.run( + constants.SIMPLY_BLOCK_DOCKER_IMAGE, ["sh", "-c", script], user="root", + volumes=[f"{directory}:{directory}" for directory in volumes], + remove=True, detach=False) + logger.info("Provisioning files written on %s", node.mgmt_ip) + except Exception as e: + failed[node.mgmt_ip] = str(e) + + if failed: + raise RuntimeError( + "Could not write the provisioning files on: " + + "; ".join(f"{ip} ({reason})" for ip, reason in failed.items()) + + ". Fix those nodes and re-run; Grafana provisions nothing on a node it cannot " + "read them from") + + +def set_event_alerts(cluster_id, enabled=True, log_limit=None, interval=None, pending_period=None, + plugin_url=None, plugin_preinstalled=False) -> None: + """Provision (or remove) the Grafana alert rules read from the cluster event log. + + Unlike the Thanos-backed rules in alerting/alert_rules.yaml, which ship + with the package and are provisioned unconditionally, these are opt-in: + they query the control plane's REST API, which needs a Grafana plugin the + deployed image does not carry and downloads once. + + Runs against the live stack -- files onto every management node, then the + Grafana task recreated so it re-reads them -- so it configures a cluster + that already exists. + """ + cluster = db_controller.get_cluster_by_id(cluster_id) + + if cluster.mode != "docker": + raise ValueError( + "Event log alerts are provisioned by the simplyblock-operator Helm chart on " + "kubernetes; this command configures the docker monitoring stack only") + + if cluster.disable_monitoring: + raise ValueError("This cluster was created with --disable-monitoring, so it has no Grafana") + + settings = _event_alert_settings(enabled, log_limit, interval, pending_period, + plugin_url, plugin_preinstalled) + + cluster_docker = utils.get_docker_client(cluster_id) + scripts_dir, alerting_dir = _grafana_provisioning_dirs(cluster_docker) + + # The secret is the bearer token the data source sends, and reaches + # plaintext only here, on its way into a file Grafana reads -- the same + # treatment prometheus.yml gets. + rendered = utils.render_event_alert_configs( + settings, [(cluster.get_id(), cluster.secret.get_secret_value())]) + datasource_path = os.path.join(scripts_dir, utils.EVENT_ALERT_DATASOURCE_FILE) + _install_provisioning_on_all_managers({ + os.path.join(alerting_dir, utils.EVENT_ALERT_RULES_FILE): + rendered[utils.EVENT_ALERT_RULES_FILE], + datasource_path: rendered[utils.EVENT_ALERT_DATASOURCE_FILE], + }) + + # --force recreates the task, which is what makes Grafana re-read + # provisioning; a single-file bind mount also tracks the inode it started + # with, so the rewritten data source is only picked up here. Mounts are + # keyed by target and env by name, so re-running updates both in place. + # Disabling leaves them: both files are empty now, and keeping the plugin + # makes re-enabling a restart instead of another download. + update = ["sudo", "docker", "service", "update", "--force"] + if enabled: + if not settings['plugin']['preinstalled']: + update += ["--env-add", f"GF_INSTALL_PLUGINS={settings['plugin']['url']};" + f"{constants.GRAFANA_EVENT_ALERTS_PLUGIN_ID}"] + update += ["--mount-add", f"type=bind,src={datasource_path}," + f"dst={GRAFANA_EVENT_DATASOURCE_TARGET},readonly"] + + logger.info("Restarting Grafana...") + subprocess.check_call(update + [constants.MONITORING_GRAFANA_SERVICE]) + + def get_cluster(cl_id) -> dict: return db_controller.get_cluster_by_id(cl_id).get_clean_dict() diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 918455dce..3fcc708b4 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -588,6 +588,20 @@ def get_config_var(name, default=None): CR_GROUP = "storage.simplyblock.io" CR_VERSION = "v1alpha1" +# Grafana alert rules read from the cluster event log rather than from Thanos, +# provisioned by `sbctl cluster event-alerts`. The plugin id is both the folder +# Grafana installs the plugin into and the data source `type` the rules use. +# 2.12.2 is the last Infinity release compatible with Grafana 10.0.12. +GRAFANA_EVENT_ALERTS_PLUGIN_ID = "yesoreyeram-infinity-datasource" +GRAFANA_EVENT_ALERTS_PLUGIN_URL = ( + "https://grafana.com/api/plugins/yesoreyeram-infinity-datasource/versions/2.12.2/download") + +# The control plane as reached from the monitoring stack. HAProxy's default +# backend is the web API, so /api/v2 needs no route of its own; the same host +# prometheus.yml.j2 scrapes /cluster/metrics from. +MONITORING_CONTROL_PLANE_ADDR = "http://HAProxy" +MONITORING_GRAFANA_SERVICE = "monitoring_grafana" + GRAFANA_K8S_ENDPOINT = "http://simplyblock-grafana:3000" GRAYLOG_K8S_ENDPOINT = "http://simplyblock-graylog:9000" OS_K8S_ENDPOINT = "http://opensearch-cluster-master:9200" diff --git a/simplyblock_core/scripts/alerting/event_alert_rules.yaml.j2 b/simplyblock_core/scripts/alerting/event_alert_rules.yaml.j2 new file mode 100644 index 000000000..131659b15 --- /dev/null +++ b/simplyblock_core/scripts/alerting/event_alert_rules.yaml.j2 @@ -0,0 +1,363 @@ +# Grafana alert rules derived from the control plane's cluster event log, +# rendered by `sbctl cluster event-alerts` into the alerting folder Grafana is +# given as a whole, next to the checked-in, Thanos-backed alert_rules.yaml. +# +# Each query folds the newest log records into the current state of every node, +# device and cluster and returns one row per entity that is currently wrong, so +# healing is structural: the reverting event removes the row, and noDataState: +# OK turns an empty result into the resolved state. The fold reads transitions +# because the cause is not in the payload -- get_logs drops caused_by, and +# set_node_status writes every transition under the same one -- so what +# separates an operator's shutdown from a fault is the state it passes through. +# +# Kept in step with the same rules in the simplyblock-operator Helm chart +# (helm-charts/charts/simplyblock-operator/templates/controlplane_configmap.yaml). +{% macro rule(cluster, ea, uid_suffix, title, columns, severity, summary, description) %} + - uid: sbev-{{ cluster.uid_prefix }}-{{ uid_suffix }} + # Titles must be unique within a group, and each cluster adds a copy. + title: {{ title }}_{{ cluster.id }} + condition: B + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: {{ cluster.ds_uid }} + model: + refId: A + type: json + source: url + format: table + # Only the backend parser runs server-side, which is what an + # alert rule needs. root_selector is tried as gjson first and + # falls back to JSONata; opening with a bracket forces that. + parser: backend + url: /api/v2/clusters/{{ cluster.id }}/logs + url_options: + method: GET + params: + - key: limit + value: "{{ ea.logLimit }}" + root_selector: |- +{{ caller() | trim | indent(18, first=True) }} + # One numeric field and the rest strings is what makes Grafana + # read the frame as a numeric table, turning each string field + # into a label of its own alert instance. + columns: +{% for name, type in columns %} + - selector: {{ name }} + text: {{ name }} + type: {{ type }} +{% endfor %} + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + refId: B + type: threshold + datasource: + type: __expr__ + uid: __expr__ + expression: A + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: last + type: query + # An empty result is the healthy state; a failing query is not. + noDataState: OK + execErrState: Error + for: {{ ea['for'] }} + annotations: + # Single-quoted so the Go template expressions Grafana expands at + # notification time survive as written. + summary: '{{ summary | replace("'", "''") }}' + description: '{{ description | replace("'", "''") }}' + labels: + app: simplyblock + cluster: {{ cluster.id }} + severity: {{ severity }} + isPaused: false +{% endmacro %} +apiVersion: 1 +{% if EVENT_ALERT_CLUSTERS %} +groups: + - orgId: 1 + name: simplyblock_events + folder: grafana_folder + interval: {{ EVENT_ALERTS.interval }} + rules: +{% for cluster in EVENT_ALERT_CLUSTERS %} +{% call rule(cluster, EVENT_ALERTS, 'node-left-online', 'StorageNode_left_online', + [('node', 'string'), ('state', 'string'), ('since', 'string'), ('value', 'number')], + 'critical', + 'Storage node {{ $labels.node }} left the online state and is now {{ $labels.state }}.', + 'The rule folds the cluster event log into the current state of every storage node and' + ' fires while a node sits in anything other than online or in_creation. A shutdown or' + ' removal an operator asked for is exempt: those reach in_shutdown, pending_removal,' + ' in_removal, or removed, which no fault path does. The alert clears when the node' + ' returns to online.') %} +( + $planned := ['in_shutdown', 'in_removal', 'pending_removal', 'removed']; + $healthy := ['online', 'in_creation']; + $ev := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Storage node status changed from: ')]]; + $fold := function($evs) { + $reduce($evs, function($a, $e) { + ( + $to := $substringAfter($e.Message, ' to: '); + { + 'state': $to, + 'since': $e.Date, + 'planned': $to in $planned or ($a.planned and $to = 'offline') + } + ) + }, { 'state': 'online', 'since': '', 'planned': false }) + }; + $nodes := $count($ev) = 0 ? [] : [$each($ev{ NodeId: [$] }, function($evs, $id) { + $merge([{ 'node': $id }, $fold($evs)]) + })]; + [$nodes[$not(state in $healthy) and $not(planned)].{ + 'node': node, + 'state': state, + 'since': since, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'device-unavailable', 'Device_became_unavailable', + [('device', 'string'), ('order', 'string'), ('since', 'string'), ('value', 'number')], + 'warning', + 'Device {{ $labels.device }} (cluster device order {{ $labels.order }}) is unavailable.', + 'The rule fires for a device that is currently unavailable and did not become so as' + ' part of a whole node going down. A node leaving online cascades every one of its' + ' devices to unavailable within seconds, and that is reported once as the storage node' + ' alert rather than once per device. The alert clears when the device returns to' + ' online.') %} +( + $window := 120000; + $ms := function($d) { $toMillis($substring($d & '.000', 0, 23), '[Y0001]-[M01]-[D01] [H01]:[m01]:[s01].[f001]') }; + $healthy := ['online', 'in_creation']; + $nodeEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Storage node status changed from: ')]]; + $cascades := [$map($nodeEv[$not($substringAfter(Message, ' to: ') in $healthy)], function($e) { $ms($e.Date) })]; + $devEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Device status changed from: ')]]; + $fold := function($evs) { + $reduce($evs, function($a, $e) { + { + 'state': $substringAfter($e.Message, ' to: '), + 'since': $e.Date, + 'at': $ms($e.Date), + 'order': $e.Storage_ID + } + }, { 'state': 'online', 'since': '', 'at': 0, 'order': '' }) + }; + $devs := $count($devEv) = 0 ? [] : [$each($devEv{ NodeId: [$] }, function($evs, $id) { + ( + $f := $fold($evs); + $merge([ + { 'device': $id, 'cascade': $count($cascades[$ <= $f.at and $f.at - $ <= $window]) > 0 }, + $f + ]) + ) + })]; + [$devs[state = 'unavailable' and $not(cascade)].{ + 'device': device, + 'order': order, + 'since': since, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'device-removed', 'Device_removed', + [('device', 'string'), ('order', 'string'), ('since', 'string'), ('value', 'number')], + 'critical', + 'Device {{ $labels.device }} (cluster device order {{ $labels.order }}) was removed.', + 'The rule fires for every device whose latest state is removed, whatever the cause. It' + ' clears when the device is brought back online, and otherwise ages out a day after the' + ' removal, since a device that is genuinely gone has no reversion to wait for.') %} +( + $recent := 86400000; + $ms := function($d) { $toMillis($substring($d & '.000', 0, 23), '[Y0001]-[M01]-[D01] [H01]:[m01]:[s01].[f001]') }; + $now := $millis(); + $devEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Device status changed from: ')]]; + $fold := function($evs) { + $reduce($evs, function($a, $e) { + { + 'state': $substringAfter($e.Message, ' to: '), + 'since': $e.Date, + 'at': $ms($e.Date), + 'order': $e.Storage_ID + } + }, { 'state': 'online', 'since': '', 'at': 0, 'order': '' }) + }; + $devs := $count($devEv) = 0 ? [] : [$each($devEv{ NodeId: [$] }, function($evs, $id) { + $merge([{ 'device': $id }, $fold($evs)]) + })]; + [$devs[state = 'removed' and $now - at <= $recent].{ + 'device': device, + 'order': order, + 'since': since, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'cluster-degraded', 'Cluster_became_degraded', + [('state', 'string'), ('since', 'string'), ('value', 'number')], + 'warning', + 'The cluster is degraded.', + "The rule fires while the cluster's latest status is degraded. A degradation an operator" + ' caused by shutting a node down is exempt, decided by whether any node currently out of' + ' service got there through in_shutdown or a removal state. The alert clears when the' + ' cluster returns to active.') %} +( + $planned := ['in_shutdown', 'in_removal', 'pending_removal', 'removed']; + $healthy := ['online', 'in_creation']; + $nodeEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Storage node status changed from: ')]]; + $fold := function($evs) { + $reduce($evs, function($a, $e) { + ( + $to := $substringAfter($e.Message, ' to: '); + { + 'state': $to, + 'planned': $to in $planned or ($a.planned and $to = 'offline') + } + ) + }, { 'state': 'online', 'planned': false }) + }; + $nodes := $count($nodeEv) = 0 ? [] : [$each($nodeEv{ NodeId: [$] }, function($evs, $id) { $fold($evs) })]; + $operatorDown := $count($nodes[$not(state in $healthy) and planned]) > 0; + $cluEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Cluster status changed from ')]]; + $last := $cluEv[-1]; + $state := $exists($last) ? $substringAfter($last.Message, ' to ') : 'active'; + [$state = 'degraded' and $not($operatorDown) ? { + 'state': $state, + 'since': $last.Date, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'cluster-suspended', 'Cluster_became_suspended', + [('state', 'string'), ('since', 'string'), ('value', 'number')], + 'critical', + 'The cluster is suspended.', + "The rule fires while the cluster's latest status is suspended, whatever the cause," + " including a suspension that followed an operator's node shutdown. The alert clears" + ' when the cluster leaves suspended.') %} +( + $cluEv := [$[Event = 'STATUS_CHANGE' and $contains(Message, 'Cluster status changed from ')]]; + $last := $cluEv[-1]; + $state := $exists($last) ? $substringAfter($last.Message, ' to ') : 'active'; + [$state = 'suspended' ? { + 'state': $state, + 'since': $last.Date, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'cluster-capacity', 'Cluster_capacity_reached', + [('kind', 'string'), ('severity', 'string'), ('value', 'number')], + 'warning', + '{{ $labels.kind }} cluster capacity is at {{ $value }}% ({{ $labels.severity }}).', + 'The rule reports the newest capacity event per kind, absolute and provisioned, and' + ' carries the utilization as its value. The control plane emits no back-to-normal' + ' event, so the alert clears by the event ceasing: the capacity monitor re-emits every' + ' 30 seconds while over a threshold, except an absolute critical which it throttles to' + ' once every 15 minutes, and the rule allows for both.') %} +( + $ms := function($d) { $toMillis($substring($d & '.000', 0, 23), '[Y0001]-[M01]-[D01] [H01]:[m01]:[s01].[f001]') }; + $now := $millis(); + $ev := [$[Event = 'CAPACITY']]; + $latest := $count($ev) = 0 ? [] : [$each( + $ev{ ($contains(Message, 'provisioned capacity') ? 'provisioned' : 'absolute'): [$] }, + function($evs, $kind) { + ( + $e := $evs[-1]; + $window := ($kind = 'absolute' and $e.Level = 'Critical') ? 1200000 : 180000; + { + 'kind': $kind, + 'severity': $e.Level, + 'value': $substringBefore($substringAfter($e.Message, 'reached: '), '%'), + 'fresh': $now - $ms($e.Date) <= $window + } + ) + })]; + [$latest[fresh and (severity = 'Warning' or severity = 'Critical')].{ + 'kind': kind, + 'severity': severity, + 'value': value + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'jm-records-threshold', 'JM_records_threshold_exceeded', + [('node', 'string'), ('since', 'string'), ('value', 'number')], + 'critical', + 'The journal on storage node {{ $labels.node }} holds more records awaiting compression' + ' than the configured threshold.', + 'Compression is not keeping up with the journal, and journal replay on the next restart' + ' or failover grows with every record. The control plane latches this alert on the' + ' upward crossing and re-arms silently, emitting nothing when the backlog drains, so the' + ' alert clears an hour after the last crossing rather than on a recovery event.') %} +( + $recent := 3600000; + $ms := function($d) { $toMillis($substring($d & '.000', 0, 23), '[Y0001]-[M01]-[D01] [H01]:[m01]:[s01].[f001]') }; + $now := $millis(); + $ev := [$[Event = 'JM_COMPRESSION_BACKLOG']]; + $latest := $count($ev) = 0 ? [] : [$each($ev{ NodeId: [$] }, function($evs, $id) { + ( + $e := $evs[-1]; + { 'node': $id, 'since': $e.Date, 'at': $ms($e.Date) } + ) + })]; + [$latest[$now - at <= $recent].{ + 'node': node, + 'since': since, + 'value': 1 + }] +) +{% endcall %} +{% call rule(cluster, EVENT_ALERTS, 'jm-compression-error', 'JM_compression_error', + [('node', 'string'), ('jm_vuid', 'string'), ('detail', 'string'), ('value', 'number')], + 'critical', + 'Journal compression failed on storage node {{ $labels.node }} for jm_vuid' + ' {{ $labels.jm_vuid }}: {{ $labels.detail }}.', + 'The rule tracks the newest compression event per node and journal and fires while that' + ' event is an error, which is either a compression_failed status or a non-zero error' + ' code. The alert clears when the same journal reports a clean compression run.') %} +( + $ev := [$[Event = 'jm_compression']]; + $latest := $count($ev) = 0 ? [] : [$each($ev{ (NodeId & '/' & VUID): [$] }, function($evs, $key) { + ( + $e := $evs[-1]; + { + 'node': $substringBefore($key, '/'), + 'jm_vuid': $substringAfter($key, '/'), + 'level': $e.Level, + 'detail': $e.Message + } + ) + })]; + [$latest[level = 'Error'].{ + 'node': node, + 'jm_vuid': jm_vuid, + 'detail': detail, + 'value': 1 + }] +) +{% endcall %} +{% endfor %} +{% else %} +# No cluster to read an event log from, or the event log alerts are off, so +# nothing to provision. Rendered rather than skipped, so that `--disable` +# overwrites the rules an earlier run left on this management node instead of +# leaving Grafana to provision them again on its next restart. +groups: [] +{% endif %} diff --git a/simplyblock_core/scripts/datasource-events.yml.j2 b/simplyblock_core/scripts/datasource-events.yml.j2 new file mode 100644 index 000000000..6dab0d0ba --- /dev/null +++ b/simplyblock_core/scripts/datasource-events.yml.j2 @@ -0,0 +1,37 @@ +# Grafana data sources for the cluster event log alert rules, rendered by +# `sbctl cluster event-alerts`. One per cluster: /api/v2 authenticates with the +# cluster secret as a bearer token, and Grafana binds credentials to a data +# source rather than to a query. The secret goes through secureJsonData, which +# Grafana stores encrypted and never returns to the browser. +apiVersion: 1 +{% if EVENT_ALERT_CLUSTERS %} +datasources: +{% for cluster in EVENT_ALERT_CLUSTERS %} + - name: Simplyblock Events {{ cluster.id }} + type: yesoreyeram-infinity-datasource + uid: {{ cluster.ds_uid }} + url: {{ CONTROL_PLANE_ADDR }} + access: proxy + editable: false + jsonData: + httpHeaderName1: Authorization + # Infinity refuses every URL query until the host is allowlisted, and + # reports the refusal as a per-rule evaluation error rather than as a + # configuration warning. Matched by prefix, so the base address covers + # every /api/v2 path the rules query. + allowedHosts: + - {{ CONTROL_PLANE_ADDR }} + # Otherwise the health check probes the bare base URL, which the control + # plane answers with a string Infinity cannot read as a table. + customHealthCheckEnabled: false + tlsSkipVerify: true + timeoutInSeconds: 30 + secureJsonData: + httpHeaderValue1: {{ ('Bearer ' ~ cluster.secret) | tojson }} +{% endfor %} +{% else %} +# Off, so nothing to provision. Written rather than deleted: the file stays +# bind-mounted, and a bind mount whose source is gone comes back as a +# directory, which Grafana refuses to start on. +datasources: [] +{% endif %} diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 50183c6ec..babc1aef2 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1,5 +1,6 @@ # coding=utf-8 import glob +import hashlib import json import logging import math @@ -28,7 +29,7 @@ from docker.errors import APIError, DockerException, ImageNotFound, NotFound import tempfile -from jinja2 import Environment, FileSystemLoader +from jinja2 import Environment, FileSystemLoader, StrictUndefined from simplyblock_core import constants from simplyblock_core import shell_utils @@ -61,6 +62,11 @@ ALERT_RESOURCES_FILE = "alert_resources.yaml" ALERTS_TEMPLATE_FOLDER = "simplyblock_core/scripts/alerting/" +SCRIPTS_FOLDER = "simplyblock_core/scripts/" + +# Provisioning files for the cluster event log alerts (`cluster event-alerts`). +EVENT_ALERT_RULES_FILE = "event_alert_rules.yaml" +EVENT_ALERT_DATASOURCE_FILE = "datasource-events.yml" def get_env_var(name, default=None, is_required=False): if not name: @@ -2567,6 +2573,10 @@ def _alerts_template_folder() -> str: return os.path.join(_top_dir(), ALERTS_TEMPLATE_FOLDER) +def _scripts_folder() -> str: + return os.path.join(_top_dir(), SCRIPTS_FOLDER) + + def _top_dir() -> str: return os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) @@ -2608,6 +2618,45 @@ def render_configfile_alerting(alert_config: Dict[str, Any]) -> str: return template.render(alert_config) +def render_event_alert_configs(settings: Dict[str, Any], + clusters: Iterable[Tuple[str, str]]) -> Dict[str, str]: + """Render the Grafana provisioning files for the cluster event log alerts. + + Returns {filename: content} for both files, always, so that disabling the + alerts overwrites what enabling them wrote. A cluster missing either its id + or its secret is skipped: the API checks that the two agree, so a + half-configured data source fails every evaluation with a 401. Both uids + are derived from the cluster id so the two files agree on them and a re-run + updates the same objects instead of adding a second set. + + StrictUndefined because a missing value would reach Grafana as empty YAML, + which it answers by refusing to start. + """ + entries = [] + if settings.get('enabled'): + for cluster_id, secret in clusters: + if not cluster_id or not secret: + continue + digest = hashlib.sha256(cluster_id.encode('utf-8')).hexdigest() + entries.append({'id': cluster_id, 'secret': secret, + 'ds_uid': f'sbev-{digest[:12]}', 'uid_prefix': digest[:8]}) + + values = { + 'EVENT_ALERTS': settings, + 'EVENT_ALERT_CLUSTERS': entries, + 'CONTROL_PLANE_ADDR': constants.MONITORING_CONTROL_PLANE_ADDR, + } + + rendered = {} + for folder, filename in ((_alerts_template_folder(), EVENT_ALERT_RULES_FILE), + (_scripts_folder(), EVENT_ALERT_DATASOURCE_FILE)): + env = Environment(loader=FileSystemLoader(folder), trim_blocks=True, lstrip_blocks=True, + undefined=StrictUndefined) + rendered[filename] = env.get_template(f'{filename}.j2').render(values) + + return rendered + + def render_and_deploy_alerting_configs(alert_config: Optional[Dict[str, Any]], contact_point: Optional[str], grafana_endpoint, cluster_uuid, cluster_secret): top_dir = _top_dir()