feat(alerting): cluster event log alert rules for docker clusters

simplyblock_cli/cli-reference.yaml

@@ -1586,6 +1586,53 @@ commands:
             dest: force
             type: bool
             action: store_true
+      - name: event-alerts
+        help: "Provisions the Grafana alert rules read from the cluster event log."
+        usage: >
+          Adds eight alert rules that read the cluster event log through /api/v2/clusters/<id>/logs
+          instead of the Thanos metrics the rules in alert_rules.yaml read. The event log carries
+          the transition an entity made, which is what lets these rules tell an operator's shutdown
+          apart from a fault that ended in the same state, and lets them report a device removal
+          and the two journal-compression conditions that no metric carries.
+          Opt-in: the rules need a REST data source, so Grafana downloads a plugin (~74 MB) on the
+          next restart. Works on a running cluster -- the provisioning files are written to every
+          management node and the Grafana task is recreated, so Grafana is unavailable for a few
+          seconds. Run it once, from any management node; re-run it to change a setting.
+        arguments:
+          - name: "cluster_id"
+            help: "The cluster id."
+            dest: cluster_id
+            type: str
+            completer: _completer_get_cluster_list
+          - name: "--disable"
+            help: "Remove the event log alert rules and the data source."
+            dest: disable
+            type: bool
+            action: store_true
+          - name: "--log-limit"
+            help: "How many of the newest event log records each rule evaluation reads. Default: `1000`."
+            description: >
+              Too low on a cluster that produces events quickly and an alert heals itself as soon
+              as the transition that opened it scrolls out of the window.
+            dest: log_limit
+            type: int
+          - name: "--interval"
+            help: "How often the rules run, as a Grafana duration. Default: `1m`."
+            dest: interval
+            type: str
+          - name: "--pending-period"
+            help: "How long a condition must hold before it notifies, as a Grafana duration. Default: `1m`."
+            dest: pending_period
+            type: str
+          - name: "--plugin-url"
+            help: "Where to fetch the Infinity data source plugin from."
+            dest: plugin_url
+            type: str
+          - name: "--plugin-preinstalled"
+            help: "The data source plugin is already in the Grafana image; do not download it."
+            dest: plugin_preinstalled
+            type: bool
+            action: store_true
       - name: switch-write-protection
         help: "Activate v2 distrib write protection cluster-wide."
         usage: "Needed once after upgrading a cluster from a release without v2; new clusters are created on v2 already. Sends the runtime distr_write_protection_v2 RPC to every ONLINE storage node -- the only way an already-created distrib gains v2, since a create parameter cannot retrofit an existing bdev. Only if every online node succeeds is the generation recorded, on the cluster row and on every node's lvstore_stack distrib entries, so later (re-)creations use the v2 parameter. Partial success records nothing, so a retry is a plain re-run. Offline nodes are not an error: they have no running bdev to migrate and come back on v2 at their next restart."

simplyblock_cli/cli.py

@@ -387,6 +387,7 @@ def init_cluster(self):
         if self.developer_mode:
             self.init_cluster__set(subparser)
         self.init_cluster__set_shared_placement(subparser)
+        self.init_cluster__event_alerts(subparser)
         self.init_cluster__switch_write_protection(subparser)
         self.init_cluster__change_name(subparser)
         self.init_cluster__add_replication(subparser)
@@ -621,6 +622,16 @@ def init_cluster__set_shared_placement(self, subparser):
         subcommand.add_argument('--disable', help='Reverse transition (per-chunk -> per-page). Debug only; only safe on a balanced or empty bdev. Requires --force.', dest='disable', action='store_true')
         subcommand.add_argument('--force', help='Bypass the rebalancing / non-online-node guards. Required when --disable is passed.', dest='force', action='store_true')
 
+    def init_cluster__event_alerts(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'event-alerts', 'Provisions the Grafana alert rules read from the cluster event log.')
+        subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list
+        subcommand.add_argument('--disable', help='Remove the event log alert rules and the data source.', dest='disable', action='store_true')
+        subcommand.add_argument('--log-limit', help='How many of the newest event log records each rule evaluation reads. Default: `1000`.', type=int, dest='log_limit')
+        subcommand.add_argument('--interval', help='How often the rules run, as a Grafana duration. Default: `1m`.', type=str, dest='interval')
+        subcommand.add_argument('--pending-period', help='How long a condition must hold before it notifies, as a Grafana duration. Default: `1m`.', type=str, dest='pending_period')
+        subcommand.add_argument('--plugin-url', help='Where to fetch the Infinity data source plugin from.', type=str, dest='plugin_url')
+        subcommand.add_argument('--plugin-preinstalled', help='The data source plugin is already in the Grafana image; do not download it.', dest='plugin_preinstalled', action='store_true')
+
     def init_cluster__switch_write_protection(self, subparser):
         subcommand = self.add_sub_command(subparser, 'switch-write-protection', 'Activate v2 distrib write protection cluster-wide.')
         subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list
@@ -1489,6 +1500,8 @@ def run(self):
                         ret = self.cluster__set(sub_command, args)
                 elif sub_command in ['set-shared-placement']:
                     ret = self.cluster__set_shared_placement(sub_command, args)
+                elif sub_command in ['event-alerts']:
+                    ret = self.cluster__event_alerts(sub_command, args)
                 elif sub_command in ['switch-write-protection']:
                     ret = self.cluster__switch_write_protection(sub_command, args)
                 elif sub_command in ['change-name']:

simplyblock_cli/clibase.py

@@ -472,6 +472,22 @@ def cluster__activate(self, sub_command, args):
             return False
         return True
 
+    def cluster__event_alerts(self, sub_command, args):
+        cluster_ops.set_event_alerts(
+            args.cluster_id,
+            enabled=not args.disable,
+            log_limit=args.log_limit,
+            interval=args.interval,
+            pending_period=args.pending_period,
+            plugin_url=args.plugin_url,
+            plugin_preinstalled=args.plugin_preinstalled,
+        )
+        if args.disable:
+            return "Event log alert rules removed from Grafana."
+        return ("Event log alert rules provisioned. Grafana was restarted; the rules start "
+                "evaluating once the data source plugin has loaded, which takes a minute or two "
+                "the first time.")
+
     def cluster__op_stop(self, sub_command, args):
         return cluster_ops.set_object_ops(args.cluster_id, True)
 

simplyblock_core/cluster_ops.py

@@ -1,6 +1,9 @@
 # coding=utf-8
+import base64
 import json
 import os
+import re
+import shlex
 import socket
 import subprocess
 import threading
@@ -2551,6 +2554,175 @@ def get_logs(cluster_id, limit=50, **kwargs) -> t.List[dict]:
     return out
 
 
+# Grafana's provisioning paths inside the monitoring container, as mounted by
+# docker-compose-swarm-monitoring.yml.
+GRAFANA_DATASOURCE_TARGET = "/etc/grafana/provisioning/datasources/datasource.yaml"
+GRAFANA_ALERTING_TARGET = "/etc/grafana/provisioning/alerting"
+GRAFANA_EVENT_DATASOURCE_TARGET = "/etc/grafana/provisioning/datasources/datasource-events.yaml"
+
+_GRAFANA_DURATION = re.compile(r'^(?:\d+(?:ms|[smhdwy]))+$')
+
+
+def _event_alert_settings(enabled, log_limit, interval, pending_period,
+                          plugin_url, plugin_preinstalled) -> t.Dict[str, t.Any]:
+    """Validate the tuning knobs and shape them for the templates.
+
+    Validated here because Grafana answers a malformed duration or limit by
+    refusing to start, long after the command that caused it.
+    """
+    settings = {
+        'enabled': enabled,
+        'logLimit': 1000 if log_limit is None else log_limit,
+        'interval': interval or '1m',
+        'for': pending_period or '1m',
+        'plugin': {
+            'url': constants.GRAFANA_EVENT_ALERTS_PLUGIN_URL if plugin_url is None else plugin_url,
+            'preinstalled': plugin_preinstalled,
+        },
+    }
+
+    if not enabled:
+        return settings
+
+    if settings['logLimit'] < 1:
+        raise ValueError(f"--log-limit must be at least 1, got {settings['logLimit']}")
+
+    for flag, key in (('--interval', 'interval'), ('--pending-period', 'for')):
+        if not _GRAFANA_DURATION.match(settings[key]):
+            raise ValueError(
+                f"{flag} must be a Grafana duration such as 30s, 1m or 1h, got {settings[key]!r}")
+
+    if not settings['plugin']['preinstalled'] and not settings['plugin']['url']:
+        raise ValueError(
+            "--plugin-url is empty and --plugin-preinstalled was not given, so the data source "
+            "plugin the rules query would never be installed")
+
+    return settings
+
+
+def _grafana_provisioning_dirs(cluster_docker) -> t.Tuple[str, str]:
+    """The host directories the running Grafana reads its provisioning from.
+
+    Read off the service rather than computed from this package's location, so
+    the files land where the cluster's own mounts point.
+    """
+    try:
+        service = cluster_docker.services.get(constants.MONITORING_GRAFANA_SERVICE)
+    except docker.errors.NotFound as e:
+        raise ValueError(
+            f"Service {constants.MONITORING_GRAFANA_SERVICE} not found: this cluster has no "
+            f"monitoring stack") from e
+
+    mounts = service.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Mounts') or []
+    sources = {mount.get('Target'): mount.get('Source') for mount in mounts}
+    for target in (GRAFANA_DATASOURCE_TARGET, GRAFANA_ALERTING_TARGET):
+        if not sources.get(target):
+            raise ValueError(
+                f"Service {constants.MONITORING_GRAFANA_SERVICE} has no bind mount at {target}; "
+                f"redeploy the monitoring stack before provisioning event log alerts")
+
+    return os.path.dirname(sources[GRAFANA_DATASOURCE_TARGET]), sources[GRAFANA_ALERTING_TARGET]
+
+
+def _install_provisioning_on_all_managers(files: t.Dict[str, str]) -> None:
+    """Write Grafana's provisioning files onto every management node.
+
+    Grafana is constrained to node.role == manager and may be rescheduled to
+    any of them, and these are host paths, so a manager that lacks them
+    provisions no rules the first time Grafana lands there. They cannot ship
+    with the package the way alert_rules.yaml does, carrying this cluster's id
+    and secret. Written through each node's docker API, the same way SNodeAPI
+    is started on a remote node; base64 keeps the YAML out of shell quoting,
+    and user=root because the image runs as USER simplyblock, which cannot
+    write into the package directory.
+    """
+    script = "; ".join(
+        f"umask 022 && echo {base64.b64encode(content.encode('utf-8')).decode('ascii')} "
+        f"| base64 -d > {shlex.quote(path)}"
+        for path, content in files.items()
+    )
+    volumes = sorted({os.path.dirname(path) for path in files})
+
+    failed = {}
+    for node in db_controller.get_mgmt_nodes():
+        try:
+            node_docker = docker.DockerClient(
+                base_url=f"tcp://{node.docker_ip_port}", version="auto", timeout=60)
+            node_docker.containers.run(
+                constants.SIMPLY_BLOCK_DOCKER_IMAGE, ["sh", "-c", script], user="root",
+                volumes=[f"{directory}:{directory}" for directory in volumes],
+                remove=True, detach=False)
+            logger.info("Provisioning files written on %s", node.mgmt_ip)
+        except Exception as e:
+            failed[node.mgmt_ip] = str(e)
+
+    if failed:
+        raise RuntimeError(
+            "Could not write the provisioning files on: "
+            + "; ".join(f"{ip} ({reason})" for ip, reason in failed.items())
+            + ". Fix those nodes and re-run; Grafana provisions nothing on a node it cannot "
+              "read them from")
+
+
+def set_event_alerts(cluster_id, enabled=True, log_limit=None, interval=None, pending_period=None,
+                     plugin_url=None, plugin_preinstalled=False) -> None:
+    """Provision (or remove) the Grafana alert rules read from the cluster event log.
+
+    Unlike the Thanos-backed rules in alerting/alert_rules.yaml, which ship
+    with the package and are provisioned unconditionally, these are opt-in:
+    they query the control plane's REST API, which needs a Grafana plugin the
+    deployed image does not carry and downloads once.
+
+    Runs against the live stack -- files onto every management node, then the
+    Grafana task recreated so it re-reads them -- so it configures a cluster
+    that already exists.
+    """
+    cluster = db_controller.get_cluster_by_id(cluster_id)
+
+    if cluster.mode != "docker":
+        raise ValueError(
+            "Event log alerts are provisioned by the simplyblock-operator Helm chart on "
+            "kubernetes; this command configures the docker monitoring stack only")
+
+    if cluster.disable_monitoring:
+        raise ValueError("This cluster was created with --disable-monitoring, so it has no Grafana")
+
+    settings = _event_alert_settings(enabled, log_limit, interval, pending_period,
+                                     plugin_url, plugin_preinstalled)
+
+    cluster_docker = utils.get_docker_client(cluster_id)
+    scripts_dir, alerting_dir = _grafana_provisioning_dirs(cluster_docker)
+
+    # The secret is the bearer token the data source sends, and reaches
+    # plaintext only here, on its way into a file Grafana reads -- the same
+    # treatment prometheus.yml gets.
+    rendered = utils.render_event_alert_configs(
+        settings, [(cluster.get_id(), cluster.secret.get_secret_value())])
+    datasource_path = os.path.join(scripts_dir, utils.EVENT_ALERT_DATASOURCE_FILE)
+    _install_provisioning_on_all_managers({
+        os.path.join(alerting_dir, utils.EVENT_ALERT_RULES_FILE):
+            rendered[utils.EVENT_ALERT_RULES_FILE],
+        datasource_path: rendered[utils.EVENT_ALERT_DATASOURCE_FILE],
+    })
+
+    # --force recreates the task, which is what makes Grafana re-read
+    # provisioning; a single-file bind mount also tracks the inode it started
+    # with, so the rewritten data source is only picked up here. Mounts are
+    # keyed by target and env by name, so re-running updates both in place.
+    # Disabling leaves them: both files are empty now, and keeping the plugin
+    # makes re-enabling a restart instead of another download.
+    update = ["sudo", "docker", "service", "update", "--force"]
+    if enabled:
+        if not settings['plugin']['preinstalled']:
+            update += ["--env-add", f"GF_INSTALL_PLUGINS={settings['plugin']['url']};"
+                                    f"{constants.GRAFANA_EVENT_ALERTS_PLUGIN_ID}"]
+        update += ["--mount-add", f"type=bind,src={datasource_path},"
+                                  f"dst={GRAFANA_EVENT_DATASOURCE_TARGET},readonly"]
+
+    logger.info("Restarting Grafana...")
+    subprocess.check_call(update + [constants.MONITORING_GRAFANA_SERVICE])
+
+
 def get_cluster(cl_id) -> dict:
     return db_controller.get_cluster_by_id(cl_id).get_clean_dict()
 

simplyblock_core/constants.py

@@ -588,6 +588,20 @@ def get_config_var(name, default=None):
 CR_GROUP = "storage.simplyblock.io"
 CR_VERSION  = "v1alpha1"
 
+# Grafana alert rules read from the cluster event log rather than from Thanos,
+# provisioned by `sbctl cluster event-alerts`. The plugin id is both the folder
+# Grafana installs the plugin into and the data source `type` the rules use.
+# 2.12.2 is the last Infinity release compatible with Grafana 10.0.12.
+GRAFANA_EVENT_ALERTS_PLUGIN_ID = "yesoreyeram-infinity-datasource"
+GRAFANA_EVENT_ALERTS_PLUGIN_URL = (
+    "https://grafana.com/api/plugins/yesoreyeram-infinity-datasource/versions/2.12.2/download")
+
+# The control plane as reached from the monitoring stack. HAProxy's default
+# backend is the web API, so /api/v2 needs no route of its own; the same host
+# prometheus.yml.j2 scrapes /cluster/metrics from.
+MONITORING_CONTROL_PLANE_ADDR = "http://HAProxy"
+MONITORING_GRAFANA_SERVICE = "monitoring_grafana"
+
 GRAFANA_K8S_ENDPOINT = "http://simplyblock-grafana:3000"
 GRAYLOG_K8S_ENDPOINT = "http://simplyblock-graylog:9000"
 OS_K8S_ENDPOINT = "http://opensearch-cluster-master:9200"