From 79b2aebf48e1bd62d7ca0372342d0682e37349c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Fri, 14 Aug 2026 15:31:00 -0300
Subject: [PATCH 1/5] docs: add inventory file for multi-server-devnet skill

---
 .claude/skills/multi-server-devnet/SKILL.md   |  34 +++-
 .../scripts/devnet.env.example                |   6 -
 .../scripts/devnet.inventory.example          |  50 ++++++
 .../multi-server-devnet/scripts/inventory.sh  | 152 ++++++++++++++++++
 .gitignore                                    |   3 +
 5 files changed, 232 insertions(+), 13 deletions(-)
 create mode 100644 .claude/skills/multi-server-devnet/scripts/devnet.inventory.example
 create mode 100755 .claude/skills/multi-server-devnet/scripts/inventory.sh

diff --git a/.claude/skills/multi-server-devnet/SKILL.md b/.claude/skills/multi-server-devnet/SKILL.md
index 4246dcbb..3d02d572 100644
--- a/.claude/skills/multi-server-devnet/SKILL.md
+++ b/.claude/skills/multi-server-devnet/SKILL.md
@@ -50,12 +50,31 @@ Nothing about the servers is hardcoded. Establish these from the operator:
 Put those values in `scripts/devnet.env` (copy `scripts/devnet.env.example`;
 gitignored) instead of retyping them: the operator-side scripts source it via
 `scripts/devnet-env.sh`, and an env var exported in the shell still wins over the
-file. It is the one place a deployment's hosts, urls, and Grafana ids live.
-Per-devnet `NODES`/`SUBNETS` are recorded there too, but as the operator's
-inventory — the scripts take them as positional args, and the authority on a
-running devnet is always its own `genesis/config.yaml`
-(`ATTESTATION_COMMITTEE_COUNT`), which `start-devnet.sh` and `convert.sh` check
-against.
+file. It is the one place a deployment's urls and Grafana ids live.
+
+**The servers themselves live in `scripts/devnet.inventory`** (copy
+`scripts/devnet.inventory.example`; gitignored) — name, ip, tags, and per-devnet
+`NODES`/`SUBNETS`, one row per host. Query it with `scripts/inventory.sh` rather
+than reading it by hand:
+
+```bash
+inventory.sh --tag devnet-ab --field ip        # ips, one per line, for a loop
+inventory.sh --tag devnet-ab --tag aggregator  # AND across tags
+inventory.sh --tag validator                   # derived, see below
+SERVERS=$(inventory.sh --field name | tr '\n' ' ')
+```
+
+Tag conventions: a `devnet-*` tag names the chain a host's nodes belong to (two
+hosts sharing one means the split-chain model, *not* two like-named devnets), and
+`aggregator` is a whole-server role. `validator` is **derived** — "has a `devnet-*`
+tag and is not tagged `aggregator`" — so it can never disagree with the aggregator
+tag. An unknown tag exits 2 with the known-tag list rather than returning nothing,
+because a typo that yields an empty loop reports success while doing nothing.
+
+`NODES`/`SUBNETS` there are still the operator's inventory: the scripts take them
+as positional args, and the authority on a running devnet is always its own
+`genesis/config.yaml` (`ATTESTATION_COMMITTEE_COUNT`), which `start-devnet.sh` and
+`convert.sh` check against.
 
 Per devnet, node `n` (0 ≤ n < NODES) on its host:
 
@@ -473,7 +492,8 @@ Swap (persistent): `fallocate -l 16G /swapfile && chmod 600 && mkswap && swapon`
 | `promtail-config.sh` | operator | `NETWORK HOST_IP LOKI_PUSH_URL [N:client ...]` | Emit a per-host promtail.yml (docker_sd → central Loki, labels mirror prometheus, backlog guard) |
 | `sweep.sh` | operator | `CENTRAL_PROM_URL` | Cross-devnet audit: head/justified/finalized + client mix from the central Prometheus |
 | `deploy-finality-alert.sh` | operator | `[WEBHOOK_FILE]`; `METRICS_HOST`, `GRAFANA_*`, `PROM_DS_UID`; `DRY_RUN` | Render + ship the "lost finality" Slack alert to the central Grafana |
-| `devnet-env.sh` / `devnet.env.example` | operator | `$DEVNET_ENV`, `./devnet.env`, script dir | Load this deployment's hosts/urls/Grafana ids as defaults; exported vars win. Copy the example to `devnet.env` (gitignored) once |
+| `devnet-env.sh` / `devnet.env.example` | operator | `$DEVNET_ENV`, `./devnet.env`, script dir | Load this deployment's urls/Grafana ids as defaults; exported vars win. Copy the example to `devnet.env` (gitignored) once |
+| `inventory.sh` / `devnet.inventory.example` | operator | `[--tag T]... [--not-tag T]... [--field F] [--file P] [--count]`; `$DEVNET_INVENTORY`, `./devnet.inventory`, script dir | Select servers by tag from the fleet inventory (name, ip, tags, NODES, SUBNETS). `devnet-*` tags name the chain, `aggregator` is a server role, `validator` is derived (`devnet-*` and not `aggregator`). Unknown tag → exit 2 + known-tag list, so a typo can't masquerade as an empty fleet. Copy the example to `devnet.inventory` (gitignored) once |
 
 **Grafana dashboards** (copy into the central Grafana's dashboards dir —
 `GRAFANA_DASHBOARDS_DIR`, *not* the provisioning tree; they auto-load in ~30s and
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet.env.example b/.claude/skills/multi-server-devnet/scripts/devnet.env.example
index ee6a4405..5e1ff70e 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet.env.example
+++ b/.claude/skills/multi-server-devnet/scripts/devnet.env.example
@@ -8,12 +8,6 @@
 SERVERS="host-a host-b"                 # ssh targets, any count/names
 SSH_USER=user                           # login user; docker is invoked with sudo
 
-# Per-devnet size, as INVENTORY: the scripts take NODES/SUBNETS as positional
-# args, and the authority for a running devnet is its own genesis/config.yaml
-# (ATTESTATION_COMMITTEE_COUNT). Recorded here so a restart doesn't have to guess.
-#   host-a: NODES=32 SUBNETS=4
-#   host-b: NODES=16 SUBNETS=2
-
 # --- central metrics / logs stack -------------------------------------------
 # sweep.sh reads CENTRAL_PROM_URL; the other two are args you pass to
 # prometheus-config.sh / promtail-config.sh.
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
new file mode 100644
index 00000000..530fdf44
--- /dev/null
+++ b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
@@ -0,0 +1,50 @@
+# The deployment's servers: what each one is called, where it is, what it belongs
+# to, and how big its devnet is. Copy to devnet.inventory (gitignored) and fill in.
+#
+# This is the operator's inventory, in the same sense devnet.env's values are: the
+# authority on a RUNNING devnet is always its own genesis/config.yaml
+# (ATTESTATION_COMMITTEE_COUNT), and the scripts still take NODES/SUBNETS as
+# positional args. Recorded here so a restart doesn't have to guess, and so
+# "which hosts are in devnet-c" has one answer instead of living in someone's head.
+#
+# Read it with inventory.sh rather than by hand:
+#
+#   inventory.sh --tag devnet-ab --field ip        # ips, one per line, for a loop
+#   inventory.sh --tag devnet-ab --tag aggregator  # AND: both tags must be present
+#   inventory.sh --tag validator                   # derived, see TAGS below
+#   SERVERS=$(inventory.sh --field name | tr '\n' ' ')
+#
+# FORMAT — whitespace-separated columns, aligned however you like. Blank lines and
+# lines whose first non-space character is '#' are ignored. Five columns:
+#
+#   name      ssh target / display name. Need not equal the machine's hostname,
+#             and often doesn't after a host is repurposed -- inventory.sh never
+#             resolves it, so a stale name here misroutes every ssh that uses it.
+#   ip        address. The one field worth re-deriving from the fleet rather than
+#             trusting after a rebuild.
+#   tags      COMMA-separated, NO spaces (a space would start the next column).
+#   nodes     NODES on this server, or '-' if it runs no devnet.
+#   subnets   ATTESTATION_COMMITTEE_COUNT, or '-'.
+#
+# TAGS — free-form, with two conventions inventory.sh knows about:
+#
+#   devnet-*    names the chain this server's nodes belong to. Two servers sharing
+#               one tag means they share one genesis, i.e. the split-chain model
+#               (nodes 0..N on one, N+1.. on the other) -- NOT two devnets that
+#               happen to be named alike. Also the `network` Prometheus label.
+#   aggregator  this server runs the aggregator duty. A whole-server role, not a
+#               per-node one.
+#
+#   `validator` is DERIVED, never written here: it means "has a devnet-* tag and
+#   is not tagged aggregator". Writing it as a literal tag would let it disagree
+#   with the aggregator column, so inventory.sh computes it instead.
+#
+# The example below shows both supported topologies at once: host-a + host-b share
+# the chain `devnet-ab` (64 nodes total, 32 each), host-c runs its own independent
+# `devnet-c`, and metrics-1 carries the Grafana/Prometheus/Loki stack and no nodes.
+
+# name        ip              tags                    nodes  subnets
+host-a        10.0.0.1        devnet-ab,aggregator    32     8
+host-b        10.0.0.2        devnet-ab               32     8
+host-c        10.0.0.3        devnet-c                16     2
+metrics-1     10.0.0.4        tooling                 -      -
diff --git a/.claude/skills/multi-server-devnet/scripts/inventory.sh b/.claude/skills/multi-server-devnet/scripts/inventory.sh
new file mode 100755
index 00000000..1472dab7
--- /dev/null
+++ b/.claude/skills/multi-server-devnet/scripts/inventory.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+# Select servers from the deployment's inventory by tag.
+#
+#   inventory.sh [--tag T]... [--not-tag T]... [--field F] [--file P] [--count]
+#
+# Lookup order, first hit wins: --file, $DEVNET_INVENTORY, ./devnet.inventory,
+# <scripts dir>/devnet.inventory. Copy devnet.inventory.example to
+# devnet.inventory (gitignored) and fill it in; that file documents the format.
+#
+#   --tag T       keep rows carrying T. Repeatable, and a single argument may hold
+#                 several whitespace-separated tags. All of them must match (AND).
+#   --not-tag T   drop rows carrying T. Same repeat/multi-value rules.
+#   --field F     name | ip | tags | nodes | subnets | all. Default prints name +
+#                 ip aligned; `all` prints every column aligned.
+#   --count       print how many rows matched, nothing else.
+#
+# `validator` is a DERIVED tag: "has a devnet-* tag and is not tagged aggregator".
+# It is computed, not read, so it can never disagree with the aggregator tag.
+# `--tag validator` is exactly `--tag <that devnet> --not-tag aggregator`.
+#
+# An unknown tag is an ERROR, not an empty result. A typo ('devnet5' for
+# 'devnet-5') that quietly returns no hosts turns `for h in $(inventory.sh ...)`
+# into a loop that does nothing and reports success, which is the same class of
+# failure devnet-env.sh guards against: acting on the wrong deployment, silently.
+# A tag that IS known but matches nothing is likewise exit 1 -- a real fleet has
+# no empty groups, so an empty match means the inventory is stale.
+set -u
+
+usage() { sed -n '2,26p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; }
+
+file=""; field="default"; count=0
+tags=(); nottags=()
+
+while [ $# -gt 0 ]; do
+  case $1 in
+    --tag)     [ $# -ge 2 ] || { echo "$1 needs a value" >&2; exit 2; }
+               # Split on whitespace so --tag 'devnet-5 aggregator' is two tags.
+               for t in $2; do tags+=("$t"); done; shift 2 ;;
+    --not-tag) [ $# -ge 2 ] || { echo "$1 needs a value" >&2; exit 2; }
+               for t in $2; do nottags+=("$t"); done; shift 2 ;;
+    --field)   [ $# -ge 2 ] || { echo "$1 needs a value" >&2; exit 2; }
+               field=$2; shift 2 ;;
+    --file)    [ $# -ge 2 ] || { echo "$1 needs a value" >&2; exit 2; }
+               file=$2; shift 2 ;;
+    --count)   count=1; shift ;;
+    -h|--help) usage 0 ;;
+    *)         echo "unknown argument: $1" >&2; usage 2 ;;
+  esac
+done
+
+case $field in
+  name|ip|tags|nodes|subnets|all|default) ;;
+  *) echo "--field $field: expected name|ip|tags|nodes|subnets|all" >&2; exit 2 ;;
+esac
+
+# An explicit path that doesn't exist is a typo, not a reason to fall back to some
+# other inventory and act on the wrong fleet -- same rule devnet-env.sh applies.
+if [ -n "$file" ]; then
+  [ -f "$file" ] || { echo "--file $file does not exist" >&2; exit 2; }
+elif [ -n "${DEVNET_INVENTORY:-}" ]; then
+  file=$DEVNET_INVENTORY
+  [ -f "$file" ] || { echo "DEVNET_INVENTORY=$file does not exist" >&2; exit 2; }
+else
+  dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+  for candidate in "./devnet.inventory" "$dir/devnet.inventory"; do
+    [ -f "$candidate" ] && { file=$candidate; break; }
+  done
+  [ -n "$file" ] || {
+    echo "no inventory file found (tried \$DEVNET_INVENTORY, ./devnet.inventory, $dir/devnet.inventory)" >&2
+    echo "copy $dir/devnet.inventory.example to $dir/devnet.inventory and fill it in" >&2
+    exit 2
+  }
+fi
+
+awk -v want="${tags[*]-}" -v nowant="${nottags[*]-}" \
+    -v field="$field" -v docount="$count" -v src="$file" '
+  # Tags are matched against ",a,b," so "agg" never matches "aggregator".
+  function hastag(t, x) {
+    if (x == "validator")
+      return (index(t, ",devnet-") > 0 && index(t, ",aggregator,") == 0)
+    return index(t, "," x ",") > 0
+  }
+  BEGIN { nw = split(want, W, " "); nn = split(nowant, NW, " ") }
+
+  { sub(/^[[:space:]]+/, "") }
+  /^#/ || /^$/ { next }
+
+  {
+    if (NF < 3) {
+      # Named rather than skipped in silence: a row the parser drops is a host
+      # that vanishes from every tag it belonged to.
+      printf "%s:%d: ignoring '\''%s'\'', need at least name/ip/tags\n", src, FNR, $0 > "/dev/stderr"
+      next
+    }
+    name = $1; ip = $2; tags = $3
+    nodes   = (NF >= 4 ? $4 : "-")
+    subnets = (NF >= 5 ? $5 : "-")
+    t = "," tags ","
+
+    # Every literal tag in the file, so a typo can be told from a real absence.
+    n = split(tags, TT, ",")
+    for (i = 1; i <= n; i++) if (TT[i] != "") seen[TT[i]] = 1
+
+    for (i = 1; i <= nw; i++) if (!hastag(t, W[i])) next
+    for (i = 1; i <= nn; i++) if ( hastag(t, NW[i])) next
+
+    m++
+    N[m] = name; I[m] = ip; T[m] = tags; O[m] = nodes; S[m] = subnets
+    if (length(name)  > wN) wN = length(name)
+    if (length(ip)    > wI) wI = length(ip)
+    if (length(tags)  > wT) wT = length(tags)
+    if (length(nodes) > wO) wO = length(nodes)
+  }
+
+  END {
+    bad = 0
+    for (i = 1; i <= nw; i++)
+      if (W[i] != "validator" && !(W[i] in seen)) {
+        printf "no such tag in %s: %s\n", src, W[i] > "/dev/stderr"; bad = 1
+      }
+    for (i = 1; i <= nn; i++)
+      if (NW[i] != "validator" && !(NW[i] in seen)) {
+        printf "no such tag in %s: %s\n", src, NW[i] > "/dev/stderr"; bad = 1
+      }
+    if (bad) {
+      printf "known tags: " > "/dev/stderr"
+      for (k in seen) printf "%s ", k > "/dev/stderr"
+      printf "(+ derived: validator)\n" > "/dev/stderr"
+      exit 2
+    }
+
+    if (docount) { print m + 0; exit (m ? 0 : 1) }
+
+    for (i = 1; i <= m; i++) {
+      if      (field == "name")    print N[i]
+      else if (field == "ip")      print I[i]
+      else if (field == "tags")    print T[i]
+      else if (field == "nodes")   print O[i]
+      else if (field == "subnets") print S[i]
+      else if (field == "all")
+        printf "%-*s  %-*s  %-*s  %-*s  %s\n", wN, N[i], wI, I[i], wT, T[i], wO, O[i], S[i]
+      else
+        printf "%-*s  %s\n", wN, N[i], I[i]
+    }
+
+    if (m == 0) {
+      printf "no host in %s matches%s%s\n", src, \
+        (nw ? " tags: " want : ""), (nn ? " not: " nowant : "") > "/dev/stderr"
+      exit 1
+    }
+  }
+' "$file"
diff --git a/.gitignore b/.gitignore
index b3f89904..d8ed6cf6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,6 @@ finality-alerts.yaml
 
 # Filled-in copy of the multi-server-devnet deployment env (names hosts/paths)
 devnet.env
+
+# Filled-in copy of the multi-server-devnet server inventory (names hosts/ips)
+devnet.inventory

From 245ea8dde6595a44b266c2ca1a71363e87709a45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Wed, 19 Aug 2026 15:54:00 -0300
Subject: [PATCH 2/5] fix(tooling): stop inventory.sh from answering wrongly on
 bad input

Six ways the selector could hand a caller the wrong host list, all of them
silent, which is the failure mode the script exists to prevent:

- `--tag ""` matched nothing, so the filter was dropped and the ENTIRE fleet
  came back at exit 0. An unset `$DEVNET` in `--tag "$DEVNET"` would act on
  hosts on other chains, the mirror image of the typo case already guarded.
- The unknown-argument path printed help to stdout, and the documented idiom
  captures stdout: a mistyped flag fed 1.4 KB of usage text into a host loop.
- `--not-tag T` hard-failed when no host carried T, contradicting the script's
  own `--tag validator` == `--tag <devnet> --not-tag aggregator` equivalence.
  An exclusion matching nothing is well-defined; only `--tag` is checked now.
- Nothing validated nodes/subnets, so one space in the tags column shifted every
  later field: a role landed in `nodes`, `--tag validator` returned the
  aggregator, and a word reached a caller expecting a count. The file is now
  refused with the row named, rather than answered from.
- Tag arguments were glob-expanded against the cwd, so `--tag 'devnet-*'` could
  silently resolve to a different group than typed.
- `usage()` sliced help from a hardcoded line range; it now ends at the first
  non-comment line, so editing the header cannot truncate `--help`.

The documented recipes were also unrunnable as written: `inventory.sh` is not on
PATH, and the `| tr` pipeline discarded the exit code the script sets so
carefully, turning any error into an empty-but-successful host list.
---
 .claude/skills/multi-server-devnet/SKILL.md   | 14 +++--
 .../scripts/devnet.inventory.example          | 12 ++--
 .../multi-server-devnet/scripts/inventory.sh  | 63 ++++++++++++++++---
 3 files changed, 68 insertions(+), 21 deletions(-)

diff --git a/.claude/skills/multi-server-devnet/SKILL.md b/.claude/skills/multi-server-devnet/SKILL.md
index 3d02d572..4a5b17d1 100644
--- a/.claude/skills/multi-server-devnet/SKILL.md
+++ b/.claude/skills/multi-server-devnet/SKILL.md
@@ -58,18 +58,20 @@ file. It is the one place a deployment's urls and Grafana ids live.
 than reading it by hand:
 
 ```bash
-inventory.sh --tag devnet-ab --field ip        # ips, one per line, for a loop
-inventory.sh --tag devnet-ab --tag aggregator  # AND across tags
-inventory.sh --tag validator                   # derived, see below
-SERVERS=$(inventory.sh --field name | tr '\n' ' ')
+bash scripts/inventory.sh --tag devnet-ab --field ip        # ips, for a loop
+bash scripts/inventory.sh --tag devnet-ab --tag aggregator  # AND across tags
+bash scripts/inventory.sh --tag validator                   # derived, see below
+SERVERS=$(bash scripts/inventory.sh --field name) || exit   # exit 2 = typo'd tag
+SERVERS=${SERVERS//$'\n'/ }                                 # newlines -> spaces
 ```
 
 Tag conventions: a `devnet-*` tag names the chain a host's nodes belong to (two
 hosts sharing one means the split-chain model, *not* two like-named devnets), and
 `aggregator` is a whole-server role. `validator` is **derived** — "has a `devnet-*`
 tag and is not tagged `aggregator`" — so it can never disagree with the aggregator
-tag. An unknown tag exits 2 with the known-tag list rather than returning nothing,
-because a typo that yields an empty loop reports success while doing nothing.
+tag. An unknown or empty `--tag` exits 2 rather than returning nothing (or, worse,
+everything): a typo that yields an empty loop reports success while doing nothing,
+and `--tag "$UNSET"` would act on hosts the caller never named.
 
 `NODES`/`SUBNETS` there are still the operator's inventory: the scripts take them
 as positional args, and the authority on a running devnet is always its own
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
index 530fdf44..b046d957 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
+++ b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
@@ -9,10 +9,11 @@
 #
 # Read it with inventory.sh rather than by hand:
 #
-#   inventory.sh --tag devnet-ab --field ip        # ips, one per line, for a loop
-#   inventory.sh --tag devnet-ab --tag aggregator  # AND: both tags must be present
-#   inventory.sh --tag validator                   # derived, see TAGS below
-#   SERVERS=$(inventory.sh --field name | tr '\n' ' ')
+#   bash scripts/inventory.sh --tag devnet-ab --field ip        # ips, for a loop
+#   bash scripts/inventory.sh --tag devnet-ab --tag aggregator  # AND: both needed
+#   bash scripts/inventory.sh --tag validator                   # derived, see TAGS
+#   SERVERS=$(bash scripts/inventory.sh --field name) || exit   # exit 2 = typo'd tag
+#   SERVERS=${SERVERS//$'\n'/ }                                 # newlines -> spaces
 #
 # FORMAT — whitespace-separated columns, aligned however you like. Blank lines and
 # lines whose first non-space character is '#' are ignored. Five columns:
@@ -22,7 +23,8 @@
 #             resolves it, so a stale name here misroutes every ssh that uses it.
 #   ip        address. The one field worth re-deriving from the fleet rather than
 #             trusting after a rebuild.
-#   tags      COMMA-separated, NO spaces (a space would start the next column).
+#   tags      COMMA-separated, NO spaces (a space would start the next column;
+#             inventory.sh rejects the file rather than answer from shifted rows).
 #   nodes     NODES on this server, or '-' if it runs no devnet.
 #   subnets   ATTESTATION_COMMITTEE_COUNT, or '-'.
 #
diff --git a/.claude/skills/multi-server-devnet/scripts/inventory.sh b/.claude/skills/multi-server-devnet/scripts/inventory.sh
index 1472dab7..0ec20fc4 100755
--- a/.claude/skills/multi-server-devnet/scripts/inventory.sh
+++ b/.claude/skills/multi-server-devnet/scripts/inventory.sh
@@ -18,15 +18,44 @@
 # It is computed, not read, so it can never disagree with the aggregator tag.
 # `--tag validator` is exactly `--tag <that devnet> --not-tag aggregator`.
 #
-# An unknown tag is an ERROR, not an empty result. A typo ('devnet5' for
+# An unknown `--tag` is an ERROR, not an empty result. A typo ('devnet5' for
 # 'devnet-5') that quietly returns no hosts turns `for h in $(inventory.sh ...)`
 # into a loop that does nothing and reports success, which is the same class of
 # failure devnet-env.sh guards against: acting on the wrong deployment, silently.
-# A tag that IS known but matches nothing is likewise exit 1 -- a real fleet has
-# no empty groups, so an empty match means the inventory is stale.
+# An EMPTY --tag value is an error for the mirror-image reason: an unset variable
+# in `--tag "$DEVNET"` would drop the filter and return the whole fleet, exit 0,
+# including hosts on other chains. A tag that IS known but matches nothing is
+# exit 1 -- a real fleet has no empty groups, so that means a stale inventory.
+#
+# `--not-tag` is deliberately NOT checked against the file: an exclusion that
+# matches nothing is well-defined, and demanding the tag exist would break the
+# documented `--tag <devnet> --not-tag aggregator` on any fleet whose aggregator
+# role is currently unassigned.
 set -u
 
-usage() { sed -n '2,26p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; }
+# Sliced from the header block above, ending at the first non-comment line, so
+# editing the comment can't silently truncate --help.
+help_text() { awk 'NR == 1 { next } !/^#/ { exit } { sub(/^# ?/, ""); print }' "$0"; }
+
+# Help goes to stdout for -h, but to stderr on the error path: callers capture
+# this script's stdout ($(inventory.sh --field name)), so help text printed there
+# becomes a list of "hosts" to ssh into.
+usage() {
+  code=${1:-0}
+  if [ "$code" = 0 ]; then help_text; else help_text >&2; fi
+  exit "$code"
+}
+
+# Whitespace-split a tag argument into SPLIT with globbing OFF: a tag is data, and
+# `--tag 'devnet-*'` must not expand against whatever happens to sit in the cwd.
+split_tags() {
+  case $2 in
+    *[![:space:]]*) ;;
+    *) echo "$1 needs a non-empty value (an empty tag would match every host)" >&2
+       exit 2 ;;
+  esac
+  set -f; SPLIT=($2); set +f
+}
 
 file=""; field="default"; count=0
 tags=(); nottags=()
@@ -35,9 +64,9 @@ while [ $# -gt 0 ]; do
   case $1 in
     --tag)     [ $# -ge 2 ] || { echo "$1 needs a value" >&2; exit 2; }
                # Split on whitespace so --tag 'devnet-5 aggregator' is two tags.
-               for t in $2; do tags+=("$t"); done; shift 2 ;;
+               split_tags "$1" "$2"; tags+=("${SPLIT[@]}"); shift 2 ;;
     --not-tag) [ $# -ge 2 ] || { echo "$1 needs a value" >&2; exit 2; }
-               for t in $2; do nottags+=("$t"); done; shift 2 ;;
+               split_tags "$1" "$2"; nottags+=("${SPLIT[@]}"); shift 2 ;;
     --field)   [ $# -ge 2 ] || { echo "$1 needs a value" >&2; exit 2; }
                field=$2; shift 2 ;;
     --file)    [ $# -ge 2 ] || { echo "$1 needs a value" >&2; exit 2; }
@@ -95,6 +124,17 @@ awk -v want="${tags[*]-}" -v nowant="${nottags[*]-}" \
     name = $1; ip = $2; tags = $3
     nodes   = (NF >= 4 ? $4 : "-")
     subnets = (NF >= 5 ? $5 : "-")
+
+    # A space inside the tags column (or a trailing comment on a data row) shifts
+    # every later field: $3 loses tags, and a ROLE lands in nodes. Unlike a short
+    # row, which only drops one host, that silently reassigns roles and hands a
+    # word to a caller expecting a count -- so refuse the whole file instead.
+    if (nodes !~ /^([0-9]+|-)$/ || subnets !~ /^([0-9]+|-)$/) {
+      printf "%s:%d: nodes/subnets must be a number or '\''-'\'', got '\''%s'\'' '\''%s'\''", \
+        src, FNR, nodes, subnets > "/dev/stderr"
+      printf " -- a space in the tags column shifts the columns right\n" > "/dev/stderr"
+      malformed = 1; next
+    }
     t = "," tags ","
 
     # Every literal tag in the file, so a typo can be told from a real absence.
@@ -113,15 +153,18 @@ awk -v want="${tags[*]-}" -v nowant="${nottags[*]-}" \
   }
 
   END {
+    # Answering from a file whose columns do not line up would mean answering
+    # about tags that were never written; the row numbers are already on stderr.
+    if (malformed) {
+      printf "%s: refusing to answer from a file that does not parse\n", src > "/dev/stderr"
+      exit 2
+    }
+
     bad = 0
     for (i = 1; i <= nw; i++)
       if (W[i] != "validator" && !(W[i] in seen)) {
         printf "no such tag in %s: %s\n", src, W[i] > "/dev/stderr"; bad = 1
       }
-    for (i = 1; i <= nn; i++)
-      if (NW[i] != "validator" && !(NW[i] in seen)) {
-        printf "no such tag in %s: %s\n", src, NW[i] > "/dev/stderr"; bad = 1
-      }
     if (bad) {
       printf "known tags: " > "/dev/stderr"
       for (k in seen) printf "%s ", k > "/dev/stderr"

From e80e24e50d3ea612d8366e69292ff1a02fa34a91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Wed, 19 Aug 2026 16:15:10 -0300
Subject: [PATCH 3/5] fix(tooling): make inventory.sh's derived tag and lookup
 single-sourced

Follow-up to the input-validation pass, closing the rest of the review:

- `validator` was special-cased in three places (a branch in hastag, plus two
  exemptions in the unknown-tag check) with `devnet-` hardcoded inside the
  branch, so a second derived tag meant touching all three and a fleet that
  names chains differently got a silent empty answer. It is now one table entry
  (prefix + disqualifying tag), and DEVNET_TAG_PREFIX names the prefix.
- A literal `validator` in the file was silently ignored, which is exactly the
  disagreement the derivation exists to rule out: the row said one thing and the
  computation another. It now names the line and refuses.
- `--tag validator` against a file carrying no chain tag at all said only "no
  host matches", when the actionable fact is that the tag can never match there.
- The file-lookup ladder was a copy of devnet-env.sh's, comment included, and had
  already drifted (return-0 vs exit-2 on nothing found). Both now call one
  `devnet_find_file`, so the two config files of a deployment cannot end up with
  two ideas of where they live.
- The known-tag list, whose whole job is to be read next to the operator's typo,
  printed in awk's hash order.

Docs: devnet.inventory is the record of which hosts exist and `SERVERS` is a
working set filled from a query against it, rather than two hand-kept host lists
with no stated precedence. Also notes that the `for h in $SERVERS` workflows rely
on bash word-splitting, which zsh does not do, and drops the em-dashes this
branch added.
---
 .claude/skills/multi-server-devnet/SKILL.md   |  18 +--
 .../multi-server-devnet/scripts/devnet-env.sh |  40 ++++---
 .../scripts/devnet.env.example                |   3 +
 .../scripts/devnet.inventory.example          |   4 +-
 .../multi-server-devnet/scripts/inventory.sh  | 105 ++++++++++++++----
 5 files changed, 126 insertions(+), 44 deletions(-)

diff --git a/.claude/skills/multi-server-devnet/SKILL.md b/.claude/skills/multi-server-devnet/SKILL.md
index 4a5b17d1..c09987c9 100644
--- a/.claude/skills/multi-server-devnet/SKILL.md
+++ b/.claude/skills/multi-server-devnet/SKILL.md
@@ -40,8 +40,10 @@ B owns `k+1..N-1`). Everything below still applies with three changes, and
 
 Nothing about the servers is hardcoded. Establish these from the operator:
 
-- `SERVERS` — the SSH targets (any count, any names), one devnet each.
-- `SSH_USER` — login user. `docker` is invoked with `sudo`.
+- `SERVERS`: the SSH targets for the operation at hand (any count, any names),
+  one devnet each. A working set taken FROM the inventory below, not a second
+  list of hosts to keep in step with it.
+- `SSH_USER`: login user. `docker` is invoked with `sudo`.
 - Per-devnet `NODES` (validators on that server) and `SUBNETS`
   (`ATTESTATION_COMMITTEE_COUNT`); these can differ between servers.
 - A central host for Grafana + the federating Prometheus (often one of the
@@ -53,7 +55,7 @@ gitignored) instead of retyping them: the operator-side scripts source it via
 file. It is the one place a deployment's urls and Grafana ids live.
 
 **The servers themselves live in `scripts/devnet.inventory`** (copy
-`scripts/devnet.inventory.example`; gitignored) — name, ip, tags, and per-devnet
+`scripts/devnet.inventory.example`; gitignored): name, ip, tags, and per-devnet
 `NODES`/`SUBNETS`, one row per host. Query it with `scripts/inventory.sh` rather
 than reading it by hand:
 
@@ -67,9 +69,10 @@ SERVERS=${SERVERS//$'\n'/ }                                 # newlines -> spaces
 
 Tag conventions: a `devnet-*` tag names the chain a host's nodes belong to (two
 hosts sharing one means the split-chain model, *not* two like-named devnets), and
-`aggregator` is a whole-server role. `validator` is **derived** — "has a `devnet-*`
-tag and is not tagged `aggregator`" — so it can never disagree with the aggregator
-tag. An unknown or empty `--tag` exits 2 rather than returning nothing (or, worse,
+`aggregator` is a whole-server role. `validator` is **derived**: "has a `devnet-*`
+tag and is not tagged `aggregator`", so it can never disagree with the aggregator
+tag, and writing it literally in the file is an error rather than a second opinion.
+An unknown or empty `--tag` exits 2 rather than returning nothing (or, worse,
 everything): a typo that yields an empty loop reports success while doing nothing,
 and `--tag "$UNSET"` would act on hosts the caller never named.
 
@@ -151,7 +154,8 @@ redirect must run under sudo).
 ## Workflows
 
 Examples assume `SSH_USER` is set and you iterate over `SERVERS`. Per server you
-pass its own `NODES`/`SUBNETS`.
+pass its own `NODES`/`SUBNETS`. `for h in $SERVERS` splits under bash; zsh does not
+split parameter expansions, so there write `for h in $(echo $SERVERS)`.
 
 ### Pull the latest images on all servers
 ```bash
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet-env.sh b/.claude/skills/multi-server-devnet/scripts/devnet-env.sh
index 2ba9c8fc..d08e8651 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet-env.sh
+++ b/.claude/skills/multi-server-devnet/scripts/devnet-env.sh
@@ -4,6 +4,9 @@
 #
 #   . "$SCRIPT_DIR/devnet-env.sh"; devnet_load_env
 #
+# Also provides devnet_find_file, the file-lookup ladder both this and
+# inventory.sh use.
+#
 # Lookup order, first hit wins: $DEVNET_ENV, ./devnet.env, <scripts dir>/devnet.env.
 # Copy devnet.env.example -> devnet.env and fill it in (devnet.env is gitignored,
 # since it names your hosts and may point at a webhook file).
@@ -17,21 +20,32 @@
 # fine. A line that looks like an assignment but whose name isn't usable is
 # reported on stderr rather than dropped in silence, because a config line that
 # goes unread is how you deploy against the wrong deployment.
-devnet_load_env() {
-  local dir file line key val
+
+# Resolve one of the deployment's config files. First hit wins: the path in $2 (an
+# env var NAME), ./<name>, <scripts dir>/<name>. Prints the path on stdout.
+# Returns 1 when nothing exists, and 2 when the env var names a path that doesn't
+# -- an explicit path that isn't there is a typo, not a licence to fall back to
+# another deployment's file. Shared with inventory.sh so devnet.env and
+# devnet.inventory can't end up with two ideas of where they live.
+devnet_find_file() {
+  local name=$1 envvar=$2 dir explicit candidate
   dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-  file=""
-  if [ -n "${DEVNET_ENV:-}" ]; then
-    # An explicit path that doesn't exist is a typo, not a reason to silently fall
-    # back to some other devnet.env and deploy against the wrong deployment.
-    [ -f "$DEVNET_ENV" ] || { echo "DEVNET_ENV=$DEVNET_ENV does not exist" >&2; return 1; }
-    file=$DEVNET_ENV
-  else
-    for candidate in "./devnet.env" "$dir/devnet.env"; do
-      [ -f "$candidate" ] && { file=$candidate; break; }
-    done
+  explicit=${!envvar:-}
+  if [ -n "$explicit" ]; then
+    [ -f "$explicit" ] || { echo "$envvar=$explicit does not exist" >&2; return 2; }
+    printf '%s\n' "$explicit"; return 0
   fi
-  [ -n "$file" ] || return 0
+  for candidate in "./$name" "$dir/$name"; do
+    [ -f "$candidate" ] && { printf '%s\n' "$candidate"; return 0; }
+  done
+  return 1
+}
+
+devnet_load_env() {
+  local file line key val rc
+  file=$(devnet_find_file devnet.env DEVNET_ENV); rc=$?
+  [ "$rc" -eq 2 ] && return 1        # DEVNET_ENV names a missing file: reported
+  [ "$rc" -eq 0 ] || return 0        # no env file at all is not an error
 
   while IFS= read -r line || [ -n "$line" ]; do
     line=${line#"${line%%[![:space:]]*}"}          # ltrim, so an indented line is read
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet.env.example b/.claude/skills/multi-server-devnet/scripts/devnet.env.example
index 5e1ff70e..8a55b5bb 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet.env.example
+++ b/.claude/skills/multi-server-devnet/scripts/devnet.env.example
@@ -5,6 +5,9 @@
 # your shell overrides the value here.
 
 # --- fleet (one independent devnet per server) -------------------------------
+# devnet.inventory is the record of which hosts exist; SERVERS is just the working
+# set for these scripts, so fill it FROM a query rather than maintaining a second
+# list: SERVERS=$(bash scripts/inventory.sh --tag devnet-ab --field name) || exit
 SERVERS="host-a host-b"                 # ssh targets, any count/names
 SSH_USER=user                           # login user; docker is invoked with sudo
 
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
index b046d957..3517540d 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
+++ b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
@@ -15,7 +15,7 @@
 #   SERVERS=$(bash scripts/inventory.sh --field name) || exit   # exit 2 = typo'd tag
 #   SERVERS=${SERVERS//$'\n'/ }                                 # newlines -> spaces
 #
-# FORMAT — whitespace-separated columns, aligned however you like. Blank lines and
+# FORMAT: whitespace-separated columns, aligned however you like. Blank lines and
 # lines whose first non-space character is '#' are ignored. Five columns:
 #
 #   name      ssh target / display name. Need not equal the machine's hostname,
@@ -28,7 +28,7 @@
 #   nodes     NODES on this server, or '-' if it runs no devnet.
 #   subnets   ATTESTATION_COMMITTEE_COUNT, or '-'.
 #
-# TAGS — free-form, with two conventions inventory.sh knows about:
+# TAGS: free-form, with two conventions inventory.sh knows about:
 #
 #   devnet-*    names the chain this server's nodes belong to. Two servers sharing
 #               one tag means they share one genesis, i.e. the split-chain model
diff --git a/.claude/skills/multi-server-devnet/scripts/inventory.sh b/.claude/skills/multi-server-devnet/scripts/inventory.sh
index 0ec20fc4..e1807290 100755
--- a/.claude/skills/multi-server-devnet/scripts/inventory.sh
+++ b/.claude/skills/multi-server-devnet/scripts/inventory.sh
@@ -15,8 +15,10 @@
 #   --count       print how many rows matched, nothing else.
 #
 # `validator` is a DERIVED tag: "has a devnet-* tag and is not tagged aggregator".
-# It is computed, not read, so it can never disagree with the aggregator tag.
-# `--tag validator` is exactly `--tag <that devnet> --not-tag aggregator`.
+# It is computed from the row, never read from it, so it can never disagree with
+# the aggregator tag; writing it literally in the file is an error rather than a
+# second opinion. `--tag validator` is `--tag <that devnet> --not-tag aggregator`.
+# Set DEVNET_TAG_PREFIX if a fleet names its chains something other than devnet-*.
 #
 # An unknown `--tag` is an ERROR, not an empty result. A typo ('devnet5' for
 # 'devnet-5') that quietly returns no hosts turns `for h in $(inventory.sh ...)`
@@ -82,34 +84,60 @@ case $field in
   *) echo "--field $field: expected name|ip|tags|nodes|subnets|all" >&2; exit 2 ;;
 esac
 
-# An explicit path that doesn't exist is a typo, not a reason to fall back to some
-# other inventory and act on the wrong fleet -- same rule devnet-env.sh applies.
 if [ -n "$file" ]; then
+  # An explicit path that doesn't exist is a typo, not a reason to fall back to
+  # some other inventory and act on the wrong fleet.
   [ -f "$file" ] || { echo "--file $file does not exist" >&2; exit 2; }
-elif [ -n "${DEVNET_INVENTORY:-}" ]; then
-  file=$DEVNET_INVENTORY
-  [ -f "$file" ] || { echo "DEVNET_INVENTORY=$file does not exist" >&2; exit 2; }
 else
+  # Same ladder, from the same helper, that finds devnet.env: two config files of
+  # one deployment should not have two ideas of where they live. Sourcing only
+  # defines functions.
   dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-  for candidate in "./devnet.inventory" "$dir/devnet.inventory"; do
-    [ -f "$candidate" ] && { file=$candidate; break; }
-  done
-  [ -n "$file" ] || {
+  . "$dir/devnet-env.sh"
+  file=$(devnet_find_file devnet.inventory DEVNET_INVENTORY); rc=$?
+  # rc 2 means DEVNET_INVENTORY names a file that isn't there, already reported.
+  [ "$rc" -eq 2 ] && exit 2
+  [ "$rc" -eq 0 ] || {
     echo "no inventory file found (tried \$DEVNET_INVENTORY, ./devnet.inventory, $dir/devnet.inventory)" >&2
     echo "copy $dir/devnet.inventory.example to $dir/devnet.inventory and fill it in" >&2
     exit 2
   }
 fi
 
-awk -v want="${tags[*]-}" -v nowant="${nottags[*]-}" \
+# Only the derived `validator` tag depends on how chains are named, so a fleet
+# that doesn't use devnet-* can say so instead of getting an empty answer.
+chain_prefix=${DEVNET_TAG_PREFIX:-devnet-}
+
+awk -v want="${tags[*]-}" -v nowant="${nottags[*]-}" -v chain="$chain_prefix" \
     -v field="$field" -v docount="$count" -v src="$file" '
-  # Tags are matched against ",a,b," so "agg" never matches "aggregator".
+  # Tags are matched against ",a,b," so "agg" never matches "aggregator". A
+  # DERIVED tag is computed from the row instead: it carries some chain tag and
+  # is not disqualified by a role tag.
   function hastag(t, x) {
-    if (x == "validator")
-      return (index(t, ",devnet-") > 0 && index(t, ",aggregator,") == 0)
+    if (x in DPREFIX)
+      return (index(t, "," DPREFIX[x]) > 0 && !hastag(t, DEXCLUDE[x]))
     return index(t, "," x ",") > 0
   }
-  BEGIN { nw = split(want, W, " "); nn = split(nowant, NW, " ") }
+
+  # `for (k in a)` walks a hash order, and these lists get read by a human next to
+  # their own typo. Insertion sort: the tag list is a few dozen entries at most.
+  function sortkeys(a, out,   i, j, n, v) {
+    n = 0
+    for (i in a) out[++n] = i
+    for (i = 2; i <= n; i++) {
+      v = out[i]
+      for (j = i - 1; j >= 1 && out[j] > v; j--) out[j+1] = out[j]
+      out[j+1] = v
+    }
+    return n
+  }
+
+  BEGIN {
+    nw = split(want, W, " "); nn = split(nowant, NW, " ")
+    # One entry per derived tag: the tag PREFIX a row must carry, and the tag that
+    # disqualifies it. Adding a derived tag is a line here, not a new branch.
+    DPREFIX["validator"] = chain; DEXCLUDE["validator"] = "aggregator"
+  }
 
   { sub(/^[[:space:]]+/, "") }
   /^#/ || /^$/ { next }
@@ -137,9 +165,14 @@ awk -v want="${tags[*]-}" -v nowant="${nottags[*]-}" \
     }
     t = "," tags ","
 
-    # Every literal tag in the file, so a typo can be told from a real absence.
+    # Every literal tag in the file, so a typo can be told from a real absence,
+    # and where it was first written, so both can be pointed at a line.
     n = split(tags, TT, ",")
-    for (i = 1; i <= n; i++) if (TT[i] != "") seen[TT[i]] = 1
+    for (i = 1; i <= n; i++) if (TT[i] != "") {
+      if (!(TT[i] in seen)) seenrow[TT[i]] = FNR
+      seen[TT[i]] = 1
+      if (substr(TT[i], 1, length(chain)) == chain) chainseen = 1
+    }
 
     for (i = 1; i <= nw; i++) if (!hastag(t, W[i])) next
     for (i = 1; i <= nn; i++) if ( hastag(t, NW[i])) next
@@ -161,14 +194,42 @@ awk -v want="${tags[*]-}" -v nowant="${nottags[*]-}" \
     }
 
     bad = 0
-    for (i = 1; i <= nw; i++)
-      if (W[i] != "validator" && !(W[i] in seen)) {
+
+    # A derived tag written as a literal is the exact disagreement the derivation
+    # exists to rule out: the row would claim one thing and the computation
+    # another. Say which line, rather than ignoring what the operator wrote.
+    nd = sortkeys(DPREFIX, D)
+    for (i = 1; i <= nd; i++)
+      if (D[i] in seen) {
+        printf "%s:%d: '\''%s'\'' is a DERIVED tag (%s* and not %s), remove it from the file\n", \
+          src, seenrow[D[i]], D[i], DPREFIX[D[i]], DEXCLUDE[D[i]] > "/dev/stderr"
+        bad = 1
+      }
+
+    for (i = 1; i <= nw; i++) {
+      if (W[i] in DPREFIX) {
+        # Derived, so never "unknown" -- but if no row carries a chain tag it can
+        # never match, and that is a stale inventory, not a query to answer.
+        if (!chainseen) {
+          printf "no %s* tag in %s, so %s matches nothing (set DEVNET_TAG_PREFIX?)\n", \
+            chain, src, W[i] > "/dev/stderr"
+          bad = 1
+        }
+        continue
+      }
+      if (!(W[i] in seen)) {
         printf "no such tag in %s: %s\n", src, W[i] > "/dev/stderr"; bad = 1
       }
+    }
+
     if (bad) {
       printf "known tags: " > "/dev/stderr"
-      for (k in seen) printf "%s ", k > "/dev/stderr"
-      printf "(+ derived: validator)\n" > "/dev/stderr"
+      nk = sortkeys(seen, K)
+      # A derived name that leaked into the file is listed as derived, not twice.
+      for (i = 1; i <= nk; i++) if (!(K[i] in DPREFIX)) printf "%s ", K[i] > "/dev/stderr"
+      printf "(+ derived:" > "/dev/stderr"
+      for (i = 1; i <= nd; i++) printf " %s", D[i] > "/dev/stderr"
+      printf ")\n" > "/dev/stderr"
       exit 2
     }
 

From 64714520e206f3837745a5858278b7865a8cdabf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Wed, 19 Aug 2026 16:22:30 -0300
Subject: [PATCH 4/5] refactor(tooling): drop devnet.env's fleet section for
 the inventory

Nothing read `SERVERS`: no script did, and a shell variable can't survive from
one command to the next anyway, so as a stored config value it was only ever a
second host list to keep in step with devnet.inventory. The workflows now query
the inventory in the same command that loops over it, guarded with `|| exit` so a
typo'd tag aborts instead of leaving a loop that does nothing and reports
success, and `$(echo $hosts)` because zsh doesn't split parameter expansions.

`SSH_USER` stays: it isn't a host, the inventory has no column for it, and every
documented ssh needs it. Its section is named for what it holds now.
---
 .claude/skills/multi-server-devnet/SKILL.md   | 26 ++++++++++++-------
 .../references/operations.md                  |  6 ++---
 .../multi-server-devnet/scripts/devnet-env.sh |  6 ++---
 .../scripts/devnet.env.example                |  8 +++---
 .../scripts/devnet.inventory.example          |  4 +--
 5 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/.claude/skills/multi-server-devnet/SKILL.md b/.claude/skills/multi-server-devnet/SKILL.md
index c09987c9..8e02dd28 100644
--- a/.claude/skills/multi-server-devnet/SKILL.md
+++ b/.claude/skills/multi-server-devnet/SKILL.md
@@ -40,16 +40,15 @@ B owns `k+1..N-1`). Everything below still applies with three changes, and
 
 Nothing about the servers is hardcoded. Establish these from the operator:
 
-- `SERVERS`: the SSH targets for the operation at hand (any count, any names),
-  one devnet each. A working set taken FROM the inventory below, not a second
-  list of hosts to keep in step with it.
+- The hosts: one row each in `scripts/devnet.inventory` (below), queried with
+  `scripts/inventory.sh`. There is no second host list to hold in step with it.
 - `SSH_USER`: login user. `docker` is invoked with `sudo`.
 - Per-devnet `NODES` (validators on that server) and `SUBNETS`
   (`ATTESTATION_COMMITTEE_COUNT`); these can differ between servers.
 - A central host for Grafana + the federating Prometheus (often one of the
   servers); each server's per-host Prometheus remote-writes to it.
 
-Put those values in `scripts/devnet.env` (copy `scripts/devnet.env.example`;
+Put the non-host values in `scripts/devnet.env` (copy `scripts/devnet.env.example`;
 gitignored) instead of retyping them: the operator-side scripts source it via
 `scripts/devnet-env.sh`, and an env var exported in the shell still wins over the
 file. It is the one place a deployment's urls and Grafana ids live.
@@ -63,8 +62,8 @@ than reading it by hand:
 bash scripts/inventory.sh --tag devnet-ab --field ip        # ips, for a loop
 bash scripts/inventory.sh --tag devnet-ab --tag aggregator  # AND across tags
 bash scripts/inventory.sh --tag validator                   # derived, see below
-SERVERS=$(bash scripts/inventory.sh --field name) || exit   # exit 2 = typo'd tag
-SERVERS=${SERVERS//$'\n'/ }                                 # newlines -> spaces
+hosts=$(bash scripts/inventory.sh --tag devnet-ab --field name) || exit
+for h in $(echo $hosts); do ssh "$SSH_USER@$h" uptime; done  # splits in bash + zsh
 ```
 
 Tag conventions: a `devnet-*` tag names the chain a host's nodes belong to (two
@@ -153,13 +152,20 @@ redirect must run under sudo).
 
 ## Workflows
 
-Examples assume `SSH_USER` is set and you iterate over `SERVERS`. Per server you
-pass its own `NODES`/`SUBNETS`. `for h in $SERVERS` splits under bash; zsh does not
-split parameter expansions, so there write `for h in $(echo $SERVERS)`.
+Examples assume `SSH_USER` is set and `$h` is one host. Per server you pass its own
+`NODES`/`SUBNETS`. Keep the host query and the loop in ONE command, since a shell's
+variables don't outlive it, and `|| exit` so a typo'd tag can't leave a loop that
+does nothing and reports success:
+
+```bash
+hosts=$(bash scripts/inventory.sh --tag <devnet> --field name) || exit
+for h in $(echo $hosts); do ...; done   # $(echo ...) splits under bash and zsh
+```
 
 ### Pull the latest images on all servers
 ```bash
-for h in $SERVERS; do ssh "$SSH_USER@$h" 'sudo docker pull ghcr.io/lambdaclass/ethlambda:devnet5'; done
+hosts=$(bash scripts/inventory.sh --tag <devnet> --field name) || exit
+for h in $(echo $hosts); do ssh "$SSH_USER@$h" 'sudo docker pull ghcr.io/lambdaclass/ethlambda:devnet5'; done
 ```
 `devnet5` is the current devnet tag; it tracks the chain's leanVM/proof format,
 so it (and the other clients' tags in `references/clients.md`) move together on a
diff --git a/.claude/skills/multi-server-devnet/references/operations.md b/.claude/skills/multi-server-devnet/references/operations.md
index cefd3b14..5afada2f 100644
--- a/.claude/skills/multi-server-devnet/references/operations.md
+++ b/.claude/skills/multi-server-devnet/references/operations.md
@@ -6,9 +6,9 @@ SKILL.md has the workflows; this file has the depth and the failure modes.
 
 ## Topology model
 
-- Operator supplies SSH targets via `SERVERS` (any count, any names) and
-  `SSH_USER`; `docker` runs under `sudo`. No host count, address, or spec is
-  assumed.
+- SSH targets come from `scripts/devnet.inventory`, queried with
+  `scripts/inventory.sh` (any count, any names); the operator supplies `SSH_USER`
+  and `docker` runs under `sudo`. No host count, address, or spec is assumed.
 - **Each server runs its own complete devnet** — its own genesis, GENESIS_TIME,
   fork choice, and finality. No cross-server peering: every ENR is pinned to
   `127.0.0.1`, so discovery never leaves the host even though all devnets share
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet-env.sh b/.claude/skills/multi-server-devnet/scripts/devnet-env.sh
index d08e8651..dd496462 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet-env.sh
+++ b/.claude/skills/multi-server-devnet/scripts/devnet-env.sh
@@ -64,7 +64,7 @@ devnet_load_env() {
     esac
     # Quotes, when present, DELIMIT the value, so a '#' or a trailing space inside
     # them survives. Unquoted, a `# comment` tail is dropped: it used to end up
-    # inside the value, and for SERVERS that means sweep.sh ssh-ing to a host '#'.
+    # inside the value, and for METRICS_HOST that means ssh-ing to 'host # ssh'.
     case $val in
       \"*) val=${val#\"}; val=${val%%\"*} ;;
       \'*) val=${val#\'}; val=${val%%\'*} ;;
@@ -73,8 +73,8 @@ devnet_load_env() {
     esac
     # File provides defaults only: keep an already-set value. No eval anywhere --
     # `${!key}` is bash indirect expansion, and `export` gets ONE quoted
-    # name=value word, so a multi-word value (SERVERS="host-a host-b") arrives
-    # whole and nothing in it is ever reparsed as shell.
+    # name=value word, so a multi-word value (a quoted path with a space, say)
+    # arrives whole and nothing in it is ever reparsed as shell.
     [ -n "${!key:-}" ] || export "$key=$val"
   done < "$file"
   echo "loaded deployment env from $file" >&2
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet.env.example b/.claude/skills/multi-server-devnet/scripts/devnet.env.example
index 8a55b5bb..e22c0b63 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet.env.example
+++ b/.claude/skills/multi-server-devnet/scripts/devnet.env.example
@@ -4,11 +4,9 @@
 # Loaded by the operator-side scripts via devnet-env.sh; an env var already set in
 # your shell overrides the value here.
 
-# --- fleet (one independent devnet per server) -------------------------------
-# devnet.inventory is the record of which hosts exist; SERVERS is just the working
-# set for these scripts, so fill it FROM a query rather than maintaining a second
-# list: SERVERS=$(bash scripts/inventory.sh --tag devnet-ab --field name) || exit
-SERVERS="host-a host-b"                 # ssh targets, any count/names
+# --- ssh ---------------------------------------------------------------------
+# WHICH hosts exist is devnet.inventory's business, not this file's: query it with
+# scripts/inventory.sh instead of keeping a second host list here to hold in step.
 SSH_USER=user                           # login user; docker is invoked with sudo
 
 # --- central metrics / logs stack -------------------------------------------
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
index 3517540d..67291525 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
+++ b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
@@ -12,8 +12,8 @@
 #   bash scripts/inventory.sh --tag devnet-ab --field ip        # ips, for a loop
 #   bash scripts/inventory.sh --tag devnet-ab --tag aggregator  # AND: both needed
 #   bash scripts/inventory.sh --tag validator                   # derived, see TAGS
-#   SERVERS=$(bash scripts/inventory.sh --field name) || exit   # exit 2 = typo'd tag
-#   SERVERS=${SERVERS//$'\n'/ }                                 # newlines -> spaces
+#   hosts=$(bash scripts/inventory.sh --tag devnet-ab --field name) || exit
+#   for h in $(echo $hosts); do ssh "$SSH_USER@$h" uptime; done  # bash + zsh
 #
 # FORMAT: whitespace-separated columns, aligned however you like. Blank lines and
 # lines whose first non-space character is '#' are ignored. Five columns:

From 84e2fac44a3ddf1fd2dd78b355a112bfb0ab34d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Wed, 19 Aug 2026 16:25:58 -0300
Subject: [PATCH 5/5] refactor(tooling): keep SERVERS in the workflows, drop it
 only from devnet.env

Narrows the previous commit: replacing the `for h in $SERVERS` loops with an
inline inventory query per command went further than intended. Which hosts a
command runs against is the caller's to pass, and `SERVERS` is how they pass it,
so the workflows are back to taking it from the caller and the query that fills
it stays documented where inventory.sh is introduced.

What devnet.env loses stands: no script reads SERVERS or SSH_USER, so storing
them there only created a host list to keep in step with devnet.inventory.
---
 .claude/skills/multi-server-devnet/SKILL.md   | 30 ++++++++-----------
 .../references/operations.md                  |  6 ++--
 .../multi-server-devnet/scripts/devnet-env.sh |  6 ++--
 .../scripts/devnet.env.example                |  7 ++---
 .../scripts/devnet.inventory.example          |  4 +--
 5 files changed, 24 insertions(+), 29 deletions(-)

diff --git a/.claude/skills/multi-server-devnet/SKILL.md b/.claude/skills/multi-server-devnet/SKILL.md
index 8e02dd28..dd5285fa 100644
--- a/.claude/skills/multi-server-devnet/SKILL.md
+++ b/.claude/skills/multi-server-devnet/SKILL.md
@@ -40,16 +40,19 @@ B owns `k+1..N-1`). Everything below still applies with three changes, and
 
 Nothing about the servers is hardcoded. Establish these from the operator:
 
-- The hosts: one row each in `scripts/devnet.inventory` (below), queried with
-  `scripts/inventory.sh`. There is no second host list to hold in step with it.
+- `SERVERS`: the SSH targets for the operation at hand (any count, any names),
+  one devnet each. A working set taken FROM the inventory below, not a second
+  list of hosts to keep in step with it.
 - `SSH_USER`: login user. `docker` is invoked with `sudo`.
 - Per-devnet `NODES` (validators on that server) and `SUBNETS`
   (`ATTESTATION_COMMITTEE_COUNT`); these can differ between servers.
 - A central host for Grafana + the federating Prometheus (often one of the
   servers); each server's per-host Prometheus remote-writes to it.
 
-Put the non-host values in `scripts/devnet.env` (copy `scripts/devnet.env.example`;
-gitignored) instead of retyping them: the operator-side scripts source it via
+`SERVERS`/`SSH_USER` are the caller's to pass per command: no script reads them, so
+which hosts a command runs against is decided at the call rather than stored. The
+rest goes in `scripts/devnet.env` (copy `scripts/devnet.env.example`; gitignored)
+instead of being retyped: the operator-side scripts source it via
 `scripts/devnet-env.sh`, and an env var exported in the shell still wins over the
 file. It is the one place a deployment's urls and Grafana ids live.
 
@@ -62,8 +65,8 @@ than reading it by hand:
 bash scripts/inventory.sh --tag devnet-ab --field ip        # ips, for a loop
 bash scripts/inventory.sh --tag devnet-ab --tag aggregator  # AND across tags
 bash scripts/inventory.sh --tag validator                   # derived, see below
-hosts=$(bash scripts/inventory.sh --tag devnet-ab --field name) || exit
-for h in $(echo $hosts); do ssh "$SSH_USER@$h" uptime; done  # splits in bash + zsh
+SERVERS=$(bash scripts/inventory.sh --field name) || exit   # exit 2 = typo'd tag
+SERVERS=${SERVERS//$'\n'/ }                                 # newlines -> spaces
 ```
 
 Tag conventions: a `devnet-*` tag names the chain a host's nodes belong to (two
@@ -152,20 +155,13 @@ redirect must run under sudo).
 
 ## Workflows
 
-Examples assume `SSH_USER` is set and `$h` is one host. Per server you pass its own
-`NODES`/`SUBNETS`. Keep the host query and the loop in ONE command, since a shell's
-variables don't outlive it, and `|| exit` so a typo'd tag can't leave a loop that
-does nothing and reports success:
-
-```bash
-hosts=$(bash scripts/inventory.sh --tag <devnet> --field name) || exit
-for h in $(echo $hosts); do ...; done   # $(echo ...) splits under bash and zsh
-```
+Examples assume `SSH_USER` is set and you iterate over `SERVERS`. Per server you
+pass its own `NODES`/`SUBNETS`. `for h in $SERVERS` splits under bash; zsh does not
+split parameter expansions, so there write `for h in $(echo $SERVERS)`.
 
 ### Pull the latest images on all servers
 ```bash
-hosts=$(bash scripts/inventory.sh --tag <devnet> --field name) || exit
-for h in $(echo $hosts); do ssh "$SSH_USER@$h" 'sudo docker pull ghcr.io/lambdaclass/ethlambda:devnet5'; done
+for h in $SERVERS; do ssh "$SSH_USER@$h" 'sudo docker pull ghcr.io/lambdaclass/ethlambda:devnet5'; done
 ```
 `devnet5` is the current devnet tag; it tracks the chain's leanVM/proof format,
 so it (and the other clients' tags in `references/clients.md`) move together on a
diff --git a/.claude/skills/multi-server-devnet/references/operations.md b/.claude/skills/multi-server-devnet/references/operations.md
index 5afada2f..cefd3b14 100644
--- a/.claude/skills/multi-server-devnet/references/operations.md
+++ b/.claude/skills/multi-server-devnet/references/operations.md
@@ -6,9 +6,9 @@ SKILL.md has the workflows; this file has the depth and the failure modes.
 
 ## Topology model
 
-- SSH targets come from `scripts/devnet.inventory`, queried with
-  `scripts/inventory.sh` (any count, any names); the operator supplies `SSH_USER`
-  and `docker` runs under `sudo`. No host count, address, or spec is assumed.
+- Operator supplies SSH targets via `SERVERS` (any count, any names) and
+  `SSH_USER`; `docker` runs under `sudo`. No host count, address, or spec is
+  assumed.
 - **Each server runs its own complete devnet** — its own genesis, GENESIS_TIME,
   fork choice, and finality. No cross-server peering: every ENR is pinned to
   `127.0.0.1`, so discovery never leaves the host even though all devnets share
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet-env.sh b/.claude/skills/multi-server-devnet/scripts/devnet-env.sh
index dd496462..d08e8651 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet-env.sh
+++ b/.claude/skills/multi-server-devnet/scripts/devnet-env.sh
@@ -64,7 +64,7 @@ devnet_load_env() {
     esac
     # Quotes, when present, DELIMIT the value, so a '#' or a trailing space inside
     # them survives. Unquoted, a `# comment` tail is dropped: it used to end up
-    # inside the value, and for METRICS_HOST that means ssh-ing to 'host # ssh'.
+    # inside the value, and for SERVERS that means sweep.sh ssh-ing to a host '#'.
     case $val in
       \"*) val=${val#\"}; val=${val%%\"*} ;;
       \'*) val=${val#\'}; val=${val%%\'*} ;;
@@ -73,8 +73,8 @@ devnet_load_env() {
     esac
     # File provides defaults only: keep an already-set value. No eval anywhere --
     # `${!key}` is bash indirect expansion, and `export` gets ONE quoted
-    # name=value word, so a multi-word value (a quoted path with a space, say)
-    # arrives whole and nothing in it is ever reparsed as shell.
+    # name=value word, so a multi-word value (SERVERS="host-a host-b") arrives
+    # whole and nothing in it is ever reparsed as shell.
     [ -n "${!key:-}" ] || export "$key=$val"
   done < "$file"
   echo "loaded deployment env from $file" >&2
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet.env.example b/.claude/skills/multi-server-devnet/scripts/devnet.env.example
index e22c0b63..8c365f84 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet.env.example
+++ b/.claude/skills/multi-server-devnet/scripts/devnet.env.example
@@ -4,10 +4,9 @@
 # Loaded by the operator-side scripts via devnet-env.sh; an env var already set in
 # your shell overrides the value here.
 
-# --- ssh ---------------------------------------------------------------------
-# WHICH hosts exist is devnet.inventory's business, not this file's: query it with
-# scripts/inventory.sh instead of keeping a second host list here to hold in step.
-SSH_USER=user                           # login user; docker is invoked with sudo
+# Only what a script actually reads lives here. SERVERS/SSH_USER don't: which hosts
+# a command runs against is the caller's to pass per invocation, and which hosts
+# EXIST is devnet.inventory's record, queried with scripts/inventory.sh.
 
 # --- central metrics / logs stack -------------------------------------------
 # sweep.sh reads CENTRAL_PROM_URL; the other two are args you pass to
diff --git a/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
index 67291525..3517540d 100644
--- a/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
+++ b/.claude/skills/multi-server-devnet/scripts/devnet.inventory.example
@@ -12,8 +12,8 @@
 #   bash scripts/inventory.sh --tag devnet-ab --field ip        # ips, for a loop
 #   bash scripts/inventory.sh --tag devnet-ab --tag aggregator  # AND: both needed
 #   bash scripts/inventory.sh --tag validator                   # derived, see TAGS
-#   hosts=$(bash scripts/inventory.sh --tag devnet-ab --field name) || exit
-#   for h in $(echo $hosts); do ssh "$SSH_USER@$h" uptime; done  # bash + zsh
+#   SERVERS=$(bash scripts/inventory.sh --field name) || exit   # exit 2 = typo'd tag
+#   SERVERS=${SERVERS//$'\n'/ }                                 # newlines -> spaces
 #
 # FORMAT: whitespace-separated columns, aligned however you like. Blank lines and
 # lines whose first non-space character is '#' are ignored. Five columns: