Scope audit: attribute every hop, verify declared regions, derive region keys (M0-M2)

cmd/ingestor/client_reception.go

@@ -3,6 +3,7 @@ package main
 import (
 	"database/sql"
 	"encoding/json"
+	"fmt"
 	"log"
 	"regexp"
 	"strings"
@@ -23,7 +24,7 @@ var clientPubkeyRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`)
 // companion reports WHERE it directly heard a node, so we write a
 // client_receptions row and never touch the observers/observations tables.
 // rxPubkey is the companion pubkey from the topic (ACL-bound by the broker).
-func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string, regionKeys map[string][]byte) {
+func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string, regionSet *regionKeySet) {
 	// The companion identity IS the (ACL-bound) topic pubkey. Reject non-hex
 	// topic segments so a no-ACL broker can't pollute the coverage tables, and
 	// never fall back to a payload-supplied id (that would defeat the ACL trust
@@ -93,7 +94,7 @@ func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map
 		// collapse them and ON CONFLICT DO NOTHING would silently drop all but
 		// the first.
 		rxAtMillis := rxTime.Format(rxTimeMillisLayout)
-		if obs := buildClientRxObservation(direction, rxPubkey, rawHex, rxAtMillis, ingestedAt, decoded, regionKeys, snrPtr, rssiPtr, lat, lon, accPtr); obs != nil {
+		if obs := buildClientRxObservation(direction, rxPubkey, rawHex, rxAtMillis, ingestedAt, decoded, regionSet, snrPtr, rssiPtr, lat, lon, accPtr); obs != nil {
 			if _, err := store.InsertClientRxObservation(obs); err != nil {
 				log.Printf("MQTT [%s] client observation insert: %v", tag, err)
 			}
@@ -422,7 +423,7 @@ type ClientRxObservation struct {
 // signal — per-flood row multiplicity meant to measure forwarder
 // amplification of traffic actually heard over the air.
 func buildClientRxObservation(
-	direction, rxPubkey, rawHex, rxAt, ingestedAt string, decoded *DecodedPacket, regionKeys map[string][]byte,
+	direction, rxPubkey, rawHex, rxAt, ingestedAt string, decoded *DecodedPacket, regionSet *regionKeySet,
 	snr *float64, rssi *int, lat, lon float64, posAccM *float64,
 ) *ClientRxObservation {
 	if !strings.EqualFold(direction, "rx") {
@@ -447,7 +448,9 @@ func buildClientRxObservation(
 		obs.Code1 = &decoded.TransportCodes.Code1
 		obs.Code2 = &decoded.TransportCodes.Code2
 		if decoded.TransportCodes.Code1 != "0000" {
-			sn := matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1)
+			m := regionSet.snapshot().match(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1)
+			recordScopeMatch(m)
+			sn := m.Name
 			obs.ScopeName = &sn
 		}
 	}
@@ -567,3 +570,64 @@ func (s *Store) CurrentDeclaredRegions(target string) (*ClientDeclaredRegions, e
 	o.Truncated = truncated == 1
 	return &o, nil
 }
+
+// DeclaredRegionStats returns every region name currently declared anywhere on
+// the network, with the two facts the derived-tier cap ranks on: how many
+// distinct repeaters declare it, and the most recent observed_at among them.
+//
+// Only the LATEST answer per target counts - the same rule
+// CurrentDeclaredRegions follows, by observed_at and never ingested_at, so a
+// drive buffered offline cannot resurrect a region a repeater has since
+// dropped. The window function is covered by idx_ndr_target(target,
+// observed_at).
+//
+// CSV splitting and aggregation happen in Go rather than SQL: regions_csv is
+// written with strings.Join, and unpicking it in SQLite would need a recursive
+// CTE for no gain at this row count (~200 targets).
+func (s *Store) DeclaredRegionStats() ([]declaredRegionStat, error) {
+	rows, err := s.db.Query(`
+		WITH ranked AS (
+			SELECT target, observed_at, regions_csv,
+				ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC) AS rn
+			FROM node_declared_regions
+		)
+		SELECT target, observed_at, regions_csv FROM ranked WHERE rn = 1
+	`)
+	if err != nil {
+		return nil, fmt.Errorf("declared region stats: %w", err)
+	}
+	defer rows.Close()
+
+	agg := map[string]*declaredRegionStat{}
+	for rows.Next() {
+		var target, observedAt, csv string
+		if err := rows.Scan(&target, &observedAt, &csv); err != nil {
+			return nil, fmt.Errorf("declared region stats scan: %w", err)
+		}
+		seenHere := map[string]bool{} // one target counts once per name
+		for _, name := range splitDeclaredRegionsCSV(csv) {
+			if name == "*" || seenHere[name] {
+				continue // '*' is the wildcard, not a region name
+			}
+			seenHere[name] = true
+			st, ok := agg[name]
+			if !ok {
+				st = &declaredRegionStat{Name: name}
+				agg[name] = st
+			}
+			st.Declarers++
+			if observedAt > st.LastSeen {
+				st.LastSeen = observedAt
+			}
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("declared region stats rows: %w", err)
+	}
+
+	out := make([]declaredRegionStat, 0, len(agg))
+	for _, st := range agg {
+		out = append(out, *st)
+	}
+	return out, nil
+}

cmd/ingestor/client_reception_test.go

@@ -703,7 +703,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) {
 		"timestamp": ts1,
 		"gps":       map[string]interface{}{"lat": 51.2, "lon": 4.4},
 	}
-	handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg, nil, regionKeys)
+	handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg, nil, regionSetFromKeys(regionKeys))
 
 	// ComputeContentHash deliberately excludes the transport-code bytes (so the
 	// same content dedups across scopes), so raw and raw2 below share one
@@ -732,7 +732,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) {
 		"timestamp": ts2,
 		"gps":       map[string]interface{}{"lat": 51.2, "lon": 4.4},
 	}
-	handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionKeys)
+	handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionSetFromKeys(regionKeys))
 
 	var code1b, scopeNameB sql.NullString
 	if err := s.db.QueryRow(`SELECT code1, scope_name FROM client_rx_observations WHERE rx_at = ?`, ts2).

cmd/ingestor/config.go

@@ -59,6 +59,7 @@ type Config struct {
 	ClientRxObservations *ClientRxObservationsConfig `json:"clientRxObservations,omitempty"`
 	ClientRfSamples      *ClientRfSamplesConfig      `json:"clientRfSamples,omitempty"`
 	ClientRegions        *ClientRegionsConfig        `json:"clientRegions,omitempty"`
+	AutoRegionKeys       *AutoRegionKeysConfig       `json:"autoRegionKeys,omitempty"`
 	GeoFilter            *GeoFilterConfig            `json:"geo_filter,omitempty"`
 	// PathTrust configures the minimum path-hash prefix length trusted as
 	// mapping/topology evidence (issue #1784).
@@ -185,6 +186,61 @@ func (c *Config) ClientRegionsEnabled() bool {
 	return c.ClientRegions != nil && c.ClientRegions.Enabled
 }
 
+// AutoRegionKeysConfig controls the opt-in derivation of region keys from the
+// names repeaters declare over RF (node_declared_regions), on top of the
+// explicit hashRegions list.
+//
+// TOP-LEVEL BLOCK, a sibling of hashRegions — not nested inside it. Config
+// loading is plain json.Unmarshal with no DisallowUnknownFields, so a
+// mis-nested key is silently ignored and derivation stays off with no error,
+// the same trap clientRxObservations documents.
+type AutoRegionKeysConfig struct {
+	Enabled        bool `json:"enabled"`
+	MaxDerived     int  `json:"maxDerived"`
+	RefreshMinutes int  `json:"refreshMinutes"`
+}
+
+// autoRegionKeysDefaultMaxDerived bounds the derived tier. Every added key
+// raises the random ambiguity rate by 1/65536 per scoped packet (code1 is two
+// bytes) and costs one more HMAC per transport-scoped packet — the match is
+// O(keys) and cannot be indexed, because the code depends on the payload. 256
+// on top of a typical explicit set puts the ambiguity rate near 0.5%, which is
+// the ceiling this design accepts.
+const autoRegionKeysDefaultMaxDerived = 256
+
+// autoRegionKeysDefaultRefreshMinutes is how often the derived tier is rebuilt
+// from the database. Declared-region answers arrive at human pace (a drive-by
+// with a companion app, or a 24h observer report), so minutes-scale staleness
+// is irrelevant and a tighter interval only burns queries.
+const autoRegionKeysDefaultRefreshMinutes = 15
+
+// AutoRegionKeysEnabled reports whether region keys may be derived from
+// declared-region answers. Default false.
+func (c *Config) AutoRegionKeysEnabled() bool {
+	return c.AutoRegionKeys != nil && c.AutoRegionKeys.Enabled
+}
+
+// AutoRegionKeysMaxDerived returns the derived-tier cap, falling back to the
+// default for absent, zero, or negative values — a zero is indistinguishable
+// from "key omitted" after json.Unmarshal, and neither should silently mean
+// "derive nothing" when the operator has switched the feature on.
+func (c *Config) AutoRegionKeysMaxDerived() int {
+	if c.AutoRegionKeys == nil || c.AutoRegionKeys.MaxDerived <= 0 {
+		return autoRegionKeysDefaultMaxDerived
+	}
+	return c.AutoRegionKeys.MaxDerived
+}
+
+// AutoRegionKeysRefreshMinutes returns the refresh interval in minutes,
+// falling back to the default for absent, zero, or negative values — a zero
+// here would otherwise panic time.NewTicker.
+func (c *Config) AutoRegionKeysRefreshMinutes() int {
+	if c.AutoRegionKeys == nil || c.AutoRegionKeys.RefreshMinutes <= 0 {
+		return autoRegionKeysDefaultRefreshMinutes
+	}
+	return c.AutoRegionKeys.RefreshMinutes
+}
+
 // RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes.
 type RetentionConfig struct {
 	NodeDays     int `json:"nodeDays"`