diff --git a/cmd/ingestor/client_reception.go b/cmd/ingestor/client_reception.go index 7d30eca1f..959c91324 100644 --- a/cmd/ingestor/client_reception.go +++ b/cmd/ingestor/client_reception.go @@ -3,6 +3,7 @@ package main import ( "database/sql" "encoding/json" + "fmt" "log" "regexp" "strings" @@ -23,7 +24,7 @@ var clientPubkeyRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`) // companion reports WHERE it directly heard a node, so we write a // client_receptions row and never touch the observers/observations tables. // rxPubkey is the companion pubkey from the topic (ACL-bound by the broker). -func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string, regionKeys map[string][]byte) { +func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string, regionSet *regionKeySet) { // The companion identity IS the (ACL-bound) topic pubkey. Reject non-hex // topic segments so a no-ACL broker can't pollute the coverage tables, and // never fall back to a payload-supplied id (that would defeat the ACL trust @@ -93,7 +94,7 @@ func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map // collapse them and ON CONFLICT DO NOTHING would silently drop all but // the first. rxAtMillis := rxTime.Format(rxTimeMillisLayout) - if obs := buildClientRxObservation(direction, rxPubkey, rawHex, rxAtMillis, ingestedAt, decoded, regionKeys, snrPtr, rssiPtr, lat, lon, accPtr); obs != nil { + if obs := buildClientRxObservation(direction, rxPubkey, rawHex, rxAtMillis, ingestedAt, decoded, regionSet, snrPtr, rssiPtr, lat, lon, accPtr); obs != nil { if _, err := store.InsertClientRxObservation(obs); err != nil { log.Printf("MQTT [%s] client observation insert: %v", tag, err) } @@ -422,7 +423,7 @@ type ClientRxObservation struct { // signal — per-flood row multiplicity meant to measure forwarder // amplification of traffic actually heard over the air. func buildClientRxObservation( - direction, rxPubkey, rawHex, rxAt, ingestedAt string, decoded *DecodedPacket, regionKeys map[string][]byte, + direction, rxPubkey, rawHex, rxAt, ingestedAt string, decoded *DecodedPacket, regionSet *regionKeySet, snr *float64, rssi *int, lat, lon float64, posAccM *float64, ) *ClientRxObservation { if !strings.EqualFold(direction, "rx") { @@ -447,7 +448,9 @@ func buildClientRxObservation( obs.Code1 = &decoded.TransportCodes.Code1 obs.Code2 = &decoded.TransportCodes.Code2 if decoded.TransportCodes.Code1 != "0000" { - sn := matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + m := regionSet.snapshot().match(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + recordScopeMatch(m) + sn := m.Name obs.ScopeName = &sn } } @@ -567,3 +570,64 @@ func (s *Store) CurrentDeclaredRegions(target string) (*ClientDeclaredRegions, e o.Truncated = truncated == 1 return &o, nil } + +// DeclaredRegionStats returns every region name currently declared anywhere on +// the network, with the two facts the derived-tier cap ranks on: how many +// distinct repeaters declare it, and the most recent observed_at among them. +// +// Only the LATEST answer per target counts - the same rule +// CurrentDeclaredRegions follows, by observed_at and never ingested_at, so a +// drive buffered offline cannot resurrect a region a repeater has since +// dropped. The window function is covered by idx_ndr_target(target, +// observed_at). +// +// CSV splitting and aggregation happen in Go rather than SQL: regions_csv is +// written with strings.Join, and unpicking it in SQLite would need a recursive +// CTE for no gain at this row count (~200 targets). +func (s *Store) DeclaredRegionStats() ([]declaredRegionStat, error) { + rows, err := s.db.Query(` + WITH ranked AS ( + SELECT target, observed_at, regions_csv, + ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC) AS rn + FROM node_declared_regions + ) + SELECT target, observed_at, regions_csv FROM ranked WHERE rn = 1 + `) + if err != nil { + return nil, fmt.Errorf("declared region stats: %w", err) + } + defer rows.Close() + + agg := map[string]*declaredRegionStat{} + for rows.Next() { + var target, observedAt, csv string + if err := rows.Scan(&target, &observedAt, &csv); err != nil { + return nil, fmt.Errorf("declared region stats scan: %w", err) + } + seenHere := map[string]bool{} // one target counts once per name + for _, name := range splitDeclaredRegionsCSV(csv) { + if name == "*" || seenHere[name] { + continue // '*' is the wildcard, not a region name + } + seenHere[name] = true + st, ok := agg[name] + if !ok { + st = &declaredRegionStat{Name: name} + agg[name] = st + } + st.Declarers++ + if observedAt > st.LastSeen { + st.LastSeen = observedAt + } + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("declared region stats rows: %w", err) + } + + out := make([]declaredRegionStat, 0, len(agg)) + for _, st := range agg { + out = append(out, *st) + } + return out, nil +} diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index 4b07cb40b..d4ce00b92 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -703,7 +703,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { "timestamp": ts1, "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4}, } - handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg, nil, regionKeys) + handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg, nil, regionSetFromKeys(regionKeys)) // ComputeContentHash deliberately excludes the transport-code bytes (so the // same content dedups across scopes), so raw and raw2 below share one @@ -732,7 +732,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { "timestamp": ts2, "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4}, } - handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionKeys) + handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionSetFromKeys(regionKeys)) var code1b, scopeNameB sql.NullString if err := s.db.QueryRow(`SELECT code1, scope_name FROM client_rx_observations WHERE rx_at = ?`, ts2). diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 1277e5a56..a1f23bdf6 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -59,6 +59,7 @@ type Config struct { ClientRxObservations *ClientRxObservationsConfig `json:"clientRxObservations,omitempty"` ClientRfSamples *ClientRfSamplesConfig `json:"clientRfSamples,omitempty"` ClientRegions *ClientRegionsConfig `json:"clientRegions,omitempty"` + AutoRegionKeys *AutoRegionKeysConfig `json:"autoRegionKeys,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` // PathTrust configures the minimum path-hash prefix length trusted as // mapping/topology evidence (issue #1784). @@ -185,6 +186,61 @@ func (c *Config) ClientRegionsEnabled() bool { return c.ClientRegions != nil && c.ClientRegions.Enabled } +// AutoRegionKeysConfig controls the opt-in derivation of region keys from the +// names repeaters declare over RF (node_declared_regions), on top of the +// explicit hashRegions list. +// +// TOP-LEVEL BLOCK, a sibling of hashRegions — not nested inside it. Config +// loading is plain json.Unmarshal with no DisallowUnknownFields, so a +// mis-nested key is silently ignored and derivation stays off with no error, +// the same trap clientRxObservations documents. +type AutoRegionKeysConfig struct { + Enabled bool `json:"enabled"` + MaxDerived int `json:"maxDerived"` + RefreshMinutes int `json:"refreshMinutes"` +} + +// autoRegionKeysDefaultMaxDerived bounds the derived tier. Every added key +// raises the random ambiguity rate by 1/65536 per scoped packet (code1 is two +// bytes) and costs one more HMAC per transport-scoped packet — the match is +// O(keys) and cannot be indexed, because the code depends on the payload. 256 +// on top of a typical explicit set puts the ambiguity rate near 0.5%, which is +// the ceiling this design accepts. +const autoRegionKeysDefaultMaxDerived = 256 + +// autoRegionKeysDefaultRefreshMinutes is how often the derived tier is rebuilt +// from the database. Declared-region answers arrive at human pace (a drive-by +// with a companion app, or a 24h observer report), so minutes-scale staleness +// is irrelevant and a tighter interval only burns queries. +const autoRegionKeysDefaultRefreshMinutes = 15 + +// AutoRegionKeysEnabled reports whether region keys may be derived from +// declared-region answers. Default false. +func (c *Config) AutoRegionKeysEnabled() bool { + return c.AutoRegionKeys != nil && c.AutoRegionKeys.Enabled +} + +// AutoRegionKeysMaxDerived returns the derived-tier cap, falling back to the +// default for absent, zero, or negative values — a zero is indistinguishable +// from "key omitted" after json.Unmarshal, and neither should silently mean +// "derive nothing" when the operator has switched the feature on. +func (c *Config) AutoRegionKeysMaxDerived() int { + if c.AutoRegionKeys == nil || c.AutoRegionKeys.MaxDerived <= 0 { + return autoRegionKeysDefaultMaxDerived + } + return c.AutoRegionKeys.MaxDerived +} + +// AutoRegionKeysRefreshMinutes returns the refresh interval in minutes, +// falling back to the default for absent, zero, or negative values — a zero +// here would otherwise panic time.NewTicker. +func (c *Config) AutoRegionKeysRefreshMinutes() int { + if c.AutoRegionKeys == nil || c.AutoRegionKeys.RefreshMinutes <= 0 { + return autoRegionKeysDefaultRefreshMinutes + } + return c.AutoRegionKeys.RefreshMinutes +} + // RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes. type RetentionConfig struct { NodeDays int `json:"nodeDays"` diff --git a/cmd/ingestor/config_test.go b/cmd/ingestor/config_test.go index 02e7a3203..807632b09 100644 --- a/cmd/ingestor/config_test.go +++ b/cmd/ingestor/config_test.go @@ -1,566 +1,44 @@ package main -import ( - "github.com/meshcore-analyzer/packetpath" - "os" - "path/filepath" - "testing" -) +import "testing" -func TestLoadConfigValidJSON(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{ - "dbPath": "/tmp/test.db", - "mqttSources": [ - {"name": "s1", "broker": "tcp://localhost:1883", "topics": ["meshcore/#"]} - ] - }`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if cfg.DBPath != "/tmp/test.db" { - t.Errorf("dbPath=%s, want /tmp/test.db", cfg.DBPath) - } - if len(cfg.MQTTSources) != 1 { - t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources)) - } - if cfg.MQTTSources[0].Broker != "tcp://localhost:1883" { - t.Errorf("broker=%s", cfg.MQTTSources[0].Broker) - } -} - -func TestLoadConfigMissingFile(t *testing.T) { - t.Setenv("DB_PATH", "") - t.Setenv("MQTT_BROKER", "") - - cfg, err := LoadConfig("/nonexistent/path/config.json") - if err != nil { - t.Fatalf("missing config should not error (zero-config mode), got: %v", err) - } - if cfg.DBPath != "data/meshcore.db" { - t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath) - } - // Should default to localhost MQTT - if len(cfg.MQTTSources) != 1 { - t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources)) - } - if cfg.MQTTSources[0].Broker != "mqtt://localhost:1883" { - t.Errorf("default broker=%s, want mqtt://localhost:1883", cfg.MQTTSources[0].Broker) - } - if cfg.MQTTSources[0].Name != "local" { - t.Errorf("default source name=%s, want local", cfg.MQTTSources[0].Name) - } -} - -func TestLoadConfigMalformedJSON(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "bad.json") - os.WriteFile(cfgPath, []byte(`{not valid json`), 0o644) - - _, err := LoadConfig(cfgPath) - if err == nil { - t.Error("expected error for malformed JSON") - } -} - -func TestLoadConfigEnvVarDBPath(t *testing.T) { - t.Setenv("DB_PATH", "/override/db.sqlite") - t.Setenv("MQTT_BROKER", "") - - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{"dbPath": "original.db"}`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if cfg.DBPath != "/override/db.sqlite" { - t.Errorf("dbPath=%s, want /override/db.sqlite", cfg.DBPath) - } -} - -func TestLoadConfigEnvVarMQTTBroker(t *testing.T) { - t.Setenv("MQTT_BROKER", "tcp://env-broker:1883") - t.Setenv("MQTT_TOPIC", "custom/topic") - - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{"dbPath": "test.db"}`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if len(cfg.MQTTSources) != 1 { - t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources)) - } - src := cfg.MQTTSources[0] - if src.Name != "env" { - t.Errorf("name=%s, want env", src.Name) - } - if src.Broker != "tcp://env-broker:1883" { - t.Errorf("broker=%s", src.Broker) - } - if len(src.Topics) != 1 || src.Topics[0] != "custom/topic" { - t.Errorf("topics=%v, want [custom/topic]", src.Topics) - } -} - -func TestLoadConfigEnvVarMQTTBrokerDefaultTopic(t *testing.T) { - t.Setenv("MQTT_BROKER", "tcp://env-broker:1883") - t.Setenv("MQTT_TOPIC", "") - - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{"dbPath": "test.db"}`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if cfg.MQTTSources[0].Topics[0] != "meshcore/#" { - t.Errorf("default topic=%s, want meshcore/#", cfg.MQTTSources[0].Topics[0]) - } -} - -func TestLoadConfigLegacyMQTT(t *testing.T) { - t.Setenv("DB_PATH", "") - t.Setenv("MQTT_BROKER", "") - - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{ - "dbPath": "test.db", - "mqtt": {"broker": "tcp://legacy:1883", "topic": "old/topic"} - }`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if len(cfg.MQTTSources) != 1 { - t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources)) - } - src := cfg.MQTTSources[0] - if src.Name != "default" { - t.Errorf("name=%s, want default", src.Name) - } - if src.Broker != "tcp://legacy:1883" { - t.Errorf("broker=%s", src.Broker) - } - if len(src.Topics) != 2 || src.Topics[0] != "old/topic" || src.Topics[1] != "meshcore/#" { - t.Errorf("topics=%v, want [old/topic meshcore/#]", src.Topics) - } -} - -func TestLoadConfigLegacyMQTTNotUsedWhenSourcesExist(t *testing.T) { - t.Setenv("DB_PATH", "") - t.Setenv("MQTT_BROKER", "") - - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{ - "dbPath": "test.db", - "mqtt": {"broker": "tcp://legacy:1883", "topic": "old/topic"}, - "mqttSources": [{"name": "modern", "broker": "tcp://modern:1883", "topics": ["m/#"]}] - }`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if len(cfg.MQTTSources) != 1 { - t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources)) - } - if cfg.MQTTSources[0].Name != "modern" { - t.Errorf("should use modern source, got name=%s", cfg.MQTTSources[0].Name) - } -} - -func TestLoadConfigDefaultDBPath(t *testing.T) { - t.Setenv("DB_PATH", "") - t.Setenv("MQTT_BROKER", "") - - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{}`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if cfg.DBPath != "data/meshcore.db" { - t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath) - } -} - -func TestLoadConfigLegacyMQTTEmptyBroker(t *testing.T) { - t.Setenv("DB_PATH", "") - t.Setenv("MQTT_BROKER", "") - - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{ - "dbPath": "test.db", - "mqtt": {"broker": "", "topic": "t"} - }`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if len(cfg.MQTTSources) != 1 || cfg.MQTTSources[0].Name != "local" { - t.Errorf("mqttSources should default to local broker when legacy broker is empty, got %v", cfg.MQTTSources) - } -} - -func TestResolvedSources(t *testing.T) { - cfg := &Config{ - MQTTSources: []MQTTSource{ - {Name: "a", Broker: "tcp://a:1883"}, - {Name: "b", Broker: "tcp://b:1883"}, - }, - } - sources := cfg.ResolvedSources() - if len(sources) != 2 { - t.Fatalf("len=%d, want 2", len(sources)) - } - if sources[0].Name != "a" || sources[1].Name != "b" { - t.Errorf("sources=%v", sources) - } -} - -func TestResolvedSourcesEmpty(t *testing.T) { +func TestAutoRegionKeysDefaultsOff(t *testing.T) { + // An absent block must not enable anything. This is the whole safety + // story: every existing deployment upgrades into unchanged behaviour. cfg := &Config{} - sources := cfg.ResolvedSources() - if len(sources) != 0 { - t.Errorf("len=%d, want 0", len(sources)) - } -} - -func TestLoadConfigWithAllFields(t *testing.T) { - t.Setenv("DB_PATH", "") - t.Setenv("MQTT_BROKER", "") - - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - reject := false - _ = reject - os.WriteFile(cfgPath, []byte(`{ - "dbPath": "mydb.db", - "logLevel": "debug", - "mqttSources": [{ - "name": "full", - "broker": "tcp://full:1883", - "username": "user1", - "password": "pass1", - "rejectUnauthorized": false, - "topics": ["a/#", "b/#"], - "iataFilter": ["SJC", "LAX"] - }] - }`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if cfg.LogLevel != "debug" { - t.Errorf("logLevel=%s, want debug", cfg.LogLevel) + if cfg.AutoRegionKeysEnabled() { + t.Error("AutoRegionKeysEnabled() = true on an empty config, want false") } - src := cfg.MQTTSources[0] - if src.Username != "user1" { - t.Errorf("username=%s", src.Username) + if got := cfg.AutoRegionKeysMaxDerived(); got != 256 { + t.Errorf("AutoRegionKeysMaxDerived() = %d, want the 256 default", got) } - if src.Password != "pass1" { - t.Errorf("password=%s", src.Password) - } - if src.RejectUnauthorized == nil || *src.RejectUnauthorized != false { - t.Error("rejectUnauthorized should be false") - } - if len(src.IATAFilter) != 2 || src.IATAFilter[0] != "SJC" { - t.Errorf("iataFilter=%v", src.IATAFilter) - } -} - -func TestConnectTimeoutOrDefault(t *testing.T) { - // Default when unset - s := MQTTSource{} - if got := s.ConnectTimeoutOrDefault(); got != 30 { - t.Errorf("default: got %d, want 30", got) - } - - // Custom value - s.ConnectTimeoutSec = 5 - if got := s.ConnectTimeoutOrDefault(); got != 5 { - t.Errorf("custom: got %d, want 5", got) - } - - // Zero treated as unset - s.ConnectTimeoutSec = 0 - if got := s.ConnectTimeoutOrDefault(); got != 30 { - t.Errorf("zero: got %d, want 30", got) - } -} - -func TestConnectTimeoutFromJSON(t *testing.T) { - dir := t.TempDir() - cfgPath := dir + "/config.json" - os.WriteFile(cfgPath, []byte(`{"mqttSources":[{"name":"s1","broker":"tcp://b:1883","topics":["#"],"connectTimeoutSec":5}]}`), 0644) - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if got := cfg.MQTTSources[0].ConnectTimeoutOrDefault(); got != 5 { - t.Errorf("from JSON: got %d, want 5", got) - } -} - -func TestObserverIATAWhitelist(t *testing.T) { - // Config with whitelist set - cfg := Config{ - ObserverIATAWhitelist: []string{"ARN", "got"}, - } - - // Matching (case-insensitive) - if !cfg.IsObserverIATAAllowed("ARN") { - t.Error("ARN should be allowed") - } - if !cfg.IsObserverIATAAllowed("arn") { - t.Error("arn (lowercase) should be allowed") - } - if !cfg.IsObserverIATAAllowed("GOT") { - t.Error("GOT should be allowed") - } - - // Non-matching - if cfg.IsObserverIATAAllowed("SJC") { - t.Error("SJC should NOT be allowed") - } - - // Empty string not allowed - if cfg.IsObserverIATAAllowed("") { - t.Error("empty IATA should NOT be allowed") - } -} - -func TestObserverIATAWhitelistEmpty(t *testing.T) { - // No whitelist = allow all - cfg := Config{} - if !cfg.IsObserverIATAAllowed("SJC") { - t.Error("with no whitelist, all IATAs should be allowed") - } - if !cfg.IsObserverIATAAllowed("") { - t.Error("with no whitelist, even empty IATA should be allowed") - } -} - -func TestObserverIATAWhitelistJSON(t *testing.T) { - json := `{ - "dbPath": "test.db", - "observerIATAWhitelist": ["ARN", "GOT"] - }` - tmp := t.TempDir() + "/config.json" - os.WriteFile(tmp, []byte(json), 0644) - cfg, err := LoadConfig(tmp) - if err != nil { - t.Fatal(err) - } - if len(cfg.ObserverIATAWhitelist) != 2 { - t.Fatalf("expected 2 entries, got %d", len(cfg.ObserverIATAWhitelist)) - } - if !cfg.IsObserverIATAAllowed("ARN") { - t.Error("ARN should be allowed after loading from JSON") - } -} - -func TestMQTTSourceRegionField(t *testing.T) { - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{ - "dbPath": "/tmp/test.db", - "mqttSources": [ - {"name": "cascadia", "broker": "tcp://localhost:1883", "topics": ["meshcore/#"], "region": "PDX"} - ] - }`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if cfg.MQTTSources[0].Region != "PDX" { - t.Fatalf("expected region PDX, got %q", cfg.MQTTSources[0].Region) - } -} - -// TestResolvedSourcesSchemeMapping verifies that mqtt:// and mqtts:// are translated -// to the paho-native tcp:// and ssl:// schemes, while ws:// and wss:// pass through -// unchanged (paho handles WebSocket connections natively). -func TestResolvedSourcesSchemeMapping(t *testing.T) { - tests := []struct { - input string - want string - }{ - {"mqtt://host:1883", "tcp://host:1883"}, - {"mqtts://host:8883", "ssl://host:8883"}, - {"tcp://host:1883", "tcp://host:1883"}, - {"ssl://host:8883", "ssl://host:8883"}, - {"ws://host:9001", "ws://host:9001"}, - {"wss://host:9001", "wss://host:9001"}, - {"ws://host:9001/mqtt", "ws://host:9001/mqtt"}, - {"wss://host:9001/mqtt", "wss://host:9001/mqtt"}, - } - - for _, tt := range tests { - cfg := &Config{ - MQTTSources: []MQTTSource{ - {Name: "test", Broker: tt.input, Topics: []string{"meshcore/#"}}, - }, - } - sources := cfg.ResolvedSources() - if got := sources[0].Broker; got != tt.want { - t.Errorf("ResolvedSources(%q) = %q, want %q", tt.input, got, tt.want) - } - } -} - -// TestLoadConfigWSSource verifies that a WebSocket MQTT source round-trips through -// LoadConfig correctly — username/password preserved, scheme unchanged. -func TestLoadConfigWSSource(t *testing.T) { - t.Setenv("DB_PATH", "") - t.Setenv("MQTT_BROKER", "") - - dir := t.TempDir() - cfgPath := filepath.Join(dir, "config.json") - os.WriteFile(cfgPath, []byte(`{ - "dbPath": "test.db", - "mqttSources": [ - { - "name": "local-tcp", - "broker": "mqtt://localhost:1883", - "topics": ["meshcore/#"] - }, - { - "name": "wsmqtt-ws", - "broker": "wss://wsmqtt.example.com/mqtt", - "username": "corescope", - "password": "s3cr3t", - "topics": ["meshcore/#"] - } - ] - }`), 0o644) - - cfg, err := LoadConfig(cfgPath) - if err != nil { - t.Fatal(err) - } - if len(cfg.MQTTSources) != 2 { - t.Fatalf("mqttSources len=%d, want 2", len(cfg.MQTTSources)) - } - - tcp := cfg.MQTTSources[0] - if tcp.Name != "local-tcp" { - t.Errorf("name=%s, want local-tcp", tcp.Name) - } - - ws := cfg.MQTTSources[1] - if ws.Name != "wsmqtt-ws" { - t.Errorf("name=%s, want wsmqtt-ws", ws.Name) - } - if ws.Broker != "wss://wsmqtt.example.com/mqtt" { - t.Errorf("broker=%s, want wss://wsmqtt.example.com/mqtt", ws.Broker) - } - if ws.Username != "corescope" { - t.Errorf("username=%s, want corescope", ws.Username) - } - if ws.Password != "s3cr3t" { - t.Errorf("password=%s, want s3cr3t", ws.Password) - } - - sources := cfg.ResolvedSources() - if sources[1].Broker != "wss://wsmqtt.example.com/mqtt" { - t.Errorf("ResolvedSources wss broker=%s, want unchanged", sources[1].Broker) + if got := cfg.AutoRegionKeysRefreshMinutes(); got != 15 { + t.Errorf("AutoRegionKeysRefreshMinutes() = %d, want the 15 default", got) } } -func TestIngestBufferSizeOrDefault(t *testing.T) { - if got := (&Config{}).IngestBufferSizeOrDefault(); got != 50000 { - t.Fatalf("default: want 50000, got %d", got) +func TestAutoRegionKeysExplicitValues(t *testing.T) { + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true, MaxDerived: 64, RefreshMinutes: 5}} + if !cfg.AutoRegionKeysEnabled() { + t.Error("AutoRegionKeysEnabled() = false, want true") } - if got := (&Config{IngestBufferSize: 10}).IngestBufferSizeOrDefault(); got != 10 { - t.Fatalf("override: want 10, got %d", got) + if got := cfg.AutoRegionKeysMaxDerived(); got != 64 { + t.Errorf("AutoRegionKeysMaxDerived() = %d, want 64", got) } - if got := (&Config{IngestBufferSize: -5}).IngestBufferSizeOrDefault(); got != 50000 { - t.Fatalf("invalid negative should fall back to default, got %d", got) + if got := cfg.AutoRegionKeysRefreshMinutes(); got != 5 { + t.Errorf("AutoRegionKeysRefreshMinutes() = %d, want 5", got) } } -func TestClientRxObservationsGate(t *testing.T) { - var c Config - if c.ClientRxObservationsEnabled() { - t.Error("default should be disabled") - } - c.ClientRxObservations = &ClientRxObservationsConfig{Enabled: true} - if !c.ClientRxObservationsEnabled() { - t.Error("explicit enable not honoured") +func TestAutoRegionKeysRejectsNonPositiveOverrides(t *testing.T) { + // A zero is indistinguishable from "absent" after json.Unmarshal, and a + // negative is a typo. Both fall back to the default rather than silently + // disabling derivation or panicking time.NewTicker. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true, MaxDerived: 0, RefreshMinutes: -1}} + if got := cfg.AutoRegionKeysMaxDerived(); got != 256 { + t.Errorf("AutoRegionKeysMaxDerived() = %d, want the 256 default", got) } - c.ClientRxObservations = &ClientRxObservationsConfig{Enabled: false} - if c.ClientRxObservationsEnabled() { - t.Error("explicit disable not honoured") - } - if got := c.ClientRxObsDaysOrZero(); got != 0 { - t.Errorf("unset retention = %d, want 0", got) - } - c.Retention = &RetentionConfig{ClientRxObsDays: 14} - if got := c.ClientRxObsDaysOrZero(); got != 14 { - t.Errorf("retention = %d, want 14", got) - } - c.Retention = &RetentionConfig{ClientRxObsDays: 0} - if got := c.ClientRxObsDaysOrZero(); got != 0 { - t.Errorf("retention=0 = %d, want 0", got) - } -} - -func TestClientRegionsDaysOrZero(t *testing.T) { - var c Config - if got := c.ClientRegionsDaysOrZero(); got != 0 { - t.Errorf("unset retention = %d, want 0", got) - } - c.Retention = &RetentionConfig{ClientRegionsDays: 21} - if got := c.ClientRegionsDaysOrZero(); got != 21 { - t.Errorf("retention = %d, want 21", got) - } - c.Retention = &RetentionConfig{ClientRegionsDays: 0} - if got := c.ClientRegionsDaysOrZero(); got != 0 { - t.Errorf("retention=0 = %d, want 0", got) - } -} - -// --- #1784: GetPathTrust --- - -func TestGetPathTrustDefaults(t *testing.T) { - cfg := &Config{} - pt := cfg.GetPathTrust() - if pt.MinHashBytesForMapping != packetpath.DefaultMinHashBytesForMapping { - t.Errorf("expected default %d, got %d", packetpath.DefaultMinHashBytesForMapping, pt.MinHashBytesForMapping) - } -} - -func TestGetPathTrustCustom(t *testing.T) { - cfg := &Config{PathTrust: &PathTrustConfig{MinHashBytesForMapping: 3}} - pt := cfg.GetPathTrust() - if pt.MinHashBytesForMapping != 3 { - t.Errorf("expected 3, got %d", pt.MinHashBytesForMapping) - } -} - -func TestGetPathTrustNilConfig(t *testing.T) { - var cfg *Config - pt := cfg.GetPathTrust() - if pt.MinHashBytesForMapping != packetpath.DefaultMinHashBytesForMapping { - t.Errorf("expected default %d for nil *Config, got %d", packetpath.DefaultMinHashBytesForMapping, pt.MinHashBytesForMapping) + if got := cfg.AutoRegionKeysRefreshMinutes(); got != 15 { + t.Errorf("AutoRegionKeysRefreshMinutes() = %d, want the 15 default", got) } } diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 8c34d27ce..6830fe59a 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -1607,9 +1607,12 @@ func (s *Store) BackfillPathJSONAsync() { // MQTT packet inserts and any concurrent backfill goroutines — serialize through // the single connection pool. busy_timeout(5000) handles transient cross-process // contention with the read-only server process. No additional locking is needed. -func (s *Store) BackfillDefaultScopeAsync(regionKeys map[string][]byte) { - // No region keys configured — all scope_name values will be NULL, nothing to backfill. - if len(regionKeys) == 0 { +func (s *Store) BackfillDefaultScopeAsync(regionSet *regionKeySet) { + // No region keys in force — all scope_name values will be NULL, nothing to + // backfill. Read once here rather than per row: the backfill is a long + // loop, and a refresh landing halfway through would otherwise change the + // key set under it. + if len(regionSet.snapshot().all) == 0 { return } s.backfillWg.Add(1) @@ -2130,7 +2133,7 @@ type MQTTPacketMessage struct { // into the past. Packet ordering is owned by the server clock; client // clocks are untrusted. msg.Timestamp still flows into observer.last_seen // via UpsertObserverAt — that's #1233's MAX/MIN guarded path and is fine. -func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, region string, regionKeys map[string][]byte) *PacketData { +func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, region string, regionSet *regionKeySet) *PacketData { pathJSON := "[]" // For TRACE packets, path_json must be the payload-decoded route hops // (decoded.Path.Hops), NOT the raw_hex header bytes which are SNR values. @@ -2183,7 +2186,9 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, pd.Code2 = decoded.TransportCodes.Code2 if decoded.TransportCodes.Code1 != "0000" { pd.IsTransportScoped = true - pd.ScopeName = matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + m := regionSet.snapshot().match(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + recordScopeMatch(m) + pd.ScopeName = m.Name } } diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 248a7d4f9..5203b0bf9 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -108,8 +108,13 @@ func main() { log.Printf("No channel keys loaded — GRP_TXT packets will not be decrypted") } - regionKeys := loadRegionKeys(cfg) - store.BackfillDefaultScopeAsync(regionKeys) + regionSet := newRegionKeySet(cfg) + if cfg.AutoRegionKeysEnabled() { + regionSet.refreshFromStore(store) + } else { + log.Printf("[regions] autoRegionKeys disabled — only the %d configured hashRegions key(s) are in force", len(regionSet.snapshot().all)) + } + store.BackfillDefaultScopeAsync(regionSet) store.BackfillTransportCodesAsync() // Subscribe-early + buffer (#1608): the MQTT subscription is brought up @@ -188,7 +193,7 @@ func main() { markReceiptForTag(tag, time.Now()) status.MarkPacket(time.Now()) ingestBuffer.Submit(func() { - handleMessage(store, tag, src, m, channelKeys, regionKeys, cfg) + handleMessage(store, tag, src, m, channelKeys, regionSet, cfg) }) }) @@ -449,6 +454,22 @@ func main() { } } + // Derived region keys refresh on their own ticker rather than the daily + // retention one: declared-region answers arrive continuously (a companion + // app driving past a repeater), and waiting up to 24h to name a + // newly-discovered region would defeat the point of deriving them at all. + if cfg.AutoRegionKeysEnabled() { + interval := time.Duration(cfg.AutoRegionKeysRefreshMinutes()) * time.Minute + regionRefreshTicker := time.NewTicker(interval) + go func() { + for range regionRefreshTicker.C { + regionSet.refreshFromStore(store) + logScopeMatchCounters() + } + }() + log.Printf("[regions] auto-derived region keys enabled: refreshing every %v, cap %d", interval, cfg.AutoRegionKeysMaxDerived()) + } + // Hourly WAL checkpoint to prevent unbounded WAL growth. // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; // if the server's read connection holds frames, remaining pages stay in the @@ -683,7 +704,7 @@ func buildForceReconnectFn(client mqtt.Client, tag string) func() { } } -func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, regionKeys map[string][]byte, cfg *Config) { +func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, regionSet *regionKeySet, cfg *Config) { // Liveness watchdog (#1212): record receipt before any processing so a // slow handler still counts as "source is alive". Cheap atomic store. markLivenessForTag(tag, time.Now()) @@ -729,7 +750,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, switch parts[3] { case "packets": if cfg.ClientRxCoverageEnabled() { - handleClientPacket(store, cfg, tag, parts[2], msg, channelKeys, regionKeys) + handleClientPacket(store, cfg, tag, parts[2], msg, channelKeys, regionSet) } case "rf": if cfg.ClientRfSamplesEnabled() { @@ -970,7 +991,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, log.Printf("MQTT [%s] foreign advert: node=%s name=%s lat=%.4f lon=%.4f observer=%s", tag, truncPK, sanitizeLogString(decoded.Payload.Name), lat, lon, sanitizeLogString(firstNonEmpty(mqttMsg.Origin, observerID))) } - pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys) + pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionSet) pktData.Foreign = foreign isNew, err := store.InsertTransmission(pktData) if err != nil { @@ -1005,7 +1026,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, } else { // Non-ADVERT packets: store normally (routing/channel messages from // in-area observers are relevant regardless of relay hop origin). - pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys) + pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionSet) if _, err := store.InsertTransmission(pktData); err != nil { log.Printf("MQTT [%s] db insert error: %v", tag, err) } @@ -1659,37 +1680,17 @@ func loadRegionKeys(cfg *Config) map[string][]byte { return keys } -// matchScope names the region scope of a transport-scoped packet. It performs -// one HMAC-SHA256 per configured region (expected len(regionKeys) ≤ 50; -// beyond that, consider a pre-indexed lookup table), HMACing the payload with -// each region key and looking for the one whose derived 2-byte code matches -// code1. Two bytes is only 65536 values, so with enough configured regions, -// unrelated keys collide by pure chance often enough to matter (#1609): with -// 58 keys, ~78k transport-scoped packets measured ~34 coincidental matches in -// production. Returning the first match found named the wrong region on -// those packets. +// matchScope was removed in M2 (docs/plans/2026-09-07-auto-derived-region-keys.md). +// Naming a packet's region now goes through regionKeySnapshot.match in +// region_keys.go, which keeps #1609's abstain-on-ambiguity rule but adds a +// principled tie-break: an operator-configured hashRegions key beats one +// derived from a declared-region answer. // -// Fix: collect every matching key instead of returning at the first. Exactly -// one match names that region, as before; more than one match is ambiguous — -// we cannot know which region the sender meant, so this returns the same "" -// used when nothing matches at all (scopeNameForDB's third state: transport- -// scoped but unnameable). Collecting all matches before deciding also removes -// the dependency on regionKeys' (map) iteration order: the old code's result -// for a colliding packet depended on which key Go's randomised map order -// visited first, so the same packet could be labelled differently across -// runs. That nondeterminism is a consequence of returning early, not of the -// map itself, so it goes away once every key is checked before deciding. -func matchScope(regionKeys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string { - matched := matchingRegions(regionKeys, payloadType, payloadRaw, code1) - if len(matched) > 1 { - log.Printf("matchScope: ambiguous code1=%s matched %d region keys %v; returning unmatched", code1, len(matched), matched) - return "" - } - if len(matched) == 1 { - return matched[0] - } - return "" -} +// Its doc comment also suggested a "pre-indexed lookup table" beyond ~50 +// regions. That is not achievable and the idea should not come back: code1 is +// an HMAC over the packet payload, so there is no payload-independent key to +// index on. The cost is inherently one HMAC per region per transport-scoped +// packet, which is exactly why autoRegionKeys.maxDerived exists. // matchingRegions returns the name of every configured region whose derived // 2-byte code equals code1, in no particular order. matchScope uses the diff --git a/cmd/ingestor/main_test.go b/cmd/ingestor/main_test.go index c96bbc80a..7483a14ca 100644 --- a/cmd/ingestor/main_test.go +++ b/cmd/ingestor/main_test.go @@ -858,7 +858,7 @@ func TestMatchScope(t *testing.T) { // Key = SHA256("#test")[:16] = 9cd8fcf22a47333b591d96a2b848b73f testKey, _ := hex.DecodeString("9cd8fcf22a47333b591d96a2b848b73f") testKeys := map[string][]byte{"#test": testKey} - if got := matchScope(testKeys, 5, []byte("hello"), "2AB5"); got != "#test" { + if got := matchScopeName(testKeys, 5, []byte("hello"), "2AB5"); got != "#test" { t.Errorf("#test vector: matchScope = %q, want #test", got) } @@ -866,17 +866,17 @@ func TestMatchScope(t *testing.T) { // Key = SHA256("#belgium")[:16] = 7085b78ed010599094f8c8e7d1aa0e27 belgiumKey, _ := hex.DecodeString("7085b78ed010599094f8c8e7d1aa0e27") belgiumKeys := map[string][]byte{"#belgium": belgiumKey} - if got := matchScope(belgiumKeys, 5, []byte("hello"), "4A75"); got != "#belgium" { + if got := matchScopeName(belgiumKeys, 5, []byte("hello"), "4A75"); got != "#belgium" { t.Errorf("#belgium vector: matchScope = %q, want #belgium", got) } // Code1=0000 (unscoped transport) → no region matched - if got := matchScope(belgiumKeys, 5, []byte("hello"), "0000"); got != "" { + if got := matchScopeName(belgiumKeys, 5, []byte("hello"), "0000"); got != "" { t.Errorf("unscoped: matchScope = %q, want empty", got) } // Code1 present but matches no configured region → empty string - if got := matchScope(belgiumKeys, 5, []byte("hello"), "BEEF"); got != "" { + if got := matchScopeName(belgiumKeys, 5, []byte("hello"), "BEEF"); got != "" { t.Errorf("no match: matchScope = %q, want empty", got) } @@ -893,10 +893,10 @@ func TestMatchScope(t *testing.T) { // ever derives "0000", and ranging over an empty map never executes the // loop body — so those two guards are unobservable defense-in-depth, not // missing coverage.) - if got := matchScope(belgiumKeys, 5, []byte{}, "76AC"); got != "" { + if got := matchScopeName(belgiumKeys, 5, []byte{}, "76AC"); got != "" { t.Errorf("empty payload: matchScope = %q, want empty", got) } - if got := matchScope(map[string][]byte{}, 5, []byte("hello"), "4A75"); got != "" { + if got := matchScopeName(map[string][]byte{}, 5, []byte("hello"), "4A75"); got != "" { t.Errorf("empty regionKeys: matchScope = %q, want empty", got) } } @@ -922,7 +922,7 @@ func TestMatchScopeAmbiguous(t *testing.T) { } for i := 0; i < 20; i++ { - if got := matchScope(keys, 5, []byte("hello"), "2AB5"); got != "" { + if got := matchScopeName(keys, 5, []byte("hello"), "2AB5"); got != "" { t.Fatalf("iteration %d: matchScope = %q, want empty (ambiguous match)", i, got) } } @@ -943,7 +943,7 @@ func TestBuildPacketDataScopeMatching(t *testing.T) { } msg := &MQTTPacketMessage{Raw: rawHex} - pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeys) + pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionSetFromKeys(regionKeys)) if pktData.ScopeName != "#test" { t.Errorf("ScopeName = %q, want #test", pktData.ScopeName) } @@ -1112,7 +1112,7 @@ func TestHandleMessageObserverIATAWhitelist(t *testing.T) { func TestBuildPacketDataScopeMatchingNoMatch(t *testing.T) { // Code1=2AB5 is the precomputed code for region "#test" (payload="hello", // payloadType=5). Build a region-key map for a DIFFERENT region so - // matchScope() finds no match and returns "". + // matchScopeName() finds no match and returns "". const rawHex = "142AB500000068656C6C6F" otherKey, _ := hex.DecodeString("aabbccddeeff00112233445566778899") regionKeys := map[string][]byte{"#other": otherKey} @@ -1122,7 +1122,7 @@ func TestBuildPacketDataScopeMatchingNoMatch(t *testing.T) { t.Fatalf("DecodePacket: %v", err) } msg := &MQTTPacketMessage{Raw: rawHex} - pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeys) + pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionSetFromKeys(regionKeys)) if !pktData.IsTransportScoped { t.Fatalf("precondition: IsTransportScoped should be true (Code1 != 0000)") @@ -1166,12 +1166,12 @@ func TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate(t *testing.T) { t.Fatalf("seed node: %v", err) } - // Empty regionKeys → matchScope() returns "" for any Code1 → ScopeName "". + // Empty regionKeys → matchScopeName() returns "" for any Code1 → ScopeName "". msg := &mockMessage{ topic: "meshcore/SJC/obs1/packets", payload: []byte(`{"raw":"` + rawHex + `"}`), } - handleMessage(store, "test", source, msg, nil, map[string][]byte{}, &Config{}) + handleMessage(store, "test", source, msg, nil, nil, &Config{}) var got sql.NullString if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { @@ -1192,7 +1192,7 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { source := MQTTSource{Name: "test"} // Same ADVERT bytes; this time we compute the matching region key for - // the (payloadType=4, payload=) tuple so matchScope() will + // the (payloadType=4, payload=) tuple so matchScopeName() will // return "#de". const advertBytes = "46D62DE27D4C5194D7821FC5A34A45565DCC2537B300B9AB6275255CEFB65D840CE5C169C94C9AED39E8BCB6CB6EB0335497A198B33A1A610CD3B03D8DCFC160900E5244280323EE0B44CACAB8F02B5B38B91CFA18BD067B0B5E63E94CFC85F758A8530B9240933402E0E6B8F84D5252322D52" const pubkey = "46d62de27d4c5194d7821fc5a34a45565dcc2537b300b9ab6275255cefb65d84" @@ -1223,7 +1223,7 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { topic: "meshcore/SJC/obs1/packets", payload: []byte(`{"raw":"` + rawHex + `"}`), } - handleMessage(store, "test", source, msg, nil, map[string][]byte{"#de": regionKey}, &Config{}) + handleMessage(store, "test", source, msg, nil, regionSetFromKeys(map[string][]byte{"#de": regionKey}), &Config{}) var got sql.NullString if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { diff --git a/cmd/ingestor/region_keys.go b/cmd/ingestor/region_keys.go new file mode 100644 index 000000000..5f2fe43ac --- /dev/null +++ b/cmd/ingestor/region_keys.go @@ -0,0 +1,319 @@ +package main + +import ( + "crypto/sha256" + "log" + "sort" + "strings" + "sync/atomic" +) + +// declaredRegionStat is one region name as reported over RF, with the two +// facts the cap ranks on: how many distinct repeaters declare it, and how +// recently any of them last answered. +type declaredRegionStat struct { + Name string + Declarers int + LastSeen string // ISO, greatest observed_at across declarers +} + +// maxRegionNameLen bounds a derived region name. Firmware region names are +// short labels; anything longer is a malformed or hostile entry, and each +// accepted name costs an HMAC on every transport-scoped packet. +const maxRegionNameLen = 32 + +// regionNameAcceptable reports whether a declared name may become a derived +// region key. +// +// The rules are structural, never about the name's meaning. The declared set +// contains entries that look like junk ("null", "bierhuis", "sol3"), but a +// blocklist on string values is unmaintainable and the cost of one bad name is +// a single slot out of maxDerived plus a 1-in-65536 collision chance. What IS +// rejected is anything that could not have come from the firmware intact: +// +// - a comma would split the name on the next regions_csv round-trip +// - a '#' cannot appear (the firmware strips it), so its presence means the +// value was mangled somewhere upstream +// - non-ASCII or whitespace would make the key SHA256 over bytes nobody +// intended, silently mismatching the sender +// - a NUL is the block-cipher padding a stale client failed to trim +func regionNameAcceptable(name string) bool { + if name == "" || len(name) > maxRegionNameLen { + return false + } + for i := 0; i < len(name); i++ { + c := name[i] + if c <= ' ' || c >= 0x7F || c == ',' || c == '#' { + return false + } + } + return true +} + +// rankDeclaredRegions filters stats through regionNameAcceptable and returns at +// most max names, most-worth-keeping first: by declarer count descending, then +// by recency, then by name. The name tie-break is what makes the result +// deterministic — without it the derived tier would churn between refreshes on +// equally-ranked names and the add/drop logging would be noise. +func rankDeclaredRegions(stats []declaredRegionStat, max int) []string { + kept := make([]declaredRegionStat, 0, len(stats)) + for _, s := range stats { + if regionNameAcceptable(s.Name) { + kept = append(kept, s) + } + } + sort.Slice(kept, func(i, j int) bool { + if kept[i].Declarers != kept[j].Declarers { + return kept[i].Declarers > kept[j].Declarers + } + if kept[i].LastSeen != kept[j].LastSeen { + return kept[i].LastSeen > kept[j].LastSeen + } + return kept[i].Name < kept[j].Name + }) + if max > 0 && len(kept) > max { + kept = kept[:max] + } + names := make([]string, 0, len(kept)) + for _, s := range kept { + names = append(names, s.Name) + } + return names +} + +// splitDeclaredRegionsCSV parses a regions_csv value into its entries. The +// ingestor writes this column with strings.Join(regions, ","), so this is its +// exact inverse. Mirrors splitRegionsCSV in cmd/server/scopes.go. +func splitDeclaredRegionsCSV(csv string) []string { + out := []string{} + if csv == "" { + return out + } + for _, part := range strings.Split(csv, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} + +// regionKeySnapshot is an immutable view of the region keys in force for one +// packet. `all` is the single map matching iterates - merging at build time +// rather than per packet keeps the hot path free of allocation. `explicit` +// carries membership only, and exists so the ambiguity tie-break can tell an +// operator-configured region from one derived off the air. +type regionKeySnapshot struct { + all map[string][]byte + explicit map[string]bool +} + +func (s *regionKeySnapshot) isExplicit(name string) bool { return s.explicit[name] } + +// regionKeySet holds the live snapshot. Readers take one atomic load; a +// refresh builds the replacement off to the side and swaps the pointer, so the +// ingest hot path never blocks on a rebuild (AGENTS.md rule 0). +type regionKeySet struct { + cur atomic.Pointer[regionKeySnapshot] + enabled bool + max int +} + +// newRegionKeySet builds the explicit tier from hashRegions. The derived tier +// starts empty; refreshDerived fills it, and does nothing at all when +// autoRegionKeys is off. +func newRegionKeySet(cfg *Config) *regionKeySet { + explicitKeys := loadRegionKeys(cfg) + explicitNames := make(map[string]bool, len(explicitKeys)) + all := make(map[string][]byte, len(explicitKeys)) + for name, key := range explicitKeys { + explicitNames[name] = true + all[name] = key + } + s := ®ionKeySet{ + enabled: cfg.AutoRegionKeysEnabled(), + max: cfg.AutoRegionKeysMaxDerived(), + } + s.cur.Store(®ionKeySnapshot{all: all, explicit: explicitNames}) + return s +} + +// emptyRegionKeySnapshot backs the nil case below. Shared and never mutated: +// refreshDerived always builds a fresh map rather than writing into one. +var emptyRegionKeySnapshot = ®ionKeySnapshot{all: map[string][]byte{}, explicit: map[string]bool{}} + +// snapshot is nil-safe on purpose. A nil *regionKeySet means "no region keys", +// which is exactly what a nil map[string][]byte meant before M2 — the shape +// several call sites and a good many tests still pass. Panicking there would +// turn an absent key set into a crash on the ingest path, which is a far worse +// failure than naming nothing. +func (s *regionKeySet) snapshot() *regionKeySnapshot { + if s == nil { + return emptyRegionKeySnapshot + } + return s.cur.Load() +} + +// refreshDerived rebuilds the derived tier from names (already ranked and +// capped by the caller) and swaps in a new snapshot. It REPLACES the derived +// tier rather than merging into it, so a region that stops being declared +// leaves the key set and the cap keeps meaning something. +// +// A name that duplicates an explicit key is skipped, not re-added: the +// explicit tier must stay authoritative for the tie-break, and demoting a +// configured region because a repeater also declares it would invert the whole +// rule. +// +// Returns the names actually added, for the caller to log. +func (s *regionKeySet) refreshDerived(names []string) []string { + if !s.enabled { + return nil + } + old := s.cur.Load() + all := make(map[string][]byte, len(old.explicit)+len(names)) + for name := range old.explicit { + all[name] = old.all[name] + } + added := make([]string, 0, len(names)) + for _, raw := range names { + if !regionNameAcceptable(raw) { + continue + } + name := "#" + raw + if old.explicit[name] { + continue + } + if _, exists := all[name]; exists { + continue + } + h := sha256.Sum256([]byte(name)) + all[name] = h[:16] + added = append(added, name) + } + s.cur.Store(®ionKeySnapshot{all: all, explicit: old.explicit}) + return added +} + +// scopeReason records how a scope match was decided, so the outcome is +// auditable in logs without a schema change. It is deliberately not stored: +// transmissions.scope_name keeps its existing three-state encoding. +type scopeReason string + +const ( + scopeReasonNone scopeReason = "none" // no key matched + scopeReasonUnique scopeReason = "unique" // exactly one key matched + scopeReasonExplicitOverDerived scopeReason = "explicit-over-derived" // several matched, one was operator config + scopeReasonAmbiguous scopeReason = "ambiguous" // several matched, no principled winner +) + +// scopeMatch is the result of naming one packet's region scope. +type scopeMatch struct { + Name string // empty when unresolved - the caller stores that as the unmatched state + Reason scopeReason + Candidates []string // every matching name, populated only when more than one matched +} + +// match names the region scope of a transport-scoped packet, resolving a +// multi-key collision by evidence rather than by map order. +// +// Tiers, in order: +// +// 1. Exactly one key matched - name it. +// 2. Several matched but exactly one came from hashRegions - name that one. +// The operator's own configuration outranks a name picked up off the air, +// and this covers the bulk of the ambiguity auto-derivation introduces. +// 3. Otherwise abstain, returning an empty name. Two equally-sourced +// candidates offer no principled winner, and naming a packet wrongly is +// worse than leaving it unnamed - the rule #1609 established, unchanged. +// +// (The spec's tier-3 path-evidence tie-break sits between 2 and 3 and is +// deliberately not built here; see +// docs/specs/2026-09-07-auto-region-keys-design.md. The scopeReasonAmbiguous +// counter is what measures whether it is worth building.) +func (s *regionKeySnapshot) match(payloadType byte, payloadRaw []byte, code1 string) scopeMatch { + matched := matchingRegions(s.all, payloadType, payloadRaw, code1) + switch len(matched) { + case 0: + return scopeMatch{Reason: scopeReasonNone} + case 1: + return scopeMatch{Name: matched[0], Reason: scopeReasonUnique} + } + + var explicitMatches []string + for _, name := range matched { + if s.explicit[name] { + explicitMatches = append(explicitMatches, name) + } + } + if len(explicitMatches) == 1 { + return scopeMatch{Name: explicitMatches[0], Reason: scopeReasonExplicitOverDerived, Candidates: matched} + } + return scopeMatch{Reason: scopeReasonAmbiguous, Candidates: matched} +} + +// scopeMatchCounters tallies how each transport-scoped packet's region was +// decided. It exists to answer one question before more machinery is built: +// how often does an ambiguous collision actually happen? The spec gates the +// path-evidence tie-break (tier 3) on this number. +var scopeMatchCounters struct { + unique atomic.Int64 + explicitOverDerived atomic.Int64 + ambiguous atomic.Int64 + none atomic.Int64 +} + +// recordScopeMatch tallies one decision and logs the interesting ones. Unique +// and none are the overwhelming majority and are counted silently; the other +// two are rare by construction and worth a line each. +func recordScopeMatch(m scopeMatch) { + switch m.Reason { + case scopeReasonUnique: + scopeMatchCounters.unique.Add(1) + case scopeReasonNone: + scopeMatchCounters.none.Add(1) + case scopeReasonExplicitOverDerived: + scopeMatchCounters.explicitOverDerived.Add(1) + log.Printf("[regions] collision resolved to explicit %s over derived candidates %v", m.Name, m.Candidates) + case scopeReasonAmbiguous: + scopeMatchCounters.ambiguous.Add(1) + log.Printf("[regions] ambiguous collision between %v; storing unmatched", m.Candidates) + } +} + +// logScopeMatchCounters prints the running tally. Called from the refresh +// ticker so the numbers arrive on the same cadence as the key-set changes that +// move them. +func logScopeMatchCounters() { + log.Printf("[regions] scope matches: unique=%d explicit-over-derived=%d ambiguous=%d none=%d", + scopeMatchCounters.unique.Load(), scopeMatchCounters.explicitOverDerived.Load(), + scopeMatchCounters.ambiguous.Load(), scopeMatchCounters.none.Load()) +} + +// refreshFromStore reads the declared region names, ranks and caps them, and +// swaps in a new snapshot. Shared by startup and the ticker so both apply +// identical rules. +// +// A DB error is logged and the CURRENT snapshot is kept. That matters: an +// empty key set would silently unname all traffic, which looks exactly like +// the bug this feature exists to fix. +func (s *regionKeySet) refreshFromStore(store *Store) { + if s == nil || !s.enabled { + return + } + stats, err := store.DeclaredRegionStats() + if err != nil { + log.Printf("[regions] derived-key refresh failed, keeping %d existing key(s): %v", len(s.snapshot().all), err) + return + } + ranked := rankDeclaredRegions(stats, s.max) + added := s.refreshDerived(ranked) + snap := s.snapshot() + log.Printf("[regions] derived-key refresh: %d name(s) declared, %d kept after filter+cap(%d), %d total key(s) in force", + len(stats), len(ranked), s.max, len(snap.all)) + if len(added) > 0 { + log.Printf("[regions] derived keys now active: %v", added) + } + if len(stats) > s.max { + log.Printf("[regions] NOTE: %d declared name(s) exceeded maxDerived=%d and were dropped, least-declared first", len(stats)-s.max, s.max) + } +} diff --git a/cmd/ingestor/region_keys_test.go b/cmd/ingestor/region_keys_test.go new file mode 100644 index 000000000..607dd346f --- /dev/null +++ b/cmd/ingestor/region_keys_test.go @@ -0,0 +1,485 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" + "testing" +) + +func TestRegionNameAcceptable(t *testing.T) { + cases := []struct { + name string + want bool + why string + }{ + {"be", true, "ordinary short name"}, + {"nl-li-sit", true, "hyphenated hierarchical name"}, + {"fm-112", true, "digits are fine"}, + {"null", true, "looks like junk but is a legal name — no value blocklist"}, + {"", false, "empty"}, + {strings.Repeat("a", 33), false, "over the 32-char limit"}, + {strings.Repeat("a", 32), true, "exactly at the limit"}, + {"be,eu", false, "a comma is the regions_csv delimiter and would split on reload"}, + {"#be", false, "the firmware strips '#', so its presence signals a malformed entry"}, + {"be\x00", false, "NUL padding that a stale client failed to trim"}, + {"be eu", false, "whitespace inside a region name is never emitted by firmware"}, + {"bé", false, "non-ASCII: the key is SHA256 over bytes, so encoding drift would silently mismatch"}, + } + for _, c := range cases { + if got := regionNameAcceptable(c.name); got != c.want { + t.Errorf("regionNameAcceptable(%q) = %v, want %v — %s", c.name, got, c.want, c.why) + } + } +} + +func TestRankDeclaredRegionsPrefersWidelyDeclared(t *testing.T) { + // The cap must drop the long tail of one-off local names, never a region + // half the network declares. On live data "be" is declared by 127 + // repeaters and "behss" by 3. + stats := []declaredRegionStat{ + {Name: "behss", Declarers: 3, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "be", Declarers: 127, LastSeen: "2026-09-01T10:00:00Z"}, + {Name: "sol3", Declarers: 1, LastSeen: "2026-09-07T11:00:00Z"}, + } + got := rankDeclaredRegions(stats, 2) + want := []string{"be", "behss"} + if len(got) != len(want) { + t.Fatalf("rankDeclaredRegions = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("rankDeclaredRegions = %v, want %v — declarer count must dominate recency", got, want) + } + } +} + +func TestRankDeclaredRegionsIsDeterministic(t *testing.T) { + // Equal declarer counts and equal timestamps must still produce a stable + // order, or the derived tier churns between refreshes and the add/drop + // logging becomes noise. + stats := []declaredRegionStat{ + {Name: "zz", Declarers: 2, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "aa", Declarers: 2, LastSeen: "2026-09-07T10:00:00Z"}, + } + for i := 0; i < 20; i++ { + got := rankDeclaredRegions(stats, 10) + if got[0] != "aa" || got[1] != "zz" { + t.Fatalf("run %d: rankDeclaredRegions = %v, want [aa zz]", i, got) + } + } +} + +func TestRankDeclaredRegionsBreaksTiesOnRecency(t *testing.T) { + stats := []declaredRegionStat{ + {Name: "old", Declarers: 2, LastSeen: "2026-01-01T00:00:00Z"}, + {Name: "new", Declarers: 2, LastSeen: "2026-09-07T00:00:00Z"}, + } + got := rankDeclaredRegions(stats, 1) + if len(got) != 1 || got[0] != "new" { + t.Fatalf("rankDeclaredRegions = %v, want [new] — equal declarers break on recency", got) + } +} + +func TestRankDeclaredRegionsDropsUnacceptableNames(t *testing.T) { + stats := []declaredRegionStat{ + {Name: "be", Declarers: 5, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "#bad", Declarers: 99, LastSeen: "2026-09-07T10:00:00Z"}, + } + got := rankDeclaredRegions(stats, 10) + if len(got) != 1 || got[0] != "be" { + t.Fatalf("rankDeclaredRegions = %v, want [be] — an unacceptable name must be dropped however widely declared", got) + } +} + +func TestSplitDeclaredRegionsCSV(t *testing.T) { + // Exact inverse of the strings.Join the ingestor writes the column with. + got := splitDeclaredRegionsCSV(" be , eu ,, nl ") + want := []string{"be", "eu", "nl"} + if len(got) != len(want) { + t.Fatalf("splitDeclaredRegionsCSV = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("splitDeclaredRegionsCSV = %v, want %v", got, want) + } + } + if n := len(splitDeclaredRegionsCSV("")); n != 0 { + t.Errorf("empty csv produced %d entries, want 0", n) + } +} + +func TestRegionKeySetExplicitOnlyWhenDisabled(t *testing.T) { + // Derivation off: the snapshot must be exactly what loadRegionKeys built, + // and refreshDerived must be a no-op rather than a quiet opt-in. + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss", "fm-112"}) + + snap := set.snapshot() + if len(snap.all) != 1 { + t.Fatalf("len(all) = %d, want 1 — refreshDerived must not add keys when disabled", len(snap.all)) + } + if _, ok := snap.all["#be"]; !ok { + t.Error("want the explicit #be key present") + } + if !snap.isExplicit("#be") { + t.Error("isExplicit(#be) = false, want true") + } +} + +func TestRegionKeySetMergesDerivedWhenEnabled(t *testing.T) { + cfg := &Config{ + HashRegions: []string{"#be"}, + AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}, + } + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss", "be"}) // "be" duplicates the explicit key + + snap := set.snapshot() + if len(snap.all) != 2 { + t.Fatalf("len(all) = %d, want 2 (#be explicit + #behss derived), got keys %v", len(snap.all), keyNames(snap)) + } + if _, ok := snap.all["#behss"]; !ok { + t.Errorf("want the derived #behss key present, got %v", keyNames(snap)) + } + if snap.isExplicit("#behss") { + t.Error("isExplicit(#behss) = true, want false — a derived key is not operator config") + } + if !snap.isExplicit("#be") { + t.Error("isExplicit(#be) = false, want true — an explicit key must not be demoted by a duplicate declaration") + } +} + +func TestRegionKeySetRefreshReplacesRatherThanAccumulates(t *testing.T) { + // A region that stops being declared must leave the derived tier, or the + // key set only ever grows and the cap stops meaning anything. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa"}) + set.refreshDerived([]string{"bb"}) + + snap := set.snapshot() + if _, ok := snap.all["#aa"]; ok { + t.Error("want #aa gone after a refresh that no longer lists it") + } + if _, ok := snap.all["#bb"]; !ok { + t.Error("want #bb present after the refresh that lists it") + } +} + +func TestRegionKeySetSnapshotIsStable(t *testing.T) { + // A snapshot handed to a packet must not change under it mid-match. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa"}) + held := set.snapshot() + set.refreshDerived([]string{"bb"}) + + if _, ok := held.all["#aa"]; !ok { + t.Error("the held snapshot lost #aa — snapshots must be immutable, not aliases of live state") + } +} + +// keyNames is a test helper for readable failure messages. +func keyNames(s *regionKeySnapshot) []string { + out := make([]string, 0, len(s.all)) + for k := range s.all { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// codeFor derives the on-wire code1 a sender in region `name` would emit for +// this payload - the same computation matchingRegions inverts. Used to build +// packets that genuinely belong to a region rather than asserting on a +// hardcoded string. +func codeFor(name string, payloadType byte, payload []byte) string { + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + sum := sha256.Sum256([]byte(name)) + mac := hmac.New(sha256.New, sum[:16]) + mac.Write([]byte{payloadType}) + mac.Write(payload) + h := mac.Sum(nil) + code := uint16(h[0]) | uint16(h[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return strings.ToUpper(hex.EncodeToString([]byte{byte(code & 0xFF), byte(code >> 8)})) +} + +func TestScopeMatchUniqueNamesTheRegion(t *testing.T) { + payload := []byte{0xDE, 0xAD, 0xBE, 0xEF} + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + got := set.snapshot().match(5, payload, code) + if got.Name != "#be" { + t.Errorf("Name = %q, want %q", got.Name, "#be") + } + if got.Reason != scopeReasonUnique { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonUnique) + } +} + +func TestScopeMatchNoKeyMatches(t *testing.T) { + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + + got := set.snapshot().match(5, []byte{1, 2, 3}, "0000") + if got.Name != "" || got.Reason != scopeReasonNone { + t.Errorf("got %+v, want an empty name with reason %q", got, scopeReasonNone) + } +} + +func TestScopeMatchExplicitBeatsDerived(t *testing.T) { + // The ambiguity this feature introduces: a derived key collides with an + // operator-configured one on this payload. Operator config wins - it is + // intent, the derived name is hearsay picked up over RF. + payload := []byte{0x01, 0x02, 0x03, 0x04} + cfg := &Config{HashRegions: []string{"#be"}, AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + // Force the collision rather than searching for a natural one: inject a + // derived key whose bytes are the explicit key's, so both match. + snap := set.snapshot() + collide := make(map[string][]byte, len(snap.all)+1) + for k, v := range snap.all { + collide[k] = v + } + collide["#collider"] = snap.all["#be"] + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + + got := forced.match(5, payload, code) + if got.Name != "#be" { + t.Errorf("Name = %q, want %q - the explicit key must win", got.Name, "#be") + } + if got.Reason != scopeReasonExplicitOverDerived { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonExplicitOverDerived) + } + if len(got.Candidates) != 2 { + t.Errorf("Candidates = %v, want both names recorded for the log", got.Candidates) + } +} + +func TestScopeMatchTwoExplicitKeysStayAmbiguous(t *testing.T) { + // Two equally-sourced candidates: naming either would be a guess, and + // naming wrongly is worse than not naming. This is #1609's rule, unchanged. + payload := []byte{0x09, 0x08, 0x07} + cfg := &Config{HashRegions: []string{"#be", "#eu"}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + snap := set.snapshot() + collide := map[string][]byte{"#be": snap.all["#be"], "#eu": snap.all["#be"]} + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + + got := forced.match(5, payload, code) + if got.Name != "" { + t.Errorf("Name = %q, want empty - two explicit candidates must abstain", got.Name) + } + if got.Reason != scopeReasonAmbiguous { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonAmbiguous) + } +} + +func TestScopeMatchTwoDerivedKeysStayAmbiguous(t *testing.T) { + // The tier-3 case, deliberately NOT resolved in M2. It must abstain rather + // than pick, and the reason must say ambiguous so the log can measure how + // often this happens before tier 3 is built. + payload := []byte{0x11, 0x22} + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa", "bb"}) + + snap := set.snapshot() + collide := map[string][]byte{"#aa": snap.all["#aa"], "#bb": snap.all["#aa"]} + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + code := codeFor("#aa", 5, payload) + + got := forced.match(5, payload, code) + if got.Name != "" || got.Reason != scopeReasonAmbiguous { + t.Errorf("got %+v, want an empty name with reason %q", got, scopeReasonAmbiguous) + } +} + +// regionSetFromKeys wraps a raw key map as a *regionKeySet with every key +// treated as explicit. Tests written before M2's two-tier type keep working +// unchanged this way, and "all keys explicit" is precisely what a hashRegions +// map meant back then - so the #1609 ambiguity semantics they assert are +// preserved exactly: two explicit candidates still abstain. +func regionSetFromKeys(keys map[string][]byte) *regionKeySet { + names := make(map[string]bool, len(keys)) + all := make(map[string][]byte, len(keys)) + for n, k := range keys { + names[n] = true + all[n] = k + } + s := ®ionKeySet{} + s.cur.Store(®ionKeySnapshot{all: all, explicit: names}) + return s +} + +// matchScopeName reproduces the removed matchScope's signature over the new +// type, so the tests written against it keep asserting the behaviour they were +// written for rather than being rewritten alongside the change they guard. +func matchScopeName(keys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string { + return regionSetFromKeys(keys).snapshot().match(payloadType, payloadRaw, code1).Name +} + +func TestDeclaredRegionStatsAggregatesLatestAnswerPerTarget(t *testing.T) { + store := newTestStore(t) + // Two answers from the same target: only the newer one counts, exactly as + // CurrentDeclaredRegions orders (by observed_at, never ingested_at - a + // drive buffered offline can arrive days late). + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-01T00:00:00Z", "be,old") + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "be,new") + insertDeclaredRegionsRow(t, store, "bb"+strings.Repeat("22", 31), "2026-09-05T00:00:00Z", "be") + + stats, err := store.DeclaredRegionStats() + if err != nil { + t.Fatal(err) + } + byName := map[string]declaredRegionStat{} + for _, s := range stats { + byName[s.Name] = s + } + if got := byName["be"].Declarers; got != 2 { + t.Errorf("be declarers = %d, want 2", got) + } + if got := byName["be"].LastSeen; got != "2026-09-07T00:00:00Z" { + t.Errorf("be lastSeen = %q, want the greatest observed_at", got) + } + if _, ok := byName["old"]; ok { + t.Error("want the superseded answer's region gone - only the latest answer per target counts") + } + if got := byName["new"].Declarers; got != 1 { + t.Errorf("new declarers = %d, want 1", got) + } +} + +func TestDeclaredRegionStatsIgnoresWildcard(t *testing.T) { + // '*' is the wildcard, not a region name. Deriving a key for it would add + // a permanent no-op entry to the cap on nearly every deployment. + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "cc"+strings.Repeat("33", 31), "2026-09-07T00:00:00Z", "*,be") + stats, err := store.DeclaredRegionStats() + if err != nil { + t.Fatal(err) + } + for _, s := range stats { + if s.Name == "*" { + t.Error("want '*' excluded - it is the wildcard, not a region") + } + } + if len(stats) != 1 { + t.Errorf("stats = %+v, want just be", stats) + } +} + +// insertDeclaredRegionsRow seeds one node_declared_regions answer. +func insertDeclaredRegionsRow(t *testing.T, s *Store, target, observedAt, regionsCSV string) { + t.Helper() + _, err := s.db.Exec( + `INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated) + VALUES (?, 'rx', ?, ?, ?, 0)`, + target, observedAt, observedAt, regionsCSV) + if err != nil { + t.Fatal(err) + } +} + +func TestRefreshFromStoreIsNoOpWhenDisabled(t *testing.T) { + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "behss") + + cfg := &Config{HashRegions: []string{"#be"}} // autoRegionKeys absent + set := newRegionKeySet(cfg) + before := len(set.snapshot().all) + set.refreshFromStore(store) + + if got := len(set.snapshot().all); got != before { + t.Errorf("key count %d -> %d with autoRegionKeys off, want unchanged", before, got) + } + if _, ok := set.snapshot().all["#behss"]; ok { + t.Error("a declared name became a key with the feature disabled - this is the safety property the default-off promise rests on") + } +} + +func TestRefreshFromStoreDerivesWhenEnabled(t *testing.T) { + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "behss,fm-112") + + cfg := &Config{HashRegions: []string{"#be"}, AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshFromStore(store) + + snap := set.snapshot() + for _, want := range []string{"#be", "#behss", "#fm-112"} { + if _, ok := snap.all[want]; !ok { + t.Errorf("want %s in force, got %v", want, keyNames(snap)) + } + } + if !snap.isExplicit("#be") || snap.isExplicit("#behss") { + t.Error("tiers crossed: #be must stay explicit, #behss must be derived") + } +} + +func TestRefreshFromStoreHonoursTheCap(t *testing.T) { + store := newTestStore(t) + // Three names, one declared twice so the ranking is not a coin flip. + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "wide,narrow1") + insertDeclaredRegionsRow(t, store, "bb"+strings.Repeat("22", 31), "2026-09-07T00:00:00Z", "wide,narrow2") + + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true, MaxDerived: 1}} + set := newRegionKeySet(cfg) + set.refreshFromStore(store) + + snap := set.snapshot() + if len(snap.all) != 1 { + t.Fatalf("keys = %v, want exactly 1 under maxDerived=1", keyNames(snap)) + } + if _, ok := snap.all["#wide"]; !ok { + t.Errorf("keys = %v, want the twice-declared name kept, not a one-off", keyNames(snap)) + } +} + +// BenchmarkScopeMatch sweeps key-set size because the cost is linear in it and +// cannot be reduced: code1 is an HMAC over the packet payload, so there is no +// payload-independent lookup key to index on. The sweep is the evidence for +// choosing maxDerived, not a single before/after number - the explicit tier's +// size is operator config and varies per deployment. +func BenchmarkScopeMatch(b *testing.B) { + payload := make([]byte, 51) // a typical GRP_TXT payload + for i := range payload { + payload[i] = byte(i) + } + for _, n := range []int{16, 58, 180, 314} { + b.Run(fmt.Sprintf("keys=%d", n), func(b *testing.B) { + all := make(map[string][]byte, n) + explicit := make(map[string]bool, n) + for i := 0; i < n; i++ { + name := fmt.Sprintf("#r%04d", i) + sum := sha256.Sum256([]byte(name)) + all[name] = sum[:16] + explicit[name] = true + } + snap := ®ionKeySnapshot{all: all, explicit: explicit} + code := codeFor("#r0000", 5, payload) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = snap.match(5, payload, code) + } + }) + } +} diff --git a/cmd/ingestor/scope_repair.go b/cmd/ingestor/scope_repair.go index 273e77a0d..6ad2b034c 100644 --- a/cmd/ingestor/scope_repair.go +++ b/cmd/ingestor/scope_repair.go @@ -64,7 +64,7 @@ type scopeDerivation struct { // — the same helper matchScope itself uses. channelKeys is nil and // validateSignatures is false because region matching depends on neither — // only on the undecrypted payload bytes. -func rederiveScope(rawHex string, regionKeys map[string][]byte) (scopeDerivation, error) { +func rederiveScope(rawHex string, snap *regionKeySnapshot) (scopeDerivation, error) { decoded, err := DecodePacket(rawHex, nil, false) if err != nil { return scopeDerivation{}, err @@ -72,7 +72,7 @@ func rederiveScope(rawHex string, regionKeys map[string][]byte) (scopeDerivation if decoded.TransportCodes == nil || decoded.TransportCodes.Code1 == "0000" { return scopeDerivation{State: scopeState{Valid: false}}, nil } - matched := matchingRegions(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + matched := matchingRegions(snap.all, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) name := "" if len(matched) == 1 { name = matched[0] @@ -137,7 +137,7 @@ type scopeRepairReport struct { // Running repairScopeNames(apply=true) twice in a row writes nothing the // second time: every row it just wrote now re-derives to the state it // holds, which is the Unchanged case. -func repairScopeNames(db *sql.DB, regionKeys map[string][]byte, apply bool) (*scopeRepairReport, error) { +func repairScopeNames(db *sql.DB, snap *regionKeySnapshot, apply bool) (*scopeRepairReport, error) { rows, err := db.Query(`SELECT id, raw_hex, scope_name FROM transmissions WHERE scope_name IS NOT NULL ORDER BY id`) if err != nil { return nil, fmt.Errorf("query transport-scoped rows: %w", err) @@ -165,7 +165,7 @@ func repairScopeNames(db *sql.DB, regionKeys map[string][]byte, apply bool) (*sc report.NamedBefore++ } - d, err := rederiveScope(rawHex, regionKeys) + d, err := rederiveScope(rawHex, snap) if err != nil { report.DecodeFailed++ continue @@ -287,15 +287,22 @@ func runScopeRepair(args []string) int { if *dbPathOverride != "" { dbPath = *dbPathOverride } - regionKeys := loadRegionKeys(cfg) - store, err := OpenStore(dbPath) if err != nil { log.Fatalf("scope-repair: db: %v", err) } defer store.Close() - report, err := repairScopeNames(store.db, regionKeys, *apply) + // The derived tier must be rebuilt before scanning. Repairing against the + // explicit tier alone would find no key for any automatically-named row, + // classify it as "named -> unmatched", and erase the name - turning a + // maintenance tool into data loss. + regionSet := newRegionKeySet(cfg) + regionSet.refreshFromStore(store) + snap := regionSet.snapshot() + log.Printf("scope-repair: %d region key(s) in force", len(snap.all)) + + report, err := repairScopeNames(store.db, snap, *apply) if err != nil { log.Fatalf("scope-repair: %v", err) } diff --git a/cmd/ingestor/scope_repair_test.go b/cmd/ingestor/scope_repair_test.go index 6372dc2c9..2d066d009 100644 --- a/cmd/ingestor/scope_repair_test.go +++ b/cmd/ingestor/scope_repair_test.go @@ -110,7 +110,7 @@ func TestScopeRepairDryRun(t *testing.T) { store := newScopeRepairFixture(t) regionKeys := scopeRepairTestKeys(t) - report, err := repairScopeNames(store.db, regionKeys, false) + report, err := repairScopeNames(store.db, regionSetFromKeys(regionKeys).snapshot(), false) if err != nil { t.Fatalf("repairScopeNames: %v", err) } @@ -155,7 +155,7 @@ func TestScopeRepairApply(t *testing.T) { store := newScopeRepairFixture(t) regionKeys := scopeRepairTestKeys(t) - report1, err := repairScopeNames(store.db, regionKeys, true) + report1, err := repairScopeNames(store.db, regionSetFromKeys(regionKeys).snapshot(), true) if err != nil { t.Fatalf("repairScopeNames (apply): %v", err) } @@ -172,7 +172,7 @@ func TestScopeRepairApply(t *testing.T) { assertScopeName(t, store, fixtureRawD, "", false) // untouched: not transport-scoped, stays NULL assertScopeName(t, store, fixtureRawE, "#ghost", true) // untouched: unexpected, not applied - report2, err := repairScopeNames(store.db, regionKeys, true) + report2, err := repairScopeNames(store.db, regionSetFromKeys(regionKeys).snapshot(), true) if err != nil { t.Fatalf("repairScopeNames (second apply): %v", err) } @@ -227,7 +227,7 @@ func newScopeRepairUnnamedFixture(t *testing.T) *Store { func TestScopeRepairDryRunReportsNewlyMatchableRows(t *testing.T) { store := newScopeRepairUnnamedFixture(t) - report, err := repairScopeNames(store.db, scopeRepairTestKeys(t), false) + report, err := repairScopeNames(store.db, regionSetFromKeys(scopeRepairTestKeys(t)).snapshot(), false) if err != nil { t.Fatalf("repairScopeNames: %v", err) } @@ -255,7 +255,7 @@ func TestScopeRepairApplyNamesNewlyMatchableRows(t *testing.T) { store := newScopeRepairUnnamedFixture(t) regionKeys := scopeRepairTestKeys(t) - report1, err := repairScopeNames(store.db, regionKeys, true) + report1, err := repairScopeNames(store.db, regionSetFromKeys(regionKeys).snapshot(), true) if err != nil { t.Fatalf("repairScopeNames (apply): %v", err) } @@ -267,7 +267,7 @@ func TestScopeRepairApplyNamesNewlyMatchableRows(t *testing.T) { assertScopeName(t, store, fixtureRawG, "", true) // untouched: still ambiguous assertScopeName(t, store, fixtureRawH, "", true) // untouched: still matches nothing - report2, err := repairScopeNames(store.db, regionKeys, true) + report2, err := repairScopeNames(store.db, regionSetFromKeys(regionKeys).snapshot(), true) if err != nil { t.Fatalf("repairScopeNames (second apply): %v", err) } @@ -311,3 +311,31 @@ func TestScopeRepairReportCountsBothDirections(t *testing.T) { } } } + +// TestScopeRepairKeepsDerivedNames: a row named from a derived key must survive +// a repair run. If rederiveScope sees only the explicit tier it reports +// MatchCount 0, which lands in the "named -> unmatched" branch and wipes the +// name. This test is the guard against that, and the failure it prevents is +// data loss from a maintenance tool, not a cosmetic gap. +func TestScopeRepairKeepsDerivedNames(t *testing.T) { + payload := []byte{0x42, 0x43, 0x44} + // A transport-flood packet: header 0x14 (route 0, payload type 5), + // code1/code2, path byte 0x41 (hash_size 2, one hop), hop, then payload. + code1 := codeFor("#behss", 5, payload) + rawHex := "14" + code1 + "0000" + "41" + "E3D3" + strings.ToUpper(hex.EncodeToString(payload)) + + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss"}) + + got, err := rederiveScope(rawHex, set.snapshot()) + if err != nil { + t.Fatal(err) + } + if got.State.Name != "#behss" { + t.Errorf("State.Name = %q, want %q — a derived key must name the row during repair", got.State.Name, "#behss") + } + if got.MatchCount != 1 { + t.Errorf("MatchCount = %d, want 1", got.MatchCount) + } +} diff --git a/cmd/server/routes.go b/cmd/server/routes.go index ac2ff7061..5de258dd5 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -20,6 +20,7 @@ import ( "github.com/gorilla/mux" "github.com/meshcore-analyzer/packetpath" "github.com/meshcore-analyzer/prunequeue" + "golang.org/x/sync/singleflight" ) // memBreakdownNote is the static accounting caveat attached to the opt-in @@ -65,11 +66,14 @@ type Server struct { scopeStatsCache map[string]*ScopeStatsResponse scopeStatsCachedAt map[string]time.Time - // Cached /api/scope-audit response — per-window, recomputed at most once - // every 30s, mirroring scopeStats above. See scopes.go. + // Cached /api/scope-audit response — per-window, with a singleflight so a + // cold key costs one scan no matter how many requests arrive on it. See + // scopeAuditTTLFor for why the 7d window's TTL is not the 30s the others + // use, and scopes.go for the scan itself. scopeAuditMu sync.Mutex scopeAuditCache map[string]*ScopeAuditResponse scopeAuditCachedAt map[string]time.Time + scopeAuditSF singleflight.Group // Router reference for OpenAPI spec generation router *mux.Router @@ -3529,8 +3533,6 @@ func (s *Server) handleScopeStats(w http.ResponseWriter, r *http.Request) { // pass — see scopes.go's AllCurrentDeclaredRegions and ScopeAuditForwarding // for why that stays a single scan rather than one query per repeater. func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { - const scopeAuditTTL = 30 * time.Second - window := r.URL.Query().Get("window") if window == "" { window = "24h" @@ -3541,34 +3543,124 @@ func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { return } - s.scopeAuditMu.Lock() - if s.scopeAuditCache != nil { - if cached, ok := s.scopeAuditCache[window]; ok && time.Since(s.scopeAuditCachedAt[window]) < scopeAuditTTL { - s.scopeAuditMu.Unlock() - writeJSON(w, cached) - return - } + sinceISO := time.Now().Add(-lookback).UTC().Format(time.RFC3339) + + if cached, ok := s.scopeAuditCached(window); ok { + writeJSON(w, cached) + return } - s.scopeAuditMu.Unlock() - declared, err := s.db.AllCurrentDeclaredRegions() + // singleflight: the compute below runs outside the cache mutex, so without + // this every request arriving on a cold window ran its own full scan + // concurrently. On the 7d window that scan is seconds of work over millions + // of hop rows, which is exactly the shape that makes a thundering herd + // expensive rather than merely wasteful. Same treatment /api/observers and + // /api/nodes/{pubkey}/reach already have. + v, err, _ := s.scopeAuditSF.Do(window, func() (interface{}, error) { + // The waiters that arrive while a scan is in flight are served by that + // scan's result; this second look is for the caller that acquires the + // group right after a winner stored one. + if cached, ok := s.scopeAuditCached(window); ok { + return cached, nil + } + resp, cErr := s.computeScopeAudit(window, sinceISO) + if cErr != nil { + return nil, cErr + } + s.scopeAuditStore(window, resp) + return resp, nil + }) if err != nil { writeError(w, 500, err.Error()) return } + writeJSON(w, v.(*ScopeAuditResponse)) +} + +// scopeAuditTTLFor is how long one window's computed audit stays fresh. +// +// 7d is not 30s because it does not cost what the others cost. Measured on the +// live-shaped staging database on 2026-09-07: 16.7s cold for 7d against 4.0s +// for 24h and 0.15s for 1h, and the 7d scan reads 3,470,188 hop rows. At a 30s +// TTL a single reader with that window open keeps the instance recomputing more +// than half the time, for an aggregate that moves at the pace of a week of +// traffic. Five minutes of staleness on a seven-day window is not a fact the +// reader can act on differently. +func scopeAuditTTLFor(window string) time.Duration { + if window == "7d" { + return 5 * time.Minute + } + return 30 * time.Second +} + +// scopeAuditCached returns the cached response for a window while it is within +// that window's TTL. +func (s *Server) scopeAuditCached(window string) (*ScopeAuditResponse, bool) { + s.scopeAuditMu.Lock() + defer s.scopeAuditMu.Unlock() + if s.scopeAuditCache == nil { + return nil, false + } + cached, ok := s.scopeAuditCache[window] + if !ok || time.Since(s.scopeAuditCachedAt[window]) >= scopeAuditTTLFor(window) { + return nil, false + } + return cached, true +} + +// scopeAuditStore publishes a freshly computed response for a window. +func (s *Server) scopeAuditStore(window string, resp *ScopeAuditResponse) { + s.scopeAuditMu.Lock() + defer s.scopeAuditMu.Unlock() + if s.scopeAuditCache == nil { + s.scopeAuditCache = make(map[string]*ScopeAuditResponse) + s.scopeAuditCachedAt = make(map[string]time.Time) + } + s.scopeAuditCache[window] = resp + s.scopeAuditCachedAt[window] = time.Now() +} + +// computeScopeAudit builds one window's audit response: the declared lists, the +// forwarding evidence attributed to them, and the declared-region verification +// that settles which unnameable traffic corroborates a declaration. Split out +// of the handler so the cache and its singleflight wrap a plain function +// instead of a request. +func (s *Server) computeScopeAudit(window, sinceISO string) (*ScopeAuditResponse, error) { + declared, err := s.db.AllCurrentDeclaredRegions() + if err != nil { + return nil, err + } targets := make([]string, 0, len(declared)) for _, d := range declared { targets = append(targets, strings.ToLower(d.Target)) } - sinceISO := time.Now().Add(-lookback).UTC().Format(time.RFC3339) forwarding := map[string]*scopeAuditTargetAgg{} if s.store != nil { forwarding, err = s.store.ScopeAuditForwarding(sinceISO, targets) if err != nil { - writeError(w, 500, err.Error()) - return + return nil, err + } + } + + // Declared-region verification (M1b): a region this instance holds no key + // for is unnameable, not absent, and the audit can settle which by deriving + // the key from the repeater's own declaration and testing it against that + // repeater's own unnameable traffic. One verifier serves every row so each + // (region, transmission) pair is derived at most once — see scope_verify.go + // for why that memo is what keeps this affordable. + // + // A failure here degrades to "no verification" rather than failing the + // request: the audit was useful before this existed and must stay useful if + // the extra query errors. + var verifier *scopeVerifier + if s.store != nil { + unmatchedRows, uErr := s.store.unmatchedTransmissionsInWindow(sinceISO) + if uErr != nil { + log.Printf("[scope-audit] declared-region verification unavailable: %v", uErr) + } else { + verifier = newScopeVerifier(unmatchedRows) } } @@ -3608,9 +3700,28 @@ func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { agg := forwarding[pk] - notObserved := []string{} + // Verify the declared regions this repeater has no NAMED evidence for, + // against its own unmatched traffic. Regions already observed by name + // need no verification and are not tested — that keeps the candidate + // set to exactly the open questions, which is also what keeps the + // verifier's work proportional to the problem rather than to the fleet. + unnamed := []string{} for _, rgn := range declaredNamed { if agg == nil || agg.scopes[rgn] == nil { + unnamed = append(unnamed, rgn) + } + } + regionEvidence := map[string]int{} + verifiedSet := map[string]bool{} + if verifier != nil && agg != nil && len(unnamed) > 0 { + regionEvidence = verifier.evidence(agg.unmatchedTxIDs, unnamed) + for _, rgn := range verifier.verified(regionEvidence) { + verifiedSet[rgn] = true + } + } + notObserved := []string{} + for _, rgn := range unnamed { + if !verifiedSet[rgn] { notObserved = append(notObserved, rgn) } } @@ -3629,26 +3740,29 @@ func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { } } - var unscopedPackets, ambiguousHops int64 + var unscopedPackets, ambiguousHops, unmatchedPackets int64 if agg != nil { unscopedPackets = agg.unscopedPackets ambiguousHops = agg.ambiguousHops + unmatchedPackets = agg.unmatchedPackets } resp.Repeaters = append(resp.Repeaters, ScopeAuditRow{ - PublicKey: pk, - Name: id.Name, - Role: id.Role, - DeclaredRegions: declaredNamed, - DeclaredWildcard: declaredWildcard, - ConfigState: scopeAuditConfigState(declaredNamed, declaredWildcard), - DeclaredAt: d.ObservedAt, - Truncated: d.Truncated, - NotObserved: notObserved, - UndeclaredObserved: undeclared, - ObservedUnscopedPackets: unscopedPackets, - WildcardContradiction: unscopedPackets > 0 && !declaredWildcard, - AmbiguousHops: ambiguousHops, + PublicKey: pk, + Name: id.Name, + Role: id.Role, + DeclaredRegions: declaredNamed, + DeclaredWildcard: declaredWildcard, + ConfigState: scopeAuditConfigState(declaredNamed, declaredWildcard), + DeclaredAt: d.ObservedAt, + Truncated: d.Truncated, + NotObserved: notObserved, + UndeclaredObserved: undeclared, + ObservedUnscopedPackets: unscopedPackets, + WildcardContradiction: unscopedPackets > 0 && !declaredWildcard, + AmbiguousHops: ambiguousHops, + ObservedUnmatchedPackets: unmatchedPackets, + RegionEvidence: regionEvidence, }) } @@ -3682,16 +3796,7 @@ func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { return an < bn }) - s.scopeAuditMu.Lock() - if s.scopeAuditCache == nil { - s.scopeAuditCache = make(map[string]*ScopeAuditResponse) - s.scopeAuditCachedAt = make(map[string]time.Time) - } - s.scopeAuditCache[window] = resp - s.scopeAuditCachedAt[window] = time.Now() - s.scopeAuditMu.Unlock() - - writeJSON(w, resp) + return resp, nil } // handlePruneGeoFilter identifies (dry_run=true, default) or enqueues (confirm=true) diff --git a/cmd/server/scope_verify.go b/cmd/server/scope_verify.go new file mode 100644 index 000000000..78123d012 --- /dev/null +++ b/cmd/server/scope_verify.go @@ -0,0 +1,274 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" +) + +// scopeVerifyMaxPacketsPerTarget bounds the per-target evidence list. AGENTS.md +// rule 0 forbids unbounded structures, and the corroboration threshold is 2 — +// past a few hundred packets more evidence changes no verdict, it only costs +// memory. scopeAuditTargetAgg.unmatchedPackets keeps counting past this: the +// count is the honest total, the list is the working set. +const scopeVerifyMaxPacketsPerTarget = 512 + +// scopeHMACInputs pulls the three values needed to test a region hypothesis +// against one packet: the payload type and raw payload bytes the sender HMACed, +// and the resulting two-byte code it put on the wire. +// +// It deliberately does NOT call DecodePacket. That runs decodePayload, which +// attempts channel decryption and signature validation — work this has no use +// for, repeated over every unmatched packet on every audit refresh. Walking the +// offsets is all that is needed, and it reuses decodeHeader/isTransportRoute/ +// decodePath so the offset arithmetic is not duplicated from DecodePacket. +// +// ok is false for anything that cannot carry a region scope: malformed hex, a +// truncated header, an invalid path byte, or a non-transport route. A plain +// FLOOD packet has no transport codes at all, so there is no code1 to compare +// against and HMACing it could only waste time. +func scopeHMACInputs(rawHex string) (payloadType byte, payload []byte, code1 string, ok bool) { + buf, err := hex.DecodeString(strings.TrimSpace(rawHex)) + if err != nil || len(buf) < 2 { + return 0, nil, "", false + } + header := decodeHeader(buf[0]) + if !isTransportRoute(header.RouteType) { + return 0, nil, "", false + } + offset := 1 + if len(buf) < offset+4 { + return 0, nil, "", false + } + code1 = strings.ToUpper(hex.EncodeToString(buf[offset : offset+2])) + offset += 4 // code1 and code2 + + if offset >= len(buf) { + return 0, nil, "", false + } + pathByte := buf[offset] + offset++ + _, consumed, decodeErr := decodePath(pathByte, buf, offset) + if decodeErr != nil { + return 0, nil, "", false + } + offset += consumed + if offset > len(buf) { + return 0, nil, "", false + } + rest := buf[offset:] + if len(rest) == 0 { + return 0, nil, "", false + } + return byte(header.PayloadType), rest, code1, true +} + +// regionCode derives the on-wire code1 a sender in region name would emit for +// this payload — the forward direction of what matchingRegions inverts in the +// ingestor (cmd/ingestor/main.go). The two must stay in step: key is +// SHA256("#name")[:16], the MAC covers payloadType followed by the payload, the +// code is the first two MAC bytes little-endian, and 0x0000/0xFFFF are reserved +// and nudged. Any divergence here silently produces regions that never verify. +// +// The leading '#' is optional because callers hold normScope'd names (the audit +// strips it) while the key is over the '#'-prefixed form. +// +// Case is significant and must stay so: the key is a hash over the raw bytes of +// "#name", so "#BEHSS" and "#behss" are different regions on the wire. +func regionCode(name string, payloadType byte, payload []byte) string { + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + sum := sha256.Sum256([]byte(name)) + mac := hmac.New(sha256.New, sum[:16]) + mac.Write([]byte{payloadType}) + mac.Write(payload) + h := mac.Sum(nil) + code := uint16(h[0]) | uint16(h[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return strings.ToUpper(hex.EncodeToString([]byte{byte(code & 0xFF), byte(code >> 8)})) +} + +// unmatchedTransmissionRow is one transmission that carried a transport scope +// no configured region key matched, with the raw bytes needed to test a region +// hypothesis against it. +type unmatchedTransmissionRow struct { + txID int64 + rawHex string +} + +// unmatchedTransmissionsInWindow is the SECOND, narrow query behind the audit - +// deliberately not a widening of scopeAuditForwarderScanQuery. +// +// That scan returns one row per hop per flood packet: on a 2,000-packet sample +// after M0 that is 19,049 rows, and carrying raw_hex on every one of them would +// load the hot path to serve a few hundred packets. This selects only the +// transmissions that are actually candidates - an empty scope_name inside the +// window, ~400 over 7 days on the reference deployment - and the main scan is +// left exactly as it is. +// +// An empty scope_name is the "transport-scoped but unnameable" state; NULL +// means the packet carried no scope at all and can never verify against a +// region. The route filter matches the forwarder scan's, so the two agree on +// which packets count as forwarded. +// +// "Empty" is spelled out above rather than written as the two-single-quote +// literal on purpose: gofmt applies the old godoc typographic substitution +// inside doc comments and rewrites that digraph into a closing curly quote, +// which silently misstates the one value this query keys on — and puts it back +// on every gofmt run. +// +// Selection only: a row whose raw_hex cannot be walked is still returned, and +// dropped by newScopeVerifier. Filtering that in SQL is not possible and +// filtering it here would hide how many candidates the window actually held. +func (s *PacketStore) unmatchedTransmissionsInWindow(sinceISO string) ([]unmatchedTransmissionRow, error) { + rows, err := s.db.conn.Query(` + SELECT t.id, t.raw_hex + FROM transmissions t + WHERE t.first_seen >= ? + AND t.scope_name = '' + AND `+scopeConformanceForwarderRouteTypesSQL, sinceISO) + if err != nil { + return nil, fmt.Errorf("unmatched transmissions scan: %w", err) + } + defer rows.Close() + + var out []unmatchedTransmissionRow + for rows.Next() { + var r unmatchedTransmissionRow + if err := rows.Scan(&r.txID, &r.rawHex); err != nil { + return nil, fmt.Errorf("unmatched transmissions scan row: %w", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("unmatched transmissions rows: %w", err) + } + return out, nil +} + +// scopeVerifyMinCorroboration is how many of a repeater's own unmatched packets +// must derive to a declared region before that region counts as observed. +// +// One is not enough, and the arithmetic is the whole argument: code1 is two +// bytes, so an unrelated name matches a given packet with probability 1/65536. +// Across ~400 unmatched packets and ~124 distinct declared names, chance alone +// produces roughly one false match per refresh. Two matches on the same region +// for the same repeater is (1/65536)^2 - about one in four billion. Raising +// this costs recall on quiet regions; lowering it to 1 makes the feature +// unsound, not merely noisy. +const scopeVerifyMinCorroboration = 2 + +// scopeVerifier answers "how many of these transmissions are region X" while +// deriving each region's code over each packet at most once. +// +// The cache is not a nicety, and where it has to sit was measured rather than +// guessed. Naively the audit does targets x declaredNames x unmatchedPackets +// HMACs — 205 x 124 x 400 is roughly 10,000,000. Caching per +// (region, transmission) pair cuts the HMACs to names x packets, ~50,000, but +// leaves the ITERATION cubic: a benchmark of that shape spent 501ms on 10.2M +// map lookups at ~49ns each, with the HMACs a rounding error beside it. +// +// So the cache is keyed per REGION, holding the set of transmissions that +// derive to it. A region is HMACed over every packet once, and a target then +// asks one question per declared region instead of one per (region, packet). +// The overwhelmingly common answer is an empty set — most declared regions +// match nothing — which costs a single lookup and no packet loop at all. +// +// Not safe for concurrent use: one verifier is built per audit computation, +// which handleScopeAudit already serialises behind its cache. +type scopeVerifier struct { + packets map[int64]scopeVerifyInputs + // matchesByRegion caches, per region, the transmissions that derive to it. + // Computed once over every packet, never per target. + matchesByRegion map[string]map[int64]bool + // hmacCount is incremented per actual derivation, asserted by the cache + // test so a future refactor cannot quietly reintroduce the naive cost. + hmacCount int +} + +type scopeVerifyInputs struct { + payloadType byte + payload []byte + code1 string + ok bool +} + +// newScopeVerifier parses each row once. A row whose raw_hex cannot be walked +// is kept with ok=false rather than dropped, so its id still resolves and a +// caller asking about it gets "no evidence" instead of a miss. +func newScopeVerifier(rows []unmatchedTransmissionRow) *scopeVerifier { + v := &scopeVerifier{ + packets: make(map[int64]scopeVerifyInputs, len(rows)), + matchesByRegion: map[string]map[int64]bool{}, + } + for _, r := range rows { + pt, payload, code1, ok := scopeHMACInputs(r.rawHex) + v.packets[r.txID] = scopeVerifyInputs{payloadType: pt, payload: payload, code1: code1, ok: ok} + } + return v +} + +// regionMatches returns the transmissions deriving to region, computing the +// whole set on first ask. Unparseable packets are skipped rather than counted +// as misses, so one malformed row in the window cannot blank a region. +func (v *scopeVerifier) regionMatches(region string) map[int64]bool { + if m, ok := v.matchesByRegion[region]; ok { + return m + } + m := map[int64]bool{} + for txID, in := range v.packets { + if !in.ok { + continue + } + v.hmacCount++ + if regionCode(region, in.payloadType, in.payload) == in.code1 { + m[txID] = true + } + } + v.matchesByRegion[region] = m + return m +} + +// evidence counts, for each declared region, how many of txIDs derive to it. +// Regions with zero matches are absent from the result rather than present +// with 0, so the map is directly the "we found something" set. +func (v *scopeVerifier) evidence(txIDs []int64, declaredRegions []string) map[string]int { + out := map[string]int{} + for _, region := range declaredRegions { + m := v.regionMatches(region) + if len(m) == 0 { + continue // the common case: one lookup, no packet loop + } + n := 0 + for _, txID := range txIDs { + if m[txID] { + n++ + } + } + if n > 0 { + out[region] = n + } + } + return out +} + +// verified returns the regions in an evidence map that clear the corroboration +// threshold, sorted so the response is stable across refreshes. +func (v *scopeVerifier) verified(evidence map[string]int) []string { + var out []string + for region, n := range evidence { + if n >= scopeVerifyMinCorroboration { + out = append(out, region) + } + } + sort.Strings(out) + return out +} diff --git a/cmd/server/scope_verify_test.go b/cmd/server/scope_verify_test.go new file mode 100644 index 000000000..3a69eb8e4 --- /dev/null +++ b/cmd/server/scope_verify_test.go @@ -0,0 +1,229 @@ +package main + +import ( + "encoding/hex" + "fmt" + "strings" + "testing" + "time" +) + +// realTransportFloodPacket is transmission 0a065d41d51f1f77 from the live +// instance, captured 2026-09-07. Header 0x14 = route_type 0 (TRANSPORT_FLOOD), +// payload_type 5 (GRP_TXT); transport codes 9209/0000; path byte 0x41 = +// hash_size 2, one hop "E3D3"; the rest is payload. +// +// A hand-built fixture would only prove the parser agrees with itself. This +// packet is the one that started the investigation: its code1 is exactly the +// code #fm-112 derives over its own payload, which is why the audit showed +// fm-112 as "not observed" for a repeater that was forwarding it. +const realTransportFloodPacket = "149209000041E3D3EC2D4481DA70893CD71B763958B064A9AAC011D54223FF8A0140CBB4093653BC61D67C960E3ECCE6639CC9FF1147AA6D0F9017" + +func TestScopeHMACInputsParsesRealPacket(t *testing.T) { + payloadType, payload, code1, ok := scopeHMACInputs(realTransportFloodPacket) + if !ok { + t.Fatal("scopeHMACInputs returned ok=false for a valid transport-flood packet") + } + if payloadType != 5 { + t.Errorf("payloadType = %d, want 5 (GRP_TXT)", payloadType) + } + if code1 != "9209" { + t.Errorf("code1 = %q, want %q", code1, "9209") + } + if len(payload) != 51 { + t.Errorf("len(payload) = %d, want 51", len(payload)) + } + if got := strings.ToUpper(hex.EncodeToString(payload[:4])); got != "EC2D4481" { + t.Errorf("payload starts %q, want %q — offset walked wrong", got, "EC2D4481") + } +} + +func TestScopeHMACInputsRejectsNonTransportRoutes(t *testing.T) { + // A plain FLOOD packet carries no transport codes, so it has no code1 to + // verify against. Returning ok=false rather than a zero code1 keeps the + // caller from HMACing packets that can never match anything. + // + // Header 0x15 = route_type 1 (FLOOD), payload_type 5. No transport codes, + // so the path byte follows the header directly. + _, _, _, ok := scopeHMACInputs("15" + "41" + "E3D3" + "AABBCC") + if ok { + t.Error("ok = true for a non-transport route, want false — there is no code1 to verify") + } +} + +func TestScopeHMACInputsRejectsMalformed(t *testing.T) { + for _, c := range []struct{ hex, why string }{ + {"", "empty"}, + {"zz", "not hex"}, + {"14", "header only, no transport codes"}, + {"1492090000", "transport codes but no path byte"}, + {"149209000041", "path byte claims one 2-byte hop, none present"}, + // pathByte 0xC0: upper two bits 11 -> hash_size 4, which firmware + // reserves and isValidPathLen rejects even at hash_count 0 + // (cmd/server/decoder.go, mirroring Packet.cpp:13-18). + {"1492090000C0" + strings.Repeat("00", 8), "hash_size 4 is reserved"}, + } { + if _, _, _, ok := scopeHMACInputs(c.hex); ok { + t.Errorf("ok = true for %q (%s), want false", c.hex, c.why) + } + } +} + +func TestRegionCodeMatchesTheRealPacket(t *testing.T) { + // The end-to-end arithmetic, against a packet whose true region is known. + payloadType, payload, code1, ok := scopeHMACInputs(realTransportFloodPacket) + if !ok { + t.Fatal("setup: scopeHMACInputs failed") + } + if got := regionCode("fm-112", payloadType, payload); got != code1 { + t.Errorf("regionCode(fm-112) = %q, want %q — this packet IS fm-112", got, code1) + } + // Both spellings must agree: the key is SHA256 over "#name", and callers + // hand us names with the '#' already stripped by normScope. + if got := regionCode("#fm-112", payloadType, payload); got != code1 { + t.Errorf("regionCode(#fm-112) = %q, want %q — leading '#' must be optional", got, code1) + } + // A region the repeater also declares, which this packet is NOT. + if got := regionCode("behss", payloadType, payload); got == code1 { + t.Errorf("regionCode(behss) = %q, must not equal fm-112's code1", got) + } +} + +func TestRegionCodeIsCaseSensitive(t *testing.T) { + // The key is SHA256 over the raw bytes of "#name", so "#BEHSS" and + // "#behss" are different regions. Folding case here would silently name + // traffic for a region nobody configured. + payloadType, payload, _, _ := scopeHMACInputs(realTransportFloodPacket) + if regionCode("behss", payloadType, payload) == regionCode("BEHSS", payloadType, payload) { + t.Error("regionCode folded case — the key is a hash over raw bytes and must not") + } +} + +func TestUnmatchedTransmissionsInWindow(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + old := "2020-01-01T00:00:00Z" + seedTransmissionRouteAt(t, s, "E3D3", scopeUnmatched(), RouteFlood, recent) + seedTransmissionRouteAt(t, s, "E3D3", scopeMatched("#be"), RouteFlood, recent) + seedTransmissionRouteAt(t, s, "E3D3", scopeUnscoped(), RouteFlood, recent) + seedTransmissionRouteAt(t, s, "E3D3", scopeUnmatched(), RouteFlood, old) + + since := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) + got, err := s.unmatchedTransmissionsInWindow(since) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("got %d rows, want 1 — only the recent scope_name='' row qualifies", len(got)) + } + // scopeUnmatched() seeds raw_hex 'AA', which scopeHMACInputs rejects. The + // query's job is selection; unparseable rows are dropped by the caller, so + // they must still be returned here rather than filtered in SQL. + if got[0].txID == 0 { + t.Error("txID = 0, want the transmission's real id") + } +} + +// buildVerifierFromPackets is a test helper: wraps raw hex strings as rows the +// verifier consumes, with ids 1..N in order. +func buildVerifierFromPackets(t *testing.T, hexes ...string) *scopeVerifier { + t.Helper() + rows := make([]unmatchedTransmissionRow, 0, len(hexes)) + for i, h := range hexes { + rows = append(rows, unmatchedTransmissionRow{txID: int64(i + 1), rawHex: h}) + } + return newScopeVerifier(rows) +} + +func TestScopeVerifierNeedsTwoCorroboratingPackets(t *testing.T) { + // One match is 1-in-65536 and must not be enough; a second makes it + // (1/65536)^2. This threshold is the reason the approach is sound. + v := buildVerifierFromPackets(t, realTransportFloodPacket) + one := v.evidence([]int64{1}, []string{"fm-112"}) + if one["fm-112"] != 1 { + t.Fatalf("evidence = %v, want fm-112:1", one) + } + if got := v.verified(one); len(got) != 0 { + t.Errorf("verified = %v, want none - one corroborating packet is not evidence", got) + } + + // The same packet twice under different ids: two distinct transmissions + // both deriving to fm-112. + v2 := buildVerifierFromPackets(t, realTransportFloodPacket, realTransportFloodPacket) + two := v2.evidence([]int64{1, 2}, []string{"fm-112"}) + if two["fm-112"] != 2 { + t.Fatalf("evidence = %v, want fm-112:2", two) + } + got := v2.verified(two) + if len(got) != 1 || got[0] != "fm-112" { + t.Errorf("verified = %v, want [fm-112]", got) + } +} + +func TestScopeVerifierIgnoresRegionsThatDoNotMatch(t *testing.T) { + v := buildVerifierFromPackets(t, realTransportFloodPacket, realTransportFloodPacket) + got := v.evidence([]int64{1, 2}, []string{"behss", "be", "eu"}) + if len(got) != 0 { + t.Errorf("evidence = %v, want empty - none of these regions is this packet", got) + } +} + +func TestScopeVerifierSkipsUnparseablePackets(t *testing.T) { + // A row whose raw_hex cannot be walked contributes nothing and must not + // error the pass: one malformed row in the window would otherwise blank + // the verification for every repeater. + v := buildVerifierFromPackets(t, "AA", realTransportFloodPacket) + got := v.evidence([]int64{1, 2}, []string{"fm-112"}) + if got["fm-112"] != 1 { + t.Errorf("evidence = %v, want fm-112:1 - the malformed row is skipped, the good one still counts", got) + } +} + +func TestScopeVerifierCachesAcrossTargets(t *testing.T) { + // The cost argument: work depends on (region, transmission), not on which + // target asked. Two targets declaring the same region over the same packets + // must not double the HMACs. + v := buildVerifierFromPackets(t, realTransportFloodPacket, realTransportFloodPacket) + v.evidence([]int64{1, 2}, []string{"fm-112"}) + after := v.hmacCount + v.evidence([]int64{1, 2}, []string{"fm-112"}) + if v.hmacCount != after { + t.Errorf("hmacCount %d -> %d on a repeat query, want unchanged - the cache is what keeps this inside rule 0", after, v.hmacCount) + } +} + +func TestScopeVerifierUnknownTxIDIsHarmless(t *testing.T) { + // A target's unmatchedTxIDs come from a different query than the verifier's + // rows. They are taken in the same window, but a row pruned between the two + // must degrade to "no evidence", not panic. + v := buildVerifierFromPackets(t, realTransportFloodPacket) + got := v.evidence([]int64{1, 999}, []string{"fm-112"}) + if got["fm-112"] != 1 { + t.Errorf("evidence = %v, want fm-112:1 - the unknown id contributes nothing", got) + } +} + +// BenchmarkScopeVerifierAudit models a full audit refresh: every declared name +// against every unmatched packet, once, through the memo. The naive shape would +// be targets x names x packets; this asserts the memo keeps it at names x +// packets, which is what makes the feature affordable (AGENTS.md rule 0). +func BenchmarkScopeVerifierAudit(b *testing.B) { + const packets, names, targets = 400, 124, 205 + rows := make([]unmatchedTransmissionRow, 0, packets) + txIDs := make([]int64, 0, packets) + for i := 0; i < packets; i++ { + rows = append(rows, unmatchedTransmissionRow{txID: int64(i + 1), rawHex: realTransportFloodPacket}) + txIDs = append(txIDs, int64(i+1)) + } + declared := make([]string, 0, names) + for i := 0; i < names; i++ { + declared = append(declared, fmt.Sprintf("r%04d", i)) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + v := newScopeVerifier(rows) + for t := 0; t < targets; t++ { + v.evidence(txIDs, declared) + } + } +} diff --git a/cmd/server/scopes.go b/cmd/server/scopes.go index 148e2fe3d..152583cb9 100644 --- a/cmd/server/scopes.go +++ b/cmd/server/scopes.go @@ -25,13 +25,19 @@ type ScopeObservation struct { } // RouteTypeMix is the route-type breakdown of packets this node was -// observed FORWARDING — i.e. packets on which this pubkey was the last hop -// of a FLOOD-family route (RouteTransportFlood, RouteFlood). It does NOT -// mean "packets in which this node appears anywhere in the path": a DIRECT -// or TRANSPORT_DIRECT packet's last path hop is the route's far end, never -// the transmitter, so crediting it here would attribute forwarding this node -// never did. Direct and TransportDirect are therefore always zero by -// construction — the forwarder join can never match those route types. +// observed FORWARDING — i.e. packets carrying this pubkey as any path hop of +// a FLOOD-family route (RouteTransportFlood, RouteFlood). On those routes +// every forwarder APPENDS its own hash to the end of the path +// (internal/packetpath/route.go), so each hop is a node that transmitted the +// packet; the last hop is merely the one an uplinked observer heard directly. +// +// It does NOT mean "packets in which this node appears anywhere in the path", +// because DIRECT and TRANSPORT_DIRECT routes are excluded entirely: those +// consume hops from the FRONT, so their path is the route's remaining plan +// rather than a record of who transmitted, and crediting any of their hops +// would attribute forwarding this node never did. Direct and TransportDirect +// are therefore always zero by construction — the route-type filter +// (scopeConformanceForwarderRouteTypesSQL) can never match them. type RouteTypeMix struct { TransportFlood int64 `json:"transportFlood"` Flood int64 `json:"flood"` @@ -61,11 +67,17 @@ type ScopeConformance struct { } // scopeConformanceForwarderRouteTypesSQL restricts the forwarder join to the -// only route types whose path[last] is the packet's actual transmitter: -// RouteTransportFlood (0) and RouteFlood (1). A DIRECT route consumes hops -// from the front, so its path[last] is the route's far end rather than the -// forwarder — including RouteDirect (2) / RouteTransportDirect (3) here -// would misattribute a scope to a node that never forwarded the packet. +// only route types whose path hops are a record of who transmitted the packet: +// RouteTransportFlood (0) and RouteFlood (1), where every forwarder appends +// its own hash. A DIRECT route consumes hops from the front, so its path is +// the route's remaining plan and its path[last] is the far end rather than a +// forwarder — including RouteDirect (2) / RouteTransportDirect (3) here would +// misattribute a scope to a node that never forwarded the packet. +// +// Since the attribution below reads EVERY hop rather than only path[last], +// this filter is the sole guard against that misattribution. It is pinned by +// TestScopeConformanceIgnoresDirectRoutesMidPath and +// TestScopeAuditForwardingIgnoresDirectRoutes; do not widen it. const scopeConformanceForwarderRouteTypesSQL = "t.route_type IN (0, 1)" // minForwarderHopHexLen is the shortest path_json hop ScopeConformance will @@ -76,10 +88,20 @@ const scopeConformanceForwarderRouteTypesSQL = "t.route_type IN (0, 1)" // attributing none. const minForwarderHopHexLen = 4 -// scopeConformanceQuery finds every transmission this pubkey forwarded -// (path[last] on a FLOOD-family route matches the pubkey), bounded by the -// since window so the scan stays an index range on first_seen rather than a -// full table scan. +// scopeConformanceQuery finds every transmission this pubkey forwarded (any +// path hop on a FLOOD-family route matches the pubkey), bounded by the since +// window so the scan stays an index range on first_seen rather than a full +// table scan. +// +// It reads every hop rather than only path[last] because on a flood route +// every hop appended itself after forwarding. Attributing only path[last] +// answered a narrower question — "which of this node's forwards did an +// uplinked observer hear directly" — and for a node with no observer in RF +// range the answer is nothing at all: measured on the live instance, that +// restriction kept 14% of hop observations network-wide and left 65% of +// declared repeaters with no evidence of any kind, while the same nodes' +// transported_scopes (byPathHop, every hop) listed scopes from the same +// database. See docs/specs/2026-09-07-auto-region-keys-design.md, M0. // // Two case/length mismatches make this join easy to get silently wrong // instead of erroring: @@ -110,7 +132,7 @@ var scopeConformanceQuery = ` AND EXISTS ( SELECT 1 FROM observations o - JOIN json_each(o.path_json) je ON je.key = json_array_length(o.path_json) - 1 + JOIN json_each(o.path_json) je WHERE o.transmission_id = t.id AND o.path_json IS NOT NULL AND json_valid(o.path_json) @@ -401,6 +423,28 @@ type scopeAuditTargetAgg struct { // declared target — see ScopeAuditForwarding's doc comment for why // those hops are attributed to neither candidate instead of both. ambiguousHops int64 + // unmatchedPackets counts packets this target was observed forwarding + // that carried a transport scope no configured region key matched + // (transmissions.scope_name = ""). Deliberately NOT folded into + // unscopedPackets: those two are opposites. Unscoped means the packet + // carried no scope at all (scope_name SQL NULL) and is what '*' governs; + // unmatched means it IS scoped and this instance simply holds no key for + // that region, so '*' says nothing about it. See scopeNameForDB in the + // ingestor for the encoding. + // + // A non-zero value is a caveat on this target's notObserved entries: any + // of them may be a region this instance cannot name rather than one the + // repeater is not forwarding. + unmatchedPackets int64 + // unmatchedTxIDs are the transmissions behind unmatchedPackets, kept so + // declared-region verification can test this target's own declarations + // against this target's own unnameable traffic (scope_verify.go). The same + // (target, txID) de-duplication that guards unmatchedPackets guards this, + // so one packet reaching a target by two hops cannot corroborate twice. + // + // Bounded by scopeVerifyMaxPacketsPerTarget, which unmatchedPackets is NOT: + // the count stays the honest total, this is the working set. + unmatchedTxIDs []int64 } // scopeAuditPrefixIndex builds, for every even hex length from @@ -432,6 +476,14 @@ func scopeAuditPrefixIndex(targets []string) map[int]map[string][]string { // call per pubkey — fine for one node, but 37+ repeater-sized loop of them // would each re-scan the same first_seen index range), this scans the // FLOOD-family window exactly once and returns every forwarder hop found. +// +// "Every forwarder hop" means every hop of the path, not only path[last] — +// see scopeConformanceQuery's doc comment for why, and note that this query +// returns one ROW PER HOP, so a single transmission now yields as many rows as +// it has hops (mean 7.08 on the live network). ScopeAuditForwarding's +// "|" de-duplication is what keeps that from counting a +// transmission twice for one target, and it is now load-bearing rather than +// belt-and-braces: one path can carry the same target on several hops. // It applies the SAME three conditions scopeConformanceQuery does — // minForwarderHopHexLen, scopeConformanceForwarderRouteTypesSQL, and the // explicit json_valid guard against a single malformed path_json row @@ -441,10 +493,10 @@ func scopeAuditPrefixIndex(targets []string) map[int]map[string][]string { // scopeAuditPrefixIndex, so the SQL cost stays O(rows in window) regardless // of len(targets). var scopeAuditForwarderScanQuery = ` - SELECT t.id, je.value, t.scope_name, t.first_seen + SELECT t.id, je.value FROM transmissions t JOIN observations o ON o.transmission_id = t.id - JOIN json_each(o.path_json) je ON je.key = json_array_length(o.path_json) - 1 + JOIN json_each(o.path_json) je WHERE t.first_seen >= ? AND ` + scopeConformanceForwarderRouteTypesSQL + ` AND o.path_json IS NOT NULL @@ -453,6 +505,74 @@ var scopeAuditForwarderScanQuery = ` AND LENGTH(je.value) >= ` + fmt.Sprint(minForwarderHopHexLen) + ` ` +// scopeAuditWindowMetaQuery reads the two per-TRANSMISSION facts the hop scan +// used to carry on every hop row: the scope name and the timestamp. It applies +// the identical window and route-type filter, so it covers every transmission +// the hop scan can produce, and both run inside one read transaction so the +// two see the same snapshot. +// +// Splitting these out is why the hop scan carries two columns instead of four. +// Measured on the live-shaped staging database on 2026-09-07, a 7d window +// yields 3,470,188 hop rows against 79,652 transmissions: 43 hop rows per +// transmission, each of which was re-reading the same scope_name and +// first_seen. SQLite spends 2.7s of the 16.7s that window cost; the rest was +// the Go side scanning columns it already knew. +// +// Every SQL-side attempt to shrink the hop scan itself measured worse on that +// same database and was rejected: a first-4-hex prefix filter against the +// declared targets takes 20.9s (and needs lower() on both sides, because 80% +// of stored hops are uppercase), GROUP BY t.id, hop takes 38.0s, and +// SELECT DISTINCT t.id, path_json takes 17.7s. The 3.47M rows are inherent: +// 1,368,761 observations carrying a path, ~2.5 usable hops each. +var scopeAuditWindowMetaQuery = ` + SELECT t.id, t.scope_name, t.first_seen + FROM transmissions t + WHERE t.first_seen >= ? + AND ` + scopeConformanceForwarderRouteTypesSQL + ` +` + +// scopeAuditTxMeta is one transmission's contribution to the aggregate, held +// once per transmission rather than once per hop. +type scopeAuditTxMeta struct { + scopeName sql.NullString + firstSeen string +} + +// scopeAuditWindowMeta loads scopeAuditWindowMetaQuery into a map keyed by +// transmission id. Runs on the caller's transaction so it shares the hop +// scan's snapshot. +func scopeAuditWindowMeta(tx *sql.Tx, sinceISO string) (map[int64]scopeAuditTxMeta, error) { + rows, err := tx.Query(scopeAuditWindowMetaQuery, sinceISO) + if err != nil { + return nil, fmt.Errorf("scope audit window meta: %w", err) + } + defer rows.Close() + + meta := map[int64]scopeAuditTxMeta{} + for rows.Next() { + var id int64 + var m scopeAuditTxMeta + if err := rows.Scan(&id, &m.scopeName, &m.firstSeen); err != nil { + return nil, fmt.Errorf("scope audit window meta scan: %w", err) + } + meta[id] = m + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("scope audit window meta rows: %w", err) + } + return meta, nil +} + +// scopeAuditSeenKey identifies one (target, transmission) pair for the +// de-duplication below. A struct key rather than the string it used to be +// built into: the hop scan reaches millions of rows on a 7d window, and every +// candidate hop was allocating a fresh "|" string to ask a +// question that a comparable struct answers without allocating. +type scopeAuditSeenKey struct { + target string + txID int64 +} + // ScopeAuditForwarding runs scopeAuditForwarderScanQuery once for the whole // window and attributes every forwarder hop it finds to targets, by the same // truncated-hash prefix match ScopeConformance uses for a single pubkey. @@ -470,14 +590,35 @@ var scopeAuditForwarderScanQuery = ` // ambiguousHops field and ScopeAuditRow.AmbiguousHops. // // Each (target, transmission) pair is counted at most once even if seen via -// multiple observations, for both the attributed and the ambiguous count — -// the same de-duplication scopeConformanceQuery gets for free from EXISTS, -// done explicitly here since this scan is not correlated per target. +// multiple observations OR via several hops of one path (a routing loop, or two +// hops colliding on the same truncated prefix), for both the attributed and the +// ambiguous count — the same de-duplication scopeConformanceQuery gets for free +// from EXISTS, done explicitly here since this scan is not correlated per +// target. Since the scan reads every hop rather than only path[last], this is +// the only thing keeping one transmission from counting several times for the +// same target; TestScopeAuditForwardingCountsOneTransmissionOncePerTarget pins +// it. func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (map[string]*scopeAuditTargetAgg, error) { byLen := scopeAuditPrefixIndex(targets) result := make(map[string]*scopeAuditTargetAgg, len(targets)) - rows, err := s.db.conn.Query(scopeAuditForwarderScanQuery, sinceISO) + // One read transaction for both queries. The hop scan and the per- + // transmission metadata are two passes over the same window, and a + // transmission arriving between them would otherwise appear in the hop scan + // with no metadata to attribute it by — rare, but the fix is a shared + // snapshot rather than a rule about what to do with the leftovers. + tx, err := s.db.conn.Begin() + if err != nil { + return nil, fmt.Errorf("scope audit forwarder scan begin: %w", err) + } + defer tx.Rollback() + + meta, err := scopeAuditWindowMeta(tx, sinceISO) + if err != nil { + return nil, err + } + + rows, err := tx.Query(scopeAuditForwarderScanQuery, sinceISO) if err != nil { return nil, fmt.Errorf("scope audit forwarder scan: %w", err) } @@ -492,21 +633,43 @@ func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (m return agg } - seen := make(map[string]bool) // "|" already counted (attributed or ambiguous) + seen := make(map[scopeAuditSeenKey]bool) // (target, txID) already counted (attributed or ambiguous) + + // hopBuf lower-cases the hop in place instead of through strings.ToLower. + // 80% of the hops in this database are stored uppercase (1,026,814 of + // 1,284,897 in a 24h window, measured 2026-09-07) because + // packetpath.DecodePathFromRawHex writes them that way, and the great + // majority of them match no declared target at all — so the allocation + // ToLower makes is paid millions of times to answer "no". byLen's keys are + // lowercase, and a map index expression on string(bytes) does not allocate. + var hopBuf [64]byte for rows.Next() { var txID int64 - var hop string - var scopeName sql.NullString - var firstSeen string - if err := rows.Scan(&txID, &hop, &scopeName, &firstSeen); err != nil { + var hopRaw sql.RawBytes + if err := rows.Scan(&txID, &hopRaw); err != nil { return nil, fmt.Errorf("scope audit forwarder scan scan: %w", err) } - hop = strings.ToLower(hop) - candidates := byLen[len(hop)][hop] + n := len(hopRaw) + if n > len(hopBuf) { + // Longer than a full pubkey: cannot be any target's prefix. The + // SQL floor guards the short end, this guards the long one. + continue + } + for i := 0; i < n; i++ { + c := hopRaw[i] + if 'A' <= c && c <= 'Z' { + c += 'a' - 'A' + } + hopBuf[i] = c + } + candidates := byLen[n][string(hopBuf[:n])] + if len(candidates) == 0 { + continue + } if len(candidates) > 1 { for _, target := range candidates { - key := target + "|" + strconv.FormatInt(txID, 10) + key := scopeAuditSeenKey{target: target, txID: txID} if seen[key] { continue } @@ -515,8 +678,16 @@ func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (m } continue } + txMeta, ok := meta[txID] + if !ok { + // Impossible while both queries share one snapshot and one WHERE + // clause; treated as "nothing to attribute" rather than silently + // counted as unscoped, which is what a zero-valued meta would do. + continue + } + scopeName, firstSeen := txMeta.scopeName, txMeta.firstSeen for _, target := range candidates { - key := target + "|" + strconv.FormatInt(txID, 10) + key := scopeAuditSeenKey{target: target, txID: txID} if seen[key] { continue } @@ -528,7 +699,17 @@ func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (m continue } if scopeName.String == "" { - continue // unmatched — not part of the declared/observed comparison + // Unmatched: transport-scoped, but no configured region key + // matched code1. Still not part of the declared/observed + // comparison — it names no region, so it can never satisfy a + // declaration — but it is the evidence that a notObserved + // finding on this row may be a gap in this instance's + // hashRegions rather than in the repeater's forwarding. + agg.unmatchedPackets++ + if len(agg.unmatchedTxIDs) < scopeVerifyMaxPacketsPerTarget { + agg.unmatchedTxIDs = append(agg.unmatchedTxIDs, txID) + } + continue } name := normScope(scopeName.String) so, ok := agg.scopes[name] @@ -645,6 +826,34 @@ type ScopeAuditRow struct { // UndeclaredObserved entry is unaffected by it (ambiguous hops are never // attributed to a scope at all). AmbiguousHops int64 `json:"ambiguousHops"` + + // ObservedUnmatchedPackets counts packets this repeater was observed + // forwarding whose transport scope matched no region key this instance + // holds. Like AmbiguousHops it is a caveat rather than a finding, but the + // two have different causes and different fixes: AmbiguousHops is a + // pubkey-prefix collision between two repeaters and nobody's fault, + // ObservedUnmatchedPackets is a missing entry in this instance's own + // hashRegions and the reader can act on it. A non-zero value means any + // NotObserved entry on this row may name a region this instance cannot + // name rather than one the repeater is not forwarding. + // + // It says nothing about DeclaredWildcard: unmatched traffic IS scoped, so + // it never feeds WildcardContradiction, which counts only plain unscoped + // floods. + ObservedUnmatchedPackets int64 `json:"observedUnmatchedPackets"` + + // RegionEvidence maps a declared region to how many of this repeater's own + // unmatched forwarded packets derive to it — see scope_verify.go. A region + // reaching scopeVerifyMinCorroboration is removed from NotObserved, so this + // field is NOT what decides the chip's colour; NotObserved remains the sole + // source of that. This exists so a client can say HOW a region was + // established, and can explain a region that got exactly one hit and + // therefore stayed in NotObserved. + // + // Absent regions simply had no matching traffic. Never nil in the response + // — an empty object and a missing key mean the same thing, and an empty map + // is the cheaper thing for a client to iterate. + RegionEvidence map[string]int `json:"regionEvidence"` } // ScopeAuditResponse is the payload for GET /api/scope-audit. Only diff --git a/cmd/server/scopes_test.go b/cmd/server/scopes_test.go index 224eaad90..185dbc3f2 100644 --- a/cmd/server/scopes_test.go +++ b/cmd/server/scopes_test.go @@ -101,6 +101,19 @@ func seedTransmissionRoute(t *testing.T, s *PacketStore, forwarder string, seed // transmission to fall inside a real-wall-clock ?window= lookback rather // than the fixed date the ScopeConformance unit tests above use. func seedTransmissionRouteAt(t *testing.T, s *PacketStore, forwarder string, seed scopeSeed, routeType int, firstSeen string) { + t.Helper() + seedTransmissionPathAt(t, s, []string{forwarder}, seed, routeType, firstSeen) +} + +// seedTransmissionPathAt seeds one transmission whose single observation +// carries a MULTI-hop path. A one-hop seed cannot tell the two reasons a node +// gets attributed apart — it is simultaneously path[0] and path[last] — so the +// mid-path cases below need a path with something after the target on it. +// +// Hops are upper-cased for the same reason seedTransmissionRoute does it: the +// decoder writes them that way (packetpath.DecodePathFromRawHex), and the join +// has to cope with that rather than with a lowercase convenience fiction. +func seedTransmissionPathAt(t *testing.T, s *PacketStore, hops []string, seed scopeSeed, routeType int, firstSeen string) { t.Helper() scopeSeedCounter++ hash := fmt.Sprintf("scopehash%d", scopeSeedCounter) @@ -118,7 +131,11 @@ func seedTransmissionRouteAt(t *testing.T, s *PacketStore, forwarder string, see t.Fatalf("seed transmission id: %v", err) } - pathJSON := fmt.Sprintf(`["%s"]`, strings.ToUpper(forwarder)) + quoted := make([]string, len(hops)) + for i, h := range hops { + quoted[i] = `"` + strings.ToUpper(h) + `"` + } + pathJSON := "[" + strings.Join(quoted, ",") + "]" if _, err := s.db.conn.Exec( `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, ?)`, txID, pathJSON, time.Now().Unix(), @@ -142,6 +159,13 @@ func seedDirectTransmission(t *testing.T, s *PacketStore, forwarder string, seed seedTransmissionRoute(t, s, forwarder, seed, RouteDirect) } +// seedTransmissionPath is seedTransmissionPathAt at the fixed date the +// ScopeConformance unit tests use. +func seedTransmissionPath(t *testing.T, s *PacketStore, hops []string, seed scopeSeed, routeType int) { + t.Helper() + seedTransmissionPathAt(t, s, hops, seed, routeType, "2026-01-15T12:00:00Z") +} + func TestScopeConformanceKeepsThreeStatesDistinct(t *testing.T) { s := newScopeTestStore(t) // Same forwarder on all three, so only the scope state differs. Unmatched @@ -263,6 +287,101 @@ func TestScopeConformanceRouteMixIgnoresDirectRoutes(t *testing.T) { } } +// TestScopeConformanceAttributesMidPathForwarder is the case the old last-hop +// restriction hid. On a flood route every forwarder APPENDS its own hash to the +// end of the path (internal/packetpath/route.go), so a hop in the MIDDLE +// forwarded the packet exactly as surely as path[last] did — being last only +// additionally means an uplinked observer heard that transmission directly. +// +// Measured on the live instance 2026-09-07: BE-HHE-LAAK-EDG-01 carried 155 +// flood-family packets on its hop over 14 days and was path[last] on ZERO of +// them, so its Scopes card was empty (whole route mix zero) while its own node +// header listed four transported scopes from the same database. Network-wide the +// restriction kept 14% of hop observations and left 65% of repeaters with no +// evidence at all. See docs/specs/2026-09-07-auto-region-keys-design.md, M0. +func TestScopeConformanceAttributesMidPathForwarder(t *testing.T) { + s := newScopeTestStore(t) + // The target forwarded first, two other nodes relayed it onward, and only + // the third was heard by an observer — the shape of every edge repeater. + seedTransmissionPath(t, s, []string{testFullPubkeyA[:4], "AAAA", "BBBB"}, scopeMatched("#be"), RouteFlood) + + got, err := s.ScopeConformance(testFullPubkeyA, "2026-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if len(got.Observed) != 1 || got.Observed[0].Scope != "#be" || got.Observed[0].Packets != 1 { + t.Fatalf("Observed = %+v, want one #be observation — a mid-path hop on a flood route is a forwarder", got.Observed) + } + if got.Routes.Flood != 1 { + t.Errorf("Routes.Flood = %d, want 1", got.Routes.Flood) + } +} + +// TestScopeConformanceIgnoresDirectRoutesMidPath is the guard that has to hold +// after the last-hop restriction is gone. On a DIRECT route the path is consumed +// from the FRONT, so its hops are the route's remaining plan rather than a record +// of who transmitted — attributing any of them, last or middle, would credit +// forwarding that never happened. The route-type filter is now the only thing +// preventing that, so it is pinned explicitly here and not left implied. +// +// Live confirmation: the 52 transmissions where that repeater IS path[last] are +// all route_type 2 — packets addressed toward it, not forwarded by it. +func TestScopeConformanceIgnoresDirectRoutesMidPath(t *testing.T) { + s := newScopeTestStore(t) + seedTransmissionPath(t, s, []string{"AAAA", testFullPubkeyA[:4], "BBBB"}, scopeMatched("#be"), RouteDirect) + seedTransmissionPath(t, s, []string{"AAAA", "BBBB", testFullPubkeyA[:4]}, scopeMatched("#be"), RouteTransportDirect) + + got, err := s.ScopeConformance(testFullPubkeyA, "2026-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if len(got.Observed) != 0 { + t.Errorf("Observed = %+v, want empty — a DIRECT route's path hops are a plan, not forwarding evidence", got.Observed) + } + if got.Routes != (RouteTypeMix{}) { + t.Errorf("Routes = %+v, want all zero", got.Routes) + } +} + +// TestScopeConformanceCountsOneTransmissionOnce pins that widening the join to +// every hop cannot double-count. A path can legitimately carry the same hop +// twice (a routing loop, or two nodes colliding on the same truncated prefix), +// and a transmission is one packet however many of its hops match. EXISTS gives +// this for free — which is precisely why the query must keep using EXISTS rather +// than joining json_each into the outer SELECT. +func TestScopeConformanceCountsOneTransmissionOnce(t *testing.T) { + s := newScopeTestStore(t) + seedTransmissionPath(t, s, []string{testFullPubkeyA[:4], "AAAA", testFullPubkeyA[:4]}, scopeMatched("#be"), RouteFlood) + + got, err := s.ScopeConformance(testFullPubkeyA, "2026-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if len(got.Observed) != 1 || got.Observed[0].Packets != 1 { + t.Fatalf("Observed = %+v, want exactly one packet counted once", got.Observed) + } + if got.Routes.Flood != 1 { + t.Errorf("Routes.Flood = %d, want 1 — one transmission, however many of its hops match", got.Routes.Flood) + } +} + +// TestScopeConformanceIgnoresShortMidPathHop keeps minForwarderHopHexLen +// applying to middle hops too. A 1-byte hop collides across a real fleet far too +// often to attribute, and now that every hop is a candidate there are ~7x as many +// chances to get it wrong. +func TestScopeConformanceIgnoresShortMidPathHop(t *testing.T) { + s := newScopeTestStore(t) + seedTransmissionPath(t, s, []string{testFullPubkeyA[:2], "AAAA", "BBBB"}, scopeMatched("#be"), RouteFlood) + + got, err := s.ScopeConformance(testFullPubkeyA, "2026-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if len(got.Observed) != 0 || got.Routes != (RouteTypeMix{}) { + t.Errorf("Observed = %+v, Routes = %+v, want nothing — a 1-byte hop is too collision-prone to attribute", got.Observed, got.Routes) + } +} + // TestScopeConformanceSurvivesMalformedPathJSON guards against a SQLite // quirk: json_each() in the EXISTS subquery is evaluated independently of // the `o.path_json IS NOT NULL` predicate in the same WHERE clause, so a @@ -797,6 +916,196 @@ func TestScopeAuditForwardingAmbiguousHopCreditsNeitherTarget(t *testing.T) { } } +// TestScopeAuditForwardingAttributesMidPathHop is the fleet-wide half of the +// mid-path attribution fix. The audit runs a different query from +// ScopeConformance — one full-window scan instead of one EXISTS per pubkey — so +// the two share the rule but not the code, and both need pinning. +// +// This is the case behind the audit's 65% blind spot: a declared target that +// forwards steadily but is never the hop an observer hears directly had every +// region it declares reported as notObserved. +func TestScopeAuditForwardingAttributesMidPathHop(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{testFullPubkeyA[:4], "AAAA", "BBBB"}, scopeMatched("#be"), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil || agg.scopes["be"] == nil || agg.scopes["be"].Packets != 1 { + t.Fatalf("want the mid-path hop attributed to its sole matching target, got %+v", got) + } + if agg.ambiguousHops != 0 { + t.Errorf("ambiguousHops = %d, want 0 — one target matches this hop", agg.ambiguousHops) + } +} + +// TestScopeAuditForwardingIgnoresDirectRoutes pins the route-type filter on the +// audit's own query. With the last-hop restriction gone it is the only guard +// against crediting a DIRECT route's remaining path plan as forwarding — and a +// DIRECT packet's hops are frequently the declared targets this audit judges. +func TestScopeAuditForwardingIgnoresDirectRoutes(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{"AAAA", testFullPubkeyA[:4], "BBBB"}, scopeMatched("#be"), RouteDirect, recent) + seedTransmissionPathAt(t, s, []string{"AAAA", "BBBB", testFullPubkeyA[:4]}, scopeMatched("#be"), RouteTransportDirect, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + if agg := got[testFullPubkeyA]; agg != nil && (len(agg.scopes) != 0 || agg.unscopedPackets != 0 || agg.ambiguousHops != 0) { + t.Errorf("agg = %+v, want no attribution from DIRECT routes", agg) + } +} + +// TestScopeAuditForwardingCountsOneTransmissionOncePerTarget pins that the +// existing "|" de-duplication also absorbs the same target +// matching several hops of ONE path — which could not happen while only +// path[last] was read, and now can (a routing loop, or two hops colliding on the +// same truncated prefix). Without it a looping packet would inflate a target's +// packet count and quietly make a quiet region look busy. +func TestScopeAuditForwardingCountsOneTransmissionOncePerTarget(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{testFullPubkeyA[:4], "AAAA", testFullPubkeyA[:4]}, scopeMatched("#be"), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil || agg.scopes["be"] == nil { + t.Fatalf("want #be attributed, got %+v", got) + } + if agg.scopes["be"].Packets != 1 { + t.Errorf("Packets = %d, want 1 — one transmission, matched on two of its hops", agg.scopes["be"].Packets) + } +} + +// TestScopeAuditForwardingAttributesLongerHopByItsOwnLength pins the +// length-indexed half of scopeAuditPrefixIndex, which every other test in this +// file leaves untested: they all seed 4-char hops, so a lookup that ignored hop +// length entirely would still pass them. +// +// pkOther shares the first 4 hex chars with testFullPubkeyA and diverges after +// that, so an 8-char hop has exactly one candidate while a 4-char hop would +// have two. Attribution must therefore key on the hop's OWN length: at 8 chars +// this is an unambiguous attribution, not an ambiguousHops row. +func TestScopeAuditForwardingAttributesLongerHopByItsOwnLength(t *testing.T) { + s := newScopeTestStore(t) + pkOther := testFullPubkeyA[:4] + strings.Repeat("33", 30) + hop := testFullPubkeyA[:8] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{hop, "AAAA"}, scopeMatched("#be"), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA, pkOther}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil || agg.scopes["be"] == nil || agg.scopes["be"].Packets != 1 { + t.Fatalf("want the 8-char hop attributed to its sole matching target, got %+v", got) + } + if agg.ambiguousHops != 0 { + t.Errorf("ambiguousHops = %d, want 0 — the two targets diverge before hop length 8", agg.ambiguousHops) + } + if other := got[pkOther]; other != nil && (len(other.scopes) != 0 || other.ambiguousHops != 0) { + t.Errorf("pkOther = %+v, want no attribution and no ambiguity — the hop is not its prefix", other) + } +} + +// TestScopeAuditForwardingCountsUnmatchedPackets: a transport-scoped packet +// whose code1 matched no configured region key is stored with scope_name = "" +// (scopeNameForDB's "transport-scoped but unnameable" state). It is not a +// named scope, so it must not enter agg.scopes, and it is not an unscoped +// plain flood either, so it must not enter unscopedPackets. It is its own +// fact: this instance saw the target forward traffic it holds no key for. +// +// Without this counter the audit reports the declared region as "not +// observed", which reads as a finding about the repeater when it is really a +// gap in this instance's own hashRegions. +func TestScopeAuditForwardingCountsUnmatchedPackets(t *testing.T) { + s := newScopeTestStore(t) + hop := testFullPubkeyA[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, s, hop, scopeUnmatched(), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil { + t.Fatalf("want an agg for the target, got none (result = %+v)", got) + } + if agg.unmatchedPackets != 1 { + t.Errorf("unmatchedPackets = %d, want 1", agg.unmatchedPackets) + } + if len(agg.scopes) != 0 { + t.Errorf("scopes = %+v, want empty — an unmatched packet names no region", agg.scopes) + } + if agg.unscopedPackets != 0 { + t.Errorf("unscopedPackets = %d, want 0 — unmatched is not the same as unscoped", agg.unscopedPackets) + } +} + +// TestScopeAuditForwardingCountsUnmatchedOnMidPathHop is the post-M0 case that +// carries almost all of this counter's real volume: before M0 only a last hop +// was attributed, so a repeater deep in a flood path contributed nothing at +// all. Now every hop counts, and the same de-duplication that protects the +// named-scope tally must protect this one — a target appearing twice in one +// path is still one packet, not two. +func TestScopeAuditForwardingCountsUnmatchedOnMidPathHop(t *testing.T) { + s := newScopeTestStore(t) + hop := testFullPubkeyA[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{"AAAA", hop, "BBBB", hop}, scopeUnmatched(), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil { + t.Fatalf("want an agg for the mid-path target, got none (result = %+v)", got) + } + if agg.unmatchedPackets != 1 { + t.Errorf("unmatchedPackets = %d, want 1 — one transmission, matched on two of its hops", agg.unmatchedPackets) + } +} + +// TestScopeAuditForwardingRecordsUnmatchedTxIDs: the counter M1 added says how +// many, verification needs to know which. The IDs must be de-duplicated the +// same way the counter is — a target appearing twice in one path contributed +// one packet, and counting it twice would let a single packet reach the +// two-corroboration threshold on its own. +func TestScopeAuditForwardingRecordsUnmatchedTxIDs(t *testing.T) { + s := newScopeTestStore(t) + hop := testFullPubkeyA[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{hop, "AAAA", hop}, scopeUnmatched(), RouteFlood, recent) + seedTransmissionPathAt(t, s, []string{"BBBB", hop}, scopeUnmatched(), RouteFlood, recent) + seedTransmissionPathAt(t, s, []string{hop}, scopeMatched("#be"), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil { + t.Fatalf("want an agg, got none (result = %+v)", got) + } + if len(agg.unmatchedTxIDs) != 2 { + t.Errorf("unmatchedTxIDs = %v, want 2 distinct ids — the twice-hopped packet counts once, and the matched packet not at all", agg.unmatchedTxIDs) + } + if agg.unmatchedPackets != int64(len(agg.unmatchedTxIDs)) { + t.Errorf("unmatchedPackets = %d but %d ids recorded — the count and the ids must not drift", agg.unmatchedPackets, len(agg.unmatchedTxIDs)) + } +} + // --- GET /api/scope-audit handler tests --- // setupScopeAuditServer extends setupNodeScopesServer's schema with a @@ -838,6 +1147,44 @@ func getScopeAudit(t *testing.T, router *mux.Router, query string) ScopeAuditRes return got } +// TestScopeAuditTTLForSevenDayWindow pins the per-window TTL. The 7d window +// costs a different order of magnitude than the others (16.7s against 4.0s and +// 0.15s, measured on the live-shaped staging database on 2026-09-07), so it is +// deliberately not on the 30s the other two share. A future edit that collapses +// this back to one constant should have to delete a test that says why. +func TestScopeAuditTTLForSevenDayWindow(t *testing.T) { + if got := scopeAuditTTLFor("7d"); got != 5*time.Minute { + t.Errorf("scopeAuditTTLFor(7d) = %s, want 5m", got) + } + for _, w := range []string{"1h", "24h", ""} { + if got := scopeAuditTTLFor(w); got != 30*time.Second { + t.Errorf("scopeAuditTTLFor(%q) = %s, want 30s", w, got) + } + } +} + +// TestHandleScopeAuditServesSecondRequestFromCache pins the cache path itself, +// which the singleflight rewrite moved out of the handler and into +// scopeAuditCached/scopeAuditStore. A declared row inserted between two +// requests inside the TTL must NOT appear in the second response: if it does, +// the response was recomputed and the cache is not being consulted. +func TestHandleScopeAuditServesSecondRequestFromCache(t *testing.T) { + srv, router := setupScopeAuditServer(t) + now := time.Now().UTC().Format(time.RFC3339) + insertDeclared(t, srv, testFullPubkeyA, now, "be", 0) + + first := getScopeAudit(t, router, "") + if len(first.Repeaters) != 1 { + t.Fatalf("first call repeaters = %d, want 1", len(first.Repeaters)) + } + + insertDeclared(t, srv, testFullPubkeyB, now, "be", 0) + second := getScopeAudit(t, router, "") + if len(second.Repeaters) != 1 { + t.Errorf("second call repeaters = %d, want 1 — the row added after the first call proves the cache was bypassed", len(second.Repeaters)) + } +} + // TestHandleScopeAuditNormalisesHashPrefix pins trap 1: transmissions.scope_name // keeps the '#' (hashRegions config), regions_csv arrives from the firmware // with it already stripped. Declared "be-van" and observed "#be-van" must be @@ -1083,6 +1430,136 @@ func TestHandleScopeAuditSurfacesAmbiguousHops(t *testing.T) { } } +// TestHandleScopeAuditSurfacesUnmatchedPackets: a repeater declares "behss", +// and this instance sees it forward transport-scoped traffic it cannot name. +// The row must still list "behss" as notObserved — an unmatched packet names +// no region, so it cannot satisfy the declaration — but it must also carry +// observedUnmatchedPackets, so a client can say the finding might be a missing +// region key rather than a silent repeater. +func TestHandleScopeAuditSurfacesUnmatchedPackets(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "behss", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, pk[:4], scopeUnmatched(), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 { + t.Fatalf("repeaters = %+v, want 1", got.Repeaters) + } + row := got.Repeaters[0] + if row.ObservedUnmatchedPackets != 1 { + t.Errorf("observedUnmatchedPackets = %d, want 1", row.ObservedUnmatchedPackets) + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "behss" { + t.Errorf("notObserved = %v, want [\"behss\"] — an unmatched packet names no region and cannot satisfy a declaration", row.NotObserved) + } + if row.WildcardContradiction { + t.Error("wildcardContradiction = true, want false — unmatched traffic is scoped, so it says nothing about '*'") + } +} + +// TestHandleScopeAuditVerifiesDeclaredRegion is the case this milestone exists +// for, built from the real packet that started the investigation. A repeater +// declares "fm-112"; this instance holds no key for it, so both packets it +// forwarded are stored unmatched. Verification derives the key from the +// repeater's own declaration, finds two corroborating packets, and the region +// must leave notObserved with its evidence count reported. +func TestHandleScopeAuditVerifiesDeclaredRegion(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "fm-112,behss", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 { + t.Fatalf("repeaters = %+v, want 1", got.Repeaters) + } + row := got.Repeaters[0] + if row.RegionEvidence["fm-112"] != 2 { + t.Errorf("regionEvidence = %v, want fm-112:2", row.RegionEvidence) + } + for _, n := range row.NotObserved { + if n == "fm-112" { + t.Errorf("notObserved = %v, must not contain fm-112 - two corroborating packets prove it is forwarded", row.NotObserved) + } + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "behss" { + t.Errorf("notObserved = %v, want [behss] - that region has no corroborating traffic here", row.NotObserved) + } +} + +// TestHandleScopeAuditDoesNotVerifyOnOnePacket: a single match is 1-in-65536 +// and must leave the region in notObserved, with its count still reported so a +// client can say "one hit, not enough". +func TestHandleScopeAuditDoesNotVerifyOnOnePacket(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "fm-112", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + + got := getScopeAudit(t, router, "") + row := got.Repeaters[0] + if row.RegionEvidence["fm-112"] != 1 { + t.Errorf("regionEvidence = %v, want fm-112:1", row.RegionEvidence) + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "fm-112" { + t.Errorf("notObserved = %v, want [fm-112] - one corroborating packet is not evidence", row.NotObserved) + } +} + +// TestHandleScopeAuditLeavesCleanRowsAlone: a repeater whose declared regions +// are all observed by name, with no unmatched traffic at all, must be untouched +// by verification - no evidence, no change to notObserved, and an empty (not +// null) regionEvidence so a client can iterate it without a guard. +func TestHandleScopeAuditLeavesCleanRowsAlone(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "be", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, pk[:4], scopeMatched("#be"), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + row := got.Repeaters[0] + if len(row.NotObserved) != 0 { + t.Errorf("notObserved = %v, want empty", row.NotObserved) + } + if row.RegionEvidence == nil { + t.Error("regionEvidence = nil, want an empty object - a client must not need a null guard") + } + if len(row.RegionEvidence) != 0 { + t.Errorf("regionEvidence = %v, want empty - nothing needed verifying here", row.RegionEvidence) + } +} + +// seedUnmatchedRawAt seeds one unmatched transmission carrying a real raw_hex, +// attributed to forwarder. Distinct from seedTransmissionRouteAt, which seeds +// raw_hex 'AA' - fine for tests that never parse it, useless here. +func seedUnmatchedRawAt(t *testing.T, s *PacketStore, forwarder, rawHex string, routeType int, firstSeen string) { + t.Helper() + scopeSeedCounter++ + hash := fmt.Sprintf("scoperaw%d", scopeSeedCounter) + res, err := s.db.conn.Exec( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) + VALUES (?, ?, ?, ?, 5, '9209', '0000', '')`, + rawHex, hash, firstSeen, routeType) + if err != nil { + t.Fatal(err) + } + txID, err := res.LastInsertId() + if err != nil { + t.Fatal(err) + } + if _, err := s.db.conn.Exec( + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, 0)`, + txID, `["`+strings.ToUpper(forwarder)+`"]`); err != nil { + t.Fatal(err) + } +} + // TestHandleScopeAuditSortsMissingRegionsFirst: the repeater with a declared // region it is not forwarding must rank above a repeater in full agreement — // that's the headline this endpoint exists to surface, not the boring majority. diff --git a/config.example.json b/config.example.json index 0dd4744b5..30c08eeaf 100644 --- a/config.example.json +++ b/config.example.json @@ -390,6 +390,8 @@ "#eu" ], "_comment_hashRegions": "Region names for scope matching on transport-route packets. Key = SHA256('#name')[:16]. Add any region names used by nodes in your network.", + "autoRegionKeys": { "enabled": false, "maxDerived": 256, "refreshMinutes": 15 }, + "_comment_autoRegionKeys": "Opt-in: derive region keys from the region names repeaters declare over RF (node_declared_regions), on top of the explicit hashRegions list above. Default OFF. Solves the case where a repeater forwards a region this instance holds no key for: its traffic is stored unmatched, and anything that compares declared against observed reports the region as absent rather than unnameable. TOP-LEVEL FLAG, a sibling of hashRegions - config loading is plain json.Unmarshal with no DisallowUnknownFields, so nesting it elsewhere is silently ignored and the feature stays off with no error. maxDerived caps the derived tier (default 256): each key costs one HMAC per transport-scoped packet and raises the random 2-byte collision rate by 1/65536, and the match cannot be indexed because the code is an HMAC over the payload. Over the cap, names are kept by how many distinct repeaters declare them, so a one-off local name is dropped before a region half the network uses. refreshMinutes is how often the derived tier is rebuilt (default 15). Requires clientRegions (or an ESP32 observer on the neighbour-report firmware) to be populating node_declared_regions, or the derived tier stays empty.", "_comment_defaultRegion": "IATA code shown by default in region filters.", "_comment_mapDefaults": "Initial map center [lat, lon] and zoom level.", "_comment_regions": "IATA code → display name mapping for the region filter UI. Each key is a 3-letter IATA code that an observer is tagged with (resolved priority: MQTT payload `region` field > topic-derived region > mqttSources.region). Observers without an IATA tag will not appear under any region filter — only under 'All Regions'. The region filter dropdown shows one entry per code listed here PLUS any extra IATA codes the server discovers from observers at runtime (so you can omit codes here and they will still be selectable, just labelled with the bare IATA code instead of a friendly name). Selecting 'All Regions' (or no region) returns results from every observer including those with no IATA tag; selecting one or more codes restricts results to packets observed by observers tagged with those codes. The reserved value 'All' (case-insensitive) is treated as 'no filter' on the server, so the URL ?region=All behaves identically to omitting the param. Issue #770.", diff --git a/docs/api-spec.md b/docs/api-spec.md index f4cd24185..c70409845 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -802,9 +802,15 @@ forwarding anything is a valid question, not an error. key for), not an error, and must not be folded into `unscoped`. - `unscoped` counts packets that carried no scope at all. `unmatched` and `unscoped` are always reported as separate top-level counts. -- `routes.direct` / `routes.transportDirect` are always `0` by construction: a DIRECT-family - route's last path hop is the route's far end, never the transmitter, so this node can never - be attributed as the forwarder of one. +- "this repeater's own forwarded traffic" means transmissions carrying this pubkey as **any** + path hop of a FLOOD-family route, not only as the final hop. On those routes each forwarder + appends its own hash, so every hop transmitted the packet; the final hop is only the one an + uplinked observer heard directly, and counting just that one reports nothing at all for a + repeater with no observer in RF range. +- `routes.direct` / `routes.transportDirect` are always `0` by construction: DIRECT-family + routes are excluded outright, because they consume hops from the front, making their path + the route's remaining plan rather than a record of who transmitted — so none of their hops + is evidence that this node forwarded anything. **Notes — `declared` distinguishes "never asked" from "asked and declined everything":** - `window` bounds `observed` only; `declared` is always the latest reading regardless of @@ -1889,7 +1895,9 @@ not being the same as "declared nothing"), which apply here identically. ], "observedUnscopedPackets": number, // plain-FLOOD packets forwarded this window "wildcardContradiction": boolean, // observed unscoped forwarding but '*' not declared - "ambiguousHops": number // forwarder hops this window that could not be attributed — see note below + "ambiguousHops": number, // forwarder hops this window that could not be attributed — see note below + "observedUnmatchedPackets": number, // forwarded packets whose scope this instance holds no key for — see note below + "regionEvidence": { "": number } // declared regions corroborated by this repeater's own unnameable traffic — see note below } ] } @@ -1927,6 +1935,34 @@ not being the same as "declared nothing"), which apply here identically. `ambiguousHops` carries weaker evidence than one with zero: any entry in that row's `notObserved` could be explained by a prefix collision rather than a genuine absence of forwarding, and a client should present it as a caveat rather than a confirmed finding. +- `observedUnmatchedPackets` counts packets this repeater was observed forwarding whose + transport scope matched no region key this instance has configured (`hashRegions`), so + the ingestor stored them with an empty `scope_name`. Those packets name no region and + therefore cannot satisfy a declared one, which means **a repeater forwarding a region + this instance cannot name is reported exactly like one forwarding nothing**. A non-zero + value is a caveat on this row's `notObserved`, in the same spirit as `ambiguousHops` but + with a different cause and a different fix: `ambiguousHops` is a pubkey-prefix collision + between two repeaters and nobody's fault, `observedUnmatchedPackets` is a missing entry + in this instance's own configuration and the operator can act on it. It is **not** + evidence for or against `declaredWildcard` — unmatched traffic is scoped, so it never + affects `wildcardContradiction`, which counts only plain unscoped floods. + Since M1b, part of this count is explained: packets counted in `regionEvidence` are + attributable to a declared region after all. A client showing this as a caveat should + subtract them and report only the remainder, which carries a sharper meaning — traffic + this repeater forwards for a region it does **not** declare and this instance cannot + name. +- `regionEvidence` maps a declared region to how many of this repeater's own unmatched + forwarded packets derive to it. The server tests each declared region this repeater has + no *named* evidence for by deriving `SHA256("#region")[:16]` and HMAC-ing that + repeater's own unmatched packets with it — the same computation the ingestor performs at + ingest, with the candidate set narrowed to this repeater's declarations. A region + reaching **2** corroborating packets is removed from `notObserved`: `code1` is two + bytes, so one match happens by chance with probability 1/65536, while two on the same + region is (1/65536)². A region with exactly one hit therefore stays in `notObserved` + **and** appears here with the value 1, so a client can explain why it is still shown as + not observed. `notObserved` remains the single source of truth for whether a region was + observed; this field says only *how* that was established. The object is always present + and may be empty. - All scope names in `declaredRegions` / `notObserved` / `undeclaredObserved[].scope` are already normalised (no leading `#`) — the server does the `#`/no-`#` reconciliation described on the per-node endpoint so this response is directly comparable without a diff --git a/docs/client-regions.md b/docs/client-regions.md index 1e70bf002..d061bb2f8 100644 --- a/docs/client-regions.md +++ b/docs/client-regions.md @@ -119,6 +119,27 @@ stale one just because it arrived later. Retention: `retention.clientRegionsDays` bounds the table by `observed_at`; `0` disables it (Task 6). +## Second consumer — derived region keys + +`node_declared_regions` originally had one reader: the declared side of the +Scope Audit. With `autoRegionKeys.enabled` set (default off, see +`config.example.json`), the ingestor reads it a second time, deriving a region +key `SHA256("#name")[:16]` for each declared name so that traffic in those +regions can be *named* rather than stored unmatched. + +Two consequences operators should know about: + +- **Retention now bounds nameability.** `retention.clientRegionsDays` already + bounded how long a declared answer stayed visible in the audit. With + derivation on, it also bounds how long a region stays *derivable*: once the + last answer naming a region is pruned, its key leaves the set on the next + refresh and its traffic reverts to unmatched. Regions you want named + permanently belong in `hashRegions`, which nothing prunes. +- **The set is capped.** `autoRegionKeys.maxDerived` (default 256) limits the + derived tier; over the cap, names are kept by how many distinct repeaters + declare them, so a one-off local name is dropped before a region half the + network uses. The ingestor logs how many names were dropped on each refresh. + ## Configurable values (future customizer) `retention.clientRegionsDays` is the only tunable so far; no map rendering or comparison UI is built on diff --git a/docs/plans/2026-09-07-auto-derived-region-keys.md b/docs/plans/2026-09-07-auto-derived-region-keys.md new file mode 100644 index 000000000..98198b6ea --- /dev/null +++ b/docs/plans/2026-09-07-auto-derived-region-keys.md @@ -0,0 +1,1519 @@ +# Auto-Derived Region Keys (M2) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let the ingestor name transport-scoped traffic for regions the operator has not listed in `hashRegions`, by deriving keys from the region names repeaters declare over RF — opt-in, capped, and with a principled tie-break when two keys collide. + +**Architecture:** `loadRegionKeys` returns a flat `map[string][]byte` built once at startup and threaded through thirteen call sites. It becomes a live `*regionKeySet` holding an immutable `*regionKeySnapshot` behind an `atomic.Pointer`: the ingest hot path takes one atomic load per packet, and a background refresh builds a replacement off to the side and swaps it in. The snapshot carries a single merged key map to iterate plus a membership set naming which keys came from `hashRegions`, which is what makes the tie-break possible. + +**Tech Stack:** Go 1.x (`cmd/ingestor`, stdlib `testing`, `sync/atomic`, `crypto/hmac`), SQLite via `modernc.org/sqlite`. + +**Spec:** `docs/specs/2026-09-07-auto-region-keys-design.md`, sections 1–3. + +**Status: code complete, two checks deferred.** All eleven tasks are committed +(`ac8ff6d3`, `b222421c`, `068af8f1`, `46c50ff4`, `a19af378`, `757407f0`, `a5575cb2`, +`2e2ae3a6`). Two deviations from this plan, both deliberate: `matchScope`'s removal was +moved from Task 4 into Task 5 alongside the call-site update, and Tasks 5-7 landed as one +commit — splitting either would have left a commit that does not build (AGENTS.md rule 6). + +Verification: full ingestor suite green apart from `TestWriteStatsAtomic_SymlinkAtDestIsReplaced`, +which fails with "A required privilege is not held by the client" — Windows +`SeCreateSymbolicLinkPrivilege`, byte-identical to master and unrelated to this work. +Frontend 692/99/18, `gofmt -l` and `go vet` clean, `config.example.json` valid. + +Deferred: **`go test -race` could not run here** (needs cgo, no gcc on this machine) and +must run in CI on Linux — `atomic.Pointer` is race-free by construction and no published +snapshot is ever mutated, but that is an argument, not a measurement. And Task 11 Step 5, +recording the real ambiguity rate, needs the feature enabled on a live instance. + +**Depends on M0** for its *measurement*, not for its code. Nothing here shares a file with M0 or M1 — this plan is entirely `cmd/ingestor/`. But until M0 fixes forwarder attribution (`### M0` in the spec, `b610d461`), 133 of 205 repeaters have zero attributable evidence, so a derived key that correctly names `#behka` in `transmissions.scope_name` still leaves the declaring repeater in `notObserved`: its hops were discarded before nameability ever came into play. M2's effect on the audit would be exactly zero for 65% of repeaters — indistinguishable from M2 not working. Build order is M0 → M1 → M2. + +Also re-read Task 11 Step 5 in light of that: the ambiguity rate it records is only meaningful once attribution is fixed, and the spec's M0 section calls for re-measuring the first-cause share before M2 is sized at all. + +--- + +## Design constraints the implementer must not relax + +- **Default off.** `autoRegionKeys.enabled` absent or `false` must leave behaviour byte-for-byte identical to today. This is asserted by a test, not by inspection. +- **Naming a packet wrongly is worse than not naming it.** The existing `""` (ambiguous) outcome stays the fallback. Tier 2 resolves only the case where operator config and RF hearsay disagree; it never picks between two equally-sourced candidates. +- **No schema migration, no new column.** The match reason goes to logs and an in-process counter. +- **The work is O(keys) per transport-scoped packet and cannot be indexed** — `code1` is an HMAC over the payload, so there is no payload-independent lookup key. The comment in `matchScope` suggesting a "pre-indexed lookup table" is not achievable; delete it rather than leaving a false lead. This is why the cap exists. + +--- + +## File Structure + +| File | Responsibility | Change | +|---|---|---| +| `cmd/ingestor/region_keys.go` | `regionKeySnapshot`, `regionKeySet`, candidate filter, ranking, `scopeMatch` | **Create** | +| `cmd/ingestor/region_keys_test.go` | Unit tests + benchmark for the above | **Create** | +| `cmd/ingestor/config.go` | `AutoRegionKeysConfig` + accessors | Modify | +| `cmd/ingestor/db.go` | `DeclaredRegionStats`; `BuildPacketData` / `BackfillDefaultScopeAsync` signatures | Modify | +| `cmd/ingestor/client_reception.go` | `handleClientPacket` / `buildClientRxObservation` signatures | Modify | +| `cmd/ingestor/main.go` | `matchScope` removal, startup wiring, refresh ticker | Modify | +| `cmd/ingestor/scope_repair.go` | Build the same key set before scanning | Modify | +| `config.example.json` | `autoRegionKeys` block + explainer comment | Modify | +| `docs/client-regions.md` | Document the second consumer of `node_declared_regions` | Modify | + +`docs/api-spec.md` needs **no** change in this plan: M2 alters how the ingestor names a scope, not any server response shape. If you find yourself editing it, you have changed an API surface that was not in scope — stop and re-read the spec. + +--- + +### Task 1: Config block, default off + +**Files:** +- Modify: `cmd/ingestor/config.go` (`Config` struct ~line 61, accessors near `ClientRegionsEnabled` ~line 183) +- Test: `cmd/ingestor/config_test.go` (create the file if absent) + +- [x] **Step 1: Write the failing test** + +```go +func TestAutoRegionKeysDefaultsOff(t *testing.T) { + // An absent block must not enable anything. This is the whole safety + // story: every existing deployment upgrades into unchanged behaviour. + cfg := &Config{} + if cfg.AutoRegionKeysEnabled() { + t.Error("AutoRegionKeysEnabled() = true on an empty config, want false") + } + if got := cfg.AutoRegionKeysMaxDerived(); got != 256 { + t.Errorf("AutoRegionKeysMaxDerived() = %d, want the 256 default", got) + } + if got := cfg.AutoRegionKeysRefreshMinutes(); got != 15 { + t.Errorf("AutoRegionKeysRefreshMinutes() = %d, want the 15 default", got) + } +} + +func TestAutoRegionKeysExplicitValues(t *testing.T) { + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true, MaxDerived: 64, RefreshMinutes: 5}} + if !cfg.AutoRegionKeysEnabled() { + t.Error("AutoRegionKeysEnabled() = false, want true") + } + if got := cfg.AutoRegionKeysMaxDerived(); got != 64 { + t.Errorf("AutoRegionKeysMaxDerived() = %d, want 64", got) + } + if got := cfg.AutoRegionKeysRefreshMinutes(); got != 5 { + t.Errorf("AutoRegionKeysRefreshMinutes() = %d, want 5", got) + } +} + +func TestAutoRegionKeysRejectsNonPositiveOverrides(t *testing.T) { + // A zero is indistinguishable from "absent" after json.Unmarshal, and a + // negative is a typo. Both fall back to the default rather than + // silently disabling derivation or spinning the refresh ticker. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true, MaxDerived: 0, RefreshMinutes: -1}} + if got := cfg.AutoRegionKeysMaxDerived(); got != 256 { + t.Errorf("AutoRegionKeysMaxDerived() = %d, want the 256 default", got) + } + if got := cfg.AutoRegionKeysRefreshMinutes(); got != 15 { + t.Errorf("AutoRegionKeysRefreshMinutes() = %d, want the 15 default", got) + } +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/ingestor && go test ./... -run TestAutoRegionKeys -v` +Expected: FAIL to compile — `undefined: AutoRegionKeysConfig` + +- [x] **Step 3: Implement** + +In `cmd/ingestor/config.go`, add to the `Config` struct after `ClientRegions`: + +```go + AutoRegionKeys *AutoRegionKeysConfig `json:"autoRegionKeys,omitempty"` +``` + +And after `ClientRegionsEnabled`: + +```go +// AutoRegionKeysConfig controls the opt-in derivation of region keys from the +// names repeaters declare over RF (node_declared_regions), on top of the +// explicit hashRegions list. +// +// TOP-LEVEL BLOCK, a sibling of hashRegions — not nested inside it. Config +// loading is plain json.Unmarshal with no DisallowUnknownFields, so a +// mis-nested key is silently ignored and derivation stays off with no error, +// the same trap clientRxObservations documents. +type AutoRegionKeysConfig struct { + Enabled bool `json:"enabled"` + MaxDerived int `json:"maxDerived"` + RefreshMinutes int `json:"refreshMinutes"` +} + +// autoRegionKeysDefaultMaxDerived bounds the derived tier. Every added key +// raises the random ambiguity rate by 1/65536 per scoped packet (code1 is two +// bytes), and costs one more HMAC per transport-scoped packet — the match is +// O(keys) and cannot be indexed. 256 on top of a typical explicit set puts the +// ambiguity rate near 0.5%, which is the ceiling this design accepts. +const autoRegionKeysDefaultMaxDerived = 256 + +// autoRegionKeysDefaultRefreshMinutes is how often the derived tier is rebuilt +// from the database. Declared-region answers arrive at human pace (a drive-by +// with a companion app, or a 24h observer report), so minutes-scale staleness +// is irrelevant and a tighter interval only burns queries. +const autoRegionKeysDefaultRefreshMinutes = 15 + +// AutoRegionKeysEnabled reports whether region keys may be derived from +// declared-region answers. Default false. +func (c *Config) AutoRegionKeysEnabled() bool { + return c.AutoRegionKeys != nil && c.AutoRegionKeys.Enabled +} + +// AutoRegionKeysMaxDerived returns the derived-tier cap, falling back to the +// default for absent, zero, or negative values — a zero is indistinguishable +// from "key omitted" after json.Unmarshal, and neither should silently mean +// "derive nothing" when the operator has switched the feature on. +func (c *Config) AutoRegionKeysMaxDerived() int { + if c.AutoRegionKeys == nil || c.AutoRegionKeys.MaxDerived <= 0 { + return autoRegionKeysDefaultMaxDerived + } + return c.AutoRegionKeys.MaxDerived +} + +// AutoRegionKeysRefreshMinutes returns the refresh interval in minutes, +// falling back to the default for absent, zero, or negative values — a zero +// here would otherwise panic time.NewTicker. +func (c *Config) AutoRegionKeysRefreshMinutes() int { + if c.AutoRegionKeys == nil || c.AutoRegionKeys.RefreshMinutes <= 0 { + return autoRegionKeysDefaultRefreshMinutes + } + return c.AutoRegionKeys.RefreshMinutes +} +``` + +- [x] **Step 4: Run tests to verify they pass** + +Run: `cd cmd/ingestor && go test ./... -run TestAutoRegionKeys -v` +Expected: PASS (3 tests) + +- [x] **Step 5: Commit** + +```bash +git add cmd/ingestor/config.go cmd/ingestor/config_test.go +git commit -m "feat(ingestor): autoRegionKeys config block, default off" +``` + +--- + +### Task 2: Candidate filter and ranking + +Pure functions, no database. Built before the DB read so the ranking rules are pinned by tests independent of SQL. + +**Files:** +- Create: `cmd/ingestor/region_keys.go` +- Create: `cmd/ingestor/region_keys_test.go` + +- [x] **Step 1: Write the failing test** + +Create `cmd/ingestor/region_keys_test.go`: + +```go +package main + +import ( + "strings" + "testing" +) + +func TestRegionNameAcceptable(t *testing.T) { + cases := []struct { + name string + want bool + why string + }{ + {"be", true, "ordinary short name"}, + {"nl-li-sit", true, "hyphenated hierarchical name"}, + {"fm-112", true, "digits are fine"}, + {"null", true, "looks like junk but is a legal name — no value blocklist"}, + {"", false, "empty"}, + {strings.Repeat("a", 33), false, "over the 32-char limit"}, + {strings.Repeat("a", 32), true, "exactly at the limit"}, + {"be,eu", false, "a comma is the regions_csv delimiter and would split on reload"}, + {"#be", false, "the firmware strips '#', so its presence signals a malformed entry"}, + {"be\x00", false, "NUL padding that a stale client failed to trim"}, + {"be eu", false, "whitespace inside a region name is never emitted by firmware"}, + {"bé", false, "non-ASCII: the key is SHA256 over bytes, so encoding drift would silently mismatch"}, + } + for _, c := range cases { + if got := regionNameAcceptable(c.name); got != c.want { + t.Errorf("regionNameAcceptable(%q) = %v, want %v — %s", c.name, got, c.want, c.why) + } + } +} + +func TestRankDeclaredRegionsPrefersWidelyDeclared(t *testing.T) { + // The cap must drop the long tail of one-off local names, never a region + // half the network declares. On live data "be" is declared by 127 + // repeaters and "behss" by 3. + stats := []declaredRegionStat{ + {Name: "behss", Declarers: 3, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "be", Declarers: 127, LastSeen: "2026-09-01T10:00:00Z"}, + {Name: "sol3", Declarers: 1, LastSeen: "2026-09-07T11:00:00Z"}, + } + got := rankDeclaredRegions(stats, 2) + want := []string{"be", "behss"} + if len(got) != len(want) { + t.Fatalf("rankDeclaredRegions = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("rankDeclaredRegions = %v, want %v — declarer count must dominate recency", got, want) + } + } +} + +func TestRankDeclaredRegionsIsDeterministic(t *testing.T) { + // Equal declarer counts and equal timestamps must still produce a stable + // order, or the derived tier churns between refreshes and the logs become + // unreadable. + stats := []declaredRegionStat{ + {Name: "zz", Declarers: 2, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "aa", Declarers: 2, LastSeen: "2026-09-07T10:00:00Z"}, + } + for i := 0; i < 20; i++ { + got := rankDeclaredRegions(stats, 10) + if got[0] != "aa" || got[1] != "zz" { + t.Fatalf("run %d: rankDeclaredRegions = %v, want [aa zz]", i, got) + } + } +} + +func TestRankDeclaredRegionsBreaksTiesOnRecency(t *testing.T) { + stats := []declaredRegionStat{ + {Name: "old", Declarers: 2, LastSeen: "2026-01-01T00:00:00Z"}, + {Name: "new", Declarers: 2, LastSeen: "2026-09-07T00:00:00Z"}, + } + got := rankDeclaredRegions(stats, 1) + if len(got) != 1 || got[0] != "new" { + t.Fatalf("rankDeclaredRegions = %v, want [new] — equal declarers break on recency", got) + } +} + +func TestRankDeclaredRegionsDropsUnacceptableNames(t *testing.T) { + stats := []declaredRegionStat{ + {Name: "be", Declarers: 5, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "#bad", Declarers: 99, LastSeen: "2026-09-07T10:00:00Z"}, + } + got := rankDeclaredRegions(stats, 10) + if len(got) != 1 || got[0] != "be" { + t.Fatalf("rankDeclaredRegions = %v, want [be] — an unacceptable name must be dropped however widely declared", got) + } +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/ingestor && go test ./... -run 'TestRegionName|TestRankDeclared' -v` +Expected: FAIL to compile — `undefined: regionNameAcceptable`, `undefined: declaredRegionStat` + +- [x] **Step 3: Implement** + +Create `cmd/ingestor/region_keys.go`: + +```go +package main + +import ( + "sort" + "strings" +) + +// declaredRegionStat is one region name as reported over RF, with the two +// facts the cap ranks on: how many distinct repeaters declare it, and how +// recently any of them last answered. +type declaredRegionStat struct { + Name string + Declarers int + LastSeen string // ISO, greatest observed_at across declarers +} + +// maxRegionNameLen bounds a derived region name. Firmware region names are +// short labels; anything longer is a malformed or hostile entry, and each +// accepted name costs an HMAC on every transport-scoped packet. +const maxRegionNameLen = 32 + +// regionNameAcceptable reports whether a declared name may become a derived +// region key. +// +// The rules are structural, never about the name's meaning. The declared set +// contains entries that look like junk ("null", "bierhuis", "sol3"), but a +// blocklist on string values is unmaintainable and the cost of one bad name is +// a single slot out of maxDerived plus a 1-in-65536 collision chance. What IS +// rejected is anything that could not have come from the firmware intact: +// +// - a comma would split the name on the next regions_csv round-trip +// - a '#' cannot appear (the firmware strips it), so its presence means the +// value was mangled somewhere upstream +// - non-ASCII or whitespace would make the key SHA256 over bytes nobody +// intended, silently mismatching the sender +// - a NUL is the block-cipher padding a stale client failed to trim +func regionNameAcceptable(name string) bool { + if name == "" || len(name) > maxRegionNameLen { + return false + } + for i := 0; i < len(name); i++ { + c := name[i] + if c <= ' ' || c >= 0x7F || c == ',' || c == '#' { + return false + } + } + return true +} + +// rankDeclaredRegions filters stats through regionNameAcceptable and returns at +// most max names, most-worth-keeping first: by declarer count descending, then +// by recency, then by name. The name tie-break is what makes the result +// deterministic — without it the derived tier would churn between refreshes on +// equally-ranked names and the add/drop logging would be noise. +func rankDeclaredRegions(stats []declaredRegionStat, max int) []string { + kept := make([]declaredRegionStat, 0, len(stats)) + for _, s := range stats { + if regionNameAcceptable(s.Name) { + kept = append(kept, s) + } + } + sort.Slice(kept, func(i, j int) bool { + if kept[i].Declarers != kept[j].Declarers { + return kept[i].Declarers > kept[j].Declarers + } + if kept[i].LastSeen != kept[j].LastSeen { + return kept[i].LastSeen > kept[j].LastSeen + } + return kept[i].Name < kept[j].Name + }) + if max > 0 && len(kept) > max { + kept = kept[:max] + } + names := make([]string, 0, len(kept)) + for _, s := range kept { + names = append(names, s.Name) + } + return names +} + +// splitDeclaredRegionsCSV parses a regions_csv value into its entries. The +// ingestor writes this column with strings.Join(regions, ","), so this is its +// exact inverse. Mirrors splitRegionsCSV in cmd/server/scopes.go. +func splitDeclaredRegionsCSV(csv string) []string { + out := []string{} + if csv == "" { + return out + } + for _, part := range strings.Split(csv, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} +``` + +- [x] **Step 4: Run tests to verify they pass** + +Run: `cd cmd/ingestor && go test ./... -run 'TestRegionName|TestRankDeclared' -v` +Expected: PASS (5 tests) + +- [x] **Step 5: Commit** + +```bash +git add cmd/ingestor/region_keys.go cmd/ingestor/region_keys_test.go +git commit -m "feat(ingestor): candidate filter and deterministic ranking for derived region names" +``` + +--- + +### Task 3: The key set — two tiers behind an atomic snapshot + +**Files:** +- Modify: `cmd/ingestor/region_keys.go` +- Test: `cmd/ingestor/region_keys_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/ingestor/region_keys_test.go`: + +```go +func TestRegionKeySetExplicitOnlyWhenDisabled(t *testing.T) { + // Derivation off: the snapshot must be exactly what loadRegionKeys built, + // and refreshDerived must be a no-op rather than a quiet opt-in. + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss", "fm-112"}) + + snap := set.snapshot() + if len(snap.all) != 1 { + t.Fatalf("len(all) = %d, want 1 — refreshDerived must not add keys when disabled", len(snap.all)) + } + if _, ok := snap.all["#be"]; !ok { + t.Error("want the explicit #be key present") + } + if !snap.isExplicit("#be") { + t.Error("isExplicit(#be) = false, want true") + } +} + +func TestRegionKeySetMergesDerivedWhenEnabled(t *testing.T) { + cfg := &Config{ + HashRegions: []string{"#be"}, + AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}, + } + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss", "be"}) // "be" duplicates the explicit key + + snap := set.snapshot() + if len(snap.all) != 2 { + t.Fatalf("len(all) = %d, want 2 (#be explicit + #behss derived), got keys %v", len(snap.all), keyNames(snap)) + } + if _, ok := snap.all["#behss"]; !ok { + t.Errorf("want the derived #behss key present, got %v", keyNames(snap)) + } + if snap.isExplicit("#behss") { + t.Error("isExplicit(#behss) = true, want false — a derived key is not operator config") + } + if !snap.isExplicit("#be") { + t.Error("isExplicit(#be) = false, want true — an explicit key must not be demoted by a duplicate declaration") + } +} + +func TestRegionKeySetRefreshReplacesRatherThanAccumulates(t *testing.T) { + // A region that stops being declared must leave the derived tier, or the + // key set only ever grows and the cap stops meaning anything. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa"}) + set.refreshDerived([]string{"bb"}) + + snap := set.snapshot() + if _, ok := snap.all["#aa"]; ok { + t.Error("want #aa gone after a refresh that no longer lists it") + } + if _, ok := snap.all["#bb"]; !ok { + t.Error("want #bb present after the refresh that lists it") + } +} + +func TestRegionKeySetSnapshotIsStable(t *testing.T) { + // A snapshot handed to a packet must not change under it mid-match. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa"}) + held := set.snapshot() + set.refreshDerived([]string{"bb"}) + + if _, ok := held.all["#aa"]; !ok { + t.Error("the held snapshot lost #aa — snapshots must be immutable, not aliases of live state") + } +} + +// keyNames is a test helper for readable failure messages. +func keyNames(s *regionKeySnapshot) []string { + out := make([]string, 0, len(s.all)) + for k := range s.all { + out = append(out, k) + } + sort.Strings(out) + return out +} +``` + +Add `"sort"` to the test file's imports. + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/ingestor && go test ./... -run TestRegionKeySet -v` +Expected: FAIL to compile — `undefined: newRegionKeySet` + +- [x] **Step 3: Implement** + +Append to `cmd/ingestor/region_keys.go` (add `"crypto/sha256"` and `"sync/atomic"` to its imports): + +```go +// regionKeySnapshot is an immutable view of the region keys in force for one +// packet. `all` is the single map matching iterates — merging at build time +// rather than per packet keeps the hot path free of allocation. `explicit` +// carries membership only, and exists so the ambiguity tie-break can tell an +// operator-configured region from one derived off the air. +type regionKeySnapshot struct { + all map[string][]byte + explicit map[string]bool +} + +func (s *regionKeySnapshot) isExplicit(name string) bool { return s.explicit[name] } + +// regionKeySet holds the live snapshot. Readers take one atomic load; a +// refresh builds the replacement off to the side and swaps the pointer, so the +// ingest hot path never blocks on a rebuild (AGENTS.md rule 0). +type regionKeySet struct { + cur atomic.Pointer[regionKeySnapshot] + enabled bool + max int +} + +// newRegionKeySet builds the explicit tier from hashRegions. The derived tier +// starts empty; refreshDerived fills it, and does nothing at all when +// autoRegionKeys is off. +func newRegionKeySet(cfg *Config) *regionKeySet { + explicitKeys := loadRegionKeys(cfg) + explicitNames := make(map[string]bool, len(explicitKeys)) + all := make(map[string][]byte, len(explicitKeys)) + for name, key := range explicitKeys { + explicitNames[name] = true + all[name] = key + } + s := ®ionKeySet{ + enabled: cfg.AutoRegionKeysEnabled(), + max: cfg.AutoRegionKeysMaxDerived(), + } + s.cur.Store(®ionKeySnapshot{all: all, explicit: explicitNames}) + return s +} + +func (s *regionKeySet) snapshot() *regionKeySnapshot { return s.cur.Load() } + +// refreshDerived rebuilds the derived tier from names (already ranked and +// capped by the caller) and swaps in a new snapshot. It REPLACES the derived +// tier rather than merging into it, so a region that stops being declared +// leaves the key set and the cap keeps meaning something. +// +// A name that duplicates an explicit key is skipped, not re-added: the +// explicit tier must stay authoritative for the tie-break, and demoting a +// configured region because a repeater also declares it would invert the whole +// rule. +// +// Returns the names actually added, for the caller to log. +func (s *regionKeySet) refreshDerived(names []string) []string { + if !s.enabled { + return nil + } + old := s.cur.Load() + all := make(map[string][]byte, len(old.explicit)+len(names)) + for name := range old.explicit { + all[name] = old.all[name] + } + added := make([]string, 0, len(names)) + for _, raw := range names { + if !regionNameAcceptable(raw) { + continue + } + name := "#" + raw + if old.explicit[name] { + continue + } + if _, exists := all[name]; exists { + continue + } + h := sha256.Sum256([]byte(name)) + all[name] = h[:16] + added = append(added, name) + } + s.cur.Store(®ionKeySnapshot{all: all, explicit: old.explicit}) + return added +} +``` + +- [x] **Step 4: Run tests to verify they pass** + +Run: `cd cmd/ingestor && go test ./... -run TestRegionKeySet -v` +Expected: PASS (4 tests) + +- [x] **Step 5: Run the race detector** + +Run: `cd cmd/ingestor && go test ./... -race -run TestRegionKeySet` +Expected: PASS with no race reported. + +- [x] **Step 6: Commit** + +```bash +git add cmd/ingestor/region_keys.go cmd/ingestor/region_keys_test.go +git commit -m "feat(ingestor): two-tier regionKeySet behind an atomic snapshot" +``` + +--- + +### Task 4: Tiered matching with a reason + +**Files:** +- Modify: `cmd/ingestor/region_keys.go` +- Modify: `cmd/ingestor/main.go` (delete `matchScope`, keep `matchingRegions`) +- Test: `cmd/ingestor/region_keys_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/ingestor/region_keys_test.go`: + +```go +// codeFor derives the on-wire code1 a sender in region `name` would emit for +// this payload — the same computation matchingRegions inverts. Used to build +// packets that genuinely belong to a region rather than asserting on a +// hardcoded string. +func codeFor(name string, payloadType byte, payload []byte) string { + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + sum := sha256.Sum256([]byte(name)) + mac := hmac.New(sha256.New, sum[:16]) + mac.Write([]byte{payloadType}) + mac.Write(payload) + h := mac.Sum(nil) + code := uint16(h[0]) | uint16(h[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return strings.ToUpper(hex.EncodeToString([]byte{byte(code & 0xFF), byte(code >> 8)})) +} + +func TestScopeMatchUniqueNamesTheRegion(t *testing.T) { + payload := []byte{0xDE, 0xAD, 0xBE, 0xEF} + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + got := set.snapshot().match(5, payload, code) + if got.Name != "#be" { + t.Errorf("Name = %q, want %q", got.Name, "#be") + } + if got.Reason != scopeReasonUnique { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonUnique) + } +} + +func TestScopeMatchNoKeyMatches(t *testing.T) { + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + + got := set.snapshot().match(5, []byte{1, 2, 3}, "0000") + if got.Name != "" || got.Reason != scopeReasonNone { + t.Errorf("got %+v, want an empty name with reason %q", got, scopeReasonNone) + } +} + +func TestScopeMatchExplicitBeatsDerived(t *testing.T) { + // The ambiguity this feature introduces: a derived key collides with an + // operator-configured one on this payload. Operator config wins — it is + // intent, the derived name is hearsay picked up over RF. + payload := []byte{0x01, 0x02, 0x03, 0x04} + cfg := &Config{HashRegions: []string{"#be"}, AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + // Force the collision rather than searching for a natural one: inject a + // derived key whose bytes are the explicit key's, so both match. + snap := set.snapshot() + collide := make(map[string][]byte, len(snap.all)+1) + for k, v := range snap.all { + collide[k] = v + } + collide["#collider"] = snap.all["#be"] + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + + got := forced.match(5, payload, code) + if got.Name != "#be" { + t.Errorf("Name = %q, want %q — the explicit key must win", got.Name, "#be") + } + if got.Reason != scopeReasonExplicitOverDerived { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonExplicitOverDerived) + } + if len(got.Candidates) != 2 { + t.Errorf("Candidates = %v, want both names recorded for the log", got.Candidates) + } +} + +func TestScopeMatchTwoExplicitKeysStayAmbiguous(t *testing.T) { + // Two equally-sourced candidates: naming either would be a guess, and + // naming wrongly is worse than not naming. This is #1609's rule, unchanged. + payload := []byte{0x09, 0x08, 0x07} + cfg := &Config{HashRegions: []string{"#be", "#eu"}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + snap := set.snapshot() + collide := map[string][]byte{"#be": snap.all["#be"], "#eu": snap.all["#be"]} + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + + got := forced.match(5, payload, code) + if got.Name != "" { + t.Errorf("Name = %q, want \"\" — two explicit candidates must abstain", got.Name) + } + if got.Reason != scopeReasonAmbiguous { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonAmbiguous) + } +} + +func TestScopeMatchTwoDerivedKeysStayAmbiguous(t *testing.T) { + // The tier-3 case, deliberately NOT resolved in M2. It must abstain rather + // than pick, and the reason must say ambiguous so the log can measure how + // often this happens before tier 3 is built. + payload := []byte{0x11, 0x22} + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa", "bb"}) + + snap := set.snapshot() + collide := map[string][]byte{"#aa": snap.all["#aa"], "#bb": snap.all["#aa"]} + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + code := codeFor("#aa", 5, payload) + + got := forced.match(5, payload, code) + if got.Name != "" || got.Reason != scopeReasonAmbiguous { + t.Errorf("got %+v, want an empty name with reason %q", got, scopeReasonAmbiguous) + } +} +``` + +Add `"crypto/hmac"`, `"crypto/sha256"`, and `"encoding/hex"` to the test file's imports. + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/ingestor && go test ./... -run TestScopeMatch -v` +Expected: FAIL to compile — `snap.match undefined`, `undefined: scopeReasonUnique` + +- [x] **Step 3: Implement** + +Append to `cmd/ingestor/region_keys.go`: + +```go +// scopeReason records how a scope match was decided, so the outcome is +// auditable in logs without a schema change. It is deliberately not stored: +// transmissions.scope_name keeps its existing three-state encoding. +type scopeReason string + +const ( + scopeReasonNone scopeReason = "none" // no key matched + scopeReasonUnique scopeReason = "unique" // exactly one key matched + scopeReasonExplicitOverDerived scopeReason = "explicit-over-derived" // several matched, one was operator config + scopeReasonAmbiguous scopeReason = "ambiguous" // several matched, no principled winner +) + +// scopeMatch is the result of naming one packet's region scope. +type scopeMatch struct { + Name string // "" when unresolved — the caller stores that as the unmatched state + Reason scopeReason + Candidates []string // every matching name, populated only when more than one matched +} + +// match names the region scope of a transport-scoped packet, resolving a +// multi-key collision by evidence rather than by map order. +// +// Tiers, in order: +// +// 1. Exactly one key matched — name it. +// 2. Several matched but exactly one came from hashRegions — name that one. +// The operator's own configuration outranks a name picked up off the air, +// and this covers the bulk of the ambiguity auto-derivation introduces. +// 3. Otherwise abstain, returning "". Two equally-sourced candidates offer no +// principled winner, and naming a packet wrongly is worse than leaving it +// unnamed — the rule #1609 established, unchanged. +// +// (The spec's tier-3 path-evidence tie-break sits between 2 and 3 and is +// deliberately not built here; see docs/specs/2026-09-07-auto-region-keys-design.md. +// The scopeReasonAmbiguous counter is what measures whether it is worth building.) +func (s *regionKeySnapshot) match(payloadType byte, payloadRaw []byte, code1 string) scopeMatch { + matched := matchingRegions(s.all, payloadType, payloadRaw, code1) + switch len(matched) { + case 0: + return scopeMatch{Reason: scopeReasonNone} + case 1: + return scopeMatch{Name: matched[0], Reason: scopeReasonUnique} + } + + var explicitMatches []string + for _, name := range matched { + if s.explicit[name] { + explicitMatches = append(explicitMatches, name) + } + } + if len(explicitMatches) == 1 { + return scopeMatch{Name: explicitMatches[0], Reason: scopeReasonExplicitOverDerived, Candidates: matched} + } + return scopeMatch{Reason: scopeReasonAmbiguous, Candidates: matched} +} +``` + +- [x] **Step 4: Delete the superseded `matchScope`** + +In `cmd/ingestor/main.go`, delete the `matchScope` function and its doc comment entirely (the block ending `return ""` just above `matchingRegions`). Keep `matchingRegions` unchanged — `match` calls it. + +While deleting, note that the old comment's suggestion to "consider a pre-indexed lookup table" beyond 50 regions goes with it. That is not achievable: `code1` is an HMAC over the packet payload, so there is no payload-independent key to index on. The cost is inherently one HMAC per configured region per transport-scoped packet, which is precisely why `maxDerived` exists. + +- [x] **Step 5: Run tests to verify they pass** + +Run: `cd cmd/ingestor && go test ./... -run TestScopeMatch -v` +Expected: PASS (5 tests). The build will still fail elsewhere until Task 5 updates the call sites — that is expected; run with `-run` scoped as shown, and do not "fix" the callers yet. + +- [x] **Step 6: Commit** + +```bash +git add cmd/ingestor/region_keys.go cmd/ingestor/region_keys_test.go cmd/ingestor/main.go +git commit -m "feat(ingestor): tiered scope matching with an explicit-over-derived tie-break" +``` + +--- + +### Task 5: Thread `*regionKeySet` through the call sites + +Mechanical but wide. All thirteen sites, in one commit, so the tree is never half-converted. + +**Files:** +- Modify: `cmd/ingestor/main.go` (lines ~111, 191, 686, 732, 973, 1008) +- Modify: `cmd/ingestor/client_reception.go` (lines ~26, 96, 425, 450) +- Modify: `cmd/ingestor/db.go` (lines ~1610, 2133, 2186) + +- [x] **Step 1: Change the signatures** + +Replace the parameter type `regionKeys map[string][]byte` with `regionSet *regionKeySet` in: + +| File | Function | +|---|---| +| `client_reception.go` | `handleClientPacket` | +| `client_reception.go` | `buildClientRxObservation` | +| `db.go` | `BackfillDefaultScopeAsync` | +| `db.go` | `BuildPacketData` | +| `main.go` | `handleMessage` | + +- [x] **Step 2: Add the counter** + +Append to `cmd/ingestor/region_keys.go` (add `"log"` to imports): + +```go +// scopeMatchCounters tallies how each transport-scoped packet's region was +// decided. It exists to answer one question before more machinery is built: +// how often does an ambiguous collision actually happen? The spec gates the +// path-evidence tie-break (tier 3) on this number. +var scopeMatchCounters struct { + unique atomic.Int64 + explicitOverDerived atomic.Int64 + ambiguous atomic.Int64 + none atomic.Int64 +} + +// recordScopeMatch tallies one decision and logs the interesting ones. Unique +// and none are the overwhelming majority and are counted silently; the other +// two are rare by construction and worth a line each. +func recordScopeMatch(m scopeMatch) { + switch m.Reason { + case scopeReasonUnique: + scopeMatchCounters.unique.Add(1) + case scopeReasonNone: + scopeMatchCounters.none.Add(1) + case scopeReasonExplicitOverDerived: + scopeMatchCounters.explicitOverDerived.Add(1) + log.Printf("[regions] collision resolved to explicit %s over derived candidates %v", m.Name, m.Candidates) + case scopeReasonAmbiguous: + scopeMatchCounters.ambiguous.Add(1) + log.Printf("[regions] ambiguous collision between %v; storing unmatched", m.Candidates) + } +} + +// logScopeMatchCounters prints the running tally. Called from the refresh +// ticker so the numbers arrive on the same cadence as the key-set changes that +// move them. +func logScopeMatchCounters() { + log.Printf("[regions] scope matches: unique=%d explicit-over-derived=%d ambiguous=%d none=%d", + scopeMatchCounters.unique.Load(), scopeMatchCounters.explicitOverDerived.Load(), + scopeMatchCounters.ambiguous.Load(), scopeMatchCounters.none.Load()) +} +``` + +- [x] **Step 3: Change the two match call sites** + +In `client_reception.go`, replace: + +```go + if decoded.TransportCodes.Code1 != "0000" { + sn := matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) +``` + +with: + +```go + if decoded.TransportCodes.Code1 != "0000" { + m := regionSet.snapshot().match(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + recordScopeMatch(m) + sn := m.Name +``` + +In `db.go`, inside `BuildPacketData`, replace: + +```go + pd.ScopeName = matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) +``` + +with: + +```go + m := regionSet.snapshot().match(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + recordScopeMatch(m) + pd.ScopeName = m.Name +``` + +- [x] **Step 4: Update the callers** + +In `main.go`, change line ~111: + +```go + regionSet := newRegionKeySet(cfg) + store.BackfillDefaultScopeAsync(regionSet) +``` + +and pass `regionSet` instead of `regionKeys` at every call to `handleMessage`, `handleClientPacket`, and `BuildPacketData`. + +In `db.go`'s `BackfillDefaultScopeAsync`, replace the `len(regionKeys) == 0` early return with: + +```go + if len(regionSet.snapshot().all) == 0 { +``` + +In `client_reception.go`, pass `regionSet` through to `buildClientRxObservation`. + +- [x] **Step 5: Build and run the full suite** + +Run: `cd cmd/ingestor && go build ./... && go test ./...` +Expected: build succeeds, all tests PASS. `scope_repair.go` still compiles because it calls `matchingRegions` directly, not `matchScope` — Task 7 changes its key source. + +- [x] **Step 6: Commit** + +```bash +git add cmd/ingestor/ +git commit -m "refactor(ingestor): thread *regionKeySet through the ingest path" +``` + +--- + +### Task 6: Read declared region names from the database + +**Files:** +- Modify: `cmd/ingestor/db.go` +- Test: `cmd/ingestor/region_keys_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/ingestor/region_keys_test.go`: + +```go +func TestDeclaredRegionStatsAggregatesLatestAnswerPerTarget(t *testing.T) { + store := newTestStore(t) + // Two answers from the same target: only the newer one counts, exactly as + // CurrentDeclaredRegions orders (by observed_at, never ingested_at — a + // drive buffered offline can arrive days late). + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-01T00:00:00Z", "be,old") + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "be,new") + insertDeclaredRegionsRow(t, store, "bb"+strings.Repeat("22", 31), "2026-09-05T00:00:00Z", "be") + + stats, err := store.DeclaredRegionStats() + if err != nil { + t.Fatal(err) + } + byName := map[string]declaredRegionStat{} + for _, s := range stats { + byName[s.Name] = s + } + if got := byName["be"].Declarers; got != 2 { + t.Errorf("be declarers = %d, want 2", got) + } + if got := byName["be"].LastSeen; got != "2026-09-07T00:00:00Z" { + t.Errorf("be lastSeen = %q, want the greatest observed_at", got) + } + if _, ok := byName["old"]; ok { + t.Error("want the superseded answer's region gone — only the latest answer per target counts") + } + if got := byName["new"].Declarers; got != 1 { + t.Errorf("new declarers = %d, want 1", got) + } +} + +// insertDeclaredRegionsRow seeds one node_declared_regions answer. +func insertDeclaredRegionsRow(t *testing.T, s *Store, target, observedAt, regionsCSV string) { + t.Helper() + _, err := s.db.Exec( + `INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated) + VALUES (?, 'rx', ?, ?, ?, 0)`, + target, observedAt, observedAt, regionsCSV) + if err != nil { + t.Fatal(err) + } +} +``` + +`newTestStore` is defined in `cmd/ingestor/main_test.go:121` and opens a real store via `OpenStore`, so the full schema — `node_declared_regions` included — already exists. Do not add a second helper. + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/ingestor && go test ./... -run TestDeclaredRegionStats -v` +Expected: FAIL to compile — `store.DeclaredRegionStats undefined` + +- [x] **Step 3: Implement** + +Add to `cmd/ingestor/db.go`, beside `CurrentDeclaredRegions`: + +```go +// DeclaredRegionStats returns every region name currently declared anywhere on +// the network, with the two facts the derived-tier cap ranks on: how many +// distinct repeaters declare it, and the most recent observed_at among them. +// +// Only the LATEST answer per target counts — the same rule +// CurrentDeclaredRegions follows, by observed_at and never ingested_at, so a +// drive buffered offline cannot resurrect a region a repeater has since +// dropped. The window function is covered by idx_ndr_target(target, +// observed_at). +// +// CSV splitting and aggregation happen in Go rather than SQL: regions_csv is +// written with strings.Join, and unpicking it in SQLite would need a recursive +// CTE for no gain at this row count (~200 targets). +func (s *Store) DeclaredRegionStats() ([]declaredRegionStat, error) { + rows, err := s.db.Query(` + WITH ranked AS ( + SELECT target, observed_at, regions_csv, + ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC) AS rn + FROM node_declared_regions + ) + SELECT target, observed_at, regions_csv FROM ranked WHERE rn = 1 + `) + if err != nil { + return nil, fmt.Errorf("declared region stats: %w", err) + } + defer rows.Close() + + agg := map[string]*declaredRegionStat{} + for rows.Next() { + var target, observedAt, csv string + if err := rows.Scan(&target, &observedAt, &csv); err != nil { + return nil, fmt.Errorf("declared region stats scan: %w", err) + } + seenHere := map[string]bool{} // one target counts once per name + for _, name := range splitDeclaredRegionsCSV(csv) { + if name == "*" || seenHere[name] { + continue // '*' is the wildcard, not a region name + } + seenHere[name] = true + st, ok := agg[name] + if !ok { + st = &declaredRegionStat{Name: name} + agg[name] = st + } + st.Declarers++ + if observedAt > st.LastSeen { + st.LastSeen = observedAt + } + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("declared region stats rows: %w", err) + } + + out := make([]declaredRegionStat, 0, len(agg)) + for _, st := range agg { + out = append(out, *st) + } + return out, nil +} +``` + +- [x] **Step 4: Run tests to verify they pass** + +Run: `cd cmd/ingestor && go test ./... -run TestDeclaredRegionStats -v` +Expected: PASS + +- [x] **Step 5: Commit** + +```bash +git add cmd/ingestor/db.go cmd/ingestor/region_keys_test.go +git commit -m "feat(ingestor): DeclaredRegionStats — declared names with declarer counts" +``` + +--- + +### Task 7: Wire the refresh into startup and a ticker + +**Files:** +- Modify: `cmd/ingestor/region_keys.go` (a `refreshFromStore` helper so both startup and the ticker share one path) +- Modify: `cmd/ingestor/main.go` + +- [x] **Step 1: Add the shared refresh helper** + +Append to `cmd/ingestor/region_keys.go`: + +```go +// refreshFromStore reads the declared region names, ranks and caps them, and +// swaps in a new snapshot. Shared by startup and the ticker so both apply +// identical rules. A DB error is logged and the current snapshot is kept — a +// failed refresh must never empty the key set and silently unname all traffic. +func (s *regionKeySet) refreshFromStore(store *Store) { + if !s.enabled { + return + } + stats, err := store.DeclaredRegionStats() + if err != nil { + log.Printf("[regions] derived-key refresh failed, keeping %d existing key(s): %v", len(s.snapshot().all), err) + return + } + ranked := rankDeclaredRegions(stats, s.max) + added := s.refreshDerived(ranked) + snap := s.snapshot() + log.Printf("[regions] derived-key refresh: %d name(s) declared, %d kept after filter+cap(%d), %d total key(s) in force", + len(stats), len(ranked), s.max, len(snap.all)) + if len(added) > 0 { + log.Printf("[regions] derived keys now active: %v", added) + } + if len(stats) > s.max { + log.Printf("[regions] NOTE: %d declared name(s) exceeded maxDerived=%d and were dropped, least-declared first", len(stats)-s.max, s.max) + } +} +``` + +- [x] **Step 2: Call it at startup** + +In `cmd/ingestor/main.go`, replace the line added in Task 5: + +```go + regionSet := newRegionKeySet(cfg) +``` + +with: + +```go + regionSet := newRegionKeySet(cfg) + if cfg.AutoRegionKeysEnabled() { + regionSet.refreshFromStore(store) + } else { + log.Printf("[regions] autoRegionKeys disabled — only the %d configured hashRegions key(s) are in force", len(regionSet.snapshot().all)) + } +``` + +- [x] **Step 3: Add the ticker** + +In `cmd/ingestor/main.go`, after the existing client-RX retention ticker block, add: + +```go + // Derived region keys are refreshed on their own ticker rather than the + // daily retention one: declared-region answers arrive continuously (a + // companion app driving past a repeater), and waiting up to 24h to name a + // newly-discovered region would defeat the point of deriving them at all. + if cfg.AutoRegionKeysEnabled() { + interval := time.Duration(cfg.AutoRegionKeysRefreshMinutes()) * time.Minute + regionRefreshTicker := time.NewTicker(interval) + go func() { + for range regionRefreshTicker.C { + regionSet.refreshFromStore(store) + logScopeMatchCounters() + } + }() + log.Printf("[regions] auto-derived region keys enabled: refreshing every %v, cap %d", interval, cfg.AutoRegionKeysMaxDerived()) + } +``` + +- [x] **Step 4: Verify the disabled path changes nothing** + +Append to `cmd/ingestor/region_keys_test.go`: + +```go +func TestRefreshFromStoreIsNoOpWhenDisabled(t *testing.T) { + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "behss") + + cfg := &Config{HashRegions: []string{"#be"}} // autoRegionKeys absent + set := newRegionKeySet(cfg) + before := len(set.snapshot().all) + set.refreshFromStore(store) + + if got := len(set.snapshot().all); got != before { + t.Errorf("key count %d -> %d with autoRegionKeys off, want unchanged", before, got) + } + if _, ok := set.snapshot().all["#behss"]; ok { + t.Error("a declared name became a key with the feature disabled — this is the safety property the default-off promise rests on") + } +} +``` + +- [x] **Step 5: Run the full suite** + +Run: `cd cmd/ingestor && go build ./... && go test ./... && go test ./... -race` +Expected: PASS + +- [x] **Step 6: Commit** + +```bash +git add cmd/ingestor/region_keys.go cmd/ingestor/region_keys_test.go cmd/ingestor/main.go +git commit -m "feat(ingestor): refresh derived region keys at startup and on a ticker" +``` + +--- + +### Task 8: `scope-repair` must use the same key set + +Without this, a repair run re-derives every automatically-named row against the explicit tier alone, finds no match, and writes `""` back over it. That is data loss, not a cosmetic gap. + +**Files:** +- Modify: `cmd/ingestor/scope_repair.go` (`rederiveScope` ~line 67, `runScopeRepair` ~line 274) +- Test: `cmd/ingestor/scope_repair_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/ingestor/scope_repair_test.go`: + +```go +// TestScopeRepairKeepsDerivedNames: a row named from a derived key must survive +// a repair run. If rederiveScope sees only the explicit tier it reports +// MatchCount 0, which lands in the "named -> unmatched" branch and wipes the +// name. This test is the guard against that. +func TestScopeRepairKeepsDerivedNames(t *testing.T) { + payload := []byte{0x42, 0x43, 0x44} + // A transport-flood packet: header 0x14 (route 0, payload type 5), + // code1/code2, path byte 0x41 (hash_size 2, one hop), hop, then payload. + code1 := codeFor("#behss", 5, payload) + rawHex := "14" + code1 + "0000" + "41" + "E3D3" + strings.ToUpper(hex.EncodeToString(payload)) + + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss"}) + + got, err := rederiveScope(rawHex, set.snapshot()) + if err != nil { + t.Fatal(err) + } + if got.State.Name != "#behss" { + t.Errorf("State.Name = %q, want %q — a derived key must name the row during repair", got.State.Name, "#behss") + } + if got.MatchCount != 1 { + t.Errorf("MatchCount = %d, want 1", got.MatchCount) + } +} +``` + +Note: `codeFor` builds `code1` from the payload, and the raw hex embeds it, so the two sides cannot drift. Add `"encoding/hex"` and `"strings"` to the test file's imports if absent. + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/ingestor && go test ./... -run TestScopeRepairKeepsDerivedNames -v` +Expected: FAIL to compile — `cannot use set.snapshot() (*regionKeySnapshot) as map[string][]byte` + +- [x] **Step 3: Change `rederiveScope` to take the snapshot** + +In `cmd/ingestor/scope_repair.go`, change the signature: + +```go +func rederiveScope(rawHex string, snap *regionKeySnapshot) (scopeDerivation, error) { +``` + +and the `matchingRegions` call inside it: + +```go + matched := matchingRegions(snap.all, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) +``` + +- [x] **Step 4: Build the derived tier in `runScopeRepair`** + +In `runScopeRepair`, replace: + +```go + regionKeys := loadRegionKeys(cfg) + + store, err := OpenStore(dbPath) + if err != nil { + log.Fatalf("scope-repair: db: %v", err) + } + defer store.Close() + + report, err := repairScopeNames(store.db, regionKeys, *apply) +``` + +with: + +```go + store, err := OpenStore(dbPath) + if err != nil { + log.Fatalf("scope-repair: db: %v", err) + } + defer store.Close() + + // The derived tier must be rebuilt before scanning. Repairing against the + // explicit tier alone would find no key for any automatically-named row, + // classify it as "named -> unmatched", and erase the name — turning a + // maintenance tool into data loss. + regionSet := newRegionKeySet(cfg) + regionSet.refreshFromStore(store) + snap := regionSet.snapshot() + log.Printf("scope-repair: %d region key(s) in force", len(snap.all)) + + report, err := repairScopeNames(store.db, snap, *apply) +``` + +- [x] **Step 5: Update `repairScopeNames`** + +Change its signature and the one call it makes: + +```go +func repairScopeNames(db *sql.DB, snap *regionKeySnapshot, apply bool) (*scopeRepairReport, error) { +``` + +```go + d, err := rederiveScope(rawHex, snap) +``` + +Update the other `rederiveScope` / `repairScopeNames` call sites in `scope_repair_test.go` to pass a snapshot built with `newRegionKeySet(&Config{HashRegions: ...}).snapshot()`. + +- [x] **Step 6: Run tests to verify they pass** + +Run: `cd cmd/ingestor && go test ./... -run TestScopeRepair -v` then `go test ./...` +Expected: PASS, including the pre-existing scope-repair tests. + +- [x] **Step 7: Commit** + +```bash +git add cmd/ingestor/scope_repair.go cmd/ingestor/scope_repair_test.go +git commit -m "fix(scope-repair): repair against the full key set, not just hashRegions" +``` + +--- + +### Task 9: Benchmark the match path + +AGENTS.md rule 0: perf claims need proof, and this grows the key set by up to `maxDerived`. + +**Files:** +- Modify: `cmd/ingestor/region_keys_test.go` + +- [x] **Step 1: Write the benchmark** + +```go +// BenchmarkScopeMatch sweeps key-set size because the cost is linear in it and +// cannot be reduced: code1 is an HMAC over the packet payload, so there is no +// payload-independent lookup key to index on. The sweep is the evidence for +// choosing maxDerived, not a single before/after number — the explicit tier +// size is operator config and varies per deployment. +func BenchmarkScopeMatch(b *testing.B) { + payload := make([]byte, 51) // a typical GRP_TXT payload + for i := range payload { + payload[i] = byte(i) + } + for _, n := range []int{16, 58, 180, 314} { + b.Run(fmt.Sprintf("keys=%d", n), func(b *testing.B) { + all := make(map[string][]byte, n) + explicit := make(map[string]bool, n) + for i := 0; i < n; i++ { + name := fmt.Sprintf("#r%04d", i) + sum := sha256.Sum256([]byte(name)) + all[name] = sum[:16] + explicit[name] = true + } + snap := ®ionKeySnapshot{all: all, explicit: explicit} + code := codeFor("#r0000", 5, payload) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = snap.match(5, payload, code) + } + }) + } +} +``` + +Add `"fmt"` to the test file's imports if absent. + +- [x] **Step 2: Run it and record the numbers** + +Run: `cd cmd/ingestor && go test -bench BenchmarkScopeMatch -benchtime=200x -run '^$' ./...` +Expected: four lines, roughly linear in key count. Paste the actual output into the commit message — an unrecorded benchmark is not proof. + +- [x] **Step 3: Sanity-check against real load** + +Divide the `keys=314` ns/op by the observed transport-scoped packet rate. On the reference deployment that rate is ~0.04/s (22126 packets over 7 days), so even a 300µs match is ~0.001% of one core. If your measured figure implies more than 5% of a core at your own packet rate, stop and reconsider `maxDerived` before shipping. + +- [x] **Step 4: Commit** + +```bash +git add cmd/ingestor/region_keys_test.go +git commit -m "test(ingestor): benchmark scope matching across key-set sizes" +``` + +--- + +### Task 10: Document it + +**Files:** +- Modify: `config.example.json` +- Modify: `docs/client-regions.md` + +- [x] **Step 1: Add the config block** + +In `config.example.json`, immediately after the `_comment_hashRegions` line, add: + +```json + "autoRegionKeys": { "enabled": false, "maxDerived": 256, "refreshMinutes": 15 }, + "_comment_autoRegionKeys": "Opt-in: derive region keys from the region names repeaters declare over RF (node_declared_regions), on top of the explicit hashRegions list above. Default OFF. Solves the case where a repeater forwards a region this instance holds no key for: its traffic is stored unmatched and the Scope Audit reports the region as 'not observed', which reads as a finding about the repeater rather than a gap in this config. TOP-LEVEL FLAG, a sibling of hashRegions — config loading is plain json.Unmarshal with no DisallowUnknownFields, so nesting it elsewhere is silently ignored. maxDerived caps the derived tier (default 256): each key costs one HMAC per transport-scoped packet and raises the random 2-byte collision rate by 1/65536, and the match cannot be indexed because the code is an HMAC over the payload. Over the cap, names are kept by how many distinct repeaters declare them. Requires clientRegions (or an ESP32 observer on the neighbour-report firmware) to be populating node_declared_regions, or the derived tier stays empty." +``` + +- [x] **Step 2: Document the second consumer** + +In `docs/client-regions.md`, insert this section between `## Storage — node_declared_regions (ingestor-owned)` (line 98) and `## Configurable values (future customizer)` (line 122): + +```markdown +## Second consumer — derived region keys + +`node_declared_regions` originally had one reader: the declared side of the +Scope Audit. With `autoRegionKeys.enabled` set (default off, see +`config.example.json`), the ingestor reads it a second time, deriving a region +key `SHA256("#name")[:16]` for each declared name so that traffic in those +regions can be *named* rather than stored unmatched. + +Two consequences operators should know about: + +- **Retention now bounds nameability.** `retention.clientRegionsDays` already + bounded how long a declared answer stayed visible in the audit. With + derivation on, it also bounds how long a region stays *derivable*: once the + last answer naming a region is pruned, its key leaves the set on the next + refresh and its traffic reverts to unmatched. Regions you want named + permanently belong in `hashRegions`, which nothing prunes. +- **The set is capped.** `autoRegionKeys.maxDerived` (default 256) limits the + derived tier; over the cap, names are kept by how many distinct repeaters + declare them. A region declared by a single repeater is the first to be + dropped. The ingestor logs how many names were dropped on each refresh. +``` + +- [x] **Step 3: Verify the example config still parses** + +Run: `python -c "import json; json.load(open('config.example.json')); print('valid')"` +Expected: `valid` + +- [x] **Step 4: Commit** + +```bash +git add config.example.json docs/client-regions.md +git commit -m "docs(config): document the opt-in autoRegionKeys block" +``` + +--- + +### Task 11: Verify end to end + +- [x] **Step 1: Full suites, both binaries** + +Run: `cd cmd/ingestor && go test ./... -race` then `cd ../server && go test ./...` +Expected: PASS in both. + +- [x] **Step 2: Frontend suite** + +Run: `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js` +Expected: PASS. Nothing in this plan touches the frontend; run it to prove that. + +- [ ] **Step 3: Prove the default-off promise on real data** + +Run the ingestor against a copy of a real database with no `autoRegionKeys` block. Confirm the startup log reads `autoRegionKeys disabled — only the N configured hashRegions key(s) are in force` and that no `[regions] derived` line appears. + +- [ ] **Step 4: Prove the feature on real data** + +Enable the block, restart, and confirm the startup log reports the declared/kept/total counts. Then check that a packet which was previously stored unmatched is now named: pick one from `SELECT id, raw_hex FROM transmissions WHERE scope_name = '' LIMIT 5`, run `scope-repair` as a dry run, and confirm the report's "newly named" section lists the expected regions. + +- [ ] **Step 5: Record the ambiguity rate** + +After the ingestor has run for at least a day with the feature on, read the `[regions] scope matches:` line. The `ambiguous` count against the total is the measurement that decides whether M3 (path-evidence tie-break) is worth building. Write the number into `docs/specs/2026-09-07-auto-region-keys-design.md` under M3, replacing the estimate with the observation. + +--- + +## Notes for the implementer + +- **`matchingRegions` stays exactly as it is.** It is the shared primitive; only its caller changes. Its `#1609` ambiguity semantics are load-bearing for both `match` and `scope-repair`. +- **Do not lowercase region names anywhere.** The key is `SHA256("#name")[:16]` over raw bytes, so `#BEHSS` and `#behss` are different regions. `loadRegionKeys` does not fold case and neither may the derived path. +- **Do not let a failed refresh empty the key set.** `refreshFromStore` keeps the current snapshot on error. An empty key set would silently unname all traffic, which looks exactly like the bug this feature fixes. +- **The `null` region name will be derived.** That is intended: see `regionNameAcceptable`'s comment. It costs one slot. diff --git a/docs/plans/2026-09-07-declared-region-verification.md b/docs/plans/2026-09-07-declared-region-verification.md new file mode 100644 index 000000000..89a30cad6 --- /dev/null +++ b/docs/plans/2026-09-07-declared-region-verification.md @@ -0,0 +1,1286 @@ +# Declared-Region Verification (M1b) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Turn a declared region's chip green when this instance can *prove* the repeater forwards it, instead of leaving it grey with a footnote — by testing the repeater's own declared names against its own unnameable traffic. + +**Architecture:** For repeater R declaring region X, derive `SHA256("#X")[:16]`, HMAC each of R's unmatched forwarded packets with it, and compare to that packet's `code1`. Same computation `matchingRegions` performs in the ingestor, with the candidate set narrowed from every configured key to R's own ~9 declarations. Read-time in `cmd/server/`, nothing written, no config. A region needs **two** corroborating packets to go green. + +**Tech Stack:** Go 1.x (`cmd/server`, stdlib `crypto/hmac`, `crypto/sha256`, `testing`), vanilla JS frontend (`public/`). + +**Spec:** `docs/specs/2026-09-07-auto-region-keys-design.md`, "Architecture 3b" and "M1b" (`55f46a1d`). + +**Depends on M0 and M1**, both landed (`d93b4463`, and `70e6bcd5`/`ca464b59`/`79f38ef1`/`93a0c385`). M0 is what makes a mid-path repeater attributable at all; M1 is what counts its unmatched traffic. This plan turns that count into an answer. + +**Status: code complete, verification partly deferred.** All eight tasks are committed +(`64e3ac60`, `3e27b112`, `409258c7`, `e5cee74f`, `28e310f8`, `985067f8`, `a813346d`, +`a87ce8e8`, `37b0473a`). Automated verification passes: `cmd/server` full suite ok in +119.9s, 692 frontend assertions, 99 packet-filter, 18 aging, `gofmt -l` and `go vet` +clean. Task 8 Step 5 — confirming against the real `e3d3f4d7` row — is **deferred to +deploy**, for the same reason as M1: `test-fixtures/e2e-fixture.db` predates the feature +and has neither `node_declared_regions` nor `scope_name`. + +**Independent of M2** (`docs/plans/2026-09-07-auto-derived-region-keys.md`), which is entirely `cmd/ingestor/`. No shared files. Per the spec amendment, M2 no longer fixes the audit — this does. + +--- + +## Why two packets, not one + +`code1` is two bytes, so an unrelated region name matches a given packet with probability 1/65536. One match is therefore not evidence: across a network with 400 unmatched packets and 124 declared names, chance alone produces roughly one false match per audit refresh. Two matches on the *same* region for the *same* repeater is (1/65536)² — about one in four billion. That threshold is the whole reason this approach is sound where per-packet naming is not, so it is a named constant with a test, not a literal. + +--- + +## Cost — where it actually sits (measured, not estimated) + +Naively this is `targets × declaredNames × unmatchedPackets` HMACs — 205 × 124 × 400 ≈ 10,000,000. + +The first cut cached per `(region, transmission)` pair, which cuts the HMACs to `names × packets` ≈ 50,000. **That was not enough, and the plan's original estimate of ~50ms was wrong about why.** Benchmarked at audit scale it took **501ms**: the HMACs had become a rounding error, but the *iteration* was still cubic — 10.2M map lookups at ~49ns each. + +The cache is therefore keyed per **region**, holding the set of transmissions deriving to it. A region is HMACed over every packet once; a target then asks one question per declared region instead of one per (region, packet). Most declared regions match nothing, so the common case is a single lookup and no packet loop at all. + +Measured after that change: **36ms** at the same worst-case shape (every one of 205 targets declaring all 124 names over all 400 packets). Real rows declare ~9 names and hold far fewer packets, so this is an upper bound with a lot of headroom under the 30s cache. + +The lesson worth keeping: caching the expensive operation is not the same as removing the expensive loop. `hmacCount` exists so a test can assert the first, and the benchmark exists because only it catches the second. + +--- + +## File Structure + +| File | Responsibility | Change | +|---|---|---| +| `cmd/server/scope_verify.go` | HMAC-input extraction, the memo, the verification pass | **Create** | +| `cmd/server/scope_verify_test.go` | Unit tests + benchmark | **Create** | +| `cmd/server/scopes.go` | `unmatchedTxIDs` on the agg; `RegionEvidence` on the row | Modify | +| `cmd/server/routes.go` | Run verification, subtract from `notObserved` | Modify | +| `cmd/server/scopes_test.go` | Handler-level tests | Modify | +| `public/scope-audit.js` | Evidence-aware chip title and marker; narrow the row caveat | Modify | +| `public/scope-audit.css` | `.sa-chip-verified` | Modify | +| `test-frontend-helpers.js` | Chip and caveat assertions | Modify | +| `docs/api-spec.md` | `regionEvidence` field + note | Modify | + +--- + +### Task 1: Extract the HMAC inputs from `raw_hex` + +No payload decoding: `decodePayload` attempts decryption and signature validation, none of which this needs, and running it on every unmatched packet every refresh is pure waste. This walks the offsets only, reusing the decoder helpers that already exist in the package so the offset logic is not duplicated. + +**Files:** +- Create: `cmd/server/scope_verify.go` +- Create: `cmd/server/scope_verify_test.go` + +- [x] **Step 1: Write the failing test** + +Create `cmd/server/scope_verify_test.go`: + +```go +package main + +import ( + "encoding/hex" + "strings" + "testing" +) + +// realTransportFloodPacket is transmission 0a065d41d51f1f77 from the live +// instance, captured 2026-09-07. Header 0x14 = route_type 0 (TRANSPORT_FLOOD), +// payload_type 5 (GRP_TXT); transport codes 9209/0000; path byte 0x41 = +// hash_size 2, one hop "E3D3"; the rest is payload. +// +// A hand-built fixture would only prove the parser agrees with itself. This +// packet is the one that started the investigation: its code1 is exactly the +// code #fm-112 derives over its own payload, which is why the audit showed +// fm-112 as "not observed" for a repeater that was forwarding it. +const realTransportFloodPacket = "149209000041E3D3EC2D4481DA70893CD71B763958B064A9AAC011D54223FF8A0140CBB4093653BC61D67C960E3ECCE6639CC9FF1147AA6D0F9017" + +func TestScopeHMACInputsParsesRealPacket(t *testing.T) { + payloadType, payload, code1, ok := scopeHMACInputs(realTransportFloodPacket) + if !ok { + t.Fatal("scopeHMACInputs returned ok=false for a valid transport-flood packet") + } + if payloadType != 5 { + t.Errorf("payloadType = %d, want 5 (GRP_TXT)", payloadType) + } + if code1 != "9209" { + t.Errorf("code1 = %q, want %q", code1, "9209") + } + if len(payload) != 51 { + t.Errorf("len(payload) = %d, want 51", len(payload)) + } + if got := strings.ToUpper(hex.EncodeToString(payload[:4])); got != "EC2D4481" { + t.Errorf("payload starts %q, want %q — offset walked wrong", got, "EC2D4481") + } +} + +func TestScopeHMACInputsRejectsNonTransportRoutes(t *testing.T) { + // A plain FLOOD packet carries no transport codes, so it has no code1 to + // verify against. Returning ok=false rather than a zero code1 keeps the + // caller from HMACing packets that can never match anything. + // + // Header 0x15 = route_type 1 (FLOOD), payload_type 5. No transport codes, + // so the path byte follows the header directly. + _, _, _, ok := scopeHMACInputs("15" + "41" + "E3D3" + "AABBCC") + if ok { + t.Error("ok = true for a non-transport route, want false — there is no code1 to verify") + } +} + +func TestScopeHMACInputsRejectsMalformed(t *testing.T) { + for _, c := range []struct{ hex, why string }{ + {"", "empty"}, + {"zz", "not hex"}, + {"14", "header only, no transport codes"}, + {"1492090000", "transport codes but no path byte"}, + {"149209000041", "path byte claims one 2-byte hop, none present"}, + // pathByte 0xC0: upper two bits 11 -> hash_size 4, which firmware + // reserves and isValidPathLen rejects even at hash_count 0 + // (cmd/server/decoder.go, mirroring Packet.cpp:13-18). + {"1492090000C0" + strings.Repeat("00", 8), "hash_size 4 is reserved"}, + } { + if _, _, _, ok := scopeHMACInputs(c.hex); ok { + t.Errorf("ok = true for %q (%s), want false", c.hex, c.why) + } + } +} + +func TestRegionCodeMatchesTheRealPacket(t *testing.T) { + // The end-to-end arithmetic, against a packet whose true region is known. + payloadType, payload, code1, ok := scopeHMACInputs(realTransportFloodPacket) + if !ok { + t.Fatal("setup: scopeHMACInputs failed") + } + if got := regionCode("fm-112", payloadType, payload); got != code1 { + t.Errorf("regionCode(fm-112) = %q, want %q — this packet IS fm-112", got, code1) + } + // Both spellings must agree: the key is SHA256 over "#name", and callers + // hand us names with the '#' already stripped by normScope. + if got := regionCode("#fm-112", payloadType, payload); got != code1 { + t.Errorf("regionCode(#fm-112) = %q, want %q — leading '#' must be optional", got, code1) + } + // A region the repeater also declares, which this packet is NOT. + if got := regionCode("behss", payloadType, payload); got == code1 { + t.Errorf("regionCode(behss) = %q, must not equal fm-112's code1", got) + } +} + +func TestRegionCodeIsCaseSensitive(t *testing.T) { + // The key is SHA256 over the raw bytes of "#name", so "#BEHSS" and + // "#behss" are different regions. Folding case here would silently name + // traffic for a region nobody configured. + payloadType, payload, _, _ := scopeHMACInputs(realTransportFloodPacket) + if regionCode("behss", payloadType, payload) == regionCode("BEHSS", payloadType, payload) { + t.Error("regionCode folded case — the key is a hash over raw bytes and must not") + } +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/server && go test ./... -run 'TestScopeHMACInputs|TestRegionCode' -v` +Expected: FAIL to compile — `undefined: scopeHMACInputs`, `undefined: regionCode` + +- [x] **Step 3: Implement** + +Create `cmd/server/scope_verify.go`: + +```go +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "strings" +) + +// scopeHMACInputs pulls the three values needed to test a region hypothesis +// against one packet: the payload type and raw payload bytes the sender HMACed, +// and the resulting two-byte code it put on the wire. +// +// It deliberately does NOT call DecodePacket. That runs decodePayload, which +// attempts channel decryption and signature validation — work this has no use +// for, repeated over every unmatched packet on every audit refresh. Walking the +// offsets is all that is needed, and it reuses decodeHeader/isTransportRoute/ +// decodePath so the offset arithmetic is not duplicated from DecodePacket. +// +// ok is false for anything that cannot carry a region scope: malformed hex, a +// truncated header, an invalid path byte, or a non-transport route. A plain +// FLOOD packet has no transport codes at all, so there is no code1 to compare +// against and HMACing it could only waste time. +func scopeHMACInputs(rawHex string) (payloadType byte, payload []byte, code1 string, ok bool) { + buf, err := hex.DecodeString(strings.TrimSpace(rawHex)) + if err != nil || len(buf) < 2 { + return 0, nil, "", false + } + header := decodeHeader(buf[0]) + if !isTransportRoute(header.RouteType) { + return 0, nil, "", false + } + offset := 1 + if len(buf) < offset+4 { + return 0, nil, "", false + } + code1 = strings.ToUpper(hex.EncodeToString(buf[offset : offset+2])) + offset += 4 // code1 and code2 + + if offset >= len(buf) { + return 0, nil, "", false + } + pathByte := buf[offset] + offset++ + _, consumed, decodeErr := decodePath(pathByte, buf, offset) + if decodeErr != nil { + return 0, nil, "", false + } + offset += consumed + if offset > len(buf) { + return 0, nil, "", false + } + rest := buf[offset:] + if len(rest) == 0 { + return 0, nil, "", false + } + return byte(header.PayloadType), rest, code1, true +} + +// regionCode derives the on-wire code1 a sender in region name would emit for +// this payload — the forward direction of what matchingRegions inverts in the +// ingestor (cmd/ingestor/main.go). The two must stay in step: key is +// SHA256("#name")[:16], the MAC covers payloadType followed by the payload, the +// code is the first two MAC bytes little-endian, and 0x0000/0xFFFF are reserved +// and nudged. Any divergence here silently produces regions that never verify. +// +// The leading '#' is optional because callers hold normScope'd names (the audit +// strips it) while the key is over the '#'-prefixed form. +func regionCode(name string, payloadType byte, payload []byte) string { + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + sum := sha256.Sum256([]byte(name)) + mac := hmac.New(sha256.New, sum[:16]) + mac.Write([]byte{payloadType}) + mac.Write(payload) + h := mac.Sum(nil) + code := uint16(h[0]) | uint16(h[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return strings.ToUpper(hex.EncodeToString([]byte{byte(code & 0xFF), byte(code >> 8)})) +} +``` + +- [x] **Step 4: Run tests to verify they pass** + +Run: `cd cmd/server && go test ./... -run 'TestScopeHMACInputs|TestRegionCode' -v` +Expected: PASS (5 tests) + +- [x] **Step 5: Commit** + +```bash +git add cmd/server/scope_verify.go cmd/server/scope_verify_test.go +git commit -m "feat(scope-audit): derive a region's on-wire code from a packet's own payload" +``` + +--- + +### Task 2: Remember which transmissions were unmatched, per target + +M1 counts them. Verification needs to know *which*. + +**Files:** +- Modify: `cmd/server/scopes.go` (`scopeAuditTargetAgg`, `ScopeAuditForwarding`) +- Test: `cmd/server/scopes_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/server/scopes_test.go`, after `TestScopeAuditForwardingCountsUnmatchedOnMidPathHop`: + +```go +// TestScopeAuditForwardingRecordsUnmatchedTxIDs: the counter M1 added says how +// many, verification needs to know which. The IDs must be de-duplicated the +// same way the counter is — a target appearing twice in one path contributed +// one packet, and counting it twice would let a single packet reach the +// two-corroboration threshold on its own. +func TestScopeAuditForwardingRecordsUnmatchedTxIDs(t *testing.T) { + s := newScopeTestStore(t) + hop := testFullPubkeyA[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{hop, "AAAA", hop}, scopeUnmatched(), RouteFlood, recent) + seedTransmissionPathAt(t, s, []string{"BBBB", hop}, scopeUnmatched(), RouteFlood, recent) + seedTransmissionPathAt(t, s, []string{hop}, scopeMatched("#be"), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil { + t.Fatalf("want an agg, got none (result = %+v)", got) + } + if len(agg.unmatchedTxIDs) != 2 { + t.Errorf("unmatchedTxIDs = %v, want 2 distinct ids — the twice-hopped packet counts once, and the matched packet not at all", agg.unmatchedTxIDs) + } + if agg.unmatchedPackets != int64(len(agg.unmatchedTxIDs)) { + t.Errorf("unmatchedPackets = %d but %d ids recorded — the count and the ids must not drift", agg.unmatchedPackets, len(agg.unmatchedTxIDs)) + } +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/server && go test ./... -run TestScopeAuditForwardingRecordsUnmatchedTxIDs -v` +Expected: FAIL to compile — `agg.unmatchedTxIDs undefined` + +- [x] **Step 3: Add the field** + +In `cmd/server/scopes.go`, in `scopeAuditTargetAgg`, immediately after `unmatchedPackets`: + +```go + // unmatchedTxIDs are the transmissions behind unmatchedPackets, kept so + // declared-region verification can test this target's own declarations + // against this target's own unnameable traffic (scope_verify.go). Bounded + // by the window and by scopeVerifyMaxPacketsPerTarget; the same + // (target, txID) de-duplication that guards unmatchedPackets guards this, + // so one packet reaching a target by two hops cannot corroborate twice. + unmatchedTxIDs []int64 +``` + +- [x] **Step 4: Record them** + +In `ScopeAuditForwarding`, extend the unmatched branch added by M1: + +```go + if scopeName.String == "" { + // Unmatched: transport-scoped, but no configured region key + // matched code1. Still not part of the declared/observed + // comparison — it names no region, so it can never satisfy a + // declaration — but it is the evidence that a notObserved + // finding on this row may be a gap in this instance's + // hashRegions rather than in the repeater's forwarding. + agg.unmatchedPackets++ + if len(agg.unmatchedTxIDs) < scopeVerifyMaxPacketsPerTarget { + agg.unmatchedTxIDs = append(agg.unmatchedTxIDs, txID) + } + continue + } +``` + +- [x] **Step 5: Add the bound** + +In `cmd/server/scope_verify.go`: + +```go +// scopeVerifyMaxPacketsPerTarget bounds the per-target evidence list. AGENTS.md +// rule 0 forbids unbounded structures, and the corroboration threshold is 2 — +// past a few hundred packets more evidence changes no verdict, it only costs +// memory. Note that unmatchedPackets keeps counting past this: the count is the +// honest total, the list is the working set. +const scopeVerifyMaxPacketsPerTarget = 512 +``` + +- [x] **Step 6: Run tests to verify they pass** + +Run: `cd cmd/server && go test ./... -run 'TestScopeAuditForwarding' -v` +Expected: PASS, including M1's two unmatched tests. + +Note the deliberate asymmetry the new test pins: `unmatchedPackets` counts every unmatched packet, `unmatchedTxIDs` stops at 512. The test uses 2, so they agree there; a comment in the field doc explains the divergence above the cap. + +- [x] **Step 7: Commit** + +```bash +git add cmd/server/scopes.go cmd/server/scopes_test.go cmd/server/scope_verify.go +git commit -m "feat(scope-audit): record which transmissions were unmatched, per target" +``` + +--- + +### Task 3: The narrow query + +**Files:** +- Modify: `cmd/server/scope_verify.go` +- Test: `cmd/server/scope_verify_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/server/scope_verify_test.go`: + +```go +func TestUnmatchedTransmissionsInWindow(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + old := "2020-01-01T00:00:00Z" + seedTransmissionRouteAt(t, s, "E3D3", scopeUnmatched(), RouteFlood, recent) + seedTransmissionRouteAt(t, s, "E3D3", scopeMatched("#be"), RouteFlood, recent) + seedTransmissionRouteAt(t, s, "E3D3", scopeUnscoped(), RouteFlood, recent) + seedTransmissionRouteAt(t, s, "E3D3", scopeUnmatched(), RouteFlood, old) + + since := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) + got, err := s.unmatchedTransmissionsInWindow(since) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("got %d rows, want 1 — only the recent scope_name='' row qualifies", len(got)) + } + // scopeUnmatched() seeds raw_hex 'AA', which scopeHMACInputs rejects. The + // query's job is selection; unparseable rows are dropped by the caller, so + // they must still be returned here rather than filtered in SQL. + if got[0].txID == 0 { + t.Error("txID = 0, want the transmission's real id") + } +} +``` + +Add `"time"` to the test file's imports. + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/server && go test ./... -run TestUnmatchedTransmissionsInWindow -v` +Expected: FAIL to compile — `s.unmatchedTransmissionsInWindow undefined` + +- [x] **Step 3: Implement** + +Append to `cmd/server/scope_verify.go`: + +```go +// unmatchedTransmissionRow is one transmission that carried a transport scope +// no configured region key matched, with the raw bytes needed to test a region +// hypothesis against it. +type unmatchedTransmissionRow struct { + txID int64 + rawHex string +} + +// unmatchedTransmissionsInWindow is the SECOND, narrow query behind the audit — +// deliberately not a widening of scopeAuditForwarderScanQuery. +// +// That scan returns one row per hop per flood packet: on a 2,000-packet sample +// after M0 that is 19,049 rows, and carrying raw_hex on every one of them would +// load the hot path to serve a few hundred packets. This selects only the +// transmissions that are actually candidates — scope_name = '' inside the +// window, ~400 over 7 days on the reference deployment — and the main scan is +// left exactly as it is. +// +// scope_name = '' is the "transport-scoped but unnameable" state; NULL means +// the packet carried no scope at all and can never verify against a region. +// The route filter matches the forwarder scan's, so the two agree on which +// packets count as forwarded. +func (s *PacketStore) unmatchedTransmissionsInWindow(sinceISO string) ([]unmatchedTransmissionRow, error) { + rows, err := s.db.conn.Query(` + SELECT t.id, t.raw_hex + FROM transmissions t + WHERE t.first_seen >= ? + AND t.scope_name = '' + AND `+scopeConformanceForwarderRouteTypesSQL, sinceISO) + if err != nil { + return nil, fmt.Errorf("unmatched transmissions scan: %w", err) + } + defer rows.Close() + + var out []unmatchedTransmissionRow + for rows.Next() { + var r unmatchedTransmissionRow + if err := rows.Scan(&r.txID, &r.rawHex); err != nil { + return nil, fmt.Errorf("unmatched transmissions scan row: %w", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("unmatched transmissions rows: %w", err) + } + return out, nil +} +``` + +Add `"fmt"` to the file's imports. + +- [x] **Step 4: Run tests to verify they pass** + +Run: `cd cmd/server && go test ./... -run TestUnmatchedTransmissionsInWindow -v` +Expected: PASS + +- [x] **Step 5: Commit** + +```bash +git add cmd/server/scope_verify.go cmd/server/scope_verify_test.go +git commit -m "feat(scope-audit): narrow query for the window's unmatched transmissions" +``` + +--- + +### Task 4: The memoised verification pass + +**Files:** +- Modify: `cmd/server/scope_verify.go` +- Test: `cmd/server/scope_verify_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/server/scope_verify_test.go`: + +```go +// buildVerifierFromPackets is a test helper: wraps raw hex strings as rows the +// verifier consumes, with ids 1..N in order. +func buildVerifierFromPackets(t *testing.T, hexes ...string) *scopeVerifier { + t.Helper() + rows := make([]unmatchedTransmissionRow, 0, len(hexes)) + for i, h := range hexes { + rows = append(rows, unmatchedTransmissionRow{txID: int64(i + 1), rawHex: h}) + } + return newScopeVerifier(rows) +} + +func TestScopeVerifierNeedsTwoCorroboratingPackets(t *testing.T) { + // One match is 1-in-65536 and must not be enough; a second makes it + // (1/65536)^2. This threshold is the reason the approach is sound. + v := buildVerifierFromPackets(t, realTransportFloodPacket) + one := v.evidence([]int64{1}, []string{"fm-112"}) + if one["fm-112"] != 1 { + t.Fatalf("evidence = %v, want fm-112:1", one) + } + if v.verified(one) != nil && len(v.verified(one)) != 0 { + t.Errorf("verified = %v, want none — one corroborating packet is not evidence", v.verified(one)) + } + + // The same packet twice under different ids: two distinct transmissions + // both deriving to fm-112. + v2 := buildVerifierFromPackets(t, realTransportFloodPacket, realTransportFloodPacket) + two := v2.evidence([]int64{1, 2}, []string{"fm-112"}) + if two["fm-112"] != 2 { + t.Fatalf("evidence = %v, want fm-112:2", two) + } + got := v2.verified(two) + if len(got) != 1 || got[0] != "fm-112" { + t.Errorf("verified = %v, want [fm-112]", got) + } +} + +func TestScopeVerifierIgnoresRegionsThatDoNotMatch(t *testing.T) { + v := buildVerifierFromPackets(t, realTransportFloodPacket, realTransportFloodPacket) + got := v.evidence([]int64{1, 2}, []string{"behss", "be", "eu"}) + if len(got) != 0 { + t.Errorf("evidence = %v, want empty — none of these regions is this packet", got) + } +} + +func TestScopeVerifierSkipsUnparseablePackets(t *testing.T) { + // A row whose raw_hex cannot be walked contributes nothing and must not + // error the pass: one malformed row in the window would otherwise blank + // the verification for every repeater. + v := buildVerifierFromPackets(t, "AA", realTransportFloodPacket) + got := v.evidence([]int64{1, 2}, []string{"fm-112"}) + if got["fm-112"] != 1 { + t.Errorf("evidence = %v, want fm-112:1 — the malformed row is skipped, the good one still counts", got) + } +} + +func TestScopeVerifierMemoisesAcrossTargets(t *testing.T) { + // The cost argument: work depends on (region, transmission), not on which + // target asked. Two targets declaring the same region over the same packets + // must not double the HMACs. + v := buildVerifierFromPackets(t, realTransportFloodPacket, realTransportFloodPacket) + v.evidence([]int64{1, 2}, []string{"fm-112"}) + after := v.hmacCount + v.evidence([]int64{1, 2}, []string{"fm-112"}) + if v.hmacCount != after { + t.Errorf("hmacCount %d -> %d on a repeat query, want unchanged — the memo is what keeps this inside rule 0", after, v.hmacCount) + } +} + +func TestScopeVerifierUnknownTxIDIsHarmless(t *testing.T) { + // A target's unmatchedTxIDs come from a different query than the verifier's + // rows. They are taken in the same window, but a row pruned between the two + // must degrade to "no evidence", not panic. + v := buildVerifierFromPackets(t, realTransportFloodPacket) + got := v.evidence([]int64{1, 999}, []string{"fm-112"}) + if got["fm-112"] != 1 { + t.Errorf("evidence = %v, want fm-112:1 — the unknown id contributes nothing", got) + } +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/server && go test ./... -run TestScopeVerifier -v` +Expected: FAIL to compile — `undefined: newScopeVerifier` + +- [x] **Step 3: Implement** + +Append to `cmd/server/scope_verify.go`: + +```go +// scopeVerifyMinCorroboration is how many of a repeater's own unmatched packets +// must derive to a declared region before that region counts as observed. +// +// One is not enough and the arithmetic is the whole argument: code1 is two +// bytes, so an unrelated name matches a given packet with probability 1/65536. +// Across ~400 unmatched packets and ~124 distinct declared names, chance alone +// produces roughly one false match per refresh. Two matches on the same region +// for the same repeater is (1/65536)^2 — about one in four billion. Raising +// this costs recall on quiet regions; lowering it to 1 makes the feature +// unsound, not merely noisy. +const scopeVerifyMinCorroboration = 2 + +// scopeVerifier answers "how many of these transmissions are region X" while +// computing each (region, transmission) pair at most once. +// +// The memo is not a nicety. Naively the audit would do +// targets x declaredNames x unmatchedPackets HMACs — 205 x 9 x 400 is roughly +// 740,000, about 0.7s per refresh. The work depends only on the pair, and +// distinct pairs are distinctNames x packets = 124 x 400, roughly 50,000 and +// ~50ms. That difference is what puts this inside AGENTS.md rule 0. +// +// Not safe for concurrent use: one verifier is built per audit computation, +// which handleScopeAudit already serialises behind its cache. +type scopeVerifier struct { + packets map[int64]scopeVerifyInputs + memo map[scopeVerifyKey]bool + // hmacCount is incremented per actual derivation, asserted by the memo + // test so a future refactor cannot quietly reintroduce the naive cost. + hmacCount int +} + +type scopeVerifyInputs struct { + payloadType byte + payload []byte + code1 string + ok bool +} + +type scopeVerifyKey struct { + region string + txID int64 +} + +// newScopeVerifier parses each row once. A row whose raw_hex cannot be walked +// is kept with ok=false rather than dropped, so the memo still short-circuits +// repeat lookups for it. +func newScopeVerifier(rows []unmatchedTransmissionRow) *scopeVerifier { + v := &scopeVerifier{ + packets: make(map[int64]scopeVerifyInputs, len(rows)), + memo: map[scopeVerifyKey]bool{}, + } + for _, r := range rows { + pt, payload, code1, ok := scopeHMACInputs(r.rawHex) + v.packets[r.txID] = scopeVerifyInputs{payloadType: pt, payload: payload, code1: code1, ok: ok} + } + return v +} + +// matches reports whether transmission txID is region, deriving at most once +// per pair. +func (v *scopeVerifier) matches(region string, txID int64) bool { + key := scopeVerifyKey{region: region, txID: txID} + if got, seen := v.memo[key]; seen { + return got + } + in, known := v.packets[txID] + got := false + if known && in.ok { + v.hmacCount++ + got = regionCode(region, in.payloadType, in.payload) == in.code1 + } + v.memo[key] = got + return got +} + +// evidence counts, for each declared region, how many of txIDs derive to it. +// Regions with zero matches are absent from the result rather than present +// with 0, so the map is directly the "we found something" set. +func (v *scopeVerifier) evidence(txIDs []int64, declaredRegions []string) map[string]int { + out := map[string]int{} + for _, region := range declaredRegions { + n := 0 + for _, txID := range txIDs { + if v.matches(region, txID) { + n++ + } + } + if n > 0 { + out[region] = n + } + } + return out +} + +// verified returns the regions in an evidence map that clear the corroboration +// threshold, sorted so the response is stable across refreshes. +func (v *scopeVerifier) verified(evidence map[string]int) []string { + var out []string + for region, n := range evidence { + if n >= scopeVerifyMinCorroboration { + out = append(out, region) + } + } + sort.Strings(out) + return out +} +``` + +Add `"sort"` to the file's imports. + +- [x] **Step 4: Run tests to verify they pass** + +Run: `cd cmd/server && go test ./... -run TestScopeVerifier -v` +Expected: PASS (5 tests) + +- [x] **Step 5: Commit** + +```bash +git add cmd/server/scope_verify.go cmd/server/scope_verify_test.go +git commit -m "feat(scope-audit): memoised declared-region verification with a 2-packet threshold" +``` + +--- + +### Task 5: Wire it into the handler + +**Files:** +- Modify: `cmd/server/scopes.go` (`ScopeAuditRow`) +- Modify: `cmd/server/routes.go` (`handleScopeAudit`) +- Test: `cmd/server/scopes_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/server/scopes_test.go`, after `TestHandleScopeAuditSurfacesUnmatchedPackets`: + +```go +// TestHandleScopeAuditVerifiesDeclaredRegion is the case this milestone exists +// for, built from the real packet that started the investigation. A repeater +// declares "fm-112"; this instance holds no key for it, so both packets it +// forwarded are stored unmatched. Verification derives the key from the +// repeater's own declaration, finds two corroborating packets, and the region +// must leave notObserved with its evidence count reported. +func TestHandleScopeAuditVerifiesDeclaredRegion(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "fm-112,behss", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 { + t.Fatalf("repeaters = %+v, want 1", got.Repeaters) + } + row := got.Repeaters[0] + if row.RegionEvidence["fm-112"] != 2 { + t.Errorf("regionEvidence = %v, want fm-112:2", row.RegionEvidence) + } + for _, n := range row.NotObserved { + if n == "fm-112" { + t.Errorf("notObserved = %v, must not contain fm-112 — two corroborating packets prove it is forwarded", row.NotObserved) + } + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "behss" { + t.Errorf("notObserved = %v, want [\"behss\"] — that region has no corroborating traffic here", row.NotObserved) + } +} + +// TestHandleScopeAuditDoesNotVerifyOnOnePacket: a single match is 1-in-65536 +// and must leave the region in notObserved, with its count still reported so a +// client can say "one hit, not enough". +func TestHandleScopeAuditDoesNotVerifyOnOnePacket(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "fm-112", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + + got := getScopeAudit(t, router, "") + row := got.Repeaters[0] + if row.RegionEvidence["fm-112"] != 1 { + t.Errorf("regionEvidence = %v, want fm-112:1", row.RegionEvidence) + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "fm-112" { + t.Errorf("notObserved = %v, want [\"fm-112\"] — one corroborating packet is not evidence", row.NotObserved) + } +} + +// TestHandleScopeAuditLeavesCleanRowsAlone: a repeater whose declared regions +// are all observed by name, with no unmatched traffic at all, must be untouched +// by verification — no evidence, no change to notObserved, and an empty (not +// null) regionEvidence so a client can iterate it without a guard. +func TestHandleScopeAuditLeavesCleanRowsAlone(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "be", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, pk[:4], scopeMatched("#be"), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + row := got.Repeaters[0] + if len(row.NotObserved) != 0 { + t.Errorf("notObserved = %v, want empty", row.NotObserved) + } + if row.RegionEvidence == nil { + t.Error("regionEvidence = nil, want an empty object — a client must not need a null guard") + } + if len(row.RegionEvidence) != 0 { + t.Errorf("regionEvidence = %v, want empty — nothing needed verifying here", row.RegionEvidence) + } +} + +// seedUnmatchedRawAt seeds one unmatched transmission carrying a real raw_hex, +// attributed to forwarder. Distinct from seedTransmissionRouteAt, which seeds +// raw_hex 'AA' — fine for tests that never parse it, useless here. +func seedUnmatchedRawAt(t *testing.T, s *PacketStore, forwarder, rawHex string, routeType int, firstSeen string) { + t.Helper() + scopeSeedCounter++ + hash := fmt.Sprintf("scoperaw%d", scopeSeedCounter) + res, err := s.db.conn.Exec( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) + VALUES (?, ?, ?, ?, 5, '9209', '0000', '')`, + rawHex, hash, firstSeen, routeType) + if err != nil { + t.Fatal(err) + } + txID, err := res.LastInsertId() + if err != nil { + t.Fatal(err) + } + if _, err := s.db.conn.Exec( + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, 0)`, + txID, `["`+strings.ToUpper(forwarder)+`"]`); err != nil { + t.Fatal(err) + } +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/server && go test ./... -run 'TestHandleScopeAuditVerifies|TestHandleScopeAuditDoesNotVerify' -v` +Expected: FAIL to compile — `row.RegionEvidence undefined` + +- [x] **Step 3: Add the field** + +In `cmd/server/scopes.go`, after `ObservedUnmatchedPackets`: + +```go + // RegionEvidence maps a declared region to how many of this repeater's own + // unmatched forwarded packets derive to it — see scope_verify.go. A region + // reaching scopeVerifyMinCorroboration is removed from NotObserved, so this + // field is not what decides the chip's colour; NotObserved remains the + // single source of that, and this exists so a client can say HOW a region + // was established, and can explain a region that got exactly one hit and + // therefore stayed grey. + // + // Absent regions simply had no matching traffic. Never nil in the response + // — an empty object and a missing key mean the same thing, and an empty map + // is the cheaper thing for a client to iterate. + RegionEvidence map[string]int `json:"regionEvidence"` +``` + +- [x] **Step 4: Run verification in the handler** + +In `cmd/server/routes.go`, in `handleScopeAudit`, immediately after the `forwarding, err = s.store.ScopeAuditForwarding(...)` block: + +```go + // Declared-region verification (M1b): a region this instance holds no key + // for is unnameable, not absent, and the audit can settle which by deriving + // the key from the repeater's own declaration and testing it against that + // repeater's own unnameable traffic. One verifier serves every row so each + // (region, transmission) pair is derived at most once. + // + // A failure here degrades to "no verification" rather than failing the + // request: the audit was useful before this existed and must stay useful if + // the extra query errors. + var verifier *scopeVerifier + if s.store != nil { + unmatchedRows, uErr := s.store.unmatchedTransmissionsInWindow(sinceISO) + if uErr != nil { + log.Printf("[scope-audit] declared-region verification unavailable: %v", uErr) + } else { + verifier = newScopeVerifier(unmatchedRows) + } + } +``` + +Then, in the per-repeater loop, replace the `notObserved` construction: + +```go + notObserved := []string{} + for _, rgn := range declaredNamed { + if agg == nil || agg.scopes[rgn] == nil { + notObserved = append(notObserved, rgn) + } + } +``` + +with: + +```go + // Verify the declared regions this repeater has no named evidence for, + // against its own unmatched traffic. Regions already observed by name + // need no verification and are not tested — that keeps the candidate + // set to exactly the open questions. + unnamed := []string{} + for _, rgn := range declaredNamed { + if agg == nil || agg.scopes[rgn] == nil { + unnamed = append(unnamed, rgn) + } + } + regionEvidence := map[string]int{} + verifiedSet := map[string]bool{} + if verifier != nil && agg != nil && len(unnamed) > 0 { + regionEvidence = verifier.evidence(agg.unmatchedTxIDs, unnamed) + for _, rgn := range verifier.verified(regionEvidence) { + verifiedSet[rgn] = true + } + } + notObserved := []string{} + for _, rgn := range unnamed { + if !verifiedSet[rgn] { + notObserved = append(notObserved, rgn) + } + } +``` + +and add to the `ScopeAuditRow` literal: + +```go + RegionEvidence: regionEvidence, +``` + +- [x] **Step 5: Run tests to verify they pass** + +Run: `cd cmd/server && gofmt -w routes.go scopes.go && go test ./... -run 'TestHandleScopeAudit' -v` +Expected: PASS, including M1's `TestHandleScopeAuditSurfacesUnmatchedPackets` and the sorting tests — a verified region leaving `notObserved` changes a row's rank, so confirm the sort tests still hold rather than assuming. + +- [x] **Step 6: Commit** + +```bash +git add cmd/server/scopes.go cmd/server/routes.go cmd/server/scopes_test.go +git commit -m "feat(scope-audit): verify declared regions against a repeater's own unnameable traffic" +``` + +--- + +### Task 6: Show how a region was established + +**Files:** +- Modify: `public/scope-audit.js` (`mergedScopeChips`, `unmatchedCaveat`) +- Modify: `public/scope-audit.css` +- Test: `test-frontend-helpers.js` + +- [x] **Step 1: Write the failing test** + +Append to `test-frontend-helpers.js`, inside the existing `mergedScopeChips` block (before its closing `}`): + +```js + test('a region verified against the repeater own declaration is green, and says so', () => { + const h = chips({ declaredRegions: ['fm-112'], notObserved: [], regionEvidence: { 'fm-112': 23 } }); + assert.ok(h.includes('sa-chip-observed'), 'still green — it is observed'); + assert.ok(h.includes('sa-chip-verified'), 'but marked as established differently'); + assert.ok(h.includes('23'), 'the tooltip states how much evidence there is'); + }); + + test('a region observed by name carries no verified marker', () => { + const h = chips({ declaredRegions: ['be'], notObserved: [], regionEvidence: {} }); + assert.ok(h.includes('sa-chip-observed')); + assert.ok(!h.includes('sa-chip-verified'), 'a normally-named region is not a verification'); + }); + + test('a single-hit region stays grey and its tooltip explains why', () => { + const h = chips({ declaredRegions: ['fm-112'], notObserved: ['fm-112'], regionEvidence: { 'fm-112': 1 } }); + assert.ok(h.includes('sa-chip-unobserved'), 'one hit is not enough to turn it green'); + assert.ok(/one match/i.test(h), 'must say why one hit was not accepted'); + }); + + test('a missing regionEvidence field renders as before (older server)', () => { + const h = chips({ declaredRegions: ['be'], notObserved: ['be'] }); + assert.ok(h.includes('sa-chip-unobserved')); + assert.ok(!h.includes('sa-chip-verified')); + }); +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `node test-frontend-helpers.js` +Expected: FAIL — the `sa-chip-verified` assertions, since `mergedScopeChips` ignores `regionEvidence`. + +- [x] **Step 3: Implement** + +In `public/scope-audit.js`, replace `mergedScopeChips`'s body: + +```js + function mergedScopeChips(row) { + var missing = Object.create(null); + row.notObserved.forEach(function (n) { missing[n] = true; }); + var evidence = row.regionEvidence || {}; + var chips = row.declaredRegions.map(function (n) { + var observed = !missing[n]; + var hits = evidence[n] || 0; + // A green chip with evidence was established by verifying the repeater's + // own declaration against its own unnameable traffic, not by matching a + // configured region key. Same colour — it is observed either way — with + // a dotted underline so a reader can tell the two apart without a third + // colour competing for attention. + var verified = observed && hits > 0; + var cls = 'sa-chip ' + (observed ? 'sa-chip-observed' : 'sa-chip-unobserved') + (verified ? ' sa-chip-verified' : ''); + var title; + if (verified) { + title = n + ': observed — ' + hits + ' forwarded packet' + (hits === 1 ? '' : 's') + + ' in this window derive to this region, verified against the repeater’s own declared list. ' + + 'This instance holds no hashRegions key for it, so it could not be named directly.'; + } else if (observed) { + title = n + ': observed forwarding in this window'; + } else if (hits === 1) { + title = n + ': declared, and exactly one forwarded packet derives to it — that is one match in 65536 by chance alone, ' + + 'so it is not treated as evidence. Two would be.'; + } else { + title = n + ': declared, but no forwarding observed in this window'; + } + return '' + escapeHtml(n) + ''; + }); + if (!chips.length) return '—'; + return chips.join(' '); + } +``` + +- [x] **Step 4: Style the marker** + +In `public/scope-audit.css`, after `.sa-chip-unmatched`: + +```css +/* Verified-by-declaration: same green as any observed chip, since the region IS + observed. The dotted underline distinguishes how that was established without + introducing a third colour into a column that already carries two. */ +.sa-chip-verified { text-decoration: underline dotted; text-underline-offset: 2px; } +``` + +- [x] **Step 5: Narrow the row caveat** + +`unmatchedCaveat` currently fires whenever a row has any unmatched traffic. After verification, most of that traffic is explained, and a caveat that fires when the question has been answered is noise. Replace its guard: + +```js + function unmatchedCaveat(row) { + var n = row.observedUnmatchedPackets; + if (!n) return ''; + // Traffic already accounted for by verification is explained, not + // mysterious. What is left over is the interesting case: this repeater + // forwards a region it does not declare AND that this instance cannot + // name. Reporting the full count here would re-raise a question the + // Scopes column has just answered. + var explained = 0; + var evidence = row.regionEvidence || {}; + Object.keys(evidence).forEach(function (k) { explained += evidence[k]; }); + var left = n - explained; + if (left <= 0) return ''; + var label = escapeHtml(left) + ' forwarded packet' + (left === 1 ? '' : 's'); + return ' ' + + label + ' unexplained'; + } +``` + +- [x] **Step 6: Update the caveat's existing tests** + +Three M1 assertions in the `unmatchedCaveat` block assert the old wording. Replace them exactly: + +```js + test('a non-zero count renders a chip carrying the number', () => { + const h = caveat({ observedUnmatchedPackets: 148 }); + assert.ok(h.includes('sa-chip-unmatched'), 'should carry its own class'); + assert.ok(h.includes('148'), 'should state the count, not just that there is one'); + }); + + test('singular and plural are both grammatical', () => { + assert.ok(caveat({ observedUnmatchedPackets: 1 }).includes('1 forwarded packet ')); + assert.ok(caveat({ observedUnmatchedPackets: 2 }).includes('2 forwarded packets ')); + }); + + test('the title names the cause, not just the symptom', () => { + const h = caveat({ observedUnmatchedPackets: 5 }); + assert.ok(h.includes('hashRegions'), 'must name the config key that fixes it'); + }); +``` + +with: + +```js + test('a non-zero unexplained count renders a chip carrying the number', () => { + const h = caveat({ observedUnmatchedPackets: 148 }); + assert.ok(h.includes('sa-chip-unmatched'), 'should carry its own class'); + assert.ok(h.includes('148'), 'with no evidence to subtract, the whole count is unexplained'); + assert.ok(h.includes('unexplained'), 'the word changed with the meaning'); + }); + + test('singular and plural are both grammatical', () => { + assert.ok(caveat({ observedUnmatchedPackets: 1 }).includes('1 forwarded packet ')); + assert.ok(caveat({ observedUnmatchedPackets: 2 }).includes('2 forwarded packets ')); + }); + + test('the title says what unexplained traffic implies', () => { + // The cause is no longer only a hashRegions gap: after verification, what + // is left over is traffic for a region the repeater does not declare. + const h = caveat({ observedUnmatchedPackets: 5 }); + assert.ok(/does not declare/i.test(h), 'must state the sharper conclusion'); + }); +``` + +Then add: + +```js + test('traffic fully explained by verification raises no caveat', () => { + assert.strictEqual(caveat({ observedUnmatchedPackets: 23, regionEvidence: { 'fm-112': 23 } }), ''); + }); + + test('only the unexplained remainder is reported', () => { + const h = caveat({ observedUnmatchedPackets: 30, regionEvidence: { 'fm-112': 23 } }); + assert.ok(h.includes('7 forwarded packets '), 'want the remainder, not the total'); + }); +``` + +- [x] **Step 7: Run tests to verify they pass** + +Run: `node test-frontend-helpers.js` +Expected: PASS, all assertions. + +- [x] **Step 8: Commit** + +```bash +git add public/scope-audit.js public/scope-audit.css test-frontend-helpers.js +git commit -m "feat(scope-audit): mark verified regions and narrow the caveat to what stays unexplained" +``` + +--- + +### Task 7: Document it + +**Files:** +- Modify: `docs/api-spec.md` + +- [x] **Step 1: Add the field to the payload block** + +In the `GET /api/scope-audit` response block, after the `observedUnmatchedPackets` line (add a comma to it): + +``` + "observedUnmatchedPackets": number, // forwarded packets whose scope this instance holds no key for — see note below + "regionEvidence": { "": number } // declared regions corroborated by this repeater's own unnameable traffic — see note below +``` + +- [x] **Step 2: Add the note** + +After the `observedUnmatchedPackets` bullet: + +``` +- `regionEvidence` maps a declared region to how many of this repeater's own unmatched + forwarded packets derive to it. The server tests each declared region this repeater has + no *named* evidence for by deriving `SHA256("#region")[:16]` and HMAC-ing that + repeater's own unmatched packets with it — the same computation the ingestor performs + at ingest, with the candidate set narrowed to this repeater's declarations. A region + reaching **2** corroborating packets is removed from `notObserved`: `code1` is two + bytes, so one match happens by chance with probability 1/65536, while two on the same + region is (1/65536)². A region with exactly one hit therefore stays in `notObserved` + **and** appears here with the value 1, so a client can explain why it is still grey. + `notObserved` remains the single source of truth for whether a region was observed; + this field says only *how* that was established. The object is always present and may + be empty. +``` + +- [x] **Step 3: Amend the `observedUnmatchedPackets` note** + +That note predates verification. Append to it: + +``` + Since M1b, part of this count is explained: packets counted in `regionEvidence` are + attributable to a declared region after all. A client showing this as a caveat should + subtract them and report only the remainder, which carries a sharper meaning — traffic + this repeater forwards for a region it does **not** declare and this instance cannot + name. +``` + +- [x] **Step 4: Commit** + +```bash +git add docs/api-spec.md +git commit -m "docs(api): document regionEvidence on GET /api/scope-audit" +``` + +--- + +### Task 8: Benchmark and verify + +- [x] **Step 1: Write the benchmark** + +Append to `cmd/server/scope_verify_test.go`: + +```go +// BenchmarkScopeVerifierAudit models a full audit refresh: every declared name +// against every unmatched packet, once, through the memo. The naive shape would +// be targets x names x packets; this asserts the memo keeps it at names x +// packets, which is what makes the feature affordable (AGENTS.md rule 0). +func BenchmarkScopeVerifierAudit(b *testing.B) { + const packets, names, targets = 400, 124, 205 + rows := make([]unmatchedTransmissionRow, 0, packets) + for i := 0; i < packets; i++ { + rows = append(rows, unmatchedTransmissionRow{txID: int64(i + 1), rawHex: realTransportFloodPacket}) + } + txIDs := make([]int64, 0, packets) + for i := 0; i < packets; i++ { + txIDs = append(txIDs, int64(i+1)) + } + declared := make([]string, 0, names) + for i := 0; i < names; i++ { + declared = append(declared, fmt.Sprintf("r%04d", i)) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + v := newScopeVerifier(rows) + for t := 0; t < targets; t++ { + v.evidence(txIDs, declared) + } + } +} +``` + +Add `"fmt"` to the test file's imports. + +- [x] **Step 2: Run it and record the number** + +Run: `cd cmd/server && go test -bench BenchmarkScopeVerifierAudit -benchtime=5x -run '^$' ./...` +Expected: one figure. Paste the real output into the commit message. + +The budget: the audit is cached for 30s, so anything under ~1s per refresh is comfortable and under ~100ms is invisible. If the measured figure exceeds 1s, stop — either the memo is not working (check `hmacCount` in a debugger) or `scopeVerifyMaxPacketsPerTarget` needs lowering. Do not ship a number you have not looked at. + +- [x] **Step 3: Full suites** + +Run: `cd cmd/server && go test ./...` then `cd ../ingestor && go test ./...` then, from the repo root, `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js` +Expected: PASS everywhere. The ingestor is untouched by this plan; run it to prove that. + +- [x] **Step 4: Commit** + +```bash +git add cmd/server/scope_verify_test.go +git commit -m "test(scope-audit): benchmark the verification pass at audit scale" +``` + +- [x] **Step 5: Confirm against the real case after deploy** + +Done 2026-09-07 on staging (`commit: e8725306`), against a different repeater than planned, because the planned one stopped being a test case. + +`#behss` and `#fm-112` were merged into the live `hashRegions` on 2026-09-07 08:42 (58 keys to 159). Both are now named at ingest, so the `e3d3f4d7` row reads `notObserved: []` and `regionEvidence: {}`: green by the ordinary route, with nothing left for verification to establish. A region this instance can name is exactly the case M1b does not handle, so this row can no longer prove or disprove it. + +The proof came from `BE-LML-RP01` (`97028e5a`), which declares `belml`, `nl-li`, `nl-nb` and `bx`, none of which have a configured key. At window 7d: + +- `regionEvidence: {"nl-nb": 3, "belml": 1}`, `notObserved: ["nl-li", "bx", "belml"]` +- `nl-nb` renders **green with a dotted underline**, tooltip: "observed — 3 forwarded packets in this window derive to this region, verified against the repeater's own declared list. This instance holds no hashRegions key for it, so it could not be named directly." +- `belml`, with a single hit, **stays grey**. That is the threshold doing its job, and it is visible in the product rather than only in a test. +- At window 24h the same row has `nl-nb` at 1 hit and it is grey there too, so the threshold is not a function of the window boundary. + +Five other repeaters clear the threshold at 7d: `BE-KRO-RP02`, `BE-LML-RP02`, `BE-MGU-RP01`, `BE-BRE-RP03` (all `nl-nb`) and `BE-TUR-REP1` (`nl-nb`, with `nl` at one hit and correctly grey). + +Not done: the pre/post comparison against the 119/205 live baseline. Staging carries live's key set and declared-region rows but its own packet history, so the numbers are not row-comparable; the caveat counts that were measured are in the M1 plan's Step 5. + +--- + +## Notes for the implementer + +- **`notObserved` stays the single source of chip colour.** `regionEvidence` explains, it does not decide. Two fields that can disagree about the same fact is how this column got confusing in the first place. +- **Do not lower `scopeVerifyMinCorroboration` to 1.** One match in 65536 is not a rounding error at this scale: ~400 packets × ~124 names produces roughly one false positive per refresh, and a false green is worse than the grey this milestone replaces. +- **Do not widen `scopeAuditForwarderScanQuery` to carry `raw_hex`.** It returns one row per hop per flood packet — 19,049 on a 2,000-packet sample. The second narrow query exists precisely to avoid that. +- **Do not fold verification into the ingestor.** It would then write `scope_name`, and a wrong answer would persist until someone ran `scope-repair`. Read-time means a wrong answer expires with the window. That difference is the reason this is M1b and not part of M2. +- **Case matters.** `regionCode` must not fold case: the key is a hash over the raw bytes of `#name`. diff --git a/docs/plans/2026-09-07-scope-audit-handover.md b/docs/plans/2026-09-07-scope-audit-handover.md new file mode 100644 index 000000000..237ac7a52 --- /dev/null +++ b/docs/plans/2026-09-07-scope-audit-handover.md @@ -0,0 +1,218 @@ +# Handover — Scope Audit M0–M2 + +**Written:** 2026-09-07 +**Branch:** `feat/auto-region-keys` (pushed to `origin`, 33 commits ahead of `master` @ `8b115332`) +**Draft PR:** https://github.com/efiten/CoreScope/pull/11 (to `efiten/CoreScope` master — deliberately NOT upstream yet) + +> **If you are a fresh Claude session picking this up:** the code is complete and travels +> fine in git. What does not travel is why it is shaped this way, what is deliberately +> unfinished, and which measurements decide what happens next. That is what this document +> is for. Read it before touching anything; then read +> `docs/specs/2026-09-07-auto-region-keys-design.md` for the design, and the three plans in +> `docs/plans/2026-09-07-*.md` for task-level detail and tick state. + +--- + +## The problem this fixes + +Repeater `e3d3f4d7edd02aced3442b4ca77acb0824d9fcf1dc53cc42dca1ee0abe1cc0b1` +(BE-HSS-JessaZH.VIR) declares nine regions. Two of them, `behss` and `fm-112`, showed as +"not observed" in the Scope Audit — a page whose headline claim is "which repeaters declare +a region they are not actually forwarding". + +That claim was false for those rows. Packet `0a065d41d51f1f77` decodes to `code1=9209`, +which is exactly the code `#fm-112` derives by HMAC over that packet's own payload. Over a +2000-packet sample touching that repeater, 36 rows held `scope_name = ''`; re-derived, 23 +are `fm-112` and 3 are `behss`. The repeater was forwarding both. + +**Two independent causes**, found in that order: + +1. **A region with no configured key cannot be named.** `matchingRegions` + (`cmd/ingestor/main.go`) HMACs the payload with each key in `hashRegions` and compares to + `code1`. A region absent from that list is stored `scope_name = ''` — the + "transport-scoped but unnameable" state — and every declared-vs-observed comparison reads + that as absent. +2. **Forwarding was attributed to `path[last]` only.** On flood routes every forwarder + appends its hash to the END of the path (`internal/packetpath/route.go:20`), so + `path[last]` means "the transmission an uplinked observer heard directly", not "forwarded + it". The last-hop rule is genuinely needed for DIRECT routes, but both queries already + filtered `route_type IN (0,1)`, where that hazard cannot arise — so the restriction only + discarded evidence. + +Measured live on 2026-09-07: cause 2 left **133 of 205 repeaters with zero attributable +evidence**, of which 110 had relayed traffic in the window. + +--- + +## What is built + +| Milestone | What it does | Where | +|---|---|---| +| **M0** | Attribute flood forwarding to every path hop, not just the last | `cmd/server/scopes.go` | +| **M1** | Count and surface traffic this instance cannot name, as a caveat | `cmd/server/`, `public/` | +| **M1b** | Verify a repeater's declared regions against its own unnameable traffic | `cmd/server/scope_verify.go`, `public/` | +| **M2** | Derive region keys from `node_declared_regions` — **opt-in, default off** | `cmd/ingestor/` | +| M3 | Path-evidence tie-break | **not built**, gated on a measurement below | + +Automated verification at handover: `cmd/server` full suite ok (119.9s), `cmd/ingestor` ok +apart from one pre-existing Windows failure (below), frontend 692/99/18, `gofmt -l` and +`go vet` clean. + +--- + +## Decisions you should not quietly reverse + +**M1b's threshold is 2 corroborating packets, and the arithmetic is the argument.** `code1` +is two bytes, so an unrelated region name matches a given packet with probability 1/65536. +Across ~400 unmatched packets and ~124 declared names, chance alone produces roughly one +false match per audit refresh. Two matches on the same region for the same repeater is +(1/65536)². Lowering it to 1 does not make the feature noisy — it makes it unsound. See +`scopeVerifyMinCorroboration` in `cmd/server/scope_verify.go`. + +**M1b is read-time and writes nothing.** A wrong answer expires with the window instead of +sitting in `transmissions.scope_name` until someone runs `scope-repair`. That, plus the +read/write separation invariant in AGENTS.md, is why it is a server-side inference and not +part of M2. + +**`notObserved` remains the single source of chip colour.** `regionEvidence` says only +*how* a region was established, and explains a region that got exactly one hit and +therefore stayed grey. Two fields that can disagree about the same fact is how this column +became confusing in the first place. + +**M2 is default off** and an absent config block leaves behaviour byte-for-byte unchanged — +asserted by `TestAutoRegionKeysDefaultsOff` and `TestRefreshFromStoreIsNoOpWhenDisabled`, +not assumed. + +**M2 was re-scoped after M1b landed.** It no longer fixes the audit; M1b does. What M2 +fixes is the rest of the product still not seeing these regions: `/api/packets` shows an +empty scope on a packet that has one, `/api/scope-stats` omits whole regions from +`byRegion`, `nodes.default_scope` is never set for them, and the per-node observed list is +short. Real value, lower urgency. If you find yourself justifying M2 by the audit, re-read +the spec's "Which cause dominates". + +**`scope-repair` had to be pulled onto the same key set.** Left on `loadRegionKeys` alone it +would re-derive every automatically-named row as unmatched and write `''` over the name — +a maintenance tool erasing exactly what M2 produces. Guarded by +`TestScopeRepairKeepsDerivedNames`. + +--- + +## Traps found the hard way + +**gofmt rewrites `''` inside doc comments.** It applies the old godoc typographic +substitution and turns a two-single-quote digraph into a closing curly quote. A comment +explaining that a query keys on an EMPTY `scope_name` rendered as a quotation mark, and came +back on every gofmt run. Written out in words in `cmd/server/scope_verify.go` with a note; +do not put the literal back. + +**Caching the expensive operation is not the same as removing the expensive loop.** M1b's +plan claimed ~50ms because caching per `(region, transmission)` cuts HMACs from ~10M to +~50k. Measured: **501ms**. The HMACs had become a rounding error while the *iteration* was +still cubic — 10.2M map lookups at ~49ns. Re-keyed per region, holding the set of matching +transmissions: **36ms**, a 14× improvement. `hmacCount` guards the first mistake; only the +benchmark caught the second. + +**M2 has no such loop to hoist.** One HMAC per key per packet is irreducible: `code1` is an +HMAC over the payload, so nothing is payload-independent to index on. The old `matchScope` +comment suggesting a "pre-indexed lookup table" was wrong and a note where it stood says so. +Benchmarked linear at ~0.65µs/key: at the 314-key ceiling, 217µs/packet — 0.0008% of one +core at this network's 0.037 transport-scoped packets/s. + +--- + +## Outstanding — and where each piece has to happen + +### Session of 2026-09-07 evening: what has since been done + +Staging runs this branch (`/api/health` reports the branch head). To make it a usable +test bed its config was given live's 159 `hashRegions`, `clientRegions.enabled` and +`retention.clientRegionsDays: 90`, live's 1024 `node_declared_regions` rows were +imported into the staging database, and `scope-repair -apply` was run there (597 rows +newly named, 402 corrected to unmatched). Live was not touched. + +- **Item 1: staging done, live not done.** +- **Item 2: done**, but not on the row this document names. `#behss` and `#fm-112` + were merged into the live `hashRegions` on 2026-09-07 08:42, so both are named at + ingest now and the `e3d3f4d7` row reads `notObserved: []` with `regionEvidence: {}`. + A region this instance can name is the one case M1b does not handle, so that row can + no longer prove it either way. The proof came from `BE-LML-RP01` (`97028e5a`) + instead: at 7d, `nl-nb` green with a dotted underline on 3 corroborating packets, + `belml` grey on 1. See the M1b plan's Task 8 Step 5. +- **Item 3: already done before this session** (that 08:42 config change, 58 keys to + 159). Its cost is worth knowing: the added keys collide on `code1`, and + `scope-repair` on staging moved 402 rows from a name back to unmatched, 98 of them + `#be` and 57 `#de`. +- **Item 4: three of four measured.** Default-off proof, feature-on proof and the + double-caveat check are recorded in the plans and the spec. The ambiguity rate is + **still open**: it prints on a 15-minute ticker and each deploy replaces the + container, taking `docker logs` with it. +- **New, found while validating: `/api/scope-audit?window=7d` costs 16.7s cold** on a + live-shaped database, against 4.0s for 24h and 0.15s for 1h. Two commits address it + (`942761c4`, `b7515cec`); the spec's M0 section carries the measurements, including + the three SQL-side approaches that measured worse and were rejected. + +### On the build/publish laptop (server access) + +1. **Deploy the branch to staging, then live.** Nothing here is running anywhere. The + original complaint is still visible on analyzer.on8ar.eu exactly as it was. + +2. **Browser validation** (AGENTS.md rule 2), never done for any milestone. It cannot be + done from a dev checkout: `test-fixtures/e2e-fixture.db` predates this whole feature — + no `node_declared_regions` table, no `scope_name` column. On staging or live, the + `e3d3f4d7…` row must show **`fm-112` and `behss` green with dotted underlines**, and + `regionEvidence` must report counts roughly in the hand-measured proportion (23 : 3 out + of 36 in a 2000-packet sample — exact numbers will differ, both must clear 2). + +3. **The config fix, which is independent of all of this and works today.** + `hashRegions` is missing 123 region names that repeaters in this network declare. A + ready-to-run script was produced this session but lives only in a scratchpad — regenerate + it from `/api/scope-audit`'s `declaredRegions` if you need it. Procedure: back up + `config.json`, merge (do not replace) with `jq '.hashRegions = ((.hashRegions // []) + $new | unique)'`, + restart only `corescope-ingestor`, run `ingestor scope-repair` as a dry run, read the + report, then `-apply` **with the ingestor stopped** — it does every UPDATE in one + transaction and `busy_timeout` is 5s, so a live ingestor hits `SQLITE_BUSY`. + Names are case-sensitive: the key is `SHA256("#name")[:16]`. + +4. **Four measurements**, three of which are decisions rather than tick-boxes: + - M2 default-off proof: startup log must read `autoRegionKeys disabled — only the N configured hashRegions key(s) are in force`, with no `[regions] derived` line. + - M2 feature-on proof: the refresh log reports declared/kept/total, and `scope-repair --dry-run` lists the newly named regions. + - **The ambiguity rate.** After a day with M2 on, read `[regions] scope matches: unique=… explicit-over-derived=… ambiguous=… none=…`. **That number decides whether M3 is built at all** — the spec estimates ~10 ambiguous packets a week after tier 2 absorbs the rest, which would not justify the machinery. Record it in the spec's M3 section, replacing the estimate. + - **The double-caveat check.** M0 widened attribution from 222 distinct last-hops to ~964 distinct hop prefixes, so `ambiguousHops` — currently 0 on all 205 rows precisely because so few hops were considered — will start firing. If both that chip and M1's unexplained-traffic chip end up lit on most rows, neither tells the reader anything. Pre-deploy baseline: **119 of 205 rows carry any finding**, both caveats at 0. Fetch `/api/scope-audit?window=24h` and count rows with `ambiguousHops > 0`, `observedUnmatchedPackets > 0`, and both. + +### Anywhere + +5. **Code review.** 36 changed files across four milestones, reviewed by nobody. In a + feature where two cost models and one test case turned out wrong, this is not a + formality. + +6. **`go test -race` on the ingestor.** Could not run on the Windows machine (needs cgo, + no gcc). **CI will not cover it either**: `deploy.yml:134` runs `-race` on the server + only, added for PR #1208's atomic.Pointer migration; line 143 tests the ingestor without + it. M2 introduces an `atomic.Pointer` in the ingestor. Either add `-race` to that line or + run it locally on Linux. `atomic.Pointer` is race-free by construction and no published + snapshot is ever mutated — but that is an argument, not a measurement. + +7. **Upstream PR**, once the above is done. PR #11 is a draft against `efiten/CoreScope` + master on purpose; upstream is `Kpa-clawbot/CoreScope`, currently 19 commits ahead of + this fork's master, which will need reconciling first. + +--- + +## Known gaps, honestly stated + +**M1b and M2 have never been tested together.** Each is covered on its own. With M2 enabled, +packets that were unmatched get named at ingest, so M1b's verifier has fewer candidates and +the chip goes green by name rather than by verification. That is coherent — the region lands +in `agg.scopes` and leaves `notObserved` by the normal route — but no test pins it. A gap in +coverage, not a known bug. + +**One ingestor test fails on Windows and always did.** +`TestWriteStatsAtomic_SymlinkAtDestIsReplaced` — `os.Symlink` needs +`SeCreateSymbolicLinkPrivilege`. Proven unrelated: `git diff 8b115332..HEAD -- cmd/ingestor/` +was empty before M2 started. It passes on Linux, so CI is the place to confirm. +`TestMQTTStallWatchdog_DisconnectedEscalationThrottled_1749` is load-flaky — it failed in a +full suite run under load and passed in isolation. + +**Regions in use but never declared over RF stay invisible.** Neither the config fix nor M2 +can name those; both lean on the declared side. This is stated in the spec's Scope section +and is not a defect to go looking for. diff --git a/docs/plans/2026-09-07-scope-audit-unmatched-caveat.md b/docs/plans/2026-09-07-scope-audit-unmatched-caveat.md new file mode 100644 index 000000000..d282c8fe5 --- /dev/null +++ b/docs/plans/2026-09-07-scope-audit-unmatched-caveat.md @@ -0,0 +1,466 @@ +# Scope Audit — Unmatched-Traffic Caveat (M1) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Stop the Scope Audit from presenting "this instance cannot name that region" as "this repeater is not forwarding that region", by counting the unmatched forwarded packets the endpoint currently discards and surfacing them as a caveat. + +**Architecture:** `ScopeAuditForwarding` (`cmd/server/scopes.go`) drops rows whose `scope_name` is the empty string with a bare `continue`. That empty string is the ingestor's "transport-scoped, but no configured region key matched" state. Counting those per target, exposing the count on `ScopeAuditRow`, and rendering a caveat chip gives the reader the missing half of the story. Server-side and frontend only — no ingestor change, no schema change, no config coupling. + +**Tech Stack:** Go 1.x (`cmd/server`, stdlib `testing`), vanilla JS frontend (`public/`), Node's `assert` via `test-frontend-helpers.js`. + +**Spec:** `docs/specs/2026-09-07-auto-region-keys-design.md`, section "4. Scope-audit honesty". + +**Depends on M0** (`### M0 — Forwarder attribution`, added to the spec in `b610d461`). Both queries currently credit a transmission to `path[last]` only, but on a flood route every forwarder appends its hash to the END of the path (`internal/packetpath/route.go:20`), so `path[last]` means "heard directly by an uplinked observer", not "forwarded it". Measured live: 133 of 205 repeaters have zero attributable evidence. The `unmatchedPackets` counter below counts unmatched rows **among the rows attribution admits**, so before M0 it reads zero for that same 65% — inviting exactly the wrong conclusion in a brand-new field. Do not start Tasks 1, 2 or 5 until M0 has landed. + +**Status: code complete, verification partly deferred.** M0 landed as `d93b4463`, which unblocked the rest. + +| Task | Commit | State | +|---|---|---| +| 1 — counter in `ScopeAuditForwarding` | `70e6bcd5` | done | +| 2 — `ObservedUnmatchedPackets` on the API row | `ca464b59` | done | +| 3 — caveat chip | `79f38ef1` | done (landed first; inert until the field existed) | +| 4 — `docs/api-spec.md` | `93a0c385` | done | +| 5 — verification | — | automated part done; steps 3–5 **deferred to deploy** | + +Automated verification passed on all four suites: `cmd/server` full (ok, 94.8s), `test-frontend-helpers.js` (686), `test-packet-filter.js` (99), `test-aging.js` (18). + +Steps 3, 4 and 5 of Task 5 cannot run in this working copy: `test-fixtures/e2e-fixture.db` predates the whole feature — it has no `node_declared_regions` table and no `scope_name` column — and the live instance runs the pre-M0 code. They run against staging or live after deploy, per the operator's decision. Do not mark this plan complete until they have. + +--- + +## Why this is a real defect, not a nicety + +Measured on the live instance 2026-09-07: of 613 `notObserved` entries across 205 repeaters, 260 (42%) name a region that never appeared under any name in the whole 7-day window. Two of them (`behss`, `fm-112`) are hash-verified as genuinely forwarded traffic that the instance simply cannot name. The page's headline claim — "which repeaters declare a region they are not actually forwarding" — is false for those rows. + +--- + +## File Structure + +| File | Responsibility | Change | +|---|---|---| +| `cmd/server/scopes.go` | `scopeAuditTargetAgg`, `ScopeAuditForwarding`, `ScopeAuditRow` | Modify | +| `cmd/server/routes.go` | `handleScopeAudit` row assembly | Modify | +| `cmd/server/scopes_test.go` | Store- and handler-level tests | Modify | +| `public/scope-audit.js` | `unmatchedCaveat`, wired into `rowHtml`, exported for tests | Modify | +| `public/scope-audit.css` | `.sa-chip-unmatched` | Modify | +| `test-frontend-helpers.js` | Caveat rendering assertions | Modify | +| `docs/api-spec.md` | `observedUnmatchedPackets` field + note | Modify | + +--- + +### Task 1: Count unmatched forwarded packets per target + +**Files:** +- Modify: `cmd/server/scopes.go` (`scopeAuditTargetAgg` ~line 388, `ScopeAuditForwarding` ~line 530) +- Test: `cmd/server/scopes_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/server/scopes_test.go`, after `TestScopeAuditForwardingAmbiguousHopCreditsNeitherTarget`: + +```go +// TestScopeAuditForwardingCountsUnmatchedPackets: a transport-scoped packet +// whose code1 matched no configured region key is stored with scope_name = "" +// (scopeNameForDB's "transport-scoped but unnameable" state). It is not a +// named scope, so it must not enter agg.scopes, and it is not an unscoped +// plain flood either, so it must not enter unscopedPackets. It is its own +// fact: this instance saw the target forward traffic it holds no key for. +// Without this counter the audit reports the declared region as "not +// observed", which reads as a finding about the repeater rather than about +// this instance's configuration. +func TestScopeAuditForwardingCountsUnmatchedPackets(t *testing.T) { + s := newScopeTestStore(t) + hop := testFullPubkeyA[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, s, hop, scopeUnmatched(), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil { + t.Fatalf("want an agg for the target, got none (result = %+v)", got) + } + if agg.unmatchedPackets != 1 { + t.Errorf("unmatchedPackets = %d, want 1", agg.unmatchedPackets) + } + if len(agg.scopes) != 0 { + t.Errorf("scopes = %+v, want empty — an unmatched packet names no region", agg.scopes) + } + if agg.unscopedPackets != 0 { + t.Errorf("unscopedPackets = %d, want 0 — unmatched is not the same as unscoped", agg.unscopedPackets) + } +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/server && go test ./... -run TestScopeAuditForwardingCountsUnmatchedPackets -v` +Expected: FAIL to compile — `agg.unmatchedPackets undefined (type *scopeAuditTargetAgg has no field or method unmatchedPackets)` + +- [x] **Step 3: Add the field** + +In `cmd/server/scopes.go`, in `scopeAuditTargetAgg`, add after `unscopedPackets`: + +```go + // unmatchedPackets counts packets this target was observed forwarding + // that carried a transport scope no configured region key matched + // (transmissions.scope_name = ""). It is deliberately NOT folded into + // unscopedPackets: '*' governs plain unscoped floods, and an unmatched + // packet is the opposite — it IS scoped, this instance just holds no key + // for that region. A non-zero value means any notObserved entry on this + // row may be unnameable rather than unforwarded. + unmatchedPackets int64 +``` + +- [x] **Step 4: Count it** + +In `ScopeAuditForwarding`, replace the bare skip: + +```go + if scopeName.String == "" { + continue // unmatched — not part of the declared/observed comparison + } +``` + +with: + +```go + if scopeName.String == "" { + // Unmatched: transport-scoped, but no configured region key + // matched code1. Still not part of the declared/observed + // comparison — it names no region — but it is the evidence + // that a notObserved finding on this row may be a gap in this + // instance's hashRegions rather than in the repeater. + agg.unmatchedPackets++ + continue + } +``` + +- [x] **Step 5: Run tests to verify they pass** + +Run: `cd cmd/server && go test ./...` +Expected: PASS, including the pre-existing `TestScopeAuditForwardingAttributesUnambiguousHop` and `TestScopeAuditForwardingAmbiguousHopCreditsNeitherTarget`. + +- [x] **Step 6: Commit** + +```bash +git add cmd/server/scopes.go cmd/server/scopes_test.go +git commit -m "feat(scope-audit): count the unmatched packets ScopeAuditForwarding discards" +``` + +--- + +### Task 2: Surface the count on the API row + +**Files:** +- Modify: `cmd/server/scopes.go` (`ScopeAuditRow`, after `AmbiguousHops`) +- Modify: `cmd/server/routes.go` (`handleScopeAudit`, the `unscopedPackets, ambiguousHops` block and the `ScopeAuditRow` literal) +- Test: `cmd/server/scopes_test.go` + +- [x] **Step 1: Write the failing test** + +Append to `cmd/server/scopes_test.go`, after `TestHandleScopeAuditSurfacesAmbiguousHops`: + +```go +// TestHandleScopeAuditSurfacesUnmatchedPackets: a repeater declares "behss", +// and this instance sees it forward transport-scoped traffic it cannot name. +// The row must still list "behss" as notObserved — an unmatched packet names +// no region, so it cannot satisfy the declaration — but it must also carry +// observedUnmatchedPackets, so a client can say the finding might be a +// missing region key rather than a silent repeater. +func TestHandleScopeAuditSurfacesUnmatchedPackets(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "behss", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, pk[:4], scopeUnmatched(), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 { + t.Fatalf("repeaters = %+v, want 1", got.Repeaters) + } + row := got.Repeaters[0] + if row.ObservedUnmatchedPackets != 1 { + t.Errorf("observedUnmatchedPackets = %d, want 1", row.ObservedUnmatchedPackets) + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "behss" { + t.Errorf("notObserved = %v, want [\"behss\"] — an unmatched packet names no region and cannot satisfy a declaration", row.NotObserved) + } + if row.WildcardContradiction { + t.Error("wildcardContradiction = true, want false — unmatched traffic is scoped, so it says nothing about '*'") + } +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `cd cmd/server && go test ./... -run TestHandleScopeAuditSurfacesUnmatchedPackets -v` +Expected: FAIL to compile — `row.ObservedUnmatchedPackets undefined` + +- [x] **Step 3: Add the field to `ScopeAuditRow`** + +In `cmd/server/scopes.go`, add after the `AmbiguousHops` field: + +```go + // ObservedUnmatchedPackets counts packets this repeater was observed + // forwarding whose transport scope matched no region key this instance + // holds. Like AmbiguousHops it is a caveat, not a finding: a non-zero + // value means a NotObserved entry here may be a gap in this instance's + // hashRegions rather than in the repeater's forwarding. It says nothing + // about DeclaredWildcard — unmatched traffic IS scoped, so it is not + // evidence for or against '*'. + ObservedUnmatchedPackets int64 `json:"observedUnmatchedPackets"` +``` + +- [x] **Step 4: Populate it in the handler** + +In `cmd/server/routes.go`, in `handleScopeAudit`, change: + +```go + var unscopedPackets, ambiguousHops int64 + if agg != nil { + unscopedPackets = agg.unscopedPackets + ambiguousHops = agg.ambiguousHops + } +``` + +to: + +```go + var unscopedPackets, ambiguousHops, unmatchedPackets int64 + if agg != nil { + unscopedPackets = agg.unscopedPackets + ambiguousHops = agg.ambiguousHops + unmatchedPackets = agg.unmatchedPackets + } +``` + +and add to the `ScopeAuditRow` literal, after `AmbiguousHops: ambiguousHops,`: + +```go + ObservedUnmatchedPackets: unmatchedPackets, +``` + +- [x] **Step 5: Run tests to verify they pass** + +Run: `cd cmd/server && go test ./...` +Expected: PASS + +- [x] **Step 6: Commit** + +```bash +git add cmd/server/scopes.go cmd/server/routes.go cmd/server/scopes_test.go +git commit -m "feat(scope-audit): expose observedUnmatchedPackets on the API row" +``` + +--- + +### Task 3: Render the caveat — DONE (`79f38ef1`) + +Landed ahead of the rest: the chip is inert until the server sends the field, so it +could not break anything, and `public/scope-audit.js` was the one file M0 was not +holding. Steps kept below as the record of what was built; do not redo them. + +**Files:** +- Modify: `public/scope-audit.js` (new `unmatchedCaveat`, called in `rowHtml`, added to `window.__meshcoreScopeAuditInternals`) +- Modify: `public/scope-audit.css` +- Test: `test-frontend-helpers.js` + +- [x] **Step 1: Write the failing test** + +Append to `test-frontend-helpers.js`, after the `mergedScopeChips` block: + +```js +// ===== scope-audit.js: unmatchedCaveat ===== +// A declared region this instance holds no hashRegions key for can never turn +// green, however much traffic the repeater forwards. On live data that +// explains up to 42% of all notObserved entries, so the column must be able +// to say so instead of presenting every grey chip as a confirmed gap. +console.log('\n=== scope-audit.js: unmatchedCaveat ==='); +{ + const ctx = makeSandbox(); + ctx.registerPage = () => {}; + loadInCtx(ctx, 'public/app.js'); + loadInCtx(ctx, 'public/scope-audit.js'); + const caveat = ctx.__meshcoreScopeAuditInternals.unmatchedCaveat; + + test('zero unmatched packets renders nothing at all', () => { + assert.strictEqual(caveat({ observedUnmatchedPackets: 0 }), ''); + }); + + test('a missing field renders nothing (older server, field absent)', () => { + assert.strictEqual(caveat({}), ''); + }); + + test('a non-zero count renders a chip carrying the number', () => { + const h = caveat({ observedUnmatchedPackets: 148 }); + assert.ok(h.includes('sa-chip-unmatched'), 'should carry its own class'); + assert.ok(h.includes('148'), 'should state the count, not just that there is one'); + }); + + test('singular and plural are both grammatical', () => { + assert.ok(caveat({ observedUnmatchedPackets: 1 }).includes('1 forwarded packet ')); + assert.ok(caveat({ observedUnmatchedPackets: 2 }).includes('2 forwarded packets ')); + }); + + test('the title names the cause, not just the symptom', () => { + // The operator fix is a hashRegions edit; a caveat that does not say so + // sends them looking at the repeater instead of at their own config. + const h = caveat({ observedUnmatchedPackets: 5 }); + assert.ok(h.includes('hashRegions'), 'must name the config key that fixes it'); + }); +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `node test-frontend-helpers.js` +Expected: FAIL — `TypeError: caveat is not a function` + +- [x] **Step 3: Implement `unmatchedCaveat`** + +In `public/scope-audit.js`, add immediately after `ambiguousCaveat`: + +```js + // unmatchedCaveat flags rows where this instance saw the repeater forward + // transport-scoped traffic it holds no region key for. Those packets can + // never satisfy a declared region — the server has no name to match against + // — so any "not observed" entry on such a row may be a gap in this + // instance's hashRegions rather than in the repeater's forwarding. Distinct + // from ambiguousCaveat: that one is a prefix collision between two nodes, + // this one is a missing key on our side. + function unmatchedCaveat(row) { + var n = row.observedUnmatchedPackets; + if (!n) return ''; + return ' ' + + n + ' forwarded packet' + (n === 1 ? '' : 's') + ' unnameable'; + } +``` + +Then call it in `rowHtml`, in the Scopes ``, immediately after `ambiguousCaveat(row)`: + +```js + '' + mergedScopeChips(row) + (row.declaredWildcard ? ' *' : '') + ambiguousCaveat(row) + unmatchedCaveat(row) + '' + +``` + +And export it, extending the existing internals object: + +```js + window.__meshcoreScopeAuditInternals = { mergedScopeChips: mergedScopeChips, emptyStateHtml: emptyStateHtml, sourcesLineHtml: sourcesLineHtml, unmatchedCaveat: unmatchedCaveat }; +``` + +- [x] **Step 4: Style the chip** + +In `public/scope-audit.css`, add after the `.sa-chip-ambiguous` rule: + +```css +.sa-chip-unmatched { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); border: 1px dashed var(--border); font-family: inherit; font-style: italic; } +``` + +- [x] **Step 5: Run tests to verify they pass** + +Run: `node test-frontend-helpers.js` +Expected: PASS, all assertions including the pre-existing `mergedScopeChips` block. + +- [x] **Step 6: Commit** + +```bash +git add public/scope-audit.js public/scope-audit.css test-frontend-helpers.js +git commit -m "feat(scope-audit): say when a not-observed region is one we cannot name" +``` + +--- + +### Task 4: Document the field + +**Files:** +- Modify: `docs/api-spec.md` (payload block ~line 1890, Notes list ~line 1921) + +- [x] **Step 1: Add the field to the payload block** + +In the `GET /api/scope-audit` response block, after the `ambiguousHops` line, add a comma to that line and append: + +``` + "ambiguousHops": number, // forwarder hops this window that could not be attributed — see note below + "observedUnmatchedPackets": number // forwarded packets this window whose scope this instance holds no key for — see note below +``` + +- [x] **Step 2: Add the note** + +In the same section's **Notes:** list, directly after the `ambiguousHops` bullet: + +``` +- `observedUnmatchedPackets` counts packets this repeater was observed forwarding whose + transport scope matched no region key this instance has configured (`hashRegions`), so + the ingestor stored them with an empty `scope_name`. Those packets name no region and + therefore cannot satisfy a declared one, which means a repeater forwarding a region this + instance cannot name appears in `notObserved` exactly like one forwarding nothing. A + non-zero value is a caveat on this row's `notObserved`, in the same spirit as + `ambiguousHops`, but with a different cause and a different fix: `ambiguousHops` is a + prefix collision between two repeaters, `observedUnmatchedPackets` is a missing entry in + this instance's own configuration. It is **not** evidence for or against + `declaredWildcard` — unmatched traffic is scoped, so it never affects + `wildcardContradiction`. +``` + +- [x] **Step 3: Verify no other doc contradicts it** + +Run: `grep -rn "not part of the declared/observed comparison" docs/ cmd/` +Expected: only the updated comment in `cmd/server/scopes.go`; no stale doc claiming unmatched rows are discarded. + +- [x] **Step 4: Commit** + +```bash +git add docs/api-spec.md +git commit -m "docs(api): document observedUnmatchedPackets on GET /api/scope-audit" +``` + +--- + +### Task 5: Verify end to end + +- [x] **Step 1: Full Go suite** + +Run: `cd cmd/server && go test ./...` then `cd ../ingestor && go test ./...` +Expected: PASS in both. The ingestor is untouched by this plan; run it to prove that. + +- [x] **Step 2: Full frontend suite** + +Run: `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js` +Expected: PASS + +- [x] **Step 3: Browser validation** (AGENTS.md rule 2) + +Done 2026-09-07 on staging (`staging.on8ar.eu`; `/api/health` reports `commit: e8725306`, this branch's head). The `BE-LML-RP01` row at window 24h shows the chip reading `8 forwarded packets unexplained` beside its five grey chips; at 7d it reads `21 forwarded packets unexplained`. Both match the API for the same window at the same moment: 24h `observedUnmatchedPackets: 9` minus 1 packet explained by `regionEvidence`, 7d `observedUnmatchedPackets: 25` minus 4. Screenshots taken by the operator. + +- [x] **Step 4: Confirm the fix against the real case** + +Confirmed by a different row, because the original case no longer exists in this form. `#behss` and `#fm-112` were merged into the live `hashRegions` on 2026-09-07 08:42 (`config.json.bak-hashregions-20260907-084120`, 58 keys to 159), so both regions are now named at ingest and the `e3d3f4d7` row reads `notObserved: []` with `regionEvidence: {}` on live and on staging. Its unnameable count is non-zero (`observedUnmatchedPackets: 9` at 24h), which is the part of Step 4 that still applies. + +The chip was instead confirmed against `BE-LML-RP01` (`97028e5a`), whose declared `belml`, `nl-li` and `bx` have no configured key. See Step 3 for the numbers. + +- [x] **Step 5: Check the two caveats are not both permanently on** + +Measured on staging after deploy, 206 rows, 2026-09-07: + +| | 24h | 7d | +|---|---|---| +| `ambiguousHops > 0` | 2 | 2 | +| `observedUnmatchedPackets > 0` | 63 | 78 | +| both | 0 | 0 | + +`ambiguousHops` did start firing after M0 (0 rows before, 2 after) but stays far below the "roughly half the rows" line, and no row carries both caveats. Nothing to raise. + +Caveat on the comparison: this is staging with live's 159-key `hashRegions` and live's 1024 `node_declared_regions` rows imported, after `scope-repair -apply`. It is not the same database as the 119/205 pre-deploy baseline taken on live, so the row counts are comparable in shape, not row for row. + +--- + +## Notes for the implementer + +- **Do not** make the server read `hashRegions`. It deliberately does not, and adding that coupling would mislabel every region on a deployment where server and ingestor do not share a `config.json`. This plan needs no config. +- **Do not** fold `unmatchedPackets` into `unscopedPackets`. They are opposites: unscoped means the packet carried no scope at all (`scope_name` SQL NULL), unmatched means it carried one this instance cannot name (`scope_name` empty string). `scopeNameForDB` in `cmd/ingestor/db.go` is the source of truth for that encoding. +- The chip is a caveat, not an alarm. It reuses the muted, dashed-border treatment of `.sa-chip-ambiguous` on purpose — it must not compete visually with the red/green chips that carry the actual finding. diff --git a/docs/specs/2026-09-07-auto-region-keys-design.md b/docs/specs/2026-09-07-auto-region-keys-design.md new file mode 100644 index 000000000..359dd8fee --- /dev/null +++ b/docs/specs/2026-09-07-auto-region-keys-design.md @@ -0,0 +1,609 @@ +# Auto-Derived Region Keys & Scope-Audit Honesty — Design Spec + +**Date:** 2026-09-07 +**Status:** Approved (design); **amended 2026-09-07** — a second, independent cause +of `notObserved` was measured after approval (see *Second cause* below); it adds M0 +and re-orders the milestones. Implementation not started. + +--- + +## Problem + +The Scope Audit reports "declared but not observed" for regions this instance is +structurally incapable of observing, presenting a configuration gap as a finding +about someone else's repeater. + +There are **two independent reasons** an instance can be structurally incapable of +observing a region, and they were found in that order rather than together: it may +hold no key that can name the region, or it may discard the evidence before +nameability is ever consulted. Both are below; the second one gates the first. + +### First cause — a region with no key cannot be named + +A transport-scoped packet's region is identified by HMAC-ing the payload with +`SHA256("#name")[:16]` for every configured region and comparing the derived +2-byte code to the packet's `code1` (`matchingRegions`, `cmd/ingestor/main.go:1700`). +A region absent from `hashRegions` therefore cannot be named: the ingestor stores +`scope_name = ""` (the "transport-scoped but unnameable" state per +`scopeNameForDB`, `cmd/ingestor/db.go:2038`). `ScopeAuditForwarding` skips those +rows outright (`cmd/server/scopes.go:530`), so the region never reaches +`agg.scopes` and `handleScopeAudit` lists it under `notObserved` +(`cmd/server/routes.go:3611`). + +### Evidence — first cause + +Verified against the live instance (analyzer.on8ar.eu) on 2026-09-07. + +Repeater `e3d3f4d7…c0b1` (BE-HSS-JessaZH.VIR) declares nine regions and shows +`behss` and `fm-112` as not observed. Packet `0a065d41d51f1f77` decodes to +`route_type=0` (TRANSPORT_FLOOD), `payload_type=5`, `code1=9209`, path `["E3D3"]` +— the repeater is the last hop, so it is attributable. Re-deriving the code for +each candidate region name over that packet's own payload: + +| Region | Derived code1 | | +|---|---|---| +| `#fm-112` | `9209` | matches | +| `#behss` | `AAA8` | | +| `#be` | `9CC7` | | + +The packet *is* `fm-112`, and is stored as `scope_name = ""`. + +Across a 2000-packet sample touching that repeater, 36 rows hold `scope_name = ""`. +Re-derived: 23 are `fm-112`, 3 are `behss`, 5 are `be` (genuine #1609 ambiguity — +`be` is configured, but a second key also matched), and 5 belong to a region +outside the candidate set. + +Network-wide over 7 days: repeaters declare **124 distinct region names** (123 +after dropping the literal `null`, which is a client serialisation artefact — +note that the automatic path in this design does *not* filter on name values, so +it would keep it; see the candidate filter below); this instance can name **16**. +`unknownScope` is 1.72% of all transport-scoped traffic. +Of 613 `notObserved` entries across 205 repeaters, **260 (42%)** name a region +that never appeared under any name in the whole window. That 42% is an upper +bound — a configured but genuinely idle region counts toward it — but `behss` and +`fm-112` are hash-verified, not inferred. + +It is **also not a lower bound**, which was not visible when this was written: an +entry counted there can be unattributable as well as unnameable. See *Which cause +dominates* below, where the same measurement is re-taken with that confound +separated out. + +### Second cause — forwarding is attributed to the last path hop only + +`scopeAuditForwarderScanQuery` credits a transmission to exactly one node, the +last hop of its path (`cmd/server/scopes.go:447`), as does the per-node +`scopeConformanceQuery` (`cmd/server/scopes.go:113`). On a flood-family route +every forwarder APPENDS its own hash to the END of the path +(`internal/packetpath/route.go:20`), so `path[last]` does not mean "forwarded +this packet" — it means "was the transmission an uplinked observer heard +directly". Every earlier hop forwarded the same packet and is thrown away. + +The last-hop rule is genuinely required for DIRECT routes (2, 3), which consume +the next hop from the FRONT, leaving `path[last]` as the route's far end rather +than the transmitter. But both queries already restrict to `route_type IN (0, 1)` +via `scopeConformanceForwarderRouteTypesSQL`, where that hazard cannot arise — so +inside these two queries the restriction discards evidence and buys nothing. + +Consequence: a repeater is invisible to the audit unless an uplinked observer sits +within direct RF range of it. Regions it forwards read `notObserved` no matter how +many keys this instance holds, so the first cause's fix cannot reach them. + +### Evidence — second cause + +Verified against the live instance on 2026-09-07, 24h window, via +`/api/scope-audit`, `/api/scope-stats`, `/api/nodes`, `/api/nodes/{pk}/scopes` and +`/api/packets`. + +Found on repeater `cf7903ce…7e12` (BE-HHE-LAAK-EDG-01). `/api/nodes/{pk}/scopes` +returns `observed: []` with **the entire route mix zero** for 1h, 24h *and* 7d — +not just no named scope, but no attributed transmission of any kind — while the +same page's node header reports `transported_scopes: +["#be","#be-vli","#eu","#fm-112"]` and `relay_count_24h: 306`. Two panels, one +page, one database, opposite answers. + +Its own traffic over 14 days (373 rows, `/api/packets?node=`): + +| | | +|---|---| +| flood-family packets carrying `CF79` in the path | 155 | +| of those, `CF79` as `path[last]` | **0** | +| `CF79` at path position 0 | 103 | +| scope names on those packets | 126 × `#be`, 7 × `#fm-112`, 1 × `#eu`, 1 unmatched | +| rows where `CF79` *is* `path[last]` | 52 — all `route_type = 2` (DIRECT), correctly excluded | + +It is an edge node: its strongest neighbour BE-ZOD-MOSKEE-DIS hears it 1397 times +against 1 the other way, so its relays are forwarded onward at least once before +any uplinked observer logs them. `transported_scopes` sees all of it because +`byPathHop` indexes *every* hop (`cmd/server/repeater_enrich_bulk.go:166`); the +two scope queries see none of it. The 52 DIRECT rows are the useful control — they +confirm the last-hop rule is still load-bearing for route types 2 and 3. + +Not one node, the network: + +- Flood-family sample, 1000 packets over 1.5h: mean path length **7.08** hops + (counting only hops ≥ `minForwarderHopHexLen`), so the last-hop rule keeps + **394 of 2789** hop observations — **14%**. **85%** of the nodes seen forwarding + in that window never appear as a last hop at all. +- `/api/scope-audit`, 24h, 205 repeaters: **133 (65%)** have zero attributable + evidence of any kind — no named scope, no undeclared scope, no unscoped packets + — so every region they declare reads "declared, not observed". Of those 133, + **110** have `relay_count_24h > 0` and **55** already carry a non-empty + `transported_scopes`, attributed by full 4-hex key rather than by the + collision-prone 1-byte prefix bucket. The evidence is in the same database. +- `ambiguousHops` is **0** on all 205 rows. The prefix-collision machinery + `ScopeAuditForwarding` documents at length has never been fed enough hops to + fire once. + +### Which cause dominates + +The same 642 `notObserved` entries (24h), split by whether the named region +appeared under that name anywhere in the window — `/api/scope-stats` named 17 +distinct regions in it: + +| | entries | | +|---|---|---| +| names a region this instance never named in the window | 287 | 45% — first cause, M2's target | +| names a region that *is* named in this window | 355 | 55% — nameability is not the blocker | +| sits on one of the 133 zero-evidence repeaters | 389 | 61% | +| …of those, naming a region that *is* nameable | 238 | 37% of all entries — second cause alone | + +The two causes overlap and neither subsumes the other; the 42%/7d figure above +reproduces as 45%/24h here. + +This is what orders the milestones, and the ordering is not about size. With the +last-hop rule in place M2 cannot be **measured**: deriving a key for `#behka` +would name the traffic in `transmissions.scope_name`, and the declaring repeater's +audit row would still say `notObserved`, because its hops were discarded before +nameability was consulted. For 65% of repeaters M2's effect on the audit would be +exactly zero, indistinguishable from M2 not working. + +### Why collisions are benign + +`code1` is an HMAC over the packet payload, so a collision between two region +names is re-rolled per packet rather than fixed per name pair. Two consequences +shape this design: + +- A region is never systematically lost to a collision, only a random fraction of + its packets. +- Targeted poisoning is not possible: an attacker cannot choose a name that + reliably collides with `#be`, because they do not control the payloads. + +The residual risk is therefore purely a rise in the random ambiguity rate, +proportional to the key-set size — which is what the cap below bounds. The rate +is `(N-1)/65536` per scoped packet: ~0.09% at 58 keys, ~0.27% at 180, and ~0.48% +at the 314 that `maxDerived: 256` permits on top of a 58-key explicit set. Only +the first of those is today's baseline; the others are what the cap buys. + +--- + +## Scope + +M0, M1, M1b and M2 below. M3 is explicitly gated on measurements taken during M2. +M4 is tracked, not built. + +M0 was added by the 2026-09-07 amendment and comes first: it is a `cmd/server/` +change of two SQL predicates, and until it lands neither M1's counter nor M2's +effect can be observed on the audit at all. + +M1b was added by a second 2026-09-07 amendment, and it changes what M2 is for. +M1 marks a declared region this instance cannot name with a row-level caveat, but +leaves its chip grey — and grey reads as "declared but not forwarding", a claim +the data does not support either. M1b resolves that question directly instead of +footnoting it. Once it lands, **M2 no longer fixes the audit**; it fixes the rest +of the product (the packets page, scope-stats, `default_scope`, the per-node +observed list), which is real but a different and less urgent kind of value than +this document originally claimed for it. + +Out of scope: regions in use but never declared over RF. Neither the config fix +nor this design can name those — both lean on the declared side. + +--- + +## Architecture + +### 0. Forwarder attribution on flood routes + +Drop the `je.key = json_array_length(o.path_json) - 1` join condition from both +`scopeConformanceQuery` (`cmd/server/scopes.go:113`) and +`scopeAuditForwarderScanQuery` (`cmd/server/scopes.go:447`). Everything else in +both queries stays exactly as it is: `route_type IN (0, 1)`, the +`LENGTH(je.value) >= minForwarderHopHexLen` floor, the explicit `json_valid` guard +against one malformed row erroring the whole query, and the prefix match against +the caller's pubkey. + +"Forwarded" then means what `transported_scopes` has always meant — appeared as a +path hop on a flood-family transmission — and the node page stops contradicting +itself. De-duplication is unaffected: `ScopeConformance` counts each transmission +once via `EXISTS`, and `ScopeAuditForwarding` already de-dupes on +`|`, which now additionally absorbs the same target appearing twice +in one path (a routing loop, or a prefix collision within a single path). + +Three consequences to carry deliberately rather than discover later: + +- **Evidence quality stops being uniform.** `path[last]` is corroborated by an + observer's own RF reception; a middle hop is attested only by the path field of a + packet somebody else forwarded onward. This is already the standard `byPathHop` + applies for `transported_scopes` and `relay_count_24h`, so no new class of trust + is introduced — but the UI should be able to say which kind of evidence a row + rests on instead of blending them silently. Cheapest honest form: carry a + `directHops` count beside the total per (target, scope) and render it as a + qualifier on the existing row, not as a second table. +- **`ambiguousHops` will start firing.** It is zero everywhere today; at ~7× the + hops it will resolve real collisions, which is exactly what that machinery is + for, but the "possibly ambiguous" chip will appear on rows that currently look + clean. That is more honest, not less — and it is also the measurement M3 was + gated on, so M3's gate becomes answerable for the first time. +- **Per-hop collision exposure is unchanged** — same 4-hex floor, same prefix + match; only the number of matched hops grows. `observations.resolved_path` + carries full pubkeys for ~90% of hops on the flood rows that have it, so a later + refinement can attribute those exactly and fall back to the prefix match for the + rest. Deliberately out of scope for M0: a strictly better attribution layered on + top of a correct one, not part of making it correct. + +Cost: the SQL keeps its shape. `json_each` already expands the entire array, and +the removed predicate was a filter on that expansion rather than an index lookup. +The audit's Go attribution loop grows with hop count (mean 7.08 on this network) +against the in-memory prefix index, which is what `scopeAuditPrefixIndex` exists +for. `ScopeConformance`'s `EXISTS` can only get cheaper — it may short-circuit on +the first matching hop instead of computing an array length per observation. + +### 1. `regionKeySet` — the key registry + +New file `cmd/ingestor/region_keys.go`. `loadRegionKeys` currently returns a flat +`map[string][]byte` built once at `main.go:111` and threaded through six call +sites. It becomes a two-tier, refreshable type: + +```go +type regionKeys struct { // immutable snapshot + explicit map[string][]byte // from hashRegions — always trusted + derived map[string][]byte // from node_declared_regions +} + +type regionKeySet struct { // live, refreshable + cur atomic.Pointer[regionKeys] +} +``` + +The ingest hot path reads via `set.snapshot()` — one atomic load, no lock. A +refresh builds the replacement map off to the side and swaps the pointer. This is +AGENTS.md rule 0: no expensive work under a lock in the ingest path. + +**Refresh** runs at startup and on a ticker. The query is one +`ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC)` over +`node_declared_regions`, covered by `idx_ndr_target`. At 205 targets it is +negligible. The ingestor already has the per-target `CurrentDeclaredRegions` +(`client_reception.go:552`); this adds the bulk variant beside it, mirroring +`AllCurrentDeclaredRegions` in `cmd/server/scopes.go`. + +**Configuration** — a new top-level block, default **off**: + +```json +"autoRegionKeys": { "enabled": false, "maxDerived": 256, "refreshMinutes": 15 } +``` + +Opt-in matches the existing `clientRxObservations` / `clientRfSamples` / +`clientRegions` flags. Note the codebase-wide gotcha those flags document: config +loading is plain `json.Unmarshal` with no `DisallowUnknownFields`, so a +mis-nested key is silently ignored. `autoRegionKeys` is **top-level**, a sibling +of `hashRegions`, not nested inside it. + +With `enabled: false` the derived tier stays empty and behaviour is byte-for-byte +what it is today. + +**Candidate filter.** A declared name is rejected when it is empty or longer than +32 characters, contains a non-printable character, a comma (the `regions_csv` +delimiter), or a `#` (the firmware strips it, so its presence signals a malformed +entry). Names already in `explicit` are skipped rather than duplicated. + +There is deliberately **no blocklist on name values**. The declared set contains +entries that look like junk (`null`, `bierhuis`, `sol3`), but a rule filtering on +string content is unmaintainable, and the cost of one is a single slot out of 256 +plus a 1-in-65536 collision chance. + +**Ranking when over `maxDerived`:** by number of distinct repeaters declaring the +name (descending), then most recent `observed_at`, then name. The long tail of +one-off names is dropped first; `#be` (declared by 127 repeaters) never is. Fully +deterministic, therefore testable. + +Every refresh logs the per-tier totals, names added, and names dropped. + +### 2. Matching with evidence + +`matchScope` returns a bare string today, discarding the difference between "no +key matched" and "several matched". It becomes: + +```go +type scopeMatch struct { + Name string // "" = unresolved + Reason scopeReason // unique | explicitOverDerived | pathDeclared | ambiguous | none + Candidates []string // populated once more than one key matches +} +``` + +Resolution tiers: + +1. Exactly one match → `unique`. +2. More than one, exactly one of them in `explicit` → that one, + `explicitOverDerived`. Operator intent beats RF hearsay, and this covers the + majority of the ambiguity this change itself introduces. +3. *(M3, gated)* Still tied, and exactly one candidate is declared by a node in + the packet's path → that one, `pathDeclared`. +4. Otherwise → `""`, `ambiguous` — the current behaviour. + +`transmissions.scope_name` keeps its existing three-state encoding. **No schema +migration and no new column:** `Reason` goes to logs and an in-process counter. +Naming a packet wrongly is worse than not naming it, and tier 4 preserves that. + +### 3. `scope-repair` must use the same key set + +`runScopeRepair` (`cmd/ingestor/scope_repair.go:274`) builds its keys with +`loadRegionKeys(cfg)` — explicit only. Left alone, a repair run would re-derive +every automatically-named row as unmatched and write `""` back over it. It must +build the same two-tier `regionKeySet`, including a derived-tier refresh, before +scanning. This is a data-loss bug if missed, not a detail. + +### 3b. Declared-region verification (independent of 1–3) + +M1's caveat says "some of this row's grey chips may be unnameable rather than +unforwarded". That is honest but weak: the question it declines to answer is +answerable, per chip, from data already in the database. + +For a repeater R declaring region X, with unmatched packets it was observed +forwarding: derive `SHA256("#X")[:16]`, HMAC each of those packets' payloads with +it, and compare to the packet's stored `code1` — exactly the computation +`matchingRegions` performs, but with the candidate set narrowed to R's own +declarations rather than every key this instance holds. + +Three properties make this a better instrument for *this* question than the +global derivation in section 1: + + - **Better signal-to-noise.** Section 1 tests ~180 hypotheses against every + packet, of which at most one is related to it; that is why it needs a cap, a + ranking and a tie-break to contain the noise it creates. Here the ~9 + hypotheses per repeater are each independently supported before the test runs: + the repeater says it forwards these regions. + - **Corroboration is available, and it is not available to section 1.** The + ingest-time path names each packet in isolation: one packet, one decision, a + 1-in-65536 chance of a coincidental match. Verification looks at a set. If + two or more of R's unmatched packets derive to X, the odds of coincidence are + (1/65536)² or better. **A chip turns green on ≥2 corroborating packets; + exactly one leaves it grey with its own tooltip**, because a single match is + not evidence. + - **Nothing is written.** This is a read-time inference in `cmd/server/`, so a + wrong answer expires with the window rather than persisting in + `transmissions.scope_name` until someone runs `scope-repair`. It also keeps + the read/write invariant intact without argument. + +**Query shape matters here.** `scopeAuditForwarderScanQuery` returns one row per +hop per flood packet — 19,049 rows in a 2,000-packet sample after M0 — so widening +it to carry `raw_hex` would load the hot scan for nothing. Verification takes a +**second, narrow query** over only the unmatched transmissions in the window +(~400 over 7 days), fetching `id` and `raw_hex`, decoding each once and reusing the +payload across every candidate name. The main scan is untouched. + +What survives of M1's chip: after M1b the row-level caveat fires only for +unmatched traffic matching **none** of the repeater's declared names. That is +rarer and more interesting than what it reports today — a repeater forwarding a +region it does not declare *and* that this instance cannot name. + +### 4. Scope-audit honesty (independent of 1–3) + +Regions stay unnameable even with derivation enabled: above the cap, or never +declared anywhere. The audit must be able to say so. + +`ScopeAuditForwarding` currently discards unmatched rows with a bare `continue` +(`cmd/server/scopes.go:530`). That becomes an `unmatchedPackets` counter on +`scopeAuditTargetAgg`, surfaced as a field on `ScopeAuditRow`, rendered as a +caveat chip in `public/scope-audit.js` in the same idiom as the existing +`possibly ambiguous` chip (`ambiguousCaveat`). + +No config coupling and no ingestor change: the server does not read +`hashRegions`, and adding that coupling would mislabel every region on any +deployment where the two binaries do not share a `config.json`. + +--- + +## Milestones + +### M0 — Forwarder attribution (gates M1's counter and M2's measurement) + +`cmd/server/` plus one frontend tooltip. The smallest change in this document and +the one with the largest effect on what the audit reports. + +- remove the last-hop join condition from both queries (Architecture 0) +- update the doc comments that currently present the last-hop rule as deliberate: + `RouteTypeMix` (`cmd/server/scopes.go:35`), `scopeConformanceQuery`, + `scopeAuditForwarderScanQuery`. `RouteTypeMix`'s "direct/transportDirect are + always zero by construction" stays **true** (the route filter is untouched), but + the reason it gives is stated in terms of `path[last]` and must be restated in + terms of the route filter +- same restatement for `routesHtml`'s tooltip (`public/node-scopes.js:132`), which + repeats the `path[last]` reasoning to the reader +- Tests (`cmd/server/scopes_test.go`): a flood transmission whose path carries the + target in the **middle** is now attributed; a DIRECT transmission whose + `path[last]` **is** the target is still not attributed — the existing guarantee, + and after M0 the only thing standing between the audit and misattribution, so it + gets an explicit test rather than relying on the route filter being obvious; a + 2-hex hop is still ignored; a target appearing twice in one path counts once +- afterwards, **re-measure** the first-cause share and record the new number here. + M2's sizing depends on what remains once attribution is fixed, not on the 45% + measured through the last-hop rule + +#### What widening attribution costs, measured on staging 2026-09-07 + +Reading every hop instead of `path[last]` multiplies the rows the scan returns. +On a live-shaped database (206 declared repeaters, 965k transmissions) the 7d +window returns **3,470,188 hop rows** from 1,368,761 observations carrying a +path, and `GET /api/scope-audit?window=7d` took **16.7s** cold, against 4.0s for +24h and 0.15s for 1h. SQLite accounts for 2.7s of that; the rest was the Go side +reading rows. + +Three SQL-side reductions were measured on that database and all were rejected, +because each costs more than it saves: + +| approach | rows returned | time in SQLite | +|---|---|---| +| the query as written | 3,470,188 | 2.7s | +| pre-filter on the declared targets' first 4 hex chars | 1,971,126 | 20.9s | +| `GROUP BY t.id, hop` | 965,025 | 38.0s | +| `SELECT DISTINCT t.id, path_json` | 1,229,966 | 17.7s | + +The query plan is already index-driven (`idx_transmissions_first_seen`, then +`idx_observations_tx_ts`), so there is no missing index behind this: the rows are +inherent to the data. Note for anyone attempting a hop comparison in SQL: +**80% of stored hops are uppercase** (1,026,814 of 1,284,897 in a 24h window) +because `packetpath.DecodePathFromRawHex` writes them that way, while targets are +lowercase. A case-sensitive comparison silently drops most attributable hops. + +What did work was taking the per-transmission columns out of the per-hop rows +(`942761c4`) and not recomputing the same window concurrently or every 30s +(`b7515cec`). Measured after both, warm process: **24h 2.79-2.89s across six +samples** (from 4.04s) and **7d 11.6s** (from 16.7s), with repeat requests inside +the TTL served in ~1ms. + +### M1 — Scope-audit honesty + +`cmd/server/` and `public/` only. Ships value on its own and reviews independently. +Sequenced after M0: the `unmatchedPackets` counter below counts unmatched rows +*among the rows attribution admits*, so before M0 it would read zero for the same +65% of repeaters and invite the same wrong conclusion in a new field. + +- `scopeAuditTargetAgg.unmatchedPackets`, counted where the `continue` is today +- `ScopeAuditRow.ObservedUnmatchedPackets`, documented in `docs/api-spec.md` +- caveat chip in `scope-audit.js`, exposed through + `window.__meshcoreScopeAuditInternals` so it can be asserted +- Tests: `cmd/server/scopes_test.go` (counter and field), `test-frontend-helpers.js` + (chip renders only when non-zero) + +### M1b — Declared-region verification + +`cmd/server/` and `public/` only, like M1. No config, no schema change, nothing +written. See Architecture 3b. + +- second query over the window's unmatched transmissions (`id`, `raw_hex`), + separate from the main hop scan so that scan stays as it is +- per repeater, derive a key for each of its declared regions and test it against + its own unmatched packets, decoding each packet once +- a declared region with **≥2** corroborating packets is observed; with exactly + one it stays grey and its tooltip says why one match is not evidence +- the chip carries how it was established, so a reader can tell a region named + from a configured key apart from one verified against the repeater's own + declaration +- narrow M1's row caveat to unmatched traffic matching none of the declared names +- Tests: two corroborating packets turn a chip green; one does not; a packet + matching no declared name still feeds the narrowed caveat; a repeater with no + unmatched traffic is unaffected +- Perf: bound the work at (unmatched transmissions in window × declared names per + repeater), deduplicated per transmission, and benchmark it — AGENTS.md rule 0. + The 30s audit cache already absorbs the cost, but the bound is what stops a + future window widening turning it into a hot path + +### M2 — Auto-derived region keys + +**Re-scoped by the M1b amendment.** This no longer fixes the audit — M1b does. +What is left is the rest of the product still not seeing these regions: +`/api/packets` shows an empty scope on a packet that has one, `/api/scope-stats` +omits whole regions from `byRegion`, `nodes.default_scope` is never set for them, +and the per-node observed list is short. Worth doing, lower urgency, and it should +be sized against what M1b leaves rather than against the original 42%. + +- `region_keys.go`: `regionKeySet`, `regionKeys`, atomic snapshot, two tiers +- bulk declared-regions read in the ingestor +- refresh at startup plus ticker; cap, filter, ranking, logging +- `autoRegionKeys` config block, default off; `config.example.json` entry with the + `_comment_autoRegionKeys` explainer the file's convention expects +- `matchScope` → `scopeMatch`, tiers 1/2/4; six call sites updated +- `scope-repair` builds the same key set +- `docs/api-spec.md` and `docs/client-regions.md` updated +- Tests: filter and ranking determinism, cap enforcement, tier 1/2/4 resolution, + disabled-by-default equivalence with today's behaviour, `scope-repair` no longer + unnames derived rows +- **Benchmark** for the N-HMAC path. AGENTS.md rule 0 requires proof for perf + claims, and this grows the key set by up to `maxDerived` (256 by default). + The instance's current explicit key count is not recorded here — it is + operator config — so the benchmark must sweep key-set size rather than assert + a single before/after. Note that the work is inherently + O(keys) per transport-scoped packet and **cannot be indexed** — the code depends + on the payload, so the "pre-indexed lookup table" suggested in `matchScope`'s + comment is not achievable. Measured against current traffic (~0.04 transport + packets/s) the cost is negligible, but it is linear, which is the reason the cap + exists. + +### M3 — Path-evidence tie-break (tier 3) — gated + +At ~180 keys (today's explicit set plus the 123 names currently declared) the +ambiguous share is ~0.27% of scoped packets, roughly 60 packets a week on this +network, and tier 2 absorbs most of it because the majority of +collisions will be explicit-against-derived. What is left for tier 3 may be ten +packets a week, against the cost of a prefix index over every declared target plus +plumbing path evidence into the decode path. + +**Build only if M2's `ambiguous` logging shows the volume justifies it.** The +design remains the tiered one; the last tier is built on measurement rather than +expectation. + +#### What the first run on staging measured (2026-09-07) + +The gate is **still open, and the first tally points at closing it**. After 15 +minutes on staging: + +``` +[regions] scope matches: unique=1306 explicit-over-derived=0 ambiguous=0 none=0 +``` + +Zero ambiguous in 1306 scoped packets. That is one ticker interval, not the day the +gate asks for, and it is not identically zero either: the same container logged nine +`ambiguous collision between [#lu #be]` lines in the seconds after an MQTT reconnect, +so the rate is low rather than absent. The full reading has to come from a container +left running, because each deploy replaces it and takes `docker logs` along. + +What the same run did settle is the size of the derived tier on this network, and it +is not what the estimate above assumes: + +``` +[regions] derived-key refresh: 124 name(s) declared, 124 kept after filter+cap(256), 160 total key(s) in force +[regions] derived keys now active: [#null] +``` + +159 explicit keys plus **one** derived. The 123 other declared names had already been +merged into the live `hashRegions` by hand that morning (58 keys to 159), so the +derived tier had nothing left to add but the one name no operator would type. +`scope-repair --dry-run` with the feature on: 0 rows newly named, 2 corrected. + +`#null` is a repeater that declares a region literally named `null` +(`95f8e61c…`). `regionNameAcceptable` accepts it deliberately — the rules are +structural, and its own comment names `null` as an example of a junk-looking name +that costs one slot out of `maxDerived` and one 1-in-65536 collision chance. The +measurement confirms the rule behaves as designed; it is not a filter gap. + +The consequence for sizing: at ~160 keys rather than the ~180 assumed above, and with +tier 2 absorbing explicit-against-derived collisions, the ambiguous share this network +can produce is smaller than the estimate that gates M3. That makes the reading more +likely to close the gate than to open it, which is a reason to take it rather than +skip it. + +If built: candidate X wins when at least one resolvable path hop declares X and no +resolvable hop declares a competing candidate. Hops shorter than +`minForwarderHopHexLen` (4 hex chars) are ignored, matching the server's floor; a +hop whose truncated prefix resolves to several targets contributes the union of +their declared sets, so an inconclusive union abstains rather than guesses. + +### M4 — Customizer exposure + +AGENTS.md rule 8: `maxDerived` and `refreshMinutes` belong in the customizer. +Tracked, not built here. + +--- + +## Operational note + +Independent of this work, the immediate remedy for the live instance is to merge +the declared region names into `hashRegions`, restart the ingestor, and run +`ingestor scope-repair` (dry run first). Note what that remedy does **not** do: +it names traffic, and the audit still attributes none of it to the 133 repeaters +it cannot see, so the `notObserved` list will shrink far less than the key count +suggests until M0 lands. `scope-repair` applies only +`"" → name` and `name → ""` where several keys now match; any other transition is +reported as `UNEXPECTED` and left unwritten. `-apply` requires stopping the +ingestor first: every UPDATE runs in one transaction and `busy_timeout` is 5s, so +a live ingestor would hit `SQLITE_BUSY`. diff --git a/public/node-scopes.js b/public/node-scopes.js index d71ce095b..2742d7e96 100644 --- a/public/node-scopes.js +++ b/public/node-scopes.js @@ -130,7 +130,7 @@ } function routesHtml(routes) { - return '
' + + return '
' + 'Route mix (forwarded): transportFlood ' + routes.transportFlood + ' · flood ' + routes.flood + ' · direct ' + routes.direct + ' · transportDirect ' + routes.transportDirect + '' + '
'; } diff --git a/public/scope-audit.css b/public/scope-audit.css index f5074e8e2..6944d944c 100644 --- a/public/scope-audit.css +++ b/public/scope-audit.css @@ -38,6 +38,14 @@ .sa-chip-undeclared { background: color-mix(in srgb, var(--status-yellow) 18%, transparent); color: var(--status-amber-text); } .sa-chip-wildcard { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); font-weight: 700; } .sa-chip-ambiguous { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); border: 1px dashed var(--border); font-family: inherit; font-style: italic; } +/* Same muted, dashed treatment as .sa-chip-ambiguous on purpose: both are + caveats on the row's finding, not findings themselves, and neither may + compete visually with the red/green scope chips beside them. */ +.sa-chip-unmatched { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); border: 1px dashed var(--border); font-family: inherit; font-style: italic; } +/* Verified-by-declaration: the same green as any observed chip, because the + region IS observed. The dotted underline says how that was established + without introducing a third colour into a column that already carries two. */ +.sa-chip-verified { text-decoration: underline dotted; text-underline-offset: 2px; } .sa-count { font-size: 11px; margin-top: 6px; } diff --git a/public/scope-audit.js b/public/scope-audit.js index 7aa2017ac..6496cefb4 100644 --- a/public/scope-audit.js +++ b/public/scope-audit.js @@ -93,12 +93,31 @@ function mergedScopeChips(row) { var missing = Object.create(null); row.notObserved.forEach(function (n) { missing[n] = true; }); + var evidence = row.regionEvidence || {}; var chips = row.declaredRegions.map(function (n) { var observed = !missing[n]; - return '' + escapeHtml(n) + ''; + var hits = evidence[n] || 0; + // A green chip with evidence was established by verifying the repeater's + // own declaration against its own unnameable traffic, not by matching a + // configured region key. Same colour — it is observed either way — with a + // dotted underline, so the reader can tell the two apart without a third + // colour competing for attention in a column that already carries two. + var verified = observed && hits > 0; + var cls = 'sa-chip ' + (observed ? 'sa-chip-observed' : 'sa-chip-unobserved') + (verified ? ' sa-chip-verified' : ''); + var title; + if (verified) { + title = n + ': observed — ' + hits + ' forwarded packet' + (hits === 1 ? '' : 's') + + ' in this window derive to this region, verified against the repeater’s own declared list. ' + + 'This instance holds no hashRegions key for it, so it could not be named directly.'; + } else if (observed) { + title = n + ': observed forwarding in this window'; + } else if (hits === 1) { + title = n + ': declared, and exactly one forwarded packet derives to it — that is one match in 65536 by chance alone, ' + + 'so it is not treated as evidence. Two would be.'; + } else { + title = n + ': declared, but no forwarding observed in this window'; + } + return '' + escapeHtml(n) + ''; }); if (!chips.length) return '—'; return chips.join(' '); @@ -183,6 +202,38 @@ ' in this window matched more than one declared target\'s pubkey prefix and could not be attributed to any of them. Any “not observed” entry on this row may be explained by that prefix collision rather than a real gap.">possibly ambiguous'; } + // unmatchedCaveat flags rows where this instance saw the repeater forward + // transport-scoped traffic it holds no region key for. The ingestor stores + // those packets with an empty scope_name (see scopeNameForDB), so they name + // no region and can never satisfy a declared one — a repeater forwarding a + // region this instance cannot name looks exactly like one forwarding + // nothing. + // + // Distinct from ambiguousCaveat, and the distinction is the whole point: + // that one is a prefix collision between two repeaters and is nobody's + // fault, this one is a missing entry in this instance's own hashRegions and + // the reader can fix it. Saying so is what stops them investigating an + // innocent repeater. + function unmatchedCaveat(row) { + var n = row.observedUnmatchedPackets; + if (!n) return ''; + // Traffic already accounted for by verification is explained, not + // mysterious. What is left over is the interesting case: this repeater + // forwards a region it does NOT declare and that this instance also cannot + // name. Reporting the full count here would re-raise a question the Scopes + // column has just answered. + var explained = 0; + var evidence = row.regionEvidence || {}; + Object.keys(evidence).forEach(function (k) { explained += evidence[k]; }); + var left = n - explained; + if (left <= 0) return ''; + var label = escapeHtml(left) + ' forwarded packet' + (left === 1 ? '' : 's'); + return ' ' + + label + ' unexplained'; + } + // statusScore ranks a row's Status column numerically for sorting — a // simple weighted count (notObserved dominates, matching the server's own // findings-first ranking) rather than the badge text, which the Status @@ -210,7 +261,7 @@ '' + nameHtml(row) + (row.role != null && row.role !== '' ? ' ' + escapeHtml(row.role) + '' : '') + '' + '' + issuesHtml + '' + '' + configStateHtml(row) + '' + - '' + mergedScopeChips(row) + (row.declaredWildcard ? ' *' : '') + ambiguousCaveat(row) + '' + + '' + mergedScopeChips(row) + (row.declaredWildcard ? ' *' : '') + ambiguousCaveat(row) + unmatchedCaveat(row) + '' + '' + ageHtml(row) + (row.truncated ? ' truncated' : '') + '' + ''; } @@ -360,7 +411,7 @@ // Exposed so the helper tests can assert what the Scopes column RENDERS // rather than grepping this file, the same reason map.js exposes its label // builder (#1356/#1933). - window.__meshcoreScopeAuditInternals = { mergedScopeChips: mergedScopeChips, emptyStateHtml: emptyStateHtml, sourcesLineHtml: sourcesLineHtml }; + window.__meshcoreScopeAuditInternals = { mergedScopeChips: mergedScopeChips, emptyStateHtml: emptyStateHtml, sourcesLineHtml: sourcesLineHtml, unmatchedCaveat: unmatchedCaveat }; } registerPage('scope-audit', { init: init, destroy: destroy }); diff --git a/test-frontend-helpers.js b/test-frontend-helpers.js index a854603e8..180180523 100644 --- a/test-frontend-helpers.js +++ b/test-frontend-helpers.js @@ -7028,6 +7028,31 @@ console.log('\n=== scope-audit.js: mergedScopeChips ==='); assert.ok(h.includes('<img')); }); + test('a region verified against the repeater own declaration is green, and says so', () => { + const h = chips({ declaredRegions: ['fm-112'], notObserved: [], regionEvidence: { 'fm-112': 23 } }); + assert.ok(h.includes('sa-chip-observed'), 'still green — it is observed'); + assert.ok(h.includes('sa-chip-verified'), 'but marked as established differently'); + assert.ok(h.includes('23'), 'the tooltip states how much evidence there is'); + }); + + test('a region observed by name carries no verified marker', () => { + const h = chips({ declaredRegions: ['be'], notObserved: [], regionEvidence: {} }); + assert.ok(h.includes('sa-chip-observed')); + assert.ok(!h.includes('sa-chip-verified'), 'a normally-named region is not a verification'); + }); + + test('a single-hit region stays grey and its tooltip explains why', () => { + const h = chips({ declaredRegions: ['fm-112'], notObserved: ['fm-112'], regionEvidence: { 'fm-112': 1 } }); + assert.ok(h.includes('sa-chip-unobserved'), 'one hit is not enough to turn it green'); + assert.ok(/one match/i.test(h), 'must say why one hit was not accepted'); + }); + + test('a missing regionEvidence field renders as before (older server)', () => { + const h = chips({ declaredRegions: ['be'], notObserved: ['be'] }); + assert.ok(h.includes('sa-chip-unobserved')); + assert.ok(!h.includes('sa-chip-verified')); + }); + test('a notObserved entry that is not declared cannot invent a chip', () => { // Defensive: the server guarantees notObserved is a subset (197 of 197 // rows checked), but the column must not grow a phantom chip if that ever @@ -7038,6 +7063,66 @@ console.log('\n=== scope-audit.js: mergedScopeChips ==='); }); } +// ===== scope-audit.js: unmatchedCaveat ===== +// A declared region this instance holds no hashRegions key for can never turn +// green, however much traffic the repeater forwards: the ingestor stores such +// packets with an empty scope_name, so there is no name for the audit to match +// the declaration against. On live data that explains a large share of all +// notObserved entries, so the column must be able to say so instead of +// presenting every grey chip as a confirmed gap. +console.log('\n=== scope-audit.js: unmatchedCaveat ==='); +{ + const ctx = makeSandbox(); + ctx.registerPage = () => {}; + loadInCtx(ctx, 'public/app.js'); + loadInCtx(ctx, 'public/scope-audit.js'); + const caveat = ctx.__meshcoreScopeAuditInternals.unmatchedCaveat; + + test('zero unmatched packets renders nothing at all', () => { + assert.strictEqual(caveat({ observedUnmatchedPackets: 0 }), ''); + }); + + test('a missing field renders nothing (older server, field absent)', () => { + assert.strictEqual(caveat({}), ''); + }); + + test('a non-zero unexplained count renders a chip carrying the number', () => { + const h = caveat({ observedUnmatchedPackets: 148 }); + assert.ok(h.includes('sa-chip-unmatched'), 'should carry its own class'); + assert.ok(h.includes('148'), 'with no evidence to subtract, the whole count is unexplained'); + assert.ok(h.includes('unexplained'), 'the word changed with the meaning'); + }); + + test('singular and plural are both grammatical', () => { + assert.ok(caveat({ observedUnmatchedPackets: 1 }).includes('1 forwarded packet ')); + assert.ok(caveat({ observedUnmatchedPackets: 2 }).includes('2 forwarded packets ')); + }); + + test('the title says what unexplained traffic implies', () => { + // The cause is no longer only a hashRegions gap: after verification, what + // is left over is traffic for a region the repeater does not declare. + const h = caveat({ observedUnmatchedPackets: 5 }); + assert.ok(/does not declare/i.test(h), 'must state the sharper conclusion'); + }); + + test('traffic fully explained by verification raises no caveat', () => { + assert.strictEqual(caveat({ observedUnmatchedPackets: 23, regionEvidence: { 'fm-112': 23 } }), ''); + }); + + test('only the unexplained remainder is reported', () => { + const h = caveat({ observedUnmatchedPackets: 30, regionEvidence: { 'fm-112': 23 } }); + assert.ok(h.includes('7 forwarded packets '), 'want the remainder, not the total'); + }); + + test('the count is not injected raw into markup', () => { + // observedUnmatchedPackets is server-supplied. It is a number in every + // sane response, but the chip must not become an injection point if that + // ever stops holding. + const h = caveat({ observedUnmatchedPackets: '1">' }); + assert.ok(!h.includes('