From 28ad67b85bb424a8756d767109bec3844e2a382b Mon Sep 17 00:00:00 2001 From: Florent Tapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:27:47 +0200 Subject: [PATCH] fix: add stricter companion series to benches 004 and 008 Both benches score a proxy for the thing they claim to measure. Bench 004 scores the logo field as "non-empty string", which any provider wins by construction by rewriting logos onto its own CDN at a deterministic path. The current beneficiary is Mobula, at 100% logo on every chain against 22.9/37.8/78.9 for a provider returning upstream URLs. Adds logo_resolved: HEAD the URL, fall back to a ranged GET on 403/405, cache 6h so we do not hammer third-party CDNs. Bench 008 scores a hit as "any non-generic name", which cannot tell a curated entity from a personal name-service record. Measured: 25% of Serialized's hits and 25.4% of Mobula's named something other than the curated entity. The anchor list already carries a Hint for every address and the scoring path ignored it. Adds wallet_labels_accurate_total scored against that Hint. Both land as SEPARATE series, not as redefinitions, so the published leaderboards and their history stay intact while the stricter numbers build up alongside. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CpArutAtXuBb1BVNUDXoYA --- .../cmd/script/logo_resolve.go | 150 ++++++++++++++++++ .../cmd/script/metadata_coverage_monitor.go | 4 + .../wallet-labels/cmd/script/accuracy.go | 117 ++++++++++++++ .../wallet-labels/cmd/script/accuracy_test.go | 38 +++++ .../wallet-labels/cmd/script/anchor_feeder.go | 1 + harnesses/wallet-labels/cmd/script/metrics.go | 28 ++++ harnesses/wallet-labels/cmd/script/monitor.go | 7 + 7 files changed, 345 insertions(+) create mode 100644 harnesses/metadata-coverage/cmd/script/logo_resolve.go create mode 100644 harnesses/wallet-labels/cmd/script/accuracy.go create mode 100644 harnesses/wallet-labels/cmd/script/accuracy_test.go diff --git a/harnesses/metadata-coverage/cmd/script/logo_resolve.go b/harnesses/metadata-coverage/cmd/script/logo_resolve.go new file mode 100644 index 000000000..61a0065bb --- /dev/null +++ b/harnesses/metadata-coverage/cmd/script/logo_resolve.go @@ -0,0 +1,150 @@ +package main + +import ( + "context" + "net/http" + "strings" + "sync" + "time" +) + +// The bench scores the logo field as "did the provider return a non-empty +// string". That is not the same question as "does this token have a logo", +// and the difference is not academic: Mobula rewrites every logo onto +// metadata.mobula.io at a deterministic path derived from chain and +// address, so its logo field is non-empty for every token by construction, +// whether or not an image exists behind the URL. Providers that return the +// upstream source URL (ipfs.io, cdn.dexscreener.com, launchpad CDNs, twimg) +// are scored on whether the upstream actually has the asset. +// +// Measured when this was written: Mobula 100% logo on all three chains, +// against 22.9% / 37.8% / 78.9% for a provider returning upstream URLs. A +// HEAD sweep of 12 distinct Mobula logo URLs resolved 11 and 404'd 1. +// +// Any provider can win the current rule by rewriting to its own CDN, and +// the current beneficiary is our own product, so this is published as a +// SEPARATE `logo_resolved` field rather than silently redefining `logo`. +// The existing series and its history stay intact; the stricter one builds +// alongside until there is enough of it to move the headline in a +// documented change. + +var logoHTTPClient = &http.Client{ + Timeout: 4 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 4 { + return http.ErrUseLastResponse + } + return nil + }, +} + +type logoCacheEntry struct { + ok bool + at time.Time +} + +var ( + logoCacheMu sync.Mutex + logoCache = map[string]logoCacheEntry{} +) + +const ( + logoCacheTTL = 6 * time.Hour + logoCacheMaxSize = 20000 +) + +// logoResolves reports whether the URL actually serves an image. Empty +// URLs are false without a request. Results are cached because launchpad +// and CDN URLs repeat heavily across fresh tokens and we should not hammer +// third-party hosts from a monitor. +func logoResolves(rawURL string) bool { + u := strings.TrimSpace(rawURL) + if u == "" { + return false + } + if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") { + // data: URIs and relative paths are not verifiable from here. + // Count them as unresolved rather than silently passing. + return false + } + + logoCacheMu.Lock() + if e, ok := logoCache[u]; ok && time.Since(e.at) < logoCacheTTL { + logoCacheMu.Unlock() + return e.ok + } + logoCacheMu.Unlock() + + ok := probeLogo(u) + + logoCacheMu.Lock() + if len(logoCache) >= logoCacheMaxSize { + logoCache = map[string]logoCacheEntry{} + } + logoCache[u] = logoCacheEntry{ok: ok, at: time.Now()} + logoCacheMu.Unlock() + return ok +} + +func probeLogo(u string) bool { + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil) + if err != nil { + return false + } + req.Header.Set("User-Agent", "OpenChainBench-logo-probe/1.0 (+https://openchainbench.com)") + resp, err := logoHTTPClient.Do(req) + if err == nil { + defer resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return imageish(resp.Header.Get("Content-Type")) + } + // A number of CDNs reject HEAD with 403/405 while serving GET + // fine. Retry those with a 1-byte ranged GET rather than + // recording a false negative. + if resp.StatusCode != http.StatusMethodNotAllowed && resp.StatusCode != http.StatusForbidden { + return false + } + } + + ctx2, cancel2 := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel2() + req2, err := http.NewRequestWithContext(ctx2, http.MethodGet, u, nil) + if err != nil { + return false + } + req2.Header.Set("User-Agent", "OpenChainBench-logo-probe/1.0 (+https://openchainbench.com)") + req2.Header.Set("Range", "bytes=0-0") + resp2, err := logoHTTPClient.Do(req2) + if err != nil { + return false + } + defer resp2.Body.Close() + if resp2.StatusCode >= 200 && resp2.StatusCode < 300 { + return imageish(resp2.Header.Get("Content-Type")) + } + return false +} + +// imageish accepts anything that plausibly renders in an . An empty +// Content-Type is accepted because several IPFS gateways omit it on +// ranged responses; a hard reject there would penalise providers that +// return honest upstream URLs, which is the opposite of the point. +func imageish(ct string) bool { + c := strings.ToLower(strings.TrimSpace(ct)) + if c == "" { + return true + } + if i := strings.IndexByte(c, ';'); i >= 0 { + c = strings.TrimSpace(c[:i]) + } + switch { + case strings.HasPrefix(c, "image/"): + return true + case c == "binary/octet-stream", c == "application/octet-stream": + return true + } + return false +} diff --git a/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go b/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go index 57e555301..d17018169 100644 --- a/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go +++ b/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go @@ -661,6 +661,7 @@ func checkTokenMetadata(token TokenToCheck, config *Config) { // Record Prometheus metrics for Mobula RecordMetadataCoverage("mobula", chainName, "logo", mobulaResult.HasLogo, config.MonitorRegion) + RecordMetadataCoverage("mobula", chainName, "logo_resolved", mobulaResult.HasLogo && logoResolves(mobulaResult.LogoURL), config.MonitorRegion) RecordMetadataCoverage("mobula", chainName, "description", mobulaResult.HasDescription, config.MonitorRegion) RecordMetadataCoverage("mobula", chainName, "twitter", mobulaResult.HasTwitter, config.MonitorRegion) RecordMetadataCoverage("mobula", chainName, "website", mobulaResult.HasWebsite, config.MonitorRegion) @@ -676,6 +677,7 @@ func checkTokenMetadata(token TokenToCheck, config *Config) { // Record Prometheus metrics for Codex RecordMetadataCoverage("codex", chainName, "logo", codexResult.HasLogo, config.MonitorRegion) + RecordMetadataCoverage("codex", chainName, "logo_resolved", codexResult.HasLogo && logoResolves(codexResult.LogoURL), config.MonitorRegion) RecordMetadataCoverage("codex", chainName, "description", codexResult.HasDescription, config.MonitorRegion) RecordMetadataCoverage("codex", chainName, "twitter", codexResult.HasTwitter, config.MonitorRegion) RecordMetadataCoverage("codex", chainName, "website", codexResult.HasWebsite, config.MonitorRegion) @@ -689,6 +691,7 @@ func checkTokenMetadata(token TokenToCheck, config *Config) { // Record Prometheus metrics for Jupiter RecordMetadataCoverage("jupiter", chainName, "logo", jupiterResult.HasLogo, config.MonitorRegion) + RecordMetadataCoverage("jupiter", chainName, "logo_resolved", jupiterResult.HasLogo && logoResolves(jupiterResult.LogoURL), config.MonitorRegion) RecordMetadataCoverage("jupiter", chainName, "description", jupiterResult.HasDescription, config.MonitorRegion) RecordMetadataCoverage("jupiter", chainName, "twitter", jupiterResult.HasTwitter, config.MonitorRegion) RecordMetadataCoverage("jupiter", chainName, "website", jupiterResult.HasWebsite, config.MonitorRegion) @@ -706,6 +709,7 @@ func checkTokenMetadata(token TokenToCheck, config *Config) { updateStats("serialized", serializedResult) RecordMetadataCoverage("serialized", chainName, "logo", serializedResult.HasLogo, config.MonitorRegion) + RecordMetadataCoverage("serialized", chainName, "logo_resolved", serializedResult.HasLogo && logoResolves(serializedResult.LogoURL), config.MonitorRegion) RecordMetadataCoverage("serialized", chainName, "description", serializedResult.HasDescription, config.MonitorRegion) RecordMetadataCoverage("serialized", chainName, "twitter", serializedResult.HasTwitter, config.MonitorRegion) RecordMetadataCoverage("serialized", chainName, "website", serializedResult.HasWebsite, config.MonitorRegion) diff --git a/harnesses/wallet-labels/cmd/script/accuracy.go b/harnesses/wallet-labels/cmd/script/accuracy.go new file mode 100644 index 000000000..c785b8dfa --- /dev/null +++ b/harnesses/wallet-labels/cmd/script/accuracy.go @@ -0,0 +1,117 @@ +package main + +import "strings" + +// The bench scores a "hit" whenever a provider returns any non-generic +// name. That rule cannot tell a curated entity label from a personal +// name-service record: a provider that resolves `dex.davywoodfi.eth` +// against Permit2, or `bonklanatoken.sol` against the Raydium authority, +// scores exactly like one that answers "Permit2" and "Raydium". +// +// Measured at the time this was written, on the 100 anchors covered by +// the two API-key providers: 25% of Serialized's hits and 25.4% of +// Mobula's named something other than the curated entity. The bias is +// symmetric, so this is a property of the scoring rule rather than of +// any one vendor. +// +// accurateLabel adds the check the harness could always have made: the +// anchor list already carries a curated Hint for every address and the +// scoring path ignored it. This is published as a SEPARATE series +// (wallet_labels_accurate_total) rather than folded into +// wallet_labels_success_total, so the existing leaderboard and its +// history stay intact while the stricter number builds up alongside. + +// normalizeLabel lowercases and strips everything that is not +// alphanumeric, so "Uniswap: Universal Router" and "uniswap universal +// router" compare equal. +func normalizeLabel(s string) string { + var b strings.Builder + for _, r := range strings.ToLower(s) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + } + } + return b.String() +} + +// genericHintWords are words that appear in curated hints but carry no +// entity signal on their own. Without this list "USDC (Base native)" +// would match "jakie.base.eth" through the word "base", scoring a +// personal Basename as a correct label for a token contract. +var genericHintWords = map[string]bool{ + "base": true, "solana": true, "ethereum": true, "polygon": true, "arbitrum": true, + "optimism": true, "avalanche": true, "bitcoin": true, "stellar": true, "native": true, + "token": true, "contract": true, "wallet": true, "hot": true, "cold": true, + "chain": true, "mint": true, "address": true, "factory": true, "proxy": true, + "deployer": true, "treasury": true, "bridge": true, "pool": true, "vault": true, + "router": true, "exchange": true, "protocol": true, "official": true, "main": true, +} + +// hintTokens splits a curated hint into the words that carry entity +// signal. Bare indices are dropped so "Binance 14" matches on "binance" +// and never on "14", otherwise "Bitstamp 14" would score as a correct +// answer. Generic and chain words are dropped for the same reason. The +// length floor is 3 and not 4: "OKX" is a real entity name. +func hintTokens(hint string) []string { + repl := strings.NewReplacer(":", " ", "-", " ", "/", " ", "(", " ", ")", " ", ".", " ", "_", " ") + var out []string + for _, w := range strings.Fields(strings.ToLower(repl.Replace(hint))) { + if len(w) < 3 || allDigits(w) || genericHintWords[w] { + continue + } + out = append(out, w) + } + return out +} + +// labelTokens splits a returned label the same way, so matching happens +// on whole words. Substring matching would let "base" inside +// "jakie.base.eth" pass, which is exactly the false positive this +// series exists to avoid. +func labelTokens(label string) []string { + repl := strings.NewReplacer(":", " ", "-", " ", "/", " ", "(", " ", ")", " ", ".", " ", "_", " ") + return strings.Fields(strings.ToLower(repl.Replace(label))) +} + +func allDigits(s string) bool { + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return len(s) > 0 +} + +// accurateLabel reports whether the returned label plausibly names the +// entity the anchor was curated for. Permissive on form ("Binance" +// matches "Binance 14", "OKX 7" matches "OKX 1") and strict on identity +// ("Bittrex 3" does not match "OKX 1", "jakie.base.eth" does not match +// "USDC (Base native)"). +func accurateLabel(hint, label string) bool { + if hint == "" || label == "" { + return false + } + h, l := normalizeLabel(hint), normalizeLabel(label) + if h == "" || l == "" { + return false + } + if h == l { + return true + } + ht := hintTokens(hint) + if len(ht) == 0 { + // Hint carried no signal word (e.g. "Binance 14" reduced to + // nothing would be a bug, but "1" alone would not). Fall back to + // whole-string containment rather than matching everything. + return strings.Contains(l, h) || strings.Contains(h, l) + } + lt := labelTokens(label) + for _, hw := range ht { + for _, lw := range lt { + if hw == lw || strings.HasPrefix(lw, hw) || strings.HasPrefix(hw, lw) { + return true + } + } + } + return false +} diff --git a/harnesses/wallet-labels/cmd/script/accuracy_test.go b/harnesses/wallet-labels/cmd/script/accuracy_test.go new file mode 100644 index 000000000..f8e08b949 --- /dev/null +++ b/harnesses/wallet-labels/cmd/script/accuracy_test.go @@ -0,0 +1,38 @@ +package main + +import "testing" + +func TestAccurateLabel(t *testing.T) { + cases := []struct { + hint, label string + want bool + why string + }{ + {"Binance 14", "Binance", true, "curated entity with an index, provider returns the bare name"}, + {"Uniswap V3 Router 2", "Uniswap: Universal Router", true, "same protocol, different punctuation"}, + {"vitalik.eth", "vitalik.eth", true, "exact"}, + {"OKX 1", "Bittrex 3", false, "different exchange must not pass"}, + {"OKX 1", "OKX 7", true, "same entity, different hot wallet index"}, + {"Bitfinex", "Polygon", false, "unrelated"}, + {"Permit2", "dex.davywoodfi.eth", false, "personal ENS on a known contract"}, + {"USDC (Base native)", "jakie.base.eth", false, "personal basename on a token contract"}, + {"Raydium Authority", "bonklanatoken.sol", false, "personal .sol on a program authority"}, + {"USDT (BSC)", "Fake_Phishing6512", false, "explorer warning tag is not the entity"}, + {"Binance 14", "", false, "no label"}, + {"", "Binance", false, "no hint"}, + {"Binance 8", "Binance 8", true, "exact with index"}, + {"Coinbase 1", "Coinbase 10", true, "same entity"}, + } + for _, c := range cases { + if got := accurateLabel(c.hint, c.label); got != c.want { + t.Errorf("accurateLabel(%q, %q) = %v, want %v (%s)", c.hint, c.label, got, c.want, c.why) + } + } +} + +func TestHintTokensDropsIndices(t *testing.T) { + got := hintTokens("Binance 14") + if len(got) != 1 || got[0] != "binance" { + t.Fatalf("hintTokens(\"Binance 14\") = %v, want [binance]; a bare index must never be a match token", got) + } +} diff --git a/harnesses/wallet-labels/cmd/script/anchor_feeder.go b/harnesses/wallet-labels/cmd/script/anchor_feeder.go index 851fb2dce..14c3b3391 100644 --- a/harnesses/wallet-labels/cmd/script/anchor_feeder.go +++ b/harnesses/wallet-labels/cmd/script/anchor_feeder.go @@ -36,6 +36,7 @@ func runAnchorFeeder(ctx context.Context, q *queue) { address: a.Address, chain: a.Chain, kind: a.Kind, + hint: a.Hint, discoveredAt: time.Now(), }) { // queue full — wait a bit so workers can catch up. diff --git a/harnesses/wallet-labels/cmd/script/metrics.go b/harnesses/wallet-labels/cmd/script/metrics.go index cc9d05ec9..c5a200244 100644 --- a/harnesses/wallet-labels/cmd/script/metrics.go +++ b/harnesses/wallet-labels/cmd/script/metrics.go @@ -26,6 +26,21 @@ var ( ConstLabels: commonLabels, }, []string{"provider", "chain", "kind"}) + // Companion series to successTotal. Same denominator, stricter rule: + // the label must actually name the curated entity for the anchor, not + // merely be non-generic. See accuracy.go for why this exists. + accuracyChecksTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "wallet_labels_accuracy_checks_total", + Help: "Label checks scored for accuracy against the curated anchor hint.", + ConstLabels: commonLabels, + }, []string{"provider", "chain", "kind"}) + + accurateTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "wallet_labels_accurate_total", + Help: "Checks where the returned label names the curated entity for that anchor.", + ConstLabels: commonLabels, + }, []string{"provider", "chain", "kind"}) + apiLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "wallet_labels_api_latency_milliseconds", Help: "Provider API response time in milliseconds.", @@ -67,6 +82,19 @@ func recordSkipped(provider, chain string) { skippedTotal.WithLabelValues(provider, chain).Inc() } +// recordAccuracy feeds the companion series that scores a label against +// the curated Hint rather than against "is it non-generic". Same label +// set as checksTotal so the two ratios share a denominator. +func recordAccuracy(provider, chain, kind string, accurate bool) { + if kind == "" { + kind = "unknown" + } + accuracyChecksTotal.WithLabelValues(provider, chain, kind).Inc() + if accurate { + accurateTotal.WithLabelValues(provider, chain, kind).Inc() + } +} + func recordCheck(provider, chain, kind string, hasLabel bool, latencyMs float64, err error) { if kind == "" { kind = "unknown" diff --git a/harnesses/wallet-labels/cmd/script/monitor.go b/harnesses/wallet-labels/cmd/script/monitor.go index 9ca9e027d..0bb2fa20c 100644 --- a/harnesses/wallet-labels/cmd/script/monitor.go +++ b/harnesses/wallet-labels/cmd/script/monitor.go @@ -12,6 +12,7 @@ type sample struct { address string chain string kind string // "contract" | "eoa" — carried into Prom labels so the bench can split by anchor kind + hint string // curated entity name for this anchor; scores the accuracy series, never the hit rule discoveredAt time.Time } @@ -105,6 +106,12 @@ func lookupAll(ctx context.Context, providers []Provider, s sample) { continue } recordCheck(r.Provider, r.Chain, s.kind, r.HasLabel, float64(r.LatencyMs), r.Err) + // Stricter companion series: a hit only counts when the label + // actually names the curated entity. Published alongside, never + // folded into, the hit rate. See accuracy.go. + if r.Err == nil { + recordAccuracy(r.Provider, r.Chain, s.kind, r.HasLabel && accurateLabel(s.hint, r.Label)) + } recordDebug(debugEntry{ Provider: r.Provider, Chain: r.Chain, Address: r.Address, HasLabel: r.HasLabel, LatencyMs: r.LatencyMs,