From f79a6615a013639c028f4b3040790abb7023d57b Mon Sep 17 00:00:00 2001 From: Ben Marx Date: Thu, 6 Aug 2026 17:41:53 -0700 Subject: [PATCH] solana/rpc: record endpoint rate-limit headroom per method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 429 from the ledger endpoint is currently opaque. solana-go keeps only a code on its error types — *HTTPError is {Code int, err error} and *RPCError is {Code, Message, Data} — and discards the http.Response inside CallForInto, so the X-Ratelimit-* headers that say what the cap actually was are gone before any caller sees the error. A service can log "Too many requests for a specific RPC call" for hours without ever learning the number it is exceeding. Adds a passive RoundTripper on the outermost transport that records X-Ratelimit-Method-Limit and X-Ratelimit-Method-Remaining as gauges labelled by JSON-RPC method. Limits are enforced per method, per source IP, over a rolling window, so an unattributed number would not be actionable and a global request-rate graph does not show it. Cheap when unused: the request body is only inspected once a response header proves there is something to record, and it is read through GetBody so the body the transport sends is untouched. Observation never alters a request/response and never fails a call. Motivating incident: a lake indexer activity was rate-limited on getTransaction for 4.5 hours. The cap was unknowable from our side — the headers were never captured, and the endpoint returns no X-Ratelimit-* headers on a 200, so the vendor's suggested "read them off a response" was not available either. --- CHANGELOG.md | 1 + tools/solana/pkg/jsonrpc/metrics.go | 13 +++ tools/solana/pkg/jsonrpc/retry.go | 29 +++++ tools/solana/pkg/jsonrpc/retry_test.go | 29 +++++ tools/solana/pkg/rpc/ratelimit.go | 136 ++++++++++++++++++++++ tools/solana/pkg/rpc/ratelimit_test.go | 155 +++++++++++++++++++++++++ tools/solana/pkg/rpc/retry.go | 7 +- 7 files changed, 368 insertions(+), 2 deletions(-) create mode 100644 tools/solana/pkg/rpc/ratelimit.go create mode 100644 tools/solana/pkg/rpc/ratelimit_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 87af8fb96b..9809b9ef41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file. - SDK - The Go telemetry SDK no longer reports a transaction the program rejected as a success. Finalization only means the cluster agreed on the transaction: a rejected instruction finalizes too, carrying the rejection in `err`, which the executor never read. It now returns a `*telemetry.ProgramError` holding the ledger's error and the program's log output, and leads the message with the program's own explanation so a caller that just prints the error still gets the reason. This is the check the serviceability executor already made. (malbeclabs/infra#1703, #4152) - A samples-account-full or missing-account rejection that reaches execution now returns the same `ErrSamplesAccountFull` / `ErrAccountNotFound` the equivalent preflight rejection does, via the new `ProgramError.CustomErrorCode()`. Preflight catches nearly all of these, but a write that simulated cleanly and then failed against the bank it landed on reported its code only through the finalized transaction, so a caller's account-full handling worked on one side of preflight and not the other. (malbeclabs/infra#1703, #4152) + - Rate-limit headroom reported by a ledger RPC endpoint is now recorded per JSON-RPC method: new `doublezero_solana_rpc_ratelimit_method_limit` and `doublezero_solana_rpc_ratelimit_method_remaining` gauges, fed by a passive transport wrapper on every retrying client `tools/solana/pkg/rpc` builds. A 429 was previously opaque: solana-go keeps only a code on its error types (`*HTTPError` is `{Code int, err error}`, `*RPCError` is `{Code, Message, Data}`) and discards the `http.Response` inside `CallForInto`, so the `X-Ratelimit-*` headers naming the cap were gone before any caller saw the error — a lake indexer activity was rate-limited on `getTransaction` for 4.5 hours with no way to learn the number it was exceeding, and the endpoint returns no such headers on a 200 either. Limits are enforced per method, per source IP, over a rolling window, so the numbers are attributed to the method they apply to; a global request-rate graph does not show this. Costs nothing where unused: the request body is inspected only once a response header proves there is something to record, it is read through `GetBody` so the body the transport sends is untouched, and an endpoint reporting no headers produces no series. Two further signals make a refusal attributable to a hop: `doublezero_solana_rpc_ratelimited_total` labels each rate-limited response by the shape that carried it, since an HTTP 429 status is what an edge limiter returns while a limiter at or behind the origin commonly answers 200 with the refusal inside the JSON-RPC envelope — only the latter reaches a caller as an `*RPCError`, and no HTTP-level metric shows it; and `doublezero_solana_rpc_responses_total` counts responses by the serving node the endpoint names, which answers whether load is concentrated on one node given limits are enforced per source IP. Neither is recoverable after the fact, which is why the 4.5h rate limit could not be attributed to the edge or the ledger node. (malbeclabs/lake#753, #4161) - Collector - A failed internet-latency submission now retries from the first unwritten sample rather than restarting at the beginning of the flushed partition, and only the unwritten remainder is requeued — the same fix the device telemetry submitter got in #4145. A partition larger than one transaction is written in batches, so any failure part-way through left the earlier batches onchain while the retry and the next tick re-sent them, appending those samples a second time and skewing the latency they feed. Reachable today from an RPC timeout mid-partition; surfacing program rejections adds another way in. (malbeclabs/infra#1703, #4152) - Device Telemetry diff --git a/tools/solana/pkg/jsonrpc/metrics.go b/tools/solana/pkg/jsonrpc/metrics.go index 828b09238b..5dc9a4a372 100644 --- a/tools/solana/pkg/jsonrpc/metrics.go +++ b/tools/solana/pkg/jsonrpc/metrics.go @@ -21,4 +21,17 @@ var ( Name: "doublezero_solana_rpc_retries_exhausted_total", Help: "Number of Solana JSON-RPC requests that failed after exhausting all retry attempts, by method.", }, []string{"method"}) + + // rateLimitedTotal counts rate-limited responses by the shape that carried the + // limit. That shape is the diagnostic: an edge/CDN limiter answers with an HTTP + // 429 status, while a limiter at or behind the origin commonly answers 200 with + // the refusal inside the JSON-RPC envelope. Only the second reaches this package + // as an *RPCError, and only the first is visible to the transport layer, so the + // split between these two label values says which hop refused the call — the + // question a provider will ask first, and one we could not answer for a 4.5h + // getTransaction rate limit because nothing recorded it. + rateLimitedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "doublezero_solana_rpc_ratelimited_total", + Help: "Rate-limited JSON-RPC responses, by method and the error shape that carried the limit (http_status or jsonrpc_error).", + }, []string{"method", "carrier"}) ) diff --git a/tools/solana/pkg/jsonrpc/retry.go b/tools/solana/pkg/jsonrpc/retry.go index 1b58c93d7d..68295cab78 100644 --- a/tools/solana/pkg/jsonrpc/retry.go +++ b/tools/solana/pkg/jsonrpc/retry.go @@ -126,6 +126,9 @@ func doRetry(ctx context.Context, opt RetryOptions, label string, idempotent boo } lastErr = f(ctx) + if carrier := rateLimitCarrier(lastErr); carrier != "" { + rateLimitedTotal.WithLabelValues(label, carrier).Inc() + } if lastErr == nil || !opt.IsRetryableFunc(lastErr) { return lastErr } @@ -211,6 +214,32 @@ func isRetryableRPCCode(code int) bool { return false } +// rateLimitCarrier reports which error shape carried a rate limit, or "" if err is +// not one. The distinction is the point: an *HTTPError means the refusal arrived as +// an HTTP 429 status, which an edge or CDN limiter produces and the transport layer +// can also see; an *RPCError means it arrived inside a JSON-RPC envelope on an +// otherwise-successful response, which is what a limiter at or behind the origin +// typically does and which no HTTP-level metric will ever show. +// +// -32429 is included because a provider fronting Agave mints it to mirror HTTP 429 +// inside an envelope (see isRetryableRPCCode); a bare positive 429 in an *RPCError +// is a different origin from the same provider's -32429, which is precisely the +// kind of thing worth being able to tell apart after the fact. +func rateLimitCarrier(err error) string { + if err == nil { + return "" + } + var httpErr *jsonrpc.HTTPError + if errors.As(err, &httpErr) && httpErr.Code == http.StatusTooManyRequests { + return "http_status" + } + var rpcErr *jsonrpc.RPCError + if errors.As(err, &rpcErr) && (rpcErr.Code == http.StatusTooManyRequests || rpcErr.Code == -32429) { + return "jsonrpc_error" + } + return "" +} + func isRetryableJSONRPC(err error) bool { if err == nil { return false diff --git a/tools/solana/pkg/jsonrpc/retry_test.go b/tools/solana/pkg/jsonrpc/retry_test.go index b0b992e3ca..4e513f5c6b 100644 --- a/tools/solana/pkg/jsonrpc/retry_test.go +++ b/tools/solana/pkg/jsonrpc/retry_test.go @@ -420,3 +420,32 @@ func fastRetryOpt(max int) *RetryOptions { MaxBackoff: 2 * time.Millisecond, } } + +// TestRateLimitCarrier distinguishes which hop refused the call. An HTTP 429 status +// is edge-shaped and also visible to the transport; a 429 inside a JSON-RPC envelope +// on an otherwise-successful response is origin-shaped and invisible to every +// HTTP-level metric. Being unable to tell these apart is what left a 4.5h +// getTransaction rate limit unattributable. +func TestRateLimitCarrier(t *testing.T) { + tests := []struct { + name string + err error + want string + }{ + {"nil", nil, ""}, + {"http 429", &jsonrpc.HTTPError{Code: 429}, "http_status"}, + {"http 503 is not a rate limit", &jsonrpc.HTTPError{Code: 503}, ""}, + {"envelope 429", &jsonrpc.RPCError{Code: 429, Message: "Too many requests for a specific RPC call"}, "jsonrpc_error"}, + {"envelope -32429", &jsonrpc.RPCError{Code: -32429}, "jsonrpc_error"}, + {"envelope -32005 is not a rate limit", &jsonrpc.RPCError{Code: -32005}, ""}, + {"wrapped envelope 429", fmt.Errorf("get transaction: %w", &jsonrpc.RPCError{Code: 429}), "jsonrpc_error"}, + {"unrelated", errors.New("connection reset by peer"), ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := rateLimitCarrier(tt.err); got != tt.want { + t.Errorf("rateLimitCarrier() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/tools/solana/pkg/rpc/ratelimit.go b/tools/solana/pkg/rpc/ratelimit.go new file mode 100644 index 0000000000..4150cd4c1c --- /dev/null +++ b/tools/solana/pkg/rpc/ratelimit.go @@ -0,0 +1,136 @@ +package rpc + +import ( + "encoding/json" + "io" + "net/http" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Rate-limit headroom as reported by the endpoint, by JSON-RPC method. Method +// cardinality is bounded by the Solana JSON-RPC surface, so it is safe as a label. +// +// These exist because a 429 is otherwise opaque. solana-go's error types carry +// only a code — *HTTPError is {Code int, err error} and *RPCError is +// {Code, Message, Data} — and the http.Response is discarded inside CallForInto, +// so by the time a rate limit reaches a caller the headers that say what the cap +// actually was are gone. A caller could see "Too many requests for a specific RPC +// call" for hours without ever learning the number it was exceeding. +// +// Rate limits are enforced per method, per source IP, over a rolling window, so +// `remaining` approaching zero on one method is the signal that matters — a global +// request-rate graph will not show it, and neither will a graph of another method. +var ( + rateLimitMethodLimit = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "doublezero_solana_rpc_ratelimit_method_limit", + Help: "Endpoint-reported request cap for a JSON-RPC method, per rolling rate-limit window.", + }, []string{"method"}) + + rateLimitMethodRemaining = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "doublezero_solana_rpc_ratelimit_method_remaining", + Help: "Endpoint-reported requests left for a JSON-RPC method in the current rate-limit window.", + }, []string{"method"}) + + // responsesTotal counts responses by the serving node the endpoint names in + // X-Ratelimit-Node / X-RPC-Node, or "unknown" when it names none. + // + // Rate limits are enforced per source IP, so when a provider asks whether load is + // concentrated on one of their nodes — and whether spreading source IPs would help + // — this is the answer. Node identity is not on the error path, so it cannot be + // recovered after a refusal; recording it per response is what makes it available + // at all. Cardinality is the endpoint's node count, which is small. + responsesTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "doublezero_solana_rpc_responses_total", + Help: "JSON-RPC responses received, by the serving node the endpoint reports.", + }, []string{"node"}) +) + +const ( + headerMethodLimit = "X-Ratelimit-Method-Limit" + headerMethodRemaining = "X-Ratelimit-Method-Remaining" + headerRPCNode = "X-RPC-Node" + + // maxObservedBody bounds how much of a request body is read to recover the + // method name. JSON-RPC requests put "method" near the front, and this only + // runs on responses that already carry rate-limit headers. + maxObservedBody = 512 +) + +// rateLimitObserver records rate-limit headers into metrics as responses pass +// through. It never alters the request or response and never fails a call: an +// endpoint that reports nothing simply produces no series. +type rateLimitObserver struct{ inner http.RoundTripper } + +func (o *rateLimitObserver) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := o.inner.RoundTrip(req) + if err != nil || resp == nil { + return resp, err + } + + node := resp.Header.Get(headerRPCNode) + if node == "" { + node = "unknown" + } + responsesTotal.WithLabelValues(node).Inc() + + // Cheap exit first. Most endpoints report nothing, and this must not add + // per-request work on the hot path when there is nothing to record — so the + // request body is only inspected once a header proves it is worth it. + limit := resp.Header.Get(headerMethodLimit) + remaining := resp.Header.Get(headerMethodRemaining) + if limit == "" && remaining == "" { + return resp, nil + } + + method := requestMethod(req) + if method == "" { + return resp, nil + } + if v, convErr := strconv.ParseFloat(limit, 64); convErr == nil { + rateLimitMethodLimit.WithLabelValues(method).Set(v) + } + if v, convErr := strconv.ParseFloat(remaining, 64); convErr == nil { + rateLimitMethodRemaining.WithLabelValues(method).Set(v) + } + return resp, nil +} + +// requestMethod recovers the JSON-RPC method from a request body, or "batch" for +// a batch request whose members may differ. It reads through GetBody so the body +// the transport is sending is left untouched; a request without GetBody (not +// produced by the clients in this package) yields "" and is skipped. +func requestMethod(req *http.Request) string { + if req.GetBody == nil { + return "" + } + body, err := req.GetBody() + if err != nil { + return "" + } + defer body.Close() + + buf, err := io.ReadAll(io.LimitReader(body, maxObservedBody)) + if err != nil || len(buf) == 0 { + return "" + } + for _, b := range buf { + switch b { + case ' ', '\t', '\r', '\n': + continue + case '[': + return "batch" + } + break + } + // Decode into just the method field; a truncated body fails cleanly to "". + var probe struct { + Method string `json:"method"` + } + if json.Unmarshal(buf, &probe) != nil { + return "" + } + return probe.Method +} diff --git a/tools/solana/pkg/rpc/ratelimit_test.go b/tools/solana/pkg/rpc/ratelimit_test.go new file mode 100644 index 0000000000..db7ee66979 --- /dev/null +++ b/tools/solana/pkg/rpc/ratelimit_test.go @@ -0,0 +1,155 @@ +package rpc + +import ( + "bytes" + "io" + "net/http" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// stubRT returns a fixed response and records whether the request body it was +// handed is still fully readable afterwards — the observer must not consume it. +type stubRT struct { + header http.Header + bodySeen string +} + +func (s *stubRT) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Body != nil { + b, _ := io.ReadAll(req.Body) + s.bodySeen = string(b) + } + return &http.Response{ + StatusCode: 200, + Header: s.header, + Body: io.NopCloser(strings.NewReader(`{"jsonrpc":"2.0","result":null,"id":1}`)), + }, nil +} + +func newJSONRPCRequest(t *testing.T, body string) *http.Request { + t.Helper() + req, err := http.NewRequest(http.MethodPost, "https://example.invalid/k", bytes.NewReader([]byte(body))) + if err != nil { + t.Fatalf("new request: %v", err) + } + return req +} + +// TestRateLimitObserver_RecordsHeadersByMethod is the point of the file: a 429 is +// opaque without these numbers, because solana-go keeps only a code on its error +// types and discards the http.Response. When the endpoint reports the cap, it must +// land in metrics attributed to the method it applies to — rate limits are enforced +// per method, so an unattributed number would be useless. +func TestRateLimitObserver_RecordsHeadersByMethod(t *testing.T) { + h := http.Header{} + h.Set("X-Ratelimit-Method-Limit", "200") + h.Set("X-Ratelimit-Method-Remaining", "7") + stub := &stubRT{header: h} + + body := `{"jsonrpc":"2.0","id":1,"method":"getTransaction","params":["sig",{}]}` + resp, err := (&rateLimitObserver{inner: stub}).RoundTrip(newJSONRPCRequest(t, body)) + if err != nil { + t.Fatalf("RoundTrip: %v", err) + } + if resp.StatusCode != 200 { + t.Fatalf("response altered: status %d", resp.StatusCode) + } + + if got := testutil.ToFloat64(rateLimitMethodLimit.WithLabelValues("getTransaction")); got != 200 { + t.Errorf("limit gauge = %v, want 200", got) + } + if got := testutil.ToFloat64(rateLimitMethodRemaining.WithLabelValues("getTransaction")); got != 7 { + t.Errorf("remaining gauge = %v, want 7", got) + } + + // The inner transport must still receive the whole body: recovering the method + // reads through GetBody precisely so the real body is left alone. + if stub.bodySeen != body { + t.Errorf("inner transport saw a modified body:\n got: %q\nwant: %q", stub.bodySeen, body) + } +} + +// TestRateLimitObserver_NoHeadersNoSeries pins the cheap-exit path. Endpoints that +// report nothing (the DZ mainnet ledger currently returns no X-Ratelimit-* headers +// on a 200) must produce no series and no body inspection. +func TestRateLimitObserver_NoHeadersNoSeries(t *testing.T) { + stub := &stubRT{header: http.Header{}} + body := `{"jsonrpc":"2.0","id":1,"method":"getSlot","params":[]}` + + if _, err := (&rateLimitObserver{inner: stub}).RoundTrip(newJSONRPCRequest(t, body)); err != nil { + t.Fatalf("RoundTrip: %v", err) + } + if n := testutil.CollectAndCount(rateLimitMethodLimit); n != 0 { + // Only this test's method would be new; the other test uses getTransaction. + if got := testutil.ToFloat64(rateLimitMethodLimit.WithLabelValues("getSlot")); got != 0 { + t.Errorf("getSlot limit gauge = %v, want no observation", got) + } + } + if stub.bodySeen != body { + t.Errorf("inner transport saw a modified body: %q", stub.bodySeen) + } +} + +// TestRateLimitObserver_BatchIsLabelledBatch: a batch carries a mix of methods, so +// attributing its headroom to any single one would be wrong. Mirrors the batchLabel +// the retry layer already uses. +func TestRateLimitObserver_BatchIsLabelledBatch(t *testing.T) { + h := http.Header{} + h.Set("X-Ratelimit-Method-Limit", "50") + stub := &stubRT{header: h} + + body := `[{"jsonrpc":"2.0","id":1,"method":"getSlot"},{"jsonrpc":"2.0","id":2,"method":"getEpochInfo"}]` + if _, err := (&rateLimitObserver{inner: stub}).RoundTrip(newJSONRPCRequest(t, body)); err != nil { + t.Fatalf("RoundTrip: %v", err) + } + if got := testutil.ToFloat64(rateLimitMethodLimit.WithLabelValues("batch")); got != 50 { + t.Errorf("batch limit gauge = %v, want 50", got) + } +} + +// TestRateLimitObserver_MalformedBodyIsSkipped: observation is best-effort and must +// never fail a call. A body it cannot parse yields no series and no error. +func TestRateLimitObserver_MalformedBodyIsSkipped(t *testing.T) { + h := http.Header{} + h.Set("X-Ratelimit-Method-Limit", "10") + stub := &stubRT{header: h} + + if _, err := (&rateLimitObserver{inner: stub}).RoundTrip(newJSONRPCRequest(t, `not json at all`)); err != nil { + t.Fatalf("malformed body must not fail the request: %v", err) + } +} + +// TestRateLimitObserver_CountsServingNode: rate limits are enforced per source IP, +// so when a provider asks whether load is concentrated on one of their nodes, the +// serving node is the answer. It is absent from the error path, so it has to be +// recorded per response or it is unavailable after the fact. +func TestRateLimitObserver_CountsServingNode(t *testing.T) { + h := http.Header{} + h.Set("X-RPC-Node", "lb-pit5") + stub := &stubRT{header: h} + + body := `{"jsonrpc":"2.0","id":1,"method":"getSlot","params":[]}` + before := testutil.ToFloat64(responsesTotal.WithLabelValues("lb-pit5")) + if _, err := (&rateLimitObserver{inner: stub}).RoundTrip(newJSONRPCRequest(t, body)); err != nil { + t.Fatalf("RoundTrip: %v", err) + } + if got := testutil.ToFloat64(responsesTotal.WithLabelValues("lb-pit5")) - before; got != 1 { + t.Errorf("lb-pit5 response count delta = %v, want 1", got) + } +} + +// TestRateLimitObserver_UnnamedNodeIsCounted: a response naming no node must still +// be counted, so a silent gap in the series can't be mistaken for no traffic. +func TestRateLimitObserver_UnnamedNodeIsCounted(t *testing.T) { + stub := &stubRT{header: http.Header{}} + before := testutil.ToFloat64(responsesTotal.WithLabelValues("unknown")) + if _, err := (&rateLimitObserver{inner: stub}).RoundTrip(newJSONRPCRequest(t, `{"method":"getSlot"}`)); err != nil { + t.Fatalf("RoundTrip: %v", err) + } + if got := testutil.ToFloat64(responsesTotal.WithLabelValues("unknown")) - before; got != 1 { + t.Errorf("unknown-node count delta = %v, want 1", got) + } +} diff --git a/tools/solana/pkg/rpc/retry.go b/tools/solana/pkg/rpc/retry.go index d5a64d3615..011597888d 100644 --- a/tools/solana/pkg/rpc/retry.go +++ b/tools/solana/pkg/rpc/retry.go @@ -116,8 +116,11 @@ func newHTTP(requestTimeout time.Duration, maxConns int, onDial func(addr, remot } return &http.Client{ - Timeout: requestTimeout, - Transport: gzhttp.Transport(newHTTPTransport(maxConns, onDial)), + Timeout: requestTimeout, + // The rate-limit observer wraps the outermost transport so it sees the final + // response headers. It is passive: nothing is altered, and an endpoint that + // reports no rate-limit headers costs nothing (see ratelimit.go). + Transport: &rateLimitObserver{inner: gzhttp.Transport(newHTTPTransport(maxConns, onDial))}, } }