From e3998e4b20c85c9743f6cf912a00577419714ad9 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Tue, 17 Nov 2026 13:30:14 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=90=9B=20fix:=20preserve=20MongoDB=20?= =?UTF-8?q?command=20and=20cursor=20semantics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/mongodb.md | 19 +++- mongodb/command_cursor_test.go | 201 +++++++++++++++++++++++++++++++++ mongodb/mongodb.go | 65 +++++++++-- 3 files changed, 270 insertions(+), 15 deletions(-) create mode 100644 mongodb/command_cursor_test.go diff --git a/docs/mongodb.md b/docs/mongodb.md index dcfbffd..912fba1 100644 --- a/docs/mongodb.md +++ b/docs/mongodb.md @@ -99,11 +99,20 @@ estimate of a whole collection, run the `count` command with `DB.Command`. ## Commands -`DB.Command` runs any command in the database and returns its reply. A reply with -`ok: 0`, `writeErrors` or a `writeConcernError` comes back together with a -`*CommandError` holding its code, code name and message. `DB.Cursor` runs a -command that opens a cursor, such as `aggregate` or `listIndexes`, and yields its -documents the way `Find` does. +`DB.Command` runs any command in the database and returns its reply. Put the +command name first in an ordered document such as `bson.D` or `bson.Raw`. +Multi-key maps, including `bson.M`, are rejected before sending because MongoDB +uses the first key to identify the command. Maps nested in a command's filter +or other arguments are unaffected; single-key commands such as `bson.M{"ping": 1}` +are also accepted. + +A reply with `ok: 0`, `writeErrors` or a `writeConcernError` comes back together +with a `*CommandError` holding its code, code name and message. `DB.Cursor` takes +an ordered command that opens a cursor, such as `aggregate` or `listIndexes`, +and yields its documents the way `Find` does. Its `getMore` and cleanup commands +retain an explicit `lsid`, `txnNumber` and `autocommit` from the opening command, +but do not repeat `startTransaction`. The cursor is cleaned up if the loop exits +early, returns an error or panics, even after the caller's context is canceled. ```go create := bson.D{{Key: "create", Value: "users"}} diff --git a/mongodb/command_cursor_test.go b/mongodb/command_cursor_test.go new file mode 100644 index 0000000..c485300 --- /dev/null +++ b/mongodb/command_cursor_test.go @@ -0,0 +1,201 @@ +package mongodb + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/batchstream/weir-go/internal/storetest" + pb "github.com/batchstream/weir-protocol/api/weir/v1" + "go.mongodb.org/mongo-driver/v2/bson" +) + +func TestCommandsRejectUnorderedMultiKeyMapsBeforeSending(t *testing.T) { + type namedMap map[string]any + unordered := bson.M{"find": "users", "filter": bson.M{"active": true}} + plain := map[string]any{"find": "users", "limit": 1} + named := namedMap{"find": "users", "limit": 1} + pointer := &unordered + inputs := []any{unordered, &unordered, &pointer, plain, &plain, named, &named} + r := &recorder{t: t} + r.answer = func(*pb.Command) []*pb.Event { + reply := bson.D{{Key: "ok", Value: 1}} + return storetest.Native(nil, bsonDocument(t, reply).Data) + } + db := r.db() + for _, input := range inputs { + _, err := db.Command(t.Context(), input) + if err == nil || !strings.Contains(err.Error(), "multi-key map") { + t.Fatalf("Command accepted unordered %T: %v", input, err) + } + var yielded int + for document, err := range db.Cursor(t.Context(), input) { + yielded++ + if document != nil || err == nil || !strings.Contains(err.Error(), "multi-key map") { + t.Fatalf("Cursor accepted unordered %T: %v, %v", input, document, err) + } + } + if yielded != 1 { + t.Fatalf("Cursor yielded %d results for invalid %T", yielded, input) + } + } + if len(r.commands) != 0 { + t.Fatal("unordered commands reached the server", r.commands) + } +} + +func TestCommandAcceptsOrderedCommandsAndSingleKeyMaps(t *testing.T) { + ordered := bson.D{{Key: "find", Value: "users"}, {Key: "filter", Value: bson.M{"active": true, "name": "Ada"}}} + raw := bson.Raw(bsonDocument(t, ordered).Data) + ping := bson.M{"ping": 1} + inputs := []any{ordered, &ordered, raw, &raw, ping, &ping} + r := &recorder{t: t} + r.answer = func(*pb.Command) []*pb.Event { + reply := bson.D{{Key: "ok", Value: 1}} + return storetest.Native(nil, bsonDocument(t, reply).Data) + } + db := r.db() + for index, input := range inputs { + if _, err := db.Command(t.Context(), input); err != nil { + t.Fatalf("rejected ordered %T: %v", input, err) + } + sent := command(t, r.commands[index]) + want := "find" + if index >= 4 { + want = "ping" + } + if sent[0].Key != want { + t.Fatalf("changed command order for %T: %v", input, sent) + } + } +} + +type marshaledCursorCommand struct { + value bson.D + calls int +} + +func (c *marshaledCursorCommand) MarshalBSON() ([]byte, error) { + c.calls++ + return bson.Marshal(c.value) +} + +func TestCursorPreservesExplicitSessionAndTransaction(t *testing.T) { + tests := []struct { + name string + transaction bool + raw bool + marshaler bool + }{ + {name: "session"}, + {name: "transaction", transaction: true}, + {name: "raw transaction", transaction: true, raw: true}, + {name: "marshaled transaction", transaction: true, marshaler: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + id := bson.Binary{Subtype: 4, Data: []byte("0123456789abcdef")} + session := bson.D{{Key: "id", Value: id}} + concern := bson.D{{Key: "level", Value: "majority"}} + find := bson.D{{Key: "find", Value: "users"}, {Key: "lsid", Value: session}, {Key: "batchSize", Value: 1}, {Key: "readConcern", Value: concern}} + keys := []string{"lsid"} + if test.transaction { + transaction := bson.D{{Key: "txnNumber", Value: int64(7)}, {Key: "autocommit", Value: false}, {Key: "startTransaction", Value: true}} + find = append(find, transaction...) + keys = append(keys, "txnNumber", "autocommit") + } + var input any = find + if test.raw { + input = bson.Raw(bsonDocument(t, find).Data) + } + custom := &marshaledCursorCommand{value: find} + if test.marshaler { + input = custom + } + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}, {"c"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + db := r.db() + seen := 0 + for _, err := range db.Cursor(t.Context(), input) { + if err != nil { + t.Fatal(err) + } + seen++ + if seen == 2 { + break + } + } + if len(r.commands) != 3 || len(cursors.killed) != 1 { + t.Fatal("expected find, getMore and one cleanup", r.commands) + } + original := bson.Raw(r.commands[0].GetNative().GetRequest().GetData()) + for _, sent := range r.commands[1:] { + raw := bson.Raw(sent.GetNative().GetRequest().GetData()) + for _, key := range keys { + want, got := original.Lookup(key), raw.Lookup(key) + if got.Type != want.Type || !bytes.Equal(got.Value, want.Value) { + t.Fatalf("lost %s in continuation: %v", key, raw) + } + } + for _, key := range []string{"startTransaction", "readConcern", "batchSize"} { + if raw.Lookup(key).Type != 0 { + t.Fatalf("copied initial-only %s into continuation: %v", key, raw) + } + } + } + if got := cursors.killed[0][1].Value.(bson.A)[0]; got != int64(2) { + t.Fatal("did not clean up the latest cursor", got) + } + if test.marshaler && custom.calls != 1 { + t.Fatal("encoded the original command more than once", custom.calls) + } + }) + } +} + +func TestCursorPanicCleansUpLatestCursor(t *testing.T) { + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}, {"c"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + db := r.db() + find := bson.D{{Key: "find", Value: "users"}} + var recovered any + func() { + defer func() { recovered = recover() }() + seen := 0 + for _, err := range db.Cursor(t.Context(), find) { + if err != nil { + t.Fatal(err) + } + seen++ + if seen == 2 { + panic("consumer failure") + } + } + }() + if recovered != "consumer failure" { + t.Fatal("consumer panic was not preserved", recovered) + } + if len(cursors.killed) != 1 || cursors.killed[0][1].Value.(bson.A)[0] != int64(2) { + t.Fatal("panic did not clean up the latest cursor exactly once", cursors.killed) + } +} + +func TestCanceledCursorStillCleansUp(t *testing.T) { + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + db := r.db() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + find := bson.D{{Key: "find", Value: "users"}} + for _, err := range db.Cursor(ctx, find) { + if err != nil { + t.Fatal(err) + } + cancel() + break + } + if len(cursors.killed) != 1 { + t.Fatal("caller cancellation prevented cursor cleanup", cursors.killed) + } +} diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index a374673..c5f90f8 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -27,8 +27,10 @@ package mongodb import ( "context" + "errors" "fmt" "iter" + "reflect" "strings" "time" @@ -71,16 +73,37 @@ func (e *CommandError) Error() string { return fmt.Sprintf("mongodb: command failed: %s (%d): %s", e.Name, e.Code, e.Message) } -// Command runs one command in the database and returns its reply. A reply -// with ok: 0, writeErrors or a writeConcernError comes with a *CommandError. +// Command runs one command in the database and returns its reply. Commands +// must preserve field order, with the command name first; multi-key maps are +// rejected before sending. Use bson.D or another ordered BSON representation. +// A reply with ok: 0, writeErrors or a writeConcernError comes with a *CommandError. // A command is sent again only when a server refused it before it began. One // a lost connection or an unavailable backend cut short reports // weir.ErrInterrupted and may have run; send it again only when that is safe. func (db DB) Command(ctx context.Context, command any) (bson.Raw, error) { + data, err := marshalCommand(command) + if err != nil { + return nil, err + } + return db.command(ctx, data) +} + +func marshalCommand(command any) (bson.Raw, error) { + value := reflect.ValueOf(command) + for value.IsValid() && (value.Kind() == reflect.Pointer || value.Kind() == reflect.Interface) { + value = value.Elem() + } + if value.IsValid() && value.Kind() == reflect.Map && value.Len() > 1 { + return nil, errors.New("mongodb: command must be ordered; use bson.D instead of a multi-key map") + } data, err := bson.Marshal(command) if err != nil { return nil, fmt.Errorf("mongodb: encode command: %w", err) } + return bson.Raw(data), nil +} + +func (db DB) command(ctx context.Context, data bson.Raw) (bson.Raw, error) { document := &weir.Document{ContentType: bsonContentType, Data: data} request := &weir.NativeRequest{Resource: db.resource(), Request: document} result, err := db.target().Native(ctx, request) @@ -97,13 +120,20 @@ func (db DB) Command(ctx context.Context, command any) (bson.Raw, error) { return reply, nil } -// Cursor runs a command that opens a cursor, such as find, aggregate or -// listIndexes, and yields its documents. It sends getMore until the cursor -// ends, and kills the cursor when the loop stops early or fails. An error -// ends the sequence. +// Cursor runs an ordered command that opens a cursor, such as find, aggregate +// or listIndexes, and yields its documents. It preserves explicit session and +// transaction fields on getMore and cleanup commands. It sends getMore until +// the cursor ends, and kills the cursor when the loop stops early, fails or +// panics. An error ends the sequence. func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] { documents := func(yield func(bson.Raw, error) bool) { - reply, err := db.Command(ctx, command) + data, err := marshalCommand(command) + if err != nil { + yield(nil, err) + return + } + session := cursorSession(data) + reply, err := db.command(ctx, data) if err != nil { yield(nil, err) return @@ -115,10 +145,10 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] } space := parseNamespace(page.Cursor.Namespace) batch, cursor := page.Cursor.FirstBatch, page.Cursor.ID + defer func() { db.killCursor(ctx, space, cursor, session) }() for { for _, document := range batch { if !yield(document, nil) { - db.killCursor(ctx, space, cursor) return } } @@ -126,13 +156,13 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] return } getMore := bson.D{{Key: "getMore", Value: cursor}, {Key: "collection", Value: space.collection}, {Key: "$db", Value: space.database}} + getMore = append(getMore, session...) reply, err := db.Command(ctx, getMore) var next cursorPage if err == nil { next, err = readPage(reply) } if err != nil { - db.killCursor(ctx, space, cursor) yield(nil, err) return } @@ -142,6 +172,20 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] return documents } +// cursorSession retains only fields needed to continue the original session +// or transaction. In particular, startTransaction belongs to the first command. +func cursorSession(command bson.Raw) bson.D { + var session bson.D + for _, key := range []string{"lsid", "txnNumber", "autocommit"} { + value := command.Lookup(key) + if value.Type != 0 { + field := bson.E{Key: key, Value: value} + session = append(session, field) + } + } + return session +} + // namespace names the collection of a cursor. Commands that follow the cursor // name its database, as the command that opened it may have named one with // $db. @@ -180,13 +224,14 @@ func readPage(reply bson.Raw) (cursorPage, error) { // killCursor releases a cursor a loop left open. The server also times idle // cursors out, so a failure here is ignored. -func (db DB) killCursor(ctx context.Context, space namespace, cursor int64) { +func (db DB) killCursor(ctx context.Context, space namespace, cursor int64, session bson.D) { if cursor == 0 { return } cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), killTimeout) defer cancel() command := bson.D{{Key: "killCursors", Value: space.collection}, {Key: "cursors", Value: bson.A{cursor}}, {Key: "$db", Value: space.database}} + command = append(command, session...) _, _ = db.Command(cleanup, command) } From b4a95f3d0115d3361917cf8e73b7e2f3b07ae6b8 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Tue, 17 Nov 2026 13:30:14 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=90=9B=20fix:=20report=20incomplete?= =?UTF-8?q?=20Search=20responses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/search.md | 25 +++++++++++++ search/index.go | 16 +++++++++ search/search.go | 21 ++++++++++- search/search_test.go | 83 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 1 deletion(-) diff --git a/docs/search.md b/docs/search.md index d2fe9e0..6dce5c0 100644 --- a/docs/search.md +++ b/docs/search.md @@ -80,6 +80,30 @@ if len(hits) > 0 { } ``` +A successful HTTP status does not guarantee a complete search. If the backend +reports `timed_out: true` or `_shards.failed > 0`, `Search` returns a +`*search.PartialResponseError` **and any decoded hits**. An empty hit slice in +this case does not mean there were no matches. A caller requiring complete +results must reject these hits; a best-effort caller may explicitly choose to +use them after inspecting the error with `errors.As`: + +```go +hits, err := products.Search(ctx, query) +if err != nil { + var partial *search.PartialResponseError + if !errors.As(err, &partial) { return err } + // This caller accepts incomplete results; hits may still be empty. + log.Printf("partial search: timed_out=%t failed_shards=%d", partial.TimedOut, partial.FailedShards) +} +``` + +`TimedOut` and `FailedShards` report the backend's completion metadata. +`ShardFailures` holds each `_shards.failures` entry as `json.RawMessage`, +preserving its shard, index and backend-specific reason details. The backend +may omit these entries even when `FailedShards` is nonzero. This error does not +trigger an automatic retry. Complete responses still return hits and a nil +error; HTTP status and JSON decoding errors retain their existing behavior. + ## HTTP requests `Index.Do` sends any HTTP request with a path relative to the index, such as @@ -113,6 +137,7 @@ as it is for a search. | `*weir.Error` | A record failed. `weir.CodeOf(err)` gives its `FailureCode`, and `Outcome` tells whether a write applied. | | `weir.ErrInterrupted` | An outage cut the request short, so it may succeed later. It may have run: never send a write again without checking. | | `*search.StatusError` | `Search` got a status of 300 or more | +| `*search.PartialResponseError` | `Search` timed out or had failed shards despite a successful HTTP status. Any returned hits may be incomplete. | A write certainly took no effect only when its `*weir.Error` has the `Outcome` `MutationNotStarted` or `MutationNotApplied`, or, for a request, the diff --git a/search/index.go b/search/index.go index 175baa7..ee7bd49 100644 --- a/search/index.go +++ b/search/index.go @@ -128,6 +128,9 @@ func (i Index[T]) Transform(ctx context.Context, id string, program string, inpu // Search sends a search request body, such as {"query": ..., "size": 10}, to // the index and returns its hits. A status of 300 or more is a *StatusError. +// A timed-out response or failed shards returns the decoded hits alongside a +// *PartialResponseError, even with a successful HTTP status. These hits may be +// incomplete, including an empty slice that does not establish no matches. // Sort holds a hit's sort values exactly, to pass back as search_after. func (i Index[T]) Search(ctx context.Context, body any) ([]Hit[T], error) { response, err := i.Do(ctx, http.MethodPost, "/_search", body) @@ -139,6 +142,11 @@ func (i Index[T]) Search(ctx context.Context, body any) ([]Hit[T], error) { return nil, failed } var result struct { + TimedOut bool `json:"timed_out"` + Shards struct { + Failed int `json:"failed"` + Failures []json.RawMessage `json:"failures"` + } `json:"_shards"` Hits struct { Hits []Hit[T] `json:"hits"` } `json:"hits"` @@ -146,6 +154,14 @@ func (i Index[T]) Search(ctx context.Context, body any) ([]Hit[T], error) { if err := json.Unmarshal(response.Body, &result); err != nil { return nil, fmt.Errorf("search: decode %s hits: %w", i.Name, err) } + if result.TimedOut || result.Shards.Failed > 0 { + partial := &PartialResponseError{ + TimedOut: result.TimedOut, + FailedShards: result.Shards.Failed, + ShardFailures: result.Shards.Failures, + } + return result.Hits.Hits, partial + } return result.Hits.Hits, nil } diff --git a/search/search.go b/search/search.go index fa27165..9561993 100644 --- a/search/search.go +++ b/search/search.go @@ -20,7 +20,9 @@ // Set a deadline on every context. A failed record is a *weir.Error: check it // with weir.CodeOf, and with errors.Is against weir.ErrNotFound and // weir.ErrInterrupted. The package keeps no state of its own; Cluster and -// Index are plain values that are safe to copy and share. +// Index are plain values that are safe to copy and share. Search returns +// decoded hits alongside a *PartialResponseError when the backend times out +// or reports failed shards, even with a successful HTTP status. package search import ( @@ -65,6 +67,23 @@ func (e *StatusError) Error() string { return fmt.Sprintf("search: status %d: %s", e.StatusCode, e.Body) } +// PartialResponseError reports that Search received an incomplete response, +// even though its HTTP status indicated success. Search returns any decoded +// hits alongside this error; they are not a complete result set. +// Inspect it with errors.As. +type PartialResponseError struct { + TimedOut bool + FailedShards int + // ShardFailures holds each _shards.failures entry as its original JSON, + // including any backend-specific reason details. It may be absent even + // when FailedShards is nonzero. + ShardFailures []json.RawMessage +} + +func (e *PartialResponseError) Error() string { + return fmt.Sprintf("search: incomplete response: timed_out=%t, failed_shards=%d", e.TimedOut, e.FailedShards) +} + // Do sends one HTTP request to the cluster and returns its response, whatever // its status. The path starts at the cluster root, such as /_cluster/health, // and may carry a query string. A []byte or json.RawMessage body is sent as it diff --git a/search/search_test.go b/search/search_test.go index d6c83c9..f32a76f 100644 --- a/search/search_test.go +++ b/search/search_test.go @@ -191,6 +191,89 @@ func TestFailedWritesReportTheirCode(t *testing.T) { } } +func TestSearchReportsPartialResponsesAndPreservesHits(t *testing.T) { + shardFailure := `{"shard":1,"index":"products","node":"node-1","reason":{"type":"query_shard_exception","reason":"query failed","caused_by":{"type":"illegal_argument_exception","reason":"bad field"}}}` + hit := `{"_id":"p1","_score":1.5,"sort":[9007199254740993,"lamp"],"_source":{"name":"lamp","price":30}}` + tests := []struct { + name string + body string + timedOut bool + failedShards int + shardFailures []string + hitCount int + }{ + { + name: "timeout with empty hits", + body: `{"timed_out":true,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"hits":[]}}`, + timedOut: true, + }, + { + name: "failed shards with partial hits", + body: `{"timed_out":false,"_shards":{"total":2,"successful":1,"failed":1,"failures":[` + shardFailure + `]},"hits":{"hits":[` + hit + `]}}`, + failedShards: 1, + shardFailures: []string{shardFailure}, + hitCount: 1, + }, + { + name: "timeout and failed shards", + body: `{"timed_out":true,"_shards":{"total":2,"successful":1,"failed":1,"failures":[` + shardFailure + `]},"hits":{"hits":[` + hit + `]}}`, + timedOut: true, + failedShards: 1, + shardFailures: []string{shardFailure}, + hitCount: 1, + }, + { + name: "failed shards without failure details", + body: `{"_shards":{"failed":2},"hits":{"hits":[]}}`, + failedShards: 2, + }, + { + name: "complete response", + body: `{"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":1,"failed":0},"hits":{"hits":[` + hit + `]}}`, + hitCount: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &recorder{t: t} + r.answer = func(*pb.Command) []*pb.Event { return httpResponse("200 OK", tt.body) } + products := r.products() + query := map[string]any{"query": map[string]any{"match_all": map[string]any{}}} + found, err := products.Search(t.Context(), query) + if tt.timedOut || tt.failedShards > 0 { + var partial *PartialResponseError + if !errors.As(err, &partial) { + t.Fatalf("expected *PartialResponseError, got %v", err) + } + if partial.TimedOut != tt.timedOut || partial.FailedShards != tt.failedShards { + t.Fatal("wrong incompleteness details", partial) + } + if len(partial.ShardFailures) != len(tt.shardFailures) { + t.Fatal("wrong shard failures", partial.ShardFailures) + } + for n, failure := range partial.ShardFailures { + if string(failure) != tt.shardFailures[n] { + t.Fatal("shard failure details changed", string(failure)) + } + } + } else if err != nil { + t.Fatal("complete response failed", err) + } + if len(found) != tt.hitCount { + t.Fatal("wrong hits", found) + } + if tt.hitCount > 0 { + if found[0].ID != "p1" || found[0].Score != 1.5 || found[0].Document == nil || found[0].Document.Name != "lamp" || found[0].Document.Price != 30 { + t.Fatal("hit changed", found[0]) + } + if len(found[0].Sort) != 2 || string(found[0].Sort[0]) != "9007199254740993" || string(found[0].Sort[1]) != `"lamp"` { + t.Fatal("sort values changed", found[0].Sort) + } + } + }) + } +} + func TestSearchDecodesHitsAndKeepsSortValuesExact(t *testing.T) { hits := `{"hits":{"total":{"value":2},"hits":[ {"_id":"p1","_score":null,"sort":[9007199254740993,"lamp"],"_source":{"name":"lamp","price":30}}, From a667f3b7e3d4b1be1470fe1f91dd5eca3bc0ed1c Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Tue, 17 Nov 2026 13:45:37 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=90=9B=20fix:=20preserve=20BSON=20hoo?= =?UTF-8?q?ks=20and=20eager=20cursor=20failure=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/mongodb.md | 10 +++-- mongodb/command_cursor_test.go | 76 ++++++++++++++++++++++++++++++++++ mongodb/mongodb.go | 22 ++++++---- 3 files changed, 96 insertions(+), 12 deletions(-) diff --git a/docs/mongodb.md b/docs/mongodb.md index 912fba1..46d2035 100644 --- a/docs/mongodb.md +++ b/docs/mongodb.md @@ -101,10 +101,12 @@ estimate of a whole collection, run the `count` command with `DB.Command`. `DB.Command` runs any command in the database and returns its reply. Put the command name first in an ordered document such as `bson.D` or `bson.Raw`. -Multi-key maps, including `bson.M`, are rejected before sending because MongoDB -uses the first key to identify the command. Maps nested in a command's filter -or other arguments are unaffected; single-key commands such as `bson.M{"ping": 1}` -are also accepted. +Multi-key maps without a custom BSON encoder, including ordinary `bson.M`, are +rejected before sending because MongoDB uses the first key to identify the +command. Custom `bson.Marshaler` implementations remain supported and are +responsible for emitting an ordered command. Maps nested in a command's filter +or other arguments are unaffected; single-key commands such as +`bson.M{"ping": 1}` are also accepted. A reply with `ok: 0`, `writeErrors` or a `writeConcernError` comes back together with a `*CommandError` holding its code, code name and message. `DB.Cursor` takes diff --git a/mongodb/command_cursor_test.go b/mongodb/command_cursor_test.go index c485300..4b625cf 100644 --- a/mongodb/command_cursor_test.go +++ b/mongodb/command_cursor_test.go @@ -11,6 +11,54 @@ import ( "go.mongodb.org/mongo-driver/v2/bson" ) +type orderedMapCommand map[string]any + +func (command orderedMapCommand) MarshalBSON() ([]byte, error) { + ordered := bson.D{{Key: "find", Value: command["find"]}, {Key: "filter", Value: command["filter"]}} + return bson.Marshal(ordered) +} + +type pointerMapCommand map[string]any + +func (command *pointerMapCommand) MarshalBSON() ([]byte, error) { + return orderedMapCommand(*command).MarshalBSON() +} + +func TestCommandsHonorOrderedMapMarshalers(t *testing.T) { + ordered := orderedMapCommand{"find": "users", "filter": bson.M{"active": true}} + pointer := pointerMapCommand(ordered) + inputs := []any{ordered, &ordered, &pointer} + for _, input := range inputs { + if _, err := bson.Marshal(input); err != nil { + t.Fatalf("fixture %T is not a supported BSON document: %v", input, err) + } + cursors := &cursorServer{t: t, batches: [][]string{{"a"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + db := r.db() + if _, err := db.Command(t.Context(), input); err != nil { + t.Fatalf("Command rejected ordered %T: %v", input, err) + } + var seen int + for document, err := range db.Cursor(t.Context(), input) { + if err != nil { + t.Fatalf("Cursor rejected ordered %T: %v", input, err) + } + if document.Lookup("name").StringValue() != "a" { + t.Fatal("wrong cursor document", document) + } + seen++ + } + if seen != 1 || len(r.commands) != 2 { + t.Fatal("wrong custom command dispatch", seen, r.commands) + } + for _, sent := range r.commands { + if got := command(t, sent)[0].Key; got != "find" { + t.Fatal("custom command order changed", got) + } + } + } +} + func TestCommandsRejectUnorderedMultiKeyMapsBeforeSending(t *testing.T) { type namedMap map[string]any unordered := bson.M{"find": "users", "filter": bson.M{"active": true}} @@ -199,3 +247,31 @@ func TestCanceledCursorStillCleansUp(t *testing.T) { t.Fatal("caller cancellation prevented cursor cleanup", cursors.killed) } } + +func TestGetMoreFailureCleansUpBeforeYieldingError(t *testing.T) { + for _, interrupted := range []bool{false, true} { + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}}, failAt: 1} + r := &recorder{t: t} + r.answer = func(sent *pb.Command) []*pb.Event { + if interrupted && command(t, sent)[0].Key == "getMore" { + return storetest.NativeIncomplete(nil, nil, pb.FailureCode_UNAVAILABLE) + } + return cursors.answer(sent) + } + db := r.db() + find := bson.D{{Key: "find", Value: "users"}} + var failures int + for _, err := range db.Cursor(t.Context(), find) { + if err == nil { + continue + } + failures++ + if len(cursors.killed) != 1 { + t.Errorf("getMore error yielded before cleanup: interrupted=%t kills=%d", interrupted, len(cursors.killed)) + } + } + if failures != 1 || len(cursors.killed) != 1 { + t.Fatal("expected one failure and exactly one cleanup", failures, cursors.killed) + } + } +} diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index c5f90f8..ef63017 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -74,8 +74,9 @@ func (e *CommandError) Error() string { } // Command runs one command in the database and returns its reply. Commands -// must preserve field order, with the command name first; multi-key maps are -// rejected before sending. Use bson.D or another ordered BSON representation. +// must preserve field order, with the command name first; multi-key maps without +// custom BSON encoders are rejected before sending. Use bson.D or another +// ordered BSON representation. Custom BSON encoders own their field order. // A reply with ok: 0, writeErrors or a writeConcernError comes with a *CommandError. // A command is sent again only when a server refused it before it began. One // a lost connection or an unavailable backend cut short reports @@ -89,12 +90,15 @@ func (db DB) Command(ctx context.Context, command any) (bson.Raw, error) { } func marshalCommand(command any) (bson.Raw, error) { - value := reflect.ValueOf(command) - for value.IsValid() && (value.Kind() == reflect.Pointer || value.Kind() == reflect.Interface) { - value = value.Elem() - } - if value.IsValid() && value.Kind() == reflect.Map && value.Len() > 1 { - return nil, errors.New("mongodb: command must be ordered; use bson.D instead of a multi-key map") + // bson.Marshal gives an explicit document encoder control over field order. + if _, custom := command.(bson.Marshaler); !custom { + value := reflect.ValueOf(command) + for value.IsValid() && (value.Kind() == reflect.Pointer || value.Kind() == reflect.Interface) { + value = value.Elem() + } + if value.IsValid() && value.Kind() == reflect.Map && value.Len() > 1 { + return nil, errors.New("mongodb: command must be ordered; use bson.D instead of a multi-key map") + } } data, err := bson.Marshal(command) if err != nil { @@ -163,6 +167,8 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] next, err = readPage(reply) } if err != nil { + db.killCursor(ctx, space, cursor, session) + cursor = 0 yield(nil, err) return } From 1e6cc14697d9fee6f11e2a2caf24c3f27145744f Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Tue, 17 Nov 2026 16:18:19 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=90=9B=20fix:=20report=20skipped=20cl?= =?UTF-8?q?usters=20and=20reject=20Search=20responses=20without=20hits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/search.md | 24 +++++++++++++++--------- search/index.go | 35 ++++++++++++++++++++++++++--------- search/search.go | 18 +++++++++++++++--- search/search_test.go | 41 +++++++++++++++++++++++++++++++++++++++-- 4 files changed, 95 insertions(+), 23 deletions(-) diff --git a/docs/search.md b/docs/search.md index 6dce5c0..9bba78a 100644 --- a/docs/search.md +++ b/docs/search.md @@ -81,11 +81,13 @@ if len(hits) > 0 { ``` A successful HTTP status does not guarantee a complete search. If the backend -reports `timed_out: true` or `_shards.failed > 0`, `Search` returns a -`*search.PartialResponseError` **and any decoded hits**. An empty hit slice in -this case does not mean there were no matches. A caller requiring complete -results must reject these hits; a best-effort caller may explicitly choose to -use them after inspecting the error with `errors.As`: +reports `timed_out: true`, `_shards.failed > 0`, or, for a cross-cluster search, +a nonzero `_clusters.skipped`, `_clusters.partial` or `_clusters.failed`, +`Search` returns a `*search.PartialResponseError` **and any decoded hits**. A +remote cluster skipped under `skip_unavailable` is reported this way. An empty +hit slice in this case does not mean there were no matches. A caller requiring +complete results must reject these hits; a best-effort caller may explicitly +choose to use them after inspecting the error with `errors.As`: ```go hits, err := products.Search(ctx, query) @@ -100,9 +102,13 @@ if err != nil { `TimedOut` and `FailedShards` report the backend's completion metadata. `ShardFailures` holds each `_shards.failures` entry as `json.RawMessage`, preserving its shard, index and backend-specific reason details. The backend -may omit these entries even when `FailedShards` is nonzero. This error does not -trigger an automatic retry. Complete responses still return hits and a nil -error; HTTP status and JSON decoding errors retain their existing behavior. +may omit these entries even when `FailedShards` is nonzero. `SkippedClusters`, +`PartialClusters` and `FailedClusters` report the `_clusters` counts and are +zero for a search of one cluster. This error does not trigger an automatic +retry. Complete responses still return hits and a nil error; HTTP status and +JSON decoding errors retain their existing behavior. A successful response +without a `hits.hits` array, such as `{}`, is a decoding error rather than an +empty result. ## HTTP requests @@ -137,7 +143,7 @@ as it is for a search. | `*weir.Error` | A record failed. `weir.CodeOf(err)` gives its `FailureCode`, and `Outcome` tells whether a write applied. | | `weir.ErrInterrupted` | An outage cut the request short, so it may succeed later. It may have run: never send a write again without checking. | | `*search.StatusError` | `Search` got a status of 300 or more | -| `*search.PartialResponseError` | `Search` timed out or had failed shards despite a successful HTTP status. Any returned hits may be incomplete. | +| `*search.PartialResponseError` | `Search` timed out, had failed shards or did not search every cluster despite a successful HTTP status. Any returned hits may be incomplete. | A write certainly took no effect only when its `*weir.Error` has the `Outcome` `MutationNotStarted` or `MutationNotApplied`, or, for a request, the diff --git a/search/index.go b/search/index.go index ee7bd49..4f39c81 100644 --- a/search/index.go +++ b/search/index.go @@ -128,9 +128,11 @@ func (i Index[T]) Transform(ctx context.Context, id string, program string, inpu // Search sends a search request body, such as {"query": ..., "size": 10}, to // the index and returns its hits. A status of 300 or more is a *StatusError. -// A timed-out response or failed shards returns the decoded hits alongside a +// A timed-out response, failed shards, or a skipped, partial or failed cluster +// of a cross-cluster search returns the decoded hits alongside a // *PartialResponseError, even with a successful HTTP status. These hits may be -// incomplete, including an empty slice that does not establish no matches. +// incomplete, including an empty slice that does not establish no matches. A +// response without a hits.hits array is a decoding error, not an empty result. // Sort holds a hit's sort values exactly, to pass back as search_after. func (i Index[T]) Search(ctx context.Context, body any) ([]Hit[T], error) { response, err := i.Do(ctx, http.MethodPost, "/_search", body) @@ -147,22 +149,37 @@ func (i Index[T]) Search(ctx context.Context, body any) ([]Hit[T], error) { Failed int `json:"failed"` Failures []json.RawMessage `json:"failures"` } `json:"_shards"` + // Clusters is present only for a cross-cluster search. + Clusters struct { + Skipped int `json:"skipped"` + Partial int `json:"partial"` + Failed int `json:"failed"` + } `json:"_clusters"` + // Hits stays nil when the response has no hits.hits array or it is null. Hits struct { - Hits []Hit[T] `json:"hits"` + Hits *[]Hit[T] `json:"hits"` } `json:"hits"` } if err := json.Unmarshal(response.Body, &result); err != nil { return nil, fmt.Errorf("search: decode %s hits: %w", i.Name, err) } - if result.TimedOut || result.Shards.Failed > 0 { + if result.Hits.Hits == nil { + return nil, fmt.Errorf("search: decode %s hits: response has no hits.hits array", i.Name) + } + hits := *result.Hits.Hits + clusters := result.Clusters + if result.TimedOut || result.Shards.Failed > 0 || clusters.Skipped > 0 || clusters.Partial > 0 || clusters.Failed > 0 { partial := &PartialResponseError{ - TimedOut: result.TimedOut, - FailedShards: result.Shards.Failed, - ShardFailures: result.Shards.Failures, + TimedOut: result.TimedOut, + FailedShards: result.Shards.Failed, + ShardFailures: result.Shards.Failures, + SkippedClusters: clusters.Skipped, + PartialClusters: clusters.Partial, + FailedClusters: clusters.Failed, } - return result.Hits.Hits, partial + return hits, partial } - return result.Hits.Hits, nil + return hits, nil } // Do sends one HTTP request to the index and returns its response, whatever diff --git a/search/search.go b/search/search.go index 9561993..d974755 100644 --- a/search/search.go +++ b/search/search.go @@ -21,8 +21,9 @@ // with weir.CodeOf, and with errors.Is against weir.ErrNotFound and // weir.ErrInterrupted. The package keeps no state of its own; Cluster and // Index are plain values that are safe to copy and share. Search returns -// decoded hits alongside a *PartialResponseError when the backend times out -// or reports failed shards, even with a successful HTTP status. +// decoded hits alongside a *PartialResponseError when the backend times out, +// reports failed shards or does not search every cluster of a cross-cluster +// search, even with a successful HTTP status. package search import ( @@ -78,10 +79,21 @@ type PartialResponseError struct { // including any backend-specific reason details. It may be absent even // when FailedShards is nonzero. ShardFailures []json.RawMessage + // SkippedClusters, PartialClusters and FailedClusters count the clusters + // of a cross-cluster search that _clusters reports as skipped, such as an + // unavailable cluster with skip_unavailable, as returning partial results, + // or as failed. They are zero for a search of one cluster. + SkippedClusters int + PartialClusters int + FailedClusters int } func (e *PartialResponseError) Error() string { - return fmt.Sprintf("search: incomplete response: timed_out=%t, failed_shards=%d", e.TimedOut, e.FailedShards) + message := fmt.Sprintf("search: incomplete response: timed_out=%t, failed_shards=%d", e.TimedOut, e.FailedShards) + if e.SkippedClusters > 0 || e.PartialClusters > 0 || e.FailedClusters > 0 { + message += fmt.Sprintf(", skipped_clusters=%d, partial_clusters=%d, failed_clusters=%d", e.SkippedClusters, e.PartialClusters, e.FailedClusters) + } + return message } // Do sends one HTTP request to the cluster and returns its response, whatever diff --git a/search/search_test.go b/search/search_test.go index f32a76f..6a12714 100644 --- a/search/search_test.go +++ b/search/search_test.go @@ -7,6 +7,7 @@ import ( "errors" "io" "net/http" + "strings" "testing" weir "github.com/batchstream/weir-go" @@ -200,7 +201,9 @@ func TestSearchReportsPartialResponsesAndPreservesHits(t *testing.T) { timedOut bool failedShards int shardFailures []string - hitCount int + // clusters holds the skipped, partial and failed cluster counts. + clusters [3]int + hitCount int }{ { name: "timeout with empty hits", @@ -227,11 +230,27 @@ func TestSearchReportsPartialResponsesAndPreservesHits(t *testing.T) { body: `{"_shards":{"failed":2},"hits":{"hits":[]}}`, failedShards: 2, }, + { + name: "skipped remote cluster", + body: `{"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"_clusters":{"total":2,"successful":1,"skipped":1},"hits":{"hits":[` + hit + `]}}`, + clusters: [3]int{1, 0, 0}, + hitCount: 1, + }, + { + name: "partial and failed clusters", + body: `{"timed_out":false,"_shards":{"total":2,"successful":2,"failed":0},"_clusters":{"total":3,"successful":1,"skipped":0,"running":0,"partial":1,"failed":1},"hits":{"hits":[]}}`, + clusters: [3]int{0, 1, 1}, + }, { name: "complete response", body: `{"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":1,"failed":0},"hits":{"hits":[` + hit + `]}}`, hitCount: 1, }, + { + name: "complete cross-cluster response", + body: `{"timed_out":false,"_shards":{"total":2,"successful":2,"failed":0},"_clusters":{"total":2,"successful":2,"skipped":0,"running":0,"partial":0,"failed":0},"hits":{"hits":[` + hit + `]}}`, + hitCount: 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -240,7 +259,7 @@ func TestSearchReportsPartialResponsesAndPreservesHits(t *testing.T) { products := r.products() query := map[string]any{"query": map[string]any{"match_all": map[string]any{}}} found, err := products.Search(t.Context(), query) - if tt.timedOut || tt.failedShards > 0 { + if tt.timedOut || tt.failedShards > 0 || tt.clusters != [3]int{} { var partial *PartialResponseError if !errors.As(err, &partial) { t.Fatalf("expected *PartialResponseError, got %v", err) @@ -248,6 +267,10 @@ func TestSearchReportsPartialResponsesAndPreservesHits(t *testing.T) { if partial.TimedOut != tt.timedOut || partial.FailedShards != tt.failedShards { t.Fatal("wrong incompleteness details", partial) } + clusters := [3]int{partial.SkippedClusters, partial.PartialClusters, partial.FailedClusters} + if clusters != tt.clusters { + t.Fatal("wrong cluster details", partial) + } if len(partial.ShardFailures) != len(tt.shardFailures) { t.Fatal("wrong shard failures", partial.ShardFailures) } @@ -274,6 +297,20 @@ func TestSearchReportsPartialResponsesAndPreservesHits(t *testing.T) { } } +func TestSearchRejectsResponsesWithoutHits(t *testing.T) { + bodies := []string{`null`, `{}`, `{"hits":null}`, `{"hits":{}}`, `{"hits":{"hits":null}}`, `{"timed_out":true}`} + for _, body := range bodies { + r := &recorder{t: t} + r.answer = func(*pb.Command) []*pb.Event { return httpResponse("200 OK", body) } + products := r.products() + query := map[string]any{"query": map[string]any{"match_all": map[string]any{}}} + found, err := products.Search(t.Context(), query) + if err == nil || found != nil || !strings.Contains(err.Error(), "no hits.hits array") { + t.Fatalf("accepted %s: %v, %v", body, found, err) + } + } +} + func TestSearchDecodesHitsAndKeepsSortValuesExact(t *testing.T) { hits := `{"hits":{"total":{"value":2},"hits":[ {"_id":"p1","_score":null,"sort":[9007199254740993,"lamp"],"_source":{"name":"lamp","price":30}}, From a41c61e4a51e372b961645d7c3f6a8ed964cdcc7 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Tue, 17 Nov 2026 16:18:19 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=90=9B=20fix:=20repeat=20Stable=20API?= =?UTF-8?q?=20fields=20and=20batch=20size=20on=20cursor=20continuations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/mongodb.md | 10 ++-- mongodb/command_cursor_test.go | 97 ++++++++++++++++++++++++++++++++-- mongodb/mongodb.go | 71 ++++++++++++++++++------- 3 files changed, 154 insertions(+), 24 deletions(-) diff --git a/docs/mongodb.md b/docs/mongodb.md index 46d2035..6a3a0f0 100644 --- a/docs/mongodb.md +++ b/docs/mongodb.md @@ -112,9 +112,13 @@ A reply with `ok: 0`, `writeErrors` or a `writeConcernError` comes back together with a `*CommandError` holding its code, code name and message. `DB.Cursor` takes an ordered command that opens a cursor, such as `aggregate` or `listIndexes`, and yields its documents the way `Find` does. Its `getMore` and cleanup commands -retain an explicit `lsid`, `txnNumber` and `autocommit` from the opening command, -but do not repeat `startTransaction`. The cursor is cleaned up if the loop exits -early, returns an error or panics, even after the caller's context is canceled. +repeat an explicit `lsid`, `txnNumber`, `autocommit`, `apiVersion`, `apiStrict` +and `apiDeprecationErrors` from the opening command, but do not repeat +`startTransaction`. Each `getMore` also repeats a positive batch size, from +`batchSize` for `find` or `cursor.batchSize` for `aggregate` and the list +commands, so later batches stay as small as the first. The cursor is cleaned up +if the loop exits early, returns an error or panics, even after the caller's +context is canceled. ```go create := bson.D{{Key: "create", Value: "users"}} diff --git a/mongodb/command_cursor_test.go b/mongodb/command_cursor_test.go index 4b625cf..d26f0b6 100644 --- a/mongodb/command_cursor_test.go +++ b/mongodb/command_cursor_test.go @@ -3,6 +3,7 @@ package mongodb import ( "bytes" "context" + "slices" "strings" "testing" @@ -146,8 +147,16 @@ func TestCursorPreservesExplicitSessionAndTransaction(t *testing.T) { id := bson.Binary{Subtype: 4, Data: []byte("0123456789abcdef")} session := bson.D{{Key: "id", Value: id}} concern := bson.D{{Key: "level", Value: "majority"}} - find := bson.D{{Key: "find", Value: "users"}, {Key: "lsid", Value: session}, {Key: "batchSize", Value: 1}, {Key: "readConcern", Value: concern}} - keys := []string{"lsid"} + find := bson.D{ + {Key: "find", Value: "users"}, + {Key: "lsid", Value: session}, + {Key: "batchSize", Value: 1}, + {Key: "readConcern", Value: concern}, + {Key: "apiVersion", Value: "1"}, + {Key: "apiStrict", Value: true}, + {Key: "apiDeprecationErrors", Value: true}, + } + keys := []string{"lsid", "apiVersion", "apiStrict", "apiDeprecationErrors"} if test.transaction { transaction := bson.D{{Key: "txnNumber", Value: int64(7)}, {Key: "autocommit", Value: false}, {Key: "startTransaction", Value: true}} find = append(find, transaction...) @@ -186,11 +195,24 @@ func TestCursorPreservesExplicitSessionAndTransaction(t *testing.T) { t.Fatalf("lost %s in continuation: %v", key, raw) } } - for _, key := range []string{"startTransaction", "readConcern", "batchSize"} { + for _, key := range []string{"startTransaction", "readConcern"} { if raw.Lookup(key).Type != 0 { t.Fatalf("copied initial-only %s into continuation: %v", key, raw) } } + want, got := original.Lookup("batchSize"), raw.Lookup("batchSize") + switch name := command(t, sent)[0].Key; name { + case "getMore": + if got.Type != want.Type || !bytes.Equal(got.Value, want.Value) { + t.Fatalf("getMore lost batchSize: %v", raw) + } + case "killCursors": + if got.Type != 0 { + t.Fatalf("copied batchSize into cleanup: %v", raw) + } + default: + t.Fatal("unexpected continuation", name) + } } if got := cursors.killed[0][1].Value.(bson.A)[0]; got != int64(2) { t.Fatal("did not clean up the latest cursor", got) @@ -202,6 +224,75 @@ func TestCursorPreservesExplicitSessionAndTransaction(t *testing.T) { } } +func TestGetMoreRepeatsAPositiveBatchSize(t *testing.T) { + pipeline := bson.E{Key: "pipeline", Value: bson.A{}} + tests := []struct { + name string + command bson.D + want any + }{ + { + name: "find", + command: bson.D{{Key: "find", Value: "users"}, {Key: "batchSize", Value: int64(2)}}, + want: int64(2), + }, + { + name: "aggregate", + command: bson.D{{Key: "aggregate", Value: "users"}, pipeline, {Key: "cursor", Value: bson.D{{Key: "batchSize", Value: int32(3)}}}}, + want: int32(3), + }, + { + name: "server default", + command: bson.D{{Key: "aggregate", Value: "users"}, pipeline, {Key: "cursor", Value: bson.D{}}}, + }, + { + name: "empty first batch", + command: bson.D{{Key: "find", Value: "users"}, {Key: "batchSize", Value: int32(0)}}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}, {"c"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + for _, err := range r.db().Cursor(t.Context(), test.command) { + if err != nil { + t.Fatal(err) + } + } + if len(r.commands) != 3 { + t.Fatal("expected the opening command and two getMores", r.commands) + } + for _, sent := range r.commands[1:] { + var size any + for _, field := range command(t, sent) { + if field.Key == "batchSize" { + size = field.Value + } + } + if size != test.want { + t.Fatalf("getMore batchSize = %#v, want %#v", size, test.want) + } + } + }) + } +} + +func TestCursorFieldsDoNotRetainTheEncodedCommand(t *testing.T) { + session := bson.D{{Key: "id", Value: "session"}} + find := bson.D{{Key: "find", Value: "users"}, {Key: "lsid", Value: session}, {Key: "apiVersion", Value: "1"}, {Key: "batchSize", Value: int32(5)}} + data := bson.Raw(bsonDocument(t, find).Data) + fields := readCursorFields(data) + kept := slices.Concat(fields.shared, fields.batchSize) + if len(kept) != 3 { + t.Fatal("wrong cursor fields", kept) + } + want := bsonDocument(t, kept).Data + clear(data) + if got := bsonDocument(t, kept).Data; !bytes.Equal(got, want) { + t.Fatal("cursor fields share memory with the encoded command") + } +} + func TestCursorPanicCleansUpLatestCursor(t *testing.T) { cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}, {"c"}}, failAt: -1} r := &recorder{t: t, answer: cursors.answer} diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index ef63017..f756001 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -26,6 +26,7 @@ package mongodb import ( + "bytes" "context" "errors" "fmt" @@ -125,10 +126,11 @@ func (db DB) command(ctx context.Context, data bson.Raw) (bson.Raw, error) { } // Cursor runs an ordered command that opens a cursor, such as find, aggregate -// or listIndexes, and yields its documents. It preserves explicit session and -// transaction fields on getMore and cleanup commands. It sends getMore until -// the cursor ends, and kills the cursor when the loop stops early, fails or -// panics. An error ends the sequence. +// or listIndexes, and yields its documents. Its getMore and cleanup commands +// repeat the explicit session, transaction and Stable API fields of the +// command, and each getMore repeats a positive batch size. It sends getMore +// until the cursor ends, and kills the cursor when the loop stops early, fails +// or panics. An error ends the sequence. func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] { documents := func(yield func(bson.Raw, error) bool) { data, err := marshalCommand(command) @@ -136,7 +138,7 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] yield(nil, err) return } - session := cursorSession(data) + fields := readCursorFields(data) reply, err := db.command(ctx, data) if err != nil { yield(nil, err) @@ -149,7 +151,7 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] } space := parseNamespace(page.Cursor.Namespace) batch, cursor := page.Cursor.FirstBatch, page.Cursor.ID - defer func() { db.killCursor(ctx, space, cursor, session) }() + defer func() { db.killCursor(ctx, space, cursor, fields.shared) }() for { for _, document := range batch { if !yield(document, nil) { @@ -160,14 +162,15 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] return } getMore := bson.D{{Key: "getMore", Value: cursor}, {Key: "collection", Value: space.collection}, {Key: "$db", Value: space.database}} - getMore = append(getMore, session...) + getMore = append(getMore, fields.batchSize...) + getMore = append(getMore, fields.shared...) reply, err := db.Command(ctx, getMore) var next cursorPage if err == nil { next, err = readPage(reply) } if err != nil { - db.killCursor(ctx, space, cursor, session) + db.killCursor(ctx, space, cursor, fields.shared) cursor = 0 yield(nil, err) return @@ -178,18 +181,50 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] return documents } -// cursorSession retains only fields needed to continue the original session -// or transaction. In particular, startTransaction belongs to the first command. -func cursorSession(command bson.Raw) bson.D { - var session bson.D - for _, key := range []string{"lsid", "txnNumber", "autocommit"} { +// sharedCursorKeys name the fields of a cursor's opening command that its +// getMore and killCursors commands repeat: the explicit session, transaction +// and Stable API fields, which MongoDB requires to stay the same. The +// startTransaction field belongs to the first command only. +var sharedCursorKeys = []string{"lsid", "txnNumber", "autocommit", "apiVersion", "apiStrict", "apiDeprecationErrors"} + +// cursorFields are the fields of a cursor's opening command that the commands +// following it repeat. +type cursorFields struct { + // shared go on every getMore and killCursors. + shared bson.D + // batchSize goes on every getMore, as MongoDB applies a batch size only to + // the reply of the command that names it. + batchSize bson.D +} + +func readCursorFields(command bson.Raw) cursorFields { + var fields cursorFields + for _, key := range sharedCursorKeys { value := command.Lookup(key) if value.Type != 0 { - field := bson.E{Key: key, Value: value} - session = append(session, field) + field := bson.E{Key: key, Value: detach(value)} + fields.shared = append(fields.shared, field) } } - return session + // find names its batch size at the top level; aggregate and the list + // commands name it in their cursor option. A size of 0 asks only for an + // empty first batch, so it is not repeated. + size := command.Lookup("batchSize") + if size.Type == 0 { + size = command.Lookup("cursor", "batchSize") + } + if n, ok := size.AsInt64OK(); ok && n > 0 { + field := bson.E{Key: "batchSize", Value: detach(size)} + fields.batchSize = bson.D{field} + } + return fields +} + +// detach copies a value out of the command it was read from, so a cursor does +// not keep the whole encoded command alive while it runs. +func detach(value bson.RawValue) bson.RawValue { + copied := bson.RawValue{Type: value.Type, Value: bytes.Clone(value.Value)} + return copied } // namespace names the collection of a cursor. Commands that follow the cursor @@ -230,14 +265,14 @@ func readPage(reply bson.Raw) (cursorPage, error) { // killCursor releases a cursor a loop left open. The server also times idle // cursors out, so a failure here is ignored. -func (db DB) killCursor(ctx context.Context, space namespace, cursor int64, session bson.D) { +func (db DB) killCursor(ctx context.Context, space namespace, cursor int64, shared bson.D) { if cursor == 0 { return } cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), killTimeout) defer cancel() command := bson.D{{Key: "killCursors", Value: space.collection}, {Key: "cursors", Value: bson.A{cursor}}, {Key: "$db", Value: space.database}} - command = append(command, session...) + command = append(command, shared...) _, _ = db.Command(cleanup, command) } From f641949c15ab4fb0da53c51d449fd5e2e394dc5b Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Tue, 17 Nov 2026 16:32:26 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=90=9B=20fix:=20repeat=20txnNumber=20?= =?UTF-8?q?on=20cursor=20continuations=20only=20within=20transactions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/mongodb.md | 16 +++++++------ mongodb/command_cursor_test.go | 43 ++++++++++++++++++++++++++++++++++ mongodb/mongodb.go | 28 +++++++++++++++------- 3 files changed, 72 insertions(+), 15 deletions(-) diff --git a/docs/mongodb.md b/docs/mongodb.md index 6a3a0f0..ad8f30b 100644 --- a/docs/mongodb.md +++ b/docs/mongodb.md @@ -112,13 +112,15 @@ A reply with `ok: 0`, `writeErrors` or a `writeConcernError` comes back together with a `*CommandError` holding its code, code name and message. `DB.Cursor` takes an ordered command that opens a cursor, such as `aggregate` or `listIndexes`, and yields its documents the way `Find` does. Its `getMore` and cleanup commands -repeat an explicit `lsid`, `txnNumber`, `autocommit`, `apiVersion`, `apiStrict` -and `apiDeprecationErrors` from the opening command, but do not repeat -`startTransaction`. Each `getMore` also repeats a positive batch size, from -`batchSize` for `find` or `cursor.batchSize` for `aggregate` and the list -commands, so later batches stay as small as the first. The cursor is cleaned up -if the loop exits early, returns an error or panics, even after the caller's -context is canceled. +repeat an explicit `lsid`, `apiVersion`, `apiStrict` and `apiDeprecationErrors` +from the opening command. Within a multi-document transaction, marked by +`autocommit: false`, they also repeat `txnNumber` and `autocommit`, but not +`startTransaction`. The `txnNumber` of a retryable write, such as `bulkWrite`, +is not repeated, as MongoDB accepts it only on the write itself. Each `getMore` +also repeats a positive batch size, from `batchSize` for `find` or +`cursor.batchSize` for `aggregate` and the list commands, so later batches stay +as small as the first. The cursor is cleaned up if the loop exits early, +returns an error or panics, even after the caller's context is canceled. ```go create := bson.D{{Key: "create", Value: "users"}} diff --git a/mongodb/command_cursor_test.go b/mongodb/command_cursor_test.go index d26f0b6..4ec4050 100644 --- a/mongodb/command_cursor_test.go +++ b/mongodb/command_cursor_test.go @@ -224,6 +224,49 @@ func TestCursorPreservesExplicitSessionAndTransaction(t *testing.T) { } } +func TestCursorDoesNotRepeatARetryableWriteTxnNumber(t *testing.T) { + id := bson.Binary{Subtype: 4, Data: []byte("0123456789abcdef")} + session := bson.D{{Key: "id", Value: id}} + insert := bson.D{{Key: "insert", Value: 0}, {Key: "document", Value: bson.D{{Key: "name", Value: "a"}}}} + namespace := bson.D{{Key: "ns", Value: "app.users"}} + bulkWrite := bson.D{ + {Key: "bulkWrite", Value: 1}, + {Key: "ops", Value: bson.A{insert, insert}}, + {Key: "nsInfo", Value: bson.A{namespace}}, + {Key: "cursor", Value: bson.D{{Key: "batchSize", Value: int32(1)}}}, + {Key: "lsid", Value: session}, + {Key: "txnNumber", Value: int64(1)}, + } + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}, {"c"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + seen := 0 + for _, err := range r.db().Cursor(t.Context(), bulkWrite) { + if err != nil { + t.Fatal(err) + } + seen++ + if seen == 2 { + break + } + } + if len(r.commands) != 3 || len(cursors.killed) != 1 { + t.Fatal("expected bulkWrite, getMore and one cleanup", r.commands) + } + original := bson.Raw(r.commands[0].GetNative().GetRequest().GetData()) + for _, sent := range r.commands[1:] { + raw := bson.Raw(sent.GetNative().GetRequest().GetData()) + want, got := original.Lookup("lsid"), raw.Lookup("lsid") + if got.Type != want.Type || !bytes.Equal(got.Value, want.Value) { + t.Fatalf("lost lsid in continuation: %v", raw) + } + for _, key := range []string{"txnNumber", "autocommit"} { + if raw.Lookup(key).Type != 0 { + t.Fatalf("repeated a retryable write's %s: %v", key, raw) + } + } + } +} + func TestGetMoreRepeatsAPositiveBatchSize(t *testing.T) { pipeline := bson.E{Key: "pipeline", Value: bson.A{}} tests := []struct { diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index f756001..4e19e4d 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -32,6 +32,7 @@ import ( "fmt" "iter" "reflect" + "slices" "strings" "time" @@ -127,10 +128,11 @@ func (db DB) command(ctx context.Context, data bson.Raw) (bson.Raw, error) { // Cursor runs an ordered command that opens a cursor, such as find, aggregate // or listIndexes, and yields its documents. Its getMore and cleanup commands -// repeat the explicit session, transaction and Stable API fields of the -// command, and each getMore repeats a positive batch size. It sends getMore -// until the cursor ends, and kills the cursor when the loop stops early, fails -// or panics. An error ends the sequence. +// repeat the explicit session and Stable API fields of the command, and its +// transaction fields within a multi-document transaction. Each getMore also +// repeats a positive batch size. It sends getMore until the cursor ends, and +// kills the cursor when the loop stops early, fails or panics. An error ends +// the sequence. func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] { documents := func(yield func(bson.Raw, error) bool) { data, err := marshalCommand(command) @@ -182,10 +184,16 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] } // sharedCursorKeys name the fields of a cursor's opening command that its -// getMore and killCursors commands repeat: the explicit session, transaction -// and Stable API fields, which MongoDB requires to stay the same. The +// getMore and killCursors commands repeat: the explicit session and Stable API +// fields, which MongoDB requires to stay the same. +var sharedCursorKeys = []string{"lsid", "apiVersion", "apiStrict", "apiDeprecationErrors"} + +// transactionCursorKeys name the fields that getMore and killCursors also +// repeat inside a multi-document transaction, which autocommit: false marks. +// A retryable write such as bulkWrite names a txnNumber too, but MongoDB +// rejects one on getMore and killCursors outside a transaction. The // startTransaction field belongs to the first command only. -var sharedCursorKeys = []string{"lsid", "txnNumber", "autocommit", "apiVersion", "apiStrict", "apiDeprecationErrors"} +var transactionCursorKeys = []string{"txnNumber", "autocommit"} // cursorFields are the fields of a cursor's opening command that the commands // following it repeat. @@ -199,7 +207,11 @@ type cursorFields struct { func readCursorFields(command bson.Raw) cursorFields { var fields cursorFields - for _, key := range sharedCursorKeys { + keys := sharedCursorKeys + if autocommit, found := command.Lookup("autocommit").BooleanOK(); found && !autocommit { + keys = slices.Concat(sharedCursorKeys, transactionCursorKeys) + } + for _, key := range keys { value := command.Lookup(key) if value.Type != 0 { field := bson.E{Key: key, Value: detach(value)}