diff --git a/docs/mongodb.md b/docs/mongodb.md index dcfbffd..ad8f30b 100644 --- a/docs/mongodb.md +++ b/docs/mongodb.md @@ -99,11 +99,28 @@ estimate of a whole collection, run the `count` command with `DB.Command`. ## Commands -`DB.Command` runs any command in the database and returns its reply. A reply with -`ok: 0`, `writeErrors` or a `writeConcernError` comes back together with a -`*CommandError` holding its code, code name and message. `DB.Cursor` runs a -command that opens a cursor, such as `aggregate` or `listIndexes`, and yields its -documents the way `Find` does. +`DB.Command` runs any command in the database and returns its reply. Put the +command name first in an ordered document such as `bson.D` or `bson.Raw`. +Multi-key maps without a custom BSON encoder, including ordinary `bson.M`, are +rejected before sending because MongoDB uses the first key to identify the +command. Custom `bson.Marshaler` implementations remain supported and are +responsible for emitting an ordered command. Maps nested in a command's filter +or other arguments are unaffected; single-key commands such as +`bson.M{"ping": 1}` are also accepted. + +A reply with `ok: 0`, `writeErrors` or a `writeConcernError` comes back together +with a `*CommandError` holding its code, code name and message. `DB.Cursor` takes +an ordered command that opens a cursor, such as `aggregate` or `listIndexes`, +and yields its documents the way `Find` does. Its `getMore` and cleanup commands +repeat an explicit `lsid`, `apiVersion`, `apiStrict` and `apiDeprecationErrors` +from the opening command. Within a multi-document transaction, marked by +`autocommit: false`, they also repeat `txnNumber` and `autocommit`, but not +`startTransaction`. The `txnNumber` of a retryable write, such as `bulkWrite`, +is not repeated, as MongoDB accepts it only on the write itself. Each `getMore` +also repeats a positive batch size, from `batchSize` for `find` or +`cursor.batchSize` for `aggregate` and the list commands, so later batches stay +as small as the first. The cursor is cleaned up if the loop exits early, +returns an error or panics, even after the caller's context is canceled. ```go create := bson.D{{Key: "create", Value: "users"}} diff --git a/docs/search.md b/docs/search.md index d2fe9e0..9bba78a 100644 --- a/docs/search.md +++ b/docs/search.md @@ -80,6 +80,36 @@ if len(hits) > 0 { } ``` +A successful HTTP status does not guarantee a complete search. If the backend +reports `timed_out: true`, `_shards.failed > 0`, or, for a cross-cluster search, +a nonzero `_clusters.skipped`, `_clusters.partial` or `_clusters.failed`, +`Search` returns a `*search.PartialResponseError` **and any decoded hits**. A +remote cluster skipped under `skip_unavailable` is reported this way. An empty +hit slice in this case does not mean there were no matches. A caller requiring +complete results must reject these hits; a best-effort caller may explicitly +choose to use them after inspecting the error with `errors.As`: + +```go +hits, err := products.Search(ctx, query) +if err != nil { + var partial *search.PartialResponseError + if !errors.As(err, &partial) { return err } + // This caller accepts incomplete results; hits may still be empty. + log.Printf("partial search: timed_out=%t failed_shards=%d", partial.TimedOut, partial.FailedShards) +} +``` + +`TimedOut` and `FailedShards` report the backend's completion metadata. +`ShardFailures` holds each `_shards.failures` entry as `json.RawMessage`, +preserving its shard, index and backend-specific reason details. The backend +may omit these entries even when `FailedShards` is nonzero. `SkippedClusters`, +`PartialClusters` and `FailedClusters` report the `_clusters` counts and are +zero for a search of one cluster. This error does not trigger an automatic +retry. Complete responses still return hits and a nil error; HTTP status and +JSON decoding errors retain their existing behavior. A successful response +without a `hits.hits` array, such as `{}`, is a decoding error rather than an +empty result. + ## HTTP requests `Index.Do` sends any HTTP request with a path relative to the index, such as @@ -113,6 +143,7 @@ as it is for a search. | `*weir.Error` | A record failed. `weir.CodeOf(err)` gives its `FailureCode`, and `Outcome` tells whether a write applied. | | `weir.ErrInterrupted` | An outage cut the request short, so it may succeed later. It may have run: never send a write again without checking. | | `*search.StatusError` | `Search` got a status of 300 or more | +| `*search.PartialResponseError` | `Search` timed out, had failed shards or did not search every cluster despite a successful HTTP status. Any returned hits may be incomplete. | A write certainly took no effect only when its `*weir.Error` has the `Outcome` `MutationNotStarted` or `MutationNotApplied`, or, for a request, the diff --git a/mongodb/command_cursor_test.go b/mongodb/command_cursor_test.go new file mode 100644 index 0000000..4ec4050 --- /dev/null +++ b/mongodb/command_cursor_test.go @@ -0,0 +1,411 @@ +package mongodb + +import ( + "bytes" + "context" + "slices" + "strings" + "testing" + + "github.com/batchstream/weir-go/internal/storetest" + pb "github.com/batchstream/weir-protocol/api/weir/v1" + "go.mongodb.org/mongo-driver/v2/bson" +) + +type orderedMapCommand map[string]any + +func (command orderedMapCommand) MarshalBSON() ([]byte, error) { + ordered := bson.D{{Key: "find", Value: command["find"]}, {Key: "filter", Value: command["filter"]}} + return bson.Marshal(ordered) +} + +type pointerMapCommand map[string]any + +func (command *pointerMapCommand) MarshalBSON() ([]byte, error) { + return orderedMapCommand(*command).MarshalBSON() +} + +func TestCommandsHonorOrderedMapMarshalers(t *testing.T) { + ordered := orderedMapCommand{"find": "users", "filter": bson.M{"active": true}} + pointer := pointerMapCommand(ordered) + inputs := []any{ordered, &ordered, &pointer} + for _, input := range inputs { + if _, err := bson.Marshal(input); err != nil { + t.Fatalf("fixture %T is not a supported BSON document: %v", input, err) + } + cursors := &cursorServer{t: t, batches: [][]string{{"a"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + db := r.db() + if _, err := db.Command(t.Context(), input); err != nil { + t.Fatalf("Command rejected ordered %T: %v", input, err) + } + var seen int + for document, err := range db.Cursor(t.Context(), input) { + if err != nil { + t.Fatalf("Cursor rejected ordered %T: %v", input, err) + } + if document.Lookup("name").StringValue() != "a" { + t.Fatal("wrong cursor document", document) + } + seen++ + } + if seen != 1 || len(r.commands) != 2 { + t.Fatal("wrong custom command dispatch", seen, r.commands) + } + for _, sent := range r.commands { + if got := command(t, sent)[0].Key; got != "find" { + t.Fatal("custom command order changed", got) + } + } + } +} + +func TestCommandsRejectUnorderedMultiKeyMapsBeforeSending(t *testing.T) { + type namedMap map[string]any + unordered := bson.M{"find": "users", "filter": bson.M{"active": true}} + plain := map[string]any{"find": "users", "limit": 1} + named := namedMap{"find": "users", "limit": 1} + pointer := &unordered + inputs := []any{unordered, &unordered, &pointer, plain, &plain, named, &named} + r := &recorder{t: t} + r.answer = func(*pb.Command) []*pb.Event { + reply := bson.D{{Key: "ok", Value: 1}} + return storetest.Native(nil, bsonDocument(t, reply).Data) + } + db := r.db() + for _, input := range inputs { + _, err := db.Command(t.Context(), input) + if err == nil || !strings.Contains(err.Error(), "multi-key map") { + t.Fatalf("Command accepted unordered %T: %v", input, err) + } + var yielded int + for document, err := range db.Cursor(t.Context(), input) { + yielded++ + if document != nil || err == nil || !strings.Contains(err.Error(), "multi-key map") { + t.Fatalf("Cursor accepted unordered %T: %v, %v", input, document, err) + } + } + if yielded != 1 { + t.Fatalf("Cursor yielded %d results for invalid %T", yielded, input) + } + } + if len(r.commands) != 0 { + t.Fatal("unordered commands reached the server", r.commands) + } +} + +func TestCommandAcceptsOrderedCommandsAndSingleKeyMaps(t *testing.T) { + ordered := bson.D{{Key: "find", Value: "users"}, {Key: "filter", Value: bson.M{"active": true, "name": "Ada"}}} + raw := bson.Raw(bsonDocument(t, ordered).Data) + ping := bson.M{"ping": 1} + inputs := []any{ordered, &ordered, raw, &raw, ping, &ping} + r := &recorder{t: t} + r.answer = func(*pb.Command) []*pb.Event { + reply := bson.D{{Key: "ok", Value: 1}} + return storetest.Native(nil, bsonDocument(t, reply).Data) + } + db := r.db() + for index, input := range inputs { + if _, err := db.Command(t.Context(), input); err != nil { + t.Fatalf("rejected ordered %T: %v", input, err) + } + sent := command(t, r.commands[index]) + want := "find" + if index >= 4 { + want = "ping" + } + if sent[0].Key != want { + t.Fatalf("changed command order for %T: %v", input, sent) + } + } +} + +type marshaledCursorCommand struct { + value bson.D + calls int +} + +func (c *marshaledCursorCommand) MarshalBSON() ([]byte, error) { + c.calls++ + return bson.Marshal(c.value) +} + +func TestCursorPreservesExplicitSessionAndTransaction(t *testing.T) { + tests := []struct { + name string + transaction bool + raw bool + marshaler bool + }{ + {name: "session"}, + {name: "transaction", transaction: true}, + {name: "raw transaction", transaction: true, raw: true}, + {name: "marshaled transaction", transaction: true, marshaler: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + id := bson.Binary{Subtype: 4, Data: []byte("0123456789abcdef")} + session := bson.D{{Key: "id", Value: id}} + concern := bson.D{{Key: "level", Value: "majority"}} + find := bson.D{ + {Key: "find", Value: "users"}, + {Key: "lsid", Value: session}, + {Key: "batchSize", Value: 1}, + {Key: "readConcern", Value: concern}, + {Key: "apiVersion", Value: "1"}, + {Key: "apiStrict", Value: true}, + {Key: "apiDeprecationErrors", Value: true}, + } + keys := []string{"lsid", "apiVersion", "apiStrict", "apiDeprecationErrors"} + if test.transaction { + transaction := bson.D{{Key: "txnNumber", Value: int64(7)}, {Key: "autocommit", Value: false}, {Key: "startTransaction", Value: true}} + find = append(find, transaction...) + keys = append(keys, "txnNumber", "autocommit") + } + var input any = find + if test.raw { + input = bson.Raw(bsonDocument(t, find).Data) + } + custom := &marshaledCursorCommand{value: find} + if test.marshaler { + input = custom + } + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}, {"c"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + db := r.db() + seen := 0 + for _, err := range db.Cursor(t.Context(), input) { + if err != nil { + t.Fatal(err) + } + seen++ + if seen == 2 { + break + } + } + if len(r.commands) != 3 || len(cursors.killed) != 1 { + t.Fatal("expected find, getMore and one cleanup", r.commands) + } + original := bson.Raw(r.commands[0].GetNative().GetRequest().GetData()) + for _, sent := range r.commands[1:] { + raw := bson.Raw(sent.GetNative().GetRequest().GetData()) + for _, key := range keys { + want, got := original.Lookup(key), raw.Lookup(key) + if got.Type != want.Type || !bytes.Equal(got.Value, want.Value) { + t.Fatalf("lost %s in continuation: %v", key, raw) + } + } + for _, key := range []string{"startTransaction", "readConcern"} { + if raw.Lookup(key).Type != 0 { + t.Fatalf("copied initial-only %s into continuation: %v", key, raw) + } + } + want, got := original.Lookup("batchSize"), raw.Lookup("batchSize") + switch name := command(t, sent)[0].Key; name { + case "getMore": + if got.Type != want.Type || !bytes.Equal(got.Value, want.Value) { + t.Fatalf("getMore lost batchSize: %v", raw) + } + case "killCursors": + if got.Type != 0 { + t.Fatalf("copied batchSize into cleanup: %v", raw) + } + default: + t.Fatal("unexpected continuation", name) + } + } + if got := cursors.killed[0][1].Value.(bson.A)[0]; got != int64(2) { + t.Fatal("did not clean up the latest cursor", got) + } + if test.marshaler && custom.calls != 1 { + t.Fatal("encoded the original command more than once", custom.calls) + } + }) + } +} + +func TestCursorDoesNotRepeatARetryableWriteTxnNumber(t *testing.T) { + id := bson.Binary{Subtype: 4, Data: []byte("0123456789abcdef")} + session := bson.D{{Key: "id", Value: id}} + insert := bson.D{{Key: "insert", Value: 0}, {Key: "document", Value: bson.D{{Key: "name", Value: "a"}}}} + namespace := bson.D{{Key: "ns", Value: "app.users"}} + bulkWrite := bson.D{ + {Key: "bulkWrite", Value: 1}, + {Key: "ops", Value: bson.A{insert, insert}}, + {Key: "nsInfo", Value: bson.A{namespace}}, + {Key: "cursor", Value: bson.D{{Key: "batchSize", Value: int32(1)}}}, + {Key: "lsid", Value: session}, + {Key: "txnNumber", Value: int64(1)}, + } + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}, {"c"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + seen := 0 + for _, err := range r.db().Cursor(t.Context(), bulkWrite) { + if err != nil { + t.Fatal(err) + } + seen++ + if seen == 2 { + break + } + } + if len(r.commands) != 3 || len(cursors.killed) != 1 { + t.Fatal("expected bulkWrite, getMore and one cleanup", r.commands) + } + original := bson.Raw(r.commands[0].GetNative().GetRequest().GetData()) + for _, sent := range r.commands[1:] { + raw := bson.Raw(sent.GetNative().GetRequest().GetData()) + want, got := original.Lookup("lsid"), raw.Lookup("lsid") + if got.Type != want.Type || !bytes.Equal(got.Value, want.Value) { + t.Fatalf("lost lsid in continuation: %v", raw) + } + for _, key := range []string{"txnNumber", "autocommit"} { + if raw.Lookup(key).Type != 0 { + t.Fatalf("repeated a retryable write's %s: %v", key, raw) + } + } + } +} + +func TestGetMoreRepeatsAPositiveBatchSize(t *testing.T) { + pipeline := bson.E{Key: "pipeline", Value: bson.A{}} + tests := []struct { + name string + command bson.D + want any + }{ + { + name: "find", + command: bson.D{{Key: "find", Value: "users"}, {Key: "batchSize", Value: int64(2)}}, + want: int64(2), + }, + { + name: "aggregate", + command: bson.D{{Key: "aggregate", Value: "users"}, pipeline, {Key: "cursor", Value: bson.D{{Key: "batchSize", Value: int32(3)}}}}, + want: int32(3), + }, + { + name: "server default", + command: bson.D{{Key: "aggregate", Value: "users"}, pipeline, {Key: "cursor", Value: bson.D{}}}, + }, + { + name: "empty first batch", + command: bson.D{{Key: "find", Value: "users"}, {Key: "batchSize", Value: int32(0)}}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}, {"c"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + for _, err := range r.db().Cursor(t.Context(), test.command) { + if err != nil { + t.Fatal(err) + } + } + if len(r.commands) != 3 { + t.Fatal("expected the opening command and two getMores", r.commands) + } + for _, sent := range r.commands[1:] { + var size any + for _, field := range command(t, sent) { + if field.Key == "batchSize" { + size = field.Value + } + } + if size != test.want { + t.Fatalf("getMore batchSize = %#v, want %#v", size, test.want) + } + } + }) + } +} + +func TestCursorFieldsDoNotRetainTheEncodedCommand(t *testing.T) { + session := bson.D{{Key: "id", Value: "session"}} + find := bson.D{{Key: "find", Value: "users"}, {Key: "lsid", Value: session}, {Key: "apiVersion", Value: "1"}, {Key: "batchSize", Value: int32(5)}} + data := bson.Raw(bsonDocument(t, find).Data) + fields := readCursorFields(data) + kept := slices.Concat(fields.shared, fields.batchSize) + if len(kept) != 3 { + t.Fatal("wrong cursor fields", kept) + } + want := bsonDocument(t, kept).Data + clear(data) + if got := bsonDocument(t, kept).Data; !bytes.Equal(got, want) { + t.Fatal("cursor fields share memory with the encoded command") + } +} + +func TestCursorPanicCleansUpLatestCursor(t *testing.T) { + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}, {"c"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + db := r.db() + find := bson.D{{Key: "find", Value: "users"}} + var recovered any + func() { + defer func() { recovered = recover() }() + seen := 0 + for _, err := range db.Cursor(t.Context(), find) { + if err != nil { + t.Fatal(err) + } + seen++ + if seen == 2 { + panic("consumer failure") + } + } + }() + if recovered != "consumer failure" { + t.Fatal("consumer panic was not preserved", recovered) + } + if len(cursors.killed) != 1 || cursors.killed[0][1].Value.(bson.A)[0] != int64(2) { + t.Fatal("panic did not clean up the latest cursor exactly once", cursors.killed) + } +} + +func TestCanceledCursorStillCleansUp(t *testing.T) { + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}}, failAt: -1} + r := &recorder{t: t, answer: cursors.answer} + db := r.db() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + find := bson.D{{Key: "find", Value: "users"}} + for _, err := range db.Cursor(ctx, find) { + if err != nil { + t.Fatal(err) + } + cancel() + break + } + if len(cursors.killed) != 1 { + t.Fatal("caller cancellation prevented cursor cleanup", cursors.killed) + } +} + +func TestGetMoreFailureCleansUpBeforeYieldingError(t *testing.T) { + for _, interrupted := range []bool{false, true} { + cursors := &cursorServer{t: t, batches: [][]string{{"a"}, {"b"}}, failAt: 1} + r := &recorder{t: t} + r.answer = func(sent *pb.Command) []*pb.Event { + if interrupted && command(t, sent)[0].Key == "getMore" { + return storetest.NativeIncomplete(nil, nil, pb.FailureCode_UNAVAILABLE) + } + return cursors.answer(sent) + } + db := r.db() + find := bson.D{{Key: "find", Value: "users"}} + var failures int + for _, err := range db.Cursor(t.Context(), find) { + if err == nil { + continue + } + failures++ + if len(cursors.killed) != 1 { + t.Errorf("getMore error yielded before cleanup: interrupted=%t kills=%d", interrupted, len(cursors.killed)) + } + } + if failures != 1 || len(cursors.killed) != 1 { + t.Fatal("expected one failure and exactly one cleanup", failures, cursors.killed) + } + } +} diff --git a/mongodb/mongodb.go b/mongodb/mongodb.go index a374673..4e19e4d 100644 --- a/mongodb/mongodb.go +++ b/mongodb/mongodb.go @@ -26,9 +26,13 @@ package mongodb import ( + "bytes" "context" + "errors" "fmt" "iter" + "reflect" + "slices" "strings" "time" @@ -71,16 +75,41 @@ func (e *CommandError) Error() string { return fmt.Sprintf("mongodb: command failed: %s (%d): %s", e.Name, e.Code, e.Message) } -// Command runs one command in the database and returns its reply. A reply -// with ok: 0, writeErrors or a writeConcernError comes with a *CommandError. +// Command runs one command in the database and returns its reply. Commands +// must preserve field order, with the command name first; multi-key maps without +// custom BSON encoders are rejected before sending. Use bson.D or another +// ordered BSON representation. Custom BSON encoders own their field order. +// A reply with ok: 0, writeErrors or a writeConcernError comes with a *CommandError. // A command is sent again only when a server refused it before it began. One // a lost connection or an unavailable backend cut short reports // weir.ErrInterrupted and may have run; send it again only when that is safe. func (db DB) Command(ctx context.Context, command any) (bson.Raw, error) { + data, err := marshalCommand(command) + if err != nil { + return nil, err + } + return db.command(ctx, data) +} + +func marshalCommand(command any) (bson.Raw, error) { + // bson.Marshal gives an explicit document encoder control over field order. + if _, custom := command.(bson.Marshaler); !custom { + value := reflect.ValueOf(command) + for value.IsValid() && (value.Kind() == reflect.Pointer || value.Kind() == reflect.Interface) { + value = value.Elem() + } + if value.IsValid() && value.Kind() == reflect.Map && value.Len() > 1 { + return nil, errors.New("mongodb: command must be ordered; use bson.D instead of a multi-key map") + } + } data, err := bson.Marshal(command) if err != nil { return nil, fmt.Errorf("mongodb: encode command: %w", err) } + return bson.Raw(data), nil +} + +func (db DB) command(ctx context.Context, data bson.Raw) (bson.Raw, error) { document := &weir.Document{ContentType: bsonContentType, Data: data} request := &weir.NativeRequest{Resource: db.resource(), Request: document} result, err := db.target().Native(ctx, request) @@ -97,13 +126,22 @@ func (db DB) Command(ctx context.Context, command any) (bson.Raw, error) { return reply, nil } -// Cursor runs a command that opens a cursor, such as find, aggregate or -// listIndexes, and yields its documents. It sends getMore until the cursor -// ends, and kills the cursor when the loop stops early or fails. An error -// ends the sequence. +// Cursor runs an ordered command that opens a cursor, such as find, aggregate +// or listIndexes, and yields its documents. Its getMore and cleanup commands +// repeat the explicit session and Stable API fields of the command, and its +// transaction fields within a multi-document transaction. Each getMore also +// repeats a positive batch size. It sends getMore until the cursor ends, and +// kills the cursor when the loop stops early, fails or panics. An error ends +// the sequence. func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] { documents := func(yield func(bson.Raw, error) bool) { - reply, err := db.Command(ctx, command) + data, err := marshalCommand(command) + if err != nil { + yield(nil, err) + return + } + fields := readCursorFields(data) + reply, err := db.command(ctx, data) if err != nil { yield(nil, err) return @@ -115,10 +153,10 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] } space := parseNamespace(page.Cursor.Namespace) batch, cursor := page.Cursor.FirstBatch, page.Cursor.ID + defer func() { db.killCursor(ctx, space, cursor, fields.shared) }() for { for _, document := range batch { if !yield(document, nil) { - db.killCursor(ctx, space, cursor) return } } @@ -126,13 +164,16 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] return } getMore := bson.D{{Key: "getMore", Value: cursor}, {Key: "collection", Value: space.collection}, {Key: "$db", Value: space.database}} + getMore = append(getMore, fields.batchSize...) + getMore = append(getMore, fields.shared...) reply, err := db.Command(ctx, getMore) var next cursorPage if err == nil { next, err = readPage(reply) } if err != nil { - db.killCursor(ctx, space, cursor) + db.killCursor(ctx, space, cursor, fields.shared) + cursor = 0 yield(nil, err) return } @@ -142,6 +183,62 @@ func (db DB) Cursor(ctx context.Context, command any) iter.Seq2[bson.Raw, error] return documents } +// sharedCursorKeys name the fields of a cursor's opening command that its +// getMore and killCursors commands repeat: the explicit session and Stable API +// fields, which MongoDB requires to stay the same. +var sharedCursorKeys = []string{"lsid", "apiVersion", "apiStrict", "apiDeprecationErrors"} + +// transactionCursorKeys name the fields that getMore and killCursors also +// repeat inside a multi-document transaction, which autocommit: false marks. +// A retryable write such as bulkWrite names a txnNumber too, but MongoDB +// rejects one on getMore and killCursors outside a transaction. The +// startTransaction field belongs to the first command only. +var transactionCursorKeys = []string{"txnNumber", "autocommit"} + +// cursorFields are the fields of a cursor's opening command that the commands +// following it repeat. +type cursorFields struct { + // shared go on every getMore and killCursors. + shared bson.D + // batchSize goes on every getMore, as MongoDB applies a batch size only to + // the reply of the command that names it. + batchSize bson.D +} + +func readCursorFields(command bson.Raw) cursorFields { + var fields cursorFields + keys := sharedCursorKeys + if autocommit, found := command.Lookup("autocommit").BooleanOK(); found && !autocommit { + keys = slices.Concat(sharedCursorKeys, transactionCursorKeys) + } + for _, key := range keys { + value := command.Lookup(key) + if value.Type != 0 { + field := bson.E{Key: key, Value: detach(value)} + fields.shared = append(fields.shared, field) + } + } + // find names its batch size at the top level; aggregate and the list + // commands name it in their cursor option. A size of 0 asks only for an + // empty first batch, so it is not repeated. + size := command.Lookup("batchSize") + if size.Type == 0 { + size = command.Lookup("cursor", "batchSize") + } + if n, ok := size.AsInt64OK(); ok && n > 0 { + field := bson.E{Key: "batchSize", Value: detach(size)} + fields.batchSize = bson.D{field} + } + return fields +} + +// detach copies a value out of the command it was read from, so a cursor does +// not keep the whole encoded command alive while it runs. +func detach(value bson.RawValue) bson.RawValue { + copied := bson.RawValue{Type: value.Type, Value: bytes.Clone(value.Value)} + return copied +} + // namespace names the collection of a cursor. Commands that follow the cursor // name its database, as the command that opened it may have named one with // $db. @@ -180,13 +277,14 @@ func readPage(reply bson.Raw) (cursorPage, error) { // killCursor releases a cursor a loop left open. The server also times idle // cursors out, so a failure here is ignored. -func (db DB) killCursor(ctx context.Context, space namespace, cursor int64) { +func (db DB) killCursor(ctx context.Context, space namespace, cursor int64, shared bson.D) { if cursor == 0 { return } cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), killTimeout) defer cancel() command := bson.D{{Key: "killCursors", Value: space.collection}, {Key: "cursors", Value: bson.A{cursor}}, {Key: "$db", Value: space.database}} + command = append(command, shared...) _, _ = db.Command(cleanup, command) } diff --git a/search/index.go b/search/index.go index 175baa7..4f39c81 100644 --- a/search/index.go +++ b/search/index.go @@ -128,6 +128,11 @@ func (i Index[T]) Transform(ctx context.Context, id string, program string, inpu // Search sends a search request body, such as {"query": ..., "size": 10}, to // the index and returns its hits. A status of 300 or more is a *StatusError. +// A timed-out response, failed shards, or a skipped, partial or failed cluster +// of a cross-cluster search returns the decoded hits alongside a +// *PartialResponseError, even with a successful HTTP status. These hits may be +// incomplete, including an empty slice that does not establish no matches. A +// response without a hits.hits array is a decoding error, not an empty result. // Sort holds a hit's sort values exactly, to pass back as search_after. func (i Index[T]) Search(ctx context.Context, body any) ([]Hit[T], error) { response, err := i.Do(ctx, http.MethodPost, "/_search", body) @@ -139,14 +144,42 @@ func (i Index[T]) Search(ctx context.Context, body any) ([]Hit[T], error) { return nil, failed } var result struct { + TimedOut bool `json:"timed_out"` + Shards struct { + Failed int `json:"failed"` + Failures []json.RawMessage `json:"failures"` + } `json:"_shards"` + // Clusters is present only for a cross-cluster search. + Clusters struct { + Skipped int `json:"skipped"` + Partial int `json:"partial"` + Failed int `json:"failed"` + } `json:"_clusters"` + // Hits stays nil when the response has no hits.hits array or it is null. Hits struct { - Hits []Hit[T] `json:"hits"` + Hits *[]Hit[T] `json:"hits"` } `json:"hits"` } if err := json.Unmarshal(response.Body, &result); err != nil { return nil, fmt.Errorf("search: decode %s hits: %w", i.Name, err) } - return result.Hits.Hits, nil + if result.Hits.Hits == nil { + return nil, fmt.Errorf("search: decode %s hits: response has no hits.hits array", i.Name) + } + hits := *result.Hits.Hits + clusters := result.Clusters + if result.TimedOut || result.Shards.Failed > 0 || clusters.Skipped > 0 || clusters.Partial > 0 || clusters.Failed > 0 { + partial := &PartialResponseError{ + TimedOut: result.TimedOut, + FailedShards: result.Shards.Failed, + ShardFailures: result.Shards.Failures, + SkippedClusters: clusters.Skipped, + PartialClusters: clusters.Partial, + FailedClusters: clusters.Failed, + } + return hits, partial + } + return hits, nil } // Do sends one HTTP request to the index and returns its response, whatever diff --git a/search/search.go b/search/search.go index fa27165..d974755 100644 --- a/search/search.go +++ b/search/search.go @@ -20,7 +20,10 @@ // Set a deadline on every context. A failed record is a *weir.Error: check it // with weir.CodeOf, and with errors.Is against weir.ErrNotFound and // weir.ErrInterrupted. The package keeps no state of its own; Cluster and -// Index are plain values that are safe to copy and share. +// Index are plain values that are safe to copy and share. Search returns +// decoded hits alongside a *PartialResponseError when the backend times out, +// reports failed shards or does not search every cluster of a cross-cluster +// search, even with a successful HTTP status. package search import ( @@ -65,6 +68,34 @@ func (e *StatusError) Error() string { return fmt.Sprintf("search: status %d: %s", e.StatusCode, e.Body) } +// PartialResponseError reports that Search received an incomplete response, +// even though its HTTP status indicated success. Search returns any decoded +// hits alongside this error; they are not a complete result set. +// Inspect it with errors.As. +type PartialResponseError struct { + TimedOut bool + FailedShards int + // ShardFailures holds each _shards.failures entry as its original JSON, + // including any backend-specific reason details. It may be absent even + // when FailedShards is nonzero. + ShardFailures []json.RawMessage + // SkippedClusters, PartialClusters and FailedClusters count the clusters + // of a cross-cluster search that _clusters reports as skipped, such as an + // unavailable cluster with skip_unavailable, as returning partial results, + // or as failed. They are zero for a search of one cluster. + SkippedClusters int + PartialClusters int + FailedClusters int +} + +func (e *PartialResponseError) Error() string { + message := fmt.Sprintf("search: incomplete response: timed_out=%t, failed_shards=%d", e.TimedOut, e.FailedShards) + if e.SkippedClusters > 0 || e.PartialClusters > 0 || e.FailedClusters > 0 { + message += fmt.Sprintf(", skipped_clusters=%d, partial_clusters=%d, failed_clusters=%d", e.SkippedClusters, e.PartialClusters, e.FailedClusters) + } + return message +} + // Do sends one HTTP request to the cluster and returns its response, whatever // its status. The path starts at the cluster root, such as /_cluster/health, // and may carry a query string. A []byte or json.RawMessage body is sent as it diff --git a/search/search_test.go b/search/search_test.go index d6c83c9..6a12714 100644 --- a/search/search_test.go +++ b/search/search_test.go @@ -7,6 +7,7 @@ import ( "errors" "io" "net/http" + "strings" "testing" weir "github.com/batchstream/weir-go" @@ -191,6 +192,125 @@ func TestFailedWritesReportTheirCode(t *testing.T) { } } +func TestSearchReportsPartialResponsesAndPreservesHits(t *testing.T) { + shardFailure := `{"shard":1,"index":"products","node":"node-1","reason":{"type":"query_shard_exception","reason":"query failed","caused_by":{"type":"illegal_argument_exception","reason":"bad field"}}}` + hit := `{"_id":"p1","_score":1.5,"sort":[9007199254740993,"lamp"],"_source":{"name":"lamp","price":30}}` + tests := []struct { + name string + body string + timedOut bool + failedShards int + shardFailures []string + // clusters holds the skipped, partial and failed cluster counts. + clusters [3]int + hitCount int + }{ + { + name: "timeout with empty hits", + body: `{"timed_out":true,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"hits":[]}}`, + timedOut: true, + }, + { + name: "failed shards with partial hits", + body: `{"timed_out":false,"_shards":{"total":2,"successful":1,"failed":1,"failures":[` + shardFailure + `]},"hits":{"hits":[` + hit + `]}}`, + failedShards: 1, + shardFailures: []string{shardFailure}, + hitCount: 1, + }, + { + name: "timeout and failed shards", + body: `{"timed_out":true,"_shards":{"total":2,"successful":1,"failed":1,"failures":[` + shardFailure + `]},"hits":{"hits":[` + hit + `]}}`, + timedOut: true, + failedShards: 1, + shardFailures: []string{shardFailure}, + hitCount: 1, + }, + { + name: "failed shards without failure details", + body: `{"_shards":{"failed":2},"hits":{"hits":[]}}`, + failedShards: 2, + }, + { + name: "skipped remote cluster", + body: `{"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"_clusters":{"total":2,"successful":1,"skipped":1},"hits":{"hits":[` + hit + `]}}`, + clusters: [3]int{1, 0, 0}, + hitCount: 1, + }, + { + name: "partial and failed clusters", + body: `{"timed_out":false,"_shards":{"total":2,"successful":2,"failed":0},"_clusters":{"total":3,"successful":1,"skipped":0,"running":0,"partial":1,"failed":1},"hits":{"hits":[]}}`, + clusters: [3]int{0, 1, 1}, + }, + { + name: "complete response", + body: `{"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":1,"failed":0},"hits":{"hits":[` + hit + `]}}`, + hitCount: 1, + }, + { + name: "complete cross-cluster response", + body: `{"timed_out":false,"_shards":{"total":2,"successful":2,"failed":0},"_clusters":{"total":2,"successful":2,"skipped":0,"running":0,"partial":0,"failed":0},"hits":{"hits":[` + hit + `]}}`, + hitCount: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &recorder{t: t} + r.answer = func(*pb.Command) []*pb.Event { return httpResponse("200 OK", tt.body) } + products := r.products() + query := map[string]any{"query": map[string]any{"match_all": map[string]any{}}} + found, err := products.Search(t.Context(), query) + if tt.timedOut || tt.failedShards > 0 || tt.clusters != [3]int{} { + var partial *PartialResponseError + if !errors.As(err, &partial) { + t.Fatalf("expected *PartialResponseError, got %v", err) + } + if partial.TimedOut != tt.timedOut || partial.FailedShards != tt.failedShards { + t.Fatal("wrong incompleteness details", partial) + } + clusters := [3]int{partial.SkippedClusters, partial.PartialClusters, partial.FailedClusters} + if clusters != tt.clusters { + t.Fatal("wrong cluster details", partial) + } + if len(partial.ShardFailures) != len(tt.shardFailures) { + t.Fatal("wrong shard failures", partial.ShardFailures) + } + for n, failure := range partial.ShardFailures { + if string(failure) != tt.shardFailures[n] { + t.Fatal("shard failure details changed", string(failure)) + } + } + } else if err != nil { + t.Fatal("complete response failed", err) + } + if len(found) != tt.hitCount { + t.Fatal("wrong hits", found) + } + if tt.hitCount > 0 { + if found[0].ID != "p1" || found[0].Score != 1.5 || found[0].Document == nil || found[0].Document.Name != "lamp" || found[0].Document.Price != 30 { + t.Fatal("hit changed", found[0]) + } + if len(found[0].Sort) != 2 || string(found[0].Sort[0]) != "9007199254740993" || string(found[0].Sort[1]) != `"lamp"` { + t.Fatal("sort values changed", found[0].Sort) + } + } + }) + } +} + +func TestSearchRejectsResponsesWithoutHits(t *testing.T) { + bodies := []string{`null`, `{}`, `{"hits":null}`, `{"hits":{}}`, `{"hits":{"hits":null}}`, `{"timed_out":true}`} + for _, body := range bodies { + r := &recorder{t: t} + r.answer = func(*pb.Command) []*pb.Event { return httpResponse("200 OK", body) } + products := r.products() + query := map[string]any{"query": map[string]any{"match_all": map[string]any{}}} + found, err := products.Search(t.Context(), query) + if err == nil || found != nil || !strings.Contains(err.Error(), "no hits.hits array") { + t.Fatalf("accepted %s: %v, %v", body, found, err) + } + } +} + func TestSearchDecodesHitsAndKeepsSortValuesExact(t *testing.T) { hits := `{"hits":{"total":{"value":2},"hits":[ {"_id":"p1","_score":null,"sort":[9007199254740993,"lamp"],"_source":{"name":"lamp","price":30}},