diff --git a/CHANGELOG.md b/CHANGELOG.md index 59475da..98e28a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,10 @@ has a matching GitHub release whose body comes from its section below. - Each card has its own base URL on the gateway, `/cards/CARD/v1`, named by the card's name or PCI address. A request through it only reaches engines on that card, and `GET /cards/CARD/v1/models` lists only theirs. One model can now be served on several cards, one engine entry per card, and `berth config check` still refuses two - entries that serve it on one card. A request without the prefix is routed as before. + entries that serve it on one card. +- A plain `/v1` request for a model served on several cards goes to the card that costs least to reach: the engine + that is up, then one parked in RAM, then a free card, then a card whose holder drains, and last a card a VM or a + pinned lease keeps. Cards of the same rank go to the first entry in `berth.toml`. - `gpu = "gpu1"` on a `[[claim]]` holds only the cards it names, as `berth run --gpu` names them, on a node with several cards. Engines on the other cards keep running, and the gateway starts background engines there. A claim without `gpu` still holds every card. `berth config check` refuses a bad card name and a `gpu` on a diff --git a/README.md b/README.md index a64d874..9efc513 100644 --- a/README.md +++ b/README.md @@ -222,8 +222,9 @@ on any of the cards stops the engine, and its exit frees every card. A lease on yet: a drain stops it the usual way. The gateway judges each engine against the cards it runs on, and each card has its own base URL on the gateway, -`/cards/gpu1/v1`, so a client can choose the card ([One URL per card](docs/gateway.md#one-url-per-card)). Choosing -a card for a plain `/v1` request is v0.5 work. +`/cards/gpu1/v1`, so a client can choose the card ([One URL per card](docs/gateway.md#one-url-per-card)). A plain +`/v1` request for a model served on several cards goes to the cheapest card to reach: a parked copy, then a free +card, then one whose holder drains. `berth hold --gpu gpu1`, `berth vfio run --gpu gpu1` and `gpu = "gpu1"` on a `[[claim]]` take only the cards they name, so the other cards keep serving. diff --git a/docs/gateway.md b/docs/gateway.md index f8c9c48..5a35f67 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -309,7 +309,17 @@ serves = ["qwen"] A client that works on gpu1 sets its base URL to `http://127.0.0.1:9799/cards/gpu1/v1` and keeps the model name. `berth config check` still refuses two entries that serve one model on the same card, or where either names no -card. A request without the prefix goes to the first engine in `berth.toml` that serves the model, as before. +card. + +A request without the prefix picks the card itself. An engine that is up and lists the model answers first. +Otherwise the gateway takes the card that costs least to reach, in this order: + +1. the card where the model's engine is parked in RAM; +2. a free card: no lease, waiter, stowaway or active claim, and not in a VM; +3. a card whose holder drains the usual way; +4. a card a VM or a pinned lease (a game, a hold) keeps, where the request waits. + +Two cards of the same rank go to the entry that comes first in `berth.toml`. - A card the node does not have answers `404` with the node's cards. - A model no engine on that card serves answers `404` naming the card. diff --git a/docs/roadmap.md b/docs/roadmap.md index 402f4d2..da98eeb 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -112,7 +112,8 @@ v0.5 lets a client choose the card on a node with several. Each slice ships on i 1. **One base URL per card**, merged in [#278](https://github.com/T-Crypt/berth/pull/278). `/cards/CARD/v1` reaches only the engines on that card, and one model can be served on several cards, one engine entry per card. 2. **Placement for a plain `/v1` request.** When several cards serve the model, the gateway picks one: the card - where it already runs, then one where it is parked, then a free card, then the cheapest measured switch. + where it already runs, then one where it is parked, then a free card, then one whose holder drains, and last a + card a VM or a game keeps. 3. **Cordon a card.** A cordoned card takes no new grants until it is uncordoned, so it can be drained for maintenance or a VM while the others serve. 4. **A card picker on the board.** The harbor and `GET /v1/berth/board` name the card each engine can run on. diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index abe8309..e2e21ac 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -439,13 +439,36 @@ func (g *Gateway) resolve(ctx context.Context, model, card string) (target, bool return target{url: e.URL, live: true}, true } } + var serving []*config.Engine for i := range g.Engines { - e := &g.Engines[i] - if g.engineOn(st, e.Name, card) && slices.Contains(e.Serves, model) { - return target{url: e.URL, engine: e}, true + if e := &g.Engines[i]; g.engineOn(st, e.Name, card) && slices.Contains(e.Serves, model) { + serving = append(serving, e) } } - return target{}, false + if len(serving) == 0 { + return target{}, false + } + // One engine per card serves the model: the cheapest card to reach + // wins, and berth.toml's order breaks a tie. + e := slices.MinFunc(serving, func(a, b *config.Engine) int { return reach(st, a) - reach(st, b) }) + return target{url: e.URL, engine: e}, true +} + +// reach ranks what it takes to bring e up on its card, cheapest first: a +// resume from RAM, a free card, a drain of the holder, then a wait for a +// card a VM or a pinned lease (a game, a hold) keeps. +func reach(st *proto.Status, e *config.Engine) int { + v := node.View(st, e.GPU) + switch { + case parkedIn(st, e.Name): + return 0 + case cardFree(v): + return 1 + case v != nil && outOfReach(v) == "": + return 2 + default: + return 3 + } } // notServed is the 404 for a model no engine serves, on card when one is diff --git a/internal/gateway/placement_test.go b/internal/gateway/placement_test.go new file mode 100644 index 0000000..93a7ba7 --- /dev/null +++ b/internal/gateway/placement_test.go @@ -0,0 +1,86 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +//go:build unix + +package gateway + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/T-Crypt/berth/internal/broker" + "github.com/T-Crypt/berth/internal/config" + "github.com/T-Crypt/berth/internal/procs" +) + +// A plain /v1 request for a model served on two cards goes to the card that +// costs least to reach: a parked copy before a free card, a free card before +// one a lease holds, and a held card before one a game has pinned. With +// nothing to tell them apart, the first entry in berth.toml wins. +func TestPlainRequestPicksTheCheapestCard(t *testing.T) { + for _, c := range []struct { + name string + hold func(t *testing.T, gpu0, gpu1 *broker.Broker) + want string + }{ + {"both free", nil, "on-gpu0"}, + {"a game pins gpu0", func(t *testing.T, gpu0, _ *broker.Broker) { + acquire(t, gpu0, broker.Request{Engine: "game", Priority: broker.Hold, Pinned: true}) + }, "on-gpu1"}, + {"a lease holds gpu0", func(t *testing.T, gpu0, _ *broker.Broker) { + acquire(t, gpu0, broker.Request{Engine: "tts", Priority: broker.Batch}) + }, "on-gpu1"}, + {"a game pins gpu0, a lease holds gpu1", func(t *testing.T, gpu0, gpu1 *broker.Broker) { + acquire(t, gpu0, broker.Request{Engine: "game", Priority: broker.Hold, Pinned: true}) + acquire(t, gpu1, broker.Request{Engine: "tts", Priority: broker.Batch}) + }, "on-gpu1"}, + {"qwen parked on gpu1", func(t *testing.T, _, gpu1 *broker.Broker) { + pid := os.Getpid() + l, err := gpu1.Reattach(broker.Request{Engine: "qwen-b", Parked: true}, pid, procs.System{}.StartTime(pid)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(l.Release) + }, "on-gpu1"}, + } { + t.Run(c.name, func(t *testing.T) { + sock, n := twoCards(t) + if c.hold != nil { + c.hold(t, n.Cards()[0].B, n.Cards()[1].B) + } + a, b := cardEngine(t, "on-gpu0"), cardEngine(t, "on-gpu1") + g := &Gateway{Socket: sock, Ready: 5 * time.Second, Key: testKey, Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + Engines: []config.Engine{ + {Name: "qwen-a", Type: "llamacpp", URL: a.URL, GPU: "gpu0", Serves: []string{"qwen"}}, + {Name: "qwen-b", Type: "llamacpp", URL: b.URL, GPU: "gpu1", Serves: []string{"qwen"}}, + }} + gw := httptest.NewServer(g.Handler()) + defer gw.Close() + resp := call(t, "POST", gw.URL+"/v1/chat/completions", `{"model":"qwen"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if want := c.want + " /v1/chat/completions"; resp.StatusCode != http.StatusOK || string(body) != want { + t.Errorf("POST /v1/chat/completions: %d %q, want 200 %q", resp.StatusCode, body, want) + } + }) + } +} + +// acquire takes a lease on b for the rest of the test. +func acquire(t *testing.T, b *broker.Broker, r broker.Request) { + t.Helper() + r.Wait = 5 * time.Second + l, err := b.Acquire(context.Background(), r, nil) + if err != nil { + t.Fatal(fmt.Errorf("acquire %s: %w", r.Engine, err)) + } + t.Cleanup(l.Release) +}