From 83ed4053f06b05abf2e3af6c6e9dd14c1a1b0245 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:06:29 -0600 Subject: [PATCH 1/2] test: a cordoned card grants only holds and the gateway skips it The broker must keep a new batch or foreground lease waiting with a reason that names the cordon, still grant a hold, and grant the waiter after an uncordon. berth cordon and uncordon must work against a live daemon and show in berth status. The gateway must treat the card as out of reach and start no background run for it. These fail until the cordon lands. --- internal/broker/cordon_test.go | 39 +++++++++++++++++++ internal/cli/cordon_test.go | 50 ++++++++++++++++++++++++ internal/gateway/cordon_test.go | 69 +++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 internal/broker/cordon_test.go create mode 100644 internal/cli/cordon_test.go create mode 100644 internal/gateway/cordon_test.go diff --git a/internal/broker/cordon_test.go b/internal/broker/cordon_test.go new file mode 100644 index 0000000..75e90f7 --- /dev/null +++ b/internal/broker/cordon_test.go @@ -0,0 +1,39 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package broker + +import ( + "strings" + "testing" + + "github.com/T-Crypt/berth/internal/procs" +) + +// A cordoned card grants nothing new but a hold, keeps its holder, and +// says why in the waiter's reason; uncordoning lets the waiter in. +func TestCordonStopsNewGrantsUntilUncordoned(t *testing.T) { + b, _ := startTurnBroker(t, procs.Map{}) + holder := must(t, acquire(b, Request{Engine: "llm", Priority: Batch})) + b.Cordon(true) + if st := b.Status(); len(st.GPUs) == 0 || !st.GPUs[0].Cordoned { + t.Fatalf("status does not show the cordon: %+v", st.GPUs) + } + holder.Release() + + w := acquireW(b, Request{Engine: "image", Priority: Foreground}) + pending(t, w.ch) + if r := firstReason(t, w); !strings.Contains(r, "cordoned") { + t.Errorf("reason = %q, want the cordon named", r) + } + hold := must(t, acquire(b, Request{Engine: "game", Priority: Hold})) + hold.Release() + pending(t, w.ch) + + b.Cordon(false) + b.Tick(t.Context()) + must(t, w.ch) + if st := b.Status(); st.GPUs[0].Cordoned { + t.Error("status still shows the cordon") + } +} diff --git a/internal/cli/cordon_test.go b/internal/cli/cordon_test.go new file mode 100644 index 0000000..bc723d7 --- /dev/null +++ b/internal/cli/cordon_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/T-Crypt/berth/internal/proto" +) + +// berth cordon and uncordon against a live daemon: the report names the +// card, status shows the cordon, and an unknown card fails with exit 1. +func TestCordonAgainstADaemon(t *testing.T) { + sock, _, _ := startStopDaemon(t, time.Second) + cordoned := func() bool { + code, out, _ := runCLI(t, "status", "--socket", sock, "--format", "json") + if code != 0 { + t.Fatalf("status exit %d", code) + } + var st proto.Status + if err := json.Unmarshal([]byte(out), &st); err != nil { + t.Fatal(err) + } + return len(st.GPUs) > 0 && st.GPUs[0].Cordoned + } + + code, out, errOut := runCLI(t, "cordon", "gpu0", "--socket", sock) + if code != 0 || !strings.Contains(out, "gpu0") { + t.Fatalf("cordon: exit %d, out %q, err %q", code, out, errOut) + } + if !cordoned() { + t.Error("status does not show gpu0 cordoned") + } + if _, out, _ := runCLI(t, "status", "--socket", sock); !strings.Contains(out, "CORDONED") { + t.Errorf("berth status does not say CORDONED:\n%s", out) + } + if code, out, errOut = runCLI(t, "uncordon", "gpu0", "--socket", sock); code != 0 || !strings.Contains(out, "gpu0") { + t.Fatalf("uncordon: exit %d, out %q, err %q", code, out, errOut) + } + if cordoned() { + t.Error("status still shows gpu0 cordoned") + } + if code, _, errOut = runCLI(t, "cordon", "gpu7", "--socket", sock); code != ExitFail || !strings.Contains(errOut, "no card gpu7") { + t.Errorf("unknown card: exit %d, err %q, want exit %d naming the card", code, errOut, ExitFail) + } +} diff --git a/internal/gateway/cordon_test.go b/internal/gateway/cordon_test.go new file mode 100644 index 0000000..047c6c0 --- /dev/null +++ b/internal/gateway/cordon_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +//go:build unix + +package gateway + +import ( + "context" + "io" + "log/slog" + "os" + "testing" + "time" + + "github.com/T-Crypt/berth/internal/config" + "github.com/T-Crypt/berth/internal/node" + "github.com/T-Crypt/berth/internal/proto" +) + +// A cordoned card is not free: its background engine waits until berth +// uncordon, and a ComfyUI job for it is told why it cannot run. +func TestCordonedCardIsNotFree(t *testing.T) { + sock, n := twoCards(t) + if _, err := n.Cordon("gpu1", true); err != nil { + t.Fatal(err) + } + var st proto.Status + until(t, "both cards read", func() bool { + st = n.Status() + return node.View(&st, "gpu0") != nil && node.View(&st, "gpu1") != nil + }) + if why := outOfReach(nodeView(t, &st, "gpu1")); why != "the card is cordoned" { + t.Errorf("outOfReach(gpu1) = %q, want the cordon", why) + } + if why := outOfReach(nodeView(t, &st, "gpu0")); why != "" { + t.Errorf("outOfReach(gpu0) = %q, want nothing", why) + } + + addr := freeAddr(t) + g := &Gateway{Socket: sock, Ready: 10 * time.Second, Key: testKey, Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + Engines: []config.Engine{{Name: "tagger", Type: "llamacpp", URL: "http://" + addr, GPU: "gpu1", + Start: []string{"env", "BERTH_GW_ENGINE=" + addr + ",tagger-model", os.Args[0]}, + Background: true, IdleAfter: config.Duration{Duration: 50 * time.Millisecond}}}} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go g.Background(ctx, 20*time.Millisecond) + + time.Sleep(400 * time.Millisecond) + if st := n.Cards()[1].B.Status(); len(st.Leases)+len(st.Waiting) != 0 { + t.Fatalf("the gateway started a background run for the cordoned card: leases %+v, waiting %+v", st.Leases, st.Waiting) + } + if _, err := n.Cordon("gpu1", false); err != nil { + t.Fatal(err) + } + until(t, "the tagger to start once gpu1 is uncordoned", func() bool { + ls := n.Cards()[1].B.Status().Leases + return len(ls) == 1 && ls[0].Engine == "tagger" + }) +} + +func nodeView(t *testing.T, st *proto.Status, card string) *proto.Status { + t.Helper() + v := node.View(st, card) + if v == nil { + t.Fatalf("no card %s", card) + } + return v +} From 7446921cd532d7d939f40d0ce70543e27b6b2cc0 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:06:29 -0600 Subject: [PATCH 2/2] feat: berth cordon and uncordon a card berth cordon CARD stops new grants on a card, holds aside, while the node's other cards serve; berth uncordon CARD lifts it and schedules the card's waiters at once. A cordon drains nothing. Waiters say the card is cordoned, berth status marks it CORDONED, and berth events records card.cordoned and card.uncordoned. The gateway treats the card as out of reach: no background engine starts there and a ComfyUI job for it queues. The cordon lives in berthd's memory, so a restart clears it. Third slice of v0.5, per-card gateway endpoints. --- CHANGELOG.md | 4 ++ docs/gateway.md | 2 + docs/interfaces.md | 2 +- docs/leasing.md | 14 +++++++ docs/roadmap.md | 4 +- internal/broker/broker.go | 23 ++++++++++++ internal/broker/status.go | 2 +- internal/cli/commands.go | 22 +++++++++++ internal/cli/cordon.go | 68 ++++++++++++++++++++++++++++++++++ internal/cli/status.go | 3 ++ internal/gateway/background.go | 2 +- internal/gateway/deferred.go | 9 +++++ internal/node/node.go | 15 ++++++++ internal/proto/proto.go | 14 ++++++- internal/server/cordon.go | 27 ++++++++++++++ internal/server/server.go | 2 + internal/skill/SKILL.md | 1 + 17 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 internal/cli/cordon.go create mode 100644 internal/server/cordon.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 59475da..f28f6dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ has a matching GitHub release whose body comes from its section below. ### Added +- `berth cordon CARD` stops new grants on a card, holds aside, while the node's other cards serve; `berth uncordon + CARD` lifts it. What holds the card keeps running. Waiters say the card is cordoned, `berth status` marks it + `CORDONED`, and the gateway treats it as out of reach: no background engine starts there and a ComfyUI job for it + queues. A berthd restart clears the cordon. - Each card has its own base URL on the gateway, `/cards/CARD/v1`, named by the card's name or PCI address. A request through it only reaches engines on that card, and `GET /cards/CARD/v1/models` lists only theirs. One model can now be served on several cards, one engine entry per card, and `berth config check` still refuses two diff --git a/docs/gateway.md b/docs/gateway.md index f8c9c48..44a0373 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -312,6 +312,8 @@ A client that works on gpu1 sets its base URL to `http://127.0.0.1:9799/cards/gp card. A request without the prefix goes to the first engine in `berth.toml` that serves the model, as before. - A card the node does not have answers `404` with the node's cards. +- A cordoned card ([Cordon a card](leasing.md#cordon-a-card)) still takes requests through its URL, and they wait + in line. - A model no engine on that card serves answers `404` naming the card. - An engine berthd finds that has no entry in `berth.toml` belongs to no card here, so only the plain `/v1` URL reaches it. diff --git a/docs/interfaces.md b/docs/interfaces.md index eda1955..c51cad0 100644 --- a/docs/interfaces.md +++ b/docs/interfaces.md @@ -1,7 +1,7 @@ # Interfaces - **CLI:** one command tree with one output contract. `berth status`, `berth events -f`, - `berth run`, `berth hold`, `berth stop`, `berth notify`, `berth claims`, `berth pull`, `berth verify`, `berth models`, `berth gateway`, + `berth run`, `berth hold`, `berth stop`, `berth cordon|uncordon`, `berth notify`, `berth claims`, `berth pull`, `berth verify`, `berth models`, `berth gateway`, `berth wire llama-swap`, `berth engine check`, `berth config check`, `berth config show`, `berth export`, `berth key`, `berth skill install|show`, `berth update`, `berth version`, and for the node itself `berth daemon`, `berth harbor`, `berth service install|uninstall|status`, diff --git a/docs/leasing.md b/docs/leasing.md index 2cd9002..11fa9d4 100644 --- a/docs/leasing.md +++ b/docs/leasing.md @@ -126,6 +126,20 @@ - A hold cannot free memory held by something outside Berth: that memory is still reported, as contested. +## Cordon a card + +On a node with several cards, `berth cordon gpu1` stops new grants on that card while the others keep serving, for +maintenance or before a VM takes it. Name the card by its name or its PCI address. + +- What holds the card keeps running. A cordon drains nothing: `berth stop ENGINE` does that. +- Waiters stay in line, and their reason says the card is cordoned. A hold (`berth hold`, a game claim) is still + granted. +- The gateway treats the card as out of reach. A plain `/v1` request goes to another card that serves its model, + a background engine for the card does not start, and a ComfyUI job for it queues. +- `berth status` marks the card `CORDONED`, and `berth events` records `card.cordoned` and `card.uncordoned`. +- `berth uncordon gpu1` lets new work in, and the card's waiters are scheduled at once. A berthd restart also + clears the cordon. + ## Training runs A training run that loses its cards loses everything since its last checkpoint. Run it under one pinned lease: diff --git a/docs/roadmap.md b/docs/roadmap.md index 402f4d2..2a30619 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -113,8 +113,8 @@ v0.5 lets a client choose the card on a node with several. Each slice ships on i the engines on that card, and one model can be served on several cards, one engine entry per card. 2. **Placement for a plain `/v1` request.** When several cards serve the model, the gateway picks one: the card where it already runs, then one where it is parked, then a free card, then the cheapest measured switch. -3. **Cordon a card.** A cordoned card takes no new grants until it is uncordoned, so it can be drained for - maintenance or a VM while the others serve. +3. **Cordon a card.** `berth cordon gpu1` stops new grants on the card, holds aside, until `berth uncordon`, so it + can be drained for maintenance or a VM while the others serve. 4. **A card picker on the board.** The harbor and `GET /v1/berth/board` name the card each engine can run on. ## Also queued diff --git a/internal/broker/broker.go b/internal/broker/broker.go index 4732cb9..d18a8e2 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -479,6 +479,7 @@ type Broker struct { saveFailed bool verdicts map[string]verdict // loser engine key -> the juggle it lost + cordoned bool // berth cordon: grant only holds // tracked is the claim watcher's last scan: a process (and every child) // a track claim names. Its card memory is the claim's, never a stowaway. tracked map[int]string @@ -1337,6 +1338,9 @@ func (c cardBudget) avail() uint64 { return sat(c.ceiling(), c.held) } // evaluate decides whether w fits now. If not, it returns why and which // holders to drain; no victims means draining would not help, so w waits. func (b *Broker) evaluate(w *lease, now time.Time) (bool, string, []*lease) { + if b.cordoned && w.prio != Hold { + return false, "the card is cordoned; berth uncordon lets new work in", nil + } // A resumed lease is still in b.leases while it waits: its own // residual is the copy it comes back to, never a holder it waits on. hs := slices.DeleteFunc(b.holders(), func(h *lease) bool { return h == w }) @@ -1817,3 +1821,22 @@ func (b *Broker) abandon(l *lease, why string) error { b.emit(proto.Event{Kind: proto.EvRefused, Lease: l.id, Engine: l.engine, Detail: why}) return fmt.Errorf("%w: %s", ErrRefused, why) } + +// Cordon stops new grants on the card but holds (on), or lets them in again +// (off). Holders keep the card; a cordon drains nothing. A change goes on +// the event stream, and an uncordon schedules the queue at once. +func (b *Broker) Cordon(on bool) { + b.mu.Lock() + defer b.mu.Unlock() + if b.cordoned == on { + return + } + b.cordoned = on + kind := proto.EvUncordoned + if on { + kind = proto.EvCordoned + } + b.log.Info("cordon", "card", b.cfg.Card, "on", on) + b.emit(proto.Event{Kind: kind}) + b.schedule(b.now()) +} diff --git a/internal/broker/status.go b/internal/broker/status.go index e01e03c..93756f9 100644 --- a/internal/broker/status.go +++ b/internal/broker/status.go @@ -66,7 +66,7 @@ func (b *Broker) Status() proto.Status { Util: d.Util, Temp: d.Temp, Power: d.Power, PowerLimit: d.PowerLimit, Fan: d.Fan, ClockSM: d.ClockSM, ClockMem: d.ClockMem, PCIeGen: d.PCIeGen, PCIeWidth: d.PCIeWidth} if d.Index == b.snap.Index { - g.Scheduled, g.Baseline = true, b.snap.Baseline() + g.Scheduled, g.Baseline, g.Cordoned = true, b.snap.Baseline(), b.cordoned g.UnobservedPIDs = append([]int(nil), b.snap.Unobserved...) g.GTTBaseline = b.snap.GTTBaseline() } diff --git a/internal/cli/commands.go b/internal/cli/commands.go index b2e26c2..fc217e1 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -520,6 +520,28 @@ unpairs every device.`, Formats: []Format{FormatHuman, FormatJSON, FormatYAML}, Run: runPair, }, + { + Name: "cordon", + Summary: "stop new grants on a card; what holds it keeps running", + Args: "CARD [--socket P] [--format F]", + Help: `Stop new grants on the named card, by its name (gpu1) or PCI address, for +maintenance or before a VM takes it, while the node's other cards keep +serving. Waiters stay in line with a reason that names the cordon, and +the gateway sends a plain /v1 request to another card that serves its +model. A hold (berth hold, a game claim) is still granted. What holds the +card keeps running: berth stop ENGINE drains it. The cordon lasts until +berth uncordon or until berthd restarts.`, + Formats: []Format{FormatHuman, FormatJSON, FormatYAML}, + Run: runCordon, + }, + { + Name: "uncordon", + Summary: "let new work get a cordoned card again", + Args: "CARD [--socket P] [--format F]", + Help: `Lift berth cordon from the named card. Its waiters are scheduled at once.`, + Formats: []Format{FormatHuman, FormatJSON, FormatYAML}, + Run: runUncordon, + }, { Name: "stop", Summary: "stop an engine on purpose: its leases drain through their own wrappers", diff --git a/internal/cli/cordon.go b/internal/cli/cordon.go new file mode 100644 index 0000000..69cd1e8 --- /dev/null +++ b/internal/cli/cordon.go @@ -0,0 +1,68 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package cli + +// berth cordon and berth uncordon: a person stops new grants on a card, +// for maintenance or before a VM takes it, while the other cards serve. + +import ( + "strings" + + "github.com/T-Crypt/berth/internal/proto" + "github.com/T-Crypt/berth/internal/server" +) + +func runCordon(c *Context) int { return cordonCmd(c, proto.OpCordon) } +func runUncordon(c *Context) int { return cordonCmd(c, proto.OpUncordon) } + +func cordonCmd(c *Context, op string) int { + fs := c.FlagSet() + sock := fs.String("socket", defaultSocket(), "berthd socket ($BERTH_SOCKET)") + if code, done := c.ParseInterspersed(fs); done { + return code + } + if len(c.Args) != 1 { + return c.fail(Failf("berth %s takes one card", op).WithHint("usage: berth %s CARD (berth status names the cards)", op)) + } + conn, raw, err := server.Dial(*sock) + if err != nil { + return c.fail(Systemf("berthd unreachable on %s: %v", *sock, err)) + } + defer raw.Close() + if err := conn.Send(proto.Msg{Op: op, Card: c.Args[0]}); err != nil { + return c.fail(Systemf("berthd on %s: %v", *sock, err)) + } + m, err := conn.Recv() + if err != nil { + return c.fail(Systemf("no %s reply from berthd: %v", op, err)) + } + if m.Op == proto.OpRefused { + return c.fail(Failf("%s", m.Reason)) + } + return c.Write(cordonView{op: op, cards: m.Cards}, nil) +} + +type cordonView struct { + op string + cards []string +} + +func (v cordonView) Data() any { + return struct { + Cordoned bool `json:"cordoned"` + Cards []string `json:"cards"` + }{v.op == proto.OpCordon, v.cards} +} + +func (v cordonView) Human(c *Context, ok bool) error { + names := strings.Join(v.cards, ", ") + if v.op == proto.OpUncordon { + c.Printf("%s uncordoned: new work gets the card again\n", names) + return nil + } + c.Printf("%s cordoned: no new grants but holds\n", names) + c.Bullet("what holds it keeps running; berth stop ENGINE drains it") + c.Bullet("the cordon lasts until berth uncordon or a berthd restart") + return nil +} diff --git a/internal/cli/status.go b/internal/cli/status.go index 40bc5cb..7a1aa3b 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -51,6 +51,9 @@ func (v statusView) Human(c *Context, ok bool) error { role := "inventory only" if g.Scheduled { role = "leased by berth, baseline " + server.MiB(g.Baseline) + if g.Cordoned { + role = c.Style.Err("CORDONED") + ", holds only; " + role + } } c.Printf("gpu%d %s: %s used of %s, %s free (%s, read %s ago)\n", g.Index, g.Name, server.MiB(g.Used), server.MiB(g.Total), server.MiB(g.Free), role, age(g.Age)) diff --git a/internal/gateway/background.go b/internal/gateway/background.go index 12fc588..e65eebf 100644 --- a/internal/gateway/background.go +++ b/internal/gateway/background.go @@ -31,7 +31,7 @@ const DefaultIdleAfter = 60 * time.Second // beside it is the collision Berth exists to prevent. func cardFree(st *proto.Status) bool { if st == nil || st.Passthrough != nil || st.ProbeErr != "" || - len(st.Leases) > 0 || len(st.Waiting) > 0 || len(st.Squatters) > 0 { + len(st.Leases) > 0 || len(st.Waiting) > 0 || len(st.Squatters) > 0 || cordoned(st) { return false } // Status lists every claim rule; only one matching right now is in use. diff --git a/internal/gateway/deferred.go b/internal/gateway/deferred.go index 7071c33..fb31bae 100644 --- a/internal/gateway/deferred.go +++ b/internal/gateway/deferred.go @@ -19,6 +19,7 @@ import ( "io" "net/http" "os" + "slices" "strconv" "strings" "time" @@ -59,6 +60,9 @@ func outOfReach(st *proto.Status) string { if st.Passthrough != nil { return "the card is passed through to a VM" } + if cordoned(st) { + return "the card is cordoned" + } for _, l := range st.Leases { if l.Pinned { return fmt.Sprintf("%s holds the card", holderName(l)) @@ -262,3 +266,8 @@ func (g *Gateway) deferredHistory(w http.ResponseWriter, r *http.Request, id str } _ = json.NewEncoder(w).Encode(out) } + +// cordoned is whether berth cordon closed a card the status covers. +func cordoned(st *proto.Status) bool { + return slices.ContainsFunc(st.GPUs, func(g proto.GPU) bool { return g.Scheduled && g.Cordoned }) +} diff --git a/internal/node/node.go b/internal/node/node.go index bcc5e6e..0aba0fd 100644 --- a/internal/node/node.go +++ b/internal/node/node.go @@ -219,6 +219,21 @@ func (n *Node) Pick(key string) ([]*Card, error) { return out, nil } +// Cordon stops (on) or resumes new grants on the cards key picks, as Pick +// picks them, and returns their names. +func (n *Node) Cordon(key string, on bool) ([]string, error) { + cards, err := n.Pick(key) + if err != nil { + return nil, err + } + names := make([]string, len(cards)) + for i, c := range cards { + c.B.Cordon(on) + names[i] = c.Name + } + return names, nil +} + // Stop drains every lease of engine on every card. func (n *Node) Stop(engine string) ([]string, error) { var ids []string diff --git a/internal/proto/proto.go b/internal/proto/proto.go index cb6256e..17ca6d3 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -46,6 +46,12 @@ const ( // higher-priority waiter would. The server answers once with the leases // it asked. OpStop = "stop" + // OpCordon, OpUncordon: client: stop or resume new grants on the cards + // Card names (as OpAcquire names them). A cordoned card keeps its + // holders and still grants a hold. The server answers with the same op + // and Cards, the cards' names, or OpRefused for a card the node lacks. + OpCordon = "cordon" + OpUncordon = "uncordon" // OpPark: the runner holding a draining lease asks to park its engine // in RAM; the daemon answers OpParkOK or OpRefused with the why line. // OpParked: the runner's park call is done; the daemon watches for the @@ -146,7 +152,9 @@ type Msg struct { Container string `json:"container,omitempty"` Lease string `json:"lease,omitempty"` // OpStop: Leases is the reply, the leases the daemon asked to drain. - Leases []string `json:"leases,omitempty"` + Leases []string `json:"leases,omitempty"` + // OpCordon, OpUncordon: Cards is the reply, the cards' names. + Cards []string `json:"cards,omitempty"` Reason string `json:"reason,omitempty"` Status *Status `json:"status,omitempty"` Since uint64 `json:"since,omitempty"` // OpEvents: only events with a higher Seq @@ -239,6 +247,8 @@ const ( EvResumed = "lease.resumed" // a parked engine was granted the card back EvUnparked = "lease.unparked" // a parked engine loaded back onto the card with no resume; it is active again EvParkRefused = "lease.park_refused" // the RAM could not take the park; the drain runs as before + EvCordoned = "card.cordoned" // berth cordon: no new grants on the card but holds + EvUncordoned = "card.uncordoned" // berth uncordon: the card grants again EvParkEvicted = "lease.park_evicted" // the parked engine is being stopped: ttl, RAM floor or pressure EvLending = "lease.lending" // a serving holder was asked to give part of its card to a waiter EvLent = "lease.lent" // the engine gave it, and a card read after the answer shows it free @@ -422,6 +432,8 @@ type GPU struct { // Passthrough is this card's VM state while it does not serve; nil // when the card is normal. Passthrough *Passthrough `json:"passthrough,omitempty"` + // Cordoned: berth cordon stopped new grants on this card, holds aside. + Cordoned bool `json:"cordoned,omitempty"` // Telemetry as the driver reports it; -1 = not reported. Util float64 `json:"util_pct"` diff --git a/internal/server/cordon.go b/internal/server/cordon.go new file mode 100644 index 0000000..82ce820 --- /dev/null +++ b/internal/server/cordon.go @@ -0,0 +1,27 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package server + +// OpCordon and OpUncordon. The named cards stop or resume new grants; the +// reply names them. A cordon drains nothing. + +import ( + "log/slog" + + "github.com/T-Crypt/berth/internal/node" + "github.com/T-Crypt/berth/internal/proto" +) + +func cordon(conn *proto.Conn, log *slog.Logger, req proto.Msg, n *node.Node) { + if req.Card == "" { + reply(conn, log, proto.Msg{Op: proto.OpRefused, Reason: "name the card to " + req.Op}) + return + } + cards, err := n.Cordon(req.Card, req.Op == proto.OpCordon) + if err != nil { + reply(conn, log, proto.Msg{Op: proto.OpRefused, Reason: err.Error()}) + return + } + reply(conn, log, proto.Msg{Op: req.Op, Cards: cards}) +} diff --git a/internal/server/server.go b/internal/server/server.go index 4602e4a..755561c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -186,6 +186,8 @@ func handle(ctx context.Context, c net.Conn, n *node.Node, x Extras, log *slog.L reclaim(conn, log, first, n) case proto.OpStop: stop(conn, log, first, n) + case proto.OpCordon, proto.OpUncordon: + cordon(conn, log, first, n) case proto.OpPairCode, proto.OpPairRedeem, proto.OpPairCheck, proto.OpPairList, proto.OpPairRevoke: pairing(conn, log, first, x.Pair) case proto.OpResume: diff --git a/internal/skill/SKILL.md b/internal/skill/SKILL.md index a51c27c..cc07665 100644 --- a/internal/skill/SKILL.md +++ b/internal/skill/SKILL.md @@ -54,6 +54,7 @@ do not assume any engine is protected. | Name an engine that discovery does not find | an `[[engine]]` entry in `berth.toml` with `type`, `url` and, for the gateway, `start` and `serves`. A Strata entry needs `type = "strata"` (and `whole_machine = true`), or its requests never reach the ledger | | Call a model on one card of several | the card's base URL: `http://127.0.0.1:9799/cards/gpu1/v1` (same key and model names) | | Warm an engine before its first request | `GET /v1/berth/wait?model=M` on the gateway (blocks until it answers) | +| Close a card to new work while the others serve | `berth cordon gpu1` (holders keep running; `berth uncordon gpu1` reopens it) | | Stop an engine Berth started | `berth stop ENGINE [--wait]` (normal drain, parked engines included; engines outlive a gateway restart) | | Bring a parked engine back | `berth resume ENGINE` (or just call its model through the gateway) | | Read a model's file ahead into RAM | `berth prefetch MODEL` (page cache only; nothing is reserved) |