fix(distributed): exclude pinned models from cluster eviction and idle scale-down

core/application/distributed.go

@@ -365,8 +365,10 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
 
 	// All dependencies ready — build SmartRouter with all options at once
 	var conflictResolver nodes.ConcurrencyConflictResolver
+	var pinnedResolver nodes.PinnedModelResolver
 	if configLoader != nil {
 		conflictResolver = configLoader
+		pinnedResolver = configLoader
 	}
 	modelCleanup := nodes.NewModelCleanupService(registry, remoteUnloader)
 	router := nodes.NewSmartRouter(registry, nodes.SmartRouterOptions{
@@ -377,6 +379,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
 		AuthToken:        routerAuthToken,
 		DB:               authDB,
 		ConflictResolver: conflictResolver,
+		PinnedResolver:   pinnedResolver,
 		PrefixProvider:   prefixProvider,
 		PrefixConfig:     prefixCfg,
 		Pressure:         pressure,
@@ -439,6 +442,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
 		ProbeStaleAfter:   2 * time.Minute,
 		Pressure:          pressure,
 		PressureThreshold: prefixCfg.PressureScaleThreshold,
+		PinnedResolver:    pinnedResolver,
 	})
 
 	// Create ModelRouterAdapter to wire into ModelLoader

core/config/model_config_loader.go

@@ -412,6 +412,26 @@ func (bcl *ModelConfigLoader) GetModelsConflictingWith(name string) []string {
 	return conflicts
 }
 
+// GetPinnedModelNames returns the names of every configured, not-disabled
+// model with `pinned: true`. The distributed router and replica reconciler
+// consult this so cluster-wide eviction honours the same pin contract the
+// local watchdog enforces (#11101) — without it, a pinned model becomes
+// eviction-eligible the moment its in-flight count drops to zero.
+func (bcl *ModelConfigLoader) GetPinnedModelNames() []string {
+	bcl.Lock()
+	defer bcl.Unlock()
+	var pinned []string
+	for n, cfg := range bcl.configs {
+		if cfg.IsDisabled() {
+			continue
+		}
+		if cfg.IsPinned() {
+			pinned = append(pinned, n)
+		}
+	}
+	return pinned
+}
+
 // UpdateModelConfig updates an existing model config in the loader.
 // This is useful for updating runtime-detected properties like thinking support.
 func (bcl *ModelConfigLoader) UpdateModelConfig(m string, updater func(*ModelConfig)) {

core/config/model_config_loader_test.go

@@ -267,6 +267,40 @@ var _ = Describe("ModelConfigLoader.GetModelsConflictingWith", func() {
 	})
 })
 
+var _ = Describe("ModelConfigLoader.GetPinnedModelNames", func() {
+	var bcl *ModelConfigLoader
+
+	BeforeEach(func() {
+		bcl = NewModelConfigLoader("/tmp/pinned-test-models")
+	})
+
+	insert := func(cfg ModelConfig) {
+		bcl.Lock()
+		bcl.configs[cfg.Name] = cfg
+		bcl.Unlock()
+	}
+
+	boolPtr := func(b bool) *bool { return &b }
+
+	It("returns nil when nothing is pinned", func() {
+		insert(ModelConfig{Name: "a"})
+		insert(ModelConfig{Name: "b", Pinned: boolPtr(false)})
+		Expect(bcl.GetPinnedModelNames()).To(BeNil())
+	})
+
+	It("returns only pinned, enabled models", func() {
+		insert(ModelConfig{Name: "a", Pinned: boolPtr(true)})
+		insert(ModelConfig{Name: "b"})
+		insert(ModelConfig{Name: "c", Pinned: boolPtr(true)})
+		Expect(bcl.GetPinnedModelNames()).To(ConsistOf("a", "c"))
+	})
+
+	It("ignores disabled pinned models", func() {
+		insert(ModelConfig{Name: "a", Pinned: boolPtr(true), Disabled: boolPtr(true)})
+		Expect(bcl.GetPinnedModelNames()).To(BeNil())
+	})
+})
+
 var _ = Describe("ModelConfigLoader alias resolution", func() {
 	var loader *ModelConfigLoader
 

core/services/nodes/interfaces.go

@@ -46,7 +46,7 @@ type ModelRouter interface {
 	FindIdleNode(ctx context.Context) (*BackendNode, error)
 	FindLeastLoadedNode(ctx context.Context) (*BackendNode, error)
 	FindGlobalLRUModelWithZeroInFlight(ctx context.Context) (*NodeModel, error)
-	FindLRUModel(ctx context.Context, nodeID string) (*NodeModel, error)
+	FindLRUModel(ctx context.Context, nodeID string, excludeModels []string) (*NodeModel, error)
 	Get(ctx context.Context, nodeID string) (*BackendNode, error)
 	GetModelScheduling(ctx context.Context, modelName string) (*ModelSchedulingConfig, error)
 	GetGoverningScheduling(ctx context.Context, modelName string) (*ModelSchedulingConfig, error)
@@ -84,6 +84,15 @@ type ConcurrencyConflictResolver interface {
 	GetModelsConflictingWith(modelName string) []string
 }
 
+// PinnedModelResolver reports which configured models are pinned. Satisfied
+// by *config.ModelConfigLoader. The router's eviction paths and the
+// reconciler's idle scale-down exclude these models so `pinned: true` holds
+// cluster-wide, not just against the per-node watchdog (#11101). Deliberate
+// teardown (admin unload, model delete, node drain) intentionally bypasses it.
+type PinnedModelResolver interface {
+	GetPinnedModelNames() []string
+}
+
 // NodeHealthStore is used by HealthMonitor for node status management.
 type NodeHealthStore interface {
 	List(ctx context.Context) ([]BackendNode, error)

core/services/nodes/model_router_test.go

@@ -114,7 +114,7 @@ func (f *fakeModelRouterForSmartRouter) FindLeastLoadedNode(_ context.Context) (
 func (f *fakeModelRouterForSmartRouter) FindGlobalLRUModelWithZeroInFlight(_ context.Context) (*NodeModel, error) {
 	return nil, nil
 }
-func (f *fakeModelRouterForSmartRouter) FindLRUModel(_ context.Context, _ string) (*NodeModel, error) {
+func (f *fakeModelRouterForSmartRouter) FindLRUModel(_ context.Context, _ string, _ []string) (*NodeModel, error) {
 	return nil, nil
 }
 func (f *fakeModelRouterForSmartRouter) Get(_ context.Context, nodeID string) (*BackendNode, error) {

core/services/nodes/pinned_eviction_test.go

@@ -0,0 +1,235 @@
+package nodes
+
+// Regression tests for #11101: `pinned: true` must hold cluster-wide, not
+// just against the per-node watchdog. Before the fix, every distributed
+// eviction path (EvictLRU, evictLRUAndFreeNode, scaleDownIdle) was
+// pinned-blind, so a pinned model became eviction-eligible the moment its
+// in-flight count dropped to zero — observable as the backend being freed
+// immediately after every request under capacity pressure.
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/services/testutil"
+	"gorm.io/gorm"
+	"runtime"
+)
+
+type fakePinnedResolver struct {
+	names []string
+}
+
+func (f *fakePinnedResolver) GetPinnedModelNames() []string { return f.names }
+
+var _ = Describe("Pinned models vs distributed eviction (#11101)", func() {
+	Describe("EvictLRU (mock-based)", func() {
+		It("excludes pinned models at the query, evicting nothing when only a pinned model is idle", func() {
+			reg := &fakeModelRouter{
+				findLRUModel: &NodeModel{NodeID: "n1", ModelName: "pinned-model"},
+			}
+			unloader := &fakeUnloader{}
+
+			router := NewSmartRouter(reg, SmartRouterOptions{
+				Unloader:       unloader,
+				PinnedResolver: &fakePinnedResolver{names: []string{"pinned-model"}},
+			})
+
+			_, err := router.EvictLRU(context.Background(), "n1")
+			Expect(err).To(HaveOccurred())
+			// The exclusion must reach the registry query: filtering after
+			// selection would just burn the attempt instead of picking the
+			// next-oldest unpinned model.
+			Expect(reg.findLRUExclude).To(ContainElement("pinned-model"))
+			Expect(unloader.stopCalls).To(BeEmpty())
+		})
+
+		It("still evicts unpinned models when a resolver is wired", func() {
+			reg := &fakeModelRouter{
+				findLRUModel: &NodeModel{NodeID: "n1", ModelName: "plain-model"},
+			}
+			unloader := &fakeUnloader{}
+
+			router := NewSmartRouter(reg, SmartRouterOptions{
+				Unloader:       unloader,
+				PinnedResolver: &fakePinnedResolver{names: []string{"some-other-model"}},
+			})
+
+			evicted, err := router.EvictLRU(context.Background(), "n1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(evicted).To(Equal("plain-model"))
+			Expect(unloader.stopCalls).To(ContainElement("n1:plain-model"))
+		})
+	})
+
+	Describe("evictLRUAndFreeNode (integration)", func() {
+		var (
+			db       *gorm.DB
+			registry *NodeRegistry
+			node     *BackendNode
+		)
+
+		BeforeEach(func() {
+			if runtime.GOOS == "darwin" {
+				Skip("testcontainers requires Docker, not available on macOS CI")
+			}
+			db = testutil.SetupTestDB()
+			var err error
+			registry, err = NewNodeRegistry(db)
+			Expect(err).ToNot(HaveOccurred())
+
+			node = &BackendNode{
+				Name:     "pinned-evict-node",
+				NodeType: NodeTypeBackend,
+				Address:  "10.0.0.200:50051",
+			}
+			Expect(registry.Register(context.Background(), node, true)).To(Succeed())
+			Expect(registry.MarkHealthy(context.Background(), node.ID)).To(Succeed())
+		})
+
+		// setLoadedModel creates an idle loaded replica row with a controlled
+		// last_used so the specs can dictate LRU order.
+		setLoadedModel := func(name string, lastUsed time.Time) {
+			Expect(registry.SetNodeModel(context.Background(), node.ID, name, 0, "loaded", "", 0)).To(Succeed())
+			Expect(db.Model(&NodeModel{}).
+				Where("node_id = ? AND model_name = ?", node.ID, name).
+				Update("last_used", lastUsed).Error).ToNot(HaveOccurred())
+		}
+
+		It("skips the pinned LRU model and evicts the next-oldest unpinned one", func() {
+			// The pinned model is the older (= LRU) candidate. Without the
+			// exclusion it would be selected first.
+			setLoadedModel("pinned-old", time.Now().Add(-2*time.Hour))
+			setLoadedModel("plain-new", time.Now().Add(-1*time.Hour))
+
+			router := NewSmartRouter(registry, SmartRouterOptions{
+				DB:             db,
+				PinnedResolver: &fakePinnedResolver{names: []string{"pinned-old"}},
+			})
+
+			freed, err := router.evictLRUAndFreeNode(context.Background())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(freed.ID).To(Equal(node.ID))
+
+			var remaining []NodeModel
+			Expect(db.Where("node_id = ?", node.ID).Find(&remaining).Error).ToNot(HaveOccurred())
+			Expect(remaining).To(HaveLen(1))
+			Expect(remaining[0].ModelName).To(Equal("pinned-old"))
+		})
+
+		It("returns ErrEvictionBusy instead of evicting when every idle model is pinned", func() {
+			setLoadedModel("pinned-only", time.Now().Add(-2*time.Hour))
+
+			router := NewSmartRouter(registry, SmartRouterOptions{
+				DB:             db,
+				PinnedResolver: &fakePinnedResolver{names: []string{"pinned-only"}},
+			})
+
+			_, err := router.evictLRUAndFreeNode(context.Background())
+			Expect(err).To(MatchError(ErrEvictionBusy))
+
+			var count int64
+			Expect(db.Model(&NodeModel{}).Where("model_name = ?", "pinned-only").Count(&count).Error).ToNot(HaveOccurred())
+			Expect(count).To(Equal(int64(1)), "the pinned model's replica row must survive")
+		})
+
+		It("still evicts the LRU model when no resolver is wired (embedder/back-compat path)", func() {
+			setLoadedModel("plain-old", time.Now().Add(-2*time.Hour))
+
+			router := NewSmartRouter(registry, SmartRouterOptions{DB: db})
+
+			freed, err := router.evictLRUAndFreeNode(context.Background())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(freed.ID).To(Equal(node.ID))
+		})
+	})
+
+	Describe("scaleDownIdle (integration)", func() {
+		var (
+			db       *gorm.DB
+			registry *NodeRegistry
+		)
+
+		BeforeEach(func() {
+			if runtime.GOOS == "darwin" {
+				Skip("testcontainers requires Docker, not available on macOS CI")
+			}
+			db = testutil.SetupTestDB()
+			var err error
+			registry, err = NewNodeRegistry(db)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		registerNode := func(name, address string) *BackendNode {
+			node := &BackendNode{
+				Name:                name,
+				NodeType:            NodeTypeBackend,
+				Address:             address,
+				MaxReplicasPerModel: 4,
+			}
+			Expect(registry.Register(context.Background(), node, true)).To(Succeed())
+			return node
+		}
+
+		It("does not scale down idle replicas of a pinned model", func() {
+			node1 := registerNode("pin-idle-1", "10.0.1.1:50051")
+			node2 := registerNode("pin-idle-2", "10.0.1.2:50051")
+			node3 := registerNode("pin-idle-3", "10.0.1.3:50051")
+			Expect(registry.SetModelScheduling(context.Background(), &ModelSchedulingConfig{
+				ModelName: "pinned-replicated", MinReplicas: 1, MaxReplicas: 4,
+			})).To(Succeed())
+
+			// Three idle replicas above the floor of one — prime scale-down bait.
+			pastTime := time.Now().Add(-10 * time.Minute)
+			for _, n := range []*BackendNode{node1, node2, node3} {
+				Expect(registry.SetNodeModel(context.Background(), n.ID, "pinned-replicated", 0, "loaded", "", 0)).To(Succeed())
+				db.Model(&NodeModel{}).Where("node_id = ? AND model_name = ?", n.ID, "pinned-replicated").
+					Update("last_used", pastTime)
+			}
+
+			unloader := &fakeUnloader{}
+			reconciler := NewReplicaReconciler(ReplicaReconcilerOptions{
+				Registry:       registry,
+				Unloader:       unloader,
+				DB:             db,
+				ScaleDownDelay: 1 * time.Minute,
+				PinnedResolver: &fakePinnedResolver{names: []string{"pinned-replicated"}},
+			})
+
+			reconciler.reconcile(context.Background())
+
+			Expect(unloader.unloadCalls).To(BeEmpty())
+		})
+
+		It("scales down unpinned models normally with a resolver wired", func() {
+			node1 := registerNode("unpin-idle-1", "10.0.1.4:50051")
+			node2 := registerNode("unpin-idle-2", "10.0.1.5:50051")
+			Expect(registry.SetModelScheduling(context.Background(), &ModelSchedulingConfig{
+				ModelName: "plain-replicated", MinReplicas: 1, MaxReplicas: 4,
+			})).To(Succeed())
+
+			pastTime := time.Now().Add(-10 * time.Minute)
+			for _, n := range []*BackendNode{node1, node2} {
+				Expect(registry.SetNodeModel(context.Background(), n.ID, "plain-replicated", 0, "loaded", "", 0)).To(Succeed())
+				db.Model(&NodeModel{}).Where("node_id = ? AND model_name = ?", n.ID, "plain-replicated").
+					Update("last_used", pastTime)
+			}
+
+			unloader := &fakeUnloader{}
+			reconciler := NewReplicaReconciler(ReplicaReconcilerOptions{
+				Registry:       registry,
+				Unloader:       unloader,
+				DB:             db,
+				ScaleDownDelay: 1 * time.Minute,
+				PinnedResolver: &fakePinnedResolver{names: []string{"some-other-model"}},
+			})
+
+			reconciler.reconcile(context.Background())
+
+			Expect(unloader.unloadCalls).To(HaveLen(1))
+		})
+	})
+})