From bf6c642c0d8789081b68647b2288a5451beb7dc2 Mon Sep 17 00:00:00 2001 From: Tai An Date: Fri, 4 Sep 2026 15:26:06 -0700 Subject: [PATCH] fix(quantization): pin the producing backend on imported quantized models (#11875) ImportModel hands the copied GGUF to importers.ImportLocalPath, which detects the file format and defaults every GGUF to `backend: llama-cpp`. For a model this service just produced with a backend stock llama.cpp cannot read, the generated config names an engine that cannot load the file, and the import silently registers an unloadable model. Correcting `backend:` by hand makes the same file work. The job record already carries the backend that served StartQuantization, so carry it into the config instead of keeping the detected default. The gallery publishes a quantizer as a release channel of the engine that runs its output ("llama-cpp-quantization" is llama.cpp's quantizer, whose GGUF is served by "llama-cpp"), so the channel suffix is stripped to get the serving backend. A backend that both quantizes and serves ("rocmfp4") carries no suffix and passes through unchanged, as do pinned hardware variants ("rocm-rocmfp4"), which are valid values for a config's backend field. An empty job backend leaves the detected default in place. Also replace the importer's generic "Fine-tuned model (GGUF)" description for this path: the model was quantized, not fine-tuned, and the job knows the type. Signed-off-by: Tai An --- core/services/quantization/service.go | 28 +++++++++++++++++++++- core/services/quantization/service_test.go | 24 ++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/core/services/quantization/service.go b/core/services/quantization/service.go index cd9cbceadbb5..2be44b767d51 100644 --- a/core/services/quantization/service.go +++ b/core/services/quantization/service.go @@ -455,6 +455,21 @@ func sanitizeQuantModelName(s string) string { return strings.ToLower(s) } +// inferenceBackendFor returns the backend that can load what a quantization +// backend produced. +// +// The gallery publishes a quantizer as a release channel of the engine that +// runs its output: "llama-cpp-quantization" is llama.cpp's quantizer, and the +// GGUF it writes is served by "llama-cpp". The suffix is a channel marker and +// carries no engine information, so stripping it yields the backend to pin in +// the imported model's config. Names that carry no channel suffix (a backend +// that both quantizes and serves, such as "rocmfp4") are already the engine +// name and pass through unchanged, as do pinned hardware variants +// ("rocm-rocmfp4"), which are valid values for a config's `backend:`. +func inferenceBackendFor(quantBackend string) string { + return strings.TrimSuffix(config.NormalizeBackendName(quantBackend), "-quantization") +} + // ImportModel imports a quantized model into LocalAI asynchronously. func (s *QuantizationService) ImportModel(ctx context.Context, userID, jobID string, req schema.QuantizationImportRequest) (string, error) { s.mu.Lock() @@ -553,6 +568,17 @@ func (s *QuantizationService) ImportModel(ctx context.Context, userID, jobID str cfg.Name = modelName + // The importer detects the file format and defaults to llama-cpp for any + // GGUF. That is wrong for a model this service just quantized with a + // backend stock llama.cpp cannot read: the job knows which backend + // produced the file, so pin that one instead of the detected default. + if backend := inferenceBackendFor(job.Backend); backend != "" { + cfg.Backend = backend + } + if job.QuantizationType != "" { + cfg.Description = "Quantized model (" + job.QuantizationType + ", GGUF)" + } + // Write YAML config yamlData, err := yaml.Marshal(cfg) if err != nil { @@ -646,4 +672,4 @@ func (s *QuantizationService) GetOutputPath(userID, jobID string) (string, strin downloadName := filepath.Base(outputFile) return outputFile, downloadName, nil -} +} \ No newline at end of file diff --git a/core/services/quantization/service_test.go b/core/services/quantization/service_test.go index 665728614c80..b2f2bbeca49c 100644 --- a/core/services/quantization/service_test.go +++ b/core/services/quantization/service_test.go @@ -175,6 +175,28 @@ var _ = Describe("QuantizationService", func() { }) }) + Describe("imported model backend", func() { + It("strips the quantization channel suffix so the config pins the serving engine", func() { + Expect(inferenceBackendFor("llama-cpp-quantization")).To(Equal("llama-cpp")) + }) + + It("leaves a backend that both quantizes and serves unchanged", func() { + Expect(inferenceBackendFor("rocmfp4")).To(Equal("rocmfp4")) + }) + + It("keeps a pinned hardware variant, which is a valid backend value", func() { + Expect(inferenceBackendFor("rocm-rocmfp4-quantization")).To(Equal("rocm-rocmfp4")) + }) + + It("normalizes dots the way gallery names are written", func() { + Expect(inferenceBackendFor("llama.cpp-quantization")).To(Equal("llama-cpp")) + }) + + It("returns empty for an unset backend so the detected default is kept", func() { + Expect(inferenceBackendFor("")).To(BeEmpty()) + }) + }) + Describe("compile-time adapter contract", func() { It("satisfies syncstate.Store for *distributed.QuantStore", func() { // Guards against drift between the adapter and the component interface; @@ -184,4 +206,4 @@ var _ = Describe("QuantizationService", func() { Expect(&quantStoreAdapter{}).ToNot(BeNil()) }) }) -}) +}) \ No newline at end of file