diff --git a/core/services/quantization/service.go b/core/services/quantization/service.go index cd9cbceadbb5..2be44b767d51 100644 --- a/core/services/quantization/service.go +++ b/core/services/quantization/service.go @@ -455,6 +455,21 @@ func sanitizeQuantModelName(s string) string { return strings.ToLower(s) } +// inferenceBackendFor returns the backend that can load what a quantization +// backend produced. +// +// The gallery publishes a quantizer as a release channel of the engine that +// runs its output: "llama-cpp-quantization" is llama.cpp's quantizer, and the +// GGUF it writes is served by "llama-cpp". The suffix is a channel marker and +// carries no engine information, so stripping it yields the backend to pin in +// the imported model's config. Names that carry no channel suffix (a backend +// that both quantizes and serves, such as "rocmfp4") are already the engine +// name and pass through unchanged, as do pinned hardware variants +// ("rocm-rocmfp4"), which are valid values for a config's `backend:`. +func inferenceBackendFor(quantBackend string) string { + return strings.TrimSuffix(config.NormalizeBackendName(quantBackend), "-quantization") +} + // ImportModel imports a quantized model into LocalAI asynchronously. func (s *QuantizationService) ImportModel(ctx context.Context, userID, jobID string, req schema.QuantizationImportRequest) (string, error) { s.mu.Lock() @@ -553,6 +568,17 @@ func (s *QuantizationService) ImportModel(ctx context.Context, userID, jobID str cfg.Name = modelName + // The importer detects the file format and defaults to llama-cpp for any + // GGUF. That is wrong for a model this service just quantized with a + // backend stock llama.cpp cannot read: the job knows which backend + // produced the file, so pin that one instead of the detected default. + if backend := inferenceBackendFor(job.Backend); backend != "" { + cfg.Backend = backend + } + if job.QuantizationType != "" { + cfg.Description = "Quantized model (" + job.QuantizationType + ", GGUF)" + } + // Write YAML config yamlData, err := yaml.Marshal(cfg) if err != nil { @@ -646,4 +672,4 @@ func (s *QuantizationService) GetOutputPath(userID, jobID string) (string, strin downloadName := filepath.Base(outputFile) return outputFile, downloadName, nil -} +} \ No newline at end of file diff --git a/core/services/quantization/service_test.go b/core/services/quantization/service_test.go index 665728614c80..b2f2bbeca49c 100644 --- a/core/services/quantization/service_test.go +++ b/core/services/quantization/service_test.go @@ -175,6 +175,28 @@ var _ = Describe("QuantizationService", func() { }) }) + Describe("imported model backend", func() { + It("strips the quantization channel suffix so the config pins the serving engine", func() { + Expect(inferenceBackendFor("llama-cpp-quantization")).To(Equal("llama-cpp")) + }) + + It("leaves a backend that both quantizes and serves unchanged", func() { + Expect(inferenceBackendFor("rocmfp4")).To(Equal("rocmfp4")) + }) + + It("keeps a pinned hardware variant, which is a valid backend value", func() { + Expect(inferenceBackendFor("rocm-rocmfp4-quantization")).To(Equal("rocm-rocmfp4")) + }) + + It("normalizes dots the way gallery names are written", func() { + Expect(inferenceBackendFor("llama.cpp-quantization")).To(Equal("llama-cpp")) + }) + + It("returns empty for an unset backend so the detected default is kept", func() { + Expect(inferenceBackendFor("")).To(BeEmpty()) + }) + }) + Describe("compile-time adapter contract", func() { It("satisfies syncstate.Store for *distributed.QuantStore", func() { // Guards against drift between the adapter and the component interface; @@ -184,4 +206,4 @@ var _ = Describe("QuantizationService", func() { Expect(&quantStoreAdapter{}).ToNot(BeNil()) }) }) -}) +}) \ No newline at end of file