fix(quantization): pin the producing backend on imported quantized models (#11875)

core/services/quantization/service.go

@@ -455,6 +455,21 @@ func sanitizeQuantModelName(s string) string {
 	return strings.ToLower(s)
 }
 
+// inferenceBackendFor returns the backend that can load what a quantization
+// backend produced.
+//
+// The gallery publishes a quantizer as a release channel of the engine that
+// runs its output: "llama-cpp-quantization" is llama.cpp's quantizer, and the
+// GGUF it writes is served by "llama-cpp". The suffix is a channel marker and
+// carries no engine information, so stripping it yields the backend to pin in
+// the imported model's config. Names that carry no channel suffix (a backend
+// that both quantizes and serves, such as "rocmfp4") are already the engine
+// name and pass through unchanged, as do pinned hardware variants
+// ("rocm-rocmfp4"), which are valid values for a config's `backend:`.
+func inferenceBackendFor(quantBackend string) string {
+	return strings.TrimSuffix(config.NormalizeBackendName(quantBackend), "-quantization")
+}
+
 // ImportModel imports a quantized model into LocalAI asynchronously.
 func (s *QuantizationService) ImportModel(ctx context.Context, userID, jobID string, req schema.QuantizationImportRequest) (string, error) {
 	s.mu.Lock()
@@ -553,6 +568,17 @@ func (s *QuantizationService) ImportModel(ctx context.Context, userID, jobID str
 
 		cfg.Name = modelName
 
+		// The importer detects the file format and defaults to llama-cpp for any
+		// GGUF. That is wrong for a model this service just quantized with a
+		// backend stock llama.cpp cannot read: the job knows which backend
+		// produced the file, so pin that one instead of the detected default.
+		if backend := inferenceBackendFor(job.Backend); backend != "" {
+			cfg.Backend = backend
+		}
+		if job.QuantizationType != "" {
+			cfg.Description = "Quantized model (" + job.QuantizationType + ", GGUF)"
+		}
+
 		// Write YAML config
 		yamlData, err := yaml.Marshal(cfg)
 		if err != nil {
@@ -646,4 +672,4 @@ func (s *QuantizationService) GetOutputPath(userID, jobID string) (string, strin
 
 	downloadName := filepath.Base(outputFile)
 	return outputFile, downloadName, nil
-}
+}

core/services/quantization/service_test.go

@@ -175,6 +175,28 @@ var _ = Describe("QuantizationService", func() {
 		})
 	})
 
+	Describe("imported model backend", func() {
+		It("strips the quantization channel suffix so the config pins the serving engine", func() {
+			Expect(inferenceBackendFor("llama-cpp-quantization")).To(Equal("llama-cpp"))
+		})
+
+		It("leaves a backend that both quantizes and serves unchanged", func() {
+			Expect(inferenceBackendFor("rocmfp4")).To(Equal("rocmfp4"))
+		})
+
+		It("keeps a pinned hardware variant, which is a valid backend value", func() {
+			Expect(inferenceBackendFor("rocm-rocmfp4-quantization")).To(Equal("rocm-rocmfp4"))
+		})
+
+		It("normalizes dots the way gallery names are written", func() {
+			Expect(inferenceBackendFor("llama.cpp-quantization")).To(Equal("llama-cpp"))
+		})
+
+		It("returns empty for an unset backend so the detected default is kept", func() {
+			Expect(inferenceBackendFor("")).To(BeEmpty())
+		})
+	})
+
 	Describe("compile-time adapter contract", func() {
 		It("satisfies syncstate.Store for *distributed.QuantStore", func() {
 			// Guards against drift between the adapter and the component interface;
@@ -184,4 +206,4 @@ var _ = Describe("QuantizationService", func() {
 			Expect(&quantStoreAdapter{}).ToNot(BeNil())
 		})
 	})
-})
+})