From 167902d3cd002854e81a89bd7f9ff0fd0aef39b9 Mon Sep 17 00:00:00 2001 From: Mike Garde Date: Fri, 14 Aug 2026 10:19:02 -0400 Subject: [PATCH] Add LM Studio support and generalize OpenAI client - Support `lmstudio` as an LLM provider in config and setup. - Generalize `src/llm/openai.rs` to support any OpenAI-compatible provider. - Allow optional API keys and configurable model validation logic. - Handle flexible base URLs and treat blank URLs as missing values. - Update documentation and configuration tests. --- README.md | 23 +++- commitbot.toml | 11 +- src/cli_args.rs | 2 +- src/config.rs | 9 +- src/llm/openai.rs | 299 ++++++++++++++++++++++++++++++++++++++-------- src/setup.rs | 31 ++++- tests/config.rs | 57 +++++++++ 7 files changed, 375 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index c23f7b0..eb499b5 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ It can summarize diffs, ask you how each file relates to the purpose of the comm - **Interactive “ask” mode** – Classify each file as main, supporting, or consequential. - **Quick mode** – Instantly summarize staged diffs into a commit message. -- **LLM-powered** – Uses OpenAI’s GPT models to generate concise and structured messages. +- **LLM-powered** – Uses OpenAI, or a local model via [Ollama](#providers) or [LM Studio](#providers). - **Configurable** – Choose models, tweak behavior, and set defaults in a config file. - **Pull request summaries** – Generate clean, readable PR descriptions from your commit history. @@ -26,7 +26,7 @@ It can summarize diffs, ask you how each file relates to the purpose of the comm ## Installation -You’ll need an OpenAI API key set as an environment variable: +To use OpenAI (the default provider) you’ll need an API key set as an environment variable: ```bash export OPENAI_API_KEY="sk-..." @@ -126,6 +126,25 @@ model = "gpt-5-nano" --- +## Providers + +Set `provider` to choose a backend. Any of them can be overridden per repository. + +| Provider | Default `url` | API key | +|------------|--------------------------|----------| +| `openai` | `https://api.openai.com` | required | +| `ollama` | `http://localhost:11434` | not used | +| `lmstudio` | `http://localhost:1234` | optional | + +```toml +[default] +provider = "lmstudio" +model = "qwen/qwen3-coder-30b" +url = "http://192.168.1.16:1234/v1" +``` + +--- + ## Roadmap - [x] Support for local/offline LLMs (Ollama, LM Studio). diff --git a/commitbot.toml b/commitbot.toml index 17c2a88..a2e3aaf 100644 --- a/commitbot.toml +++ b/commitbot.toml @@ -7,14 +7,14 @@ model = "gpt-5-nano" # Optional: OpenAI-style API key (falls back to env OPENAI_API_KEY) openai_api_key = "your api key here" -# Optional: provider base URL (e.g. http://localhost:11434 for Ollama) +# Optional: provider base URL (e.g. http://localhost:11434 for local LLM) url = "https://api.openai.com" # 1 = fully serial, >1 = parallel API calls max_concurrent_requests = 4 -["mikegarde/commitbot"] +["company/repo"] provider = "openai" stream = false model = "gpt-4o-mini" @@ -22,6 +22,13 @@ openai_api_key = "alternative for spend identification" max_concurrent_requests = 8 +["MikeGarde/commitbot"] +provider = "lmstudio" +model = "google/gemma-4-12b-qat" +url = "http://GPU.localdomain:1234/v1" +max_concurrent_requests = 2 + + ["company/enterprise"] # Enterprise / self-hosted style config base_url = "https://enterprise.api.endpoint" diff --git a/src/cli_args.rs b/src/cli_args.rs index dd3b845..7f7544f 100644 --- a/src/cli_args.rs +++ b/src/cli_args.rs @@ -28,7 +28,7 @@ pub struct Cli { #[arg(short = 'k', long, global = true)] pub api_key: Option, - /// LLM provider / API style (openai or ollama) + /// LLM provider / API style (openai, ollama, or lmstudio) #[arg(long, global = true)] pub provider: Option, diff --git a/src/config.rs b/src/config.rs index 654bce3..3440d8d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; /// Final resolved configuration for commitbot. #[derive(Debug, Clone)] pub struct Config { - /// LLM provider (openai, ollama) + /// LLM provider (openai, ollama, lmstudio) pub provider: String, /// OpenAI API key for authentication (sensitive – redacted in logs) pub openai_api_key: Option, @@ -55,13 +55,16 @@ impl Config { let max_concurrent_requests = r.get_usize("max_concurrent_requests", 4); let stream = r.get_bool("stream", true); - let request_timeout_secs = r.get_u64("request_timeout_secs", 90); + let request_timeout_secs = r.get_u64("request_timeout_secs", 300); // Cleanup: trim stray quotes if any upstream included them let provider = provider.trim_matches('"').to_string(); let model = model.trim_matches('"').to_string(); let openai_api_key = openai_api_key.map(|s| s.trim_matches('"').to_string()); - let base_url = base_url.map(|s| s.trim_matches('"').to_string()); + // A blank url is the same as no url + let base_url = base_url + .map(|s| s.trim_matches('"').trim().to_string()) + .filter(|s| !s.is_empty()); if provider == "openai" && openai_api_key.is_none() { return Err(anyhow!( diff --git a/src/llm/openai.rs b/src/llm/openai.rs index 72d14d1..f4bc392 100644 --- a/src/llm/openai.rs +++ b/src/llm/openai.rs @@ -5,7 +5,7 @@ use crate::FileChange; use crate::git::{PrItem, PrSummaryMode}; use anyhow::{Context, Result, anyhow}; use reqwest::StatusCode; -use reqwest::blocking::Client; +use reqwest::blocking::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; use std::io::BufReader; use std::time::Duration; @@ -48,6 +48,16 @@ struct ChatUsage { total_tokens: u32, } +#[derive(Deserialize)] +struct ModelListResponse { + data: Vec, +} + +#[derive(Deserialize)] +struct ModelListEntry { + id: String, +} + #[derive(Deserialize)] struct StreamResponse { choices: Vec, @@ -63,13 +73,26 @@ struct StreamDelta { content: Option, } -/// OpenAI-based implementation of LlmClient. +/// How to confirm the configured model exists on the upstream server. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ModelValidation { + /// `GET {base}/v1/models/{model}` — OpenAI's retrieve-model endpoint. + Retrieve, + /// `GET {base}/v1/models`, then look for the id in the listing. LM Studio + /// only implements the list endpoint, not retrieve. + List, +} + +/// Client for OpenAI and any server speaking the same chat-completions API. pub struct OpenAiClient { client: Client, - api_key: String, + api_key: Option, model: String, api_base_url: String, stream: bool, + /// Name used in logs and error messages (e.g. "OpenAI", "LM Studio"). + provider_label: String, + model_validation: ModelValidation, usage: Mutex, } @@ -82,6 +105,30 @@ struct TokenUsage { impl OpenAiClient { pub fn new(api_key: String, model: String, api_base_url: String, stream: bool, timeout_secs: u64) -> Self { + Self::openai_compatible( + Some(api_key), + model, + api_base_url, + stream, + timeout_secs, + "OpenAI", + ModelValidation::Retrieve, + ) + } + + /// Client for an OpenAI-compatible server (LM Studio, self-hosted gateways). + /// + /// `api_key` is optional: LM Studio serves unauthenticated requests by + /// default, and omitting the key sends no `Authorization` header at all. + pub fn openai_compatible( + api_key: Option, + model: String, + api_base_url: String, + stream: bool, + timeout_secs: u64, + provider_label: impl Into, + model_validation: ModelValidation, + ) -> Self { let client = Client::builder() .timeout(Duration::from_secs(timeout_secs)) .build() @@ -93,26 +140,42 @@ impl OpenAiClient { model, api_base_url: api_base_url.trim_end_matches('/').to_string(), stream, + provider_label: provider_label.into(), + model_validation, usage: Mutex::new(TokenUsage::default()), } } - fn chat_url(&self) -> String { - if self.api_base_url.ends_with("/v1") { - format!("{}/chat/completions", self.api_base_url) - } else { - format!("{}/v1/chat/completions", self.api_base_url) + /// Attach bearer auth when a key is configured, otherwise send the request + /// unauthenticated. + fn authed(&self, req: RequestBuilder) -> RequestBuilder { + match &self.api_key { + Some(key) => req.bearer_auth(key), + None => req, } } - fn model_url(&self) -> String { + /// Join `path` under `/v1`, tolerating a base URL that already ends in `/v1`. + fn v1_url(&self, path: &str) -> String { if self.api_base_url.ends_with("/v1") { - format!("{}/models/{}", self.api_base_url, self.model) + format!("{}/{}", self.api_base_url, path) } else { - format!("{}/v1/models/{}", self.api_base_url, self.model) + format!("{}/v1/{}", self.api_base_url, path) } } + fn chat_url(&self) -> String { + self.v1_url("chat/completions") + } + + fn model_url(&self) -> String { + self.v1_url(&format!("models/{}", self.model)) + } + + fn models_url(&self) -> String { + self.v1_url("models") + } + fn call_chat(&self, req: &ChatRequest) -> Result { if req.stream { return self.call_chat_streaming(req); @@ -120,34 +183,35 @@ impl OpenAiClient { let url = self.chat_url(); - log::info!("Calling OpenAI model {:?}", &req.model); + log::info!("Calling {} model {:?}", self.provider_label, req.model); let t0 = std::time::Instant::now(); let resp = self - .client - .post(url) - .bearer_auth(&self.api_key) + .authed(self.client.post(&url)) .json(req) .send() - .context("failed to send request to OpenAI")?; + .with_context(|| format!("failed to send request to {} at {}", self.provider_label, url))?; if !resp.status().is_success() { let status = resp.status(); let text = resp.text().unwrap_or_default(); return Err(anyhow!( - "OpenAI API error: HTTP {} - {}", + "{} API error: HTTP {} - {}", + self.provider_label, status.as_u16(), text )); } - let chat_resp: ChatResponse = resp.json().context("failed to parse OpenAI response")?; - log::debug!("OpenAI response time: {:.2?}", t0.elapsed()); + let chat_resp: ChatResponse = resp + .json() + .with_context(|| format!("failed to parse {} response", self.provider_label))?; + log::debug!("{} response time: {:.2?}", self.provider_label, t0.elapsed()); let content = chat_resp .choices .first() .map(|c| c.message.content.clone()) - .ok_or_else(|| anyhow!("no choices returned from OpenAI"))?; + .ok_or_else(|| anyhow!("no choices returned from {}", self.provider_label))?; if let Some(usage) = &chat_resp.usage { // Recover from a poisoned mutex instead of panicking so the CLI @@ -164,22 +228,26 @@ impl OpenAiClient { fn call_chat_streaming(&self, req: &ChatRequest) -> Result { let url = self.chat_url(); - log::info!("Streaming OpenAI model {:?}", &req.model); + log::info!("Streaming {} model {:?}", self.provider_label, req.model); let t0 = std::time::Instant::now(); let resp = self - .client - .post(url) - .bearer_auth(&self.api_key) + .authed(self.client.post(&url)) .json(req) .send() - .context("failed to send streaming request to OpenAI")?; + .with_context(|| { + format!( + "failed to send streaming request to {} at {}", + self.provider_label, url + ) + })?; if !resp.status().is_success() { let status = resp.status(); let text = resp.text().unwrap_or_default(); return Err(anyhow!( - "OpenAI API error: HTTP {} - {}", + "{} API error: HTTP {} - {}", + self.provider_label, status.as_u16(), text )); @@ -187,9 +255,96 @@ impl OpenAiClient { let reader = BufReader::new(resp); let result = read_stream_to_string(reader, parse_stream_line); - log::debug!("OpenAI streaming response time: {:.2?}", t0.elapsed()); + log::debug!( + "{} streaming response time: {:.2?}", + self.provider_label, + t0.elapsed() + ); result } + + /// `GET /v1/models/{model}` — OpenAI's retrieve-model endpoint. + fn validate_model_by_retrieve(&self) -> Result<()> { + let url = self.model_url(); + let resp = self + .authed(self.client.get(&url)) + .send() + .with_context(|| { + format!( + "failed to send model validation request to {} at {}", + self.provider_label, url + ) + })?; + + if resp.status() == StatusCode::OK { + return Ok(()); + } + + let status = resp.status(); + let text = resp.text().unwrap_or_default(); + Err(anyhow!( + "{} model validation failed for {:?} at {}: HTTP {} - {}", + self.provider_label, + self.model, + url, + status.as_u16(), + text + )) + } + + /// `GET /v1/models` plus a membership check, for servers that list models + /// but do not implement retrieve-by-id. + fn validate_model_by_list(&self) -> Result<()> { + let url = self.models_url(); + let resp = self + .authed(self.client.get(&url)) + .send() + .with_context(|| { + format!( + "failed to reach {} at {} — is the server running and reachable?", + self.provider_label, url + ) + })?; + + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().unwrap_or_default(); + return Err(anyhow!( + "{} model listing failed at {}: HTTP {} - {}", + self.provider_label, + url, + status.as_u16(), + text + )); + } + + let listed: ModelListResponse = resp + .json() + .with_context(|| format!("failed to parse model list from {url}"))?; + + if listed.data.iter().any(|m| m.id == self.model) { + return Ok(()); + } + + let available = listed + .data + .iter() + .map(|m| m.id.as_str()) + .collect::>() + .join(", "); + + Err(anyhow!( + "{} at {} has no model {:?}. Available: {}", + self.provider_label, + url, + self.model, + if available.is_empty() { + "(none)" + } else { + &available + } + )) + } } fn parse_stream_line(line: &str) -> Result> { @@ -212,27 +367,10 @@ fn parse_stream_line(line: &str) -> Result> { impl LlmClient for OpenAiClient { fn validate_model(&self) -> Result<()> { - let url = self.model_url(); - let resp = self - .client - .get(&url) - .bearer_auth(&self.api_key) - .send() - .context("failed to send model validation request to OpenAI")?; - - if resp.status() == StatusCode::OK { - return Ok(()); + match self.model_validation { + ModelValidation::Retrieve => self.validate_model_by_retrieve(), + ModelValidation::List => self.validate_model_by_list(), } - - let status = resp.status(); - let text = resp.text().unwrap_or_default(); - Err(anyhow!( - "OpenAI model validation failed for {:?} at {}: HTTP {} - {}", - self.model, - url, - status.as_u16(), - text - )) } fn summarize_file( @@ -430,4 +568,71 @@ mod tests { "https://api.openai.com/v1/models/gpt-5-nano" ); } + + fn lm_studio(base_url: &str) -> OpenAiClient { + OpenAiClient::openai_compatible( + None, + "qwen/qwen3-coder-30b".into(), + base_url.into(), + false, + 90, + "LM Studio", + ModelValidation::List, + ) + } + + #[test] + fn builds_lm_studio_urls_from_v1_base() { + let client = lm_studio("http://localhost:1234/v1"); + + assert_eq!(client.models_url(), "http://localhost:1234/v1/models"); + assert_eq!( + client.chat_url(), + "http://localhost:1234/v1/chat/completions" + ); + } + + #[test] + fn builds_lm_studio_urls_from_root_base() { + // Given without the /v1 suffix, and with a trailing slash, still lands + // on the same paths. + let client = lm_studio("http://localhost:1234/"); + + assert_eq!(client.models_url(), "http://localhost:1234/v1/models"); + assert_eq!( + client.chat_url(), + "http://localhost:1234/v1/chat/completions" + ); + } + + #[test] + fn lm_studio_validates_against_the_model_list() { + // LM Studio implements GET /v1/models but not GET /v1/models/{id}, + // so validation has to go through the listing. + assert_eq!( + lm_studio("http://localhost:1234/v1").model_validation, + ModelValidation::List + ); + assert_eq!( + OpenAiClient::new("k".into(), "m".into(), "https://api.openai.com".into(), false, 90) + .model_validation, + ModelValidation::Retrieve + ); + } + + #[test] + fn decodes_model_list_payload() { + let body = r#"{"object":"list","data":[ + {"id":"qwen/qwen3-coder-30b","object":"model","owned_by":"organization_owner"}, + {"id":"text-embedding-nomic-embed-text-v1.5","object":"model","owned_by":"organization_owner"} + ]}"#; + + let parsed: ModelListResponse = serde_json::from_str(body).expect("valid model list"); + let ids: Vec<&str> = parsed.data.iter().map(|m| m.id.as_str()).collect(); + + assert_eq!( + ids, + ["qwen/qwen3-coder-30b", "text-embedding-nomic-embed-text-v1.5"] + ); + } } diff --git a/src/setup.rs b/src/setup.rs index 62444e7..b80965b 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -2,7 +2,7 @@ use anyhow::{anyhow, Result}; use crate::config::Config; use crate::llm::LlmClient; use crate::llm::ollama::OllamaClient; -use crate::llm::openai::OpenAiClient; +use crate::llm::openai::{ModelValidation, OpenAiClient}; /// Build the LLM client based on CLI + config. pub fn build_llm_client(cfg: &Config) -> Result> { @@ -52,6 +52,33 @@ pub fn build_llm_client(cfg: &Config) -> Result> { cfg.request_timeout_secs, ))) } - other => Err(anyhow!("Unknown provider: {}", other)), + "lmstudio" => { + let base_url = cfg + .base_url + .clone() + .unwrap_or_else(|| "http://localhost:1234".to_string()); + + log::debug!( + "Using LM Studio at {} with model: {} (stream={}, timeout={}s)", + base_url, + cfg.model, + cfg.stream, + cfg.request_timeout_secs + ); + + Ok(Box::new(OpenAiClient::openai_compatible( + cfg.openai_api_key.clone(), + cfg.model.clone(), + base_url, + cfg.stream, + cfg.request_timeout_secs, + "LM Studio", + ModelValidation::List, + ))) + } + other => Err(anyhow!( + "Unknown provider: {:?} (expected one of: openai, ollama, lmstudio)", + other + )), } } diff --git a/tests/config.rs b/tests/config.rs index 3a9420e..f072860 100644 --- a/tests/config.rs +++ b/tests/config.rs @@ -108,6 +108,63 @@ stream = true fs::remove_file(config_path).ok(); } +#[test] +fn lmstudio_needs_no_url_or_api_key() { + // Like ollama: the url is optional and falls back to a local default, + // and no API key is required. + let config_path = write_temp_config( + "lmstudio_defaults", + r#" +[default] +provider = "lmstudio" +model = "qwen/qwen3-coder-30b" +"#, + ); + + let cli = Cli::parse_from([ + "commitbot", + "--config", + config_path.to_str().expect("utf-8 path"), + ]); + + let cfg = Config::from_sources(&cli).expect("lmstudio config should resolve"); + assert_eq!(cfg.provider, "lmstudio"); + assert_eq!(cfg.model, "qwen/qwen3-coder-30b"); + assert_eq!(cfg.base_url, None, "the default is applied when building the client"); + + fs::remove_file(config_path).ok(); +} + +#[test] +fn lmstudio_url_can_point_at_a_remote_host() { + let config_path = write_temp_config( + "lmstudio_remote", + r#" +[default] +provider = "lmstudio" +model = "qwen/qwen3-coder-30b" +url = "http://localhost:1234/v1" +"#, + ); + + let cli = Cli::parse_from([ + "commitbot", + "--config", + config_path.to_str().expect("utf-8 path"), + "--url", + "http://gpu.example.com:1234/v1", + ]); + + let cfg = Config::from_sources(&cli).expect("cli url should override file"); + assert_eq!(cfg.provider, "lmstudio"); + assert_eq!( + cfg.base_url.as_deref(), + Some("http://gpu.example.com:1234/v1") + ); + + fs::remove_file(config_path).ok(); +} + #[test] fn invalid_openai_config_returns_error() { let config_path = write_temp_config(