diff --git a/docs/providers/sglang.mdx b/docs/providers/sglang.mdx index 4c6036fb1..aed2fba93 100644 --- a/docs/providers/sglang.mdx +++ b/docs/providers/sglang.mdx @@ -194,6 +194,10 @@ GoModel strips client authorization before forwarding and applies - Chat completions, streaming, model listing, Responses, and embeddings use SGLang's OpenAI-compatible API. +- SGLang's native `/v1/responses` endpoint rejects tool-bearing requests on + some versions. Set `api_mode: chat_compatible` in YAML (or + `SGLANG_API_MODE=chat_compatible`) to translate Responses API calls through + chat completions instead, which keeps function calling working. - Embeddings and model-specific features depend on the model loaded by SGLang. - Native batch, file, and stored-response lifecycle interfaces are not yet exposed as typed GoModel provider capabilities; use passthrough where the diff --git a/internal/providers/openai/openai.go b/internal/providers/openai/openai.go index 677a36b72..46473e3a3 100644 --- a/internal/providers/openai/openai.go +++ b/internal/providers/openai/openai.go @@ -2,6 +2,8 @@ package openai import ( + "context" + "io" "net/http" "strconv" "strings" @@ -27,12 +29,29 @@ const ( defaultBaseURL = "https://api.openai.com/v1" ) +// apiModeResponsesViaChat lists the APIMode values that switch the Responses +// API from native upstream passthrough to translation over chat completions. +// OpenAI-compatible upstreams such as SGLang or older vLLM builds expose +// /v1/responses but reject or fail on tool-bearing requests; routing through +// chat completions keeps function calling working on those upstreams. +var apiModeResponsesViaChat = map[string]bool{ + "chat": true, + "chat_completions": true, + "chat-compatible": true, + "chat_compatible": true, + "responses_via_chat": true, + "responses-via-chat": true, +} + // Provider implements the core.Provider interface for OpenAI. // Credentials and the realtime base URL are both read live from the embedded // CompatibleProvider, so SetBaseURL overrides and key rotation are honored on // the realtime websocket dial target too (see realtime.go). type Provider struct { *CompatibleProvider + // responsesViaChat translates Responses API calls through chat + // completions instead of forwarding to the upstream /responses endpoint. + responsesViaChat bool } // New creates a new OpenAI provider. @@ -44,7 +63,27 @@ func New(cfg providers.ProviderConfig, opts providers.ProviderOptions) core.Prov BaseURL: baseURL, SetHeaders: setHeaders, }), + responsesViaChat: apiModeResponsesViaChat[strings.ToLower(strings.TrimSpace(cfg.APIMode))], + } +} + +// Responses sends a Responses API request. With APIMode set to a +// chat-compatible value the request is translated through chat completions; +// otherwise it is forwarded to the upstream /responses endpoint. +func (p *Provider) Responses(ctx context.Context, req *core.ResponsesRequest) (*core.ResponsesResponse, error) { + if p.responsesViaChat { + return providers.ResponsesViaChat(ctx, p, req) + } + return p.CompatibleProvider.Responses(ctx, req) +} + +// StreamResponses streams a Responses API request, honoring the same APIMode +// translation switch as Responses. +func (p *Provider) StreamResponses(ctx context.Context, req *core.ResponsesRequest) (io.ReadCloser, error) { + if p.responsesViaChat { + return providers.StreamResponsesViaChat(ctx, p, req, "openai") } + return p.CompatibleProvider.StreamResponses(ctx, req) } // NewWithHTTPClient creates a new OpenAI provider with a custom HTTP client. diff --git a/internal/providers/openai/openai_test.go b/internal/providers/openai/openai_test.go index acb1b6188..670dda353 100644 --- a/internal/providers/openai/openai_test.go +++ b/internal/providers/openai/openai_test.go @@ -37,6 +37,108 @@ func TestNew_ReturnsProvider(t *testing.T) { } } +func TestResponses_APIModeChatCompatibleTranslatesViaChat(t *testing.T) { + var gotPath string + var gotBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Errorf("failed to decode request body: %v", err) + } + if r.URL.Path == "/responses" { + // Simulate an upstream (e.g. SGLang) whose native /responses + // endpoint cannot serve tool-bearing requests. + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`Internal Server Error`)) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": "chatcmpl_123", + "object": "chat.completion", + "created": 1677652288, + "model": "glm-5.2", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12} + }`)) + })) + defer server.Close() + + provider := New(providers.ProviderConfig{ + APIKey: "test-api-key", + BaseURL: server.URL, + APIMode: "chat_compatible", + }, providers.ProviderOptions{}) + + resp, err := provider.Responses(context.Background(), &core.ResponsesRequest{ + Model: "glm-5.2", + Input: "Reply exactly: OK", + Tools: []map[string]any{{ + "type": "function", + "name": "get_weather", + "description": "Get weather", + "parameters": map[string]any{"type": "object"}, + }}, + }) + if err != nil { + t.Fatalf("Responses returned error: %v", err) + } + if gotPath != "/chat/completions" { + t.Fatalf("request path = %q, want /chat/completions", gotPath) + } + tools, ok := gotBody["tools"].([]any) + if !ok || len(tools) != 1 { + t.Fatalf("translated chat request tools = %v, want exactly one tool", gotBody["tools"]) + } + tool := tools[0].(map[string]any) + fn, ok := tool["function"].(map[string]any) + if !ok { + t.Fatalf("translated tool = %v, want chat-completions function shape", tool) + } + if fn["name"] != "get_weather" { + t.Errorf("translated tool name = %q, want get_weather", fn["name"]) + } + if resp == nil || resp.Model != "glm-5.2" { + t.Fatalf("response = %+v, want model glm-5.2", resp) + } +} + +func TestResponses_APIModeDefaultKeepsNativePassthrough(t *testing.T) { + var gotPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": "resp_123", + "object": "response", + "created_at": 1677652288, + "model": "gpt-4o", + "status": "completed", + "output": [] + }`)) + })) + defer server.Close() + + provider := New(providers.ProviderConfig{ + APIKey: "test-api-key", + BaseURL: server.URL, + }, providers.ProviderOptions{}) + + if _, err := provider.Responses(context.Background(), &core.ResponsesRequest{ + Model: "gpt-4o", + Input: "ping", + }); err != nil { + t.Fatalf("Responses returned error: %v", err) + } + if gotPath != "/responses" { + t.Fatalf("request path = %q, want native /responses passthrough", gotPath) + } +} + func TestNilRequests_ReturnInvalidRequestError(t *testing.T) { provider := NewWithHTTPClient("test-api-key", nil, llmclient.Hooks{})