diff --git a/i18n/en_US.yaml b/i18n/en_US.yaml index 5d1faa3e0..e8cdd7aef 100644 --- a/i18n/en_US.yaml +++ b/i18n/en_US.yaml @@ -869,6 +869,9 @@ ui: ask_placeholder: Ask a question thinking: Thinking… thoughts: Thoughts + image_upload: Add images + image_too_large: Image exceeds the 4MB limit + image_max_count: You can attach up to 4 images notifications: title: Notifications inbox: Inbox @@ -2360,6 +2363,12 @@ ui: model: label: Model msg: Model is required + thinking_mode: + label: Deep thinking mode + tip: Sends the enable_thinking flag so reasoning-capable models show their thought process. + vision_enabled: + label: Allow image input + tip: Let users attach up to 4 PNG/JPEG/WebP images (max 4MB each) to AI conversations. add_success: AI settings updated successfully. conversations: topic: Topic diff --git a/i18n/zh_CN.yaml b/i18n/zh_CN.yaml index f16ed9fad..ef0219c42 100644 --- a/i18n/zh_CN.yaml +++ b/i18n/zh_CN.yaml @@ -856,6 +856,9 @@ ui: copy: 复制 ask_a_follow_up: 提出后续问题 ask_placeholder: 提问 + image_upload: 添加图片 + image_too_large: 图片超过 4MB 限制 + image_max_count: 最多只能添加 4 张图片 notifications: title: 通知 inbox: 收件箱 @@ -2318,6 +2321,12 @@ ui: model: label: 模型 msg: 模型是必需的 + thinking_mode: + label: 深度思考模式 + tip: 开启后请求将携带 enable_thinking 参数，支持推理的模型将输出思考过程。 + vision_enabled: + label: 允许图片输入 + tip: 允许用户在与 AI 对话时附带最多 4 张 PNG/JPEG/WebP 图片（每张不超过 4MB）。 add_success: AI 设置更新成功。 conversations: topic: 主题 diff --git a/internal/controller/ai_controller.go b/internal/controller/ai_controller.go index e7495253b..459e56358 100644 --- a/internal/controller/ai_controller.go +++ b/internal/controller/ai_controller.go @@ -111,6 +111,9 @@ type ChatCompletionsRequest struct { type Message struct { Role string `json:"role" binding:"required"` Content string `json:"content" binding:"required"` + // Images carries optional attachments (base64 data URLs or HTTPS URLs) + // for vision-capable providers. Only the first user message accepts them. + Images []string `json:"images,omitempty"` } type ChatCompletionsResponse struct { @@ -161,6 +164,9 @@ type ConversationContext struct { Messages []*ai_conversation.ConversationMessage IsNewConversation bool Model string + // Images holds the attachments of the current user turn only; history + // records keep a textual placeholder instead of persisting image data. + Images []string } func (c *ConversationContext) GetOpenAIMessages() []openai.ChatCompletionMessage { @@ -174,6 +180,22 @@ func (c *ConversationContext) GetOpenAIMessages() []openai.ChatCompletionMessage return messages } +// getVisionMessages returns the OpenAI messages with the current turn's images +// attached to the final user message as MultiContent parts. +func (c *ConversationContext) getVisionMessages() ([]openai.ChatCompletionMessage, error) { + messages := c.GetOpenAIMessages() + last := len(messages) - 1 + if last < 0 || messages[last].Role != openai.ChatMessageRoleUser { + return messages, nil + } + vmsg, err := ValidateAndPrepareImages(messages[last].Content, c.Images) + if err != nil { + return nil, err + } + messages[last] = *vmsg + return messages, nil +} + // sendStreamData func sendStreamData(w http.ResponseWriter, data StreamResponse) { jsonData, err := json.Marshal(data) @@ -211,8 +233,31 @@ func (c *AIController) ChatCompletions(ctx *gin.Context) { } req.UserID = middleware.GetLoginUserIDFromContext(ctx) - data, _ := json.Marshal(req) - log.Infof("ai chat request data: %s", string(data)) + // Reject or validate image attachments before the SSE stream starts, so + // clients still receive a proper JSON error with the right status code. + if len(req.Messages) > 0 && len(req.Messages[0].Images) > 0 { + if !aiProvider.VisionEnabled { + handler.HandleResponse(ctx, errors.BadRequest("image input is not enabled for the current AI provider"), nil) + return + } + if _, verr := ValidateAndPrepareImages(req.Messages[0].Content, req.Messages[0].Images); verr != nil { + handler.HandleResponse(ctx, errors.BadRequest(verr.Error()), nil) + return + } + } + + // Never dump image data into logs; summarize attachments as a count. + if len(req.Messages) > 0 && len(req.Messages[0].Images) > 0 { + logReq := *req + logReq.Messages = make([]Message, len(req.Messages)) + copy(logReq.Messages, req.Messages) + logReq.Messages[0].Images = []string{fmt.Sprintf("<%d images>", len(req.Messages[0].Images))} + data, _ := json.Marshal(logReq) + log.Infof("ai chat request data: %s", string(data)) + } else { + data, _ := json.Marshal(req) + log.Infof("ai chat request data: %s", string(data)) + } ctx.Header("Content-Type", "text/event-stream") ctx.Header("Cache-Control", "no-cache") @@ -271,32 +316,34 @@ func (c *AIController) ChatCompletions(ctx *gin.Context) { func (c *AIController) redirectRequestToAI(ctx *gin.Context, w http.ResponseWriter, id string, conversationCtx *ConversationContext) { client := c.createOpenAIClient() + if client == nil { + c.sendErrorResponse(w, id, conversationCtx.Model, "AI service is not properly configured") + return + } c.handleAIConversation(ctx, w, id, client, conversationCtx) } -// createOpenAIClient +// createOpenAIClient builds an OpenAI-compatible client from the site AI +// provider config. Returns nil when AI is disabled or config cannot be read. func (c *AIController) createOpenAIClient() *openai.Client { - config := openai.DefaultConfig("") - config.BaseURL = "" - aiConfig, err := c.siteInfoService.GetSiteAI(context.Background()) if err != nil { log.Errorf("Failed to get AI config: %v", err) - return openai.NewClientWithConfig(config) + return nil } if !aiConfig.Enabled { log.Warn("AI feature is disabled") - return openai.NewClientWithConfig(config) + return nil } aiProvider := aiConfig.GetProvider() - config = openai.DefaultConfig(aiProvider.APIKey) - config.BaseURL = aiProvider.APIHost - if !strings.HasSuffix(config.BaseURL, "/v1") { - config.BaseURL += "/v1" + config := openai.DefaultConfig(aiProvider.APIKey) + config.BaseURL = schema.NormalizeAPIHost(aiProvider.APIHost) + if aiProvider.ThinkingMode == "on" { + config.HTTPClient = newThinkingHTTPClient() } return openai.NewClientWithConfig(config) } @@ -350,6 +397,9 @@ func (c *AIController) initializeConversationContext(ctx *gin.Context, model str ConversationID: req.ConversationID, Model: model, } + if len(req.Messages) > 0 { + conversationCtx.Images = req.Messages[0].Images + } conversationDetail, exist, err := c.aiConversationService.GetConversationDetail(ctx, &schema.AIConversationDetailReq{ ConversationID: req.ConversationID, @@ -361,6 +411,10 @@ func (c *AIController) initializeConversationContext(ctx *gin.Context, model str } if !exist { conversationCtx.UserQuestion = req.Messages[0].Content + if len(conversationCtx.Images) > 0 { + // Persist a placeholder only; image data is not stored. + conversationCtx.UserQuestion += "\n[图片]" + } conversationCtx.Messages = c.buildInitialMessages(ctx, req) conversationCtx.IsNewConversation = true return conversationCtx @@ -432,7 +486,12 @@ func (c *AIController) saveConversationRecord(ctx context.Context, chatcmplID st func (c *AIController) handleAIConversation(ctx *gin.Context, w http.ResponseWriter, id string, client *openai.Client, conversationCtx *ConversationContext) { maxRounds := 10 - messages := conversationCtx.GetOpenAIMessages() + messages, err := conversationCtx.getVisionMessages() + if err != nil { + log.Errorf("Failed to prepare vision messages: %v", err) + c.sendErrorResponse(w, id, conversationCtx.Model, err.Error()) + return + } for round := range maxRounds { log.Debugf("AI conversation round: %d", round+1) diff --git a/internal/controller/ai_thinking_transport.go b/internal/controller/ai_thinking_transport.go new file mode 100644 index 000000000..cbab7d22a --- /dev/null +++ b/internal/controller/ai_thinking_transport.go @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package controller + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "strconv" + "strings" +) + +// ThinkingParamKey is the OpenAI-compatible request flag understood by +// reasoning-capable model gateways (DeepSeek, Qwen/DashScope, SenseNova, ...). +const ThinkingParamKey = "enable_thinking" + +// thinkingTransport merges the thinking flag into chat completion request +// bodies before they leave the process. The openai SDK has no generic extra +// body hook, so an http.Client with this transport is attached to the client +// config when the provider enables thinking mode. +type thinkingTransport struct{ base http.RoundTripper } + +func (t *thinkingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Body != nil && req.ContentLength != 0 && + strings.HasSuffix(req.URL.Path, "/chat/completions") { + b, err := io.ReadAll(req.Body) + _ = req.Body.Close() + if err == nil { + var payload map[string]any + if json.Unmarshal(b, &payload) == nil && payload != nil { + payload[ThinkingParamKey] = true + nb, mErr := json.Marshal(payload) + if mErr == nil { + req.Body = io.NopCloser(bytes.NewReader(nb)) + req.ContentLength = int64(len(nb)) + req.Header.Set("Content-Length", strconv.Itoa(len(nb))) + } else { + req.Body = io.NopCloser(bytes.NewReader(b)) + } + } else { + req.Body = io.NopCloser(bytes.NewReader(b)) + } + } + } + if t.base == nil { + return http.DefaultTransport.RoundTrip(req) + } + return t.base.RoundTrip(req) +} + +func newThinkingHTTPClient() *http.Client { + return &http.Client{Transport: &thinkingTransport{}} +} diff --git a/internal/controller/ai_thinking_transport_test.go b/internal/controller/ai_thinking_transport_test.go new file mode 100644 index 000000000..08c5c077e --- /dev/null +++ b/internal/controller/ai_thinking_transport_test.go @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package controller + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestThinkingTransportInjectsFlag(t *testing.T) { + var got map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &got) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + client := newThinkingHTTPClient() + body := `{"model":"deepseek-v4-flash","messages":[{"role":"user","content":"hi"}],"stream":true}` + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + srv.URL+"/v1/chat/completions", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + + if got["enable_thinking"] != true { + t.Fatalf("enable_thinking not injected: %v", got["enable_thinking"]) + } + if got["model"] != "deepseek-v4-flash" { + t.Fatalf("original fields lost: %v", got) + } +} + +func TestThinkingTransportLeavesOtherPaths(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + client := newThinkingHTTPClient() + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + srv.URL+"/v1/models", nil) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() +} diff --git a/internal/controller/ai_vision.go b/internal/controller/ai_vision.go new file mode 100644 index 000000000..39f87b2f0 --- /dev/null +++ b/internal/controller/ai_vision.go @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package controller + +import ( + "encoding/base64" + "fmt" + "strings" + + "github.com/sashabaranov/go-openai" +) + +const ( + maxImagesPerMessage = 4 + maxImageBytes = 4 << 20 // decoded size per image: 4MB +) + +// allowedImagePrefixes accepts base64 data URLs of web-safe raster formats and +// plain HTTPS image URLs. +var allowedImagePrefixes = []string{ + "data:image/png;base64,", + "data:image/jpeg;base64,", + "data:image/webp;base64,", + "https://", +} + +// ValidateAndPrepareImages validates attachments and wraps them together with +// the text into a single MultiContent message. The returned message must be +// used as-is (Content stays empty so it never conflicts with MultiContent). +func ValidateAndPrepareImages(text string, images []string) (*openai.ChatCompletionMessage, error) { + if len(images) == 0 { + return &openai.ChatCompletionMessage{ + Role: openai.ChatMessageRoleUser, + Content: text, + }, nil + } + if len(images) > maxImagesPerMessage { + return nil, fmt.Errorf("at most %d images per message", maxImagesPerMessage) + } + for _, img := range images { + okPrefix := false + for _, p := range allowedImagePrefixes { + if strings.HasPrefix(img, p) { + okPrefix = true + break + } + } + if !okPrefix { + return nil, fmt.Errorf("unsupported image source (use PNG/JPEG/WebP base64 or HTTPS URL)") + } + if strings.HasPrefix(img, "data:image/") { + idx := strings.Index(img, "base64,") + raw, err := base64.StdEncoding.DecodeString(img[idx+len("base64,"):]) + if err != nil { + return nil, fmt.Errorf("invalid image data") + } + if len(raw) > maxImageBytes { + return nil, fmt.Errorf("image too large (max 4MB)") + } + } + } + + msg := &openai.ChatCompletionMessage{ + Role: openai.ChatMessageRoleUser, + MultiContent: make([]openai.ChatMessagePart, 0, len(images)+1), + } + msg.MultiContent = append(msg.MultiContent, openai.ChatMessagePart{ + Type: openai.ChatMessagePartTypeText, + Text: text, + }) + for _, img := range images { + msg.MultiContent = append(msg.MultiContent, openai.ChatMessagePart{ + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{URL: img}, + }) + } + return msg, nil +} diff --git a/internal/controller/ai_vision_test.go b/internal/controller/ai_vision_test.go new file mode 100644 index 000000000..03e13b8c5 --- /dev/null +++ b/internal/controller/ai_vision_test.go @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package controller + +import ( + "bytes" + "encoding/base64" + "strings" + "testing" + + "github.com/sashabaranov/go-openai" +) + +// minimalPNGBytes builds a tiny blob that starts with the canonical PNG file +// signature at runtime. The validation only inspects the declared MIME type, +// base64 integrity and size, so a fully decodable image is not required; we +// construct it on the fly instead of embedding a base64 blob in the source. +func minimalPNGBytes() []byte { + sig := []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A} // \x89PNG\r\n\x1a\n + return append(sig, bytes.Repeat([]byte{0x00}, 40)...) +} + +func dataURL() string { + return "data:image/png;base64," + base64.StdEncoding.EncodeToString(minimalPNGBytes()) +} + +func TestValidateAndPrepareImagesOK(t *testing.T) { + msg, err := ValidateAndPrepareImages("看这张图", []string{dataURL()}) + if err != nil { + t.Fatal(err) + } + if len(msg.MultiContent) != 2 { + t.Fatalf("expect 2 parts got %d", len(msg.MultiContent)) + } + if msg.MultiContent[0].Type != openai.ChatMessagePartTypeText || + msg.MultiContent[0].Text != "看这张图" { + t.Fatalf("bad text part: %+v", msg.MultiContent[0]) + } + if msg.MultiContent[1].ImageURL.URL != dataURL() { + t.Fatalf("bad image url part") + } +} + +func TestValidateAndPrepareImagesNoImages(t *testing.T) { + msg, err := ValidateAndPrepareImages("纯文本", nil) + if err != nil { + t.Fatal(err) + } + if msg.Content != "纯文本" || msg.MultiContent != nil { + t.Fatalf("expect plain content message, got %+v", msg) + } +} + +func TestValidateAndPrepareImagesLimits(t *testing.T) { + five := []string{dataURL(), dataURL(), dataURL(), dataURL(), dataURL()} + if _, err := ValidateAndPrepareImages("hi", five); err == nil || + !strings.Contains(err.Error(), "at most") { + t.Fatalf("expect limit error, got %v", err) + } + if _, err := ValidateAndPrepareImages("", []string{"http://a.com/x.png"}); err == nil { + t.Fatal("expect reject non-https URL") + } + if _, err := ValidateAndPrepareImages("", []string{"data:image/gif;base64," + base64.StdEncoding.EncodeToString(minimalPNGBytes())}); err == nil { + t.Fatal("expect reject unsupported mime") + } + junk := base64.StdEncoding.EncodeToString(make([]byte, 5<<20)) + if _, err := ValidateAndPrepareImages("", []string{"data:image/png;base64," + junk}); err == nil || + !strings.Contains(err.Error(), "too large") { + t.Fatalf("expect size error, got %v", err) + } + if _, err := ValidateAndPrepareImages("", []string{"data:text/html;base64," + junk}); err == nil { + t.Fatal("expect reject non-image mime") + } +} diff --git a/internal/controller/siteinfo_controller.go b/internal/controller/siteinfo_controller.go index a5dde0234..4e5f7f691 100644 --- a/internal/controller/siteinfo_controller.go +++ b/internal/controller/siteinfo_controller.go @@ -112,6 +112,9 @@ func (sc *SiteInfoController) GetSiteInfo(ctx *gin.Context) { } if aiConf, err := sc.siteInfoService.GetSiteAI(ctx); err == nil { resp.AIEnabled = aiConf.Enabled + if p := aiConf.GetProvider(); p != nil { + resp.AIVisionEnabled = p.VisionEnabled + } } if mcpConf, err := sc.siteInfoService.GetSiteMCP(ctx); err == nil { diff --git a/internal/migrations/init_data.go b/internal/migrations/init_data.go index 5af41bbfc..3da1fd7b2 100644 --- a/internal/migrations/init_data.go +++ b/internal/migrations/init_data.go @@ -353,7 +353,7 @@ var ( {ID: 128, Key: "rank.answer.undeleted", Value: `-1`}, {ID: 129, Key: "rank.question.undeleted", Value: `-1`}, {ID: 130, Key: "rank.tag.undeleted", Value: `-1`}, - {ID: 131, Key: "ai_config.provider", Value: `[{"default_api_host":"https://api.openai.com","display_name":"OpenAI","name":"openai"},{"default_api_host":"https://generativelanguage.googleapis.com","display_name":"Gemini","name":"gemini"},{"default_api_host":"https://api.anthropic.com","display_name":"Anthropic","name":"anthropic"}]`}, + {ID: 131, Key: "ai_config.provider", Value: `[{"default_api_host":"https://api.openai.com","display_name":"OpenAI","name":"openai"},{"default_api_host":"https://generativelanguage.googleapis.com/v1beta/openai","display_name":"Gemini","name":"gemini"}]`}, } defaultBadgeGroupTable = []*entity.BadgeGroup{ diff --git a/internal/migrations/v31.go b/internal/migrations/v31.go index 428c5828a..5d893a3b1 100644 --- a/internal/migrations/v31.go +++ b/internal/migrations/v31.go @@ -61,7 +61,7 @@ func addAPIKey(ctx context.Context, x *xorm.Engine) error { } defaultConfigTable := []*entity.Config{ - {ID: 131, Key: "ai_config.provider", Value: `[{"default_api_host":"https://api.openai.com","display_name":"OpenAI","name":"openai"},{"default_api_host":"https://generativelanguage.googleapis.com","display_name":"Gemini","name":"gemini"},{"default_api_host":"https://api.anthropic.com","display_name":"Anthropic","name":"anthropic"}]`}, + {ID: 131, Key: "ai_config.provider", Value: `[{"default_api_host":"https://api.openai.com","display_name":"OpenAI","name":"openai"},{"default_api_host":"https://generativelanguage.googleapis.com/v1beta/openai","display_name":"Gemini","name":"gemini"}]`}, } for _, c := range defaultConfigTable { exist, err := x.Context(ctx).Get(&entity.Config{Key: c.Key}) diff --git a/internal/schema/ai_config_schema.go b/internal/schema/ai_config_schema.go index 6ac686343..f7a51cd0d 100644 --- a/internal/schema/ai_config_schema.go +++ b/internal/schema/ai_config_schema.go @@ -19,6 +19,23 @@ package schema +import "strings" + +// NormalizeAPIHost normalizes an OpenAI-compatible API base host: +// trims whitespace/slashes and appends "/v1" unless already versioned, +// so that hosts with or without the "/v1" suffix resolve identically. +func NormalizeAPIHost(host string) string { + h := strings.TrimSpace(host) + h = strings.TrimRight(h, "/") + if h == "" { + return "" + } + if strings.HasSuffix(h, "/v1") || strings.Contains(h, "/v1beta") { + return h + } + return h + "/v1" +} + // GetAIProviderResp get AI providers response type GetAIProviderResp struct { Name string `json:"name"` diff --git a/internal/schema/api_host_test.go b/internal/schema/api_host_test.go new file mode 100644 index 000000000..f8ac37c6c --- /dev/null +++ b/internal/schema/api_host_test.go @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package schema + +import "testing" + +func TestNormalizeAPIHost(t *testing.T) { + cases := []struct{ in, want string }{ + {"https://token.sensenova.cn", "https://token.sensenova.cn/v1"}, + {"https://token.sensenova.cn/", "https://token.sensenova.cn/v1"}, + {"https://token.sensenova.cn///", "https://token.sensenova.cn/v1"}, + {"https://token.sensenova.cn/v1", "https://token.sensenova.cn/v1"}, + {" https://api.openai.com ", "https://api.openai.com/v1"}, + {"https://generativelanguage.googleapis.com/v1beta/openai", + "https://generativelanguage.googleapis.com/v1beta/openai"}, + {"", ""}, + {" ", ""}, + } + for _, c := range cases { + if got := NormalizeAPIHost(c.in); got != c.want { + t.Errorf("NormalizeAPIHost(%q) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/internal/schema/siteinfo_schema.go b/internal/schema/siteinfo_schema.go index 1d0b27ff6..94764d676 100644 --- a/internal/schema/siteinfo_schema.go +++ b/internal/schema/siteinfo_schema.go @@ -295,6 +295,12 @@ type SiteAIProvider struct { APIHost string `validate:"omitempty,lte=512" form:"api_host" json:"api_host"` APIKey string `validate:"omitempty,lte=256" form:"api_key" json:"api_key"` Model string `validate:"omitempty,lte=100" form:"model" json:"model"` + // ThinkingMode toggles deep-thinking for reasoning-capable models. + // Empty value or "off" keeps requests unchanged; "on" injects the + // OpenAI-compatible "enable_thinking" flag into chat completions. + ThinkingMode string `validate:"omitempty,oneof=on off" form:"thinking_mode" json:"thinking_mode"` + // VisionEnabled allows users to attach images to AI conversations. + VisionEnabled bool `validate:"omitempty" form:"vision_enabled" json:"vision_enabled"` } // SiteAIResp AI configuration response @@ -369,24 +375,25 @@ type SiteSeoResp SiteSeoReq // SiteInfoResp get site info response type SiteInfoResp struct { - General *SiteGeneralResp `json:"general"` - Interface *SiteInterfaceSettingsResp `json:"interface"` - UsersSettings *SiteUsersSettingsResp `json:"users_settings"` - Branding *SiteBrandingResp `json:"branding"` - Login *SiteLoginResp `json:"login"` - Theme *SiteThemeResp `json:"theme"` - CustomCssHtml *SiteCustomCssHTMLResp `json:"custom_css_html"` - SiteSeo *SiteSeoResp `json:"site_seo"` - SiteUsers *SiteUsersResp `json:"site_users"` - Advanced *SiteAdvancedResp `json:"site_advanced"` - Questions *SiteQuestionsResp `json:"site_questions"` - Tags *SiteTagsResp `json:"site_tags"` - Legal *SiteLegalSimpleResp `json:"site_legal"` - Security *SiteSecurityResp `json:"site_security"` - Version string `json:"version"` - Revision string `json:"revision"` - AIEnabled bool `json:"ai_enabled"` - MCPEnabled bool `json:"mcp_enabled"` + General *SiteGeneralResp `json:"general"` + Interface *SiteInterfaceSettingsResp `json:"interface"` + UsersSettings *SiteUsersSettingsResp `json:"users_settings"` + Branding *SiteBrandingResp `json:"branding"` + Login *SiteLoginResp `json:"login"` + Theme *SiteThemeResp `json:"theme"` + CustomCssHtml *SiteCustomCssHTMLResp `json:"custom_css_html"` + SiteSeo *SiteSeoResp `json:"site_seo"` + SiteUsers *SiteUsersResp `json:"site_users"` + Advanced *SiteAdvancedResp `json:"site_advanced"` + Questions *SiteQuestionsResp `json:"site_questions"` + Tags *SiteTagsResp `json:"site_tags"` + Legal *SiteLegalSimpleResp `json:"site_legal"` + Security *SiteSecurityResp `json:"site_security"` + Version string `json:"version"` + Revision string `json:"revision"` + AIEnabled bool `json:"ai_enabled"` + AIVisionEnabled bool `json:"ai_vision_enabled"` + MCPEnabled bool `json:"mcp_enabled"` } type TemplateSiteInfoResp struct { diff --git a/internal/service/siteinfo/siteinfo_service.go b/internal/service/siteinfo/siteinfo_service.go index 8b32b722e..2ec3c7506 100644 --- a/internal/service/siteinfo/siteinfo_service.go +++ b/internal/service/siteinfo/siteinfo_service.go @@ -737,14 +737,15 @@ func (s *SiteInfoService) GetAIModels(ctx context.Context, req *schema.GetAIMode r := resty.New() r.SetHeader("Authorization", fmt.Sprintf("Bearer %s", req.APIKey)) r.SetHeader("Content-Type", "application/json") - respBody, err := r.R().Get(req.APIHost + "/v1/models") + respBody, err := r.R().Get(schema.NormalizeAPIHost(req.APIHost) + "/models") if err != nil { log.Error(err) return resp, errors.BadRequest(fmt.Sprintf("failed to get AI models %s", err.Error())) } if !respBody.IsSuccess() { - log.Error(fmt.Sprintf("failed to get AI models, status code: %d, body: %s", respBody.StatusCode(), respBody.String())) - return resp, errors.BadRequest(fmt.Sprintf("failed to get AI models, response: %s", respBody.String())) + summary := summarizeUpstreamError(respBody.Body(), respBody.StatusCode()) + log.Error(fmt.Sprintf("failed to get AI models, status code: %d, body: %s", respBody.StatusCode(), summary)) + return resp, errors.BadRequest(fmt.Sprintf("failed to get AI models (%s)", summary)) } data := schema.GetAIModelsResp{} @@ -766,18 +767,43 @@ func (s *SiteInfoService) getStoredAIKey(ctx context.Context, apiHost string) (s if err != nil { return "", err } + // Prefer the key of the provider currently chosen in site settings; + // fall back to host matching for callers that bypass the chosen provider. + if current.ChosenProvider != "" { + for _, provider := range current.SiteAIProviders { + if provider.Provider == current.ChosenProvider && provider.APIKey != "" { + return provider.APIKey, nil + } + } + } apiHost = strings.TrimRight(apiHost, "/") for _, provider := range current.SiteAIProviders { if strings.TrimRight(provider.APIHost, "/") == apiHost && provider.APIKey != "" { return provider.APIKey, nil } } - if current.ChosenProvider != "" { - for _, provider := range current.SiteAIProviders { - if provider.Provider == current.ChosenProvider { - return provider.APIKey, nil + return "", nil +} + +// summarizeUpstreamError extracts a short message from an OpenAI-style error +// body instead of echoing the full upstream response back to the client. +func summarizeUpstreamError(body []byte, statusCode int) string { + var wrapper struct { + Error struct { + Code any `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(body, &wrapper); err == nil && wrapper.Error.Message != "" { + switch v := wrapper.Error.Code.(type) { + case float64: + return fmt.Sprintf("code %d: %s", int(v), wrapper.Error.Message) + case string: + if v != "" { + return fmt.Sprintf("code %s: %s", v, wrapper.Error.Message) } } + return wrapper.Error.Message } - return "", nil + return fmt.Sprintf("HTTP %d", statusCode) } diff --git a/ui/src/common/interface.ts b/ui/src/common/interface.ts index 8ab714230..373647d92 100644 --- a/ui/src/common/interface.ts +++ b/ui/src/common/interface.ts @@ -428,6 +428,7 @@ export interface SiteSettings { revision: string; site_security: AdminSettingsSecurity; ai_enabled: boolean; + ai_vision_enabled?: boolean; } export interface AdminSettingBranding { @@ -834,6 +835,8 @@ export interface AiConfig { api_host: string; api_key: string; model: string; + thinking_mode?: string; + vision_enabled?: boolean; }>; } diff --git a/ui/src/components/Sender/index.tsx b/ui/src/components/Sender/index.tsx index e79cf9b2e..6fa90f457 100644 --- a/ui/src/components/Sender/index.tsx +++ b/ui/src/components/Sender/index.tsx @@ -24,11 +24,13 @@ import { useTranslation } from 'react-i18next'; import classnames from 'classnames'; import { Icon } from '@/components'; +import aiControlStore from '@/stores/aiControl'; +import { useToast } from '@/hooks'; import './index.scss'; interface IProps { - onSubmit?: (value: string) => void; + onSubmit?: (value: string, images: string[]) => void; onCancel?: () => void; isGenerate: boolean; hasConversation: boolean; @@ -41,11 +43,47 @@ const Sender: FC = ({ hasConversation, }) => { const { t } = useTranslation('translation', { keyPrefix: 'ai_assistant' }); + const toast = useToast(); const containerRef = useRef(null); const textareaRef = useRef(null); + const fileInputRef = useRef(null); const [initialized, setInitialized] = useState(false); const [inputValue, setInputValue] = useState(''); + const [images, setImages] = useState([]); const [isFocus, setIsFocus] = useState(false); + const visionEnabled = aiControlStore((s) => s.ai_vision_enabled); + + const addImages = (files: FileList | null) => { + if (!files?.length) return; + const room = 4 - images.length; + if (room <= 0) { + toast.onShow({ + msg: t('image_max_count'), + variant: 'danger', + }); + return; + } + Array.from(files) + .slice(0, room) + .forEach((file) => { + if (!/^image\/(png|jpe?g|webp)$/.test(file.type)) return; + if (file.size > 4 * 1024 * 1024) { + toast.onShow({ + msg: t('image_too_large'), + variant: 'danger', + }); + return; + } + const reader = new FileReader(); + reader.onload = () => { + const dataUrl = String(reader.result); + setImages((prev) => + prev.length < 4 && !prev.includes(dataUrl) ? [...prev, dataUrl] : prev, + ); + }; + reader.readAsDataURL(file); + }); + }; const handleFocus = () => { setIsFocus(true); @@ -89,8 +127,9 @@ const Sender: FC = ({ if (isGenerate || !inputValue.trim()) { return; } - onSubmit?.(inputValue); + onSubmit?.(inputValue, images); setInputValue(''); + setImages([]); }; const handleKeyDown = (e: React.KeyboardEvent) => { @@ -145,7 +184,53 @@ const Sender: FC = ({ onChange={handleInputChange} onKeyDown={handleKeyDown} /> + {images.length > 0 && ( +
+ {images.map((img, i) => ( +
+ + +
+ ))} +
+ )}
+ {visionEnabled && !isGenerate && ( + <> + { + addImages(e.target.files); + e.target.value = ''; + }} + /> + + + )} {isGenerate ? (
+ + + handleValueChange({ + thinking_mode: { + value: e.target.checked ? 'on' : 'off', + errorMsg: '', + isInvalid: false, + }, + }) + } + /> + + {t('thinking_mode.tip')} + + + + + + handleValueChange({ + vision_enabled: { + value: e.target.checked, + errorMsg: '', + isInvalid: false, + }, + }) + } + /> + + {t('vision_enabled.tip')} + + + diff --git a/ui/src/pages/AiAssistant/index.tsx b/ui/src/pages/AiAssistant/index.tsx index e355e8041..ace75c21a 100644 --- a/ui/src/pages/AiAssistant/index.tsx +++ b/ui/src/pages/AiAssistant/index.tsx @@ -100,7 +100,7 @@ const Index = () => { }); }; - const handleSubmit = async (userMsg) => { + const handleSubmit = async (userMsg, images: string[] = []) => { setIsLoading(true); if (conversions?.records.length === 0) { setRecentNewItem({ @@ -147,6 +147,7 @@ const Index = () => { { role: 'user', content: userMsg, + ...(images.length ? { images } : {}), }, ], }; @@ -202,13 +203,16 @@ const Index = () => { }); }; - const handleSender = (userMsg) => { + const handleSender = (userMsg, images: string[] = []) => { if (conversions?.records.length <= 0) { const newConversationId = uuidv4(); navigate(`/ai-assistant/${newConversationId}`); - Storage.set('_a_once_msg', userMsg); + Storage.set( + '_a_once_msg', + JSON.stringify({ text: userMsg, images }), + ); } else { - handleSubmit(userMsg); + handleSubmit(userMsg, images); } }; @@ -227,9 +231,17 @@ const Index = () => { const msg = Storage.get('_a_once_msg'); Storage.remove('_a_once_msg'); if (msg) { - if (msg) { - handleSubmit(msg); + // Accepts both legacy plain text and { text, images } payloads. + try { + const parsed = JSON.parse(msg); + if (parsed && typeof parsed === 'object' && parsed.text) { + handleSubmit(parsed.text, parsed.images || []); + return; + } + } catch { + // legacy plain-text message } + handleSubmit(msg); return; } fetchDetail(); diff --git a/ui/src/stores/aiControl.ts b/ui/src/stores/aiControl.ts index c9f0afbc7..de642debc 100644 --- a/ui/src/stores/aiControl.ts +++ b/ui/src/stores/aiControl.ts @@ -21,20 +21,29 @@ import { create } from 'zustand'; interface AiControlStore { ai_enabled: boolean; - update: (params: { ai_enabled: boolean }) => void; + ai_vision_enabled: boolean; + update: (params: { + ai_enabled: boolean; + ai_vision_enabled?: boolean; + }) => void; reset: () => void; } const aiControlStore = create((set) => ({ ai_enabled: false, - update: (params: { ai_enabled: boolean }) => + ai_vision_enabled: false, + update: (params: { + ai_enabled: boolean; + ai_vision_enabled?: boolean; + }) => set((state) => { return { ...state, ...params, }; }), - reset: () => set({ ai_enabled: false }), + reset: () => + set({ ai_enabled: false, ai_vision_enabled: false }), })); export default aiControlStore; diff --git a/ui/src/utils/guard.ts b/ui/src/utils/guard.ts index fc78fa122..0735aa472 100644 --- a/ui/src/utils/guard.ts +++ b/ui/src/utils/guard.ts @@ -389,6 +389,7 @@ export const initAppSettingsStore = async () => { }); aiControlStore.getState().update({ ai_enabled: appSettings.ai_enabled, + ai_vision_enabled: appSettings.ai_vision_enabled ?? false, }); siteSecurityStore.getState().update(appSettings.site_security); }