From 50b9bffc87069c205f618182ab5fbe91cdc819b6 Mon Sep 17 00:00:00 2001 From: Delije4 <191491735+Delije4@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:11:53 +0300 Subject: [PATCH 1/2] feat: add native video input processing controls --- README.md | 1 + README_en.md | 1 + docs/protocol.md | 26 ++- internal/aistudio/build.go | 80 ++++++++ internal/aistudio/event.go | 25 +++ internal/aistudio/generate.go | 6 + internal/aistudio/interaction.go | 3 + internal/aistudio/media_processing.go | 57 ++++++ internal/aistudio/media_processing_test.go | 125 ++++++++++++ internal/aistudio/models.go | 1 + internal/aistudio/request.go | 34 +++- internal/aistudio/types.go | 23 +++ internal/aistudio/video_input.go | 143 ++++++++++++++ internal/aistudio/video_input_test.go | 183 ++++++++++++++++++ internal/api/gemini.go | 104 +++++++++- internal/api/interaction_input.go | 30 ++- internal/api/interaction_output.go | 26 ++- internal/api/responses.go | 12 ++ internal/api/video_input.go | 125 ++++++++++++ internal/api/video_input_test.go | 175 +++++++++++++++++ internal/api/video_processing_history_test.go | 147 ++++++++++++++ internal/api/video_processing_output_test.go | 117 +++++++++++ 22 files changed, 1432 insertions(+), 12 deletions(-) create mode 100644 internal/aistudio/media_processing.go create mode 100644 internal/aistudio/media_processing_test.go create mode 100644 internal/aistudio/video_input.go create mode 100644 internal/aistudio/video_input_test.go create mode 100644 internal/api/video_input.go create mode 100644 internal/api/video_input_test.go create mode 100644 internal/api/video_processing_history_test.go create mode 100644 internal/api/video_processing_output_test.go diff --git a/README.md b/README.md index 2405f8b9..d08d54ba 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ - **视频生成**: 支持 Veo 视频生成和图片转视频，目录中没有 Veo 时视频长任务由 Gemini Omni 生成；Gemini Omni 通过四套生成接口接收文本、图片与视频输入，输出文本与 MP4 视频 - **Deep Research**: Pro 与 Ultra 账户可通过四套生成接口与 Interactions 的 `agent` 字段调用 Deep Research，输出研究过程、带引用的报告与图表 - **YouTube 输入**: YouTube 视频 URL 作为外部视频附件读取；开启 `YOUTUBE_LINKS` 后，消息文本中的 YouTube 链接同样作为视频读取 +- **视频理解设置**: Gemini 与 Interactions 支持逐视频的起止时间、0–24 FPS 范围内的正数采样率和静态/Agentic 处理；Agentic 返回服务端处理步骤并支持历史续接 - **智能模型切换**: 从 AI Studio 实时发现模型并按 `model` 字段路由 - **Google 工具**: 支持 Search、Image Search、URL Context、Code Execution 和 Maps - **Files 与 Transcribe**: 支持文件上传、查询、内容读取、删除和音频转录 diff --git a/README_en.md b/README_en.md index 88a519d3..2206ed96 100644 --- a/README_en.md +++ b/README_en.md @@ -62,6 +62,7 @@ - **Video Generation**: Veo video generation and image-to-video, with video tasks generated by Gemini Omni when the catalog has no Veo model; Gemini Omni accepts text, image, and video input and returns text and MP4 video through the four generation APIs - **Deep Research**: Pro and Ultra accounts can run Deep Research through the four generation APIs and the Interactions `agent` field, returning the research process, a cited report, and charts - **YouTube Input**: YouTube video URLs are read as external video attachments; with `YOUTUBE_LINKS` enabled, YouTube links in message text are read as video too +- **Video Understanding Controls**: Gemini and Interactions support per-video start/end offsets, sampling rates greater than 0 and up to 24 FPS, and static/agentic processing; agentic results preserve server-side processing steps and conversation history - **Smart Model Switching**: Discover models from AI Studio and route through the `model` field - **Google Tools**: Search, Image Search, URL Context, Code Execution, and Maps - **Files and Transcribe**: File upload, metadata, content, deletion, and audio transcription diff --git a/docs/protocol.md b/docs/protocol.md index dd4f3b8d..9f4a778e 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -308,6 +308,7 @@ field 1 是 `request_key`，field 2 是 `botguard_response`。正确 proof、损 | 52 | thinking level | 53 | music route | | 54 | image search | 58 | Google Maps | | 59 | private Interaction route | 85 | TTS 分段说话人 | +| 69 | Agentic 视频理解 | | | | 46 | Live 实时翻译 | 48 | Build 写代码助手可选 | 未知能力码按原值保留为 `capability_code_` 或 `secondary_capability_code_`。 @@ -326,6 +327,7 @@ field 1 是 `request_key`，field 2 是 `botguard_response`。正确 proof、损 | 52 | `thinking_level` | 53 | `music_route` | | 54 | `image_search` | 58 | `google_maps` | | 59 | `interaction_route` | 74 | `transcription_word_timestamps` | +| 69 | `agentic_video` | | | | 76 | `transcription_language_codes` | 77 | `transcription_output` | | 80 | `transcription_speaker_labels` | 81 | `transcription_custom_vocabulary` | | 84 | `transcription_smart` | 85 | `speech_metadata` | @@ -429,8 +431,12 @@ Content 形状： | 11 | 12 | function result `[name, Struct, callId?]` | | 12 | 13 | thought boolean | | 14 | 15 | thought signature | +| 15 | 16 | 视频静态处理元数据 `[startOffset?, endOffset?, fps?]`；offset 为 `[seconds,nanos?]` | | 17 | 18 | 上传附件 `[mime, 上传路径, 文件名, 字节数]`，字节数为十进制字符串；用于 Build 写代码助手，见 [Build 通道](build.md) 第 8 节 | | 22 | 23 | transcription metadata `[text, speaker?, timestamp spans?]` | +| 25 | 26 | 服务端媒体处理调用 `[13, name?, Struct?, callId?]` | +| 26 | 27 | 服务端媒体处理结果 `[13, name?, Struct?, callId?]` | +| 39 | 40 | 视频处理模式：`1` STATIC、`2` AGENTIC | system instruction： @@ -1969,6 +1975,7 @@ delta 联合类型为 `text_delta{text}`、`thinking_delta{thinking}`、`signatu | --- | --- | | `input` | 字符串、单个内容块、内容块数组或步骤数组；内容类型为 `text`、`image`、`audio`、`video`、`document` | | 媒体内容 | `data`（Base64）或 `uri`，可选 `mime_type` | +| 视频处理 | `processing:"static"`、`"agentic"` 或 `{type:"static",start_offset?,end_offset?,fps?}`；offset 接受秒数或带 `s` 的 Duration 字符串 | | 输入步骤 | `user_input`、`model_output`、`thought`、`function_call`、`function_result`、`code_execution_call`、`code_execution_result` 与工具 call/result 步骤 | | `system_instruction` | 当前请求的系统指令 | | `generation_config` | `temperature`、`top_p`、`top_k`、`max_output_tokens`、`seed`、`stop_sequences`、`thinking_level`、`thinking_summaries`、`speech_config`、`tool_choice`、`image_config:{aspect_ratio?,image_size?}` | @@ -1983,7 +1990,8 @@ delta 联合类型为 `text_delta{text}`、`thinking_delta{thinking}`、`signatu | 续接 | `previous_interaction_id`、`store` | - 媒体内容的 `data` 与 `uri` 二选一，复用 Gemini 文件与内联媒体解析：省略 `mime_type` 时内联数据按内容识别类型，无法识别时返回 `400 INVALID_ARGUMENT`；`uri` 为外部 http(s) URL 时由服务下载为内联媒体，规则与 GenerateContent 的 `fileData` 相同；本服务 Gemini 文件接口返回的 `uri` 作为文件引用 -- 函数结果通过 `call_id` 匹配历史调用，没有对应调用时使用自带的 `name`；`is_error:true` 的结果以 `{"error":}` 发送上游；`code_execution_call` 与 `code_execution_result` 以代码块文本并入模型轮次；`google_search`、`url_context`、`google_maps`、`file_search`、`mcp_server_tool`、`retrieval`、`processing` 的 call 与 result 步骤不发送上游 +- 函数结果通过 `call_id` 匹配历史调用，没有对应调用时使用自带的 `name`；`is_error:true` 的结果以 `{"error":}` 发送上游；`code_execution_call` 与 `code_execution_result` 以代码块文本并入模型轮次；`google_search`、`url_context`、`google_maps`、`file_search`、`mcp_server_tool`、`retrieval` 的 call 与 result 步骤不发送上游 +- `processing_call` 和 `processing_result` 保留服务端媒体处理调用的 `id`、对应的 `call_id` 与各自的 `signature`，作为模型历史续接；流式响应按 `step.start`、`step.delta`、`step.stop` 发布，签名位于 delta - 文本 `response_format` 带 `schema` 时请求结构化输出，`mime_type` 默认 `application/json`；不带 `type` 或 `type` 为 JSON Schema 类型的对象直接作为结构化输出 Schema；图片指定 `mime_type:"image/jpeg"` 时其他格式的图片输出转为 JPEG - `response_format` 中的图片比例、尺寸与文本 `mime_type` 优先于 `generation_config.image_config` 与 `response_mime_type`；`response_modalities` 中的其他取值返回 `400 INVALID_ARGUMENT` - `speech_config` 的 `language` 不发送上游，多说话人 `mode` 为 `verbatim` 或 `conversational`；`speech_metadata` 保留说话人与风格 @@ -2032,11 +2040,27 @@ Content 字段为 `role` 与 `parts`。Part oneof： | file data | `fileData:{mimeType?,fileUri,displayName}` | | function call | `functionCall:{id,name,args}` | | function response | `functionResponse:{id,name,response}` | +| server media processing call | `toolCall:{id?,toolType:"MEDIA_PROCESSING",toolName?,args?}` | +| server media processing response | `toolResponse:{id?,toolType:"MEDIA_PROCESSING",response?}` | | executable code | `executableCode:{language,code}` | | code result | `codeExecutionResult:{outcome,output,error}` | 不含数据字段且没有 `thoughtSignature` 的 Part 不发送；同一 Part 含多个数据字段时返回 `400 INVALID_ARGUMENT`。`fileData.fileUri` 为 YouTube 链接时作为外部视频，为本服务 `/upload/v1beta/files` 返回的文件 URI 或 `/v1/files` 文件 ID 时作为文件引用，为其他 http(s) URL 时由服务下载为内联媒体；其他 Gemini Files API URI 不下载。外部 URL 在同一请求内只下载一次，上限 512 MiB，MIME 依次取响应 `Content-Type`、`mimeType` 与内容识别；主机或任一重定向目标解析到本机、私网、链路本地、组播或未指定地址时不连接，按下载失败处理；下载失败返回带 URL 与原因的 `400 INVALID_ARGUMENT`。`:countTokens` 同样先下载外部 URL 再计数。 +视频 Part 可设置 `mediaProcessing` 为 `STATIC`、`AGENTIC` 或 `MEDIA_PROCESSING_UNSPECIFIED`。省略时沿用模型默认处理方式。`AGENTIC` 使用模型目录的 `agentic_video` 能力，并返回 `toolType:"MEDIA_PROCESSING"` 的服务端 `toolCall` 与 `toolResponse`；调用标识与 `thoughtSignature` 随完整模型 Part 一起用于下一轮历史。 + +静态视频的 `videoMetadata` 可设置 `startOffset`、`endOffset` 与 `fps`，与该 Part 的视频附件一起参与生成及 token 计数。offset 为非负秒数 Duration 字符串，支持最多九位小数，例如 `"2.5s"`；显式 `"0s"` 保留起点，终点须晚于起点。FPS 范围为 `(0,24]`，静态参数与 `AGENTIC` 互斥。例如： + +```json +{ + "fileData": {"mimeType":"video/*","fileUri":"https://www.youtube.com/watch?v="}, + "mediaProcessing": "STATIC", + "videoMetadata": {"startOffset":"2s","endOffset":"4s","fps":0.5} +} +``` + +这些参数与媒体处理历史适用于上游使用 GenerateContent 的模型；使用原生 CreateInteractionStream 或 CreateInteraction 的模型收到这些参数时返回参数错误。 + `generationConfig` 全字段： | 类别 | 字段 | diff --git a/internal/aistudio/build.go b/internal/aistudio/build.go index 9d0bb7d7..cdc485a1 100644 --- a/internal/aistudio/build.go +++ b/internal/aistudio/build.go @@ -166,6 +166,10 @@ func encodeBuildContents(contents []Content) ([]any, error) { } func encodeBuildPart(part Part) (map[string]any, error) { + metadata, processing, err := videoPartOptions(part) + if err != nil { + return nil, err + } wire := map[string]any{} signature := part.ThoughtSignature switch { @@ -240,6 +244,29 @@ func encodeBuildPart(part Part) (map[string]any, error) { result["output"] = output } wire["codeExecutionResult"] = result + case part.ProcessingCall != nil: + step := part.ProcessingCall + if _, err := encodeMediaProcessingStep(step); err != nil { + return nil, err + } + value := map[string]any{"toolType": "MEDIA_PROCESSING", "id": step.ID} + if step.Name != "" { + value["toolName"] = step.Name + } + if len(step.Data) != 0 { + value["args"] = step.Data + } + wire["toolCall"] = value + case part.ProcessingResult != nil: + step := part.ProcessingResult + if _, err := encodeMediaProcessingStep(step); err != nil { + return nil, err + } + value := map[string]any{"toolType": "MEDIA_PROCESSING", "id": step.ID} + if len(step.Data) != 0 { + value["response"] = step.Data + } + wire["toolResponse"] = value case signature != "": wire["text"] = "" default: @@ -248,6 +275,23 @@ func encodeBuildPart(part Part) (map[string]any, error) { if signature != "" { wire["thoughtSignature"] = signature } + if len(metadata) > 0 { + video := part.VideoMetadata + value := map[string]any{} + if video.StartOffset != "" { + value["startOffset"] = video.StartOffset + } + if video.EndOffset != "" { + value["endOffset"] = video.EndOffset + } + if video.FPS != nil { + value["fps"] = *video.FPS + } + wire["videoMetadata"] = value + } + if processing != 0 { + wire["mediaProcessing"] = part.MediaProcessing + } return wire, nil } @@ -491,6 +535,16 @@ type buildResponsePart struct { Name string `json:"name"` Args json.RawMessage `json:"args"` } `json:"functionCall"` + ToolCall *buildProcessingStep `json:"toolCall"` + ToolResponse *buildProcessingStep `json:"toolResponse"` +} + +type buildProcessingStep struct { + ID string `json:"id"` + ToolType string `json:"toolType"` + ToolName string `json:"toolName"` + Args json.RawMessage `json:"args"` + Response json.RawMessage `json:"response"` } type buildGrounding struct { @@ -664,6 +718,32 @@ func decodeBuildPart(part buildResponsePart, index int, raw []byte) ([]Event, er call := FunctionCall{ID: part.FunctionCall.ID, Name: part.FunctionCall.Name, Arguments: arguments, ThoughtSignature: signature} events = append(events, Event{Kind: EventToolCall, ToolCall: &call, ThoughtSignature: signature}) } + for _, field := range []struct { + step *buildProcessingStep + kind EventKind + name string + }{{part.ToolCall, EventMediaProcessingCall, "toolCall"}, {part.ToolResponse, EventMediaProcessingResult, "toolResponse"}} { + if field.step == nil || field.step.ToolType != "MEDIA_PROCESSING" { + continue + } + if len(events) != 0 { + return nil, &ProtocolEvidenceError{Method: buildProxyStreamedMethod, Path: fmt.Sprintf("$chunk.parts[%d]", index), Detail: "media processing part contains multiple data variants", Raw: cloneRaw(raw)} + } + step := field.step + data := step.Args + if field.kind == EventMediaProcessingResult { + data = step.Response + } + if isJSONNull(data) { + data = nil + } else { + var object map[string]json.RawMessage + if err := json.Unmarshal(data, &object); err != nil { + return nil, &ProtocolEvidenceError{Method: buildProxyStreamedMethod, Path: fmt.Sprintf("$chunk.parts[%d].%s", index, field.name), Detail: "media processing payload must be an object", Raw: cloneRaw(raw)} + } + } + events = append(events, Event{Kind: field.kind, MediaProcessing: &MediaProcessingStep{ID: step.ID, Name: step.ToolName, Data: data}, ThoughtSignature: signature}) + } if len(events) == 0 && signature != "" { events = append(events, Event{Kind: EventThoughtSignature, ThoughtSignature: signature}) } diff --git a/internal/aistudio/event.go b/internal/aistudio/event.go index ec4406e5..37af89dd 100644 --- a/internal/aistudio/event.go +++ b/internal/aistudio/event.go @@ -252,6 +252,17 @@ func (d *FrameDecoder) decodePart(raw json.RawMessage, path string, evidence jso if err != nil { return nil, withMethod(err, "GenerateContent") } + if !isJSONNull(rawAt(part, 25)) || !isJSONNull(rawAt(part, 26)) { + variants := 0 + for _, index := range []int{1, 2, 5, 6, 7, 8, 10, 11, 17, 25, 26} { + if !isJSONNull(rawAt(part, index)) { + variants++ + } + } + if variants != 1 { + return nil, d.protocolError(path, "media processing part contains multiple data variants", raw) + } + } thought := false if thoughtRaw := rawAt(part, 12); !isJSONNull(thoughtRaw) { thought, err = rawBool(thoughtRaw, path+"[12]", raw) @@ -329,6 +340,20 @@ func (d *FrameDecoder) decodePart(raw json.RawMessage, path string, evidence jso }) } } + for _, field := range []struct { + index int + kind EventKind + }{{25, EventMediaProcessingCall}, {26, EventMediaProcessingResult}} { + if value := rawAt(part, field.index); !isJSONNull(value) { + step, err := decodeMediaProcessingStep(value, fmt.Sprintf("%s[%d]", path, field.index), raw) + if err != nil { + return nil, err + } + if step != nil { + events = append(events, Event{Kind: field.kind, MediaProcessing: step, ThoughtSignature: signature}) + } + } + } if len(events) == 0 { if signature != "" { return []Event{{Kind: EventThoughtSignature, ThoughtSignature: signature}}, nil diff --git a/internal/aistudio/generate.go b/internal/aistudio/generate.go index 1157ee24..4d952cb6 100644 --- a/internal/aistudio/generate.go +++ b/internal/aistudio/generate.go @@ -645,6 +645,9 @@ func (c *Client) validateGenerateRequest(request GenerateRequest) error { // validateGenerateEntry 按 Generate 的顺序校验工具、转写、生成参数与安全设置 func validateGenerateEntry(request GenerateRequest, entry modelEntry) error { + if err := validateVideoInputs(request.Contents, entry.model); err != nil { + return fmt.Errorf("%w: %v", ErrInvalidArgument, err) + } if _, err := supportedTools(request.Tools, entry.model); err != nil { return fmt.Errorf("%w: %v", ErrInvalidArgument, err) } @@ -676,6 +679,9 @@ func (c *Client) Generate(ctx context.Context, request GenerateRequest) (<-chan if err != nil { return nil, err } + if err := validateVideoInputs(request.Contents, entry.model); err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidArgument, err) + } request.Tools, err = supportedTools(request.Tools, entry.model) if err != nil { return nil, fmt.Errorf("%w: %v", ErrInvalidArgument, err) diff --git a/internal/aistudio/interaction.go b/internal/aistudio/interaction.go index 68dacc09..5a8ac794 100644 --- a/internal/aistudio/interaction.go +++ b/internal/aistudio/interaction.go @@ -158,6 +158,9 @@ func encodeInteractionSteps(contents []Content) ([]any, []Content, bool, error) parts := make([]any, 0, len(content.Parts)) texts := make([]Part, 0, len(content.Parts)) for _, part := range content.Parts { + if part.VideoMetadata != nil || part.MediaProcessing != "" || part.ProcessingCall != nil || part.ProcessingResult != nil { + return nil, nil, false, fmt.Errorf("Interaction-routed models do not support GenerateContent video processing options or navigation history") + } switch { case part.Thought: continue diff --git a/internal/aistudio/media_processing.go b/internal/aistudio/media_processing.go new file mode 100644 index 00000000..12e99544 --- /dev/null +++ b/internal/aistudio/media_processing.go @@ -0,0 +1,57 @@ +package aistudio + +import ( + "encoding/json" + "fmt" +) + +// encodeMediaProcessingStep encodes the observed MEDIA_PROCESSING tool message. +func encodeMediaProcessingStep(step *MediaProcessingStep) ([]any, error) { + wire := []any{int64(13), nil, nil, step.ID} + if step.Name != "" { + wire[1] = step.Name + } + if len(step.Data) != 0 { + value, err := encodeWireStructJSON(step.Data) + if err != nil { + return nil, fmt.Errorf("media processing payload: %w", err) + } + wire[2] = value + } + return wire, nil +} + +// decodeMediaProcessingStep decodes the server-owned tool type observed as 13. +func decodeMediaProcessingStep(raw json.RawMessage, path string, evidence json.RawMessage) (*MediaProcessingStep, error) { + fields, err := rawArray(raw, path, evidence) + if err != nil { + return nil, withMethod(err, "GenerateContent") + } + typeID, err := rawInt64(rawAt(fields, 0), path+"[0]", evidence) + if err != nil { + return nil, withMethod(err, "GenerateContent") + } + if typeID != 13 { + return nil, nil + } + step := &MediaProcessingStep{} + if value := rawAt(fields, 1); !isJSONNull(value) { + step.Name, err = rawString(value, path+"[1]", evidence) + if err != nil { + return nil, withMethod(err, "GenerateContent") + } + } + if value := rawAt(fields, 2); !isJSONNull(value) { + step.Data, err = decodeWireStruct(value, path+"[2]", evidence) + if err != nil { + return nil, err + } + } + if value := rawAt(fields, 3); !isJSONNull(value) { + step.ID, err = rawString(value, path+"[3]", evidence) + if err != nil { + return nil, withMethod(err, "GenerateContent") + } + } + return step, nil +} diff --git a/internal/aistudio/media_processing_test.go b/internal/aistudio/media_processing_test.go new file mode 100644 index 00000000..a842dbd1 --- /dev/null +++ b/internal/aistudio/media_processing_test.go @@ -0,0 +1,125 @@ +package aistudio + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestMediaProcessingTraceRoundTrip(t *testing.T) { + for _, result := range []bool{false, true} { + part := Part{ThoughtSignature: "processing-signature"} + step := &MediaProcessingStep{ID: "media_call_1"} + index, kind := 25, EventMediaProcessingCall + if result { + part.ProcessingResult, index, kind = step, 26, EventMediaProcessingResult + } else { + part.ProcessingCall = step + } + wire, err := encodePart(part) + if err != nil { + t.Fatal(err) + } + if !jsonValueEqual(t, wire[index], []any{int64(13), nil, nil, "media_call_1"}) { + t.Fatalf("media processing wire = %#v", wire[index]) + } + raw, err := json.Marshal(wire) + if err != nil { + t.Fatal(err) + } + events, err := NewFrameDecoder().decodePart(raw, "$part", raw) + if err != nil { + t.Fatal(err) + } + if len(events) != 1 || events[0].Kind != kind || events[0].MediaProcessing == nil || events[0].MediaProcessing.ID != step.ID || events[0].ThoughtSignature != part.ThoughtSignature { + t.Fatalf("processing events = %#v", events) + } + if events[0].ToolCall != nil { + t.Fatal("server processing became a host function call") + } + build, err := encodeBuildPart(part) + if err != nil { + t.Fatal(err) + } + field := "toolCall" + if result { + field = "toolResponse" + } + value, ok := build[field].(map[string]any) + if !ok || value["toolType"] != "MEDIA_PROCESSING" || value["id"] != step.ID { + t.Fatalf("Build processing projection = %#v", build) + } + } +} + +func TestBuildMediaProcessingTrace(t *testing.T) { + const raw = `{"candidates":[{"content":{"role":"model","parts":[ + {"toolCall":{"toolType":"MEDIA_PROCESSING","id":"video_call","toolName":"inspect","args":{"start_offset":2}},"thoughtSignature":"call-signature"}, + {"toolResponse":{"toolType":"MEDIA_PROCESSING","id":"video_call","response":{"loaded":true}},"thoughtSignature":"result-signature"}, + {"text":"Video answer"} + ]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":2,"totalTokenCount":12}}` + decoder := &BuildStreamDecoder{} + events, err := decoder.Decode([]byte(raw)) + if err != nil { + t.Fatal(err) + } + if err := decoder.End(); err != nil { + t.Fatal(err) + } + if len(events) != 5 || events[0].Kind != EventMediaProcessingCall || events[1].Kind != EventMediaProcessingResult || events[2].Kind != EventText || events[3].Kind != EventUsage || events[4].Kind != EventFinish { + t.Fatalf("Build event order = %#v", events) + } + for index, signature := range []string{"call-signature", "result-signature"} { + if events[index].MediaProcessing == nil || events[index].MediaProcessing.ID != "video_call" || events[index].ThoughtSignature != signature || events[index].ToolCall != nil { + t.Fatalf("Build server processing identity/authority = %#v", events[index]) + } + } + if events[0].MediaProcessing.Name != "inspect" || !strings.Contains(string(events[0].MediaProcessing.Data), "start_offset") { + t.Fatalf("Build processing arguments = %#v", events[0].MediaProcessing) + } +} + +func TestBuildMediaProcessingMalformedPart(t *testing.T) { + for _, part := range []string{ + `{"toolCall":{"toolType":"MEDIA_PROCESSING","args":[]}}`, + `{"text":"text","toolCall":{"toolType":"MEDIA_PROCESSING"}}`, + `{"toolCall":{"toolType":"MEDIA_PROCESSING"},"toolResponse":{"toolType":"MEDIA_PROCESSING"}}`, + } { + var parsed buildResponsePart + if err := json.Unmarshal([]byte(part), &parsed); err != nil { + t.Fatal(err) + } + if _, err := decodeBuildPart(parsed, 0, []byte(part)); err == nil { + t.Fatalf("malformed Build processing part accepted: %s", part) + } + } +} + +func TestMediaProcessingPayloadAndMalformedWire(t *testing.T) { + step := &MediaProcessingStep{ID: "media_call_2", Name: "inspect", Data: json.RawMessage(`{"start":2,"end":4}`)} + call, err := encodePart(Part{ProcessingCall: step}) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(call) + events, err := NewFrameDecoder().decodePart(raw, "$part", raw) + if err != nil || len(events) != 1 { + t.Fatalf("decode = %#v, %v", events, err) + } + got := events[0].MediaProcessing + var values map[string]int + if err := json.Unmarshal(got.Data, &values); err != nil { + t.Fatal(err) + } + if got.Name != "inspect" || len(values) != 2 || values["start"] != 2 || values["end"] != 4 { + t.Fatalf("processing payload = %#v", got) + } + call[25] = []any{int64(13), nil, nil, true} + raw, _ = json.Marshal(call) + if _, err := NewFrameDecoder().decodePart(raw, "$part", raw); err == nil { + t.Fatal("malformed processing ID accepted") + } + if _, err := encodePart(Part{Text: "text", ProcessingCall: step}); err == nil { + t.Fatal("multiple part variants accepted") + } +} diff --git a/internal/aistudio/models.go b/internal/aistudio/models.go index e957d688..31b483e1 100644 --- a/internal/aistudio/models.go +++ b/internal/aistudio/models.go @@ -31,6 +31,7 @@ var capabilityNames = map[int64]string{ 54: "image_search", 58: "google_maps", 59: "interaction_route", + 69: "agentic_video", 74: "transcription_word_timestamps", 76: "transcription_language_codes", 77: "transcription_output", diff --git a/internal/aistudio/request.go b/internal/aistudio/request.go index e48d689d..bbb1a081 100644 --- a/internal/aistudio/request.go +++ b/internal/aistudio/request.go @@ -184,6 +184,11 @@ func encodeContent(content Content) ([]any, error) { } func encodePart(part Part) ([]any, error) { + if part.VideoMetadata != nil || part.MediaProcessing != "" { + if _, _, err := videoPartOptions(part); err != nil { + return nil, err + } + } variants := 0 if part.Text != "" { variants++ @@ -212,6 +217,12 @@ func encodePart(part Part) ([]any, error) { if part.CodeExecutionResult != nil { variants++ } + if part.ProcessingCall != nil { + variants++ + } + if part.ProcessingResult != nil { + variants++ + } if variants == 0 && part.ThoughtSignature != "" { return setPartThoughtSignature([]any{}, part.ThoughtSignature), nil } @@ -247,7 +258,7 @@ func encodePart(part Part) ([]any, error) { return nil, fmt.Errorf("inline data 缺少 MIME 或数据") } wire := []any{nil, nil, []any{part.InlineData.MIME, base64.StdEncoding.EncodeToString(part.InlineData.Data)}} - return setPartThoughtSignature(wire, part.ThoughtSignature), nil + return encodeVideoPart(wire, part) } if part.ExternalMedia != nil { if part.ExternalMedia.MIME == "" || part.ExternalMedia.URL == "" { @@ -255,20 +266,20 @@ func encodePart(part Part) ([]any, error) { } wire := make([]any, 7) wire[6] = []any{part.ExternalMedia.MIME, part.ExternalMedia.URL} - return setPartThoughtSignature(wire, part.ThoughtSignature), nil + return encodeVideoPart(wire, part) } if part.File != nil { wire, err := encodeFilePart(part.File) if err != nil { return nil, err } - return setPartThoughtSignature(wire, part.ThoughtSignature), nil + return encodeVideoPart(wire, part) } if upload := part.Upload; upload != nil { // Part field 18 上传附件：MIME、上传路径、文件名、字节数 wire := make([]any, 18) wire[17] = []any{upload.MIME, upload.Path, upload.Name, strconv.FormatInt(upload.Size, 10)} - return setPartThoughtSignature(wire, part.ThoughtSignature), nil + return encodeVideoPart(wire, part) } if part.FunctionCall != nil { if err := validateFunctionCall(part.FunctionCall); err != nil { @@ -306,6 +317,19 @@ func encodePart(part Part) ([]any, error) { wire[11] = result return setPartThoughtSignature(wire, part.ThoughtSignature), nil } + if part.ProcessingCall != nil || part.ProcessingResult != nil { + step, index := part.ProcessingCall, 25 + if step == nil { + step, index = part.ProcessingResult, 26 + } + value, err := encodeMediaProcessingStep(step) + if err != nil { + return nil, err + } + wire := make([]any, index+1) + wire[index] = value + return setPartThoughtSignature(wire, part.ThoughtSignature), nil + } if part.ExecutableCode != nil { language, ok := map[string]int64{"LANGUAGE_UNSPECIFIED": 0, "PYTHON": 1}[part.ExecutableCode.Language] if !ok { @@ -352,7 +376,7 @@ func countTokensNeedsGenerateRequest(contents []Content) bool { for _, content := range contents { for _, part := range content.Parts { if part.InlineData != nil || part.ExternalMedia != nil || part.File != nil || part.FunctionCall != nil || part.FunctionResult != nil || - part.ExecutableCode != nil || part.CodeExecutionResult != nil { + part.ExecutableCode != nil || part.CodeExecutionResult != nil || part.ProcessingCall != nil || part.ProcessingResult != nil { return true } } diff --git a/internal/aistudio/types.go b/internal/aistudio/types.go index 92e22e36..6634a8f0 100644 --- a/internal/aistudio/types.go +++ b/internal/aistudio/types.go @@ -45,6 +45,20 @@ type UploadRef struct { Size int64 `json:"size"` } +// VideoMetadata configures static input-video clipping and frame sampling. +type VideoMetadata struct { + StartOffset string `json:"start_offset,omitempty"` + EndOffset string `json:"end_offset,omitempty"` + FPS *float64 `json:"fps,omitempty"` +} + +// MediaProcessingStep represents a server-owned video navigation call or result. +type MediaProcessingStep struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Data json.RawMessage `json:"data,omitempty"` +} + // FunctionCall 表示模型发起的函数调用 type FunctionCall struct { ID string `json:"id"` @@ -136,6 +150,10 @@ type Part struct { Thought bool `json:"thought,omitempty"` ThoughtSignature string `json:"thought_signature,omitempty"` SpeechMetadata *SpeechMetadata `json:"speech_metadata,omitempty"` + VideoMetadata *VideoMetadata `json:"video_metadata,omitempty"` + MediaProcessing string `json:"media_processing,omitempty"` + ProcessingCall *MediaProcessingStep `json:"processing_call,omitempty"` + ProcessingResult *MediaProcessingStep `json:"processing_result,omitempty"` } // SpeechMetadata 表示语音文本分段的说话人与风格 @@ -449,6 +467,10 @@ const ( EventReasoning EventKind = "reasoning" // EventToolCall 表示函数调用 EventToolCall EventKind = "tool_call" + // EventMediaProcessingCall identifies server-owned video navigation. + EventMediaProcessingCall EventKind = "media_processing_call" + // EventMediaProcessingResult identifies the server's navigation result. + EventMediaProcessingResult EventKind = "media_processing_result" // EventExecutableCode 表示内置代码执行器生成代码 EventExecutableCode EventKind = "executable_code" // EventCodeExecutionResult 表示内置代码执行结果 @@ -474,6 +496,7 @@ type Event struct { Kind EventKind `json:"kind"` Text string `json:"text,omitempty"` ToolCall *FunctionCall `json:"tool_call,omitempty"` + MediaProcessing *MediaProcessingStep `json:"media_processing,omitempty"` ExecutableCode *ExecutableCode `json:"executable_code,omitempty"` CodeExecutionResult *CodeExecutionResult `json:"code_execution_result,omitempty"` Grounding *GroundingMetadata `json:"grounding,omitempty"` diff --git a/internal/aistudio/video_input.go b/internal/aistudio/video_input.go new file mode 100644 index 00000000..e77a52ef --- /dev/null +++ b/internal/aistudio/video_input.go @@ -0,0 +1,143 @@ +package aistudio + +import ( + "fmt" + "math" + "regexp" + "strconv" + "strings" +) + +var videoOffsetPattern = regexp.MustCompile(`^([0-9]+)(?:\.([0-9]{1,9}))?s$`) + +// videoOffset parses the protobuf Duration representation used for video offsets. +func videoOffset(value string) (int64, int64, error) { + match := videoOffsetPattern.FindStringSubmatch(value) + if match == nil { + return 0, 0, fmt.Errorf("video offset must be a nonnegative duration in seconds: %q", value) + } + seconds, err := strconv.ParseInt(match[1], 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("video offset: %w", err) + } + var nanos int64 + if match[2] != "" { + nanos, _ = strconv.ParseInt(match[2]+strings.Repeat("0", 9-len(match[2])), 10, 64) + } + return seconds, nanos, nil +} + +// videoPartOptions validates and encodes the metadata shared by video media parts. +func videoPartOptions(part Part) ([]any, int64, error) { + var metadata []any + if video := part.VideoMetadata; video != nil { + var startSeconds, startNanos int64 + if video.StartOffset != "" { + seconds, nanos, err := videoOffset(video.StartOffset) + if err != nil { + return nil, 0, err + } + startSeconds, startNanos = seconds, nanos + metadata = []any{videoDurationWire(seconds, nanos)} + } + if video.EndOffset != "" { + seconds, nanos, err := videoOffset(video.EndOffset) + if err != nil { + return nil, 0, err + } + if seconds < startSeconds || seconds == startSeconds && nanos <= startNanos { + return nil, 0, fmt.Errorf("video end offset must be greater than its start offset") + } + for len(metadata) < 2 { + metadata = append(metadata, nil) + } + metadata[1] = videoDurationWire(seconds, nanos) + } + if video.FPS != nil { + fps := *video.FPS + if math.IsNaN(fps) || math.IsInf(fps, 0) || fps <= 0 || fps > 24 { + return nil, 0, fmt.Errorf("video fps must be greater than 0 and at most 24") + } + for len(metadata) < 3 { + metadata = append(metadata, nil) + } + metadata[2] = fps + } + } + var processing int64 + switch part.MediaProcessing { + case "", "MEDIA_PROCESSING_UNSPECIFIED": + case "STATIC": + processing = 1 + case "AGENTIC": + if len(metadata) != 0 { + return nil, 0, fmt.Errorf("agentic video processing cannot use static clipping or fps") + } + processing = 2 + default: + return nil, 0, fmt.Errorf("unknown video mediaProcessing %q", part.MediaProcessing) + } + if len(metadata) != 0 || processing != 0 { + mime := "" + switch { + case part.InlineData != nil: + mime = part.InlineData.MIME + case part.ExternalMedia != nil: + mime = part.ExternalMedia.MIME + case part.File != nil: + mime = part.File.MIME + case part.Upload != nil: + mime = part.Upload.MIME + default: + return nil, 0, fmt.Errorf("video metadata requires a video media part") + } + if !strings.HasPrefix(mime, "video/") && !(mime == "" && part.File != nil) { + return nil, 0, fmt.Errorf("video metadata requires a video MIME type") + } + } + return metadata, processing, nil +} + +// videoDurationWire encodes seconds and optional nanoseconds without losing zero. +func videoDurationWire(seconds, nanos int64) []any { + if nanos == 0 { + return []any{seconds} + } + return []any{seconds, nanos} +} + +// encodeVideoPart attaches input-video options and the existing thought signature. +func encodeVideoPart(wire []any, part Part) ([]any, error) { + metadata, processing, err := videoPartOptions(part) + if err != nil { + return nil, err + } + if len(metadata) > 0 { + for len(wire) < 16 { + wire = append(wire, nil) + } + wire[15] = metadata + } + if processing != 0 { + for len(wire) < 40 { + wire = append(wire, nil) + } + wire[39] = processing + } + return setPartThoughtSignature(wire, part.ThoughtSignature), nil +} + +// validateVideoInputs checks requested agentic processing against the live model. +func validateVideoInputs(contents []Content, model Model) error { + for _, content := range contents { + for _, part := range content.Parts { + if _, _, err := videoPartOptions(part); err != nil { + return err + } + if part.MediaProcessing == "AGENTIC" && !model.Capabilities["agentic_video"] { + return fmt.Errorf("model %s does not support agentic video processing", model.ID) + } + } + } + return nil +} diff --git a/internal/aistudio/video_input_test.go b/internal/aistudio/video_input_test.go new file mode 100644 index 00000000..19856868 --- /dev/null +++ b/internal/aistudio/video_input_test.go @@ -0,0 +1,183 @@ +package aistudio + +import ( + "encoding/json" + "math" + "strings" + "testing" +) + +func TestVideoInputProtocolEncoding(t *testing.T) { + fps := 0.5 + part := Part{ + InlineData: &Blob{MIME: "video/mp4", Data: []byte("video")}, + VideoMetadata: &VideoMetadata{StartOffset: "0s", EndOffset: "2.000000003s", FPS: &fps}, + MediaProcessing: "STATIC", + ThoughtSignature: "video-signature", + } + encodedPart, err := encodePart(part) + if err != nil { + t.Fatal(err) + } + if len(encodedPart) <= 39 || !jsonValueEqual(t, encodedPart[2], []any{"video/mp4", "dmlkZW8="}) { + t.Fatalf("video data wire = %#v", encodedPart) + } + if got, want := encodedPart[14], "video-signature"; got != want { + t.Fatalf("thought signature = %#v, want %q", got, want) + } + metadata, ok := encodedPart[15].([]any) + if !ok || len(metadata) != 3 { + t.Fatalf("video metadata = %#v", encodedPart[15]) + } + if got := metadata[0]; !jsonValueEqual(t, got, []any{int64(0)}) { + t.Fatalf("explicit zero start offset = %#v", got) + } + if got := metadata[1]; !jsonValueEqual(t, got, []any{int64(2), int64(3)}) { + t.Fatalf("end offset = %#v", got) + } + if metadata[2] != fps || encodedPart[39] != int64(1) { + t.Fatalf("fps/media processing = %#v/%#v", metadata[2], encodedPart[39]) + } + + request := GenerateRequest{Model: "video-model", Contents: []Content{{Role: RoleUser, Parts: []Part{{Text: "before"}, part, {Text: "after"}}}}} + defaults := GenerationDefaults{MaxOutputTokens: 1024} + body, err := EncodeGenerateContentRequest(request, defaults, RequestContext{}) + if err != nil { + t.Fatal(err) + } + var generate []any + if err := json.Unmarshal(body, &generate); err != nil { + t.Fatal(err) + } + content := generate[1].([]any)[0].([]any) + requestParts := content[0].([]any) + if len(requestParts) != 3 || !jsonValueEqual(t, requestParts[0], []any{nil, "before"}) || !jsonValueEqual(t, requestParts[1], encodedPart) || !jsonValueEqual(t, requestParts[2], []any{nil, "after"}) { + t.Fatalf("GenerateContent part order = %#v", requestParts) + } + + tokens, err := EncodeCountTokensRequest(TokenCountRequest{Model: "video-model", Contents: request.Contents}) + if err != nil { + t.Fatal(err) + } + var count []any + if err := json.Unmarshal(tokens, &count); err != nil { + t.Fatal(err) + } + countGenerate := count[2].([]any) + countParts := countGenerate[1].([]any)[0].([]any)[0].([]any) + if len(countParts) != 3 || !jsonValueEqual(t, countParts[1], encodedPart) { + t.Fatalf("CountTokens did not preserve video part: %#v", countGenerate) + } + + buildPart, err := encodeBuildPart(part) + if err != nil { + t.Fatal(err) + } + var buildMetadata map[string]any + if err := json.Unmarshal(mustJSON(t, buildPart), &buildMetadata); err != nil { + t.Fatal(err) + } + if buildMetadata["videoMetadata"] == nil || buildMetadata["mediaProcessing"] != "STATIC" || buildMetadata["thoughtSignature"] != "video-signature" { + t.Fatalf("Build video part = %#v", buildMetadata) + } + + filePart, err := encodePart(Part{File: &FileRef{ID: "files/video"}, MediaProcessing: "STATIC"}) + if err != nil { + t.Fatalf("video file reference without MIME was rejected: %v", err) + } + if len(filePart) <= 39 || filePart[39] != int64(1) { + t.Fatalf("video file processing field = %#v", filePart) + } +} + +func TestVideoInputValidation(t *testing.T) { + fps := 24.01 + tests := []struct { + name string + part Part + }{ + {name: "bad duration syntax", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{StartOffset: "1m"}}}, + {name: "too many fractional digits", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{StartOffset: "1.1234567890s"}}}, + {name: "end not later", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{StartOffset: "2s", EndOffset: "2s"}}}, + {name: "fps out of range", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{FPS: &fps}}}, + {name: "non-finite fps", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{FPS: floatPointer(math.Inf(1))}}}, + {name: "unknown processing", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, MediaProcessing: "FAST"}}, + {name: "agentic conflicts with metadata", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{StartOffset: "0s"}, MediaProcessing: "AGENTIC"}}, + {name: "non-video media", part: Part{InlineData: &Blob{MIME: "image/png", Data: []byte("x")}, MediaProcessing: "STATIC"}}, + {name: "text part", part: Part{Text: "video", MediaProcessing: "STATIC"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := encodePart(test.part); err == nil { + t.Fatal("expected invalid video options to fail") + } + }) + } + + video := Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, MediaProcessing: "AGENTIC"} + request := GenerateRequest{Model: "m", Contents: []Content{{Role: RoleUser, Parts: []Part{video}}}} + entry := modelEntry{model: Model{ID: "m", Capabilities: map[string]bool{}}, defaults: GenerationDefaults{MaxOutputTokens: 1024}} + if err := validateGenerateEntry(request, entry); err == nil || !strings.Contains(err.Error(), "agentic video") { + t.Fatalf("unsupported agentic video error = %v", err) + } + entry.model.Capabilities["agentic_video"] = true + if err := validateGenerateEntry(request, entry); err != nil { + t.Fatalf("supported agentic video rejected: %v", err) + } +} + +func TestVideoInputLegacyWireUnchanged(t *testing.T) { + request := GenerateRequest{Model: "m", Contents: []Content{{Role: RoleUser, Parts: []Part{{Text: "hello"}}}}} + got, err := EncodeGenerateContentRequest(request, GenerationDefaults{MaxOutputTokens: 1024}, RequestContext{}) + if err != nil { + t.Fatal(err) + } + const want = `["models/m",[[[[null,"hello"]],"user"]],[[null,null,7,5],[null,null,8,5],[null,null,9,5],[null,null,10,5]],[null,null,null,1024,null,null,null,null,null,null,null,null,null,1],null,null,null,null,null,null,1]` + if string(got) != want { + t.Fatalf("legacy GenerateContent wire = %s, want %s", got, want) + } + count, err := EncodeCountTokensRequest(TokenCountRequest{Model: "m", Contents: request.Contents}) + if err != nil { + t.Fatal(err) + } + if string(count) != `["models/m",[[[[null,"hello"]],"user"]]]` { + t.Fatalf("legacy CountTokens wire = %s", count) + } +} + +func TestInteractionRoutedVideoOptionsFailInsteadOfDisappearing(t *testing.T) { + for _, part := range []Part{ + {File: &FileRef{ID: "video"}, VideoMetadata: &VideoMetadata{StartOffset: "1s"}}, + {File: &FileRef{ID: "video"}, MediaProcessing: "STATIC"}, + {ProcessingCall: &MediaProcessingStep{ID: "call"}, ThoughtSignature: "call-signature"}, + {ProcessingResult: &MediaProcessingStep{ID: "call"}, ThoughtSignature: "result-signature"}, + } { + if _, _, _, err := encodeInteractionSteps([]Content{{Role: RoleAssistant, Parts: []Part{part}}}); err == nil { + t.Fatal("private Interaction route silently accepted unsupported video options/history") + } + } +} + +func jsonValueEqual(t *testing.T, got, want any) bool { + t.Helper() + gotJSON, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + wantJSON, err := json.Marshal(want) + if err != nil { + t.Fatal(err) + } + return string(gotJSON) == string(wantJSON) +} + +func mustJSON(t *testing.T, value any) []byte { + t.Helper() + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return encoded +} + +func floatPointer(value float64) *float64 { return &value } diff --git a/internal/api/gemini.go b/internal/api/gemini.go index 51bcfd39..52fba0e2 100644 --- a/internal/api/gemini.go +++ b/internal/api/gemini.go @@ -112,8 +112,61 @@ type geminiPart struct { Output string `json:"output"` Error string `json:"error"` } `json:"codeExecutionResult"` - SpeechMetadata *aistudio.SpeechMetadata `json:"speechMetadata"` - SpeechMetadataSnake *aistudio.SpeechMetadata `json:"speech_metadata"` + SpeechMetadata *aistudio.SpeechMetadata `json:"speechMetadata"` + SpeechMetadataSnake *aistudio.SpeechMetadata `json:"speech_metadata"` + VideoMetadata *geminiVideoMetadata `json:"videoMetadata"` + VideoMetadataSnake *geminiVideoMetadata `json:"video_metadata"` + MediaProcessing string `json:"mediaProcessing"` + MediaProcessingSnake string `json:"media_processing"` + ToolCall *geminiProcessingStep `json:"toolCall"` + ToolCallSnake *geminiProcessingStep `json:"tool_call"` + ToolResponse *geminiProcessingStep `json:"toolResponse"` + ToolResponseSnake *geminiProcessingStep `json:"tool_response"` +} + +type geminiProcessingStep struct { + ID string `json:"id"` + ToolType string `json:"toolType"` + ToolTypeSnake string `json:"tool_type"` + ToolName string `json:"toolName"` + ToolNameSnake string `json:"tool_name"` + Args json.RawMessage `json:"args"` + Response json.RawMessage `json:"response"` +} + +type geminiVideoMetadata struct { + StartOffset string `json:"startOffset"` + StartOffsetSnake string `json:"start_offset"` + EndOffset string `json:"endOffset"` + EndOffsetSnake string `json:"end_offset"` + FPS *float64 `json:"fps"` +} + +// videoMetadata maps the Gemini metadata field names to canonical video options. +func (part geminiPart) videoMetadata() *aistudio.VideoMetadata { + video := part.VideoMetadata + if video == nil { + video = part.VideoMetadataSnake + } + if video == nil { + return nil + } + start, end := video.StartOffset, video.EndOffset + if start == "" { + start = video.StartOffsetSnake + } + if end == "" { + end = video.EndOffsetSnake + } + return &aistudio.VideoMetadata{StartOffset: start, EndOffset: end, FPS: video.FPS} +} + +// mediaProcessing selects the Gemini media processing field spelling. +func (part geminiPart) mediaProcessing() string { + if part.MediaProcessing != "" { + return part.MediaProcessing + } + return part.MediaProcessingSnake } func (p geminiPart) speechMetadata() *aistudio.SpeechMetadata { @@ -770,6 +823,23 @@ func mapGeminiParts(input []geminiPart) ([]aistudio.Part, bool, error) { if part.CodeExecutionResult != nil { variants++ } + processingCall, processingResult := part.ToolCall, part.ToolResponse + if processingCall == nil { + processingCall = part.ToolCallSnake + } + if processingResult == nil { + processingResult = part.ToolResponseSnake + } + if processingCall != nil { + variants++ + } + if processingResult != nil { + variants++ + } + video, processing := part.videoMetadata(), part.mediaProcessing() + if (video != nil || processing != "") && inline == nil && file == nil { + return nil, false, fmt.Errorf("parts[%d] video metadata requires inlineData or fileData", index) + } if variants == 0 { if part.ThoughtSignature != "" { parts = append(parts, aistudio.Part{ThoughtSignature: part.ThoughtSignature}) @@ -793,6 +863,7 @@ func mapGeminiParts(input []geminiPart) ([]aistudio.Part, bool, error) { parts = append(parts, aistudio.Part{ InlineData: &aistudio.Blob{MIME: mimeType, Data: data}, ThoughtSignature: part.ThoughtSignature, + VideoMetadata: video, MediaProcessing: processing, }) case file != nil: uri := file.URI() @@ -804,13 +875,14 @@ func mapGeminiParts(input []geminiPart) ([]aistudio.Part, bool, error) { uri = driveID } if media, ok := aistudio.ExternalMediaForURL(uri); ok { - parts = append(parts, aistudio.Part{ExternalMedia: media, ThoughtSignature: part.ThoughtSignature}) + parts = append(parts, aistudio.Part{ExternalMedia: media, ThoughtSignature: part.ThoughtSignature, VideoMetadata: video, MediaProcessing: processing}) } else { parts = append(parts, aistudio.Part{ File: &aistudio.FileRef{ ID: uri, Name: file.DisplayName, MIME: mime, }, ThoughtSignature: part.ThoughtSignature, + VideoMetadata: video, MediaProcessing: processing, }) } case part.FunctionCall != nil: @@ -847,6 +919,22 @@ func mapGeminiParts(input []geminiPart) ([]aistudio.Part, bool, error) { }) parts = append(parts, media...) hasResult = true + case processingCall != nil || processingResult != nil: + value := processingCall + if value == nil { + value = processingResult + } + step, err := value.canonical(processingCall != nil) + if err != nil { + return nil, false, fmt.Errorf("parts[%d]: %w", index, err) + } + mapped := aistudio.Part{ThoughtSignature: part.ThoughtSignature} + if processingCall != nil { + mapped.ProcessingCall = step + } else { + mapped.ProcessingResult = step + } + parts = append(parts, mapped) case part.ExecutableCode != nil: parts = append(parts, aistudio.Part{ ExecutableCode: &aistudio.ExecutableCode{ @@ -1145,6 +1233,10 @@ func geminiOutputParts(result generationResult) []map[string]any { if event.ToolCall != nil { appendPart(geminiFunctionCallPart(*event.ToolCall), event.ThoughtSignature) } + case aistudio.EventMediaProcessingCall, aistudio.EventMediaProcessingResult: + if part := geminiProcessingPart(event); part != nil { + appendPart(part, event.ThoughtSignature) + } case aistudio.EventExecutableCode: if event.ExecutableCode != nil { appendPart(map[string]any{"executableCode": map[string]any{ @@ -1545,6 +1637,12 @@ func geminiStreamCandidate(event aistudio.Event, index int) map[string]any { return nil } part = geminiSignedPart(geminiFunctionCallPart(*event.ToolCall), event.ThoughtSignature) + case aistudio.EventMediaProcessingCall, aistudio.EventMediaProcessingResult: + part = geminiProcessingPart(event) + if part == nil { + return nil + } + part = geminiSignedPart(part, event.ThoughtSignature) case aistudio.EventExecutableCode: if event.ExecutableCode == nil { return nil diff --git a/internal/api/interaction_input.go b/internal/api/interaction_input.go index 36f3aab0..fd8f4b4c 100644 --- a/internal/api/interaction_input.go +++ b/internal/api/interaction_input.go @@ -74,6 +74,7 @@ type interactionInput struct { Data string `json:"data"` URI string `json:"uri"` MIME string `json:"mime_type"` + Processing json.RawMessage `json:"processing"` Content []interactionInput `json:"content"` Summary []interactionInput `json:"summary"` ID string `json:"id"` @@ -360,6 +361,22 @@ func interactionContents(raw json.RawMessage) ([]aistudio.Content, error) { if item.Signature != "" { pendingSignature = item.Signature } + case "processing_call", "processing_result": + role = aistudio.RoleAssistant + if item.Type == "processing_call" && item.ID == "" || item.Type == "processing_result" && item.CallID == "" { + return nil, fmt.Errorf("processing step requires its call identifier") + } + mapped := aistudio.Part{ThoughtSignature: item.Signature} + if item.Type == "processing_call" { + mapped.ProcessingCall = &aistudio.MediaProcessingStep{ID: item.ID} + } else { + mapped.ProcessingResult = &aistudio.MediaProcessingStep{ID: item.CallID} + } + if pendingSignature != "" { + parts = append(parts, aistudio.Part{ThoughtSignature: pendingSignature}) + pendingSignature = "" + } + parts = append(parts, mapped) case "code_execution_call", "code_execution_result": role = aistudio.RoleAssistant if text := item.codeExecutionText(); text != "" { @@ -367,8 +384,7 @@ func interactionContents(raw json.RawMessage) ([]aistudio.Content, error) { } case "google_search_call", "google_search_result", "url_context_call", "url_context_result", "google_maps_call", "google_maps_result", "file_search_call", "file_search_result", - "mcp_server_tool_call", "mcp_server_tool_result", "retrieval_call", "retrieval_result", - "processing_call", "processing_result": + "mcp_server_tool_call", "mcp_server_tool_result", "retrieval_call", "retrieval_result": continue default: parts, err = item.parts() @@ -422,6 +438,16 @@ func (item interactionInput) codeExecutionText() string { // parts 复用 Gemini 的文本、媒体与文件输入映射 func (content interactionInput) parts() ([]aistudio.Part, error) { var part geminiPart + if rawJSONConfigured(content.Processing) { + if content.Type != "video" { + return nil, fmt.Errorf("processing requires video content") + } + var err error + part.VideoMetadata, part.MediaProcessing, err = interactionVideoProcessing(content.Processing) + if err != nil { + return nil, err + } + } switch content.Type { case "text": if content.Text == nil { diff --git a/internal/api/interaction_output.go b/internal/api/interaction_output.go index ecfabcc8..1697eca8 100644 --- a/internal/api/interaction_output.go +++ b/internal/api/interaction_output.go @@ -82,6 +82,22 @@ func interactionStep(event aistudio.Event, audio interactionAudioFormat) (map[st } call := event.ToolCall return map[string]any{"type": "function_call", "id": call.ID, "name": call.Name, "arguments": call.Arguments}, nil, nil + case aistudio.EventMediaProcessingCall, aistudio.EventMediaProcessingResult: + if event.MediaProcessing == nil { + return nil, nil, nil + } + stepType := "processing_call" + step := map[string]any{"type": stepType, "id": event.MediaProcessing.ID} + if event.Kind == aistudio.EventMediaProcessingResult { + stepType = "processing_result" + step = map[string]any{"type": stepType, "call_id": event.MediaProcessing.ID} + } + delta := map[string]any{"type": stepType} + if event.ThoughtSignature != "" { + step["signature"] = event.ThoughtSignature + delta["signature"] = event.ThoughtSignature + } + return step, delta, nil case aistudio.EventMedia: if event.Media == nil { return nil, nil, nil @@ -333,7 +349,7 @@ func (s *server) streamInteraction(w http.ResponseWriter, r *http.Request, reque } textLength += length } - if active != kind || kind == "function_call" { + if active != kind || kind == "function_call" || kind == "processing_call" || kind == "processing_result" { if err := closeStep(); err != nil { return err } @@ -349,6 +365,14 @@ func (s *server) streamInteraction(w http.ResponseWriter, r *http.Request, reque if kind == "function_call" { start = step } + if kind == "processing_call" || kind == "processing_result" { + start = map[string]any{"type": kind} + for _, field := range []string{"id", "call_id"} { + if value, ok := step[field]; ok { + start[field] = value + } + } + } if err := send("step.start", map[string]any{"index": index, "step": start}); err != nil { return err } diff --git a/internal/api/responses.go b/internal/api/responses.go index 071e3e30..de7235cd 100644 --- a/internal/api/responses.go +++ b/internal/api/responses.go @@ -575,6 +575,18 @@ func responseHistoryOutput(result generationResult) aistudio.Content { call.Arguments = append(json.RawMessage(nil), call.Arguments...) output.Parts = append(output.Parts, aistudio.Part{FunctionCall: &call, ThoughtSignature: event.ThoughtSignature}) } + case aistudio.EventMediaProcessingCall, aistudio.EventMediaProcessingResult: + if event.MediaProcessing != nil { + step := *event.MediaProcessing + step.Data = append(json.RawMessage(nil), step.Data...) + part := aistudio.Part{ThoughtSignature: event.ThoughtSignature} + if event.Kind == aistudio.EventMediaProcessingCall { + part.ProcessingCall = &step + } else { + part.ProcessingResult = &step + } + output.Parts = append(output.Parts, part) + } case aistudio.EventMedia: if event.Media != nil && len(event.Media.Data) > 0 { blob := &aistudio.Blob{MIME: event.Media.MIME, Data: append([]byte(nil), event.Media.Data...)} diff --git a/internal/api/video_input.go b/internal/api/video_input.go new file mode 100644 index 00000000..e72d2410 --- /dev/null +++ b/internal/api/video_input.go @@ -0,0 +1,125 @@ +package api + +import ( + "bytes" + "encoding/json" + "fmt" + "math/big" + "strings" + + "github.com/Mag1cFall/AIStudio2API/internal/aistudio" +) + +// canonical maps only the server-owned media processing tool, never a host function. +func (value geminiProcessingStep) canonical(call bool) (*aistudio.MediaProcessingStep, error) { + typeName := value.ToolType + if typeName == "" { + typeName = value.ToolTypeSnake + } + if typeName != "MEDIA_PROCESSING" { + return nil, fmt.Errorf("unsupported server tool type %q", typeName) + } + name := value.ToolName + if name == "" { + name = value.ToolNameSnake + } + data := value.Response + if call { + data = value.Args + } + return &aistudio.MediaProcessingStep{ID: value.ID, Name: name, Data: data}, nil +} + +// geminiProcessingPart preserves native video navigation calls and their results. +func geminiProcessingPart(event aistudio.Event) map[string]any { + step := event.MediaProcessing + if step == nil { + return nil + } + value := map[string]any{"toolType": "MEDIA_PROCESSING"} + if step.ID != "" { + value["id"] = step.ID + } + key, payload := "toolResponse", "response" + if event.Kind == aistudio.EventMediaProcessingCall { + key, payload = "toolCall", "args" + if step.Name != "" { + value["toolName"] = step.Name + } + } + if len(step.Data) != 0 { + value[payload] = step.Data + } + return map[string]any{key: value} +} + +// interactionVideoProcessing maps the documented string or static object union. +func interactionVideoProcessing(raw json.RawMessage) (*geminiVideoMetadata, string, error) { + var mode string + if json.Unmarshal(raw, &mode) == nil { + switch mode { + case "static": + return nil, "STATIC", nil + case "agentic": + return nil, "AGENTIC", nil + default: + return nil, "", fmt.Errorf("unknown video processing mode %q", mode) + } + } + var config struct { + Type string `json:"type"` + Start json.RawMessage `json:"start_offset"` + End json.RawMessage `json:"end_offset"` + FPS *float64 `json:"fps"` + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&config); err != nil { + return nil, "", fmt.Errorf("video processing: %w", err) + } + if config.Type != "static" { + return nil, "", fmt.Errorf("video processing object type must be static") + } + start, err := interactionVideoOffset(config.Start) + if err != nil { + return nil, "", fmt.Errorf("video start_offset: %w", err) + } + end, err := interactionVideoOffset(config.End) + if err != nil { + return nil, "", fmt.Errorf("video end_offset: %w", err) + } + return &geminiVideoMetadata{StartOffset: start, EndOffset: end, FPS: config.FPS}, "STATIC", nil +} + +// interactionVideoOffset accepts documented duration strings and numeric seconds. +func interactionVideoOffset(raw json.RawMessage) (string, error) { + if !rawJSONConfigured(raw) { + return "", nil + } + var value string + if json.Unmarshal(raw, &value) == nil { + return value, nil + } + var seconds json.Number + if err := json.Unmarshal(raw, &seconds); err != nil { + return "", fmt.Errorf("expected a duration string or numeric seconds") + } + offset, ok := new(big.Rat).SetString(seconds.String()) + if !ok || offset.Sign() < 0 { + return "", fmt.Errorf("video offset must be nonnegative numeric seconds") + } + offset.Mul(offset, big.NewRat(1_000_000_000, 1)) + if !offset.IsInt() { + return "", fmt.Errorf("video offset exceeds nanosecond precision") + } + whole, nanos := new(big.Int), new(big.Int) + whole.QuoRem(offset.Num(), big.NewInt(1_000_000_000), nanos) + if !whole.IsInt64() { + return "", fmt.Errorf("video offset seconds exceed int64 range") + } + if nanos.Sign() == 0 { + return whole.String() + "s", nil + } + fraction := strings.TrimRight(fmt.Sprintf("%09d", nanos.Int64()), "0") + return whole.String() + "." + fraction + "s", nil +} diff --git a/internal/api/video_input_test.go b/internal/api/video_input_test.go new file mode 100644 index 00000000..0f5d8acb --- /dev/null +++ b/internal/api/video_input_test.go @@ -0,0 +1,175 @@ +package api + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/Mag1cFall/AIStudio2API/internal/aistudio" +) + +func TestGeminiVideoInputFieldMappings(t *testing.T) { + for _, test := range []struct { + name string + json string + }{ + { + name: "camel case", + json: `{"inlineData":{"mimeType":"video/mp4","data":"dmlkZW8="},"videoMetadata":{"startOffset":"0s","endOffset":"3.25s","fps":0.5},"mediaProcessing":"STATIC"}`, + }, + { + name: "snake case", + json: `{"inline_data":{"mime_type":"video/mp4","data":"dmlkZW8="},"video_metadata":{"start_offset":"0s","end_offset":"3.25s","fps":0.5},"media_processing":"STATIC"}`, + }, + } { + t.Run(test.name, func(t *testing.T) { + var part geminiPart + if err := json.Unmarshal([]byte(test.json), &part); err != nil { + t.Fatal(err) + } + mapped, _, err := mapGeminiParts([]geminiPart{part}) + if err != nil { + t.Fatal(err) + } + if len(mapped) != 1 || mapped[0].VideoMetadata == nil || mapped[0].VideoMetadata.StartOffset != "0s" || mapped[0].VideoMetadata.EndOffset != "3.25s" || mapped[0].VideoMetadata.FPS == nil || *mapped[0].VideoMetadata.FPS != 0.5 { + t.Fatalf("mapped video metadata = %#v", mapped) + } + if mapped[0].MediaProcessing != "STATIC" { + t.Fatalf("media processing = %q, want STATIC", mapped[0].MediaProcessing) + } + }) + } +} + +func TestInteractionVideoInputProcessing(t *testing.T) { + tests := []struct { + name string + processing string + wantMode string + wantStart string + wantEnd string + wantFPS float64 + }{ + {name: "static string", processing: `"static"`, wantMode: "STATIC"}, + {name: "agentic string", processing: `"agentic"`, wantMode: "AGENTIC"}, + {name: "static options", processing: `{"type":"static","start_offset":2,"end_offset":4,"fps":0.5}`, wantMode: "STATIC", wantStart: "2s", wantEnd: "4s", wantFPS: 0.5}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + raw := `{"type":"video","data":"dmlkZW8=","mime_type":"video/mp4","processing":` + test.processing + `}` + var input interactionInput + if err := json.Unmarshal([]byte(raw), &input); err != nil { + t.Fatal(err) + } + parts, err := input.parts() + if err != nil { + t.Fatal(err) + } + if len(parts) != 1 || parts[0].MediaProcessing != test.wantMode { + t.Fatalf("interaction video parts = %#v", parts) + } + if test.wantStart == "" { + if parts[0].VideoMetadata != nil { + t.Fatalf("unexpected video metadata: %#v", parts[0].VideoMetadata) + } + return + } + metadata := parts[0].VideoMetadata + if metadata == nil || metadata.StartOffset != test.wantStart || metadata.EndOffset != test.wantEnd || metadata.FPS == nil || *metadata.FPS != test.wantFPS { + t.Fatalf("interaction video metadata = %#v", metadata) + } + }) + } +} + +func TestInteractionVideoInputRejectsInvalidProcessing(t *testing.T) { + for _, test := range []struct { + processing string + mapperFails bool + }{ + {processing: `"unknown"`, mapperFails: true}, + {processing: `{"type":"static","start_offset":{}}`, mapperFails: true}, + {processing: `{"type":"static","startOffset":2}`, mapperFails: true}, + {processing: `{"type":"static","start_offset":"1m"}`}, + {processing: `{"type":"static","start_offset":2,"end_offset":2}`}, + {processing: `{"type":"static","fps":0}`}, + {processing: `{"type":"static","fps":25}`}, + {processing: `{"type":"agentic","fps":1}`, mapperFails: true}, + } { + raw := `{"type":"video","data":"dmlkZW8=","mime_type":"video/mp4","processing":` + test.processing + `}` + var input interactionInput + if err := json.Unmarshal([]byte(raw), &input); err != nil { + t.Fatal(err) + } + parts, err := input.parts() + if test.mapperFails { + if err == nil { + t.Errorf("processing %s unexpectedly accepted by mapper", test.processing) + } + continue + } + if err != nil { + t.Fatalf("valid processing shape %s did not map: %v", test.processing, err) + } + request := aistudio.GenerateRequest{Model: "m", Contents: []aistudio.Content{{Role: aistudio.RoleUser, Parts: parts}}} + if _, err := aistudio.EncodeGenerateContentRequest(request, aistudio.GenerationDefaults{MaxOutputTokens: 1024}, aistudio.RequestContext{}); err == nil { + t.Errorf("processing %s unexpectedly passed canonical wire validation", test.processing) + } + } +} + +func TestInteractionNumericVideoOffsets(t *testing.T) { + for _, test := range []struct { + input string + want string + wire string + }{ + {input: "1e-3", want: "0.001s", wire: `[0,1000000]`}, + {input: "2e0", want: "2s", wire: `[2]`}, + {input: "1e-9", want: "0.000000001s", wire: `[0,1]`}, + {input: "100e-10", want: "0.00000001s", wire: `[0,10]`}, + {input: "1.25E+1", want: "12.5s", wire: `[12,500000000]`}, + {input: "9007199254740993e0", want: "9007199254740993s", wire: `[9007199254740993]`}, + {input: "9223372036854775807e0", want: "9223372036854775807s", wire: `[9223372036854775807]`}, + {input: "0e999999999", want: "0s", wire: `[0]`}, + } { + t.Run(test.input, func(t *testing.T) { + var input interactionInput + raw := `{"type":"video","data":"dmlkZW8=","mime_type":"video/mp4","processing":{"type":"static","start_offset":` + test.input + `}}` + if err := json.Unmarshal([]byte(raw), &input); err != nil { + t.Fatal(err) + } + parts, err := input.parts() + if err != nil { + t.Fatal(err) + } + if got := parts[0].VideoMetadata.StartOffset; got != test.want { + t.Fatalf("offset = %q, want %q", got, test.want) + } + request := aistudio.GenerateRequest{Model: "m", Contents: []aistudio.Content{{Role: aistudio.RoleUser, Parts: parts}}} + encoded, err := aistudio.EncodeGenerateContentRequest(request, aistudio.GenerationDefaults{MaxOutputTokens: 64}, aistudio.RequestContext{}) + if err != nil { + t.Fatal(err) + } + var wire []any + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + if err := decoder.Decode(&wire); err != nil { + t.Fatal(err) + } + part := wire[1].([]any)[0].([]any)[0].([]any)[0].([]any) + duration, err := json.Marshal(part[15].([]any)[0]) + if err != nil || string(duration) != test.wire { + t.Fatalf("offset wire = %s, want %s: %v", duration, test.wire, err) + } + }) + } +} + +func TestInteractionNumericVideoOffsetLimits(t *testing.T) { + for _, value := range []string{"-1e-3", "1e-10", "1.0000000001", "9223372036854775808", "1e1000000000", "1e-1000000000"} { + if _, err := interactionVideoOffset(json.RawMessage(value)); err == nil { + t.Errorf("unrepresentable numeric offset %s accepted", value) + } + } +} diff --git a/internal/api/video_processing_history_test.go b/internal/api/video_processing_history_test.go new file mode 100644 index 00000000..535b4d61 --- /dev/null +++ b/internal/api/video_processing_history_test.go @@ -0,0 +1,147 @@ +package api + +import ( + "encoding/json" + "net/http/httptest" + "strings" + "testing" + + "github.com/Mag1cFall/AIStudio2API/internal/aistudio" +) + +func TestGeminiMediaProcessingOutputAndHistory(t *testing.T) { + events := []aistudio.Event{ + {Kind: aistudio.EventMediaProcessingCall, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call"}, ThoughtSignature: "call-signature"}, + {Kind: aistudio.EventMediaProcessingResult, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call"}, ThoughtSignature: "result-signature"}, + {Kind: aistudio.EventText, Text: "Video answer"}, + } + parts := geminiOutputParts(generationResult{events: events}) + if len(parts) != 3 { + t.Fatalf("output parts = %#v", parts) + } + for i, key := range []string{"toolCall", "toolResponse"} { + tool, ok := parts[i][key].(map[string]any) + if !ok || tool["toolType"] != "MEDIA_PROCESSING" || tool["id"] != "video_call" { + t.Fatalf("native processing part = %#v", parts[i]) + } + if parts[i]["functionCall"] != nil || parts[i]["thoughtSignature"] != events[i].ThoughtSignature { + t.Fatalf("processing authority/signature = %#v", parts[i]) + } + stream := geminiStreamCandidate(events[i], 0) + streamParts := stream["content"].(map[string]any)["parts"].([]any) + if len(streamParts) != 1 || streamParts[0].(map[string]any)[key] == nil { + t.Fatalf("stream processing projection = %#v", stream) + } + } + raw, err := json.Marshal(parts) + if err != nil { + t.Fatal(err) + } + var input []geminiPart + if err := json.Unmarshal(raw, &input); err != nil { + t.Fatal(err) + } + canonical, hasHostResult, err := mapGeminiParts(input) + if err != nil || hasHostResult || len(canonical) != 3 { + t.Fatalf("history mapping = %#v, %v, %v", canonical, hasHostResult, err) + } + if canonical[0].ProcessingCall == nil || canonical[1].ProcessingResult == nil || canonical[0].ThoughtSignature != "call-signature" || canonical[1].ThoughtSignature != "result-signature" { + t.Fatalf("history lost processing trace: %#v", canonical) + } + if _, err := aistudio.EncodeCountTokensRequest(aistudio.TokenCountRequest{Model: "m", Contents: []aistudio.Content{{Role: aistudio.RoleAssistant, Parts: canonical}}}); err != nil { + t.Fatalf("processing history count encoding: %v", err) + } +} + +func TestInteractionProcessingHistoryPreservesSignatures(t *testing.T) { + contents, err := interactionContents(json.RawMessage(`[ + {"type":"thought","signature":"prior-signature"}, + {"type":"processing_call","id":"video_call","signature":"call-signature"}, + {"type":"processing_result","call_id":"video_call","signature":"result-signature"} + ]`)) + if err != nil { + t.Fatal(err) + } + if len(contents) != 1 || contents[0].Role != aistudio.RoleAssistant || len(contents[0].Parts) != 3 { + t.Fatalf("processing history = %#v", contents) + } + parts := contents[0].Parts + if parts[0].ThoughtSignature != "prior-signature" || parts[1].ThoughtSignature != "call-signature" || parts[2].ThoughtSignature != "result-signature" || parts[1].ProcessingCall == nil || parts[2].ProcessingResult == nil { + t.Fatalf("processing signatures were overwritten: %#v", parts) + } + for _, raw := range []string{`[{"type":"processing_call"}]`, `[{"type":"processing_result"}]`} { + if _, err := interactionContents(json.RawMessage(raw)); err == nil { + t.Fatalf("missing processing ID accepted: %s", raw) + } + } +} + +func TestStoredInteractionMediaProcessingContinuation(t *testing.T) { + for _, stream := range []bool{false, true} { + name := "nonstream" + if stream { + name = "stream" + } + t.Run(name, func(t *testing.T) { + call := &aistudio.MediaProcessingStep{ID: "video_call", Name: "inspect", Data: json.RawMessage(`{"start":2}`)} + reply := &aistudio.MediaProcessingStep{ID: "video_call", Data: json.RawMessage(`{"loaded":true}`)} + events := []aistudio.Event{ + {Kind: aistudio.EventMediaProcessingCall, MediaProcessing: call, ThoughtSignature: "call-signature"}, + {Kind: aistudio.EventMediaProcessingResult, MediaProcessing: reply, ThoughtSignature: "result-signature"}, + {Kind: aistudio.EventText, Text: "Video answer"}, + {Kind: aistudio.EventFinish, FinishReason: "stop"}, + } + store := newResponseStateStore() + s := &server{responseStates: store} + request := interactionRequest{Model: "video-model", Input: json.RawMessage(`"Inspect the video"`)} + current := []aistudio.Content{{Role: aistudio.RoleUser, Parts: []aistudio.Part{{Text: "Inspect the video"}}}} + if stream { + channel := make(chan aistudio.Event, len(events)) + for _, event := range events { + channel <- event + } + close(channel) + response := httptest.NewRecorder() + s.streamInteraction(response, httptest.NewRequest("POST", "/v1beta/interactions", nil), request, + aistudio.GenerateRequest{ID: "stored-video", Model: request.Model}, false, responseHistory{}, current, "2026-10-28T00:00:00Z", channel) + if !strings.Contains(response.Body.String(), "interaction.completed") { + t.Fatalf("stream did not complete: %s", response.Body.String()) + } + } else { + s.storeInteraction(request, responseHistory{}, current, generationResult{events: events}, map[string]any{"id": "stored-video"}) + } + call.ID, reply.ID = "changed-call", "changed-result" + call.Data[0], reply.Data[0] = '[', '[' + history, ok := store.Load("stored-video") + if !ok || !history.Interaction || len(history.Contents) != 2 { + t.Fatalf("stored interaction history = %#v", history) + } + parts := history.Contents[1].Parts + if len(parts) != 3 || parts[0].ProcessingCall == nil || parts[1].ProcessingResult == nil { + t.Fatalf("stored navigation parts = %#v", parts) + } + if parts[0].ProcessingCall.ID != "video_call" || parts[1].ProcessingResult.ID != "video_call" || + parts[0].ProcessingCall.Name != "inspect" || string(parts[0].ProcessingCall.Data) != `{"start":2}` || + string(parts[1].ProcessingResult.Data) != `{"loaded":true}` || parts[0].ThoughtSignature != "call-signature" || parts[1].ThoughtSignature != "result-signature" { + t.Fatalf("stored processing identity or payload changed: %#v", parts) + } + contents := append(history.Contents, aistudio.Content{Role: aistudio.RoleUser, Parts: []aistudio.Part{{Text: "Continue"}}}) + encoded, err := aistudio.EncodeGenerateContentRequest(aistudio.GenerateRequest{Model: request.Model, Contents: contents}, aistudio.GenerationDefaults{MaxOutputTokens: 64}, aistudio.RequestContext{}) + if err != nil { + t.Fatal(err) + } + var wire []any + if err := json.Unmarshal(encoded, &wire); err != nil { + t.Fatal(err) + } + modelParts := wire[1].([]any)[1].([]any)[0].([]any) + for index, field := range []int{25, 26} { + part := modelParts[index].([]any) + step := part[field].([]any) + if step[0] != float64(13) || step[3] != "video_call" || part[14] != parts[index].ThoughtSignature { + t.Fatalf("continuation navigation wire = %#v", part) + } + } + }) + } +} diff --git a/internal/api/video_processing_output_test.go b/internal/api/video_processing_output_test.go new file mode 100644 index 00000000..9331c7c2 --- /dev/null +++ b/internal/api/video_processing_output_test.go @@ -0,0 +1,117 @@ +package api + +import ( + "encoding/json" + "net/http/httptest" + "strings" + "testing" + + "github.com/Mag1cFall/AIStudio2API/internal/aistudio" +) + +func TestInteractionStepsProjectMediaProcessing(t *testing.T) { + result := generationResult{events: []aistudio.Event{ + {Kind: aistudio.EventMediaProcessingCall, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call_1"}, ThoughtSignature: "call-signature"}, + {Kind: aistudio.EventMediaProcessingResult, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call_1"}, ThoughtSignature: "result-signature"}, + }} + + steps, err := interactionSteps(result, interactionAudioFormat{}, true) + if err != nil { + t.Fatal(err) + } + if len(steps) != 2 { + t.Fatalf("steps = %#v, want one processing call and result", steps) + } + if got := steps[0]; got["type"] != "processing_call" || got["id"] != "video_call_1" || got["signature"] != "call-signature" { + t.Errorf("processing call = %#v", got) + } + if got := steps[1]; got["type"] != "processing_result" || got["call_id"] != "video_call_1" || got["signature"] != "result-signature" { + t.Errorf("processing result = %#v", got) + } + for _, step := range steps { + if step["type"] == "function_call" || step["type"] == "model_output" { + t.Errorf("server processing step was projected as host/model output: %#v", step) + } + } +} + +func TestStreamInteractionMediaProcessingProtocol(t *testing.T) { + store := false + request := interactionRequest{Store: &store} + generate := aistudio.GenerateRequest{ID: "interaction_1", Model: "model"} + events := make(chan aistudio.Event, 3) + events <- aistudio.Event{Kind: aistudio.EventMediaProcessingCall, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call_1"}, ThoughtSignature: "call-signature"} + events <- aistudio.Event{Kind: aistudio.EventMediaProcessingResult, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call_1"}, ThoughtSignature: "result-signature"} + events <- aistudio.Event{Kind: aistudio.EventFinish, FinishReason: "STOP"} + close(events) + + response := httptest.NewRecorder() + s := &server{} + s.streamInteraction(response, httptest.NewRequest("POST", "/v1/interactions", nil), request, generate, false, responseHistory{}, nil, "2026-10-26T00:00:00Z", events) + + type frame struct { + event string + data map[string]any + } + var frames []frame + for _, block := range strings.Split(strings.TrimSpace(response.Body.String()), "\n\n") { + if block == "" { + continue + } + var current frame + for _, line := range strings.Split(block, "\n") { + if strings.HasPrefix(line, "event: ") { + current.event = strings.TrimPrefix(line, "event: ") + } + if strings.HasPrefix(line, "data: ") { + if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), ¤t.data); err != nil { + t.Fatalf("decode stream frame: %v", err) + } + } + } + frames = append(frames, current) + } + wantEvents := []string{"interaction.created", "step.start", "step.delta", "step.stop", "step.start", "step.delta", "step.stop", "interaction.completed"} + if len(frames) != len(wantEvents) { + t.Fatalf("stream frames = %#v", frames) + } + for i, want := range wantEvents { + if frames[i].event != want { + t.Errorf("frame %d event = %q, want %q", i, frames[i].event, want) + } + } + assertFrameValue := func(index int, field string, want any) { + t.Helper() + if got := frames[index].data[field]; got != want { + t.Errorf("frame %d %s = %#v, want %#v", index, field, got, want) + } + } + for _, test := range []struct { + startIndex int + deltaIndex int + kind string + idField string + signature string + }{ + {startIndex: 1, deltaIndex: 2, kind: "processing_call", idField: "id", signature: "call-signature"}, + {startIndex: 4, deltaIndex: 5, kind: "processing_result", idField: "call_id", signature: "result-signature"}, + } { + start, _ := frames[test.startIndex].data["step"].(map[string]any) + delta, _ := frames[test.deltaIndex].data["delta"].(map[string]any) + if start["type"] != test.kind || start[test.idField] != "video_call_1" { + t.Errorf("%s start step = %#v", test.kind, start) + } + if _, ok := start["signature"]; ok { + t.Errorf("%s signature appeared before its delta: %#v", test.kind, start) + } + if delta["type"] != test.kind || delta["signature"] != test.signature { + t.Errorf("%s delta = %#v", test.kind, delta) + } + assertFrameValue(test.startIndex, "index", float64(test.startIndex/3)) + assertFrameValue(test.deltaIndex, "index", float64(test.startIndex/3)) + assertFrameValue(test.startIndex+2, "index", float64(test.startIndex/3)) + } + if strings.Contains(response.Body.String(), `"type":"function_call"`) { + t.Fatal("server media processing emitted a host function call") + } +} From 0f416014de79ef6f5a60cd2dec0a109b4fd767fe Mon Sep 17 00:00:00 2001 From: Mag1cFall Date: Tue, 27 Oct 2026 16:56:51 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20=E7=B2=BE=E7=AE=80=E8=A7=86=E9=A2=91?= =?UTF-8?q?=E5=A4=84=E7=90=86=E5=8F=82=E6=95=B0=E5=AE=9E=E7=8E=B0=E5=B9=B6?= =?UTF-8?q?=E8=A1=A5=E5=85=A8=E7=BB=AD=E6=8E=A5=E5=8E=86=E5=8F=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 删除视频参数的重复校验与本地预检，帧率、处理方式组合与模型能力交由上游判定 - 删除 Gemini 输入中已由键名归一化覆盖的 snake_case 字段与取值函数 - 合并 Playground 视频元数据与服务端处理步骤的编解码，Build 直接写出 Gemini API 字段 - 让 Responses 与 Interactions 续接历史保留代码执行、链接媒体与服务端处理步骤 - 把注释改为中文单行，测试精简为一个公开协议往返用例 --- docs/protocol.md | 2 +- internal/aistudio/build.go | 91 +++------ internal/aistudio/event.go | 11 - internal/aistudio/generate.go | 4 +- internal/aistudio/interaction.go | 5 +- internal/aistudio/media_processing.go | 57 ------ internal/aistudio/media_processing_test.go | 125 ------------ internal/aistudio/request.go | 10 +- internal/aistudio/types.go | 15 +- internal/aistudio/video_input.go | 191 +++++++++--------- internal/aistudio/video_input_test.go | 183 ----------------- internal/api/gemini.go | 115 ++++------- internal/api/interaction_input.go | 6 +- internal/api/interaction_output.go | 31 +-- internal/api/responses.go | 25 +-- internal/api/video_input.go | 101 +-------- internal/api/video_input_test.go | 191 +++--------------- internal/api/video_processing_history_test.go | 147 -------------- internal/api/video_processing_output_test.go | 117 ----------- 19 files changed, 246 insertions(+), 1181 deletions(-) delete mode 100644 internal/aistudio/media_processing.go delete mode 100644 internal/aistudio/media_processing_test.go delete mode 100644 internal/aistudio/video_input_test.go delete mode 100644 internal/api/video_processing_history_test.go delete mode 100644 internal/api/video_processing_output_test.go diff --git a/docs/protocol.md b/docs/protocol.md index 9f4a778e..54efdeb7 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1975,7 +1975,7 @@ delta 联合类型为 `text_delta{text}`、`thinking_delta{thinking}`、`signatu | --- | --- | | `input` | 字符串、单个内容块、内容块数组或步骤数组；内容类型为 `text`、`image`、`audio`、`video`、`document` | | 媒体内容 | `data`（Base64）或 `uri`，可选 `mime_type` | -| 视频处理 | `processing:"static"`、`"agentic"` 或 `{type:"static",start_offset?,end_offset?,fps?}`；offset 接受秒数或带 `s` 的 Duration 字符串 | +| 视频处理 | `processing:"static"`、`"agentic"` 或 `{type:"static",start_offset?,end_offset?,fps?}`；offset 为带 `s` 的 Duration 字符串，例如 `"2.5s"` | | 输入步骤 | `user_input`、`model_output`、`thought`、`function_call`、`function_result`、`code_execution_call`、`code_execution_result` 与工具 call/result 步骤 | | `system_instruction` | 当前请求的系统指令 | | `generation_config` | `temperature`、`top_p`、`top_k`、`max_output_tokens`、`seed`、`stop_sequences`、`thinking_level`、`thinking_summaries`、`speech_config`、`tool_choice`、`image_config:{aspect_ratio?,image_size?}` | diff --git a/internal/aistudio/build.go b/internal/aistudio/build.go index cdc485a1..13327b68 100644 --- a/internal/aistudio/build.go +++ b/internal/aistudio/build.go @@ -166,10 +166,6 @@ func encodeBuildContents(contents []Content) ([]any, error) { } func encodeBuildPart(part Part) (map[string]any, error) { - metadata, processing, err := videoPartOptions(part) - if err != nil { - return nil, err - } wire := map[string]any{} signature := part.ThoughtSignature switch { @@ -245,28 +241,9 @@ func encodeBuildPart(part Part) (map[string]any, error) { } wire["codeExecutionResult"] = result case part.ProcessingCall != nil: - step := part.ProcessingCall - if _, err := encodeMediaProcessingStep(step); err != nil { - return nil, err - } - value := map[string]any{"toolType": "MEDIA_PROCESSING", "id": step.ID} - if step.Name != "" { - value["toolName"] = step.Name - } - if len(step.Data) != 0 { - value["args"] = step.Data - } - wire["toolCall"] = value + wire["toolCall"] = buildMediaProcessingStep(part.ProcessingCall, "args") case part.ProcessingResult != nil: - step := part.ProcessingResult - if _, err := encodeMediaProcessingStep(step); err != nil { - return nil, err - } - value := map[string]any{"toolType": "MEDIA_PROCESSING", "id": step.ID} - if len(step.Data) != 0 { - value["response"] = step.Data - } - wire["toolResponse"] = value + wire["toolResponse"] = buildMediaProcessingStep(part.ProcessingResult, "response") case signature != "": wire["text"] = "" default: @@ -275,26 +252,37 @@ func encodeBuildPart(part Part) (map[string]any, error) { if signature != "" { wire["thoughtSignature"] = signature } - if len(metadata) > 0 { - video := part.VideoMetadata - value := map[string]any{} + if video := part.VideoMetadata; video != nil { + metadata := map[string]any{} if video.StartOffset != "" { - value["startOffset"] = video.StartOffset + metadata["startOffset"] = video.StartOffset } if video.EndOffset != "" { - value["endOffset"] = video.EndOffset + metadata["endOffset"] = video.EndOffset } if video.FPS != nil { - value["fps"] = *video.FPS + metadata["fps"] = *video.FPS } - wire["videoMetadata"] = value + wire["videoMetadata"] = metadata } - if processing != 0 { + if part.MediaProcessing != "" { wire["mediaProcessing"] = part.MediaProcessing } return wire, nil } +// buildMediaProcessingStep 编码视频理解处理的 toolCall 或 toolResponse，payload 为 args 或 response +func buildMediaProcessingStep(step *MediaProcessingStep, payload string) map[string]any { + value := map[string]any{"toolType": "MEDIA_PROCESSING", "id": step.ID} + if step.Name != "" { + value["toolName"] = step.Name + } + if len(step.Data) != 0 { + value[payload] = step.Data + } + return value +} + // buildJSONObject 把 JSON 值转成 Gemini API Struct，非对象值放在 result 字段 func buildJSONObject(raw json.RawMessage) (any, error) { trimmed := bytes.TrimSpace(raw) @@ -535,11 +523,12 @@ type buildResponsePart struct { Name string `json:"name"` Args json.RawMessage `json:"args"` } `json:"functionCall"` - ToolCall *buildProcessingStep `json:"toolCall"` - ToolResponse *buildProcessingStep `json:"toolResponse"` + ToolCall *buildToolStep `json:"toolCall"` + ToolResponse *buildToolStep `json:"toolResponse"` } -type buildProcessingStep struct { +// buildToolStep 表示 Gemini API 服务端工具的 toolCall 或 toolResponse +type buildToolStep struct { ID string `json:"id"` ToolType string `json:"toolType"` ToolName string `json:"toolName"` @@ -718,31 +707,11 @@ func decodeBuildPart(part buildResponsePart, index int, raw []byte) ([]Event, er call := FunctionCall{ID: part.FunctionCall.ID, Name: part.FunctionCall.Name, Arguments: arguments, ThoughtSignature: signature} events = append(events, Event{Kind: EventToolCall, ToolCall: &call, ThoughtSignature: signature}) } - for _, field := range []struct { - step *buildProcessingStep - kind EventKind - name string - }{{part.ToolCall, EventMediaProcessingCall, "toolCall"}, {part.ToolResponse, EventMediaProcessingResult, "toolResponse"}} { - if field.step == nil || field.step.ToolType != "MEDIA_PROCESSING" { - continue - } - if len(events) != 0 { - return nil, &ProtocolEvidenceError{Method: buildProxyStreamedMethod, Path: fmt.Sprintf("$chunk.parts[%d]", index), Detail: "media processing part contains multiple data variants", Raw: cloneRaw(raw)} - } - step := field.step - data := step.Args - if field.kind == EventMediaProcessingResult { - data = step.Response - } - if isJSONNull(data) { - data = nil - } else { - var object map[string]json.RawMessage - if err := json.Unmarshal(data, &object); err != nil { - return nil, &ProtocolEvidenceError{Method: buildProxyStreamedMethod, Path: fmt.Sprintf("$chunk.parts[%d].%s", index, field.name), Detail: "media processing payload must be an object", Raw: cloneRaw(raw)} - } - } - events = append(events, Event{Kind: field.kind, MediaProcessing: &MediaProcessingStep{ID: step.ID, Name: step.ToolName, Data: data}, ThoughtSignature: signature}) + if step := part.ToolCall; step != nil && step.ToolType == "MEDIA_PROCESSING" { + events = append(events, Event{Kind: EventMediaProcessingCall, MediaProcessing: &MediaProcessingStep{ID: step.ID, Name: step.ToolName, Data: step.Args}, ThoughtSignature: signature}) + } + if step := part.ToolResponse; step != nil && step.ToolType == "MEDIA_PROCESSING" { + events = append(events, Event{Kind: EventMediaProcessingResult, MediaProcessing: &MediaProcessingStep{ID: step.ID, Name: step.ToolName, Data: step.Response}, ThoughtSignature: signature}) } if len(events) == 0 && signature != "" { events = append(events, Event{Kind: EventThoughtSignature, ThoughtSignature: signature}) diff --git a/internal/aistudio/event.go b/internal/aistudio/event.go index 37af89dd..1b0dd8df 100644 --- a/internal/aistudio/event.go +++ b/internal/aistudio/event.go @@ -252,17 +252,6 @@ func (d *FrameDecoder) decodePart(raw json.RawMessage, path string, evidence jso if err != nil { return nil, withMethod(err, "GenerateContent") } - if !isJSONNull(rawAt(part, 25)) || !isJSONNull(rawAt(part, 26)) { - variants := 0 - for _, index := range []int{1, 2, 5, 6, 7, 8, 10, 11, 17, 25, 26} { - if !isJSONNull(rawAt(part, index)) { - variants++ - } - } - if variants != 1 { - return nil, d.protocolError(path, "media processing part contains multiple data variants", raw) - } - } thought := false if thoughtRaw := rawAt(part, 12); !isJSONNull(thoughtRaw) { thought, err = rawBool(thoughtRaw, path+"[12]", raw) diff --git a/internal/aistudio/generate.go b/internal/aistudio/generate.go index 4d952cb6..9f856ba8 100644 --- a/internal/aistudio/generate.go +++ b/internal/aistudio/generate.go @@ -645,7 +645,7 @@ func (c *Client) validateGenerateRequest(request GenerateRequest) error { // validateGenerateEntry 按 Generate 的顺序校验工具、转写、生成参数与安全设置 func validateGenerateEntry(request GenerateRequest, entry modelEntry) error { - if err := validateVideoInputs(request.Contents, entry.model); err != nil { + if err := validateVideoOffsets(request.Contents); err != nil { return fmt.Errorf("%w: %v", ErrInvalidArgument, err) } if _, err := supportedTools(request.Tools, entry.model); err != nil { @@ -679,7 +679,7 @@ func (c *Client) Generate(ctx context.Context, request GenerateRequest) (<-chan if err != nil { return nil, err } - if err := validateVideoInputs(request.Contents, entry.model); err != nil { + if err := validateVideoOffsets(request.Contents); err != nil { return nil, fmt.Errorf("%w: %v", ErrInvalidArgument, err) } request.Tools, err = supportedTools(request.Tools, entry.model) diff --git a/internal/aistudio/interaction.go b/internal/aistudio/interaction.go index 5a8ac794..fdb4dbb5 100644 --- a/internal/aistudio/interaction.go +++ b/internal/aistudio/interaction.go @@ -158,10 +158,9 @@ func encodeInteractionSteps(contents []Content) ([]any, []Content, bool, error) parts := make([]any, 0, len(content.Parts)) texts := make([]Part, 0, len(content.Parts)) for _, part := range content.Parts { - if part.VideoMetadata != nil || part.MediaProcessing != "" || part.ProcessingCall != nil || part.ProcessingResult != nil { - return nil, nil, false, fmt.Errorf("Interaction-routed models do not support GenerateContent video processing options or navigation history") - } switch { + case part.VideoMetadata != nil || part.MediaProcessing != "" || part.ProcessingCall != nil || part.ProcessingResult != nil: + return nil, nil, false, fmt.Errorf("Interaction 模型的 contents[%d] 不接受视频截取、采样、处理方式与处理步骤", index) case part.Thought: continue case part.ExecutableCode != nil || part.CodeExecutionResult != nil: diff --git a/internal/aistudio/media_processing.go b/internal/aistudio/media_processing.go deleted file mode 100644 index 12e99544..00000000 --- a/internal/aistudio/media_processing.go +++ /dev/null @@ -1,57 +0,0 @@ -package aistudio - -import ( - "encoding/json" - "fmt" -) - -// encodeMediaProcessingStep encodes the observed MEDIA_PROCESSING tool message. -func encodeMediaProcessingStep(step *MediaProcessingStep) ([]any, error) { - wire := []any{int64(13), nil, nil, step.ID} - if step.Name != "" { - wire[1] = step.Name - } - if len(step.Data) != 0 { - value, err := encodeWireStructJSON(step.Data) - if err != nil { - return nil, fmt.Errorf("media processing payload: %w", err) - } - wire[2] = value - } - return wire, nil -} - -// decodeMediaProcessingStep decodes the server-owned tool type observed as 13. -func decodeMediaProcessingStep(raw json.RawMessage, path string, evidence json.RawMessage) (*MediaProcessingStep, error) { - fields, err := rawArray(raw, path, evidence) - if err != nil { - return nil, withMethod(err, "GenerateContent") - } - typeID, err := rawInt64(rawAt(fields, 0), path+"[0]", evidence) - if err != nil { - return nil, withMethod(err, "GenerateContent") - } - if typeID != 13 { - return nil, nil - } - step := &MediaProcessingStep{} - if value := rawAt(fields, 1); !isJSONNull(value) { - step.Name, err = rawString(value, path+"[1]", evidence) - if err != nil { - return nil, withMethod(err, "GenerateContent") - } - } - if value := rawAt(fields, 2); !isJSONNull(value) { - step.Data, err = decodeWireStruct(value, path+"[2]", evidence) - if err != nil { - return nil, err - } - } - if value := rawAt(fields, 3); !isJSONNull(value) { - step.ID, err = rawString(value, path+"[3]", evidence) - if err != nil { - return nil, withMethod(err, "GenerateContent") - } - } - return step, nil -} diff --git a/internal/aistudio/media_processing_test.go b/internal/aistudio/media_processing_test.go deleted file mode 100644 index a842dbd1..00000000 --- a/internal/aistudio/media_processing_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package aistudio - -import ( - "encoding/json" - "strings" - "testing" -) - -func TestMediaProcessingTraceRoundTrip(t *testing.T) { - for _, result := range []bool{false, true} { - part := Part{ThoughtSignature: "processing-signature"} - step := &MediaProcessingStep{ID: "media_call_1"} - index, kind := 25, EventMediaProcessingCall - if result { - part.ProcessingResult, index, kind = step, 26, EventMediaProcessingResult - } else { - part.ProcessingCall = step - } - wire, err := encodePart(part) - if err != nil { - t.Fatal(err) - } - if !jsonValueEqual(t, wire[index], []any{int64(13), nil, nil, "media_call_1"}) { - t.Fatalf("media processing wire = %#v", wire[index]) - } - raw, err := json.Marshal(wire) - if err != nil { - t.Fatal(err) - } - events, err := NewFrameDecoder().decodePart(raw, "$part", raw) - if err != nil { - t.Fatal(err) - } - if len(events) != 1 || events[0].Kind != kind || events[0].MediaProcessing == nil || events[0].MediaProcessing.ID != step.ID || events[0].ThoughtSignature != part.ThoughtSignature { - t.Fatalf("processing events = %#v", events) - } - if events[0].ToolCall != nil { - t.Fatal("server processing became a host function call") - } - build, err := encodeBuildPart(part) - if err != nil { - t.Fatal(err) - } - field := "toolCall" - if result { - field = "toolResponse" - } - value, ok := build[field].(map[string]any) - if !ok || value["toolType"] != "MEDIA_PROCESSING" || value["id"] != step.ID { - t.Fatalf("Build processing projection = %#v", build) - } - } -} - -func TestBuildMediaProcessingTrace(t *testing.T) { - const raw = `{"candidates":[{"content":{"role":"model","parts":[ - {"toolCall":{"toolType":"MEDIA_PROCESSING","id":"video_call","toolName":"inspect","args":{"start_offset":2}},"thoughtSignature":"call-signature"}, - {"toolResponse":{"toolType":"MEDIA_PROCESSING","id":"video_call","response":{"loaded":true}},"thoughtSignature":"result-signature"}, - {"text":"Video answer"} - ]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":2,"totalTokenCount":12}}` - decoder := &BuildStreamDecoder{} - events, err := decoder.Decode([]byte(raw)) - if err != nil { - t.Fatal(err) - } - if err := decoder.End(); err != nil { - t.Fatal(err) - } - if len(events) != 5 || events[0].Kind != EventMediaProcessingCall || events[1].Kind != EventMediaProcessingResult || events[2].Kind != EventText || events[3].Kind != EventUsage || events[4].Kind != EventFinish { - t.Fatalf("Build event order = %#v", events) - } - for index, signature := range []string{"call-signature", "result-signature"} { - if events[index].MediaProcessing == nil || events[index].MediaProcessing.ID != "video_call" || events[index].ThoughtSignature != signature || events[index].ToolCall != nil { - t.Fatalf("Build server processing identity/authority = %#v", events[index]) - } - } - if events[0].MediaProcessing.Name != "inspect" || !strings.Contains(string(events[0].MediaProcessing.Data), "start_offset") { - t.Fatalf("Build processing arguments = %#v", events[0].MediaProcessing) - } -} - -func TestBuildMediaProcessingMalformedPart(t *testing.T) { - for _, part := range []string{ - `{"toolCall":{"toolType":"MEDIA_PROCESSING","args":[]}}`, - `{"text":"text","toolCall":{"toolType":"MEDIA_PROCESSING"}}`, - `{"toolCall":{"toolType":"MEDIA_PROCESSING"},"toolResponse":{"toolType":"MEDIA_PROCESSING"}}`, - } { - var parsed buildResponsePart - if err := json.Unmarshal([]byte(part), &parsed); err != nil { - t.Fatal(err) - } - if _, err := decodeBuildPart(parsed, 0, []byte(part)); err == nil { - t.Fatalf("malformed Build processing part accepted: %s", part) - } - } -} - -func TestMediaProcessingPayloadAndMalformedWire(t *testing.T) { - step := &MediaProcessingStep{ID: "media_call_2", Name: "inspect", Data: json.RawMessage(`{"start":2,"end":4}`)} - call, err := encodePart(Part{ProcessingCall: step}) - if err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(call) - events, err := NewFrameDecoder().decodePart(raw, "$part", raw) - if err != nil || len(events) != 1 { - t.Fatalf("decode = %#v, %v", events, err) - } - got := events[0].MediaProcessing - var values map[string]int - if err := json.Unmarshal(got.Data, &values); err != nil { - t.Fatal(err) - } - if got.Name != "inspect" || len(values) != 2 || values["start"] != 2 || values["end"] != 4 { - t.Fatalf("processing payload = %#v", got) - } - call[25] = []any{int64(13), nil, nil, true} - raw, _ = json.Marshal(call) - if _, err := NewFrameDecoder().decodePart(raw, "$part", raw); err == nil { - t.Fatal("malformed processing ID accepted") - } - if _, err := encodePart(Part{Text: "text", ProcessingCall: step}); err == nil { - t.Fatal("multiple part variants accepted") - } -} diff --git a/internal/aistudio/request.go b/internal/aistudio/request.go index bbb1a081..af57a314 100644 --- a/internal/aistudio/request.go +++ b/internal/aistudio/request.go @@ -184,11 +184,6 @@ func encodeContent(content Content) ([]any, error) { } func encodePart(part Part) ([]any, error) { - if part.VideoMetadata != nil || part.MediaProcessing != "" { - if _, _, err := videoPartOptions(part); err != nil { - return nil, err - } - } variants := 0 if part.Text != "" { variants++ @@ -217,10 +212,7 @@ func encodePart(part Part) ([]any, error) { if part.CodeExecutionResult != nil { variants++ } - if part.ProcessingCall != nil { - variants++ - } - if part.ProcessingResult != nil { + if part.ProcessingCall != nil || part.ProcessingResult != nil { variants++ } if variants == 0 && part.ThoughtSignature != "" { diff --git a/internal/aistudio/types.go b/internal/aistudio/types.go index 6634a8f0..36e059d1 100644 --- a/internal/aistudio/types.go +++ b/internal/aistudio/types.go @@ -45,14 +45,14 @@ type UploadRef struct { Size int64 `json:"size"` } -// VideoMetadata configures static input-video clipping and frame sampling. +// VideoMetadata 是视频 Part 的截取起止与采样帧率，起止为 Duration 字符串 type VideoMetadata struct { StartOffset string `json:"start_offset,omitempty"` EndOffset string `json:"end_offset,omitempty"` FPS *float64 `json:"fps,omitempty"` } -// MediaProcessingStep represents a server-owned video navigation call or result. +// MediaProcessingStep 是上游视频理解发起的一次服务端处理调用或其结果 type MediaProcessingStep struct { ID string `json:"id,omitempty"` Name string `json:"name,omitempty"` @@ -151,9 +151,10 @@ type Part struct { ThoughtSignature string `json:"thought_signature,omitempty"` SpeechMetadata *SpeechMetadata `json:"speech_metadata,omitempty"` VideoMetadata *VideoMetadata `json:"video_metadata,omitempty"` - MediaProcessing string `json:"media_processing,omitempty"` - ProcessingCall *MediaProcessingStep `json:"processing_call,omitempty"` - ProcessingResult *MediaProcessingStep `json:"processing_result,omitempty"` + // MediaProcessing 为视频处理方式 STATIC 或 AGENTIC + MediaProcessing string `json:"media_processing,omitempty"` + ProcessingCall *MediaProcessingStep `json:"processing_call,omitempty"` + ProcessingResult *MediaProcessingStep `json:"processing_result,omitempty"` } // SpeechMetadata 表示语音文本分段的说话人与风格 @@ -467,9 +468,9 @@ const ( EventReasoning EventKind = "reasoning" // EventToolCall 表示函数调用 EventToolCall EventKind = "tool_call" - // EventMediaProcessingCall identifies server-owned video navigation. + // EventMediaProcessingCall 表示视频理解的服务端处理调用 EventMediaProcessingCall EventKind = "media_processing_call" - // EventMediaProcessingResult identifies the server's navigation result. + // EventMediaProcessingResult 表示视频理解的服务端处理结果 EventMediaProcessingResult EventKind = "media_processing_result" // EventExecutableCode 表示内置代码执行器生成代码 EventExecutableCode EventKind = "executable_code" diff --git a/internal/aistudio/video_input.go b/internal/aistudio/video_input.go index e77a52ef..5babeb6a 100644 --- a/internal/aistudio/video_input.go +++ b/internal/aistudio/video_input.go @@ -1,24 +1,31 @@ package aistudio import ( + "encoding/json" "fmt" - "math" "regexp" "strconv" "strings" ) +// videoOffsetPattern 匹配秒数后带 s 的 Duration 字符串，小数最多九位 var videoOffsetPattern = regexp.MustCompile(`^([0-9]+)(?:\.([0-9]{1,9}))?s$`) -// videoOffset parses the protobuf Duration representation used for video offsets. +// mediaProcessingCodes 是 Part field 40 视频处理方式的枚举值 +var mediaProcessingCodes = map[string]int64{"MEDIA_PROCESSING_UNSPECIFIED": 0, "STATIC": 1, "AGENTIC": 2} + +// mediaProcessingToolType 是 ToolCall 与 ToolResponse field 1 中视频理解处理的工具类型 +const mediaProcessingToolType = 13 + +// videoOffset 把 Duration 字符串解析为秒与纳秒 func videoOffset(value string) (int64, int64, error) { match := videoOffsetPattern.FindStringSubmatch(value) if match == nil { - return 0, 0, fmt.Errorf("video offset must be a nonnegative duration in seconds: %q", value) + return 0, 0, fmt.Errorf("video offset %q 必须是秒数加 s 的 Duration，例如 \"2.5s\"", value) } seconds, err := strconv.ParseInt(match[1], 10, 64) if err != nil { - return 0, 0, fmt.Errorf("video offset: %w", err) + return 0, 0, fmt.Errorf("video offset %q: %w", value, err) } var nanos int64 if match[2] != "" { @@ -27,117 +34,119 @@ func videoOffset(value string) (int64, int64, error) { return seconds, nanos, nil } -// videoPartOptions validates and encodes the metadata shared by video media parts. -func videoPartOptions(part Part) ([]any, int64, error) { - var metadata []any - if video := part.VideoMetadata; video != nil { - var startSeconds, startNanos int64 - if video.StartOffset != "" { - seconds, nanos, err := videoOffset(video.StartOffset) - if err != nil { - return nil, 0, err +// videoOffsetWire 把 Duration 字符串编码为 google.protobuf.Duration 的 [seconds, nanos] +func videoOffsetWire(value string) ([]any, error) { + seconds, nanos, err := videoOffset(value) + if err != nil { + return nil, err + } + if nanos == 0 { + return []any{seconds}, nil + } + return []any{seconds, nanos}, nil +} + +// validateVideoOffsets 校验视频截取同时设置起止时终点晚于起点 +func validateVideoOffsets(contents []Content) error { + for _, content := range contents { + for _, part := range content.Parts { + video := part.VideoMetadata + if video == nil || video.StartOffset == "" || video.EndOffset == "" { + continue } - startSeconds, startNanos = seconds, nanos - metadata = []any{videoDurationWire(seconds, nanos)} - } - if video.EndOffset != "" { - seconds, nanos, err := videoOffset(video.EndOffset) + startSeconds, startNanos, err := videoOffset(video.StartOffset) if err != nil { - return nil, 0, err + return err } - if seconds < startSeconds || seconds == startSeconds && nanos <= startNanos { - return nil, 0, fmt.Errorf("video end offset must be greater than its start offset") + endSeconds, endNanos, err := videoOffset(video.EndOffset) + if err != nil { + return err } - for len(metadata) < 2 { - metadata = append(metadata, nil) + if endSeconds < startSeconds || endSeconds == startSeconds && endNanos <= startNanos { + return fmt.Errorf("video endOffset %s 必须晚于 startOffset %s", video.EndOffset, video.StartOffset) } - metadata[1] = videoDurationWire(seconds, nanos) } - if video.FPS != nil { - fps := *video.FPS - if math.IsNaN(fps) || math.IsInf(fps, 0) || fps <= 0 || fps > 24 { - return nil, 0, fmt.Errorf("video fps must be greater than 0 and at most 24") + } + return nil +} + +// encodeVideoPart 为媒体 Part 写入 field 16 视频元数据与 field 40 处理方式，再写入思考签名 +func encodeVideoPart(wire []any, part Part) ([]any, error) { + if video := part.VideoMetadata; video != nil { + var metadata []any + for index, offset := range []string{video.StartOffset, video.EndOffset} { + if offset == "" { + continue } - for len(metadata) < 3 { - metadata = append(metadata, nil) + value, err := videoOffsetWire(offset) + if err != nil { + return nil, err } - metadata[2] = fps + metadata = setWireField(metadata, index, value) } - } - var processing int64 - switch part.MediaProcessing { - case "", "MEDIA_PROCESSING_UNSPECIFIED": - case "STATIC": - processing = 1 - case "AGENTIC": - if len(metadata) != 0 { - return nil, 0, fmt.Errorf("agentic video processing cannot use static clipping or fps") + if video.FPS != nil { + metadata = setWireField(metadata, 2, *video.FPS) } - processing = 2 - default: - return nil, 0, fmt.Errorf("unknown video mediaProcessing %q", part.MediaProcessing) - } - if len(metadata) != 0 || processing != 0 { - mime := "" - switch { - case part.InlineData != nil: - mime = part.InlineData.MIME - case part.ExternalMedia != nil: - mime = part.ExternalMedia.MIME - case part.File != nil: - mime = part.File.MIME - case part.Upload != nil: - mime = part.Upload.MIME - default: - return nil, 0, fmt.Errorf("video metadata requires a video media part") + if metadata != nil { + wire = setWireField(wire, 15, metadata) } - if !strings.HasPrefix(mime, "video/") && !(mime == "" && part.File != nil) { - return nil, 0, fmt.Errorf("video metadata requires a video MIME type") + } + if part.MediaProcessing != "" { + code, ok := mediaProcessingCodes[part.MediaProcessing] + if !ok { + return nil, fmt.Errorf("未知 mediaProcessing %q", part.MediaProcessing) + } + if code != 0 { + wire = setWireField(wire, 39, code) } } - return metadata, processing, nil + return setPartThoughtSignature(wire, part.ThoughtSignature), nil } -// videoDurationWire encodes seconds and optional nanoseconds without losing zero. -func videoDurationWire(seconds, nanos int64) []any { - if nanos == 0 { - return []any{seconds} +// encodeMediaProcessingStep 编码 ToolCall 或 ToolResponse：工具类型、名称、Struct 与调用 ID +func encodeMediaProcessingStep(step *MediaProcessingStep) ([]any, error) { + wire := []any{int64(mediaProcessingToolType), nil, nil, step.ID} + if step.Name != "" { + wire[1] = step.Name + } + if len(step.Data) != 0 { + value, err := encodeWireStructJSON(step.Data) + if err != nil { + return nil, fmt.Errorf("media processing data: %w", err) + } + wire[2] = value } - return []any{seconds, nanos} + return wire, nil } -// encodeVideoPart attaches input-video options and the existing thought signature. -func encodeVideoPart(wire []any, part Part) ([]any, error) { - metadata, processing, err := videoPartOptions(part) +// decodeMediaProcessingStep 解码视频理解处理类型的 ToolCall 或 ToolResponse，其他工具类型返回 nil +func decodeMediaProcessingStep(raw json.RawMessage, path string, evidence json.RawMessage) (*MediaProcessingStep, error) { + fields, err := rawArray(raw, path, evidence) if err != nil { - return nil, err + return nil, withMethod(err, "GenerateContent") } - if len(metadata) > 0 { - for len(wire) < 16 { - wire = append(wire, nil) + toolType, err := rawInt64(rawAt(fields, 0), path+"[0]", evidence) + if err != nil { + return nil, withMethod(err, "GenerateContent") + } + if toolType != mediaProcessingToolType { + return nil, nil + } + step := &MediaProcessingStep{} + if value := rawAt(fields, 1); !isJSONNull(value) { + if step.Name, err = rawString(value, path+"[1]", evidence); err != nil { + return nil, withMethod(err, "GenerateContent") } - wire[15] = metadata } - if processing != 0 { - for len(wire) < 40 { - wire = append(wire, nil) + if value := rawAt(fields, 2); !isJSONNull(value) { + if step.Data, err = decodeWireStruct(value, path+"[2]", evidence); err != nil { + return nil, err } - wire[39] = processing } - return setPartThoughtSignature(wire, part.ThoughtSignature), nil -} - -// validateVideoInputs checks requested agentic processing against the live model. -func validateVideoInputs(contents []Content, model Model) error { - for _, content := range contents { - for _, part := range content.Parts { - if _, _, err := videoPartOptions(part); err != nil { - return err - } - if part.MediaProcessing == "AGENTIC" && !model.Capabilities["agentic_video"] { - return fmt.Errorf("model %s does not support agentic video processing", model.ID) - } + if value := rawAt(fields, 3); !isJSONNull(value) { + if step.ID, err = rawString(value, path+"[3]", evidence); err != nil { + return nil, withMethod(err, "GenerateContent") } } - return nil + return step, nil } diff --git a/internal/aistudio/video_input_test.go b/internal/aistudio/video_input_test.go deleted file mode 100644 index 19856868..00000000 --- a/internal/aistudio/video_input_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package aistudio - -import ( - "encoding/json" - "math" - "strings" - "testing" -) - -func TestVideoInputProtocolEncoding(t *testing.T) { - fps := 0.5 - part := Part{ - InlineData: &Blob{MIME: "video/mp4", Data: []byte("video")}, - VideoMetadata: &VideoMetadata{StartOffset: "0s", EndOffset: "2.000000003s", FPS: &fps}, - MediaProcessing: "STATIC", - ThoughtSignature: "video-signature", - } - encodedPart, err := encodePart(part) - if err != nil { - t.Fatal(err) - } - if len(encodedPart) <= 39 || !jsonValueEqual(t, encodedPart[2], []any{"video/mp4", "dmlkZW8="}) { - t.Fatalf("video data wire = %#v", encodedPart) - } - if got, want := encodedPart[14], "video-signature"; got != want { - t.Fatalf("thought signature = %#v, want %q", got, want) - } - metadata, ok := encodedPart[15].([]any) - if !ok || len(metadata) != 3 { - t.Fatalf("video metadata = %#v", encodedPart[15]) - } - if got := metadata[0]; !jsonValueEqual(t, got, []any{int64(0)}) { - t.Fatalf("explicit zero start offset = %#v", got) - } - if got := metadata[1]; !jsonValueEqual(t, got, []any{int64(2), int64(3)}) { - t.Fatalf("end offset = %#v", got) - } - if metadata[2] != fps || encodedPart[39] != int64(1) { - t.Fatalf("fps/media processing = %#v/%#v", metadata[2], encodedPart[39]) - } - - request := GenerateRequest{Model: "video-model", Contents: []Content{{Role: RoleUser, Parts: []Part{{Text: "before"}, part, {Text: "after"}}}}} - defaults := GenerationDefaults{MaxOutputTokens: 1024} - body, err := EncodeGenerateContentRequest(request, defaults, RequestContext{}) - if err != nil { - t.Fatal(err) - } - var generate []any - if err := json.Unmarshal(body, &generate); err != nil { - t.Fatal(err) - } - content := generate[1].([]any)[0].([]any) - requestParts := content[0].([]any) - if len(requestParts) != 3 || !jsonValueEqual(t, requestParts[0], []any{nil, "before"}) || !jsonValueEqual(t, requestParts[1], encodedPart) || !jsonValueEqual(t, requestParts[2], []any{nil, "after"}) { - t.Fatalf("GenerateContent part order = %#v", requestParts) - } - - tokens, err := EncodeCountTokensRequest(TokenCountRequest{Model: "video-model", Contents: request.Contents}) - if err != nil { - t.Fatal(err) - } - var count []any - if err := json.Unmarshal(tokens, &count); err != nil { - t.Fatal(err) - } - countGenerate := count[2].([]any) - countParts := countGenerate[1].([]any)[0].([]any)[0].([]any) - if len(countParts) != 3 || !jsonValueEqual(t, countParts[1], encodedPart) { - t.Fatalf("CountTokens did not preserve video part: %#v", countGenerate) - } - - buildPart, err := encodeBuildPart(part) - if err != nil { - t.Fatal(err) - } - var buildMetadata map[string]any - if err := json.Unmarshal(mustJSON(t, buildPart), &buildMetadata); err != nil { - t.Fatal(err) - } - if buildMetadata["videoMetadata"] == nil || buildMetadata["mediaProcessing"] != "STATIC" || buildMetadata["thoughtSignature"] != "video-signature" { - t.Fatalf("Build video part = %#v", buildMetadata) - } - - filePart, err := encodePart(Part{File: &FileRef{ID: "files/video"}, MediaProcessing: "STATIC"}) - if err != nil { - t.Fatalf("video file reference without MIME was rejected: %v", err) - } - if len(filePart) <= 39 || filePart[39] != int64(1) { - t.Fatalf("video file processing field = %#v", filePart) - } -} - -func TestVideoInputValidation(t *testing.T) { - fps := 24.01 - tests := []struct { - name string - part Part - }{ - {name: "bad duration syntax", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{StartOffset: "1m"}}}, - {name: "too many fractional digits", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{StartOffset: "1.1234567890s"}}}, - {name: "end not later", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{StartOffset: "2s", EndOffset: "2s"}}}, - {name: "fps out of range", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{FPS: &fps}}}, - {name: "non-finite fps", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{FPS: floatPointer(math.Inf(1))}}}, - {name: "unknown processing", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, MediaProcessing: "FAST"}}, - {name: "agentic conflicts with metadata", part: Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, VideoMetadata: &VideoMetadata{StartOffset: "0s"}, MediaProcessing: "AGENTIC"}}, - {name: "non-video media", part: Part{InlineData: &Blob{MIME: "image/png", Data: []byte("x")}, MediaProcessing: "STATIC"}}, - {name: "text part", part: Part{Text: "video", MediaProcessing: "STATIC"}}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if _, err := encodePart(test.part); err == nil { - t.Fatal("expected invalid video options to fail") - } - }) - } - - video := Part{InlineData: &Blob{MIME: "video/mp4", Data: []byte("x")}, MediaProcessing: "AGENTIC"} - request := GenerateRequest{Model: "m", Contents: []Content{{Role: RoleUser, Parts: []Part{video}}}} - entry := modelEntry{model: Model{ID: "m", Capabilities: map[string]bool{}}, defaults: GenerationDefaults{MaxOutputTokens: 1024}} - if err := validateGenerateEntry(request, entry); err == nil || !strings.Contains(err.Error(), "agentic video") { - t.Fatalf("unsupported agentic video error = %v", err) - } - entry.model.Capabilities["agentic_video"] = true - if err := validateGenerateEntry(request, entry); err != nil { - t.Fatalf("supported agentic video rejected: %v", err) - } -} - -func TestVideoInputLegacyWireUnchanged(t *testing.T) { - request := GenerateRequest{Model: "m", Contents: []Content{{Role: RoleUser, Parts: []Part{{Text: "hello"}}}}} - got, err := EncodeGenerateContentRequest(request, GenerationDefaults{MaxOutputTokens: 1024}, RequestContext{}) - if err != nil { - t.Fatal(err) - } - const want = `["models/m",[[[[null,"hello"]],"user"]],[[null,null,7,5],[null,null,8,5],[null,null,9,5],[null,null,10,5]],[null,null,null,1024,null,null,null,null,null,null,null,null,null,1],null,null,null,null,null,null,1]` - if string(got) != want { - t.Fatalf("legacy GenerateContent wire = %s, want %s", got, want) - } - count, err := EncodeCountTokensRequest(TokenCountRequest{Model: "m", Contents: request.Contents}) - if err != nil { - t.Fatal(err) - } - if string(count) != `["models/m",[[[[null,"hello"]],"user"]]]` { - t.Fatalf("legacy CountTokens wire = %s", count) - } -} - -func TestInteractionRoutedVideoOptionsFailInsteadOfDisappearing(t *testing.T) { - for _, part := range []Part{ - {File: &FileRef{ID: "video"}, VideoMetadata: &VideoMetadata{StartOffset: "1s"}}, - {File: &FileRef{ID: "video"}, MediaProcessing: "STATIC"}, - {ProcessingCall: &MediaProcessingStep{ID: "call"}, ThoughtSignature: "call-signature"}, - {ProcessingResult: &MediaProcessingStep{ID: "call"}, ThoughtSignature: "result-signature"}, - } { - if _, _, _, err := encodeInteractionSteps([]Content{{Role: RoleAssistant, Parts: []Part{part}}}); err == nil { - t.Fatal("private Interaction route silently accepted unsupported video options/history") - } - } -} - -func jsonValueEqual(t *testing.T, got, want any) bool { - t.Helper() - gotJSON, err := json.Marshal(got) - if err != nil { - t.Fatal(err) - } - wantJSON, err := json.Marshal(want) - if err != nil { - t.Fatal(err) - } - return string(gotJSON) == string(wantJSON) -} - -func mustJSON(t *testing.T, value any) []byte { - t.Helper() - encoded, err := json.Marshal(value) - if err != nil { - t.Fatal(err) - } - return encoded -} - -func floatPointer(value float64) *float64 { return &value } diff --git a/internal/api/gemini.go b/internal/api/gemini.go index 52fba0e2..5712b079 100644 --- a/internal/api/gemini.go +++ b/internal/api/gemini.go @@ -112,61 +112,36 @@ type geminiPart struct { Output string `json:"output"` Error string `json:"error"` } `json:"codeExecutionResult"` - SpeechMetadata *aistudio.SpeechMetadata `json:"speechMetadata"` - SpeechMetadataSnake *aistudio.SpeechMetadata `json:"speech_metadata"` - VideoMetadata *geminiVideoMetadata `json:"videoMetadata"` - VideoMetadataSnake *geminiVideoMetadata `json:"video_metadata"` - MediaProcessing string `json:"mediaProcessing"` - MediaProcessingSnake string `json:"media_processing"` - ToolCall *geminiProcessingStep `json:"toolCall"` - ToolCallSnake *geminiProcessingStep `json:"tool_call"` - ToolResponse *geminiProcessingStep `json:"toolResponse"` - ToolResponseSnake *geminiProcessingStep `json:"tool_response"` -} - -type geminiProcessingStep struct { - ID string `json:"id"` - ToolType string `json:"toolType"` - ToolTypeSnake string `json:"tool_type"` - ToolName string `json:"toolName"` - ToolNameSnake string `json:"tool_name"` - Args json.RawMessage `json:"args"` - Response json.RawMessage `json:"response"` + SpeechMetadata *aistudio.SpeechMetadata `json:"speechMetadata"` + SpeechMetadataSnake *aistudio.SpeechMetadata `json:"speech_metadata"` + VideoMetadata *geminiVideoMetadata `json:"videoMetadata"` + MediaProcessing string `json:"mediaProcessing"` + ToolCall *geminiToolStep `json:"toolCall"` + ToolResponse *geminiToolStep `json:"toolResponse"` } +// geminiVideoMetadata 为 Part.videoMetadata 的截取起止与采样帧率 type geminiVideoMetadata struct { - StartOffset string `json:"startOffset"` - StartOffsetSnake string `json:"start_offset"` - EndOffset string `json:"endOffset"` - EndOffsetSnake string `json:"end_offset"` - FPS *float64 `json:"fps"` + StartOffset string `json:"startOffset"` + EndOffset string `json:"endOffset"` + FPS *float64 `json:"fps"` } -// videoMetadata maps the Gemini metadata field names to canonical video options. -func (part geminiPart) videoMetadata() *aistudio.VideoMetadata { - video := part.VideoMetadata - if video == nil { - video = part.VideoMetadataSnake - } - if video == nil { - return nil - } - start, end := video.StartOffset, video.EndOffset - if start == "" { - start = video.StartOffsetSnake - } - if end == "" { - end = video.EndOffsetSnake - } - return &aistudio.VideoMetadata{StartOffset: start, EndOffset: end, FPS: video.FPS} +// geminiToolStep 为 Part.toolCall 与 Part.toolResponse 的服务端工具步骤 +type geminiToolStep struct { + ID string `json:"id"` + ToolType string `json:"toolType"` + ToolName string `json:"toolName"` + Args json.RawMessage `json:"args"` + Response json.RawMessage `json:"response"` } -// mediaProcessing selects the Gemini media processing field spelling. -func (part geminiPart) mediaProcessing() string { - if part.MediaProcessing != "" { - return part.MediaProcessing +// mediaProcessing 把视频理解处理类型的服务端工具步骤转为规范步骤，payload 为 args 或 response +func (step geminiToolStep) mediaProcessing(payload json.RawMessage) (*aistudio.MediaProcessingStep, error) { + if step.ToolType != "MEDIA_PROCESSING" { + return nil, fmt.Errorf("unsupported server tool type %q", step.ToolType) } - return part.MediaProcessingSnake + return &aistudio.MediaProcessingStep{ID: step.ID, Name: step.ToolName, Data: payload}, nil } func (p geminiPart) speechMetadata() *aistudio.SpeechMetadata { @@ -823,23 +798,13 @@ func mapGeminiParts(input []geminiPart) ([]aistudio.Part, bool, error) { if part.CodeExecutionResult != nil { variants++ } - processingCall, processingResult := part.ToolCall, part.ToolResponse - if processingCall == nil { - processingCall = part.ToolCallSnake - } - if processingResult == nil { - processingResult = part.ToolResponseSnake - } - if processingCall != nil { + if part.ToolCall != nil { variants++ } - if processingResult != nil { + if part.ToolResponse != nil { variants++ } - video, processing := part.videoMetadata(), part.mediaProcessing() - if (video != nil || processing != "") && inline == nil && file == nil { - return nil, false, fmt.Errorf("parts[%d] video metadata requires inlineData or fileData", index) - } + video, processing := (*aistudio.VideoMetadata)(part.VideoMetadata), part.MediaProcessing if variants == 0 { if part.ThoughtSignature != "" { parts = append(parts, aistudio.Part{ThoughtSignature: part.ThoughtSignature}) @@ -919,22 +884,18 @@ func mapGeminiParts(input []geminiPart) ([]aistudio.Part, bool, error) { }) parts = append(parts, media...) hasResult = true - case processingCall != nil || processingResult != nil: - value := processingCall - if value == nil { - value = processingResult - } - step, err := value.canonical(processingCall != nil) + case part.ToolCall != nil: + step, err := part.ToolCall.mediaProcessing(part.ToolCall.Args) if err != nil { return nil, false, fmt.Errorf("parts[%d]: %w", index, err) } - mapped := aistudio.Part{ThoughtSignature: part.ThoughtSignature} - if processingCall != nil { - mapped.ProcessingCall = step - } else { - mapped.ProcessingResult = step + parts = append(parts, aistudio.Part{ProcessingCall: step, ThoughtSignature: part.ThoughtSignature}) + case part.ToolResponse != nil: + step, err := part.ToolResponse.mediaProcessing(part.ToolResponse.Response) + if err != nil { + return nil, false, fmt.Errorf("parts[%d]: %w", index, err) } - parts = append(parts, mapped) + parts = append(parts, aistudio.Part{ProcessingResult: step, ThoughtSignature: part.ThoughtSignature}) case part.ExecutableCode != nil: parts = append(parts, aistudio.Part{ ExecutableCode: &aistudio.ExecutableCode{ @@ -1234,9 +1195,7 @@ func geminiOutputParts(result generationResult) []map[string]any { appendPart(geminiFunctionCallPart(*event.ToolCall), event.ThoughtSignature) } case aistudio.EventMediaProcessingCall, aistudio.EventMediaProcessingResult: - if part := geminiProcessingPart(event); part != nil { - appendPart(part, event.ThoughtSignature) - } + appendPart(geminiProcessingPart(event), event.ThoughtSignature) case aistudio.EventExecutableCode: if event.ExecutableCode != nil { appendPart(map[string]any{"executableCode": map[string]any{ @@ -1638,11 +1597,7 @@ func geminiStreamCandidate(event aistudio.Event, index int) map[string]any { } part = geminiSignedPart(geminiFunctionCallPart(*event.ToolCall), event.ThoughtSignature) case aistudio.EventMediaProcessingCall, aistudio.EventMediaProcessingResult: - part = geminiProcessingPart(event) - if part == nil { - return nil - } - part = geminiSignedPart(part, event.ThoughtSignature) + part = geminiSignedPart(geminiProcessingPart(event), event.ThoughtSignature) case aistudio.EventExecutableCode: if event.ExecutableCode == nil { return nil diff --git a/internal/api/interaction_input.go b/internal/api/interaction_input.go index fd8f4b4c..bc153ad5 100644 --- a/internal/api/interaction_input.go +++ b/internal/api/interaction_input.go @@ -439,12 +439,8 @@ func (item interactionInput) codeExecutionText() string { func (content interactionInput) parts() ([]aistudio.Part, error) { var part geminiPart if rawJSONConfigured(content.Processing) { - if content.Type != "video" { - return nil, fmt.Errorf("processing requires video content") - } var err error - part.VideoMetadata, part.MediaProcessing, err = interactionVideoProcessing(content.Processing) - if err != nil { + if part.VideoMetadata, part.MediaProcessing, err = interactionVideoProcessing(content.Processing); err != nil { return nil, err } } diff --git a/internal/api/interaction_output.go b/internal/api/interaction_output.go index 1697eca8..6a97b8e8 100644 --- a/internal/api/interaction_output.go +++ b/internal/api/interaction_output.go @@ -3,6 +3,7 @@ package api import ( "encoding/base64" "fmt" + "maps" "mime" "net/http" "strconv" @@ -83,19 +84,13 @@ func interactionStep(event aistudio.Event, audio interactionAudioFormat) (map[st call := event.ToolCall return map[string]any{"type": "function_call", "id": call.ID, "name": call.Name, "arguments": call.Arguments}, nil, nil case aistudio.EventMediaProcessingCall, aistudio.EventMediaProcessingResult: - if event.MediaProcessing == nil { - return nil, nil, nil - } - stepType := "processing_call" - step := map[string]any{"type": stepType, "id": event.MediaProcessing.ID} + step := map[string]any{"type": "processing_call", "id": event.MediaProcessing.ID} if event.Kind == aistudio.EventMediaProcessingResult { - stepType = "processing_result" - step = map[string]any{"type": stepType, "call_id": event.MediaProcessing.ID} + step = map[string]any{"type": "processing_result", "call_id": event.MediaProcessing.ID} } - delta := map[string]any{"type": stepType} + delta := map[string]any{"type": step["type"]} if event.ThoughtSignature != "" { - step["signature"] = event.ThoughtSignature - delta["signature"] = event.ThoughtSignature + step["signature"], delta["signature"] = event.ThoughtSignature, event.ThoughtSignature } return step, delta, nil case aistudio.EventMedia: @@ -349,7 +344,8 @@ func (s *server) streamInteraction(w http.ResponseWriter, r *http.Request, reque } textLength += length } - if active != kind || kind == "function_call" || kind == "processing_call" || kind == "processing_result" { + standalone := kind == "function_call" || kind == "processing_call" || kind == "processing_result" + if active != kind || standalone { if err := closeStep(); err != nil { return err } @@ -362,16 +358,9 @@ func (s *server) streamInteraction(w http.ResponseWriter, r *http.Request, reque if kind == "thought" { start["summary"] = []any{} } - if kind == "function_call" { - start = step - } - if kind == "processing_call" || kind == "processing_result" { - start = map[string]any{"type": kind} - for _, field := range []string{"id", "call_id"} { - if value, ok := step[field]; ok { - start[field] = value - } - } + if standalone { + start = maps.Clone(step) + delete(start, "signature") } if err := send("step.start", map[string]any{"index": index, "step": start}); err != nil { return err diff --git a/internal/api/responses.go b/internal/api/responses.go index de7235cd..d03906c0 100644 --- a/internal/api/responses.go +++ b/internal/api/responses.go @@ -575,23 +575,24 @@ func responseHistoryOutput(result generationResult) aistudio.Content { call.Arguments = append(json.RawMessage(nil), call.Arguments...) output.Parts = append(output.Parts, aistudio.Part{FunctionCall: &call, ThoughtSignature: event.ThoughtSignature}) } - case aistudio.EventMediaProcessingCall, aistudio.EventMediaProcessingResult: - if event.MediaProcessing != nil { - step := *event.MediaProcessing - step.Data = append(json.RawMessage(nil), step.Data...) - part := aistudio.Part{ThoughtSignature: event.ThoughtSignature} - if event.Kind == aistudio.EventMediaProcessingCall { - part.ProcessingCall = &step - } else { - part.ProcessingResult = &step - } - output.Parts = append(output.Parts, part) - } case aistudio.EventMedia: if event.Media != nil && len(event.Media.Data) > 0 { blob := &aistudio.Blob{MIME: event.Media.MIME, Data: append([]byte(nil), event.Media.Data...)} output.Parts = append(output.Parts, aistudio.Part{InlineData: blob, ThoughtSignature: event.ThoughtSignature}) + } else if event.Media != nil && event.Media.URL != "" { + file := &aistudio.FileRef{ID: event.Media.URL, Name: event.Media.Name, MIME: event.Media.MIME} + output.Parts = append(output.Parts, aistudio.Part{File: file, ThoughtSignature: event.ThoughtSignature}) } + case aistudio.EventExecutableCode: + code := *event.ExecutableCode + output.Parts = append(output.Parts, aistudio.Part{ExecutableCode: &code, ThoughtSignature: event.ThoughtSignature}) + case aistudio.EventCodeExecutionResult: + result := *event.CodeExecutionResult + output.Parts = append(output.Parts, aistudio.Part{CodeExecutionResult: &result, ThoughtSignature: event.ThoughtSignature}) + case aistudio.EventMediaProcessingCall: + output.Parts = append(output.Parts, aistudio.Part{ProcessingCall: event.MediaProcessing, ThoughtSignature: event.ThoughtSignature}) + case aistudio.EventMediaProcessingResult: + output.Parts = append(output.Parts, aistudio.Part{ProcessingResult: event.MediaProcessing, ThoughtSignature: event.ThoughtSignature}) case aistudio.EventReasoning, aistudio.EventThoughtSignature: if event.ThoughtSignature != "" { output.Parts = append(output.Parts, aistudio.Part{ThoughtSignature: event.ThoughtSignature}) diff --git a/internal/api/video_input.go b/internal/api/video_input.go index e72d2410..003fc97e 100644 --- a/internal/api/video_input.go +++ b/internal/api/video_input.go @@ -1,45 +1,17 @@ package api import ( - "bytes" "encoding/json" "fmt" - "math/big" "strings" "github.com/Mag1cFall/AIStudio2API/internal/aistudio" ) -// canonical maps only the server-owned media processing tool, never a host function. -func (value geminiProcessingStep) canonical(call bool) (*aistudio.MediaProcessingStep, error) { - typeName := value.ToolType - if typeName == "" { - typeName = value.ToolTypeSnake - } - if typeName != "MEDIA_PROCESSING" { - return nil, fmt.Errorf("unsupported server tool type %q", typeName) - } - name := value.ToolName - if name == "" { - name = value.ToolNameSnake - } - data := value.Response - if call { - data = value.Args - } - return &aistudio.MediaProcessingStep{ID: value.ID, Name: name, Data: data}, nil -} - -// geminiProcessingPart preserves native video navigation calls and their results. +// geminiProcessingPart 把视频理解处理事件投影为 Gemini 的 toolCall 或 toolResponse Part func geminiProcessingPart(event aistudio.Event) map[string]any { step := event.MediaProcessing - if step == nil { - return nil - } - value := map[string]any{"toolType": "MEDIA_PROCESSING"} - if step.ID != "" { - value["id"] = step.ID - } + value := map[string]any{"toolType": "MEDIA_PROCESSING", "id": step.ID} key, payload := "toolResponse", "response" if event.Kind == aistudio.EventMediaProcessingCall { key, payload = "toolCall", "args" @@ -53,73 +25,20 @@ func geminiProcessingPart(event aistudio.Event) map[string]any { return map[string]any{key: value} } -// interactionVideoProcessing maps the documented string or static object union. +// interactionVideoProcessing 把 video 内容的 processing 映射为截取采样设置与处理方式，取值为 static、agentic 或 static 设置对象 func interactionVideoProcessing(raw json.RawMessage) (*geminiVideoMetadata, string, error) { var mode string if json.Unmarshal(raw, &mode) == nil { - switch mode { - case "static": - return nil, "STATIC", nil - case "agentic": - return nil, "AGENTIC", nil - default: - return nil, "", fmt.Errorf("unknown video processing mode %q", mode) - } + return nil, strings.ToUpper(mode), nil } var config struct { - Type string `json:"type"` - Start json.RawMessage `json:"start_offset"` - End json.RawMessage `json:"end_offset"` - FPS *float64 `json:"fps"` + Type string `json:"type"` + StartOffset string `json:"start_offset"` + EndOffset string `json:"end_offset"` + FPS *float64 `json:"fps"` } - decoder := json.NewDecoder(bytes.NewReader(raw)) - decoder.DisallowUnknownFields() - if err := decoder.Decode(&config); err != nil { + if err := json.Unmarshal(raw, &config); err != nil { return nil, "", fmt.Errorf("video processing: %w", err) } - if config.Type != "static" { - return nil, "", fmt.Errorf("video processing object type must be static") - } - start, err := interactionVideoOffset(config.Start) - if err != nil { - return nil, "", fmt.Errorf("video start_offset: %w", err) - } - end, err := interactionVideoOffset(config.End) - if err != nil { - return nil, "", fmt.Errorf("video end_offset: %w", err) - } - return &geminiVideoMetadata{StartOffset: start, EndOffset: end, FPS: config.FPS}, "STATIC", nil -} - -// interactionVideoOffset accepts documented duration strings and numeric seconds. -func interactionVideoOffset(raw json.RawMessage) (string, error) { - if !rawJSONConfigured(raw) { - return "", nil - } - var value string - if json.Unmarshal(raw, &value) == nil { - return value, nil - } - var seconds json.Number - if err := json.Unmarshal(raw, &seconds); err != nil { - return "", fmt.Errorf("expected a duration string or numeric seconds") - } - offset, ok := new(big.Rat).SetString(seconds.String()) - if !ok || offset.Sign() < 0 { - return "", fmt.Errorf("video offset must be nonnegative numeric seconds") - } - offset.Mul(offset, big.NewRat(1_000_000_000, 1)) - if !offset.IsInt() { - return "", fmt.Errorf("video offset exceeds nanosecond precision") - } - whole, nanos := new(big.Int), new(big.Int) - whole.QuoRem(offset.Num(), big.NewInt(1_000_000_000), nanos) - if !whole.IsInt64() { - return "", fmt.Errorf("video offset seconds exceed int64 range") - } - if nanos.Sign() == 0 { - return whole.String() + "s", nil - } - fraction := strings.TrimRight(fmt.Sprintf("%09d", nanos.Int64()), "0") - return whole.String() + "." + fraction + "s", nil + return &geminiVideoMetadata{StartOffset: config.StartOffset, EndOffset: config.EndOffset, FPS: config.FPS}, strings.ToUpper(config.Type), nil } diff --git a/internal/api/video_input_test.go b/internal/api/video_input_test.go index 0f5d8acb..2f525168 100644 --- a/internal/api/video_input_test.go +++ b/internal/api/video_input_test.go @@ -1,175 +1,50 @@ package api import ( - "bytes" "encoding/json" "testing" "github.com/Mag1cFall/AIStudio2API/internal/aistudio" ) -func TestGeminiVideoInputFieldMappings(t *testing.T) { - for _, test := range []struct { - name string - json string - }{ - { - name: "camel case", - json: `{"inlineData":{"mimeType":"video/mp4","data":"dmlkZW8="},"videoMetadata":{"startOffset":"0s","endOffset":"3.25s","fps":0.5},"mediaProcessing":"STATIC"}`, - }, - { - name: "snake case", - json: `{"inline_data":{"mime_type":"video/mp4","data":"dmlkZW8="},"video_metadata":{"start_offset":"0s","end_offset":"3.25s","fps":0.5},"media_processing":"STATIC"}`, - }, - } { - t.Run(test.name, func(t *testing.T) { - var part geminiPart - if err := json.Unmarshal([]byte(test.json), &part); err != nil { - t.Fatal(err) - } - mapped, _, err := mapGeminiParts([]geminiPart{part}) - if err != nil { - t.Fatal(err) - } - if len(mapped) != 1 || mapped[0].VideoMetadata == nil || mapped[0].VideoMetadata.StartOffset != "0s" || mapped[0].VideoMetadata.EndOffset != "3.25s" || mapped[0].VideoMetadata.FPS == nil || *mapped[0].VideoMetadata.FPS != 0.5 { - t.Fatalf("mapped video metadata = %#v", mapped) - } - if mapped[0].MediaProcessing != "STATIC" { - t.Fatalf("media processing = %q, want STATIC", mapped[0].MediaProcessing) - } - }) +// TestGeminiVideoProcessingRoundTrip 保留视频截取、采样与处理方式，服务端处理步骤按 Gemini 形状输出后可作为下一轮历史读回 +func TestGeminiVideoProcessingRoundTrip(t *testing.T) { + var request geminiRequest + if err := json.Unmarshal([]byte(`{"contents":[{"role":"user","parts":[ + {"fileData":{"mimeType":"video/*","fileUri":"https://www.youtube.com/watch?v=rOVtjJkqtiA"},"videoMetadata":{"startOffset":"2s","endOffset":"4.5s","fps":0.5},"mediaProcessing":"STATIC"}, + {"text":"describe"}]}]}`), &request); err != nil { + t.Fatal(err) } -} - -func TestInteractionVideoInputProcessing(t *testing.T) { - tests := []struct { - name string - processing string - wantMode string - wantStart string - wantEnd string - wantFPS float64 - }{ - {name: "static string", processing: `"static"`, wantMode: "STATIC"}, - {name: "agentic string", processing: `"agentic"`, wantMode: "AGENTIC"}, - {name: "static options", processing: `{"type":"static","start_offset":2,"end_offset":4,"fps":0.5}`, wantMode: "STATIC", wantStart: "2s", wantEnd: "4s", wantFPS: 0.5}, + generate, err := request.toGenerateRequest("id", "m") + if err != nil { + t.Fatal(err) } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - raw := `{"type":"video","data":"dmlkZW8=","mime_type":"video/mp4","processing":` + test.processing + `}` - var input interactionInput - if err := json.Unmarshal([]byte(raw), &input); err != nil { - t.Fatal(err) - } - parts, err := input.parts() - if err != nil { - t.Fatal(err) - } - if len(parts) != 1 || parts[0].MediaProcessing != test.wantMode { - t.Fatalf("interaction video parts = %#v", parts) - } - if test.wantStart == "" { - if parts[0].VideoMetadata != nil { - t.Fatalf("unexpected video metadata: %#v", parts[0].VideoMetadata) - } - return - } - metadata := parts[0].VideoMetadata - if metadata == nil || metadata.StartOffset != test.wantStart || metadata.EndOffset != test.wantEnd || metadata.FPS == nil || *metadata.FPS != test.wantFPS { - t.Fatalf("interaction video metadata = %#v", metadata) - } - }) + video := generate.Contents[0].Parts[0] + if video.ExternalMedia == nil || video.MediaProcessing != "STATIC" || video.VideoMetadata == nil || + video.VideoMetadata.StartOffset != "2s" || video.VideoMetadata.EndOffset != "4.5s" || video.VideoMetadata.FPS == nil || *video.VideoMetadata.FPS != 0.5 { + t.Fatalf("video part = %#v", video) } -} -func TestInteractionVideoInputRejectsInvalidProcessing(t *testing.T) { - for _, test := range []struct { - processing string - mapperFails bool - }{ - {processing: `"unknown"`, mapperFails: true}, - {processing: `{"type":"static","start_offset":{}}`, mapperFails: true}, - {processing: `{"type":"static","startOffset":2}`, mapperFails: true}, - {processing: `{"type":"static","start_offset":"1m"}`}, - {processing: `{"type":"static","start_offset":2,"end_offset":2}`}, - {processing: `{"type":"static","fps":0}`}, - {processing: `{"type":"static","fps":25}`}, - {processing: `{"type":"agentic","fps":1}`, mapperFails: true}, - } { - raw := `{"type":"video","data":"dmlkZW8=","mime_type":"video/mp4","processing":` + test.processing + `}` - var input interactionInput - if err := json.Unmarshal([]byte(raw), &input); err != nil { - t.Fatal(err) - } - parts, err := input.parts() - if test.mapperFails { - if err == nil { - t.Errorf("processing %s unexpectedly accepted by mapper", test.processing) - } - continue - } - if err != nil { - t.Fatalf("valid processing shape %s did not map: %v", test.processing, err) - } - request := aistudio.GenerateRequest{Model: "m", Contents: []aistudio.Content{{Role: aistudio.RoleUser, Parts: parts}}} - if _, err := aistudio.EncodeGenerateContentRequest(request, aistudio.GenerationDefaults{MaxOutputTokens: 1024}, aistudio.RequestContext{}); err == nil { - t.Errorf("processing %s unexpectedly passed canonical wire validation", test.processing) - } + events := []aistudio.Event{ + {Kind: aistudio.EventMediaProcessingCall, MediaProcessing: &aistudio.MediaProcessingStep{ID: "call_1"}, ThoughtSignature: "call-signature"}, + {Kind: aistudio.EventMediaProcessingResult, MediaProcessing: &aistudio.MediaProcessingStep{ID: "call_1"}, ThoughtSignature: "result-signature"}, + {Kind: aistudio.EventText, Text: "answer"}, } -} - -func TestInteractionNumericVideoOffsets(t *testing.T) { - for _, test := range []struct { - input string - want string - wire string - }{ - {input: "1e-3", want: "0.001s", wire: `[0,1000000]`}, - {input: "2e0", want: "2s", wire: `[2]`}, - {input: "1e-9", want: "0.000000001s", wire: `[0,1]`}, - {input: "100e-10", want: "0.00000001s", wire: `[0,10]`}, - {input: "1.25E+1", want: "12.5s", wire: `[12,500000000]`}, - {input: "9007199254740993e0", want: "9007199254740993s", wire: `[9007199254740993]`}, - {input: "9223372036854775807e0", want: "9223372036854775807s", wire: `[9223372036854775807]`}, - {input: "0e999999999", want: "0s", wire: `[0]`}, - } { - t.Run(test.input, func(t *testing.T) { - var input interactionInput - raw := `{"type":"video","data":"dmlkZW8=","mime_type":"video/mp4","processing":{"type":"static","start_offset":` + test.input + `}}` - if err := json.Unmarshal([]byte(raw), &input); err != nil { - t.Fatal(err) - } - parts, err := input.parts() - if err != nil { - t.Fatal(err) - } - if got := parts[0].VideoMetadata.StartOffset; got != test.want { - t.Fatalf("offset = %q, want %q", got, test.want) - } - request := aistudio.GenerateRequest{Model: "m", Contents: []aistudio.Content{{Role: aistudio.RoleUser, Parts: parts}}} - encoded, err := aistudio.EncodeGenerateContentRequest(request, aistudio.GenerationDefaults{MaxOutputTokens: 64}, aistudio.RequestContext{}) - if err != nil { - t.Fatal(err) - } - var wire []any - decoder := json.NewDecoder(bytes.NewReader(encoded)) - decoder.UseNumber() - if err := decoder.Decode(&wire); err != nil { - t.Fatal(err) - } - part := wire[1].([]any)[0].([]any)[0].([]any)[0].([]any) - duration, err := json.Marshal(part[15].([]any)[0]) - if err != nil || string(duration) != test.wire { - t.Fatalf("offset wire = %s, want %s: %v", duration, test.wire, err) - } - }) + output, err := json.Marshal(map[string]any{"contents": []any{map[string]any{"role": "model", "parts": geminiOutputParts(generationResult{events: events})}}}) + if err != nil { + t.Fatal(err) } -} - -func TestInteractionNumericVideoOffsetLimits(t *testing.T) { - for _, value := range []string{"-1e-3", "1e-10", "1.0000000001", "9223372036854775808", "1e1000000000", "1e-1000000000"} { - if _, err := interactionVideoOffset(json.RawMessage(value)); err == nil { - t.Errorf("unrepresentable numeric offset %s accepted", value) - } + var history geminiRequest + if err := json.Unmarshal(output, &history); err != nil { + t.Fatal(err) + } + replayed, err := history.toGenerateRequest("id", "m") + if err != nil { + t.Fatal(err) + } + parts := replayed.Contents[0].Parts + if len(parts) != 3 || parts[0].ProcessingCall == nil || parts[0].ProcessingCall.ID != "call_1" || parts[0].ThoughtSignature != "call-signature" || + parts[1].ProcessingResult == nil || parts[1].ProcessingResult.ID != "call_1" || parts[1].ThoughtSignature != "result-signature" || parts[2].Text != "answer" { + t.Fatalf("history parts = %s", output) } } diff --git a/internal/api/video_processing_history_test.go b/internal/api/video_processing_history_test.go deleted file mode 100644 index 535b4d61..00000000 --- a/internal/api/video_processing_history_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package api - -import ( - "encoding/json" - "net/http/httptest" - "strings" - "testing" - - "github.com/Mag1cFall/AIStudio2API/internal/aistudio" -) - -func TestGeminiMediaProcessingOutputAndHistory(t *testing.T) { - events := []aistudio.Event{ - {Kind: aistudio.EventMediaProcessingCall, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call"}, ThoughtSignature: "call-signature"}, - {Kind: aistudio.EventMediaProcessingResult, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call"}, ThoughtSignature: "result-signature"}, - {Kind: aistudio.EventText, Text: "Video answer"}, - } - parts := geminiOutputParts(generationResult{events: events}) - if len(parts) != 3 { - t.Fatalf("output parts = %#v", parts) - } - for i, key := range []string{"toolCall", "toolResponse"} { - tool, ok := parts[i][key].(map[string]any) - if !ok || tool["toolType"] != "MEDIA_PROCESSING" || tool["id"] != "video_call" { - t.Fatalf("native processing part = %#v", parts[i]) - } - if parts[i]["functionCall"] != nil || parts[i]["thoughtSignature"] != events[i].ThoughtSignature { - t.Fatalf("processing authority/signature = %#v", parts[i]) - } - stream := geminiStreamCandidate(events[i], 0) - streamParts := stream["content"].(map[string]any)["parts"].([]any) - if len(streamParts) != 1 || streamParts[0].(map[string]any)[key] == nil { - t.Fatalf("stream processing projection = %#v", stream) - } - } - raw, err := json.Marshal(parts) - if err != nil { - t.Fatal(err) - } - var input []geminiPart - if err := json.Unmarshal(raw, &input); err != nil { - t.Fatal(err) - } - canonical, hasHostResult, err := mapGeminiParts(input) - if err != nil || hasHostResult || len(canonical) != 3 { - t.Fatalf("history mapping = %#v, %v, %v", canonical, hasHostResult, err) - } - if canonical[0].ProcessingCall == nil || canonical[1].ProcessingResult == nil || canonical[0].ThoughtSignature != "call-signature" || canonical[1].ThoughtSignature != "result-signature" { - t.Fatalf("history lost processing trace: %#v", canonical) - } - if _, err := aistudio.EncodeCountTokensRequest(aistudio.TokenCountRequest{Model: "m", Contents: []aistudio.Content{{Role: aistudio.RoleAssistant, Parts: canonical}}}); err != nil { - t.Fatalf("processing history count encoding: %v", err) - } -} - -func TestInteractionProcessingHistoryPreservesSignatures(t *testing.T) { - contents, err := interactionContents(json.RawMessage(`[ - {"type":"thought","signature":"prior-signature"}, - {"type":"processing_call","id":"video_call","signature":"call-signature"}, - {"type":"processing_result","call_id":"video_call","signature":"result-signature"} - ]`)) - if err != nil { - t.Fatal(err) - } - if len(contents) != 1 || contents[0].Role != aistudio.RoleAssistant || len(contents[0].Parts) != 3 { - t.Fatalf("processing history = %#v", contents) - } - parts := contents[0].Parts - if parts[0].ThoughtSignature != "prior-signature" || parts[1].ThoughtSignature != "call-signature" || parts[2].ThoughtSignature != "result-signature" || parts[1].ProcessingCall == nil || parts[2].ProcessingResult == nil { - t.Fatalf("processing signatures were overwritten: %#v", parts) - } - for _, raw := range []string{`[{"type":"processing_call"}]`, `[{"type":"processing_result"}]`} { - if _, err := interactionContents(json.RawMessage(raw)); err == nil { - t.Fatalf("missing processing ID accepted: %s", raw) - } - } -} - -func TestStoredInteractionMediaProcessingContinuation(t *testing.T) { - for _, stream := range []bool{false, true} { - name := "nonstream" - if stream { - name = "stream" - } - t.Run(name, func(t *testing.T) { - call := &aistudio.MediaProcessingStep{ID: "video_call", Name: "inspect", Data: json.RawMessage(`{"start":2}`)} - reply := &aistudio.MediaProcessingStep{ID: "video_call", Data: json.RawMessage(`{"loaded":true}`)} - events := []aistudio.Event{ - {Kind: aistudio.EventMediaProcessingCall, MediaProcessing: call, ThoughtSignature: "call-signature"}, - {Kind: aistudio.EventMediaProcessingResult, MediaProcessing: reply, ThoughtSignature: "result-signature"}, - {Kind: aistudio.EventText, Text: "Video answer"}, - {Kind: aistudio.EventFinish, FinishReason: "stop"}, - } - store := newResponseStateStore() - s := &server{responseStates: store} - request := interactionRequest{Model: "video-model", Input: json.RawMessage(`"Inspect the video"`)} - current := []aistudio.Content{{Role: aistudio.RoleUser, Parts: []aistudio.Part{{Text: "Inspect the video"}}}} - if stream { - channel := make(chan aistudio.Event, len(events)) - for _, event := range events { - channel <- event - } - close(channel) - response := httptest.NewRecorder() - s.streamInteraction(response, httptest.NewRequest("POST", "/v1beta/interactions", nil), request, - aistudio.GenerateRequest{ID: "stored-video", Model: request.Model}, false, responseHistory{}, current, "2026-10-28T00:00:00Z", channel) - if !strings.Contains(response.Body.String(), "interaction.completed") { - t.Fatalf("stream did not complete: %s", response.Body.String()) - } - } else { - s.storeInteraction(request, responseHistory{}, current, generationResult{events: events}, map[string]any{"id": "stored-video"}) - } - call.ID, reply.ID = "changed-call", "changed-result" - call.Data[0], reply.Data[0] = '[', '[' - history, ok := store.Load("stored-video") - if !ok || !history.Interaction || len(history.Contents) != 2 { - t.Fatalf("stored interaction history = %#v", history) - } - parts := history.Contents[1].Parts - if len(parts) != 3 || parts[0].ProcessingCall == nil || parts[1].ProcessingResult == nil { - t.Fatalf("stored navigation parts = %#v", parts) - } - if parts[0].ProcessingCall.ID != "video_call" || parts[1].ProcessingResult.ID != "video_call" || - parts[0].ProcessingCall.Name != "inspect" || string(parts[0].ProcessingCall.Data) != `{"start":2}` || - string(parts[1].ProcessingResult.Data) != `{"loaded":true}` || parts[0].ThoughtSignature != "call-signature" || parts[1].ThoughtSignature != "result-signature" { - t.Fatalf("stored processing identity or payload changed: %#v", parts) - } - contents := append(history.Contents, aistudio.Content{Role: aistudio.RoleUser, Parts: []aistudio.Part{{Text: "Continue"}}}) - encoded, err := aistudio.EncodeGenerateContentRequest(aistudio.GenerateRequest{Model: request.Model, Contents: contents}, aistudio.GenerationDefaults{MaxOutputTokens: 64}, aistudio.RequestContext{}) - if err != nil { - t.Fatal(err) - } - var wire []any - if err := json.Unmarshal(encoded, &wire); err != nil { - t.Fatal(err) - } - modelParts := wire[1].([]any)[1].([]any)[0].([]any) - for index, field := range []int{25, 26} { - part := modelParts[index].([]any) - step := part[field].([]any) - if step[0] != float64(13) || step[3] != "video_call" || part[14] != parts[index].ThoughtSignature { - t.Fatalf("continuation navigation wire = %#v", part) - } - } - }) - } -} diff --git a/internal/api/video_processing_output_test.go b/internal/api/video_processing_output_test.go deleted file mode 100644 index 9331c7c2..00000000 --- a/internal/api/video_processing_output_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package api - -import ( - "encoding/json" - "net/http/httptest" - "strings" - "testing" - - "github.com/Mag1cFall/AIStudio2API/internal/aistudio" -) - -func TestInteractionStepsProjectMediaProcessing(t *testing.T) { - result := generationResult{events: []aistudio.Event{ - {Kind: aistudio.EventMediaProcessingCall, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call_1"}, ThoughtSignature: "call-signature"}, - {Kind: aistudio.EventMediaProcessingResult, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call_1"}, ThoughtSignature: "result-signature"}, - }} - - steps, err := interactionSteps(result, interactionAudioFormat{}, true) - if err != nil { - t.Fatal(err) - } - if len(steps) != 2 { - t.Fatalf("steps = %#v, want one processing call and result", steps) - } - if got := steps[0]; got["type"] != "processing_call" || got["id"] != "video_call_1" || got["signature"] != "call-signature" { - t.Errorf("processing call = %#v", got) - } - if got := steps[1]; got["type"] != "processing_result" || got["call_id"] != "video_call_1" || got["signature"] != "result-signature" { - t.Errorf("processing result = %#v", got) - } - for _, step := range steps { - if step["type"] == "function_call" || step["type"] == "model_output" { - t.Errorf("server processing step was projected as host/model output: %#v", step) - } - } -} - -func TestStreamInteractionMediaProcessingProtocol(t *testing.T) { - store := false - request := interactionRequest{Store: &store} - generate := aistudio.GenerateRequest{ID: "interaction_1", Model: "model"} - events := make(chan aistudio.Event, 3) - events <- aistudio.Event{Kind: aistudio.EventMediaProcessingCall, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call_1"}, ThoughtSignature: "call-signature"} - events <- aistudio.Event{Kind: aistudio.EventMediaProcessingResult, MediaProcessing: &aistudio.MediaProcessingStep{ID: "video_call_1"}, ThoughtSignature: "result-signature"} - events <- aistudio.Event{Kind: aistudio.EventFinish, FinishReason: "STOP"} - close(events) - - response := httptest.NewRecorder() - s := &server{} - s.streamInteraction(response, httptest.NewRequest("POST", "/v1/interactions", nil), request, generate, false, responseHistory{}, nil, "2026-10-26T00:00:00Z", events) - - type frame struct { - event string - data map[string]any - } - var frames []frame - for _, block := range strings.Split(strings.TrimSpace(response.Body.String()), "\n\n") { - if block == "" { - continue - } - var current frame - for _, line := range strings.Split(block, "\n") { - if strings.HasPrefix(line, "event: ") { - current.event = strings.TrimPrefix(line, "event: ") - } - if strings.HasPrefix(line, "data: ") { - if err := json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), ¤t.data); err != nil { - t.Fatalf("decode stream frame: %v", err) - } - } - } - frames = append(frames, current) - } - wantEvents := []string{"interaction.created", "step.start", "step.delta", "step.stop", "step.start", "step.delta", "step.stop", "interaction.completed"} - if len(frames) != len(wantEvents) { - t.Fatalf("stream frames = %#v", frames) - } - for i, want := range wantEvents { - if frames[i].event != want { - t.Errorf("frame %d event = %q, want %q", i, frames[i].event, want) - } - } - assertFrameValue := func(index int, field string, want any) { - t.Helper() - if got := frames[index].data[field]; got != want { - t.Errorf("frame %d %s = %#v, want %#v", index, field, got, want) - } - } - for _, test := range []struct { - startIndex int - deltaIndex int - kind string - idField string - signature string - }{ - {startIndex: 1, deltaIndex: 2, kind: "processing_call", idField: "id", signature: "call-signature"}, - {startIndex: 4, deltaIndex: 5, kind: "processing_result", idField: "call_id", signature: "result-signature"}, - } { - start, _ := frames[test.startIndex].data["step"].(map[string]any) - delta, _ := frames[test.deltaIndex].data["delta"].(map[string]any) - if start["type"] != test.kind || start[test.idField] != "video_call_1" { - t.Errorf("%s start step = %#v", test.kind, start) - } - if _, ok := start["signature"]; ok { - t.Errorf("%s signature appeared before its delta: %#v", test.kind, start) - } - if delta["type"] != test.kind || delta["signature"] != test.signature { - t.Errorf("%s delta = %#v", test.kind, delta) - } - assertFrameValue(test.startIndex, "index", float64(test.startIndex/3)) - assertFrameValue(test.deltaIndex, "index", float64(test.startIndex/3)) - assertFrameValue(test.startIndex+2, "index", float64(test.startIndex/3)) - } - if strings.Contains(response.Body.String(), `"type":"function_call"`) { - t.Fatal("server media processing emitted a host function call") - } -}