feat(#3996): strip unsupported audio/video media parts

agent-schema.json

@@ -1738,7 +1738,7 @@
         },
         "capabilities": {
           "$ref": "#/definitions/CapabilitiesConfig",
-          "description": "Explicit attachment capability override for models the models.dev catalogue does not describe correctly (custom OpenAI-compatible providers, local models like Ollama, or dropped model versions). When set, the declared flags are authoritative and no models.dev lookup is performed; when omitted, capabilities are detected automatically. Without it, such models fall back to text-only and their image/PDF attachments are silently dropped."
+          "description": "Explicit attachment capability override for models the models.dev catalogue does not describe correctly (custom OpenAI-compatible providers, local models like Ollama, or dropped model versions). When set, the declared flags are authoritative and no models.dev lookup is performed; when omitted, capabilities are detected automatically. Without it, such models fall back to text-only and their image/PDF/audio/video attachments are silently dropped."
         },
         "cost": {
           "$ref": "#/definitions/CostConfig",
@@ -1758,6 +1758,14 @@
         "pdf": {
           "type": "boolean",
           "description": "Whether the model accepts PDF (application/pdf) attachments"
+        },
+        "audio": {
+          "type": "boolean",
+          "description": "Whether the model accepts audio attachments"
+        },
+        "video": {
+          "type": "boolean",
+          "description": "Whether the model accepts video attachments"
         }
       },
       "additionalProperties": false

docs/configuration/models/index.md

@@ -38,6 +38,8 @@ models:
     capabilities: # Optional: override attachment capabilities
       image: boolean # Optional: whether the model accepts image attachments
       pdf: boolean # Optional: whether the model accepts PDF attachments
+      audio: boolean # Optional: whether the model accepts audio attachments
+      video: boolean # Optional: whether the model accepts video attachments
     cost: # Optional: explicit token pricing (USD per 1M tokens)
       input: float # Optional: price per 1M input tokens
       output: float # Optional: price per 1M output tokens
@@ -83,9 +85,9 @@ models:
 
 For custom OpenAI-compatible providers, local models (Ollama, DMR), and any
 model the built-in catalogue does not describe, Docker Agent cannot
-auto-detect whether the endpoint accepts image or PDF attachments. When the
-model is absent from the catalogue, Docker Agent logs a diagnostic and falls
-back to text-only, silently dropping attachments.
+auto-detect whether the endpoint accepts image, PDF, audio, or video
+attachments. When the model is absent from the catalogue, Docker Agent logs a
+diagnostic and falls back to text-only, silently dropping attachments.
 
 Declare `capabilities` to make the model's attachment support authoritative
 and skip the catalogue lookup entirely:
@@ -105,19 +107,46 @@ models:
     capabilities:
       image: true
       pdf: true
+
+  proxy-multimodal:
+    provider: vision-proxy
+    model: gemini-2.5-pro
+    capabilities:
+      image: true
+      pdf: true
+      audio: true
+      video: true
 ```
 
 | Field                  | Type    | Description                                       |
-| ---------------------- | ------- | ------------------------------------------------- |
+| ---------------------- | ------- | -------------------------------------------------- |
 | `capabilities.image`   | boolean | Whether the model accepts image attachments       |
 | `capabilities.pdf`     | boolean | Whether the model accepts PDF attachments         |
+| `capabilities.audio`   | boolean | Whether the model accepts audio attachments       |
+| `capabilities.video`   | boolean | Whether the model accepts video attachments       |
 
 The flags must match what the endpoint actually accepts. Claiming a modality
 that the endpoint does not support leads to a provider-side API error. When
 `capabilities` is omitted the behaviour is unchanged (catalogue lookup then
 conservative text-only fallback).
 
-See [`examples/capability-overrides.yaml`](https://github.com/docker/docker-agent/blob/main/examples/capability-overrides.yaml) for a complete example.
+### Unsupported media is stripped before the call
+
+Before each model call, Docker Agent removes image, audio, and video message
+parts that the resolved capabilities of the active model do not cover, instead
+of letting the provider fail the whole request. Adjacent text (and PDF) parts
+are preserved in their original order, and each stripped part is reported in
+the debug log (`--debug`) with its media kind and reason.
+
+The stripping decision uses the same capability resolution as attachment
+routing: an explicit `capabilities` declaration is authoritative, so a model
+declared with `audio: true` keeps its audio parts even when the catalogue says
+otherwise. Models absent from the catalogue (without an override) resolve to
+the conservative text-only default and have their media parts stripped.
+
+See [`examples/capability-overrides.yaml`](https://github.com/docker/docker-agent/blob/main/examples/capability-overrides.yaml) for a complete example, and
+[`examples/strip-unsupported-media.yaml`](https://github.com/docker/docker-agent/blob/main/examples/strip-unsupported-media.yaml) for a fixture demonstrating the
+stripping behaviour with and without an override.
 
 ## Custom Token Pricing
 

examples/capability-overrides.yaml

@@ -1,8 +1,8 @@
 # Demonstrates the `capabilities` override for models the models.dev catalogue
 # does not describe correctly. Custom OpenAI-compatible providers and local
 # models are not (always) in models.dev, so docker-agent cannot auto-detect
-# whether they accept image/PDF attachments and conservatively falls back to
-# text-only, silently dropping attachments (issue #2741).
+# whether they accept image/PDF/audio/video attachments and conservatively
+# falls back to text-only, silently dropping attachments (issue #2741).
 #
 # Declaring `capabilities` makes the model's attachment support authoritative
 # and skips the models.dev lookup entirely. The flags must match what the
@@ -38,6 +38,18 @@ models:
       image: true
       pdf: true
 
+  # Custom gateway serving a model that also accepts audio and video
+  # attachments (e.g. a self-hosted Gemini-family model behind a proxy that
+  # models.dev has never seen).
+  proxy-multimodal:
+    provider: vision-proxy
+    model: gemini-2.5-pro
+    capabilities:
+      image: true
+      pdf: true
+      audio: true
+      video: true
+
 agents:
   root:
     model: llava-local

examples/strip-unsupported-media.yaml

@@ -0,0 +1,62 @@
+# Mixed-media fixture for the strip_unsupported_modalities transform: the
+# runtime drops image/audio/video message parts before the provider call when
+# the resolved capabilities of the active model don't cover that media kind,
+# and an explicit `capabilities` override always wins over the models.dev
+# catalogue.
+#
+# Manual validation (needs OPENAI_API_KEY in the environment):
+#
+#   task build
+#
+#   # 1. Text-only model: the attached image is stripped (grep the debug log
+#   #    for "strip_unsupported_modalities: stripped media part"); the text
+#   #    part still reaches the model.
+#   ./bin/docker-agent run --exec examples/strip-unsupported-media.yaml -a text-only \
+#       --attach <some-image.png> --debug --log-file /tmp/strip-media.log \
+#       "What do you see? Reply with one sentence."
+#
+#   # 2. Vision model: no strip entry in the log; the model describes the image.
+#   ./bin/docker-agent run --exec examples/strip-unsupported-media.yaml -a vision \
+#       --attach <some-image.png> --debug --log-file /tmp/strip-media.log \
+#       "What do you see? Reply with one sentence."
+#
+#   # 3. Override: same text-only model, but `capabilities.image: true` makes
+#   #    the declaration authoritative — no strip entry appears and the image
+#   #    reaches the provider. (gpt-3.5-turbo genuinely rejects images, so the
+#   #    provider-side 400 error is the PROOF that the override was respected
+#   #    and the runtime no longer silently protected the call.)
+#   ./bin/docker-agent run --exec examples/strip-unsupported-media.yaml -a override \
+#       --attach <some-image.png> --debug --log-file /tmp/strip-media.log \
+#       "What do you see? Reply with one sentence."
+#
+# Cleanup: rm /tmp/strip-media.log
+models:
+  text-only:
+    provider: openai
+    model: gpt-3.5-turbo # models.dev input modalities: text only
+
+  vision:
+    provider: openai
+    model: gpt-4o # models.dev input modalities: text, image, pdf
+
+  claims-vision:
+    provider: openai
+    model: gpt-3.5-turbo
+    capabilities:
+      image: true # override wins: the runtime must NOT strip images
+
+agents:
+  text-only:
+    model: text-only
+    description: Text-only model — attached images are stripped before the call
+    instruction: You are a helpful assistant. Describe any images the user attaches.
+
+  vision:
+    model: vision
+    description: Vision model — attached images reach the provider untouched
+    instruction: You are a helpful assistant. Describe any images the user attaches.
+
+  override:
+    model: claims-vision
+    description: Text-only model with an explicit image capability override
+    instruction: You are a helpful assistant. Describe any images the user attaches.

pkg/attachment/decide_test.go

@@ -11,15 +11,15 @@ import (
 
 // testCaps is a small helper that builds a ModelCapabilities directly.
 func visionCaps() modelinfo.ModelCapabilities {
-	return modelinfo.CapsWith(true, true)
+	return modelinfo.CapsWith(true, true, false, false)
 }
 
 func textOnlyCaps() modelinfo.ModelCapabilities {
-	return modelinfo.CapsWith(false, false)
+	return modelinfo.CapsWith(false, false, false, false)
 }
 
 func imageNoPDFCaps() modelinfo.ModelCapabilities {
-	return modelinfo.CapsWith(true, false)
+	return modelinfo.CapsWith(true, false, false, false)
 }
 
 func TestDecide(t *testing.T) {

pkg/config/latest/capabilities_test.go

@@ -38,6 +38,54 @@ capabilities:
 	assert.False(t, rt.Capabilities.PDF)
 }
 
+func TestModelConfigCapabilitiesAudioVideoYAMLRoundTrip(t *testing.T) {
+	t.Parallel()
+
+	const in = `provider: vision-proxy
+model: gemini-2.5-pro
+capabilities:
+  image: true
+  pdf: true
+  audio: true
+  video: true
+`
+	var f FlexibleModelConfig
+	require.NoError(t, yaml.Unmarshal([]byte(in), &f))
+
+	require.NotNil(t, f.Capabilities, "capabilities should be parsed")
+	assert.True(t, f.Capabilities.Audio)
+	assert.True(t, f.Capabilities.Video)
+
+	out, err := yaml.Marshal(f)
+	require.NoError(t, err)
+
+	var rt FlexibleModelConfig
+	require.NoError(t, yaml.Unmarshal(out, &rt))
+	require.NotNil(t, rt.Capabilities, "capabilities should survive a marshal round-trip; got:\n%s", out)
+	assert.True(t, rt.Capabilities.Audio)
+	assert.True(t, rt.Capabilities.Video)
+}
+
+// TestModelConfigCapabilitiesAudioVideoOmittedDefaultsFalse pins that, within
+// a present capabilities block, omitted audio/video resolve to false rather
+// than falling back to models.dev: any non-nil block is authoritative.
+func TestModelConfigCapabilitiesAudioVideoOmittedDefaultsFalse(t *testing.T) {
+	t.Parallel()
+
+	const in = `provider: ollama
+model: llava
+capabilities:
+  image: true
+  pdf: false
+`
+	var f FlexibleModelConfig
+	require.NoError(t, yaml.Unmarshal([]byte(in), &f))
+
+	require.NotNil(t, f.Capabilities)
+	assert.False(t, f.Capabilities.Audio)
+	assert.False(t, f.Capabilities.Video)
+}
+
 func TestModelConfigShorthandOnlyWithoutCapabilities(t *testing.T) {
 	t.Parallel()
 
@@ -80,13 +128,15 @@ func TestModelConfigCloneCopiesCapabilities(t *testing.T) {
 	orig := &ModelConfig{
 		Provider:     "my-proxy",
 		Model:        "gpt-4o",
-		Capabilities: &CapabilitiesConfig{Image: true, PDF: true},
+		Capabilities: &CapabilitiesConfig{Image: true, PDF: true, Audio: true, Video: true},
 	}
 
 	clone := orig.Clone()
 	require.NotNil(t, clone.Capabilities)
 	assert.True(t, clone.Capabilities.Image)
 	assert.True(t, clone.Capabilities.PDF)
+	assert.True(t, clone.Capabilities.Audio)
+	assert.True(t, clone.Capabilities.Video)
 
 	// Mutating the clone must not affect the original (deep copy).
 	clone.Capabilities.Image = false

pkg/config/latest/types.go

@@ -1206,7 +1206,8 @@ func (c *CostConfig) validate() error {
 // catalogue). It exists for models the catalogue does not describe correctly:
 // custom OpenAI-compatible providers, local models (e.g. Ollama), and model
 // versions that have been dropped from the catalogue. Without it, such models
-// fall back to text-only and their image/PDF attachments are silently dropped.
+// fall back to text-only and their image/PDF/audio/video attachments are
+// silently dropped.
 //
 // When set, the declared flags are authoritative and no models.dev lookup is
 // performed. When nil (the default), capabilities are detected from models.dev.
@@ -1217,6 +1218,10 @@ type CapabilitiesConfig struct {
 	Image bool `json:"image,omitempty"`
 	// PDF reports whether the model accepts PDF (application/pdf) attachments.
 	PDF bool `json:"pdf,omitempty"`
+	// Audio reports whether the model accepts audio attachments.
+	Audio bool `json:"audio,omitempty"`
+	// Video reports whether the model accepts video attachments.
+	Video bool `json:"video,omitempty"`
 }
 
 // IsFirstAvailable reports whether this model is a first-available selector