From 2f6f5fab1c5673ec495d1131f50c914eb2d5739e Mon Sep 17 00:00:00 2001 From: Victor Vargas Date: Thu, 16 Jul 2026 22:23:54 -0700 Subject: [PATCH 1/2] feat(llm): add multimodal ContentPart/Parts + per-provider serialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Message gains an optional Parts []ContentPart alongside the existing plain-text Content, so a turn can carry text plus image/video attachments. Content stays the single source of truth for every existing text-only caller (sidebar.go, memory_tools.go, etc. are untouched); Parts only matters to a provider client when non-empty. openai and llamacpp (both OpenAI-compatible) serialize Parts into the standard text/image_url content-array shape; llamacpp additionally passes video through as a best-effort video_url part, since llama.cpp itself has no video support but the whole point of this client is the user's own OpenAI-compatible server sitting in front of a video-capable model — the server decides whether it understands it, not this client. anthropic converts image parts to its base64 image content block, and rejects a video part outright with a clear error: the Messages API has no video block type at all, so sending one would just produce a confusing 400 instead. ProviderCapabilities gains SupportsVideo, true only for llamacpp. --- pkg/llm/providers/anthropic/client.go | 58 +++++++++++++++++++---- pkg/llm/providers/llamacpp/client.go | 58 +++++++++++++++++++++-- pkg/llm/providers/llamacpp/client_test.go | 7 ++- pkg/llm/providers/openai/client.go | 52 ++++++++++++++++++-- pkg/llm/types.go | 23 +++++++++ 5 files changed, 181 insertions(+), 17 deletions(-) diff --git a/pkg/llm/providers/anthropic/client.go b/pkg/llm/providers/anthropic/client.go index ab7648f..109ff5b 100644 --- a/pkg/llm/providers/anthropic/client.go +++ b/pkg/llm/providers/anthropic/client.go @@ -275,6 +275,7 @@ func (c *Client) Capabilities() llm.ProviderCapabilities { return llm.ProviderCapabilities{ SupportsTools: true, SupportsVision: true, + SupportsVideo: false, SupportsJSON: true, MaxContextWindow: c.contextWindow, } @@ -323,7 +324,29 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader Content: m.Content, }) case llm.RoleUser: - appendUserBlock(anthropicContentBlock{Type: "text", Text: m.Content}) + if len(m.Parts) == 0 { + appendUserBlock(anthropicContentBlock{Type: "text", Text: m.Content}) + break + } + for _, p := range m.Parts { + switch p.Type { + case "text": + appendUserBlock(anthropicContentBlock{Type: "text", Text: p.Text}) + case "image": + appendUserBlock(anthropicContentBlock{ + Type: "image", + Source: &anthropicImageSource{ + Type: "base64", + MediaType: p.MimeType, + Data: stripDataURIPrefix(p.MediaURL), + }, + }) + case "video": + return nil, fmt.Errorf("anthropic: video attachments are not supported by the Messages API") + default: + return nil, fmt.Errorf("anthropic: unknown content part type %q", p.Type) + } + } case llm.RoleAssistant: var blocks []anthropicContentBlock if strings.TrimSpace(m.Content) != "" { @@ -518,13 +541,32 @@ type anthropicMessage struct { } type anthropicContentBlock struct { - Type string `json:"type"` - Text string `json:"text,omitempty"` - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Input json.RawMessage `json:"input,omitempty"` - ToolUseID string `json:"tool_use_id,omitempty"` - Content string `json:"content,omitempty"` + Type string `json:"type"` + Text string `json:"text,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Input json.RawMessage `json:"input,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + Content string `json:"content,omitempty"` + Source *anthropicImageSource `json:"source,omitempty"` +} + +// anthropicImageSource is an "image" content block's base64-encoded payload. +type anthropicImageSource struct { + Type string `json:"type"` // always "base64" + MediaType string `json:"media_type"` + Data string `json:"data"` +} + +// stripDataURIPrefix strips a "data:;base64," prefix from a data URI, +// leaving just the base64 payload Anthropic's image source expects. Returns +// the input unchanged if it isn't a data URI (e.g. a caller passed a raw +// base64 string directly). +func stripDataURIPrefix(mediaURL string) string { + if idx := strings.Index(mediaURL, ";base64,"); idx != -1 { + return mediaURL[idx+len(";base64,"):] + } + return mediaURL } type anthropicTool struct { diff --git a/pkg/llm/providers/llamacpp/client.go b/pkg/llm/providers/llamacpp/client.go index aaa6f08..6e764e7 100644 --- a/pkg/llm/providers/llamacpp/client.go +++ b/pkg/llm/providers/llamacpp/client.go @@ -322,7 +322,8 @@ func (c *Client) Name() string { func (c *Client) Capabilities() llm.ProviderCapabilities { return llm.ProviderCapabilities{ SupportsTools: true, - SupportsVision: false, + SupportsVision: true, + SupportsVideo: true, SupportsJSON: true, MaxContextWindow: c.contextWindow, } @@ -332,9 +333,13 @@ func (c *Client) Capabilities() llm.ProviderCapabilities { func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader, error) { messages := make([]llamaMessage, len(req.Messages)) for i, m := range req.Messages { + content, err := buildContentValue(m) + if err != nil { + return nil, err + } messages[i] = llamaMessage{ Role: string(m.Role), - Content: m.Content, + Content: content, ToolCallID: m.ToolCallID, Name: m.Name, } @@ -477,13 +482,58 @@ type llamaStreamOptions struct { } type llamaMessage struct { - Role string `json:"role"` - Content string `json:"content"` + Role string `json:"role"` + // Content is either a plain string (the common case) or a + // []llamaContentPart when the source llm.Message carried Parts - see + // buildContentValue. + Content interface{} `json:"content"` ToolCallID string `json:"tool_call_id,omitempty"` Name string `json:"name,omitempty"` ToolCalls []llamaToolCall `json:"tool_calls,omitempty"` } +// llamaContentPart is one block of a multipart "content" array, following +// the same OpenAI-compatible shape llama.cpp's server accepts for +// vision-capable models (e.g. Qwen2-VL via its mmproj). +type llamaContentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *llamaMediaURL `json:"image_url,omitempty"` + VideoURL *llamaMediaURL `json:"video_url,omitempty"` +} + +type llamaMediaURL struct { + URL string `json:"url"` +} + +// buildContentValue converts an llm.Message's Parts into the OpenAI-style +// multipart content shape, or falls back to the plain Content string when +// there are no Parts. Unlike the openai/anthropic clients, a video part is +// passed through as a "video_url" block rather than rejected: llama.cpp +// itself has no video support, but this client's whole reason to exist is +// the user's own OpenAI-compatible server sitting in front of a +// video-capable model, so the server - not this client - is what decides +// whether it understands it. +func buildContentValue(m llm.Message) (interface{}, error) { + if len(m.Parts) == 0 { + return m.Content, nil + } + parts := make([]llamaContentPart, 0, len(m.Parts)) + for _, p := range m.Parts { + switch p.Type { + case "text": + parts = append(parts, llamaContentPart{Type: "text", Text: p.Text}) + case "image": + parts = append(parts, llamaContentPart{Type: "image_url", ImageURL: &llamaMediaURL{URL: p.MediaURL}}) + case "video": + parts = append(parts, llamaContentPart{Type: "video_url", VideoURL: &llamaMediaURL{URL: p.MediaURL}}) + default: + return nil, fmt.Errorf("llamacpp: unknown content part type %q", p.Type) + } + } + return parts, nil +} + type llamaTool struct { Type string `json:"type"` Function json.RawMessage `json:"function"` diff --git a/pkg/llm/providers/llamacpp/client_test.go b/pkg/llm/providers/llamacpp/client_test.go index 4de3eed..fb67a31 100644 --- a/pkg/llm/providers/llamacpp/client_test.go +++ b/pkg/llm/providers/llamacpp/client_test.go @@ -25,8 +25,11 @@ func TestClient_Capabilities(t *testing.T) { if !caps.SupportsTools { t.Error("expected SupportsTools to be true") } - if caps.SupportsVision { - t.Error("expected SupportsVision to be false") + if !caps.SupportsVision { + t.Error("expected SupportsVision to be true") + } + if !caps.SupportsVideo { + t.Error("expected SupportsVideo to be true") } } diff --git a/pkg/llm/providers/openai/client.go b/pkg/llm/providers/openai/client.go index fbc0079..1a27d45 100644 --- a/pkg/llm/providers/openai/client.go +++ b/pkg/llm/providers/openai/client.go @@ -243,6 +243,7 @@ func (c *Client) Capabilities() llm.ProviderCapabilities { return llm.ProviderCapabilities{ SupportsTools: true, SupportsVision: true, + SupportsVideo: false, SupportsJSON: true, MaxContextWindow: 128000, } @@ -253,9 +254,13 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader // Convert messages to OpenAI format messages := make([]openaiMessage, len(req.Messages)) for i, m := range req.Messages { + content, err := buildContentValue(m) + if err != nil { + return nil, err + } messages[i] = openaiMessage{ Role: string(m.Role), - Content: m.Content, + Content: content, ToolCallID: m.ToolCallID, Name: m.Name, } @@ -378,13 +383,54 @@ type openaiStreamOptions struct { } type openaiMessage struct { - Role string `json:"role"` - Content string `json:"content"` + Role string `json:"role"` + // Content is either a plain string (the common case) or a + // []openaiContentPart when the source llm.Message carried Parts - see + // buildContentValue. + Content interface{} `json:"content"` ToolCallID string `json:"tool_call_id,omitempty"` Name string `json:"name,omitempty"` ToolCalls []openaiToolCall `json:"tool_calls,omitempty"` } +// openaiContentPart is one block of a multipart "content" array, following +// the same shape OpenAI's vision-capable chat completions endpoint expects. +type openaiContentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *openaiMediaURL `json:"image_url,omitempty"` +} + +type openaiMediaURL struct { + URL string `json:"url"` +} + +// buildContentValue converts an llm.Message's Parts into the OpenAI +// multipart content shape, or falls back to the plain Content string when +// there are no Parts - existing callers building a plain-text Message are +// completely unaffected. A video part is rejected outright: OpenAI's chat +// completions API has no video content type, so sending one would just +// produce a confusing API error instead of this clear one. +func buildContentValue(m llm.Message) (interface{}, error) { + if len(m.Parts) == 0 { + return m.Content, nil + } + parts := make([]openaiContentPart, 0, len(m.Parts)) + for _, p := range m.Parts { + switch p.Type { + case "text": + parts = append(parts, openaiContentPart{Type: "text", Text: p.Text}) + case "image": + parts = append(parts, openaiContentPart{Type: "image_url", ImageURL: &openaiMediaURL{URL: p.MediaURL}}) + case "video": + return nil, fmt.Errorf("openai: video attachments are not supported by the chat completions API") + default: + return nil, fmt.Errorf("openai: unknown content part type %q", p.Type) + } + } + return parts, nil +} + type openaiTool struct { Type string `json:"type"` Function json.RawMessage `json:"function"` diff --git a/pkg/llm/types.go b/pkg/llm/types.go index 9bd6275..3e98b5a 100644 --- a/pkg/llm/types.go +++ b/pkg/llm/types.go @@ -29,12 +29,35 @@ type Message struct { // rejected by) the chat template - the model loses track of what it // already asked for and re-attempts it, or restarts from scratch. ToolCalls []ToolCall `json:"tool_calls,omitempty"` + // Parts, when non-empty, carries a multimodal message (text plus + // image/video attachments) and takes precedence over Content when a + // provider client serializes the wire request. Content should still be + // set to a plain-text rendition even when Parts is used, since it's what + // storage/logging/history reconstruction read - Parts only matters for + // the live request that actually goes out to the model. + Parts []ContentPart `json:"parts,omitempty"` +} + +// ContentPart is one piece of a multimodal message. +type ContentPart struct { + // Type is "text", "image", or "video". + Type string `json:"type"` + // Text is set when Type == "text". + Text string `json:"text,omitempty"` + // MediaURL is set when Type == "image"/"video": either a data URI + // (data:;base64,<...>) or an http(s) URL. + MediaURL string `json:"media_url,omitempty"` + // MimeType is the media's MIME type (e.g. "image/png"), split out + // separately from MediaURL so providers that need it apart from the data + // URI (e.g. Anthropic's base64 media_type field) don't have to re-parse it. + MimeType string `json:"mime_type,omitempty"` } // ProviderCapabilities describes what a model supports. type ProviderCapabilities struct { SupportsTools bool `json:"supports_tools"` SupportsVision bool `json:"supports_vision"` + SupportsVideo bool `json:"supports_video"` SupportsJSON bool `json:"supports_json"` MaxContextWindow int `json:"max_context_window"` } From 49353485e5611d449bcc001e3c4b7843ea2d509e Mon Sep 17 00:00:00 2001 From: Victor Vargas Date: Thu, 16 Jul 2026 22:24:03 -0700 Subject: [PATCH 2/2] feat(agent): thread the new turn through Loop as llm.Message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run/RunStream took the new turn as a bare string, which had nowhere to carry ContentPart attachments. Both now take an llm.Message (Role is forced to RoleUser regardless of what the caller sets), so a caller building a multimodal turn just fills in Content/Parts on it instead of the loop needing a second, parallel parameter. subagent.go and every test call site are updated to wrap their string prompt as llm.Message{Role: llm.RoleUser, Content: ...} — SubAgent.Run itself is untouched, it still takes a plain task string. --- pkg/agent/integration_test.go | 4 ++-- pkg/agent/loop.go | 15 +++++++++------ pkg/agent/loop_test.go | 30 ++++++++++++++--------------- pkg/agent/subagent.go | 2 +- pkg/agent/thinking_budget_test.go | 4 ++-- pkg/agent/unparsed_toolcall_test.go | 6 +++--- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/pkg/agent/integration_test.go b/pkg/agent/integration_test.go index 2a2f052..68532bf 100644 --- a/pkg/agent/integration_test.go +++ b/pkg/agent/integration_test.go @@ -103,7 +103,7 @@ func TestIntegration_AgentLoop_Generate(t *testing.T) { MaxIters: 3, }) - resp, err := loop.Run(context.Background(), "What is 20+22? Use the add_numbers tool.") + resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "What is 20+22? Use the add_numbers tool."}) if err != nil { t.Fatalf("run failed: %v", err) } @@ -130,7 +130,7 @@ func TestIntegration_AgentLoop_Stream(t *testing.T) { MaxIters: 3, }) - stream := loop.RunStream(context.Background(), "Say something interesting.") + stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Say something interesting."}) var chunks []llm.StreamChunk for chunk, err := range stream { diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 6932298..de38251 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -77,8 +77,9 @@ func New(cfg Config) *Loop { // Run executes the agent loop and returns the final response. // Optional history messages are appended after the system prompt and before -// the new user input. -func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (Response, error) { +// the new user input. input's Role is overwritten to RoleUser regardless of +// what the caller sets, so callers only need to fill in Content/Parts. +func (l *Loop) Run(ctx context.Context, input llm.Message, history ...llm.Message) (Response, error) { start := time.Now() messages := l.buildInitialMessages(input, history) @@ -174,8 +175,9 @@ func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (R // RunStream executes the agent loop with streaming output. // Optional history messages are appended after the system prompt and before -// the new user input. -func (l *Loop) RunStream(ctx context.Context, input string, history ...llm.Message) iter.Seq2[llm.StreamChunk, error] { +// the new user input. input's Role is overwritten to RoleUser regardless of +// what the caller sets, so callers only need to fill in Content/Parts. +func (l *Loop) RunStream(ctx context.Context, input llm.Message, history ...llm.Message) iter.Seq2[llm.StreamChunk, error] { return func(yield func(llm.StreamChunk, error) bool) { messages := l.buildInitialMessages(input, history) // Same as Run: the schemas are identical on every iteration. @@ -362,12 +364,13 @@ func containsUnparsedToolCall(s string) bool { return strings.Contains(s, "