Merge pull request #10 from VictorVargas/feat/multimodal-content-parts

Add multimodal (image/video) content support

pkg/agent/integration_test.go

@@ -103,7 +103,7 @@ func TestIntegration_AgentLoop_Generate(t *testing.T) {
 		MaxIters: 3,
 	})
 
-	resp, err := loop.Run(context.Background(), "What is 20+22? Use the add_numbers tool.")
+	resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "What is 20+22? Use the add_numbers tool."})
 	if err != nil {
 		t.Fatalf("run failed: %v", err)
 	}
@@ -130,7 +130,7 @@ func TestIntegration_AgentLoop_Stream(t *testing.T) {
 		MaxIters: 3,
 	})
 
-	stream := loop.RunStream(context.Background(), "Say something interesting.")
+	stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Say something interesting."})
 
 	var chunks []llm.StreamChunk
 	for chunk, err := range stream {

pkg/agent/loop.go

@@ -77,8 +77,9 @@ func New(cfg Config) *Loop {
 
 // Run executes the agent loop and returns the final response.
 // Optional history messages are appended after the system prompt and before
-// the new user input.
+// the new user input. input's Role is overwritten to RoleUser regardless of
-func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (Response, error) {
+// what the caller sets, so callers only need to fill in Content/Parts.
+func (l *Loop) Run(ctx context.Context, input llm.Message, history ...llm.Message) (Response, error) {
 	start := time.Now()
 
 	messages := l.buildInitialMessages(input, history)
@@ -174,8 +175,9 @@ func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (R
 
 // RunStream executes the agent loop with streaming output.
 // Optional history messages are appended after the system prompt and before
-// the new user input.
+// the new user input. input's Role is overwritten to RoleUser regardless of
-func (l *Loop) RunStream(ctx context.Context, input string, history ...llm.Message) iter.Seq2[llm.StreamChunk, error] {
+// what the caller sets, so callers only need to fill in Content/Parts.
+func (l *Loop) RunStream(ctx context.Context, input llm.Message, history ...llm.Message) iter.Seq2[llm.StreamChunk, error] {
 	return func(yield func(llm.StreamChunk, error) bool) {
 		messages := l.buildInitialMessages(input, history)
 		// Same as Run: the schemas are identical on every iteration.
@@ -362,12 +364,13 @@ func containsUnparsedToolCall(s string) bool {
 	return strings.Contains(s, "<tool_call") || strings.Contains(s, "<function=")
 }
 
-func (l *Loop) buildInitialMessages(input string, history []llm.Message) []llm.Message {
+func (l *Loop) buildInitialMessages(input llm.Message, history []llm.Message) []llm.Message {
 	systemPrompt := persona.AssembleSystemPrompt(l.cfg.Persona, l.cfg.AgentsMD)
 	messages := make([]llm.Message, 0, len(history)+2)
 	messages = append(messages, llm.Message{Role: llm.RoleSystem, Content: systemPrompt})
 	messages = append(messages, history...)
-	messages = append(messages, llm.Message{Role: llm.RoleUser, Content: input})
+	input.Role = llm.RoleUser
+	messages = append(messages, input)
 	return messages
 }
 

pkg/agent/loop_test.go

@@ -102,7 +102,7 @@ func TestRun_NoToolCalls(t *testing.T) {
 		Tools:   tools.NewRegistry(),
 	})
 
-	resp, err := loop.Run(context.Background(), "Hello")
+	resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Hello"})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -133,7 +133,7 @@ func TestRun_IncludesAgentsMD(t *testing.T) {
 		AgentsMD: "Never edit go.mod directly.",
 	})
 
-	if _, err := loop.Run(context.Background(), "Hello"); err != nil {
+	if _, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Hello"}); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 	if !strings.Contains(capturedSystemPrompt, "Never edit go.mod directly.") {
@@ -176,7 +176,7 @@ func TestRun_ToolCalls(t *testing.T) {
 		Tools:   registry,
 	})
 
-	resp, err := loop.Run(context.Background(), "Say hi")
+	resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Say hi"})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -233,7 +233,7 @@ func TestRun_ToolCalls_RecordsAssistantTurnAndToolCallID(t *testing.T) {
 		Tools:   registry,
 	})
 
-	if _, err := loop.Run(context.Background(), "Say hi"); err != nil {
+	if _, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Say hi"}); err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
 
@@ -315,7 +315,7 @@ func TestRun_Stream_ToolCalls_RecordsAssistantTurnAndToolCallID(t *testing.T) {
 		Tools:   registry,
 	})
 
-	for _, err := range loop.RunStream(context.Background(), "test") {
+	for _, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"}) {
 		if err != nil {
 			t.Fatalf("unexpected stream error: %v", err)
 		}
@@ -379,7 +379,7 @@ func TestRun_MaxIterations(t *testing.T) {
 		MaxIters: 3,
 	})
 
-	_, err := loop.Run(context.Background(), "test")
+	_, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
 	if err == nil {
 		t.Fatal("expected error, got nil")
 	}
@@ -408,7 +408,7 @@ func TestRun_ToolNotFound(t *testing.T) {
 		Tools:   tools.NewRegistry(),
 	})
 
-	resp, err := loop.Run(context.Background(), "test")
+	resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -451,7 +451,7 @@ func TestRun_ApprovalDenied(t *testing.T) {
 		},
 	})
 
-	resp, err := loop.Run(context.Background(), "test")
+	resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -496,7 +496,7 @@ func TestRun_SandboxViolation(t *testing.T) {
 		},
 	})
 
-	resp, err := loop.Run(context.Background(), "test")
+	resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -540,7 +540,7 @@ func TestRun_OnIterationHook(t *testing.T) {
 		},
 	})
 
-	_, err := loop.Run(context.Background(), "test")
+	_, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -570,7 +570,7 @@ func TestRun_Stream_NoToolCalls(t *testing.T) {
 	})
 
 	var chunks []llm.StreamChunk
-	stream := loop.RunStream(context.Background(), "test")
+	stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
 	for chunk, err := range stream {
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
@@ -608,7 +608,7 @@ func TestRun_Stream_ForwardsTrailingUsageOnlyChunk(t *testing.T) {
 	})
 
 	var chunks []llm.StreamChunk
-	stream := loop.RunStream(context.Background(), "test")
+	stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
 	for chunk, err := range stream {
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
@@ -661,7 +661,7 @@ func TestRun_Stream_WithToolCalls(t *testing.T) {
 	})
 
 	var chunks []llm.StreamChunk
-	stream := loop.RunStream(context.Background(), "test")
+	stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
 	for chunk, err := range stream {
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
@@ -713,7 +713,7 @@ func TestRun_Stream_MaxIterations(t *testing.T) {
 	})
 
 	var chunks []llm.StreamChunk
-	stream := loop.RunStream(context.Background(), "test")
+	stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
 	for chunk, err := range stream {
 		if err != nil {
 			// expect max iterations error
@@ -748,7 +748,7 @@ func TestRun_Timeout(t *testing.T) {
 		Tools:   tools.NewRegistry(),
 	})
 
-	_, err := loop.Run(ctx, "test")
+	_, err := loop.Run(ctx, llm.Message{Role: llm.RoleUser, Content: "test"})
 	if err == nil {
 		t.Fatal("expected timeout error, got nil")
 	}

pkg/agent/subagent.go

@@ -33,7 +33,7 @@ func (s SubAgent) Run(ctx context.Context, llmClient llm.LLMClient, agentsMD str
 	if cfg.MaxIters == 0 {
 		cfg.MaxIters = DefaultMaxIterations
 	}
-	return New(cfg).Run(ctx, task)
+	return New(cfg).Run(ctx, llm.Message{Role: llm.RoleUser, Content: task})
 }
 
 // SubAgentRegistry looks up SubAgents by name for the delegate tool.

pkg/agent/thinking_budget_test.go

@@ -28,7 +28,7 @@ func TestRunStream_RecoversFromThinkingBudgetCut(t *testing.T) {
 	loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
 
 	var final strings.Builder
-	for chunk, err := range loop.RunStream(context.Background(), "arregla x.py") {
+	for chunk, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "arregla x.py"}) {
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
@@ -62,7 +62,7 @@ func TestRunStream_ThinkingBudgetNudgeGivesUpAfterLimit(t *testing.T) {
 
 	loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
 
-	for _, err := range loop.RunStream(context.Background(), "haz algo") {
+	for _, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "haz algo"}) {
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}

pkg/agent/unparsed_toolcall_test.go

@@ -97,7 +97,7 @@ func TestRunStream_RecoversFromUnparsedToolCall(t *testing.T) {
 	loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
 
 	var final strings.Builder
-	for chunk, err := range loop.RunStream(context.Background(), "arregla x.py") {
+	for chunk, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "arregla x.py"}) {
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
@@ -133,7 +133,7 @@ func TestRunStream_NudgeGivesUpAfterLimit(t *testing.T) {
 	loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
 
 	rounds := 0
-	for _, err := range loop.RunStream(context.Background(), "haz algo") {
+	for _, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "haz algo"}) {
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
@@ -158,7 +158,7 @@ func TestRun_RecoversFromUnparsedToolCall(t *testing.T) {
 	}}
 
 	loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
-	resp, err := loop.Run(context.Background(), "arregla x.py")
+	resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "arregla x.py"})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

pkg/llm/providers/anthropic/client.go

@@ -275,6 +275,7 @@ func (c *Client) Capabilities() llm.ProviderCapabilities {
 	return llm.ProviderCapabilities{
 		SupportsTools:    true,
 		SupportsVision:   true,
+		SupportsVideo:    false,
 		SupportsJSON:     true,
 		MaxContextWindow: c.contextWindow,
 	}
@@ -323,7 +324,29 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader
 				Content:   m.Content,
 			})
 		case llm.RoleUser:
+			if len(m.Parts) == 0 {
 				appendUserBlock(anthropicContentBlock{Type: "text", Text: m.Content})
+				break
+			}
+			for _, p := range m.Parts {
+				switch p.Type {
+				case "text":
+					appendUserBlock(anthropicContentBlock{Type: "text", Text: p.Text})
+				case "image":
+					appendUserBlock(anthropicContentBlock{
+						Type: "image",
+						Source: &anthropicImageSource{
+							Type:      "base64",
+							MediaType: p.MimeType,
+							Data:      stripDataURIPrefix(p.MediaURL),
+						},
+					})
+				case "video":
+					return nil, fmt.Errorf("anthropic: video attachments are not supported by the Messages API")
+				default:
+					return nil, fmt.Errorf("anthropic: unknown content part type %q", p.Type)
+				}
+			}
 		case llm.RoleAssistant:
 			var blocks []anthropicContentBlock
 			if strings.TrimSpace(m.Content) != "" {
@@ -525,6 +548,25 @@ type anthropicContentBlock struct {
 	Input     json.RawMessage       `json:"input,omitempty"`
 	ToolUseID string                `json:"tool_use_id,omitempty"`
 	Content   string                `json:"content,omitempty"`
+	Source    *anthropicImageSource `json:"source,omitempty"`
+}
+
+// anthropicImageSource is an "image" content block's base64-encoded payload.
+type anthropicImageSource struct {
+	Type      string `json:"type"` // always "base64"
+	MediaType string `json:"media_type"`
+	Data      string `json:"data"`
+}
+
+// stripDataURIPrefix strips a "data:<mime>;base64," prefix from a data URI,
+// leaving just the base64 payload Anthropic's image source expects. Returns
+// the input unchanged if it isn't a data URI (e.g. a caller passed a raw
+// base64 string directly).
+func stripDataURIPrefix(mediaURL string) string {
+	if idx := strings.Index(mediaURL, ";base64,"); idx != -1 {
+		return mediaURL[idx+len(";base64,"):]
+	}
+	return mediaURL
 }
 
 type anthropicTool struct {

pkg/llm/providers/llamacpp/client.go

@@ -322,7 +322,8 @@ func (c *Client) Name() string {
 func (c *Client) Capabilities() llm.ProviderCapabilities {
 	return llm.ProviderCapabilities{
 		SupportsTools:    true,
-		SupportsVision:   false,
+		SupportsVision:   true,
+		SupportsVideo:    true,
 		SupportsJSON:     true,
 		MaxContextWindow: c.contextWindow,
 	}
@@ -332,9 +333,13 @@ func (c *Client) Capabilities() llm.ProviderCapabilities {
 func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader, error) {
 	messages := make([]llamaMessage, len(req.Messages))
 	for i, m := range req.Messages {
+		content, err := buildContentValue(m)
+		if err != nil {
+			return nil, err
+		}
 		messages[i] = llamaMessage{
 			Role:       string(m.Role),
-			Content:    m.Content,
+			Content:    content,
 			ToolCallID: m.ToolCallID,
 			Name:       m.Name,
 		}
@@ -478,12 +483,57 @@ type llamaStreamOptions struct {
 
 type llamaMessage struct {
 	Role string `json:"role"`
-	Content    string          `json:"content"`
+	// Content is either a plain string (the common case) or a
+	// []llamaContentPart when the source llm.Message carried Parts - see
+	// buildContentValue.
+	Content    interface{}     `json:"content"`
 	ToolCallID string          `json:"tool_call_id,omitempty"`
 	Name       string          `json:"name,omitempty"`
 	ToolCalls  []llamaToolCall `json:"tool_calls,omitempty"`
 }
 
+// llamaContentPart is one block of a multipart "content" array, following
+// the same OpenAI-compatible shape llama.cpp's server accepts for
+// vision-capable models (e.g. Qwen2-VL via its mmproj).
+type llamaContentPart struct {
+	Type     string         `json:"type"`
+	Text     string         `json:"text,omitempty"`
+	ImageURL *llamaMediaURL `json:"image_url,omitempty"`
+	VideoURL *llamaMediaURL `json:"video_url,omitempty"`
+}
+
+type llamaMediaURL struct {
+	URL string `json:"url"`
+}
+
+// buildContentValue converts an llm.Message's Parts into the OpenAI-style
+// multipart content shape, or falls back to the plain Content string when
+// there are no Parts. Unlike the openai/anthropic clients, a video part is
+// passed through as a "video_url" block rather than rejected: llama.cpp
+// itself has no video support, but this client's whole reason to exist is
+// the user's own OpenAI-compatible server sitting in front of a
+// video-capable model, so the server - not this client - is what decides
+// whether it understands it.
+func buildContentValue(m llm.Message) (interface{}, error) {
+	if len(m.Parts) == 0 {
+		return m.Content, nil
+	}
+	parts := make([]llamaContentPart, 0, len(m.Parts))
+	for _, p := range m.Parts {
+		switch p.Type {
+		case "text":
+			parts = append(parts, llamaContentPart{Type: "text", Text: p.Text})
+		case "image":
+			parts = append(parts, llamaContentPart{Type: "image_url", ImageURL: &llamaMediaURL{URL: p.MediaURL}})
+		case "video":
+			parts = append(parts, llamaContentPart{Type: "video_url", VideoURL: &llamaMediaURL{URL: p.MediaURL}})
+		default:
+			return nil, fmt.Errorf("llamacpp: unknown content part type %q", p.Type)
+		}
+	}
+	return parts, nil
+}
+
 type llamaTool struct {
 	Type     string          `json:"type"`
 	Function json.RawMessage `json:"function"`

pkg/llm/providers/llamacpp/client_test.go

@@ -25,8 +25,11 @@ func TestClient_Capabilities(t *testing.T) {
 	if !caps.SupportsTools {
 		t.Error("expected SupportsTools to be true")
 	}
-	if caps.SupportsVision {
+	if !caps.SupportsVision {
-		t.Error("expected SupportsVision to be false")
+		t.Error("expected SupportsVision to be true")
+	}
+	if !caps.SupportsVideo {
+		t.Error("expected SupportsVideo to be true")
 	}
 }
 

pkg/llm/providers/openai/client.go

@@ -243,6 +243,7 @@ func (c *Client) Capabilities() llm.ProviderCapabilities {
 	return llm.ProviderCapabilities{
 		SupportsTools:    true,
 		SupportsVision:   true,
+		SupportsVideo:    false,
 		SupportsJSON:     true,
 		MaxContextWindow: 128000,
 	}
@@ -253,9 +254,13 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader
 	// Convert messages to OpenAI format
 	messages := make([]openaiMessage, len(req.Messages))
 	for i, m := range req.Messages {
+		content, err := buildContentValue(m)
+		if err != nil {
+			return nil, err
+		}
 		messages[i] = openaiMessage{
 			Role:       string(m.Role),
-			Content:    m.Content,
+			Content:    content,
 			ToolCallID: m.ToolCallID,
 			Name:       m.Name,
 		}
@@ -379,12 +384,53 @@ type openaiStreamOptions struct {
 
 type openaiMessage struct {
 	Role string `json:"role"`
-	Content    string           `json:"content"`
+	// Content is either a plain string (the common case) or a
+	// []openaiContentPart when the source llm.Message carried Parts - see
+	// buildContentValue.
+	Content    interface{}      `json:"content"`
 	ToolCallID string           `json:"tool_call_id,omitempty"`
 	Name       string           `json:"name,omitempty"`
 	ToolCalls  []openaiToolCall `json:"tool_calls,omitempty"`
 }
 
+// openaiContentPart is one block of a multipart "content" array, following
+// the same shape OpenAI's vision-capable chat completions endpoint expects.
+type openaiContentPart struct {
+	Type     string          `json:"type"`
+	Text     string          `json:"text,omitempty"`
+	ImageURL *openaiMediaURL `json:"image_url,omitempty"`
+}
+
+type openaiMediaURL struct {
+	URL string `json:"url"`
+}
+
+// buildContentValue converts an llm.Message's Parts into the OpenAI
+// multipart content shape, or falls back to the plain Content string when
+// there are no Parts - existing callers building a plain-text Message are
+// completely unaffected. A video part is rejected outright: OpenAI's chat
+// completions API has no video content type, so sending one would just
+// produce a confusing API error instead of this clear one.
+func buildContentValue(m llm.Message) (interface{}, error) {
+	if len(m.Parts) == 0 {
+		return m.Content, nil
+	}
+	parts := make([]openaiContentPart, 0, len(m.Parts))
+	for _, p := range m.Parts {
+		switch p.Type {
+		case "text":
+			parts = append(parts, openaiContentPart{Type: "text", Text: p.Text})
+		case "image":
+			parts = append(parts, openaiContentPart{Type: "image_url", ImageURL: &openaiMediaURL{URL: p.MediaURL}})
+		case "video":
+			return nil, fmt.Errorf("openai: video attachments are not supported by the chat completions API")
+		default:
+			return nil, fmt.Errorf("openai: unknown content part type %q", p.Type)
+		}
+	}
+	return parts, nil
+}
+
 type openaiTool struct {
 	Type     string          `json:"type"`
 	Function json.RawMessage `json:"function"`

pkg/llm/types.go

@@ -29,12 +29,35 @@ type Message struct {
 	// rejected by) the chat template - the model loses track of what it
 	// already asked for and re-attempts it, or restarts from scratch.
 	ToolCalls []ToolCall `json:"tool_calls,omitempty"`
+	// Parts, when non-empty, carries a multimodal message (text plus
+	// image/video attachments) and takes precedence over Content when a
+	// provider client serializes the wire request. Content should still be
+	// set to a plain-text rendition even when Parts is used, since it's what
+	// storage/logging/history reconstruction read - Parts only matters for
+	// the live request that actually goes out to the model.
+	Parts []ContentPart `json:"parts,omitempty"`
+}
+
+// ContentPart is one piece of a multimodal message.
+type ContentPart struct {
+	// Type is "text", "image", or "video".
+	Type string `json:"type"`
+	// Text is set when Type == "text".
+	Text string `json:"text,omitempty"`
+	// MediaURL is set when Type == "image"/"video": either a data URI
+	// (data:<mime>;base64,<...>) or an http(s) URL.
+	MediaURL string `json:"media_url,omitempty"`
+	// MimeType is the media's MIME type (e.g. "image/png"), split out
+	// separately from MediaURL so providers that need it apart from the data
+	// URI (e.g. Anthropic's base64 media_type field) don't have to re-parse it.
+	MimeType string `json:"mime_type,omitempty"`
 }
 
 // ProviderCapabilities describes what a model supports.
 type ProviderCapabilities struct {
 	SupportsTools    bool `json:"supports_tools"`
 	SupportsVision   bool `json:"supports_vision"`
+	SupportsVideo    bool `json:"supports_video"`
 	SupportsJSON     bool `json:"supports_json"`
 	MaxContextWindow int  `json:"max_context_window"`
 }