victorvargas/rony-llm-agent
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #10 from VictorVargas/feat/multimodal-content-parts

Browse source 
Add multimodal (image/video) content support
This commit is contained in:
Victor Hugo Vargas Servin 2026-07-17 00:11:09 -07:00 committed by GitHub
commit cde9ac3544
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 213 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pkg/agent/integration_test.go
View file

@ -103,7 +103,7 @@ func TestIntegration_AgentLoop_Generate(t *testing.T) {
MaxIters: 3,
})
resp, err := loop.Run(context.Background(), "What is 20+22? Use the add_numbers tool.")
resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "What is 20+22? Use the add_numbers tool."})
if err != nil {
t.Fatalf("run failed: %v", err)
}
@ -130,7 +130,7 @@ func TestIntegration_AgentLoop_Stream(t *testing.T) {
MaxIters: 3,
})
stream := loop.RunStream(context.Background(), "Say something interesting.")
stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Say something interesting."})
var chunks []llm.StreamChunk
for chunk, err := range stream {

15
pkg/agent/loop.go
View file

@ -77,8 +77,9 @@ func New(cfg Config) *Loop {
// Run executes the agent loop and returns the final response.
// Optional history messages are appended after the system prompt and before
// the new user input.
func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (Response, error) {
// the new user input. input's Role is overwritten to RoleUser regardless of
// what the caller sets, so callers only need to fill in Content/Parts.
func (l *Loop) Run(ctx context.Context, input llm.Message, history ...llm.Message) (Response, error) {
start := time.Now()
messages := l.buildInitialMessages(input, history)
@ -174,8 +175,9 @@ func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (R
// RunStream executes the agent loop with streaming output.
// Optional history messages are appended after the system prompt and before
// the new user input.
func (l *Loop) RunStream(ctx context.Context, input string, history ...llm.Message) iter.Seq2[llm.StreamChunk, error] {
// the new user input. input's Role is overwritten to RoleUser regardless of
// what the caller sets, so callers only need to fill in Content/Parts.
func (l *Loop) RunStream(ctx context.Context, input llm.Message, history ...llm.Message) iter.Seq2[llm.StreamChunk, error] {
return func(yield func(llm.StreamChunk, error) bool) {
messages := l.buildInitialMessages(input, history)
// Same as Run: the schemas are identical on every iteration.
@ -362,12 +364,13 @@ func containsUnparsedToolCall(s string) bool {
return strings.Contains(s, "<tool_call") || strings.Contains(s, "<function=")
}
func (l *Loop) buildInitialMessages(input string, history []llm.Message) []llm.Message {
func (l *Loop) buildInitialMessages(input llm.Message, history []llm.Message) []llm.Message {
systemPrompt := persona.AssembleSystemPrompt(l.cfg.Persona, l.cfg.AgentsMD)
messages := make([]llm.Message, 0, len(history)+2)
messages = append(messages, llm.Message{Role: llm.RoleSystem, Content: systemPrompt})
messages = append(messages, history...)
messages = append(messages, llm.Message{Role: llm.RoleUser, Content: input})
input.Role = llm.RoleUser
messages = append(messages, input)
return messages
}

30
pkg/agent/loop_test.go
View file

@ -102,7 +102,7 @@ func TestRun_NoToolCalls(t *testing.T) {
Tools: tools.NewRegistry(),
})
resp, err := loop.Run(context.Background(), "Hello")
resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Hello"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -133,7 +133,7 @@ func TestRun_IncludesAgentsMD(t *testing.T) {
AgentsMD: "Never edit go.mod directly.",
})
if _, err := loop.Run(context.Background(), "Hello"); err != nil {
if _, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Hello"}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !strings.Contains(capturedSystemPrompt, "Never edit go.mod directly.") {
@ -176,7 +176,7 @@ func TestRun_ToolCalls(t *testing.T) {
Tools: registry,
})
resp, err := loop.Run(context.Background(), "Say hi")
resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Say hi"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -233,7 +233,7 @@ func TestRun_ToolCalls_RecordsAssistantTurnAndToolCallID(t *testing.T) {
Tools: registry,
})
if _, err := loop.Run(context.Background(), "Say hi"); err != nil {
if _, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "Say hi"}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -315,7 +315,7 @@ func TestRun_Stream_ToolCalls_RecordsAssistantTurnAndToolCallID(t *testing.T) {
Tools: registry,
})
for _, err := range loop.RunStream(context.Background(), "test") {
for _, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"}) {
if err != nil {
t.Fatalf("unexpected stream error: %v", err)
}
@ -379,7 +379,7 @@ func TestRun_MaxIterations(t *testing.T) {
MaxIters: 3,
})
_, err := loop.Run(context.Background(), "test")
_, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
if err == nil {
t.Fatal("expected error, got nil")
}
@ -408,7 +408,7 @@ func TestRun_ToolNotFound(t *testing.T) {
Tools: tools.NewRegistry(),
})
resp, err := loop.Run(context.Background(), "test")
resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -451,7 +451,7 @@ func TestRun_ApprovalDenied(t *testing.T) {
},
})
resp, err := loop.Run(context.Background(), "test")
resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -496,7 +496,7 @@ func TestRun_SandboxViolation(t *testing.T) {
},
})
resp, err := loop.Run(context.Background(), "test")
resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -540,7 +540,7 @@ func TestRun_OnIterationHook(t *testing.T) {
},
})
_, err := loop.Run(context.Background(), "test")
_, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -570,7 +570,7 @@ func TestRun_Stream_NoToolCalls(t *testing.T) {
})
var chunks []llm.StreamChunk
stream := loop.RunStream(context.Background(), "test")
stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
for chunk, err := range stream {
if err != nil {
t.Fatalf("unexpected error: %v", err)
@ -608,7 +608,7 @@ func TestRun_Stream_ForwardsTrailingUsageOnlyChunk(t *testing.T) {
})
var chunks []llm.StreamChunk
stream := loop.RunStream(context.Background(), "test")
stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
for chunk, err := range stream {
if err != nil {
t.Fatalf("unexpected error: %v", err)
@ -661,7 +661,7 @@ func TestRun_Stream_WithToolCalls(t *testing.T) {
})
var chunks []llm.StreamChunk
stream := loop.RunStream(context.Background(), "test")
stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
for chunk, err := range stream {
if err != nil {
t.Fatalf("unexpected error: %v", err)
@ -713,7 +713,7 @@ func TestRun_Stream_MaxIterations(t *testing.T) {
})
var chunks []llm.StreamChunk
stream := loop.RunStream(context.Background(), "test")
stream := loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "test"})
for chunk, err := range stream {
if err != nil {
// expect max iterations error
@ -748,7 +748,7 @@ func TestRun_Timeout(t *testing.T) {
Tools: tools.NewRegistry(),
})
_, err := loop.Run(ctx, "test")
_, err := loop.Run(ctx, llm.Message{Role: llm.RoleUser, Content: "test"})
if err == nil {
t.Fatal("expected timeout error, got nil")
}

2
pkg/agent/subagent.go
View file

@ -33,7 +33,7 @@ func (s SubAgent) Run(ctx context.Context, llmClient llm.LLMClient, agentsMD str
if cfg.MaxIters == 0 {
cfg.MaxIters = DefaultMaxIterations
}
return New(cfg).Run(ctx, task)
return New(cfg).Run(ctx, llm.Message{Role: llm.RoleUser, Content: task})
}
// SubAgentRegistry looks up SubAgents by name for the delegate tool.

4
pkg/agent/thinking_budget_test.go
View file

@ -28,7 +28,7 @@ func TestRunStream_RecoversFromThinkingBudgetCut(t *testing.T) {
loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
var final strings.Builder
for chunk, err := range loop.RunStream(context.Background(), "arregla x.py") {
for chunk, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "arregla x.py"}) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -62,7 +62,7 @@ func TestRunStream_ThinkingBudgetNudgeGivesUpAfterLimit(t *testing.T) {
loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
for _, err := range loop.RunStream(context.Background(), "haz algo") {
for _, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "haz algo"}) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

6
pkg/agent/unparsed_toolcall_test.go
View file

@ -97,7 +97,7 @@ func TestRunStream_RecoversFromUnparsedToolCall(t *testing.T) {
loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
var final strings.Builder
for chunk, err := range loop.RunStream(context.Background(), "arregla x.py") {
for chunk, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "arregla x.py"}) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -133,7 +133,7 @@ func TestRunStream_NudgeGivesUpAfterLimit(t *testing.T) {
loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
rounds := 0
for _, err := range loop.RunStream(context.Background(), "haz algo") {
for _, err := range loop.RunStream(context.Background(), llm.Message{Role: llm.RoleUser, Content: "haz algo"}) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -158,7 +158,7 @@ func TestRun_RecoversFromUnparsedToolCall(t *testing.T) {
}}
loop := New(Config{LLM: stub, Tools: editTestRegistry(t, &executed), MaxIters: 10})
resp, err := loop.Run(context.Background(), "arregla x.py")
resp, err := loop.Run(context.Background(), llm.Message{Role: llm.RoleUser, Content: "arregla x.py"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

42
pkg/llm/providers/anthropic/client.go
View file

@ -275,6 +275,7 @@ func (c *Client) Capabilities() llm.ProviderCapabilities {
return llm.ProviderCapabilities{
SupportsTools: true,
SupportsVision: true,
SupportsVideo: false,
SupportsJSON: true,
MaxContextWindow: c.contextWindow,
}
@ -323,7 +324,29 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader
Content: m.Content,
})
case llm.RoleUser:
if len(m.Parts) == 0 {
appendUserBlock(anthropicContentBlock{Type: "text", Text: m.Content})
break
}
for _, p := range m.Parts {
switch p.Type {
case "text":
appendUserBlock(anthropicContentBlock{Type: "text", Text: p.Text})
case "image":
appendUserBlock(anthropicContentBlock{
Type: "image",
Source: &anthropicImageSource{
Type: "base64",
MediaType: p.MimeType,
Data: stripDataURIPrefix(p.MediaURL),
},
})
case "video":
return nil, fmt.Errorf("anthropic: video attachments are not supported by the Messages API")
default:
return nil, fmt.Errorf("anthropic: unknown content part type %q", p.Type)
}
}
case llm.RoleAssistant:
var blocks []anthropicContentBlock
if strings.TrimSpace(m.Content) != "" {
@ -525,6 +548,25 @@ type anthropicContentBlock struct {
Input json.RawMessage `json:"input,omitempty"`
ToolUseID string `json:"tool_use_id,omitempty"`
Content string `json:"content,omitempty"`
Source *anthropicImageSource `json:"source,omitempty"`
}
// anthropicImageSource is an "image" content block's base64-encoded payload.
type anthropicImageSource struct {
Type string `json:"type"` // always "base64"
MediaType string `json:"media_type"`
Data string `json:"data"`
}
// stripDataURIPrefix strips a "data:<mime>;base64," prefix from a data URI,
// leaving just the base64 payload Anthropic's image source expects. Returns
// the input unchanged if it isn't a data URI (e.g. a caller passed a raw
// base64 string directly).
func stripDataURIPrefix(mediaURL string) string {
if idx := strings.Index(mediaURL, ";base64,"); idx != -1 {
return mediaURL[idx+len(";base64,"):]
}
return mediaURL
}
type anthropicTool struct {

56
pkg/llm/providers/llamacpp/client.go
View file

@ -322,7 +322,8 @@ func (c *Client) Name() string {
func (c *Client) Capabilities() llm.ProviderCapabilities {
return llm.ProviderCapabilities{
SupportsTools: true,
SupportsVision: false,
SupportsVision: true,
SupportsVideo: true,
SupportsJSON: true,
MaxContextWindow: c.contextWindow,
}
@ -332,9 +333,13 @@ func (c *Client) Capabilities() llm.ProviderCapabilities {
func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader, error) {
messages := make([]llamaMessage, len(req.Messages))
for i, m := range req.Messages {
content, err := buildContentValue(m)
if err != nil {
return nil, err
}
messages[i] = llamaMessage{
Role: string(m.Role),
Content: m.Content,
Content: content,
ToolCallID: m.ToolCallID,
Name: m.Name,
}
@ -478,12 +483,57 @@ type llamaStreamOptions struct {
type llamaMessage struct {
Role string `json:"role"`
Content string `json:"content"`
// Content is either a plain string (the common case) or a
// []llamaContentPart when the source llm.Message carried Parts - see
// buildContentValue.
Content interface{} `json:"content"`
ToolCallID string `json:"tool_call_id,omitempty"`
Name string `json:"name,omitempty"`
ToolCalls []llamaToolCall `json:"tool_calls,omitempty"`
}
// llamaContentPart is one block of a multipart "content" array, following
// the same OpenAI-compatible shape llama.cpp's server accepts for
// vision-capable models (e.g. Qwen2-VL via its mmproj).
type llamaContentPart struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ImageURL *llamaMediaURL `json:"image_url,omitempty"`
VideoURL *llamaMediaURL `json:"video_url,omitempty"`
}
type llamaMediaURL struct {
URL string `json:"url"`
}
// buildContentValue converts an llm.Message's Parts into the OpenAI-style
// multipart content shape, or falls back to the plain Content string when
// there are no Parts. Unlike the openai/anthropic clients, a video part is
// passed through as a "video_url" block rather than rejected: llama.cpp
// itself has no video support, but this client's whole reason to exist is
// the user's own OpenAI-compatible server sitting in front of a
// video-capable model, so the server - not this client - is what decides
// whether it understands it.
func buildContentValue(m llm.Message) (interface{}, error) {
if len(m.Parts) == 0 {
return m.Content, nil
}
parts := make([]llamaContentPart, 0, len(m.Parts))
for _, p := range m.Parts {
switch p.Type {
case "text":
parts = append(parts, llamaContentPart{Type: "text", Text: p.Text})
case "image":
parts = append(parts, llamaContentPart{Type: "image_url", ImageURL: &llamaMediaURL{URL: p.MediaURL}})
case "video":
parts = append(parts, llamaContentPart{Type: "video_url", VideoURL: &llamaMediaURL{URL: p.MediaURL}})
default:
return nil, fmt.Errorf("llamacpp: unknown content part type %q", p.Type)
}
}
return parts, nil
}
type llamaTool struct {
Type string `json:"type"`
Function json.RawMessage `json:"function"`

7
pkg/llm/providers/llamacpp/client_test.go
View file

@ -25,8 +25,11 @@ func TestClient_Capabilities(t *testing.T) {
if !caps.SupportsTools {
t.Error("expected SupportsTools to be true")
}
if caps.SupportsVision {
t.Error("expected SupportsVision to be false")
if !caps.SupportsVision {
t.Error("expected SupportsVision to be true")
}
if !caps.SupportsVideo {
t.Error("expected SupportsVideo to be true")
}
}

50
pkg/llm/providers/openai/client.go
View file

@ -243,6 +243,7 @@ func (c *Client) Capabilities() llm.ProviderCapabilities {
return llm.ProviderCapabilities{
SupportsTools: true,
SupportsVision: true,
SupportsVideo: false,
SupportsJSON: true,
MaxContextWindow: 128000,
}
@ -253,9 +254,13 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader
// Convert messages to OpenAI format
messages := make([]openaiMessage, len(req.Messages))
for i, m := range req.Messages {
content, err := buildContentValue(m)
if err != nil {
return nil, err
}
messages[i] = openaiMessage{
Role: string(m.Role),
Content: m.Content,
Content: content,
ToolCallID: m.ToolCallID,
Name: m.Name,
}
@ -379,12 +384,53 @@ type openaiStreamOptions struct {
type openaiMessage struct {
Role string `json:"role"`
Content string `json:"content"`
// Content is either a plain string (the common case) or a
// []openaiContentPart when the source llm.Message carried Parts - see
// buildContentValue.
Content interface{} `json:"content"`
ToolCallID string `json:"tool_call_id,omitempty"`
Name string `json:"name,omitempty"`
ToolCalls []openaiToolCall `json:"tool_calls,omitempty"`
}
// openaiContentPart is one block of a multipart "content" array, following
// the same shape OpenAI's vision-capable chat completions endpoint expects.
type openaiContentPart struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ImageURL *openaiMediaURL `json:"image_url,omitempty"`
}
type openaiMediaURL struct {
URL string `json:"url"`
}
// buildContentValue converts an llm.Message's Parts into the OpenAI
// multipart content shape, or falls back to the plain Content string when
// there are no Parts - existing callers building a plain-text Message are
// completely unaffected. A video part is rejected outright: OpenAI's chat
// completions API has no video content type, so sending one would just
// produce a confusing API error instead of this clear one.
func buildContentValue(m llm.Message) (interface{}, error) {
if len(m.Parts) == 0 {
return m.Content, nil
}
parts := make([]openaiContentPart, 0, len(m.Parts))
for _, p := range m.Parts {
switch p.Type {
case "text":
parts = append(parts, openaiContentPart{Type: "text", Text: p.Text})
case "image":
parts = append(parts, openaiContentPart{Type: "image_url", ImageURL: &openaiMediaURL{URL: p.MediaURL}})
case "video":
return nil, fmt.Errorf("openai: video attachments are not supported by the chat completions API")
default:
return nil, fmt.Errorf("openai: unknown content part type %q", p.Type)
}
}
return parts, nil
}
type openaiTool struct {
Type string `json:"type"`
Function json.RawMessage `json:"function"`

23
pkg/llm/types.go
View file

@ -29,12 +29,35 @@ type Message struct {
// rejected by) the chat template - the model loses track of what it
// already asked for and re-attempts it, or restarts from scratch.
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
// Parts, when non-empty, carries a multimodal message (text plus
// image/video attachments) and takes precedence over Content when a
// provider client serializes the wire request. Content should still be
// set to a plain-text rendition even when Parts is used, since it's what
// storage/logging/history reconstruction read - Parts only matters for
// the live request that actually goes out to the model.
Parts []ContentPart `json:"parts,omitempty"`
}
// ContentPart is one piece of a multimodal message.
type ContentPart struct {
// Type is "text", "image", or "video".
Type string `json:"type"`
// Text is set when Type == "text".
Text string `json:"text,omitempty"`
// MediaURL is set when Type == "image"/"video": either a data URI
// (data:<mime>;base64,<...>) or an http(s) URL.
MediaURL string `json:"media_url,omitempty"`
// MimeType is the media's MIME type (e.g. "image/png"), split out
// separately from MediaURL so providers that need it apart from the data
// URI (e.g. Anthropic's base64 media_type field) don't have to re-parse it.
MimeType string `json:"mime_type,omitempty"`
}
// ProviderCapabilities describes what a model supports.
type ProviderCapabilities struct {
SupportsTools bool `json:"supports_tools"`
SupportsVision bool `json:"supports_vision"`
SupportsVideo bool `json:"supports_video"`
SupportsJSON bool `json:"supports_json"`
MaxContextWindow int `json:"max_context_window"`
}