diff --git a/pkg/agent/integration_test.go b/pkg/agent/integration_test.go index 6de55ad..2a2f052 100644 --- a/pkg/agent/integration_test.go +++ b/pkg/agent/integration_test.go @@ -96,10 +96,10 @@ func TestIntegration_AgentLoop_Generate(t *testing.T) { }) loop := agent.New(agent.Config{ - LLM: client, - Persona: persona.DefaultPersona(), - Tools: registry, - Sandbox: &mockSandbox{}, + LLM: client, + Persona: persona.DefaultPersona(), + Tools: registry, + Sandbox: &mockSandbox{}, MaxIters: 3, }) @@ -124,9 +124,9 @@ func TestIntegration_AgentLoop_Stream(t *testing.T) { } loop := agent.New(agent.Config{ - LLM: client, - Persona: persona.DefaultPersona(), - Tools: tools.NewRegistry(), + LLM: client, + Persona: persona.DefaultPersona(), + Tools: tools.NewRegistry(), MaxIters: 3, }) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 59dc93e..f73d882 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -47,19 +47,19 @@ type Config struct { // Iteration represents a single cycle of the agent loop. type Iteration struct { - Number int - ToolCalls []llm.ToolCall - ToolsUsed int - Duration time.Duration + Number int + ToolCalls []llm.ToolCall + ToolsUsed int + Duration time.Duration } // Response is the final output of the agent loop. type Response struct { - Content string - ToolCalls []llm.ToolCall - Iterations int - Duration time.Duration - TokenUsage llm.TokenUsage + Content string + ToolCalls []llm.ToolCall + Iterations int + Duration time.Duration + TokenUsage llm.TokenUsage } // Loop is the main agent loop that orchestrates LLM calls and tool execution. @@ -82,17 +82,21 @@ func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (R start := time.Now() messages := l.buildInitialMessages(input, history) + // Tool schemas don't change between iterations, so build the JSON once + // per Run instead of re-marshaling every tool on every loop pass. + toolSchemas := l.getToolSchemas() var finalContent string var allToolCalls []llm.ToolCall var totalUsage llm.TokenUsage iterations := 0 + completed := false for iterations < l.cfg.MaxIters { iterations++ resp, err := l.cfg.LLM.Generate(ctx, llm.CompletionRequest{ Messages: messages, - Tools: l.getToolSchemas(), + Tools: toolSchemas, ChatTemplateKwargs: l.cfg.ChatTemplateKwargs, }) if err != nil { @@ -105,6 +109,7 @@ func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (R if len(resp.ToolCalls) == 0 { finalContent = resp.Content + completed = true break } @@ -138,16 +143,20 @@ func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (R } duration := time.Since(start) - if iterations >= l.cfg.MaxIters { + // Only report the max-iterations failure when the loop actually ran out + // of budget without producing a final answer — an answer that arrives + // exactly on the last allowed iteration is still a success (the old + // `iterations >= MaxIters` check threw that valid response away). + if !completed { return Response{}, fmt.Errorf("max iterations (%d) reached", l.cfg.MaxIters) } return Response{ - Content: finalContent, - ToolCalls: allToolCalls, - Iterations: iterations, - Duration: duration, - TokenUsage: totalUsage, + Content: finalContent, + ToolCalls: allToolCalls, + Iterations: iterations, + Duration: duration, + TokenUsage: totalUsage, }, nil } @@ -157,6 +166,8 @@ func (l *Loop) Run(ctx context.Context, input string, history ...llm.Message) (R func (l *Loop) RunStream(ctx context.Context, input string, history ...llm.Message) iter.Seq2[llm.StreamChunk, error] { return func(yield func(llm.StreamChunk, error) bool) { messages := l.buildInitialMessages(input, history) + // Same as Run: the schemas are identical on every iteration. + toolSchemas := l.getToolSchemas() iterations := 0 for iterations < l.cfg.MaxIters { @@ -164,7 +175,7 @@ func (l *Loop) RunStream(ctx context.Context, input string, history ...llm.Messa stream := l.cfg.LLM.Stream(ctx, llm.CompletionRequest{ Messages: messages, - Tools: l.getToolSchemas(), + Tools: toolSchemas, ChatTemplateKwargs: l.cfg.ChatTemplateKwargs, }) @@ -226,11 +237,23 @@ func (l *Loop) RunStream(ctx context.Context, input string, history ...llm.Messa // real token counts. hasContent := chunk.Delta != "" || chunk.ReasoningDelta != "" hasUsage := chunk.Usage.TotalTokens > 0 - if !hasToolCalls && (hasContent || hasUsage) { + switch { + case !hasToolCalls && (hasContent || hasUsage): responseBuilder.WriteString(chunk.Delta) if !yield(chunk, nil) { return } + case hasToolCalls && hasUsage: + // The content gate above exists to hide raw provider + // deltas during a tool-call round, but it also swallowed + // that round's token usage — so callers tracking context + // occupancy (e.g. a UI's context bar deciding when to + // compact) only ever saw the usage of the final, + // tool-free round. Forward the usage on its own, + // without the content. + if !yield(llm.StreamChunk{Usage: chunk.Usage}, nil) { + return + } } } diff --git a/pkg/agent/loop_test.go b/pkg/agent/loop_test.go index ccf5931..2135b93 100644 --- a/pkg/agent/loop_test.go +++ b/pkg/agent/loop_test.go @@ -28,8 +28,8 @@ func (m *mockLLM) Stream(ctx context.Context, req llm.CompletionRequest) iter.Se return m.streamFunc(ctx, req) } -func (m *mockLLM) Name() string { return "mock" } -func (m *mockLLM) Capabilities() llm.ProviderCapabilities { return llm.ProviderCapabilities{} } +func (m *mockLLM) Name() string { return "mock" } +func (m *mockLLM) Capabilities() llm.ProviderCapabilities { return llm.ProviderCapabilities{} } type mockSandbox struct { validateFunc func(tool tools.Tool, call llm.ToolCall) error @@ -71,9 +71,9 @@ func TestNew_CustomMaxIters(t *testing.T) { } loop := New(Config{ - LLM: mockClient, - Persona: persona.DefaultPersona(), - Tools: tools.NewRegistry(), + LLM: mockClient, + Persona: persona.DefaultPersona(), + Tools: tools.NewRegistry(), MaxIters: 10, }) @@ -373,9 +373,9 @@ func TestRun_MaxIterations(t *testing.T) { }) loop := New(Config{ - LLM: mockClient, - Persona: persona.DefaultPersona(), - Tools: registry, + LLM: mockClient, + Persona: persona.DefaultPersona(), + Tools: registry, MaxIters: 3, }) @@ -706,9 +706,9 @@ func TestRun_Stream_MaxIterations(t *testing.T) { }) loop := New(Config{ - LLM: mockClient, - Persona: persona.DefaultPersona(), - Tools: registry, + LLM: mockClient, + Persona: persona.DefaultPersona(), + Tools: registry, MaxIters: 1, }) diff --git a/pkg/config/config.go b/pkg/config/config.go index a663a54..8793a31 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,34 +7,34 @@ import ( // ProviderConfig holds provider-specific settings. type ProviderConfig struct { - Type string `yaml:"type"` - Model string `yaml:"model"` - APIKey string `yaml:"api_key"` - BaseURL string `yaml:"base_url,omitempty"` - MaxTokens int `yaml:"max_tokens,omitempty"` + Type string `yaml:"type"` + Model string `yaml:"model"` + APIKey string `yaml:"api_key"` + BaseURL string `yaml:"base_url,omitempty"` + MaxTokens int `yaml:"max_tokens,omitempty"` Temperature float32 `yaml:"temperature,omitempty"` } // ToolPolicy controls which tools are available and their permissions. type ToolPolicy struct { - DefaultPermission string `yaml:"default_permission"` - AllowList []string `yaml:"allow_list,omitempty"` - DenyList []string `yaml:"deny_list,omitempty"` + DefaultPermission string `yaml:"default_permission"` + AllowList []string `yaml:"allow_list,omitempty"` + DenyList []string `yaml:"deny_list,omitempty"` } // LoggingConfig controls logging output. type LoggingConfig struct { - Level string `yaml:"level"` - Format string `yaml:"format"` - Output string `yaml:"output"` + Level string `yaml:"level"` + Format string `yaml:"format"` + Output string `yaml:"output"` } // Config is the top-level configuration for the agent. type Config struct { - Model string `yaml:"model"` - Provider ProviderConfig `yaml:"provider"` - Tools ToolPolicy `yaml:"tools"` - Logging LoggingConfig `yaml:"logging"` + Model string `yaml:"model"` + Provider ProviderConfig `yaml:"provider"` + Tools ToolPolicy `yaml:"tools"` + Logging LoggingConfig `yaml:"logging"` } // Loader is responsible for loading configuration from various sources. diff --git a/pkg/llm/mock/mock.go b/pkg/llm/mock/mock.go index f34f625..4c56987 100644 --- a/pkg/llm/mock/mock.go +++ b/pkg/llm/mock/mock.go @@ -10,9 +10,9 @@ import ( // MockLLMClient is a deterministic implementation of llm.LLMClient for testing. type MockLLMClient struct { - GenerateFunc func(ctx context.Context, req llm.CompletionRequest) (llm.CompletionResponse, error) - StreamFunc func(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] - NameFunc func() string + GenerateFunc func(ctx context.Context, req llm.CompletionRequest) (llm.CompletionResponse, error) + StreamFunc func(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] + NameFunc func() string CapabilitiesFunc func() llm.ProviderCapabilities } diff --git a/pkg/llm/mock/mock_test.go b/pkg/llm/mock/mock_test.go index c049aeb..7a86371 100644 --- a/pkg/llm/mock/mock_test.go +++ b/pkg/llm/mock/mock_test.go @@ -167,7 +167,7 @@ func TestMockLLMClient_MatchResponse(t *testing.T) { client := NewWithMatch([]MatchResponse{ {Match: "hello", Response: "hi there!"}, {Match: "world", Response: "earth"}, - {Match: "*", Response: "default"}, + {Match: "*", Response: "default"}, }) resp, _ := client.Generate(context.Background(), llm.CompletionRequest{ diff --git a/pkg/llm/providers/anthropic/client.go b/pkg/llm/providers/anthropic/client.go index a0e9112..ab7648f 100644 --- a/pkg/llm/providers/anthropic/client.go +++ b/pkg/llm/providers/anthropic/client.go @@ -7,39 +7,48 @@ import ( "encoding/json" "fmt" "io" - "net/http" "iter" + "net/http" "strings" "github.com/VictorVargas/rony-llm-agent/pkg/llm" ) const ( - defaultBaseURL = "https://api.anthropic.com/v1" - defaultModel = "claude-opus-4-8" - defaultMaxTokens = 8192 - anthropicVersion = "2023-06-01" + defaultBaseURL = "https://api.anthropic.com/v1" + defaultModel = "claude-opus-4-8" + defaultMaxTokens = 8192 + anthropicVersion = "2023-06-01" + // defaultContextWindow is what Capabilities() reports when + // Config.ContextWindow is unset: 200k tokens, the standard window for + // Claude models. Callers use this number to decide when to compact + // their conversation, so over-reporting it (the old code assumed 1M + // for anything that wasn't haiku) meant compaction fired far too late + // and requests started overflowing the real window. + defaultContextWindow = 200000 ) // Config holds the settings needed to create an Anthropic client. type Config struct { - APIKey string - Model string // defaults to claude-opus-4-8 - BaseURL string // defaults to https://api.anthropic.com/v1 - MaxTokens int // default max_tokens sent on every request (Anthropic requires one); 0 = defaultMaxTokens - Temperature *float32 - TopP *float32 + APIKey string + Model string // defaults to claude-opus-4-8 + BaseURL string // defaults to https://api.anthropic.com/v1 + MaxTokens int // default max_tokens sent on every request (Anthropic requires one); 0 = defaultMaxTokens + ContextWindow int // model's context window in tokens (0 = defaultContextWindow); raise it only for models/plans with an extended window + Temperature *float32 + TopP *float32 } // Client implements llm.LLMClient for Anthropic. type Client struct { - apiKey string - baseURL string - model string - maxTokens int - temperature *float32 - topP *float32 - http *http.Client + apiKey string + baseURL string + model string + maxTokens int + contextWindow int + temperature *float32 + topP *float32 + http *http.Client } // New returns a new Anthropic client. @@ -63,14 +72,20 @@ func New(cfg Config) (*Client, error) { maxTokens = defaultMaxTokens } + contextWindow := cfg.ContextWindow + if contextWindow == 0 { + contextWindow = defaultContextWindow + } + return &Client{ - apiKey: cfg.APIKey, - baseURL: baseURL, - model: model, - maxTokens: maxTokens, - temperature: cfg.Temperature, - topP: cfg.TopP, - http: http.DefaultClient, + apiKey: cfg.APIKey, + baseURL: baseURL, + model: model, + maxTokens: maxTokens, + contextWindow: contextWindow, + temperature: cfg.Temperature, + topP: cfg.TopP, + http: http.DefaultClient, }, nil } @@ -257,15 +272,11 @@ func (c *Client) Name() string { } func (c *Client) Capabilities() llm.ProviderCapabilities { - maxContext := 1000000 - if strings.Contains(c.model, "haiku") { - maxContext = 200000 - } return llm.ProviderCapabilities{ SupportsTools: true, SupportsVision: true, SupportsJSON: true, - MaxContextWindow: maxContext, + MaxContextWindow: c.contextWindow, } } @@ -489,20 +500,20 @@ func mapStopReason(reason string) string { // Anthropic API types type anthropicRequest struct { - Model string `json:"model"` - Messages []anthropicMessage `json:"messages"` - System string `json:"system,omitempty"` - MaxTokens int `json:"max_tokens"` - Tools []anthropicTool `json:"tools,omitempty"` - ToolChoice json.RawMessage `json:"tool_choice,omitempty"` - Temperature *float32 `json:"temperature,omitempty"` - TopP *float32 `json:"top_p,omitempty"` - StopSequences []string `json:"stop_sequences,omitempty"` - Stream bool `json:"stream,omitempty"` + Model string `json:"model"` + Messages []anthropicMessage `json:"messages"` + System string `json:"system,omitempty"` + MaxTokens int `json:"max_tokens"` + Tools []anthropicTool `json:"tools,omitempty"` + ToolChoice json.RawMessage `json:"tool_choice,omitempty"` + Temperature *float32 `json:"temperature,omitempty"` + TopP *float32 `json:"top_p,omitempty"` + StopSequences []string `json:"stop_sequences,omitempty"` + Stream bool `json:"stream,omitempty"` } type anthropicMessage struct { - Role string `json:"role"` + Role string `json:"role"` Content []anthropicContentBlock `json:"content"` } diff --git a/pkg/llm/providers/llamacpp/client.go b/pkg/llm/providers/llamacpp/client.go index 6861ca2..d2a661b 100644 --- a/pkg/llm/providers/llamacpp/client.go +++ b/pkg/llm/providers/llamacpp/client.go @@ -2,12 +2,13 @@ package llamacpp import ( "bufio" + "bytes" "context" "encoding/json" "fmt" "io" - "net/http" "iter" + "net/http" "strings" "time" @@ -24,15 +25,15 @@ const defaultContextWindow = 32768 // Config holds the settings needed to create a llama.cpp client. type Config struct { - BaseURL string // defaults to http://localhost:8080/v1 - Model string - Timeout int // request timeout in seconds (0 = default, no timeout) - ContextWindow int // model's context window in tokens (0 = defaultContextWindow) - MaxTokens int // default max_tokens (0 = defaultMaxTokens) - TopK int // top-k sampling (0 = model/server default) - TopP float32 // nucleus sampling (0 = model/server default) - Temperature float32 - MinP float32 // min-p sampling (llama.cpp extension) + BaseURL string // defaults to http://localhost:8080/v1 + Model string + Timeout int // request timeout in seconds (0 = default, no timeout) + ContextWindow int // model's context window in tokens (0 = defaultContextWindow) + MaxTokens int // default max_tokens (0 = defaultMaxTokens) + TopK int // top-k sampling (0 = model/server default) + TopP float32 // nucleus sampling (0 = model/server default) + Temperature float32 + MinP float32 // min-p sampling (llama.cpp extension) PresencePenalty float32 RepetitionPenalty float32 // sent as the server's `repeat_penalty` field MaxThinkingTokens int // best-effort cap on reasoning tokens; ignored by servers that don't support it @@ -123,7 +124,7 @@ func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.C return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err) } - return c.toResponse(apiResp), nil + return c.toResponse(apiResp) } func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] { @@ -191,6 +192,11 @@ func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq } scanner := bufio.NewScanner(resp.Body) + // A single SSE line can exceed bufio.Scanner's 64KB default cap + // (e.g. a large tool-call arguments delta or a long reasoning + // event), which would kill the stream mid-turn with "token too + // long" — same headroom the anthropic client already reserves. + scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "data: ") { @@ -370,11 +376,16 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader if err != nil { return nil, fmt.Errorf("marshaling request: %w", err) } - return strings.NewReader(string(data)), nil + return bytes.NewReader(data), nil } // toResponse converts a llama.cpp API response to our CompletionResponse. -func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse { +func (c *Client) toResponse(resp llamaChatResponse) (llm.CompletionResponse, error) { + // Guard against a 200 response with no choices (e.g. a misbehaving + // server or proxy) — indexing Choices[0] blindly panics the whole app. + if len(resp.Choices) == 0 { + return llm.CompletionResponse{}, fmt.Errorf("llamacpp: response contained no choices") + } choice := resp.Choices[0] result := llm.CompletionResponse{ ID: resp.ID, @@ -398,32 +409,32 @@ func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse { TotalTokens: resp.Usage.TotalTokens, } - return result + return result, nil } // llama.cpp API types type llamaChatRequest struct { - Model string `json:"model"` - Messages []llamaMessage `json:"messages"` - Tools []llamaTool `json:"tools,omitempty"` - ToolChoice interface{} `json:"tool_choice,omitempty"` - Temperature float32 `json:"temperature,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - TopK int `json:"top_k,omitempty"` - TopP float32 `json:"top_p,omitempty"` - MinP float32 `json:"min_p,omitempty"` - PresencePenalty float32 `json:"presence_penalty,omitempty"` - RepeatPenalty float32 `json:"repeat_penalty,omitempty"` + Model string `json:"model"` + Messages []llamaMessage `json:"messages"` + Tools []llamaTool `json:"tools,omitempty"` + ToolChoice interface{} `json:"tool_choice,omitempty"` + Temperature float32 `json:"temperature,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` + TopK int `json:"top_k,omitempty"` + TopP float32 `json:"top_p,omitempty"` + MinP float32 `json:"min_p,omitempty"` + PresencePenalty float32 `json:"presence_penalty,omitempty"` + RepeatPenalty float32 `json:"repeat_penalty,omitempty"` // MaxThinkingTokens is a best-effort reasoning-token cap: not part of // upstream llama.cpp's server API, but harmless to send since JSON // servers ignore unrecognized fields, and some front-ends (e.g. the // proxy this model's config was written for) do honor it. - MaxThinkingTokens int `json:"max_thinking_tokens,omitempty"` - Stop []string `json:"stop,omitempty"` - Stream bool `json:"stream"` + MaxThinkingTokens int `json:"max_thinking_tokens,omitempty"` + Stop []string `json:"stop,omitempty"` + Stream bool `json:"stream"` StreamOptions *llamaStreamOptions `json:"stream_options,omitempty"` - ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` + ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` } type llamaStreamOptions struct { @@ -444,16 +455,16 @@ type llamaTool struct { } type llamaChatResponse struct { - ID string `json:"id"` - Model string `json:"model"` - Choices []llamaChoice `json:"choices"` - Usage llamaUsage `json:"usage"` + ID string `json:"id"` + Model string `json:"model"` + Choices []llamaChoice `json:"choices"` + Usage llamaUsage `json:"usage"` } type llamaChoice struct { Index int `json:"index"` Message llamaMessageResult `json:"message"` - FinishReason string `json:"finish_reason"` + FinishReason string `json:"finish_reason"` } type llamaMessageResult struct { @@ -464,14 +475,14 @@ type llamaMessageResult struct { } type llamaToolCall struct { - ID string `json:"id"` - Type string `json:"type"` - Function llamaFunction `json:"function"` + ID string `json:"id"` + Type string `json:"type"` + Function llamaFunction `json:"function"` } type llamaFunction struct { - Name string `json:"name"` - Arguments string `json:"arguments"` + Name string `json:"name"` + Arguments string `json:"arguments"` } type llamaUsage struct { @@ -483,32 +494,32 @@ type llamaUsage struct { // Stream event types type llamaStreamEvent struct { - ID string `json:"id"` + ID string `json:"id"` Choices []llamaStreamChoice `json:"choices"` - Usage *llamaUsage `json:"usage"` + Usage *llamaUsage `json:"usage"` } type llamaStreamChoice struct { - Index int `json:"index"` - Delta llamaStreamDelta `json:"delta"` - FinishReason string `json:"finish_reason"` + Index int `json:"index"` + Delta llamaStreamDelta `json:"delta"` + FinishReason string `json:"finish_reason"` } type llamaStreamDelta struct { Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` - Role string `json:"role"` - ToolCalls []llamaStreamToolCall `json:"tool_calls"` + Role string `json:"role"` + ToolCalls []llamaStreamToolCall `json:"tool_calls"` } type llamaStreamToolCall struct { - Index int `json:"index"` - ID string `json:"id"` - Type string `json:"type"` - Function llamaStreamFunction `json:"function"` + Index int `json:"index"` + ID string `json:"id"` + Type string `json:"type"` + Function llamaStreamFunction `json:"function"` } type llamaStreamFunction struct { - Name string `json:"name"` - Arguments string `json:"arguments"` + Name string `json:"name"` + Arguments string `json:"arguments"` } diff --git a/pkg/llm/providers/llamacpp/client_test.go b/pkg/llm/providers/llamacpp/client_test.go index 7e66c3b..4de3eed 100644 --- a/pkg/llm/providers/llamacpp/client_test.go +++ b/pkg/llm/providers/llamacpp/client_test.go @@ -34,8 +34,8 @@ func TestClient_Generate(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(llamaChatResponse{ - ID: "llama-123", - Model: "llama3", + ID: "llama-123", + Model: "llama3", Choices: []llamaChoice{ { Index: 0, @@ -189,8 +189,8 @@ func TestClient_Generate_ToolCall(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(llamaChatResponse{ - ID: "llama-tool-1", - Model: "llama3", + ID: "llama-tool-1", + Model: "llama3", Choices: []llamaChoice{ { Index: 0, diff --git a/pkg/llm/providers/openai/client.go b/pkg/llm/providers/openai/client.go index 291e179..fbc0079 100644 --- a/pkg/llm/providers/openai/client.go +++ b/pkg/llm/providers/openai/client.go @@ -2,12 +2,13 @@ package openai import ( "bufio" + "bytes" "context" "encoding/json" "fmt" "io" - "net/http" "iter" + "net/http" "strings" "github.com/VictorVargas/rony-llm-agent/pkg/llm" @@ -15,14 +16,15 @@ import ( // Config holds the settings needed to create an OpenAI client. type Config struct { - APIKey string - Model string - BaseURL string // defaults to https://api.openai.com/v1 + APIKey string + Model string + BaseURL string // defaults to https://api.openai.com/v1 } // Client implements llm.LLMClient for OpenAI. type Client struct { apiKey string + model string baseURL string http *http.Client } @@ -40,6 +42,7 @@ func New(cfg Config) (*Client, error) { return &Client{ apiKey: cfg.APIKey, + model: cfg.Model, baseURL: baseURL, http: http.DefaultClient, }, nil @@ -48,7 +51,7 @@ func New(cfg Config) (*Client, error) { func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.CompletionResponse, error) { endpoint := c.baseURL + "/chat/completions" - payload, err := c.buildRequest(req) + payload, err := c.buildRequest(req, false) if err != nil { return llm.CompletionResponse{}, fmt.Errorf("building request: %w", err) } @@ -76,14 +79,14 @@ func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.C return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err) } - return c.toResponse(apiResp), nil + return c.toResponse(apiResp) } func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] { return func(yield func(llm.StreamChunk, error) bool) { endpoint := c.baseURL + "/chat/completions" - payload, err := c.buildRequest(req) + payload, err := c.buildRequest(req, true) if err != nil { yield(llm.StreamChunk{}, fmt.Errorf("building request: %w", err)) return @@ -111,7 +114,45 @@ func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq return } + // toolCallAccum buffers one tool call's fragments as they stream in: + // the SSE format sends the id/name in the first delta for a given + // tool-call index and the (potentially large) arguments JSON in + // pieces across many subsequent deltas, so it can't be handed to a + // tool handler until it's fully assembled. Same accumulation the + // llamacpp client does — without it, tool calls made over a stream + // were silently dropped and the agent loop never executed them. + type toolCallAccum struct { + id string + name string + args strings.Builder + } + toolCallFrags := map[int]*toolCallAccum{} + var toolCallOrder []int + + flushToolCalls := func() []llm.ToolCall { + if len(toolCallOrder) == 0 { + return nil + } + calls := make([]llm.ToolCall, 0, len(toolCallOrder)) + for _, idx := range toolCallOrder { + frag := toolCallFrags[idx] + calls = append(calls, llm.ToolCall{ + ID: frag.id, + Name: frag.name, + Arguments: json.RawMessage(frag.args.String()), + }) + } + toolCallFrags = map[int]*toolCallAccum{} + toolCallOrder = nil + return calls + } + scanner := bufio.NewScanner(resp.Body) + // A single SSE line can exceed bufio.Scanner's 64KB default cap + // (e.g. a large tool-call arguments delta), which would kill the + // stream with "token too long" — same headroom the anthropic + // client already reserves. + scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "data: ") { @@ -128,13 +169,61 @@ func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq return } + var usage llm.TokenUsage + if event.Usage != nil { + usage = llm.TokenUsage{ + InputTokens: event.Usage.PromptTokens, + OutputTokens: event.Usage.CompletionTokens, + TotalTokens: event.Usage.TotalTokens, + } + } + + if len(event.Choices) == 0 { + // The usage-only event (per stream_options.include_usage) + // carries no choices, so it needs its own chunk. + if event.Usage != nil { + if !yield(llm.StreamChunk{Usage: usage}, nil) { + return + } + } + continue + } + for _, choice := range event.Choices { + hasFragment := len(choice.Delta.ToolCalls) > 0 + for _, tc := range choice.Delta.ToolCalls { + frag, ok := toolCallFrags[tc.Index] + if !ok { + frag = &toolCallAccum{} + toolCallFrags[tc.Index] = frag + toolCallOrder = append(toolCallOrder, tc.Index) + } + if tc.ID != "" { + frag.id = tc.ID + } + if tc.Function.Name != "" { + frag.name = tc.Function.Name + } + frag.args.WriteString(tc.Function.Arguments) + } + chunk := llm.StreamChunk{ Delta: choice.Delta.Content, + Usage: usage, } if choice.FinishReason != "" { chunk.FinishReason = choice.FinishReason + chunk.ToolCalls = flushToolCalls() } + + // A fragment-only event (a piece of a tool call's streamed + // arguments, with nothing else in this delta) has nothing + // yet for the agent loop to act on: it was buffered above, + // so skip yielding an empty chunk for it. + if hasFragment && chunk.Delta == "" && chunk.FinishReason == "" { + continue + } + if !yield(chunk, nil) { return } @@ -160,7 +249,7 @@ func (c *Client) Capabilities() llm.ProviderCapabilities { } // buildRequest converts an llm.CompletionRequest to the OpenAI API format. -func (c *Client) buildRequest(req llm.CompletionRequest) (io.Reader, error) { +func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader, error) { // Convert messages to OpenAI format messages := make([]openaiMessage, len(req.Messages)) for i, m := range req.Messages { @@ -195,10 +284,25 @@ func (c *Client) buildRequest(req llm.CompletionRequest) (io.Reader, error) { tools[i] = tool } + // The per-request model wins when set; otherwise fall back to the + // client's configured one (Config.Model used to be discarded entirely, + // so every request went out with an empty model — a hard API error on + // OpenAI, and the agent loop never sets req.Model). + model := req.Model + if model == "" { + model = c.model + } + openaiReq := openaiChatRequest{ - Model: req.Model, + Model: model, Messages: messages, - Stream: false, + Stream: stream, + } + if stream { + // Ask for a final SSE event carrying token usage (OpenAI-style + // streaming omits it otherwise), so callers can track real token + // counts per turn instead of always seeing zero. + openaiReq.StreamOptions = &openaiStreamOptions{IncludeUsage: true} } if len(tools) > 0 { openaiReq.Tools = tools @@ -222,11 +326,14 @@ func (c *Client) buildRequest(req llm.CompletionRequest) (io.Reader, error) { if err != nil { return nil, fmt.Errorf("marshaling request: %w", err) } - return strings.NewReader(string(data)), nil + return bytes.NewReader(data), nil } // toResponse converts an OpenAI API response to our CompletionResponse. -func (c *Client) toResponse(resp openaiChatResponse) llm.CompletionResponse { +func (c *Client) toResponse(resp openaiChatResponse) (llm.CompletionResponse, error) { + if len(resp.Choices) == 0 { + return llm.CompletionResponse{}, fmt.Errorf("openai: response contained no choices") + } choice := resp.Choices[0] result := llm.CompletionResponse{ ID: resp.ID, @@ -249,63 +356,68 @@ func (c *Client) toResponse(resp openaiChatResponse) llm.CompletionResponse { TotalTokens: resp.Usage.TotalTokens, } - return result + return result, nil } // OpenAI API types type openaiChatRequest struct { - Model string `json:"model"` - Messages []openaiMessage `json:"messages"` - Tools []openaiTool `json:"tools,omitempty"` - ToolChoice interface{} `json:"tool_choice,omitempty"` - Temperature *float32 `json:"temperature,omitempty"` - MaxTokens *int `json:"max_tokens,omitempty"` - Stop []string `json:"stop,omitempty"` - Stream bool `json:"stream"` + Model string `json:"model"` + Messages []openaiMessage `json:"messages"` + Tools []openaiTool `json:"tools,omitempty"` + ToolChoice interface{} `json:"tool_choice,omitempty"` + Temperature *float32 `json:"temperature,omitempty"` + MaxTokens *int `json:"max_tokens,omitempty"` + Stop []string `json:"stop,omitempty"` + Stream bool `json:"stream"` + StreamOptions *openaiStreamOptions `json:"stream_options,omitempty"` +} + +type openaiStreamOptions struct { + IncludeUsage bool `json:"include_usage"` } type openaiMessage struct { - Role string `json:"role"` - Content string `json:"content"` - ToolCallID string `json:"tool_call_id,omitempty"` - Name string `json:"name,omitempty"` - ToolCalls []openaiToolCall `json:"tool_calls,omitempty"` + Role string `json:"role"` + Content string `json:"content"` + ToolCallID string `json:"tool_call_id,omitempty"` + Name string `json:"name,omitempty"` + ToolCalls []openaiToolCall `json:"tool_calls,omitempty"` } type openaiTool struct { - Type string `json:"type"` - Function json.RawMessage `json:"function"` + Type string `json:"type"` + Function json.RawMessage `json:"function"` } type openaiChatResponse struct { - ID string `json:"id"` - Model string `json:"model"` - Choices []openaiChoice `json:"choices"` - Usage openaiUsage `json:"usage"` + ID string `json:"id"` + Model string `json:"model"` + Choices []openaiChoice `json:"choices"` + Usage openaiUsage `json:"usage"` } type openaiChoice struct { Index int `json:"index"` Message openaiMessageResult `json:"message"` - FinishReason string `json:"finish_reason"` + FinishReason string `json:"finish_reason"` } type openaiMessageResult struct { - Role string `json:"role"` - Content string `json:"content"` - ToolCalls []openaiToolCall `json:"tool_calls"` + Role string `json:"role"` + Content string `json:"content"` + ToolCalls []openaiToolCall `json:"tool_calls"` } type openaiToolCall struct { - ID string `json:"id"` - Type string `json:"type"` - Function openaiFunction `json:"function"` + ID string `json:"id"` + Type string `json:"type"` + Function openaiFunction `json:"function"` } type openaiFunction struct { - Name string `json:"name"` - Arguments string `json:"arguments"` + Name string `json:"name"` + Arguments string `json:"arguments"` } type openaiUsage struct { @@ -317,30 +429,31 @@ type openaiUsage struct { // Stream event types type openaiStreamEvent struct { - ID string `json:"id"` - Choices []openaiStreamChoice `json:"choices"` + ID string `json:"id"` + Choices []openaiStreamChoice `json:"choices"` + Usage *openaiUsage `json:"usage"` } type openaiStreamChoice struct { - Index int `json:"index"` - Delta openaiStreamDelta `json:"delta"` - FinishReason string `json:"finish_reason"` + Index int `json:"index"` + Delta openaiStreamDelta `json:"delta"` + FinishReason string `json:"finish_reason"` } type openaiStreamDelta struct { - Content string `json:"content"` - Role string `json:"role"` + Content string `json:"content"` + Role string `json:"role"` ToolCalls []openaiStreamToolCall `json:"tool_calls"` } type openaiStreamToolCall struct { - Index int `json:"index"` - ID string `json:"id"` - Type string `json:"type"` + Index int `json:"index"` + ID string `json:"id"` + Type string `json:"type"` Function openaiStreamFunction `json:"function"` } type openaiStreamFunction struct { - Name string `json:"name"` - Arguments string `json:"arguments"` + Name string `json:"name"` + Arguments string `json:"arguments"` } diff --git a/pkg/llm/providers/openai/client_test.go b/pkg/llm/providers/openai/client_test.go index dd838f6..04c7040 100644 --- a/pkg/llm/providers/openai/client_test.go +++ b/pkg/llm/providers/openai/client_test.go @@ -33,8 +33,8 @@ func TestClient_Generate(t *testing.T) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(openaiChatResponse{ - ID: "test-123", - Model: "gpt-4", + ID: "test-123", + Model: "gpt-4", Choices: []openaiChoice{ { Index: 0, @@ -55,8 +55,8 @@ func TestClient_Generate(t *testing.T) { defer server.Close() client, err := New(Config{ - APIKey: "test-key", - BaseURL: server.URL, + APIKey: "test-key", + BaseURL: server.URL, }) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -90,8 +90,8 @@ func TestClient_Generate_Error(t *testing.T) { defer server.Close() client, err := New(Config{ - APIKey: "bad-key", - BaseURL: server.URL, + APIKey: "bad-key", + BaseURL: server.URL, }) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -113,8 +113,8 @@ func TestClient_Stream(t *testing.T) { defer server.Close() client, err := New(Config{ - APIKey: "test-key", - BaseURL: server.URL + "/v1", + APIKey: "test-key", + BaseURL: server.URL + "/v1", }) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/pkg/llm/types.go b/pkg/llm/types.go index 3e571de..7c8df57 100644 --- a/pkg/llm/types.go +++ b/pkg/llm/types.go @@ -10,18 +10,18 @@ import ( type Role string const ( - RoleSystem Role = "system" - RoleUser Role = "user" + RoleSystem Role = "system" + RoleUser Role = "user" RoleAssistant Role = "assistant" - RoleTool Role = "tool" + RoleTool Role = "tool" ) // Message is a single message in a conversation. type Message struct { - Role Role `json:"role"` - Content string `json:"content"` - ToolCallID string `json:"tool_call_id,omitempty"` - Name string `json:"name,omitempty"` + Role Role `json:"role"` + Content string `json:"content"` + ToolCallID string `json:"tool_call_id,omitempty"` + Name string `json:"name,omitempty"` // ToolCalls records the calls an assistant message requested, so the // agent loop can replay them on the next request: without this, the // conversation sent back to the model has tool-result messages with no @@ -50,8 +50,8 @@ type TokenUsage struct { type ToolChoice string const ( - ToolChoiceAuto ToolChoice = "auto" - ToolChoiceNone ToolChoice = "none" + ToolChoiceAuto ToolChoice = "auto" + ToolChoiceNone ToolChoice = "none" ToolChoiceRequired ToolChoice = "required" ) @@ -73,14 +73,14 @@ func (t *ToolRef) MarshalJSON() ([]byte, error) { // CompletionRequest is sent to an LLM provider. type CompletionRequest struct { - Model string `json:"model"` - Messages []Message `json:"messages"` - Tools []json.RawMessage `json:"tools,omitempty"` - ToolChoice interface{} `json:"tool_choice,omitempty"` // ToolChoice, ToolRef, or null - Temperature *float32 `json:"temperature,omitempty"` - MaxTokens *int `json:"max_tokens,omitempty"` - Stop []string `json:"stop,omitempty"` - Metadata map[string]string `json:"metadata,omitempty"` + Model string `json:"model"` + Messages []Message `json:"messages"` + Tools []json.RawMessage `json:"tools,omitempty"` + ToolChoice interface{} `json:"tool_choice,omitempty"` // ToolChoice, ToolRef, or null + Temperature *float32 `json:"temperature,omitempty"` + MaxTokens *int `json:"max_tokens,omitempty"` + Stop []string `json:"stop,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` // model-specific chat template params, e.g. Qwen enable_thinking } @@ -98,9 +98,9 @@ type CompletionResponse struct { // StopReason values. const ( StopReasonEndTurn = "end_turn" - StopReasonToolUse = "tool_use" + StopReasonToolUse = "tool_use" StopReasonMaxTokens = "max_tokens" - StopReasonStopSeq = "stop_sequence" + StopReasonStopSeq = "stop_sequence" ) // ToolCall represents a function invocation requested by the model. diff --git a/pkg/llm/types_test.go b/pkg/llm/types_test.go index 6747890..0359137 100644 --- a/pkg/llm/types_test.go +++ b/pkg/llm/types_test.go @@ -121,7 +121,7 @@ func TestToolCall_JSON(t *testing.T) { func TestStreamChunk_JSON(t *testing.T) { chunk := StreamChunk{ - Delta: "hello", + Delta: "hello", ToolCalls: []ToolCall{}, } @@ -139,7 +139,5 @@ func TestStreamChunk_JSON(t *testing.T) { } } - - -func float32Ptr(f float32) *float32 { return &f } +func float32Ptr(f float32) *float32 { return &f } func intPtr(i int) *int { return &i } diff --git a/pkg/persona/persona.go b/pkg/persona/persona.go index c092b23..b8a72a1 100644 --- a/pkg/persona/persona.go +++ b/pkg/persona/persona.go @@ -18,7 +18,7 @@ type Persona struct { Style string Language string Constraints []string - FewShot []llm.Message + FewShot []llm.Message } // Loader loads personas from files. diff --git a/pkg/rag/backends/chroma/chroma.go b/pkg/rag/backends/chroma/chroma.go index 3900cb2..586b313 100644 --- a/pkg/rag/backends/chroma/chroma.go +++ b/pkg/rag/backends/chroma/chroma.go @@ -224,13 +224,13 @@ func stringifyMap(m map[string]interface{}) map[string]string { // chromaQueryResponse represents the structure of a ChromaDB query response. type chromaQueryResponse struct { - Names []string `json:"names"` + Names []string `json:"names"` Results []chromaQueryResults `json:"results"` } type chromaQueryResults struct { - IDs [][]string `json:"ids"` - Documents [][]string `json:"documents"` - Distances [][]float64 `json:"distances"` + IDs [][]string `json:"ids"` + Documents [][]string `json:"documents"` + Distances [][]float64 `json:"distances"` Metadatas [][]map[string]interface{} `json:"metadatas"` } diff --git a/pkg/rag/backends/chroma/chroma_test.go b/pkg/rag/backends/chroma/chroma_test.go index dfe6324..03101a4 100644 --- a/pkg/rag/backends/chroma/chroma_test.go +++ b/pkg/rag/backends/chroma/chroma_test.go @@ -54,7 +54,7 @@ func TestBackend_Search(t *testing.T) { meta := []map[string]interface{}{{"key": "value"}} metaNested := [][]map[string]interface{}{meta} mockResponse := map[string]interface{}{ - "names": []string{"rony-memory"}, + "names": []string{"rony-memory"}, "results": []map[string]interface{}{ { "ids": [][]string{{"test-id"}}, diff --git a/pkg/rag/memory_test.go b/pkg/rag/memory_test.go index dbe77f8..7bda2d4 100644 --- a/pkg/rag/memory_test.go +++ b/pkg/rag/memory_test.go @@ -193,9 +193,9 @@ func TestMemory_Search_EmbeddingErrorFallsBackToLexicalSearch(t *testing.T) { // mockBackend implements chroma.Backend for testing. type mockBackend struct { - upsertFunc func(ctx context.Context, id string, vector []float32, content string, metadata map[string]string) error - searchFunc func(ctx context.Context, query string, queryVector []float32, topK int) ([]rag.SearchResult, error) - forgetAllFunc func(ctx context.Context) error + upsertFunc func(ctx context.Context, id string, vector []float32, content string, metadata map[string]string) error + searchFunc func(ctx context.Context, query string, queryVector []float32, topK int) ([]rag.SearchResult, error) + forgetAllFunc func(ctx context.Context) error } func (m *mockBackend) Upsert(ctx context.Context, id string, vector []float32, content string, metadata map[string]string) error { diff --git a/pkg/tools/registry.go b/pkg/tools/registry.go index dc64673..955c66f 100644 --- a/pkg/tools/registry.go +++ b/pkg/tools/registry.go @@ -6,9 +6,9 @@ import ( // registry is the default implementation of Registry. type registry struct { - mu sync.RWMutex - tools map[string]Tool - order []string + mu sync.RWMutex + tools map[string]Tool + order []string } // NewRegistry returns a new empty registry. diff --git a/pkg/tools/types.go b/pkg/tools/types.go index 1db51cd..dae7c05 100644 --- a/pkg/tools/types.go +++ b/pkg/tools/types.go @@ -45,10 +45,10 @@ type ToolHandler func(ctx context.Context, args json.RawMessage) (ToolResult, er // ToolResult is returned by a ToolHandler. type ToolResult struct { - Content string - IsError bool - Metadata map[string]string - Artifacts []Artifact + Content string + IsError bool + Metadata map[string]string + Artifacts []Artifact } // Artifact represents a file or data artifact produced by a tool. @@ -60,8 +60,8 @@ type Artifact struct { // ToolExample provides few-shot examples for the LLM to improve tool usage. type ToolExample struct { - Input map[string]interface{} - Output string + Input map[string]interface{} + Output string } // Registry manages tool registration and lookup.