diff --git a/pkg/llm/providers/anthropic/client.go b/pkg/llm/providers/anthropic/client.go index a0e9112..ab7648f 100644 --- a/pkg/llm/providers/anthropic/client.go +++ b/pkg/llm/providers/anthropic/client.go @@ -7,39 +7,48 @@ import ( "encoding/json" "fmt" "io" - "net/http" "iter" + "net/http" "strings" "github.com/VictorVargas/rony-llm-agent/pkg/llm" ) const ( - defaultBaseURL = "https://api.anthropic.com/v1" - defaultModel = "claude-opus-4-8" - defaultMaxTokens = 8192 - anthropicVersion = "2023-06-01" + defaultBaseURL = "https://api.anthropic.com/v1" + defaultModel = "claude-opus-4-8" + defaultMaxTokens = 8192 + anthropicVersion = "2023-06-01" + // defaultContextWindow is what Capabilities() reports when + // Config.ContextWindow is unset: 200k tokens, the standard window for + // Claude models. Callers use this number to decide when to compact + // their conversation, so over-reporting it (the old code assumed 1M + // for anything that wasn't haiku) meant compaction fired far too late + // and requests started overflowing the real window. + defaultContextWindow = 200000 ) // Config holds the settings needed to create an Anthropic client. type Config struct { - APIKey string - Model string // defaults to claude-opus-4-8 - BaseURL string // defaults to https://api.anthropic.com/v1 - MaxTokens int // default max_tokens sent on every request (Anthropic requires one); 0 = defaultMaxTokens - Temperature *float32 - TopP *float32 + APIKey string + Model string // defaults to claude-opus-4-8 + BaseURL string // defaults to https://api.anthropic.com/v1 + MaxTokens int // default max_tokens sent on every request (Anthropic requires one); 0 = defaultMaxTokens + ContextWindow int // model's context window in tokens (0 = defaultContextWindow); raise it only for models/plans with an extended window + Temperature *float32 + TopP *float32 } // Client implements llm.LLMClient for Anthropic. type Client struct { - apiKey string - baseURL string - model string - maxTokens int - temperature *float32 - topP *float32 - http *http.Client + apiKey string + baseURL string + model string + maxTokens int + contextWindow int + temperature *float32 + topP *float32 + http *http.Client } // New returns a new Anthropic client. @@ -63,14 +72,20 @@ func New(cfg Config) (*Client, error) { maxTokens = defaultMaxTokens } + contextWindow := cfg.ContextWindow + if contextWindow == 0 { + contextWindow = defaultContextWindow + } + return &Client{ - apiKey: cfg.APIKey, - baseURL: baseURL, - model: model, - maxTokens: maxTokens, - temperature: cfg.Temperature, - topP: cfg.TopP, - http: http.DefaultClient, + apiKey: cfg.APIKey, + baseURL: baseURL, + model: model, + maxTokens: maxTokens, + contextWindow: contextWindow, + temperature: cfg.Temperature, + topP: cfg.TopP, + http: http.DefaultClient, }, nil } @@ -257,15 +272,11 @@ func (c *Client) Name() string { } func (c *Client) Capabilities() llm.ProviderCapabilities { - maxContext := 1000000 - if strings.Contains(c.model, "haiku") { - maxContext = 200000 - } return llm.ProviderCapabilities{ SupportsTools: true, SupportsVision: true, SupportsJSON: true, - MaxContextWindow: maxContext, + MaxContextWindow: c.contextWindow, } } @@ -489,20 +500,20 @@ func mapStopReason(reason string) string { // Anthropic API types type anthropicRequest struct { - Model string `json:"model"` - Messages []anthropicMessage `json:"messages"` - System string `json:"system,omitempty"` - MaxTokens int `json:"max_tokens"` - Tools []anthropicTool `json:"tools,omitempty"` - ToolChoice json.RawMessage `json:"tool_choice,omitempty"` - Temperature *float32 `json:"temperature,omitempty"` - TopP *float32 `json:"top_p,omitempty"` - StopSequences []string `json:"stop_sequences,omitempty"` - Stream bool `json:"stream,omitempty"` + Model string `json:"model"` + Messages []anthropicMessage `json:"messages"` + System string `json:"system,omitempty"` + MaxTokens int `json:"max_tokens"` + Tools []anthropicTool `json:"tools,omitempty"` + ToolChoice json.RawMessage `json:"tool_choice,omitempty"` + Temperature *float32 `json:"temperature,omitempty"` + TopP *float32 `json:"top_p,omitempty"` + StopSequences []string `json:"stop_sequences,omitempty"` + Stream bool `json:"stream,omitempty"` } type anthropicMessage struct { - Role string `json:"role"` + Role string `json:"role"` Content []anthropicContentBlock `json:"content"` } diff --git a/pkg/llm/providers/llamacpp/client.go b/pkg/llm/providers/llamacpp/client.go index 6861ca2..d2a661b 100644 --- a/pkg/llm/providers/llamacpp/client.go +++ b/pkg/llm/providers/llamacpp/client.go @@ -2,12 +2,13 @@ package llamacpp import ( "bufio" + "bytes" "context" "encoding/json" "fmt" "io" - "net/http" "iter" + "net/http" "strings" "time" @@ -24,15 +25,15 @@ const defaultContextWindow = 32768 // Config holds the settings needed to create a llama.cpp client. type Config struct { - BaseURL string // defaults to http://localhost:8080/v1 - Model string - Timeout int // request timeout in seconds (0 = default, no timeout) - ContextWindow int // model's context window in tokens (0 = defaultContextWindow) - MaxTokens int // default max_tokens (0 = defaultMaxTokens) - TopK int // top-k sampling (0 = model/server default) - TopP float32 // nucleus sampling (0 = model/server default) - Temperature float32 - MinP float32 // min-p sampling (llama.cpp extension) + BaseURL string // defaults to http://localhost:8080/v1 + Model string + Timeout int // request timeout in seconds (0 = default, no timeout) + ContextWindow int // model's context window in tokens (0 = defaultContextWindow) + MaxTokens int // default max_tokens (0 = defaultMaxTokens) + TopK int // top-k sampling (0 = model/server default) + TopP float32 // nucleus sampling (0 = model/server default) + Temperature float32 + MinP float32 // min-p sampling (llama.cpp extension) PresencePenalty float32 RepetitionPenalty float32 // sent as the server's `repeat_penalty` field MaxThinkingTokens int // best-effort cap on reasoning tokens; ignored by servers that don't support it @@ -123,7 +124,7 @@ func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.C return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err) } - return c.toResponse(apiResp), nil + return c.toResponse(apiResp) } func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] { @@ -191,6 +192,11 @@ func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq } scanner := bufio.NewScanner(resp.Body) + // A single SSE line can exceed bufio.Scanner's 64KB default cap + // (e.g. a large tool-call arguments delta or a long reasoning + // event), which would kill the stream mid-turn with "token too + // long" — same headroom the anthropic client already reserves. + scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "data: ") { @@ -370,11 +376,16 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader if err != nil { return nil, fmt.Errorf("marshaling request: %w", err) } - return strings.NewReader(string(data)), nil + return bytes.NewReader(data), nil } // toResponse converts a llama.cpp API response to our CompletionResponse. -func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse { +func (c *Client) toResponse(resp llamaChatResponse) (llm.CompletionResponse, error) { + // Guard against a 200 response with no choices (e.g. a misbehaving + // server or proxy) — indexing Choices[0] blindly panics the whole app. + if len(resp.Choices) == 0 { + return llm.CompletionResponse{}, fmt.Errorf("llamacpp: response contained no choices") + } choice := resp.Choices[0] result := llm.CompletionResponse{ ID: resp.ID, @@ -398,32 +409,32 @@ func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse { TotalTokens: resp.Usage.TotalTokens, } - return result + return result, nil } // llama.cpp API types type llamaChatRequest struct { - Model string `json:"model"` - Messages []llamaMessage `json:"messages"` - Tools []llamaTool `json:"tools,omitempty"` - ToolChoice interface{} `json:"tool_choice,omitempty"` - Temperature float32 `json:"temperature,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - TopK int `json:"top_k,omitempty"` - TopP float32 `json:"top_p,omitempty"` - MinP float32 `json:"min_p,omitempty"` - PresencePenalty float32 `json:"presence_penalty,omitempty"` - RepeatPenalty float32 `json:"repeat_penalty,omitempty"` + Model string `json:"model"` + Messages []llamaMessage `json:"messages"` + Tools []llamaTool `json:"tools,omitempty"` + ToolChoice interface{} `json:"tool_choice,omitempty"` + Temperature float32 `json:"temperature,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` + TopK int `json:"top_k,omitempty"` + TopP float32 `json:"top_p,omitempty"` + MinP float32 `json:"min_p,omitempty"` + PresencePenalty float32 `json:"presence_penalty,omitempty"` + RepeatPenalty float32 `json:"repeat_penalty,omitempty"` // MaxThinkingTokens is a best-effort reasoning-token cap: not part of // upstream llama.cpp's server API, but harmless to send since JSON // servers ignore unrecognized fields, and some front-ends (e.g. the // proxy this model's config was written for) do honor it. - MaxThinkingTokens int `json:"max_thinking_tokens,omitempty"` - Stop []string `json:"stop,omitempty"` - Stream bool `json:"stream"` + MaxThinkingTokens int `json:"max_thinking_tokens,omitempty"` + Stop []string `json:"stop,omitempty"` + Stream bool `json:"stream"` StreamOptions *llamaStreamOptions `json:"stream_options,omitempty"` - ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` + ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` } type llamaStreamOptions struct { @@ -444,16 +455,16 @@ type llamaTool struct { } type llamaChatResponse struct { - ID string `json:"id"` - Model string `json:"model"` - Choices []llamaChoice `json:"choices"` - Usage llamaUsage `json:"usage"` + ID string `json:"id"` + Model string `json:"model"` + Choices []llamaChoice `json:"choices"` + Usage llamaUsage `json:"usage"` } type llamaChoice struct { Index int `json:"index"` Message llamaMessageResult `json:"message"` - FinishReason string `json:"finish_reason"` + FinishReason string `json:"finish_reason"` } type llamaMessageResult struct { @@ -464,14 +475,14 @@ type llamaMessageResult struct { } type llamaToolCall struct { - ID string `json:"id"` - Type string `json:"type"` - Function llamaFunction `json:"function"` + ID string `json:"id"` + Type string `json:"type"` + Function llamaFunction `json:"function"` } type llamaFunction struct { - Name string `json:"name"` - Arguments string `json:"arguments"` + Name string `json:"name"` + Arguments string `json:"arguments"` } type llamaUsage struct { @@ -483,32 +494,32 @@ type llamaUsage struct { // Stream event types type llamaStreamEvent struct { - ID string `json:"id"` + ID string `json:"id"` Choices []llamaStreamChoice `json:"choices"` - Usage *llamaUsage `json:"usage"` + Usage *llamaUsage `json:"usage"` } type llamaStreamChoice struct { - Index int `json:"index"` - Delta llamaStreamDelta `json:"delta"` - FinishReason string `json:"finish_reason"` + Index int `json:"index"` + Delta llamaStreamDelta `json:"delta"` + FinishReason string `json:"finish_reason"` } type llamaStreamDelta struct { Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` - Role string `json:"role"` - ToolCalls []llamaStreamToolCall `json:"tool_calls"` + Role string `json:"role"` + ToolCalls []llamaStreamToolCall `json:"tool_calls"` } type llamaStreamToolCall struct { - Index int `json:"index"` - ID string `json:"id"` - Type string `json:"type"` - Function llamaStreamFunction `json:"function"` + Index int `json:"index"` + ID string `json:"id"` + Type string `json:"type"` + Function llamaStreamFunction `json:"function"` } type llamaStreamFunction struct { - Name string `json:"name"` - Arguments string `json:"arguments"` + Name string `json:"name"` + Arguments string `json:"arguments"` }