victorvargas/rony-llm-agent
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(llamacpp,anthropic): SSE headroom, no-choices guard, real context window

Browse source 
- llamacpp: 4MB SSE scanner buffer (the 64KB bufio.Scanner default killed
  streams whose single line exceeded it, e.g. a write tool call carrying a
  whole file) and an empty-choices guard in toResponse instead of a panic;
  request payload now uses bytes.NewReader (drops a full string copy).
- anthropic: Capabilities() reported a 1M-token context window for any
  non-haiku model. Callers use that number to decide when to compact, so
  compaction would have fired far too late and requests overflowed the
  real window. Default is now the standard 200k, configurable via
  Config.ContextWindow for extended-window models/plans.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Victor Hugo Vargas Servin 2026-07-12 16:14:15 -07:00
parent 838eef642a
commit e6830161bc
2 changed files with 115 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

93
pkg/llm/providers/anthropic/client.go
View file

@ -7,39 +7,48 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"net/http"
"iter" "iter"
"net/http"
"strings" "strings"
"github.com/VictorVargas/rony-llm-agent/pkg/llm" "github.com/VictorVargas/rony-llm-agent/pkg/llm"
) )
const ( const (
defaultBaseURL = "https://api.anthropic.com/v1" defaultBaseURL = "https://api.anthropic.com/v1"
defaultModel = "claude-opus-4-8" defaultModel = "claude-opus-4-8"
defaultMaxTokens = 8192 defaultMaxTokens = 8192
anthropicVersion = "2023-06-01" anthropicVersion = "2023-06-01"
// defaultContextWindow is what Capabilities() reports when
// Config.ContextWindow is unset: 200k tokens, the standard window for
// Claude models. Callers use this number to decide when to compact
// their conversation, so over-reporting it (the old code assumed 1M
// for anything that wasn't haiku) meant compaction fired far too late
// and requests started overflowing the real window.
defaultContextWindow = 200000
) )
// Config holds the settings needed to create an Anthropic client. // Config holds the settings needed to create an Anthropic client.
type Config struct { type Config struct {
APIKey string APIKey string
Model string // defaults to claude-opus-4-8 Model string // defaults to claude-opus-4-8
BaseURL string // defaults to https://api.anthropic.com/v1 BaseURL string // defaults to https://api.anthropic.com/v1
MaxTokens int // default max_tokens sent on every request (Anthropic requires one); 0 = defaultMaxTokens MaxTokens int // default max_tokens sent on every request (Anthropic requires one); 0 = defaultMaxTokens
Temperature *float32 ContextWindow int // model's context window in tokens (0 = defaultContextWindow); raise it only for models/plans with an extended window
TopP *float32 Temperature *float32
TopP *float32
} }
// Client implements llm.LLMClient for Anthropic. // Client implements llm.LLMClient for Anthropic.
type Client struct { type Client struct {
apiKey string apiKey string
baseURL string baseURL string
model string model string
maxTokens int maxTokens int
temperature *float32 contextWindow int
topP *float32 temperature *float32
http *http.Client topP *float32
http *http.Client
} }
// New returns a new Anthropic client. // New returns a new Anthropic client.
@ -63,14 +72,20 @@ func New(cfg Config) (*Client, error) {
maxTokens = defaultMaxTokens maxTokens = defaultMaxTokens
} }
contextWindow := cfg.ContextWindow
if contextWindow == 0 {
contextWindow = defaultContextWindow
}
return &Client{ return &Client{
apiKey: cfg.APIKey, apiKey: cfg.APIKey,
baseURL: baseURL, baseURL: baseURL,
model: model, model: model,
maxTokens: maxTokens, maxTokens: maxTokens,
temperature: cfg.Temperature, contextWindow: contextWindow,
topP: cfg.TopP, temperature: cfg.Temperature,
http: http.DefaultClient, topP: cfg.TopP,
http: http.DefaultClient,
}, nil }, nil
} }
@ -257,15 +272,11 @@ func (c *Client) Name() string {
} }
func (c *Client) Capabilities() llm.ProviderCapabilities { func (c *Client) Capabilities() llm.ProviderCapabilities {
maxContext := 1000000
if strings.Contains(c.model, "haiku") {
maxContext = 200000
}
return llm.ProviderCapabilities{ return llm.ProviderCapabilities{
SupportsTools: true, SupportsTools: true,
SupportsVision: true, SupportsVision: true,
SupportsJSON: true, SupportsJSON: true,
MaxContextWindow: maxContext, MaxContextWindow: c.contextWindow,
} }
} }
@ -489,20 +500,20 @@ func mapStopReason(reason string) string {
// Anthropic API types // Anthropic API types
type anthropicRequest struct { type anthropicRequest struct {
Model string `json:"model"` Model string `json:"model"`
Messages []anthropicMessage `json:"messages"` Messages []anthropicMessage `json:"messages"`
System string `json:"system,omitempty"` System string `json:"system,omitempty"`
MaxTokens int `json:"max_tokens"` MaxTokens int `json:"max_tokens"`
Tools []anthropicTool `json:"tools,omitempty"` Tools []anthropicTool `json:"tools,omitempty"`
ToolChoice json.RawMessage `json:"tool_choice,omitempty"` ToolChoice json.RawMessage `json:"tool_choice,omitempty"`
Temperature *float32 `json:"temperature,omitempty"` Temperature *float32 `json:"temperature,omitempty"`
TopP *float32 `json:"top_p,omitempty"` TopP *float32 `json:"top_p,omitempty"`
StopSequences []string `json:"stop_sequences,omitempty"` StopSequences []string `json:"stop_sequences,omitempty"`
Stream bool `json:"stream,omitempty"` Stream bool `json:"stream,omitempty"`
} }
type anthropicMessage struct { type anthropicMessage struct {
Role string `json:"role"` Role string `json:"role"`
Content []anthropicContentBlock `json:"content"` Content []anthropicContentBlock `json:"content"`
} }

115
pkg/llm/providers/llamacpp/client.go
View file

@ -2,12 +2,13 @@ package llamacpp
import ( import (
"bufio" "bufio"
"bytes"
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"net/http"
"iter" "iter"
"net/http"
"strings" "strings"
"time" "time"
@ -24,15 +25,15 @@ const defaultContextWindow = 32768
// Config holds the settings needed to create a llama.cpp client. // Config holds the settings needed to create a llama.cpp client.
type Config struct { type Config struct {
BaseURL string // defaults to http://localhost:8080/v1 BaseURL string // defaults to http://localhost:8080/v1
Model string Model string
Timeout int // request timeout in seconds (0 = default, no timeout) Timeout int // request timeout in seconds (0 = default, no timeout)
ContextWindow int // model's context window in tokens (0 = defaultContextWindow) ContextWindow int // model's context window in tokens (0 = defaultContextWindow)
MaxTokens int // default max_tokens (0 = defaultMaxTokens) MaxTokens int // default max_tokens (0 = defaultMaxTokens)
TopK int // top-k sampling (0 = model/server default) TopK int // top-k sampling (0 = model/server default)
TopP float32 // nucleus sampling (0 = model/server default) TopP float32 // nucleus sampling (0 = model/server default)
Temperature float32 Temperature float32
MinP float32 // min-p sampling (llama.cpp extension) MinP float32 // min-p sampling (llama.cpp extension)
PresencePenalty float32 PresencePenalty float32
RepetitionPenalty float32 // sent as the server's `repeat_penalty` field RepetitionPenalty float32 // sent as the server's `repeat_penalty` field
MaxThinkingTokens int // best-effort cap on reasoning tokens; ignored by servers that don't support it MaxThinkingTokens int // best-effort cap on reasoning tokens; ignored by servers that don't support it
@ -123,7 +124,7 @@ func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.C
return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err) return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err)
} }
return c.toResponse(apiResp), nil return c.toResponse(apiResp)
} }
func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] { func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] {
@ -191,6 +192,11 @@ func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq
} }
scanner := bufio.NewScanner(resp.Body) scanner := bufio.NewScanner(resp.Body)
// A single SSE line can exceed bufio.Scanner's 64KB default cap
// (e.g. a large tool-call arguments delta or a long reasoning
// event), which would kill the stream mid-turn with "token too
// long" — same headroom the anthropic client already reserves.
scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
for scanner.Scan() { for scanner.Scan() {
line := scanner.Text() line := scanner.Text()
if !strings.HasPrefix(line, "data: ") { if !strings.HasPrefix(line, "data: ") {
@ -370,11 +376,16 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader
if err != nil { if err != nil {
return nil, fmt.Errorf("marshaling request: %w", err) return nil, fmt.Errorf("marshaling request: %w", err)
} }
return strings.NewReader(string(data)), nil return bytes.NewReader(data), nil
} }
// toResponse converts a llama.cpp API response to our CompletionResponse. // toResponse converts a llama.cpp API response to our CompletionResponse.
func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse { func (c *Client) toResponse(resp llamaChatResponse) (llm.CompletionResponse, error) {
// Guard against a 200 response with no choices (e.g. a misbehaving
// server or proxy) — indexing Choices[0] blindly panics the whole app.
if len(resp.Choices) == 0 {
return llm.CompletionResponse{}, fmt.Errorf("llamacpp: response contained no choices")
}
choice := resp.Choices[0] choice := resp.Choices[0]
result := llm.CompletionResponse{ result := llm.CompletionResponse{
ID: resp.ID, ID: resp.ID,
@ -398,32 +409,32 @@ func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse {
TotalTokens: resp.Usage.TotalTokens, TotalTokens: resp.Usage.TotalTokens,
} }
return result return result, nil
} }
// llama.cpp API types // llama.cpp API types
type llamaChatRequest struct { type llamaChatRequest struct {
Model string `json:"model"` Model string `json:"model"`
Messages []llamaMessage `json:"messages"` Messages []llamaMessage `json:"messages"`
Tools []llamaTool `json:"tools,omitempty"` Tools []llamaTool `json:"tools,omitempty"`
ToolChoice interface{} `json:"tool_choice,omitempty"` ToolChoice interface{} `json:"tool_choice,omitempty"`
Temperature float32 `json:"temperature,omitempty"` Temperature float32 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"` MaxTokens int `json:"max_tokens,omitempty"`
TopK int `json:"top_k,omitempty"` TopK int `json:"top_k,omitempty"`
TopP float32 `json:"top_p,omitempty"` TopP float32 `json:"top_p,omitempty"`
MinP float32 `json:"min_p,omitempty"` MinP float32 `json:"min_p,omitempty"`
PresencePenalty float32 `json:"presence_penalty,omitempty"` PresencePenalty float32 `json:"presence_penalty,omitempty"`
RepeatPenalty float32 `json:"repeat_penalty,omitempty"` RepeatPenalty float32 `json:"repeat_penalty,omitempty"`
// MaxThinkingTokens is a best-effort reasoning-token cap: not part of // MaxThinkingTokens is a best-effort reasoning-token cap: not part of
// upstream llama.cpp's server API, but harmless to send since JSON // upstream llama.cpp's server API, but harmless to send since JSON
// servers ignore unrecognized fields, and some front-ends (e.g. the // servers ignore unrecognized fields, and some front-ends (e.g. the
// proxy this model's config was written for) do honor it. // proxy this model's config was written for) do honor it.
MaxThinkingTokens int `json:"max_thinking_tokens,omitempty"` MaxThinkingTokens int `json:"max_thinking_tokens,omitempty"`
Stop []string `json:"stop,omitempty"` Stop []string `json:"stop,omitempty"`
Stream bool `json:"stream"` Stream bool `json:"stream"`
StreamOptions *llamaStreamOptions `json:"stream_options,omitempty"` StreamOptions *llamaStreamOptions `json:"stream_options,omitempty"`
ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"`
} }
type llamaStreamOptions struct { type llamaStreamOptions struct {
@ -444,16 +455,16 @@ type llamaTool struct {
} }
type llamaChatResponse struct { type llamaChatResponse struct {
ID string `json:"id"` ID string `json:"id"`
Model string `json:"model"` Model string `json:"model"`
Choices []llamaChoice `json:"choices"` Choices []llamaChoice `json:"choices"`
Usage llamaUsage `json:"usage"` Usage llamaUsage `json:"usage"`
} }
type llamaChoice struct { type llamaChoice struct {
Index int `json:"index"` Index int `json:"index"`
Message llamaMessageResult `json:"message"` Message llamaMessageResult `json:"message"`
FinishReason string `json:"finish_reason"` FinishReason string `json:"finish_reason"`
} }
type llamaMessageResult struct { type llamaMessageResult struct {
@ -464,14 +475,14 @@ type llamaMessageResult struct {
} }
type llamaToolCall struct { type llamaToolCall struct {
ID string `json:"id"` ID string `json:"id"`
Type string `json:"type"` Type string `json:"type"`
Function llamaFunction `json:"function"` Function llamaFunction `json:"function"`
} }
type llamaFunction struct { type llamaFunction struct {
Name string `json:"name"` Name string `json:"name"`
Arguments string `json:"arguments"` Arguments string `json:"arguments"`
} }
type llamaUsage struct { type llamaUsage struct {
@ -483,32 +494,32 @@ type llamaUsage struct {
// Stream event types // Stream event types
type llamaStreamEvent struct { type llamaStreamEvent struct {
ID string `json:"id"` ID string `json:"id"`
Choices []llamaStreamChoice `json:"choices"` Choices []llamaStreamChoice `json:"choices"`
Usage *llamaUsage `json:"usage"` Usage *llamaUsage `json:"usage"`
} }
type llamaStreamChoice struct { type llamaStreamChoice struct {
Index int `json:"index"` Index int `json:"index"`
Delta llamaStreamDelta `json:"delta"` Delta llamaStreamDelta `json:"delta"`
FinishReason string `json:"finish_reason"` FinishReason string `json:"finish_reason"`
} }
type llamaStreamDelta struct { type llamaStreamDelta struct {
Content string `json:"content"` Content string `json:"content"`
ReasoningContent string `json:"reasoning_content"` ReasoningContent string `json:"reasoning_content"`
Role string `json:"role"` Role string `json:"role"`
ToolCalls []llamaStreamToolCall `json:"tool_calls"` ToolCalls []llamaStreamToolCall `json:"tool_calls"`
} }
type llamaStreamToolCall struct { type llamaStreamToolCall struct {
Index int `json:"index"` Index int `json:"index"`
ID string `json:"id"` ID string `json:"id"`
Type string `json:"type"` Type string `json:"type"`
Function llamaStreamFunction `json:"function"` Function llamaStreamFunction `json:"function"`
} }
type llamaStreamFunction struct { type llamaStreamFunction struct {
Name string `json:"name"` Name string `json:"name"`
Arguments string `json:"arguments"` Arguments string `json:"arguments"`
} }