victorvargas/rony-llm-agent
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix(llamacpp,anthropic): SSE headroom, no-choices guard, real context window

Browse source 
- llamacpp: 4MB SSE scanner buffer (the 64KB bufio.Scanner default killed
  streams whose single line exceeded it, e.g. a write tool call carrying a
  whole file) and an empty-choices guard in toResponse instead of a panic;
  request payload now uses bytes.NewReader (drops a full string copy).
- anthropic: Capabilities() reported a 1M-token context window for any
  non-haiku model. Callers use that number to decide when to compact, so
  compaction would have fired far too late and requests overflowed the
  real window. Default is now the standard 200k, configurable via
  Config.ContextWindow for extended-window models/plans.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Victor Hugo Vargas Servin 2026-07-12 16:14:15 -07:00
parent 838eef642a
commit e6830161bc
2 changed files with 115 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

93
pkg/llm/providers/anthropic/client.go
View file

@ -7,39 +7,48 @@ import (
"encoding/json"
"fmt"
"io"
"net/http"
"iter"
"net/http"
"strings"
"github.com/VictorVargas/rony-llm-agent/pkg/llm"
)
const (
defaultBaseURL = "https://api.anthropic.com/v1"
defaultModel = "claude-opus-4-8"
defaultMaxTokens = 8192
anthropicVersion = "2023-06-01"
defaultBaseURL = "https://api.anthropic.com/v1"
defaultModel = "claude-opus-4-8"
defaultMaxTokens = 8192
anthropicVersion = "2023-06-01"
// defaultContextWindow is what Capabilities() reports when
// Config.ContextWindow is unset: 200k tokens, the standard window for
// Claude models. Callers use this number to decide when to compact
// their conversation, so over-reporting it (the old code assumed 1M
// for anything that wasn't haiku) meant compaction fired far too late
// and requests started overflowing the real window.
defaultContextWindow = 200000
)
// Config holds the settings needed to create an Anthropic client.
type Config struct {
APIKey string
Model string // defaults to claude-opus-4-8
BaseURL string // defaults to https://api.anthropic.com/v1
MaxTokens int // default max_tokens sent on every request (Anthropic requires one); 0 = defaultMaxTokens
Temperature *float32
TopP *float32
APIKey string
Model string // defaults to claude-opus-4-8
BaseURL string // defaults to https://api.anthropic.com/v1
MaxTokens int // default max_tokens sent on every request (Anthropic requires one); 0 = defaultMaxTokens
ContextWindow int // model's context window in tokens (0 = defaultContextWindow); raise it only for models/plans with an extended window
Temperature *float32
TopP *float32
}
// Client implements llm.LLMClient for Anthropic.
type Client struct {
apiKey string
baseURL string
model string
maxTokens int
temperature *float32
topP *float32
http *http.Client
apiKey string
baseURL string
model string
maxTokens int
contextWindow int
temperature *float32
topP *float32
http *http.Client
}
// New returns a new Anthropic client.
@ -63,14 +72,20 @@ func New(cfg Config) (*Client, error) {
maxTokens = defaultMaxTokens
}
contextWindow := cfg.ContextWindow
if contextWindow == 0 {
contextWindow = defaultContextWindow
}
return &Client{
apiKey: cfg.APIKey,
baseURL: baseURL,
model: model,
maxTokens: maxTokens,
temperature: cfg.Temperature,
topP: cfg.TopP,
http: http.DefaultClient,
apiKey: cfg.APIKey,
baseURL: baseURL,
model: model,
maxTokens: maxTokens,
contextWindow: contextWindow,
temperature: cfg.Temperature,
topP: cfg.TopP,
http: http.DefaultClient,
}, nil
}
@ -257,15 +272,11 @@ func (c *Client) Name() string {
}
func (c *Client) Capabilities() llm.ProviderCapabilities {
maxContext := 1000000
if strings.Contains(c.model, "haiku") {
maxContext = 200000
}
return llm.ProviderCapabilities{
SupportsTools: true,
SupportsVision: true,
SupportsJSON: true,
MaxContextWindow: maxContext,
MaxContextWindow: c.contextWindow,
}
}
@ -489,20 +500,20 @@ func mapStopReason(reason string) string {
// Anthropic API types
type anthropicRequest struct {
Model string `json:"model"`
Messages []anthropicMessage `json:"messages"`
System string `json:"system,omitempty"`
MaxTokens int `json:"max_tokens"`
Tools []anthropicTool `json:"tools,omitempty"`
ToolChoice json.RawMessage `json:"tool_choice,omitempty"`
Temperature *float32 `json:"temperature,omitempty"`
TopP *float32 `json:"top_p,omitempty"`
StopSequences []string `json:"stop_sequences,omitempty"`
Stream bool `json:"stream,omitempty"`
Model string `json:"model"`
Messages []anthropicMessage `json:"messages"`
System string `json:"system,omitempty"`
MaxTokens int `json:"max_tokens"`
Tools []anthropicTool `json:"tools,omitempty"`
ToolChoice json.RawMessage `json:"tool_choice,omitempty"`
Temperature *float32 `json:"temperature,omitempty"`
TopP *float32 `json:"top_p,omitempty"`
StopSequences []string `json:"stop_sequences,omitempty"`
Stream bool `json:"stream,omitempty"`
}
type anthropicMessage struct {
Role string `json:"role"`
Role string `json:"role"`
Content []anthropicContentBlock `json:"content"`
}

115
pkg/llm/providers/llamacpp/client.go
View file

@ -2,12 +2,13 @@ package llamacpp
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"iter"
"net/http"
"strings"
"time"
@ -24,15 +25,15 @@ const defaultContextWindow = 32768
// Config holds the settings needed to create a llama.cpp client.
type Config struct {
BaseURL string // defaults to http://localhost:8080/v1
Model string
Timeout int // request timeout in seconds (0 = default, no timeout)
ContextWindow int // model's context window in tokens (0 = defaultContextWindow)
MaxTokens int // default max_tokens (0 = defaultMaxTokens)
TopK int // top-k sampling (0 = model/server default)
TopP float32 // nucleus sampling (0 = model/server default)
Temperature float32
MinP float32 // min-p sampling (llama.cpp extension)
BaseURL string // defaults to http://localhost:8080/v1
Model string
Timeout int // request timeout in seconds (0 = default, no timeout)
ContextWindow int // model's context window in tokens (0 = defaultContextWindow)
MaxTokens int // default max_tokens (0 = defaultMaxTokens)
TopK int // top-k sampling (0 = model/server default)
TopP float32 // nucleus sampling (0 = model/server default)
Temperature float32
MinP float32 // min-p sampling (llama.cpp extension)
PresencePenalty float32
RepetitionPenalty float32 // sent as the server's `repeat_penalty` field
MaxThinkingTokens int // best-effort cap on reasoning tokens; ignored by servers that don't support it
@ -123,7 +124,7 @@ func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.C
return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err)
}
return c.toResponse(apiResp), nil
return c.toResponse(apiResp)
}
func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] {
@ -191,6 +192,11 @@ func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq
}
scanner := bufio.NewScanner(resp.Body)
// A single SSE line can exceed bufio.Scanner's 64KB default cap
// (e.g. a large tool-call arguments delta or a long reasoning
// event), which would kill the stream mid-turn with "token too
// long" — same headroom the anthropic client already reserves.
scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "data: ") {
@ -370,11 +376,16 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader
if err != nil {
return nil, fmt.Errorf("marshaling request: %w", err)
}
return strings.NewReader(string(data)), nil
return bytes.NewReader(data), nil
}
// toResponse converts a llama.cpp API response to our CompletionResponse.
func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse {
func (c *Client) toResponse(resp llamaChatResponse) (llm.CompletionResponse, error) {
// Guard against a 200 response with no choices (e.g. a misbehaving
// server or proxy) — indexing Choices[0] blindly panics the whole app.
if len(resp.Choices) == 0 {
return llm.CompletionResponse{}, fmt.Errorf("llamacpp: response contained no choices")
}
choice := resp.Choices[0]
result := llm.CompletionResponse{
ID: resp.ID,
@ -398,32 +409,32 @@ func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse {
TotalTokens: resp.Usage.TotalTokens,
}
return result
return result, nil
}
// llama.cpp API types
type llamaChatRequest struct {
Model string `json:"model"`
Messages []llamaMessage `json:"messages"`
Tools []llamaTool `json:"tools,omitempty"`
ToolChoice interface{} `json:"tool_choice,omitempty"`
Temperature float32 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
TopK int `json:"top_k,omitempty"`
TopP float32 `json:"top_p,omitempty"`
MinP float32 `json:"min_p,omitempty"`
PresencePenalty float32 `json:"presence_penalty,omitempty"`
RepeatPenalty float32 `json:"repeat_penalty,omitempty"`
Model string `json:"model"`
Messages []llamaMessage `json:"messages"`
Tools []llamaTool `json:"tools,omitempty"`
ToolChoice interface{} `json:"tool_choice,omitempty"`
Temperature float32 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
TopK int `json:"top_k,omitempty"`
TopP float32 `json:"top_p,omitempty"`
MinP float32 `json:"min_p,omitempty"`
PresencePenalty float32 `json:"presence_penalty,omitempty"`
RepeatPenalty float32 `json:"repeat_penalty,omitempty"`
// MaxThinkingTokens is a best-effort reasoning-token cap: not part of
// upstream llama.cpp's server API, but harmless to send since JSON
// servers ignore unrecognized fields, and some front-ends (e.g. the
// proxy this model's config was written for) do honor it.
MaxThinkingTokens int `json:"max_thinking_tokens,omitempty"`
Stop []string `json:"stop,omitempty"`
Stream bool `json:"stream"`
MaxThinkingTokens int `json:"max_thinking_tokens,omitempty"`
Stop []string `json:"stop,omitempty"`
Stream bool `json:"stream"`
StreamOptions *llamaStreamOptions `json:"stream_options,omitempty"`
ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"`
ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"`
}
type llamaStreamOptions struct {
@ -444,16 +455,16 @@ type llamaTool struct {
}
type llamaChatResponse struct {
ID string `json:"id"`
Model string `json:"model"`
Choices []llamaChoice `json:"choices"`
Usage llamaUsage `json:"usage"`
ID string `json:"id"`
Model string `json:"model"`
Choices []llamaChoice `json:"choices"`
Usage llamaUsage `json:"usage"`
}
type llamaChoice struct {
Index int `json:"index"`
Message llamaMessageResult `json:"message"`
FinishReason string `json:"finish_reason"`
FinishReason string `json:"finish_reason"`
}
type llamaMessageResult struct {
@ -464,14 +475,14 @@ type llamaMessageResult struct {
}
type llamaToolCall struct {
ID string `json:"id"`
Type string `json:"type"`
Function llamaFunction `json:"function"`
ID string `json:"id"`
Type string `json:"type"`
Function llamaFunction `json:"function"`
}
type llamaFunction struct {
Name string `json:"name"`
Arguments string `json:"arguments"`
Name string `json:"name"`
Arguments string `json:"arguments"`
}
type llamaUsage struct {
@ -483,32 +494,32 @@ type llamaUsage struct {
// Stream event types
type llamaStreamEvent struct {
ID string `json:"id"`
ID string `json:"id"`
Choices []llamaStreamChoice `json:"choices"`
Usage *llamaUsage `json:"usage"`
Usage *llamaUsage `json:"usage"`
}
type llamaStreamChoice struct {
Index int `json:"index"`
Delta llamaStreamDelta `json:"delta"`
FinishReason string `json:"finish_reason"`
Index int `json:"index"`
Delta llamaStreamDelta `json:"delta"`
FinishReason string `json:"finish_reason"`
}
type llamaStreamDelta struct {
Content string `json:"content"`
ReasoningContent string `json:"reasoning_content"`
Role string `json:"role"`
ToolCalls []llamaStreamToolCall `json:"tool_calls"`
Role string `json:"role"`
ToolCalls []llamaStreamToolCall `json:"tool_calls"`
}
type llamaStreamToolCall struct {
Index int `json:"index"`
ID string `json:"id"`
Type string `json:"type"`
Function llamaStreamFunction `json:"function"`
Index int `json:"index"`
ID string `json:"id"`
Type string `json:"type"`
Function llamaStreamFunction `json:"function"`
}
type llamaStreamFunction struct {
Name string `json:"name"`
Arguments string `json:"arguments"`
Name string `json:"name"`
Arguments string `json:"arguments"`
}