From 0f835a0802493220a6091c1b772a7ce1aced3521 Mon Sep 17 00:00:00 2001 From: Victor Vargas Date: Wed, 8 Jul 2026 23:27:17 -0700 Subject: [PATCH 1/2] feat(agent): wire AGENTS.md discovery into the agent loop's system prompt Export persona.DiscoverAgentsMD and add agent.Config.AgentsMD so project and global AGENTS.md rules actually reach the model. Previously buildInitialMessages always called AssembleSystemPrompt with an empty string, so no AGENTS.md content was ever injected despite the discovery logic already existing. Co-Authored-By: Claude Sonnet 5 --- pkg/agent/loop.go | 3 ++- pkg/agent/loop_test.go | 25 +++++++++++++++++++++++++ pkg/persona/persona.go | 4 ++-- pkg/persona/persona_test.go | 4 ++-- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 0d98e22..1c0916a 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -42,6 +42,7 @@ type Config struct { OnIteration OnIterationHook ToolTimeout time.Duration ChatTemplateKwargs map[string]any // passed to the LLM provider (e.g. Qwen enable_thinking) + AgentsMD string // discovered AGENTS.md content, folded into the system prompt } // Iteration represents a single cycle of the agent loop. @@ -230,7 +231,7 @@ func (l *Loop) RunStream(ctx context.Context, input string, history ...llm.Messa } func (l *Loop) buildInitialMessages(input string, history []llm.Message) []llm.Message { - systemPrompt := persona.AssembleSystemPrompt(l.cfg.Persona, "") + systemPrompt := persona.AssembleSystemPrompt(l.cfg.Persona, l.cfg.AgentsMD) messages := make([]llm.Message, 0, len(history)+2) messages = append(messages, llm.Message{Role: llm.RoleSystem, Content: systemPrompt}) messages = append(messages, history...) diff --git a/pkg/agent/loop_test.go b/pkg/agent/loop_test.go index e81f128..4c59bcf 100644 --- a/pkg/agent/loop_test.go +++ b/pkg/agent/loop_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "iter" + "strings" "testing" "time" @@ -116,6 +117,30 @@ func TestRun_NoToolCalls(t *testing.T) { } } +func TestRun_IncludesAgentsMD(t *testing.T) { + var capturedSystemPrompt string + mockClient := &mockLLM{ + generateFunc: func(ctx context.Context, req llm.CompletionRequest) (llm.CompletionResponse, error) { + capturedSystemPrompt = req.Messages[0].Content + return llm.CompletionResponse{Content: "done"}, nil + }, + } + + loop := New(Config{ + LLM: mockClient, + Persona: persona.DefaultPersona(), + Tools: tools.NewRegistry(), + AgentsMD: "Never edit go.mod directly.", + }) + + if _, err := loop.Run(context.Background(), "Hello"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(capturedSystemPrompt, "Never edit go.mod directly.") { + t.Errorf("expected system prompt to include AGENTS.md content, got %q", capturedSystemPrompt) + } +} + func TestRun_ToolCalls(t *testing.T) { registry := tools.NewRegistry() registry.Register(tools.Tool{ diff --git a/pkg/persona/persona.go b/pkg/persona/persona.go index 0d83982..c092b23 100644 --- a/pkg/persona/persona.go +++ b/pkg/persona/persona.go @@ -79,8 +79,8 @@ func AssembleSystemPrompt(p Persona, agentsMD string) string { return strings.Join(parts, "\n\n") } -// discoverAgentsMD walks up the directory tree looking for AGENTS.md files. -func discoverAgentsMD(root string) string { +// DiscoverAgentsMD walks up the directory tree looking for AGENTS.md files. +func DiscoverAgentsMD(root string) string { var parts []string current := root diff --git a/pkg/persona/persona_test.go b/pkg/persona/persona_test.go index 86a3ffc..ed60fea 100644 --- a/pkg/persona/persona_test.go +++ b/pkg/persona/persona_test.go @@ -45,9 +45,9 @@ func TestDiscoverAgentsMD(t *testing.T) { agentsPath := filepath.Join(tmpDir, "AGENTS.md") os.WriteFile(agentsPath, []byte("test instructions"), 0644) - result := discoverAgentsMD(tmpDir) + result := DiscoverAgentsMD(tmpDir) if !contains(result, "test instructions") { - t.Error("expected discoverAgentsMD to find AGENTS.md") + t.Error("expected DiscoverAgentsMD to find AGENTS.md") } } From a2de4eb812df63caa52b4942da3f818231a7de57 Mon Sep 17 00:00:00 2001 From: Victor Vargas Date: Wed, 8 Jul 2026 23:33:20 -0700 Subject: [PATCH 2/2] feat(llm): add anthropic provider client, expand llama.cpp sampling config - pkg/llm/providers/anthropic: new client implementation (was previously imported by rony-harness but never committed here, so a fresh clone wouldn't build) - pkg/llm/providers/llamacpp: Config/Client gain the full local-model sampling surface (max_tokens, context_window, top_k/top_p/min_p, presence/repetition penalty, max_thinking_tokens) to match the llamacpp-local* entries added to configs/ai_providers.yaml Co-Authored-By: Claude Sonnet 5 --- pkg/llm/providers/anthropic/client.go | 561 ++++++++++++++++++++++++++ pkg/llm/providers/llamacpp/client.go | 99 ++++- 2 files changed, 647 insertions(+), 13 deletions(-) create mode 100644 pkg/llm/providers/anthropic/client.go diff --git a/pkg/llm/providers/anthropic/client.go b/pkg/llm/providers/anthropic/client.go new file mode 100644 index 0000000..a0e9112 --- /dev/null +++ b/pkg/llm/providers/anthropic/client.go @@ -0,0 +1,561 @@ +// Package anthropic implements llm.LLMClient for the Anthropic Messages API. +package anthropic + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "iter" + "strings" + + "github.com/VictorVargas/rony-llm-agent/pkg/llm" +) + +const ( + defaultBaseURL = "https://api.anthropic.com/v1" + defaultModel = "claude-opus-4-8" + defaultMaxTokens = 8192 + anthropicVersion = "2023-06-01" +) + +// Config holds the settings needed to create an Anthropic client. +type Config struct { + APIKey string + Model string // defaults to claude-opus-4-8 + BaseURL string // defaults to https://api.anthropic.com/v1 + MaxTokens int // default max_tokens sent on every request (Anthropic requires one); 0 = defaultMaxTokens + Temperature *float32 + TopP *float32 +} + +// Client implements llm.LLMClient for Anthropic. +type Client struct { + apiKey string + baseURL string + model string + maxTokens int + temperature *float32 + topP *float32 + http *http.Client +} + +// New returns a new Anthropic client. +func New(cfg Config) (*Client, error) { + if cfg.APIKey == "" { + return nil, fmt.Errorf("anthropic: API key is required") + } + + baseURL := cfg.BaseURL + if baseURL == "" { + baseURL = defaultBaseURL + } + + model := cfg.Model + if model == "" { + model = defaultModel + } + + maxTokens := cfg.MaxTokens + if maxTokens == 0 { + maxTokens = defaultMaxTokens + } + + return &Client{ + apiKey: cfg.APIKey, + baseURL: baseURL, + model: model, + maxTokens: maxTokens, + temperature: cfg.Temperature, + topP: cfg.TopP, + http: http.DefaultClient, + }, nil +} + +func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.CompletionResponse, error) { + endpoint := c.baseURL + "/messages" + + payload, err := c.buildRequest(req, false) + if err != nil { + return llm.CompletionResponse{}, fmt.Errorf("building request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload) + if err != nil { + return llm.CompletionResponse{}, fmt.Errorf("creating request: %w", err) + } + c.setHeaders(httpReq) + + resp, err := c.http.Do(httpReq) + if err != nil { + return llm.CompletionResponse{}, fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return llm.CompletionResponse{}, c.apiError(resp) + } + + var apiResp anthropicResponse + if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil { + return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err) + } + + return c.toResponse(apiResp), nil +} + +func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] { + return func(yield func(llm.StreamChunk, error) bool) { + endpoint := c.baseURL + "/messages" + + payload, err := c.buildRequest(req, true) + if err != nil { + yield(llm.StreamChunk{}, fmt.Errorf("building request: %w", err)) + return + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload) + if err != nil { + yield(llm.StreamChunk{}, fmt.Errorf("creating request: %w", err)) + return + } + c.setHeaders(httpReq) + httpReq.Header.Set("Accept", "text/event-stream") + + resp, err := c.http.Do(httpReq) + if err != nil { + yield(llm.StreamChunk{}, fmt.Errorf("request failed: %w", err)) + return + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + yield(llm.StreamChunk{}, c.apiError(resp)) + return + } + + // blockAccum buffers one content block's fragments as they stream + // in: text arrives piecemeal via text_delta events (yielded as we + // go), while a tool_use block's `input` arrives as fragments of a + // JSON string via input_json_delta that can't be parsed until the + // block is complete. + type blockAccum struct { + kind string // "text" | "tool_use" + id string + name string + args strings.Builder + } + blocks := map[int]*blockAccum{} + var order []int + var inputTokens int + + flushToolCalls := func() []llm.ToolCall { + var calls []llm.ToolCall + for _, idx := range order { + b := blocks[idx] + if b.kind != "tool_use" { + continue + } + args := b.args.String() + if strings.TrimSpace(args) == "" { + args = "{}" + } + calls = append(calls, llm.ToolCall{ + ID: b.id, + Name: b.name, + Arguments: json.RawMessage(args), + }) + } + return calls + } + + scanner := bufio.NewScanner(resp.Body) + scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "data: ") { + continue + } + data := strings.TrimPrefix(line, "data: ") + + var event anthropicStreamEvent + if err := json.Unmarshal([]byte(data), &event); err != nil { + yield(llm.StreamChunk{}, fmt.Errorf("decoding event: %w", err)) + return + } + + switch event.Type { + case "message_start": + if event.Message != nil { + inputTokens = event.Message.Usage.InputTokens + } + case "content_block_start": + if event.ContentBlock != nil { + blocks[event.Index] = &blockAccum{ + kind: event.ContentBlock.Type, + id: event.ContentBlock.ID, + name: event.ContentBlock.Name, + } + order = append(order, event.Index) + } + case "content_block_delta": + if event.Delta == nil { + continue + } + switch event.Delta.Type { + case "text_delta": + if !yield(llm.StreamChunk{Delta: event.Delta.Text}, nil) { + return + } + case "input_json_delta": + if b, ok := blocks[event.Index]; ok { + b.args.WriteString(event.Delta.PartialJSON) + } + } + case "message_delta": + var outputTokens int + if event.Usage != nil { + outputTokens = event.Usage.OutputTokens + } + var finishReason string + if event.Delta != nil { + finishReason = mapStopReason(event.Delta.StopReason) + } + chunk := llm.StreamChunk{ + ToolCalls: flushToolCalls(), + FinishReason: finishReason, + Usage: llm.TokenUsage{ + InputTokens: inputTokens, + OutputTokens: outputTokens, + TotalTokens: inputTokens + outputTokens, + }, + } + if !yield(chunk, nil) { + return + } + case "message_stop": + return + case "error": + msg := "unknown error" + if event.Error != nil { + msg = event.Error.Message + } + yield(llm.StreamChunk{}, fmt.Errorf("anthropic stream error: %s", msg)) + return + } + } + if err := scanner.Err(); err != nil { + yield(llm.StreamChunk{}, fmt.Errorf("stream error: %w", err)) + } + } +} + +func (c *Client) Name() string { + return "anthropic" +} + +func (c *Client) Capabilities() llm.ProviderCapabilities { + maxContext := 1000000 + if strings.Contains(c.model, "haiku") { + maxContext = 200000 + } + return llm.ProviderCapabilities{ + SupportsTools: true, + SupportsVision: true, + SupportsJSON: true, + MaxContextWindow: maxContext, + } +} + +func (c *Client) setHeaders(httpReq *http.Request) { + httpReq.Header.Set("x-api-key", c.apiKey) + httpReq.Header.Set("anthropic-version", anthropicVersion) + httpReq.Header.Set("content-type", "application/json") +} + +func (c *Client) apiError(resp *http.Response) error { + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode == http.StatusUnauthorized { + return fmt.Errorf("anthropic: authentication failed, check ANTHROPIC_API_KEY (401): %s", body) + } + return fmt.Errorf("API error %d: %s", resp.StatusCode, string(body)) +} + +// buildRequest converts an llm.CompletionRequest to the Anthropic Messages API format. +func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader, error) { + var systemParts []string + var messages []anthropicMessage + + // appendUserBlock merges consecutive content destined for a "user" turn + // (plain user text and tool_result blocks alike) into a single message, + // since Anthropic requires messages to strictly alternate user/assistant. + appendUserBlock := func(block anthropicContentBlock) { + if n := len(messages); n > 0 && messages[n-1].Role == "user" { + messages[n-1].Content = append(messages[n-1].Content, block) + return + } + messages = append(messages, anthropicMessage{Role: "user", Content: []anthropicContentBlock{block}}) + } + + for _, m := range req.Messages { + switch m.Role { + case llm.RoleSystem: + if strings.TrimSpace(m.Content) != "" { + systemParts = append(systemParts, m.Content) + } + case llm.RoleTool: + appendUserBlock(anthropicContentBlock{ + Type: "tool_result", + ToolUseID: m.ToolCallID, + Content: m.Content, + }) + case llm.RoleUser: + appendUserBlock(anthropicContentBlock{Type: "text", Text: m.Content}) + case llm.RoleAssistant: + var blocks []anthropicContentBlock + if strings.TrimSpace(m.Content) != "" { + blocks = append(blocks, anthropicContentBlock{Type: "text", Text: m.Content}) + } + for _, tc := range m.ToolCalls { + input := tc.Arguments + if len(input) == 0 { + input = json.RawMessage("{}") + } + blocks = append(blocks, anthropicContentBlock{Type: "tool_use", ID: tc.ID, Name: tc.Name, Input: input}) + } + if len(blocks) == 0 { + blocks = append(blocks, anthropicContentBlock{Type: "text", Text: ""}) + } + messages = append(messages, anthropicMessage{Role: "assistant", Content: blocks}) + } + } + + model := req.Model + if model == "" { + model = c.model + } + + maxTokens := c.maxTokens + if req.MaxTokens != nil { + maxTokens = *req.MaxTokens + } + + anthReq := anthropicRequest{ + Model: model, + Messages: messages, + System: strings.Join(systemParts, "\n\n"), + MaxTokens: maxTokens, + Temperature: c.temperature, + TopP: c.topP, + Stream: stream, + } + if req.Temperature != nil { + anthReq.Temperature = req.Temperature + } + if len(req.Stop) > 0 { + anthReq.StopSequences = req.Stop + } + + tools, err := convertTools(req.Tools) + if err != nil { + return nil, err + } + if len(tools) > 0 { + anthReq.Tools = tools + } + + if choice := convertToolChoice(req.ToolChoice); choice != nil { + anthReq.ToolChoice = choice + } + + data, err := json.Marshal(anthReq) + if err != nil { + return nil, fmt.Errorf("marshaling request: %w", err) + } + return strings.NewReader(string(data)), nil +} + +// convertTools converts the harness's OpenAI-style function-tool schemas +// ({"type":"function","function":{name,description,parameters}}) into +// Anthropic's flatter {name,description,input_schema} tool format. +func convertTools(raw []json.RawMessage) ([]anthropicTool, error) { + if len(raw) == 0 { + return nil, nil + } + tools := make([]anthropicTool, 0, len(raw)) + for i, t := range raw { + var wrapper struct { + Function struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters json.RawMessage `json:"parameters"` + } `json:"function"` + } + if err := json.Unmarshal(t, &wrapper); err != nil { + return nil, fmt.Errorf("parsing tool %d: %w", i, err) + } + tools = append(tools, anthropicTool{ + Name: wrapper.Function.Name, + Description: wrapper.Function.Description, + InputSchema: wrapper.Function.Parameters, + }) + } + return tools, nil +} + +// convertToolChoice maps the harness's provider-agnostic tool_choice value +// (llm.ToolChoice, *llm.ToolRef, or nil) to Anthropic's tool_choice shape. +func convertToolChoice(choice interface{}) json.RawMessage { + switch v := choice.(type) { + case llm.ToolChoice: + switch v { + case llm.ToolChoiceAuto: + return json.RawMessage(`{"type":"auto"}`) + case llm.ToolChoiceNone: + return json.RawMessage(`{"type":"none"}`) + case llm.ToolChoiceRequired: + return json.RawMessage(`{"type":"any"}`) + } + case *llm.ToolRef: + if v == nil { + return nil + } + data, err := json.Marshal(struct { + Type string `json:"type"` + Name string `json:"name"` + }{Type: "tool", Name: v.Name}) + if err != nil { + return nil + } + return data + } + return nil +} + +// toResponse converts an Anthropic API response to our CompletionResponse. +func (c *Client) toResponse(resp anthropicResponse) llm.CompletionResponse { + var content strings.Builder + var toolCalls []llm.ToolCall + + for _, block := range resp.Content { + switch block.Type { + case "text": + content.WriteString(block.Text) + case "tool_use": + input := block.Input + if len(input) == 0 { + input = json.RawMessage("{}") + } + toolCalls = append(toolCalls, llm.ToolCall{ + ID: block.ID, + Name: block.Name, + Arguments: input, + }) + } + } + + return llm.CompletionResponse{ + ID: resp.ID, + Model: resp.Model, + Content: content.String(), + ToolCalls: toolCalls, + StopReason: mapStopReason(resp.StopReason), + Usage: llm.TokenUsage{ + InputTokens: resp.Usage.InputTokens, + OutputTokens: resp.Usage.OutputTokens, + TotalTokens: resp.Usage.InputTokens + resp.Usage.OutputTokens, + }, + } +} + +func mapStopReason(reason string) string { + switch reason { + case "end_turn", "stop_sequence": + if reason == "stop_sequence" { + return llm.StopReasonStopSeq + } + return llm.StopReasonEndTurn + case "tool_use": + return llm.StopReasonToolUse + case "max_tokens": + return llm.StopReasonMaxTokens + default: + return reason + } +} + +// Anthropic API types + +type anthropicRequest struct { + Model string `json:"model"` + Messages []anthropicMessage `json:"messages"` + System string `json:"system,omitempty"` + MaxTokens int `json:"max_tokens"` + Tools []anthropicTool `json:"tools,omitempty"` + ToolChoice json.RawMessage `json:"tool_choice,omitempty"` + Temperature *float32 `json:"temperature,omitempty"` + TopP *float32 `json:"top_p,omitempty"` + StopSequences []string `json:"stop_sequences,omitempty"` + Stream bool `json:"stream,omitempty"` +} + +type anthropicMessage struct { + Role string `json:"role"` + Content []anthropicContentBlock `json:"content"` +} + +type anthropicContentBlock struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Input json.RawMessage `json:"input,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + Content string `json:"content,omitempty"` +} + +type anthropicTool struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + InputSchema json.RawMessage `json:"input_schema"` +} + +type anthropicResponse struct { + ID string `json:"id"` + Model string `json:"model"` + Content []anthropicContentBlock `json:"content"` + StopReason string `json:"stop_reason"` + Usage anthropicUsage `json:"usage"` +} + +type anthropicUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` +} + +// Stream event types + +type anthropicStreamEvent struct { + Type string `json:"type"` + Index int `json:"index"` + Message *struct { + Usage anthropicUsage `json:"usage"` + } `json:"message,omitempty"` + ContentBlock *struct { + Type string `json:"type"` + ID string `json:"id"` + Name string `json:"name"` + } `json:"content_block,omitempty"` + Delta *struct { + Type string `json:"type"` + Text string `json:"text"` + PartialJSON string `json:"partial_json"` + StopReason string `json:"stop_reason"` + } `json:"delta,omitempty"` + Usage *anthropicUsage `json:"usage,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} diff --git a/pkg/llm/providers/llamacpp/client.go b/pkg/llm/providers/llamacpp/client.go index 716b078..6861ca2 100644 --- a/pkg/llm/providers/llamacpp/client.go +++ b/pkg/llm/providers/llamacpp/client.go @@ -9,24 +9,50 @@ import ( "net/http" "iter" "strings" + "time" "github.com/VictorVargas/rony-llm-agent/pkg/llm" ) +// defaultMaxTokens is used when neither the Config nor the per-request +// CompletionRequest specify one, so requests never go out with an +// unbounded/zero max_tokens. +const defaultMaxTokens = 4096 + +// defaultContextWindow is reported by Capabilities() when Config.ContextWindow is unset. +const defaultContextWindow = 32768 + // Config holds the settings needed to create a llama.cpp client. type Config struct { - BaseURL string // defaults to http://localhost:8080/v1 - Model string - Timeout int // request timeout in seconds (0 = default) - TopK int // top-k sampling (0 = default) - TopP float32 + BaseURL string // defaults to http://localhost:8080/v1 + Model string + Timeout int // request timeout in seconds (0 = default, no timeout) + ContextWindow int // model's context window in tokens (0 = defaultContextWindow) + MaxTokens int // default max_tokens (0 = defaultMaxTokens) + TopK int // top-k sampling (0 = model/server default) + TopP float32 // nucleus sampling (0 = model/server default) Temperature float32 + MinP float32 // min-p sampling (llama.cpp extension) + PresencePenalty float32 + RepetitionPenalty float32 // sent as the server's `repeat_penalty` field + MaxThinkingTokens int // best-effort cap on reasoning tokens; ignored by servers that don't support it } // Client implements llm.LLMClient for llama.cpp. type Client struct { baseURL string + model string http *http.Client + + contextWindow int + maxTokens int + topK int + topP float32 + temperature float32 + minP float32 + presencePenalty float32 + repetitionPenalty float32 + maxThinkingTokens int } // New returns a new llama.cpp client. @@ -36,9 +62,34 @@ func New(cfg Config) (*Client, error) { baseURL = "http://localhost:8080/v1" } + maxTokens := cfg.MaxTokens + if maxTokens == 0 { + maxTokens = defaultMaxTokens + } + + contextWindow := cfg.ContextWindow + if contextWindow == 0 { + contextWindow = defaultContextWindow + } + + httpClient := http.DefaultClient + if cfg.Timeout > 0 { + httpClient = &http.Client{Timeout: time.Duration(cfg.Timeout) * time.Second} + } + return &Client{ - baseURL: baseURL, - http: http.DefaultClient, + baseURL: baseURL, + model: cfg.Model, + http: httpClient, + contextWindow: contextWindow, + maxTokens: maxTokens, + topK: cfg.TopK, + topP: cfg.TopP, + temperature: cfg.Temperature, + minP: cfg.MinP, + presencePenalty: cfg.PresencePenalty, + repetitionPenalty: cfg.RepetitionPenalty, + maxThinkingTokens: cfg.MaxThinkingTokens, }, nil } @@ -232,7 +283,7 @@ func (c *Client) Capabilities() llm.ProviderCapabilities { SupportsTools: true, SupportsVision: false, SupportsJSON: true, - MaxContextWindow: 32768, + MaxContextWindow: c.contextWindow, } } @@ -271,11 +322,27 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader tools = append(tools, tool) } + model := req.Model + if model == "" { + model = c.model + } + openReq := llamaChatRequest{ - Model: req.Model, + Model: model, Messages: messages, Stream: stream, ChatTemplateKwargs: req.ChatTemplateKwargs, + // Client-level sampling defaults (from Config, e.g. the local + // model's configured temperature/top_p/top_k/etc.) go first; a + // per-request override below takes precedence when set. + Temperature: c.temperature, + MaxTokens: c.maxTokens, + TopK: c.topK, + TopP: c.topP, + MinP: c.minP, + PresencePenalty: c.presencePenalty, + RepeatPenalty: c.repetitionPenalty, + MaxThinkingTokens: c.maxThinkingTokens, } if stream { // Ask for a final SSE event carrying token usage (OpenAI-style @@ -290,12 +357,10 @@ func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader openReq.ToolChoice = req.ToolChoice } if req.Temperature != nil { - tmp := *req.Temperature - openReq.Temperature = tmp + openReq.Temperature = *req.Temperature } if req.MaxTokens != nil { - tmp := *req.MaxTokens - openReq.MaxTokens = tmp + openReq.MaxTokens = *req.MaxTokens } if len(req.Stop) > 0 { openReq.Stop = req.Stop @@ -347,6 +412,14 @@ type llamaChatRequest struct { MaxTokens int `json:"max_tokens,omitempty"` TopK int `json:"top_k,omitempty"` TopP float32 `json:"top_p,omitempty"` + MinP float32 `json:"min_p,omitempty"` + PresencePenalty float32 `json:"presence_penalty,omitempty"` + RepeatPenalty float32 `json:"repeat_penalty,omitempty"` + // MaxThinkingTokens is a best-effort reasoning-token cap: not part of + // upstream llama.cpp's server API, but harmless to send since JSON + // servers ignore unrecognized fields, and some front-ends (e.g. the + // proxy this model's config was written for) do honor it. + MaxThinkingTokens int `json:"max_thinking_tokens,omitempty"` Stop []string `json:"stop,omitempty"` Stream bool `json:"stream"` StreamOptions *llamaStreamOptions `json:"stream_options,omitempty"`