package llamacpp import ( "bufio" "context" "encoding/json" "fmt" "io" "net/http" "iter" "strings" "github.com/VictorVargas/rony-llm-agent/pkg/llm" ) // Config holds the settings needed to create a llama.cpp client. type Config struct { BaseURL string // defaults to http://localhost:8080/v1 Model string Timeout int // request timeout in seconds (0 = default) TopK int // top-k sampling (0 = default) TopP float32 Temperature float32 } // Client implements llm.LLMClient for llama.cpp. type Client struct { baseURL string http *http.Client } // New returns a new llama.cpp client. func New(cfg Config) (*Client, error) { baseURL := cfg.BaseURL if baseURL == "" { baseURL = "http://localhost:8080/v1" } return &Client{ baseURL: baseURL, http: http.DefaultClient, }, nil } func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.CompletionResponse, error) { endpoint := c.baseURL + "/chat/completions" payload, err := c.buildRequest(req, false) if err != nil { return llm.CompletionResponse{}, fmt.Errorf("building request: %w", err) } httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload) if err != nil { return llm.CompletionResponse{}, fmt.Errorf("creating request: %w", err) } httpReq.Header.Set("Content-Type", "application/json") resp, err := c.http.Do(httpReq) if err != nil { return llm.CompletionResponse{}, fmt.Errorf("request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { body, _ := io.ReadAll(resp.Body) return llm.CompletionResponse{}, fmt.Errorf("API error %d: %s", resp.StatusCode, string(body)) } var apiResp llamaChatResponse if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil { return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err) } return c.toResponse(apiResp), nil } func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] { return func(yield func(llm.StreamChunk, error) bool) { endpoint := c.baseURL + "/chat/completions" payload, err := c.buildRequest(req, true) if err != nil { yield(llm.StreamChunk{}, fmt.Errorf("building request: %w", err)) return } httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload) if err != nil { yield(llm.StreamChunk{}, fmt.Errorf("creating request: %w", err)) return } httpReq.Header.Set("Content-Type", "application/json") resp, err := c.http.Do(httpReq) if err != nil { yield(llm.StreamChunk{}, fmt.Errorf("request failed: %w", err)) return } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { body, _ := io.ReadAll(resp.Body) yield(llm.StreamChunk{}, fmt.Errorf("API error %d: %s", resp.StatusCode, string(body))) return } // toolCallAccum buffers one tool call's fragments as they stream in: // the OpenAI-compatible SSE format sends the id/name in the first // delta for a given tool-call index and the (potentially large) // arguments JSON in pieces across many subsequent deltas, so it // can't be handed to a tool handler until it's fully assembled. type toolCallAccum struct { id string name string args strings.Builder } toolCallFrags := map[int]*toolCallAccum{} var toolCallOrder []int // flushToolCalls assembles the buffered fragments into complete // tool calls (called once finish_reason arrives) and resets the // accumulator for any further choices/events. flushToolCalls := func() []llm.ToolCall { if len(toolCallOrder) == 0 { return nil } calls := make([]llm.ToolCall, 0, len(toolCallOrder)) for _, idx := range toolCallOrder { frag := toolCallFrags[idx] calls = append(calls, llm.ToolCall{ ID: frag.id, Name: frag.name, Arguments: json.RawMessage(frag.args.String()), }) } toolCallFrags = map[int]*toolCallAccum{} toolCallOrder = nil return calls } scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "data: ") { continue } data := strings.TrimPrefix(line, "data: ") if data == "[DONE]" { return } var event llamaStreamEvent if err := json.Unmarshal([]byte(data), &event); err != nil { yield(llm.StreamChunk{}, fmt.Errorf("decoding event: %w", err)) return } var usage llm.TokenUsage if event.Usage != nil { usage = llm.TokenUsage{ InputTokens: event.Usage.PromptTokens, OutputTokens: event.Usage.CompletionTokens, TotalTokens: event.Usage.TotalTokens, } } if len(event.Choices) == 0 { // The usage-only event (per stream_options.include_usage) // carries no choices, so it needs its own chunk. if event.Usage != nil { if !yield(llm.StreamChunk{Usage: usage}, nil) { return } } continue } for _, choice := range event.Choices { hasFragment := len(choice.Delta.ToolCalls) > 0 for _, tc := range choice.Delta.ToolCalls { frag, ok := toolCallFrags[tc.Index] if !ok { frag = &toolCallAccum{} toolCallFrags[tc.Index] = frag toolCallOrder = append(toolCallOrder, tc.Index) } if tc.ID != "" { frag.id = tc.ID } if tc.Function.Name != "" { frag.name = tc.Function.Name } frag.args.WriteString(tc.Function.Arguments) } chunk := llm.StreamChunk{ Delta: choice.Delta.Content, ReasoningDelta: choice.Delta.ReasoningContent, Usage: usage, } if choice.FinishReason != "" { chunk.FinishReason = choice.FinishReason chunk.ToolCalls = flushToolCalls() } // A fragment-only event (a piece of a tool call's streamed // arguments, with nothing else in this delta) has nothing // yet for the agent loop to act on: it was buffered above, // so skip yielding an empty chunk for it. if hasFragment && chunk.Delta == "" && chunk.ReasoningDelta == "" && chunk.FinishReason == "" { continue } if !yield(chunk, nil) { return } } } if err := scanner.Err(); err != nil { yield(llm.StreamChunk{}, fmt.Errorf("stream error: %w", err)) } } } func (c *Client) Name() string { return "llama.cpp" } func (c *Client) Capabilities() llm.ProviderCapabilities { return llm.ProviderCapabilities{ SupportsTools: true, SupportsVision: false, SupportsJSON: true, MaxContextWindow: 32768, } } // buildRequest converts an llm.CompletionRequest to the llama.cpp API format. func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader, error) { messages := make([]llamaMessage, len(req.Messages)) for i, m := range req.Messages { messages[i] = llamaMessage{ Role: string(m.Role), Content: m.Content, ToolCallID: m.ToolCallID, Name: m.Name, } if len(m.ToolCalls) > 0 { calls := make([]llamaToolCall, len(m.ToolCalls)) for j, tc := range m.ToolCalls { calls[j] = llamaToolCall{ ID: tc.ID, Type: "function", Function: llamaFunction{ Name: tc.Name, Arguments: string(tc.Arguments), }, } } messages[i].ToolCalls = calls } } var tools []llamaTool for _, t := range req.Tools { var tool llamaTool if err := json.Unmarshal(t, &tool); err != nil { return nil, fmt.Errorf("parsing tool: %w", err) } tools = append(tools, tool) } openReq := llamaChatRequest{ Model: req.Model, Messages: messages, Stream: stream, ChatTemplateKwargs: req.ChatTemplateKwargs, } if stream { // Ask for a final SSE event carrying token usage (OpenAI-style // streaming omits it otherwise), so Rony can track real token // counts per turn instead of always seeing zero. openReq.StreamOptions = &llamaStreamOptions{IncludeUsage: true} } if len(tools) > 0 { openReq.Tools = tools } if req.ToolChoice != nil { openReq.ToolChoice = req.ToolChoice } if req.Temperature != nil { tmp := *req.Temperature openReq.Temperature = tmp } if req.MaxTokens != nil { tmp := *req.MaxTokens openReq.MaxTokens = tmp } if len(req.Stop) > 0 { openReq.Stop = req.Stop } data, err := json.Marshal(openReq) if err != nil { return nil, fmt.Errorf("marshaling request: %w", err) } return strings.NewReader(string(data)), nil } // toResponse converts a llama.cpp API response to our CompletionResponse. func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse { choice := resp.Choices[0] result := llm.CompletionResponse{ ID: resp.ID, Model: resp.Model, Content: choice.Message.Content, Reasoning: choice.Message.ReasoningContent, StopReason: choice.FinishReason, } for _, tc := range choice.Message.ToolCalls { result.ToolCalls = append(result.ToolCalls, llm.ToolCall{ ID: tc.ID, Name: tc.Function.Name, Arguments: json.RawMessage(tc.Function.Arguments), }) } result.Usage = llm.TokenUsage{ InputTokens: resp.Usage.PromptTokens, OutputTokens: resp.Usage.CompletionTokens, TotalTokens: resp.Usage.TotalTokens, } return result } // llama.cpp API types type llamaChatRequest struct { Model string `json:"model"` Messages []llamaMessage `json:"messages"` Tools []llamaTool `json:"tools,omitempty"` ToolChoice interface{} `json:"tool_choice,omitempty"` Temperature float32 `json:"temperature,omitempty"` MaxTokens int `json:"max_tokens,omitempty"` TopK int `json:"top_k,omitempty"` TopP float32 `json:"top_p,omitempty"` Stop []string `json:"stop,omitempty"` Stream bool `json:"stream"` StreamOptions *llamaStreamOptions `json:"stream_options,omitempty"` ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` } type llamaStreamOptions struct { IncludeUsage bool `json:"include_usage"` } type llamaMessage struct { Role string `json:"role"` Content string `json:"content"` ToolCallID string `json:"tool_call_id,omitempty"` Name string `json:"name,omitempty"` ToolCalls []llamaToolCall `json:"tool_calls,omitempty"` } type llamaTool struct { Type string `json:"type"` Function json.RawMessage `json:"function"` } type llamaChatResponse struct { ID string `json:"id"` Model string `json:"model"` Choices []llamaChoice `json:"choices"` Usage llamaUsage `json:"usage"` } type llamaChoice struct { Index int `json:"index"` Message llamaMessageResult `json:"message"` FinishReason string `json:"finish_reason"` } type llamaMessageResult struct { Role string `json:"role"` Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` ToolCalls []llamaToolCall `json:"tool_calls"` } type llamaToolCall struct { ID string `json:"id"` Type string `json:"type"` Function llamaFunction `json:"function"` } type llamaFunction struct { Name string `json:"name"` Arguments string `json:"arguments"` } type llamaUsage struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` TotalTokens int `json:"total_tokens"` } // Stream event types type llamaStreamEvent struct { ID string `json:"id"` Choices []llamaStreamChoice `json:"choices"` Usage *llamaUsage `json:"usage"` } type llamaStreamChoice struct { Index int `json:"index"` Delta llamaStreamDelta `json:"delta"` FinishReason string `json:"finish_reason"` } type llamaStreamDelta struct { Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` Role string `json:"role"` ToolCalls []llamaStreamToolCall `json:"tool_calls"` } type llamaStreamToolCall struct { Index int `json:"index"` ID string `json:"id"` Type string `json:"type"` Function llamaStreamFunction `json:"function"` } type llamaStreamFunction struct { Name string `json:"name"` Arguments string `json:"arguments"` }