package llamacpp import ( "bufio" "context" "encoding/json" "fmt" "io" "net/http" "iter" "strings" "github.com/VictorVargas/rony-llm-agent/pkg/llm" ) // Config holds the settings needed to create a llama.cpp client. type Config struct { BaseURL string // defaults to http://localhost:8080/v1 Model string Timeout int // request timeout in seconds (0 = default) TopK int // top-k sampling (0 = default) TopP float32 Temperature float32 } // Client implements llm.LLMClient for llama.cpp. type Client struct { baseURL string http *http.Client } // New returns a new llama.cpp client. func New(cfg Config) (*Client, error) { baseURL := cfg.BaseURL if baseURL == "" { baseURL = "http://localhost:8080/v1" } return &Client{ baseURL: baseURL, http: http.DefaultClient, }, nil } func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.CompletionResponse, error) { endpoint := c.baseURL + "/chat/completions" payload, err := c.buildRequest(req, false) if err != nil { return llm.CompletionResponse{}, fmt.Errorf("building request: %w", err) } httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload) if err != nil { return llm.CompletionResponse{}, fmt.Errorf("creating request: %w", err) } httpReq.Header.Set("Content-Type", "application/json") resp, err := c.http.Do(httpReq) if err != nil { return llm.CompletionResponse{}, fmt.Errorf("request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { body, _ := io.ReadAll(resp.Body) return llm.CompletionResponse{}, fmt.Errorf("API error %d: %s", resp.StatusCode, string(body)) } var apiResp llamaChatResponse if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil { return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err) } return c.toResponse(apiResp), nil } func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] { return func(yield func(llm.StreamChunk, error) bool) { endpoint := c.baseURL + "/chat/completions" payload, err := c.buildRequest(req, true) if err != nil { yield(llm.StreamChunk{}, fmt.Errorf("building request: %w", err)) return } httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload) if err != nil { yield(llm.StreamChunk{}, fmt.Errorf("creating request: %w", err)) return } httpReq.Header.Set("Content-Type", "application/json") resp, err := c.http.Do(httpReq) if err != nil { yield(llm.StreamChunk{}, fmt.Errorf("request failed: %w", err)) return } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { body, _ := io.ReadAll(resp.Body) yield(llm.StreamChunk{}, fmt.Errorf("API error %d: %s", resp.StatusCode, string(body))) return } scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "data: ") { continue } data := strings.TrimPrefix(line, "data: ") if data == "[DONE]" { return } var event llamaStreamEvent if err := json.Unmarshal([]byte(data), &event); err != nil { yield(llm.StreamChunk{}, fmt.Errorf("decoding event: %w", err)) return } var usage llm.TokenUsage if event.Usage != nil { usage = llm.TokenUsage{ InputTokens: event.Usage.PromptTokens, OutputTokens: event.Usage.CompletionTokens, TotalTokens: event.Usage.TotalTokens, } } if len(event.Choices) == 0 { // The usage-only event (per stream_options.include_usage) // carries no choices, so it needs its own chunk. if event.Usage != nil { if !yield(llm.StreamChunk{Usage: usage}, nil) { return } } continue } for _, choice := range event.Choices { chunk := llm.StreamChunk{ Delta: choice.Delta.Content, ReasoningDelta: choice.Delta.ReasoningContent, Usage: usage, } if choice.FinishReason != "" { chunk.FinishReason = choice.FinishReason } if !yield(chunk, nil) { return } } } if err := scanner.Err(); err != nil { yield(llm.StreamChunk{}, fmt.Errorf("stream error: %w", err)) } } } func (c *Client) Name() string { return "llama.cpp" } func (c *Client) Capabilities() llm.ProviderCapabilities { return llm.ProviderCapabilities{ SupportsTools: true, SupportsVision: false, SupportsJSON: true, MaxContextWindow: 32768, } } // buildRequest converts an llm.CompletionRequest to the llama.cpp API format. func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader, error) { messages := make([]llamaMessage, len(req.Messages)) for i, m := range req.Messages { messages[i] = llamaMessage{ Role: string(m.Role), Content: m.Content, } } var tools []llamaTool for _, t := range req.Tools { var tool llamaTool if err := json.Unmarshal(t, &tool); err != nil { return nil, fmt.Errorf("parsing tool: %w", err) } tools = append(tools, tool) } openReq := llamaChatRequest{ Model: req.Model, Messages: messages, Stream: stream, ChatTemplateKwargs: req.ChatTemplateKwargs, } if stream { // Ask for a final SSE event carrying token usage (OpenAI-style // streaming omits it otherwise), so Rony can track real token // counts per turn instead of always seeing zero. openReq.StreamOptions = &llamaStreamOptions{IncludeUsage: true} } if len(tools) > 0 { openReq.Tools = tools } if req.ToolChoice != nil { openReq.ToolChoice = req.ToolChoice } if req.Temperature != nil { tmp := *req.Temperature openReq.Temperature = tmp } if req.MaxTokens != nil { tmp := *req.MaxTokens openReq.MaxTokens = tmp } if len(req.Stop) > 0 { openReq.Stop = req.Stop } data, err := json.Marshal(openReq) if err != nil { return nil, fmt.Errorf("marshaling request: %w", err) } return strings.NewReader(string(data)), nil } // toResponse converts a llama.cpp API response to our CompletionResponse. func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse { choice := resp.Choices[0] result := llm.CompletionResponse{ ID: resp.ID, Model: resp.Model, Content: choice.Message.Content, Reasoning: choice.Message.ReasoningContent, StopReason: choice.FinishReason, } for _, tc := range choice.Message.ToolCalls { result.ToolCalls = append(result.ToolCalls, llm.ToolCall{ ID: tc.ID, Name: tc.Function.Name, Arguments: json.RawMessage(tc.Function.Arguments), }) } result.Usage = llm.TokenUsage{ InputTokens: resp.Usage.PromptTokens, OutputTokens: resp.Usage.CompletionTokens, TotalTokens: resp.Usage.TotalTokens, } return result } // llama.cpp API types type llamaChatRequest struct { Model string `json:"model"` Messages []llamaMessage `json:"messages"` Tools []llamaTool `json:"tools,omitempty"` ToolChoice interface{} `json:"tool_choice,omitempty"` Temperature float32 `json:"temperature,omitempty"` MaxTokens int `json:"max_tokens,omitempty"` TopK int `json:"top_k,omitempty"` TopP float32 `json:"top_p,omitempty"` Stop []string `json:"stop,omitempty"` Stream bool `json:"stream"` StreamOptions *llamaStreamOptions `json:"stream_options,omitempty"` ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"` } type llamaStreamOptions struct { IncludeUsage bool `json:"include_usage"` } type llamaMessage struct { Role string `json:"role"` Content string `json:"content"` } type llamaTool struct { Type string `json:"type"` Function json.RawMessage `json:"function"` } type llamaChatResponse struct { ID string `json:"id"` Model string `json:"model"` Choices []llamaChoice `json:"choices"` Usage llamaUsage `json:"usage"` } type llamaChoice struct { Index int `json:"index"` Message llamaMessageResult `json:"message"` FinishReason string `json:"finish_reason"` } type llamaMessageResult struct { Role string `json:"role"` Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` ToolCalls []llamaToolCall `json:"tool_calls"` } type llamaToolCall struct { ID string `json:"id"` Type string `json:"type"` Function llamaFunction `json:"function"` } type llamaFunction struct { Name string `json:"name"` Arguments string `json:"arguments"` } type llamaUsage struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` TotalTokens int `json:"total_tokens"` } // Stream event types type llamaStreamEvent struct { ID string `json:"id"` Choices []llamaStreamChoice `json:"choices"` Usage *llamaUsage `json:"usage"` } type llamaStreamChoice struct { Index int `json:"index"` Delta llamaStreamDelta `json:"delta"` FinishReason string `json:"finish_reason"` } type llamaStreamDelta struct { Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` Role string `json:"role"` ToolCalls []llamaStreamToolCall `json:"tool_calls"` } type llamaStreamToolCall struct { Index int `json:"index"` ID string `json:"id"` Type string `json:"type"` Function llamaStreamFunction `json:"function"` } type llamaStreamFunction struct { Name string `json:"name"` Arguments string `json:"arguments"` }