rony-llm-agent/pkg/llm/providers/llamacpp/client.go

package llamacpp

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"iter"
	"strings"

	"github.com/VictorVargas/rony-llm-agent/pkg/llm"
)

// Config holds the settings needed to create a llama.cpp client.
type Config struct {
	BaseURL   string // defaults to http://localhost:8080/v1
	Model     string
	Timeout   int    // request timeout in seconds (0 = default)
	TopK      int    // top-k sampling (0 = default)
	TopP      float32
	Temperature float32
}

// Client implements llm.LLMClient for llama.cpp.
type Client struct {
	baseURL string
	http    *http.Client
}

// New returns a new llama.cpp client.
func New(cfg Config) (*Client, error) {
	baseURL := cfg.BaseURL
	if baseURL == "" {
		baseURL = "http://localhost:8080/v1"
	}

	return &Client{
		baseURL: baseURL,
		http:    http.DefaultClient,
	}, nil
}

func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.CompletionResponse, error) {
	endpoint := c.baseURL + "/chat/completions"

	payload, err := c.buildRequest(req, false)
	if err != nil {
		return llm.CompletionResponse{}, fmt.Errorf("building request: %w", err)
	}

	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload)
	if err != nil {
		return llm.CompletionResponse{}, fmt.Errorf("creating request: %w", err)
	}
	httpReq.Header.Set("Content-Type", "application/json")

	resp, err := c.http.Do(httpReq)
	if err != nil {
		return llm.CompletionResponse{}, fmt.Errorf("request failed: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		body, _ := io.ReadAll(resp.Body)
		return llm.CompletionResponse{}, fmt.Errorf("API error %d: %s", resp.StatusCode, string(body))
	}

	var apiResp llamaChatResponse
	if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
		return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err)
	}

	return c.toResponse(apiResp), nil
}

func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] {
	return func(yield func(llm.StreamChunk, error) bool) {
		endpoint := c.baseURL + "/chat/completions"

		payload, err := c.buildRequest(req, true)
		if err != nil {
			yield(llm.StreamChunk{}, fmt.Errorf("building request: %w", err))
			return
		}

		httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload)
		if err != nil {
			yield(llm.StreamChunk{}, fmt.Errorf("creating request: %w", err))
			return
		}
		httpReq.Header.Set("Content-Type", "application/json")

		resp, err := c.http.Do(httpReq)
		if err != nil {
			yield(llm.StreamChunk{}, fmt.Errorf("request failed: %w", err))
			return
		}
		defer resp.Body.Close()

		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
			body, _ := io.ReadAll(resp.Body)
			yield(llm.StreamChunk{}, fmt.Errorf("API error %d: %s", resp.StatusCode, string(body)))
			return
		}

		scanner := bufio.NewScanner(resp.Body)
		for scanner.Scan() {
			line := scanner.Text()
			if !strings.HasPrefix(line, "data: ") {
				continue
			}
			data := strings.TrimPrefix(line, "data: ")
			if data == "[DONE]" {
				return
			}

			var event llamaStreamEvent
			if err := json.Unmarshal([]byte(data), &event); err != nil {
				yield(llm.StreamChunk{}, fmt.Errorf("decoding event: %w", err))
				return
			}

			for _, choice := range event.Choices {
				chunk := llm.StreamChunk{
					Delta:          choice.Delta.Content,
					ReasoningDelta: choice.Delta.ReasoningContent,
				}
				if choice.FinishReason != "" {
					chunk.FinishReason = choice.FinishReason
				}
				if !yield(chunk, nil) {
					return
				}
			}
		}
		if err := scanner.Err(); err != nil {
			yield(llm.StreamChunk{}, fmt.Errorf("stream error: %w", err))
		}
	}
}

func (c *Client) Name() string {
	return "llama.cpp"
}

func (c *Client) Capabilities() llm.ProviderCapabilities {
	return llm.ProviderCapabilities{
		SupportsTools:    true,
		SupportsVision:   false,
		SupportsJSON:     true,
		MaxContextWindow: 32768,
	}
}

// buildRequest converts an llm.CompletionRequest to the llama.cpp API format.
func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader, error) {
	messages := make([]llamaMessage, len(req.Messages))
	for i, m := range req.Messages {
		messages[i] = llamaMessage{
			Role:    string(m.Role),
			Content: m.Content,
		}
	}

	var tools []llamaTool
	for _, t := range req.Tools {
		var tool llamaTool
		if err := json.Unmarshal(t, &tool); err != nil {
			return nil, fmt.Errorf("parsing tool: %w", err)
		}
		tools = append(tools, tool)
	}

	openReq := llamaChatRequest{
		Model:              req.Model,
		Messages:           messages,
		Stream:             stream,
		ChatTemplateKwargs: req.ChatTemplateKwargs,
	}
	if len(tools) > 0 {
		openReq.Tools = tools
	}
	if req.ToolChoice != nil {
		openReq.ToolChoice = req.ToolChoice
	}
	if req.Temperature != nil {
		tmp := *req.Temperature
		openReq.Temperature = tmp
	}
	if req.MaxTokens != nil {
		tmp := *req.MaxTokens
		openReq.MaxTokens = tmp
	}
	if len(req.Stop) > 0 {
		openReq.Stop = req.Stop
	}

	data, err := json.Marshal(openReq)
	if err != nil {
		return nil, fmt.Errorf("marshaling request: %w", err)
	}
	return strings.NewReader(string(data)), nil
}

// toResponse converts a llama.cpp API response to our CompletionResponse.
func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse {
	choice := resp.Choices[0]
	result := llm.CompletionResponse{
		ID:         resp.ID,
		Model:      resp.Model,
		Content:    choice.Message.Content,
		Reasoning:  choice.Message.ReasoningContent,
		StopReason: choice.FinishReason,
	}

	for _, tc := range choice.Message.ToolCalls {
		result.ToolCalls = append(result.ToolCalls, llm.ToolCall{
			ID:        tc.ID,
			Name:      tc.Function.Name,
			Arguments: json.RawMessage(tc.Function.Arguments),
		})
	}

	result.Usage = llm.TokenUsage{
		InputTokens:  resp.Usage.PromptTokens,
		OutputTokens: resp.Usage.CompletionTokens,
		TotalTokens:  resp.Usage.TotalTokens,
	}

	return result
}

// llama.cpp API types

type llamaChatRequest struct {
	Model              string              `json:"model"`
	Messages           []llamaMessage      `json:"messages"`
	Tools              []llamaTool         `json:"tools,omitempty"`
	ToolChoice        interface{}         `json:"tool_choice,omitempty"`
	Temperature       float32             `json:"temperature,omitempty"`
	MaxTokens         int                `json:"max_tokens,omitempty"`
	TopK              int                `json:"top_k,omitempty"`
	TopP              float32            `json:"top_p,omitempty"`
	Stop              []string           `json:"stop,omitempty"`
	Stream            bool               `json:"stream"`
	ChatTemplateKwargs map[string]any     `json:"chat_template_kwargs,omitempty"`
}

type llamaMessage struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type llamaTool struct {
	Type     string          `json:"type"`
	Function json.RawMessage `json:"function"`
}

type llamaChatResponse struct {
	ID      string               `json:"id"`
	Model   string               `json:"model"`
	Choices []llamaChoice        `json:"choices"`
	Usage   llamaUsage           `json:"usage"`
}

type llamaChoice struct {
	Index        int                `json:"index"`
	Message      llamaMessageResult `json:"message"`
	FinishReason string              `json:"finish_reason"`
}

type llamaMessageResult struct {
	Role             string          `json:"role"`
	Content          string          `json:"content"`
	ReasoningContent string          `json:"reasoning_content"`
	ToolCalls        []llamaToolCall `json:"tool_calls"`
}

type llamaToolCall struct {
	ID       string             `json:"id"`
	Type     string             `json:"type"`
	Function llamaFunction      `json:"function"`
}

type llamaFunction struct {
	Name        string `json:"name"`
	Arguments   string `json:"arguments"`
}

type llamaUsage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}

// Stream event types

type llamaStreamEvent struct {
	ID      string               `json:"id"`
	Choices []llamaStreamChoice `json:"choices"`
}

type llamaStreamChoice struct {
	Index        int                  `json:"index"`
	Delta        llamaStreamDelta    `json:"delta"`
	FinishReason string               `json:"finish_reason"`
}

type llamaStreamDelta struct {
	Content          string                `json:"content"`
	ReasoningContent string                `json:"reasoning_content"`
	Role              string               `json:"role"`
	ToolCalls         []llamaStreamToolCall `json:"tool_calls"`
}

type llamaStreamToolCall struct {
	Index    int                   `json:"index"`
	ID       string                `json:"id"`
	Type     string                `json:"type"`
	Function llamaStreamFunction   `json:"function"`
}

type llamaStreamFunction struct {
	Name        string `json:"name"`
	Arguments   string `json:"arguments"`
}