rony-llm-agent/pkg/llm/providers/llamacpp/client.go

package llamacpp

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"iter"
	"strings"

	"github.com/VictorVargas/rony-llm-agent/pkg/llm"
)

// Config holds the settings needed to create a llama.cpp client.
type Config struct {
	BaseURL   string // defaults to http://localhost:8080/v1
	Model     string
	Timeout   int    // request timeout in seconds (0 = default)
	TopK      int    // top-k sampling (0 = default)
	TopP      float32
	Temperature float32
}

// Client implements llm.LLMClient for llama.cpp.
type Client struct {
	baseURL string
	http    *http.Client
}

// New returns a new llama.cpp client.
func New(cfg Config) (*Client, error) {
	baseURL := cfg.BaseURL
	if baseURL == "" {
		baseURL = "http://localhost:8080/v1"
	}

	return &Client{
		baseURL: baseURL,
		http:    http.DefaultClient,
	}, nil
}

func (c *Client) Generate(ctx context.Context, req llm.CompletionRequest) (llm.CompletionResponse, error) {
	endpoint := c.baseURL + "/chat/completions"

	payload, err := c.buildRequest(req, false)
	if err != nil {
		return llm.CompletionResponse{}, fmt.Errorf("building request: %w", err)
	}

	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload)
	if err != nil {
		return llm.CompletionResponse{}, fmt.Errorf("creating request: %w", err)
	}
	httpReq.Header.Set("Content-Type", "application/json")

	resp, err := c.http.Do(httpReq)
	if err != nil {
		return llm.CompletionResponse{}, fmt.Errorf("request failed: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		body, _ := io.ReadAll(resp.Body)
		return llm.CompletionResponse{}, fmt.Errorf("API error %d: %s", resp.StatusCode, string(body))
	}

	var apiResp llamaChatResponse
	if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
		return llm.CompletionResponse{}, fmt.Errorf("decoding response: %w", err)
	}

	return c.toResponse(apiResp), nil
}

func (c *Client) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamChunk, error] {
	return func(yield func(llm.StreamChunk, error) bool) {
		endpoint := c.baseURL + "/chat/completions"

		payload, err := c.buildRequest(req, true)
		if err != nil {
			yield(llm.StreamChunk{}, fmt.Errorf("building request: %w", err))
			return
		}

		httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, payload)
		if err != nil {
			yield(llm.StreamChunk{}, fmt.Errorf("creating request: %w", err))
			return
		}
		httpReq.Header.Set("Content-Type", "application/json")

		resp, err := c.http.Do(httpReq)
		if err != nil {
			yield(llm.StreamChunk{}, fmt.Errorf("request failed: %w", err))
			return
		}
		defer resp.Body.Close()

		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
			body, _ := io.ReadAll(resp.Body)
			yield(llm.StreamChunk{}, fmt.Errorf("API error %d: %s", resp.StatusCode, string(body)))
			return
		}

		// toolCallAccum buffers one tool call's fragments as they stream in:
		// the OpenAI-compatible SSE format sends the id/name in the first
		// delta for a given tool-call index and the (potentially large)
		// arguments JSON in pieces across many subsequent deltas, so it
		// can't be handed to a tool handler until it's fully assembled.
		type toolCallAccum struct {
			id   string
			name string
			args strings.Builder
		}
		toolCallFrags := map[int]*toolCallAccum{}
		var toolCallOrder []int

		// flushToolCalls assembles the buffered fragments into complete
		// tool calls (called once finish_reason arrives) and resets the
		// accumulator for any further choices/events.
		flushToolCalls := func() []llm.ToolCall {
			if len(toolCallOrder) == 0 {
				return nil
			}
			calls := make([]llm.ToolCall, 0, len(toolCallOrder))
			for _, idx := range toolCallOrder {
				frag := toolCallFrags[idx]
				calls = append(calls, llm.ToolCall{
					ID:        frag.id,
					Name:      frag.name,
					Arguments: json.RawMessage(frag.args.String()),
				})
			}
			toolCallFrags = map[int]*toolCallAccum{}
			toolCallOrder = nil
			return calls
		}

		scanner := bufio.NewScanner(resp.Body)
		for scanner.Scan() {
			line := scanner.Text()
			if !strings.HasPrefix(line, "data: ") {
				continue
			}
			data := strings.TrimPrefix(line, "data: ")
			if data == "[DONE]" {
				return
			}

			var event llamaStreamEvent
			if err := json.Unmarshal([]byte(data), &event); err != nil {
				yield(llm.StreamChunk{}, fmt.Errorf("decoding event: %w", err))
				return
			}

			var usage llm.TokenUsage
			if event.Usage != nil {
				usage = llm.TokenUsage{
					InputTokens:  event.Usage.PromptTokens,
					OutputTokens: event.Usage.CompletionTokens,
					TotalTokens:  event.Usage.TotalTokens,
				}
			}

			if len(event.Choices) == 0 {
				// The usage-only event (per stream_options.include_usage)
				// carries no choices, so it needs its own chunk.
				if event.Usage != nil {
					if !yield(llm.StreamChunk{Usage: usage}, nil) {
						return
					}
				}
				continue
			}

			for _, choice := range event.Choices {
				hasFragment := len(choice.Delta.ToolCalls) > 0
				for _, tc := range choice.Delta.ToolCalls {
					frag, ok := toolCallFrags[tc.Index]
					if !ok {
						frag = &toolCallAccum{}
						toolCallFrags[tc.Index] = frag
						toolCallOrder = append(toolCallOrder, tc.Index)
					}
					if tc.ID != "" {
						frag.id = tc.ID
					}
					if tc.Function.Name != "" {
						frag.name = tc.Function.Name
					}
					frag.args.WriteString(tc.Function.Arguments)
				}

				chunk := llm.StreamChunk{
					Delta:          choice.Delta.Content,
					ReasoningDelta: choice.Delta.ReasoningContent,
					Usage:          usage,
				}
				if choice.FinishReason != "" {
					chunk.FinishReason = choice.FinishReason
					chunk.ToolCalls = flushToolCalls()
				}

				// A fragment-only event (a piece of a tool call's streamed
				// arguments, with nothing else in this delta) has nothing
				// yet for the agent loop to act on: it was buffered above,
				// so skip yielding an empty chunk for it.
				if hasFragment && chunk.Delta == "" && chunk.ReasoningDelta == "" && chunk.FinishReason == "" {
					continue
				}

				if !yield(chunk, nil) {
					return
				}
			}
		}
		if err := scanner.Err(); err != nil {
			yield(llm.StreamChunk{}, fmt.Errorf("stream error: %w", err))
		}
	}
}

func (c *Client) Name() string {
	return "llama.cpp"
}

func (c *Client) Capabilities() llm.ProviderCapabilities {
	return llm.ProviderCapabilities{
		SupportsTools:    true,
		SupportsVision:   false,
		SupportsJSON:     true,
		MaxContextWindow: 32768,
	}
}

// buildRequest converts an llm.CompletionRequest to the llama.cpp API format.
func (c *Client) buildRequest(req llm.CompletionRequest, stream bool) (io.Reader, error) {
	messages := make([]llamaMessage, len(req.Messages))
	for i, m := range req.Messages {
		messages[i] = llamaMessage{
			Role:       string(m.Role),
			Content:    m.Content,
			ToolCallID: m.ToolCallID,
			Name:       m.Name,
		}
		if len(m.ToolCalls) > 0 {
			calls := make([]llamaToolCall, len(m.ToolCalls))
			for j, tc := range m.ToolCalls {
				calls[j] = llamaToolCall{
					ID:   tc.ID,
					Type: "function",
					Function: llamaFunction{
						Name:      tc.Name,
						Arguments: string(tc.Arguments),
					},
				}
			}
			messages[i].ToolCalls = calls
		}
	}

	var tools []llamaTool
	for _, t := range req.Tools {
		var tool llamaTool
		if err := json.Unmarshal(t, &tool); err != nil {
			return nil, fmt.Errorf("parsing tool: %w", err)
		}
		tools = append(tools, tool)
	}

	openReq := llamaChatRequest{
		Model:              req.Model,
		Messages:           messages,
		Stream:             stream,
		ChatTemplateKwargs: req.ChatTemplateKwargs,
	}
	if stream {
		// Ask for a final SSE event carrying token usage (OpenAI-style
		// streaming omits it otherwise), so Rony can track real token
		// counts per turn instead of always seeing zero.
		openReq.StreamOptions = &llamaStreamOptions{IncludeUsage: true}
	}
	if len(tools) > 0 {
		openReq.Tools = tools
	}
	if req.ToolChoice != nil {
		openReq.ToolChoice = req.ToolChoice
	}
	if req.Temperature != nil {
		tmp := *req.Temperature
		openReq.Temperature = tmp
	}
	if req.MaxTokens != nil {
		tmp := *req.MaxTokens
		openReq.MaxTokens = tmp
	}
	if len(req.Stop) > 0 {
		openReq.Stop = req.Stop
	}

	data, err := json.Marshal(openReq)
	if err != nil {
		return nil, fmt.Errorf("marshaling request: %w", err)
	}
	return strings.NewReader(string(data)), nil
}

// toResponse converts a llama.cpp API response to our CompletionResponse.
func (c *Client) toResponse(resp llamaChatResponse) llm.CompletionResponse {
	choice := resp.Choices[0]
	result := llm.CompletionResponse{
		ID:         resp.ID,
		Model:      resp.Model,
		Content:    choice.Message.Content,
		Reasoning:  choice.Message.ReasoningContent,
		StopReason: choice.FinishReason,
	}

	for _, tc := range choice.Message.ToolCalls {
		result.ToolCalls = append(result.ToolCalls, llm.ToolCall{
			ID:        tc.ID,
			Name:      tc.Function.Name,
			Arguments: json.RawMessage(tc.Function.Arguments),
		})
	}

	result.Usage = llm.TokenUsage{
		InputTokens:  resp.Usage.PromptTokens,
		OutputTokens: resp.Usage.CompletionTokens,
		TotalTokens:  resp.Usage.TotalTokens,
	}

	return result
}

// llama.cpp API types

type llamaChatRequest struct {
	Model              string              `json:"model"`
	Messages           []llamaMessage      `json:"messages"`
	Tools              []llamaTool         `json:"tools,omitempty"`
	ToolChoice        interface{}         `json:"tool_choice,omitempty"`
	Temperature       float32             `json:"temperature,omitempty"`
	MaxTokens         int                `json:"max_tokens,omitempty"`
	TopK              int                `json:"top_k,omitempty"`
	TopP              float32            `json:"top_p,omitempty"`
	Stop              []string           `json:"stop,omitempty"`
	Stream            bool               `json:"stream"`
	StreamOptions      *llamaStreamOptions `json:"stream_options,omitempty"`
	ChatTemplateKwargs map[string]any     `json:"chat_template_kwargs,omitempty"`
}

type llamaStreamOptions struct {
	IncludeUsage bool `json:"include_usage"`
}

type llamaMessage struct {
	Role       string          `json:"role"`
	Content    string          `json:"content"`
	ToolCallID string          `json:"tool_call_id,omitempty"`
	Name       string          `json:"name,omitempty"`
	ToolCalls  []llamaToolCall `json:"tool_calls,omitempty"`
}

type llamaTool struct {
	Type     string          `json:"type"`
	Function json.RawMessage `json:"function"`
}

type llamaChatResponse struct {
	ID      string               `json:"id"`
	Model   string               `json:"model"`
	Choices []llamaChoice        `json:"choices"`
	Usage   llamaUsage           `json:"usage"`
}

type llamaChoice struct {
	Index        int                `json:"index"`
	Message      llamaMessageResult `json:"message"`
	FinishReason string              `json:"finish_reason"`
}

type llamaMessageResult struct {
	Role             string          `json:"role"`
	Content          string          `json:"content"`
	ReasoningContent string          `json:"reasoning_content"`
	ToolCalls        []llamaToolCall `json:"tool_calls"`
}

type llamaToolCall struct {
	ID       string             `json:"id"`
	Type     string             `json:"type"`
	Function llamaFunction      `json:"function"`
}

type llamaFunction struct {
	Name        string `json:"name"`
	Arguments   string `json:"arguments"`
}

type llamaUsage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}

// Stream event types

type llamaStreamEvent struct {
	ID      string               `json:"id"`
	Choices []llamaStreamChoice `json:"choices"`
	Usage   *llamaUsage          `json:"usage"`
}

type llamaStreamChoice struct {
	Index        int                  `json:"index"`
	Delta        llamaStreamDelta    `json:"delta"`
	FinishReason string               `json:"finish_reason"`
}

type llamaStreamDelta struct {
	Content          string                `json:"content"`
	ReasoningContent string                `json:"reasoning_content"`
	Role              string               `json:"role"`
	ToolCalls         []llamaStreamToolCall `json:"tool_calls"`
}

type llamaStreamToolCall struct {
	Index    int                   `json:"index"`
	ID       string                `json:"id"`
	Type     string                `json:"type"`
	Function llamaStreamFunction   `json:"function"`
}

type llamaStreamFunction struct {
	Name        string `json:"name"`
	Arguments   string `json:"arguments"`
}