From 4c5cad38f823dbcfcc2b8f34f4bf40c389e1f5a4 Mon Sep 17 00:00:00 2001 From: Victor Hugo Vargas Date: Sat, 18 Jul 2026 00:07:41 -0700 Subject: [PATCH] feat(config): add auto-compaction block and CLI wiring Adds a configurable compaction section to portfolio-bot.yaml with threshold_ratio, keep_recent_turns and an optional summary prompt. Wires the new fields through config.Validate() and cmd/chat-bot/main.go into agent.Runner.WithCompaction() so the runner can opt in to auto-compaction at startup. --- cmd/chat-bot/main.go | 16 +++++++++++-- configs/portfolio-bot.yaml | 18 +++++++++++++-- internal/config/config.go | 46 +++++++++++++++++++++++++++++++++----- 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/cmd/chat-bot/main.go b/cmd/chat-bot/main.go index afd58d6..22e6f52 100644 --- a/cmd/chat-bot/main.go +++ b/cmd/chat-bot/main.go @@ -115,7 +115,13 @@ func serveCmd() *cobra.Command { if err != nil { return err } - runner := agent.New(client, p, cfg.SystemPrompt, store, cfg.RAG.TopK) + runner := agent.New(client, p, cfg.SystemPrompt, store, cfg.RAG.TopK). + WithCompaction(agent.CompactionConfig{ + Enabled: cfg.Compaction.Enabled, + ThresholdRatio: cfg.Compaction.ThresholdRatio, + KeepRecentTurns: cfg.Compaction.KeepRecentTurns, + SummarySystemPrompt: cfg.Compaction.SummarySystemPrompt, + }) h := server.NewHandlers(cfg, runner, store, version) srv := server.New(cfg, h) @@ -212,7 +218,13 @@ func runAsk(cfg *config.Config, question string, noStream bool) error { if err != nil { return err } - runner := agent.New(client, p, cfg.SystemPrompt, store, cfg.RAG.TopK) + runner := agent.New(client, p, cfg.SystemPrompt, store, cfg.RAG.TopK). + WithCompaction(agent.CompactionConfig{ + Enabled: cfg.Compaction.Enabled, + ThresholdRatio: cfg.Compaction.ThresholdRatio, + KeepRecentTurns: cfg.Compaction.KeepRecentTurns, + SummarySystemPrompt: cfg.Compaction.SummarySystemPrompt, + }) history := []agent.Message{{Role: agent.RoleUser, Content: question}} if noStream { diff --git a/configs/portfolio-bot.yaml b/configs/portfolio-bot.yaml index 2cf8004..5c26f91 100644 --- a/configs/portfolio-bot.yaml +++ b/configs/portfolio-bot.yaml @@ -21,7 +21,7 @@ providers: type: llamacpp model: qwen2.5-3b-instruct endpoint: http://localhost:9100/v1 - context_size: 4096 + context_size: 2048 max_tokens: 2048 default: true @@ -123,4 +123,18 @@ system_prompt: | logging: level: info # debug | info | warn | error format: json # json | text - output: stderr \ No newline at end of file + output: stderr + +# Auto-compaction: fold the older part of a long conversation into a single +# summary message before sending it to the model, so context overflow doesn't +# kill long threads. Triggered when the previous turn's input tokens exceed +# `threshold_ratio` of the provider's reported MaxContextWindow. +# +# Off by default — most portfolio chats are short. Turn it on for chatty +# visitors or for the small-context local models (qwen2.5-1.5b / 3b) where +# 4–6 turns is already most of the window. +compaction: + enabled: true + threshold_ratio: 0.75 # compact at 75% of context window + keep_recent_turns: 2 # last 2 user turns kept verbatim; older → summary + # summary_system_prompt: "" # leave empty for the built-in bilingual default \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index c729a52..7eeb03d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -59,13 +59,41 @@ type Logging struct { Output string `yaml:"output"` } +// Compaction controls automatic context-window management. When the most +// recent request's input tokens exceed threshold_ratio × the provider's +// reported MaxContextWindow, the older portion of the conversation is +// summarized by the LLM and replaced with a single system-role note. +// Recent turns (keep_recent_turns) are always preserved verbatim. +// +// Disabled by default; flip enabled: true in the YAML to opt in. The +// model's MaxContextWindow comes from llm.LLMClient.Capabilities(), so +// the same setting works for llamacpp, ollama, openai and anthropic +// without per-provider configuration. +type Compaction struct { + Enabled bool `yaml:"enabled"` + // ThresholdRatio is the fraction of the context window that, once + // exceeded by the previous turn's input tokens, triggers compaction. + // Range 0–1; default 0.75 (75% of the window). + ThresholdRatio float64 `yaml:"threshold_ratio"` + // KeepRecentTurns is how many of the most recent user turns (each + // being a user message plus everything up to the next user message) + // to keep verbatim after compaction. Older turns are folded into the + // summary. Default 4. + KeepRecentTurns int `yaml:"keep_recent_turns"` + // SummarySystemPrompt is the instruction sent to the LLM when it + // summarizes the older turns. Empty falls back to a built-in + // bilingual prompt that auto-matches the conversation language. + SummarySystemPrompt string `yaml:"summary_system_prompt"` +} + type Config struct { - Server Server `yaml:"server"` + Server Server `yaml:"server"` Providers []Provider `yaml:"providers"` - RAG RAG `yaml:"rag"` - Persona Persona `yaml:"persona"` - SystemPrompt string `yaml:"system_prompt"` - Logging Logging `yaml:"logging"` + RAG RAG `yaml:"rag"` + Persona Persona `yaml:"persona"` + SystemPrompt string `yaml:"system_prompt"` + Compaction Compaction `yaml:"compaction"` + Logging Logging `yaml:"logging"` } func Load(path string) (*Config, error) { @@ -120,6 +148,14 @@ func (c *Config) validate() error { if c.RAG.Tokenize == "" { c.RAG.Tokenize = "unicode61" } + if c.Compaction.Enabled { + if c.Compaction.ThresholdRatio <= 0 || c.Compaction.ThresholdRatio > 1 { + c.Compaction.ThresholdRatio = 0.75 + } + if c.Compaction.KeepRecentTurns <= 0 { + c.Compaction.KeepRecentTurns = 4 + } + } return nil }