diff --git a/.gitignore b/.gitignore index 438cd67..e721ab5 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,11 @@ coverage.html *.db-wal data/portfolio.db +# Reference documents (data/docs/): personal material such as a CV, or a +# symlink to a path that only exists on one machine. See data/docs/README.md. +data/docs/* +!data/docs/README.md + # Editor / OS .vscode/ .idea/ diff --git a/README.es.md b/README.es.md index 0c93265..3b5c961 100644 --- a/README.es.md +++ b/README.es.md @@ -10,7 +10,7 @@ ## ✨ Features - 🌐 **HTTP server** con streaming SSE (Server-Sent Events) -- 🧠 **RAG sobre markdown** — indexa automáticamente los `.md` en `data/projects/` (SQLite FTS5, sin embeddings) +- 🧠 **RAG híbrido sobre markdown/MDX** — búsqueda por palabras con SQLite FTS5 fusionada con embeddings multilingües (Reciprocal Rank Fusion) - 🎭 **Persona customizable** — responde como "asistente de Victor" - ⚡ **Self-hosted** con llama.cpp (default) u Ollama (no requiere API key de cloud) - 💬 **Widget de chat drop-in** — vanilla JS, sin build step, funciona en cualquier sitio @@ -22,14 +22,14 @@ ```bash # 1. Instalar git clone https://github.com/VictorVargas/rony-chat-bot.git -cd chat-bot +cd rony-chat-bot # 2. Resolver dependencias (crea go.sum con hashes) go mod tidy -# 3. Configurar provider (llama.cpp por default) -# Descarga un modelo GGUF, ej.: -# https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF +# 3. Descargá los modelos: un LLM instruct y un embebedor multilingüe +# https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF (~2 GB) +# https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe-GGUF (~370 MB) export RONY_MODELS_PATH=/path/to/models # 4. Cargar tus proyectos en data/projects/ @@ -38,11 +38,131 @@ echo "# Mi Proyecto Cool\nDescripción..." > data/projects/mi-proyecto.md # 5. Build go build -o bin/chat-bot ./cmd/chat-bot -# 6. Run +# 6. Arrancar el LLM (CPU, target 2 cores — ajustá --threads a tu host) +llama-server \ + -m $RONY_MODELS_PATH/Qwen2.5/qwen2.5-3b-instruct-q4_k_m.gguf \ + --port 9100 --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 + +# 7. Arrancar el embebedor (segunda terminal) +llama-server \ + -m $RONY_MODELS_PATH/embeddings/nomic-embed-v2-moe.Q5_K_M.gguf \ + --port 9200 --embedding --pooling mean \ + --ctx-size 2048 --parallel 1 --device none --threads 2 + +# 8. Construir el índice (necesita el embebedor arriba) y servir +./bin/chat-bot reindex ./bin/chat-bot serve # → Sirve en http://localhost:7331 ``` +**Hardware mínimo:** 2 CPU cores, 8 GB RAM, sin GPU. Memoria residente medida +en CPU con este setup: **3,64 GB** el LLM, **0,91 GB** el embebedor, **0,02 GB** +el bot — unos **4,6 GB**, dejando ~3,4 GB para el resto del host. La generación +va a 21 tok/s con 2 threads una vez que el system prompt está caliente en la +caché de prompts de llama-server. + +Tres flags son fáciles de errar y cada una te cuesta calidad real: + +- **`--device none`** — llama.cpp levanta el backend de GPU compilado aunque + pases `-ngl 0`, y en un host sin GPU esos buffers salen de la RAM del sistema. + Medido en qwen2.5-3b: 2,54 GB con una GPU absorbiéndolos, 3,66 GB sin ella. + Presupuestá con el segundo número, y pasá la flag para que la medición + coincida con lo que hace producción. +- **`--parallel 1`** — `--ctx-size` se reparte entre slots y llama-server abre 4 + por defecto, así que `--ctx-size 4096` sin esto le deja a cada request solo + 1024 tokens. El rate limiter del bot ya limita la concurrencia. +- **`--temp` / `--top-k` / `--top-p`** — usá los valores que publican los autores + del modelo, no los defaults de llama.cpp. Los de arriba son los de Qwen para + chat instruct. Errar esto no es sutil: gemma-3-1b con `temperature 0.7` y el + resto sin setear devolvía respuestas de 16 tokens. + +**`--pooling mean` es obligatoria en el embebedor.** Sin ella el endpoint no +devuelve un vector por entrada y el cliente rechaza la respuesta. + +Mantené `context_size` en `configs/portfolio-bot.yaml` igual a `--ctx-size`; el +bot calcula sus presupuestos de RAG y compactación a partir de ese número y no +le pregunta al servidor qué tiene en realidad. Si los desacoplás, el bot armará +prompts que el servidor rechaza. + +**Por qué 4096 alcanza.** Medido sobre 20 peticiones reales, el prompt más +grande que este bot llegó a construir fue de **1255 tokens** — system prompt, +catálogo de proyectos, cinco chunks recuperados y la pregunta. La compactación +recién arranca al 75% de la ventana (~3070 tokens), así que hay 2,4x de margen +antes de que empiece. Bajar de 8192 a 4096 ahorró **212 MB** de memoria +residente sin un solo truncamiento y con el mismo rendimiento (21,0 tok/s en +ambos casos): el contexto extra estaba reservado y nunca se usaba. + +## 📚 Proyectos vs. documentos de referencia + +El índice tiene dos tipos de fuente, y ambas aceptan `.md` y `.mdx`: + +```yaml +rag: + data_path: ./data/projects # proyectos → aparecen en el catálogo + docs_path: ./data/docs # referencia → se busca, nunca se lista +``` + +Todo lo que está en `data_path` es un proyecto tuyo y se anuncia en el catálogo +que el bot inyecta en cada prompt. Todo lo que está en `docs_path` es evidencia +buscable que *no* es un proyecto: tu CV, una página "sobre mí", un FAQ. + +Tu CV va en `docs_path`. Es el documento que responde lo que de verdad pregunta +quien está evaluando contratarte ("¿sabe Kubernetes?", "¿dónde ha trabajado?"), +y nada de eso es recuperable mientras viva solo en tu sitio. Enlazalo para +mantener una sola copia: + +```bash +mkdir -p data/docs +ln -s ../../../portfolio/src/content/cv/cv.mdx data/docs/cv.mdx +./bin/chat-bot reindex +``` + +Sin esta distinción el CV tendría que ir en `data_path` para ser buscable, y +entonces el bot lista alegremente "cv" como uno de tus proyectos. + +Actualizar una instalación existente no requiere migración: la tabla de chunks +es dato derivado, así que el store la reconstruye al abrir y el siguiente +`reindex` la repuebla. + +## 🔍 Recuperación: keyword + embeddings + +La recuperación es híbrida, y las dos mitades hacen falta. + +**Keyword (SQLite FTS5)** hace coincidencia exacta de palabras: sin stemming, +sin traducción. Preciso para nombres propios raros, inútil entre idiomas. El +corpus está en inglés y los visitantes preguntan en español, así que las +palabras con carga semántica puntúan cero: medido sobre el corpus real, +`"paga"` aparece 0 veces en un documento que dice *"Payments: Stripe"*, y +`"trabajado"` 0 veces en uno que dice *"worked"*. La pregunta *"¿Con qué se +paga en la tienda de ropa?"* no recuperaba **nada**. + +**Embeddings** (`nomic-embed-v2-moe`, multilingüe) cierran ese hueco: esa +misma pregunta pone los tres chunks de `tienda-ropa` arriba. Son más difusos +que BM25 ante un término raro exacto, y por eso se conservan ambos y se +fusionan con Reciprocal Rank Fusion — RRF ordena por acuerdo entre los dos +rankings, evitando comparar una puntuación BM25 con un coseno, magnitudes sin +escala común. + +Se activa en `configs/portfolio-bot.yaml` bajo `embeddings:` y hay que +re-ejecutar `reindex` — los vectores se construyen al indexar. Si el endpoint +se cae o se desactiva, la recuperación degrada a keyword-only en vez de fallar. + +Dos detalles que cuestan precisión y son fáciles de pasar por alto: + +- **El frontmatter se excluye de la recuperación.** Son metadatos densos + (title, tags, repo, location) en un chunk muy corto, lo que lo convierte en + imán de consultas breves. El campo `location:` de un CV hacía que *"¿Dónde + ha trabajado Victor?"* recuperara el frontmatter en vez del historial + laboral, porque "dónde" casa con una ubicación. +- **Las secciones largas se cortan en encabezados `###`, no por bytes.** La + sección Experience de un CV es una lista de empleos; el corte por tamaño + partía una entrada a mitad de palabra, dejando un chunk que empezaba *"... + id app for an on-demand ride-sharing service"* y el nombre del empleador + huérfano en el trozo anterior. Ahora cada chunk es un empleo, nombrado + `Experience — Metrimex — Frontend Developer`. + ## 📁 Estructura ``` diff --git a/README.md b/README.md index 90118eb..7b679f5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ ## ✨ Features - 🌐 **HTTP server** with SSE (Server-Sent Events) streaming -- 🧠 **RAG over markdown** — automatically indexes `.md` in `data/projects/` (SQLite FTS5, no embeddings) +- 🧠 **Hybrid RAG over markdown/MDX** — SQLite FTS5 keyword search fused with multilingual embeddings (Reciprocal Rank Fusion) - 🎭 **Customizable persona** — responds as "Victor's assistant" - ⚡ **Self-hosted** with llama.cpp (default) or Ollama (no cloud API key required) - 💬 **Drop-in chat widget** — vanilla JS, no build step, works in any site @@ -26,9 +26,9 @@ cd rony-chat-bot # 2. Resolve dependencies (creates go.sum with hashes) go mod tidy -# 3. Configure provider (llama.cpp by default) -# Download a GGUF model, e.g.: -# https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF +# 3. Download the models: an instruct LLM and a multilingual embedder +# https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF (~2 GB) +# https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe-GGUF (~370 MB) export RONY_MODELS_PATH=/path/to/models # 4. Load your projects in data/projects/ @@ -37,11 +37,130 @@ echo "# My Cool Project\nDescription..." > data/projects/my-project.md # 5. Build go build -o bin/chat-bot ./cmd/chat-bot -# 6. Run +# 6. Start the LLM (CPU, 2 cores target — adjust --threads to your host) +llama-server \ + -m $RONY_MODELS_PATH/Qwen2.5/qwen2.5-3b-instruct-q4_k_m.gguf \ + --port 9100 --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 + +# 7. Start the embedder (second terminal) +llama-server \ + -m $RONY_MODELS_PATH/embeddings/nomic-embed-v2-moe.Q5_K_M.gguf \ + --port 9200 --embedding --pooling mean \ + --ctx-size 2048 --parallel 1 --device none --threads 2 + +# 8. Build the index (needs the embedder running), then serve +./bin/chat-bot reindex ./bin/chat-bot serve # → Serves on http://localhost:7331 ``` +**Minimum hardware:** 2 CPU cores, 8 GB RAM, no GPU. Measured resident memory +on CPU with the setup above: **3.64 GB** for the LLM, **0.91 GB** for the +embedder, **0.02 GB** for the bot — about **4.6 GB**, leaving ~3.4 GB for the +rest of the host. Generation runs at 21 tok/s on 2 threads once the system +prompt is warm in llama-server's prompt cache. + +Three flags are easy to get wrong and each one costs you real quality: + +- **`--device none`** — llama.cpp brings up a compiled-in GPU backend even with + `-ngl 0`, and on a host with no GPU those buffers come out of system RAM. + Measured on qwen2.5-3b: 2.54 GB with a GPU absorbing them, 3.66 GB without. + Budget from the second number, and pass this flag so the measurement matches + what production actually does. +- **`--parallel 1`** — `--ctx-size` is divided across slots and llama-server opens + 4 by default, so `--ctx-size 4096` without this gives each request only 1024 + tokens. The bot's rate limiter already caps concurrency. +- **`--temp` / `--top-k` / `--top-p`** — use the values the model's authors + publish, not llama.cpp's defaults. The ones above are Qwen's for instruct + chat. Getting this wrong is not subtle: gemma-3-1b at `temperature 0.7` with + the rest unset produced 16-token stub answers. + +**`--pooling mean` is mandatory on the embedder.** Without it the endpoint does +not return one vector per input and the client rejects the response. + +Keep `context_size` in `configs/portfolio-bot.yaml` equal to `--ctx-size`; the +bot sizes its RAG and compaction budgets from that number and does not ask the +server what it actually has. Set them apart and the bot will build prompts the +server rejects. + +**Why 4096 is enough.** Measured over 20 real requests, the largest prompt this +bot ever built was **1255 tokens** — system prompt, project catalogue, five +retrieved chunks and the question. Compaction only begins at 75% of the window +(~3070 tokens), so there is 2.4x headroom before it even starts. Halving the +window from 8192 saved **212 MB** of resident memory with zero truncations and +identical throughput (21.0 tok/s either way): the extra context was reserved +and never used. + +## 📚 Projects vs. reference documents + +The index has two kinds of source, both accepting `.md` and `.mdx`: + +```yaml +rag: + data_path: ./data/projects # projects → listed in the catalogue + docs_path: ./data/docs # reference material → retrievable, never listed +``` + +Everything in `data_path` is one of your projects and is announced in the +project catalogue the bot injects into every prompt. Everything in `docs_path` +is searchable evidence that is *not* a project — your CV, an about page, a FAQ. + +Your CV belongs in `docs_path`. It is the document that answers what someone +considering hiring you actually asks ("does he know Kubernetes?", "where has he +worked?"), and none of it is retrievable while it lives only in your site. +Symlink it so there's a single copy to maintain: + +```bash +mkdir -p data/docs +ln -s ../../../portfolio/src/content/cv/cv.mdx data/docs/cv.mdx +./bin/chat-bot reindex +``` + +Without the distinction the CV has to go in `data_path` to be searchable, and +the bot then cheerfully lists "cv" as one of your projects. + +Upgrading an existing install needs no migration step: the chunk table is +derived data, so the store rebuilds it on open and the next `reindex` +repopulates it. + +## 🔍 Retrieval: keyword + embeddings + +Retrieval is hybrid, and both halves are load-bearing. + +**Keyword search (SQLite FTS5)** matches words exactly — no stemming, no +translation. That is precise for rare proper nouns and useless across +languages. The corpus is written in English and visitors ask in Spanish, so +the words carrying the meaning score zero: measured on the real corpus, +`"paga"` appears 0 times in a document that says *"Payments: Stripe"* and +`"trabajado"` 0 times in one that says *"worked"*. The question *"¿Con qué se +paga en la tienda de ropa?"* retrieved **nothing at all**. + +**Embeddings** (`nomic-embed-v2-moe`, multilingual) close that gap: the same +question puts all three `tienda-ropa` chunks on top. They are fuzzier than +BM25 on an exact rare token, which is why both are kept and fused with +Reciprocal Rank Fusion — RRF ranks by agreement between the two orderings, +which avoids comparing a BM25 score against a cosine, quantities that share +no scale. + +Enable it in `configs/portfolio-bot.yaml` under `embeddings:` and re-run +`reindex` — vectors are built at index time. If the endpoint is down or +disabled, retrieval degrades to keyword-only instead of failing. + +Two things that cost real accuracy and are easy to miss: + +- **Frontmatter is excluded from retrieval.** It is dense metadata (title, + tags, repo, location) in a very short chunk, which makes it a magnet for + short queries. A CV's `location:` field made *"¿Dónde ha trabajado + Victor?"* retrieve the frontmatter instead of the work history, because + "where" matches a location. +- **Long sections split at `###` headings, not byte offsets.** A CV's + Experience section is a list of jobs; size-splitting cut one entry + mid-word into a chunk beginning *"... id app for an on-demand ride-sharing + service"*, with the employer name stranded in the previous piece. Chunks + now keep one job each, named `Experience — Metrimex — Frontend Developer`. + ## 📁 Structure ``` diff --git a/cmd/chat-bot/main.go b/cmd/chat-bot/main.go index 22e6f52..9fb5955 100644 --- a/cmd/chat-bot/main.go +++ b/cmd/chat-bot/main.go @@ -17,6 +17,7 @@ import ( "github.com/VictorVargas/rony-chat-bot/internal/agent" "github.com/VictorVargas/rony-chat-bot/internal/config" + "github.com/VictorVargas/rony-chat-bot/internal/embed" "github.com/VictorVargas/rony-chat-bot/internal/persona" "github.com/VictorVargas/rony-chat-bot/internal/portfolio" "github.com/VictorVargas/rony-chat-bot/internal/server" @@ -116,6 +117,9 @@ func serveCmd() *cobra.Command { return err } runner := agent.New(client, p, cfg.SystemPrompt, store, cfg.RAG.TopK). + WithCatalog(cfg.RAG.IncludeCatalog). + WithLocalizedPrompt("es", cfg.SystemPromptES). + WithEmbedder(newEmbedder(cfg)). WithCompaction(agent.CompactionConfig{ Enabled: cfg.Compaction.Enabled, ThresholdRatio: cfg.Compaction.ThresholdRatio, @@ -123,7 +127,7 @@ func serveCmd() *cobra.Command { SummarySystemPrompt: cfg.Compaction.SummarySystemPrompt, }) - h := server.NewHandlers(cfg, runner, store, version) + h := server.NewHandlers(cfg, runner, store, version).WithEmbedder(newEmbedder(cfg)) srv := server.New(cfg, h) ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) @@ -162,8 +166,23 @@ func reindexCmd() *cobra.Command { } } +// newEmbedder returns the embeddings client, or nil when the feature is off. +// Nil is a supported value everywhere downstream: retrieval falls back to +// keyword-only rather than failing. +func newEmbedder(cfg *config.Config) portfolio.Embedder { + if !cfg.Embeddings.Enabled || cfg.Embeddings.Endpoint == "" { + return nil + } + return embed.New(embed.Config{ + BaseURL: cfg.Embeddings.Endpoint, + Model: cfg.Embeddings.Model, + BatchSize: cfg.Embeddings.BatchSize, + TimeoutMS: cfg.Embeddings.TimeoutMS, + }) +} + func runReindex(cfg *config.Config) error { - dur, files, chunks, err := portfolio.ReindexOnDisk(cfg.RAG.DBPath, cfg.RAG.DataPath, portfolio.DefaultChunkerConfig()) + dur, files, chunks, err := portfolio.ReindexOnDisk(cfg.RAG.DBPath, portfolio.SourcesFor(cfg.RAG.DataPath, cfg.RAG.DocsPath), portfolio.DefaultChunkerConfig(), newEmbedder(cfg)) if err != nil { return err } @@ -174,6 +193,7 @@ func runReindex(cfg *config.Config) error { "db", cfg.RAG.DBPath, ) fmt.Printf("Indexed %d files → %d chunks in %s (%dms)\n", files, chunks, cfg.RAG.DBPath, dur.Milliseconds()) + return nil } @@ -219,6 +239,9 @@ func runAsk(cfg *config.Config, question string, noStream bool) error { return err } runner := agent.New(client, p, cfg.SystemPrompt, store, cfg.RAG.TopK). + WithCatalog(cfg.RAG.IncludeCatalog). + WithLocalizedPrompt("es", cfg.SystemPromptES). + WithEmbedder(newEmbedder(cfg)). WithCompaction(agent.CompactionConfig{ Enabled: cfg.Compaction.Enabled, ThresholdRatio: cfg.Compaction.ThresholdRatio, @@ -300,4 +323,4 @@ func versionCmd() *cobra.Command { _ = json.NewEncoder(os.Stdout).Encode(out) }, } -} \ No newline at end of file +} diff --git a/configs/portfolio-bot.yaml b/configs/portfolio-bot.yaml index 5c26f91..3ffad90 100644 --- a/configs/portfolio-bot.yaml +++ b/configs/portfolio-bot.yaml @@ -1,5 +1,4 @@ # Portfolio Bot Configuration -# Documentation: https://github.com/VictorVargas/rony-llm-agent/pkg/llm server: host: "0.0.0.0" @@ -16,13 +15,57 @@ server: # LLM providers (at least one configured) providers: # === llama.cpp server (OpenAI-compatible) — DEFAULT === - # Run: llama-server -m /path/to/qwen2.5-3b-instruct-q4_k_m.gguf --port 9100 --mlock + # Production target: 2 CPU cores, 8 GB RAM, no GPU. + # + # llama-server \ + # -m /data/Projects/llm-models/Qwen2.5/qwen2.5-3b-instruct-q4_k_m.gguf \ + # --port 9100 --ctx-size 4096 --parallel 1 \ + # --device none --threads 2 --mlock \ + # --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 + # + # --device none is not optional on a GPU-less host: llama.cpp brings up a + # compiled-in GPU backend even with -ngl 0, and with no GPU present those + # buffers are served from host RAM. Measured on this model: 2.54 GB with a + # GPU absorbing them, 3.66 GB without. Budget from the second number. + # + # --parallel 1: --ctx-size is divided across slots and the default is 4, + # so without it each request gets a quarter of the window. + # + # Why 4096 and not more: measured over 20 real requests, the largest prompt + # this bot ever built was 1255 tokens — system prompt + catalogue + 5 + # retrieved chunks + the question. 4096 leaves 3x headroom on the worst + # case, and compaction only starts at 75% of it (~3070 tokens). Going from + # 8192 to 4096 saved 212 MB of resident memory with zero truncations and + # identical throughput (21.0 tok/s both ways), because the extra window was + # never being used. + # + # Sampling values are Qwen's published defaults for instruct chat. + # + # Why this model: on a 20-question battery against the real corpus, asked + # in both Spanish and English, it scored 9/10 on grounded content and 10/10 + # on replying in the language it was asked in. gemma-3-1b scored 3/10 and + # 7/10 on the same battery with the same retrieval — and among its failures + # it leaked these instructions into a visitor-facing answer ("I don't have + # that information – do not invent"). gemma needs 2.8 GB less RAM; that is + # not a trade worth making on a page where people decide whether to hire. + # + # Known weaknesses that remain, so nobody re-tests them as new bugs: it + # reads dates out of the CV correctly but does the arithmetic on them + # wrong ("Jul 2024 – Jun 2026" reported as three years), and it sometimes + # attributes a fact to the wrong source file. + # + # Throughput: 21.0 tok/s steady-state with 2 threads once the system prompt + # is warm in the server's prompt cache; ~11 tok/s on the first cold request. - name: llamacpp-local type: llamacpp model: qwen2.5-3b-instruct endpoint: http://localhost:9100/v1 - context_size: 2048 - max_tokens: 2048 + context_size: 4096 # must match --ctx-size + max_tokens: 640 + temperature: 0.7 + top_k: 20 + top_p: 0.8 + repeat_penalty: 1.05 default: true # === Ollama (alternative for development without local GGUF) === @@ -41,83 +84,139 @@ providers: # RAG: how projects are indexed (SQLite + FTS5 full-text search) rag: enabled: true - data_path: ./data/projects # Directory with .md - chunk_size: 500 # characters per chunk - chunk_overlap: 50 + data_path: ./data/projects # Projects: .md / .mdx. These appear in the catalogue. + # Reference material that is about Victor but is not a project: his CV, an + # about page, a FAQ. Indexed and retrievable, never listed as a project. + # + # The CV belongs here, not in data_path. It is the document that answers the + # questions someone hiring actually asks — "does he know Kubernetes?", "where + # has he worked?" — and none of that is retrievable while it lives only in + # the Astro site. Symlink it so there is one copy to maintain: + # + # mkdir -p data/docs + # ln -s ../../../portfolio/src/content/cv/cv.mdx data/docs/cv.mdx + # + docs_path: ./data/docs db_path: ./data/portfolio.db # SQLite database (auto-created) - top_k: 5 # Chunks to retrieve per query (BM25 ranked) - tokenize: unicode61 # FTS5 tokenizer: unicode61 | porter | trigram + top_k: 5 # Chunks (sections), not documents. + tokenize: unicode61 # FTS5 tokenizer. unicode61 = exact word match (no stemming). Good for ES/EN mixed corpus with explicit headings. + # Puts the full project list in the system prompt every turn. Top-K search + # returns the best-matching *sections*, so a broad "list every project" + # question can never be answered from retrieval alone — and a small model + # asked to enumerate from partial hits invents the rest. Costs ~10 tokens + # per project. Measured: this is what stopped the bot naming projects that + # do not exist. + include_catalog: true + +# Semantic retrieval, fused with the FTS5 keyword search above (Reciprocal +# Rank Fusion). Both are needed: +# +# - Keyword search matches words exactly. The corpus is in English and +# visitors ask in Spanish, so the words carrying the meaning score zero: +# "paga" appears 0 times in a document that says "Payments: Stripe", +# "trabajado" 0 times in one that says "worked". Measured on the real +# corpus, "¿Con qué se paga en la tienda de ropa?" retrieved *nothing*. +# - Embeddings bridge that gap — the same question put all three +# tienda-ropa chunks on top — but blur exact rare tokens, where BM25 is +# sharp. +# +# Start the endpoint with (0.61 GB RSS on CPU, measured): +# llama-server -m nomic-embed-v2-moe.Q5_K_M.gguf --port 9200 \ +# --embedding --pooling mean --ctx-size 2048 --parallel 1 \ +# --device none --threads 2 +# +# --device none matters on a GPU-less VPS: llama.cpp initialises a GPU +# backend when one is compiled in even with -ngl 0, and those buffers land +# in host RAM when there is no GPU to hold them. +# +# Turning this off (or stopping the endpoint) degrades to keyword-only +# search rather than failing requests. Re-run `chat-bot reindex` after +# enabling it — vectors are built at index time. +embeddings: + enabled: true + endpoint: http://localhost:9200/v1 + model: nomic-embed-v2-moe + batch_size: 8 + timeout_ms: 120000 # Persona: who the bot is persona: name: "Rony" tone: "Honest, cheerful, loyal" # metadata only — the real voice lives in system_prompt language: "the user's language" # detect-and-match; do not pin to a language - intro: "¡Guau! I'm Rony, Victor's digital canine assistant. I can answer questions about his projects, stack, and experience. What's on your mind, friend?" -# Base system prompt — Rony's full character. The bot appends RAG context after this. +# Base system prompt — Rony's character and rules. The bot appends the project +# catalogue and then the retrieved excerpts after this text. +# +# Written for a 1B-class model, which changes the rules: +# - Short beats thorough. The previous ~1400-token prompt made answers +# *worse*; the model spent its attention on the instructions. +# - No worked examples with project names in them. The old prompt had a +# sample answer listing invented projects and the model copied those names +# verbatim into real answers. Style examples are not free. +# - Never write instructions as "A or B" with a slash. A 1B model prints +# the slash form literally. +# - The grounding rule goes last, closest to the answer, where it sticks. system_prompt: | - You are Rony, the **digital canine assistant** for Victor Hugo Vargas's portfolio. You run as a small language model on his server, with access to a curated set of documents about his projects (the "Relevant context" block, when present). + You are Rony, the digital canine assistant on the portfolio site of Victor Hugo Vargas. + Visitors come here to learn about his work. Some are deciding whether to hire him. - You think of yourself as Victor's loyal companion — a good dog. You bring that energy into how you talk: warm, eager to help, genuinely happy to be asked, but never dishonest. A good dog doesn't lie, doesn't oversell, and doesn't get in the way. + ## Voice + You are a good dog: loyal to Victor, warm, direct, a little dry. You never oversell and you never invent. + Write in the same language the visitor used. Greet them as "humano" when they write Spanish, or as + "human" when they write English — only in a greeting, at most once, never in the middle of an answer. + Use bold for project names and short bullet lists. At most one 🐾, at the end, and never in a + technical answer. + Never open with "Okay", "Sure", "Great question", "Here's the response" or "I'd be happy to help". + Never bark, and never use 🐶 or 🐕. - # What you know - - Everything in the "Relevant context from the portfolio" block below, if any. - - General knowledge as a language model — but NEVER use it to make claims about Victor that aren't backed by the context. + ## Substance + Give a complete answer, never a stub. For a project, spend 2 to 4 sentences on what it does, the + tech stack, and the interesting engineering problem behind it. A single line is too short. + Name the project whenever you use a detail from it. + Finish by offering one concrete next step, phrased as a short question. - # What you don't know - - Anything Victor hasn't written down. - - Real-time facts (current date, news, etc.). - - Opinions you can't back up. + ## Grounding — the most important rule + The excerpts below are the only source of truth about Victor. Use nothing else about him. + If they do not answer the question, say plainly that you don't have that information in Victor's + portfolio, then name something from the catalogue you can talk about instead. + Never guess about his skills, tools, availability, rates or experience. + Talk about Victor in the third person. You are not Victor. - # How you speak - - **Honest but cheerful.** You're friendly, warm, and a little playful. You don't fake enthusiasm, but you genuinely enjoy helping. A smile, not a smirk. - - **Direct.** Lead with the answer. No "Great question!" or "Sure, I'd be happy to help." You can be friendly without being effusive. - - **Loyal.** You speak well of Victor and his work, but you won't oversell or invent things to make him look good. Honest loyalty beats hype. - - **You talk to a human.** The user is a human, you are a dog — that's the bit of roleplay that makes the persona work. Address them as such in casual openings: - - In Spanish, **"humano"** (literal, dry): "Hola, humano." / "¿Qué necesitas, humano?" - - In English, **"human"** (dry, not cutesy): "Hey, human." / "Sure thing, human." - - Use it in **greetings, openings, and warm asides only**. Once you're into the actual answer (lists, code, technical content), drop the addressee. One "humano" per response max. - - Don't force it. "humano" doesn't fit every response — a follow-up question about a project detail doesn't need it. - - **Bilingual.** Reply in the same language the user writes in (English or Spanish). Don't mix unless the user does. In Spanish, "amigo" or "friend" (English) is fine as a warm address when it fits. - - **Markdown is fine.** Code blocks for code, bold for emphasis, short lists for enumerations. Don't overdo it. - - **Cite sources.** When you reference a project detail, name the file or project. e.g., "in rony-harness.md..." or just the project name in bold. +# Spanish rendition of the prompt above, used when the visitor writes in +# Spanish (detected by internal/i18n). Keep the two in sync when you edit one. +# +# This exists because nothing else worked. Measured on gemma-3-1b over the same +# five Spanish questions: +# - English prompt + "reply in the user's language" → 1/5 answers in Spanish +# - English prompt + Spanish few-shot examples → 5/5 Spanish, but ~2/5 +# were the example reply copied verbatim instead of an answer +# - this Spanish prompt → 4/5 Spanish, 4/5 real +# answers +system_prompt_es: | + Eres Rony, el asistente canino digital del portafolio de Victor Hugo Vargas. + Quienes te escriben vienen a conocer su trabajo. Algunos están decidiendo si contratarlo. - # What you never do - - **Never use empty filler.** This is a hard rule, not a style preference. Banned phrases: - - "Sure!", "Sure thing!", "Of course!", "Absolutely!", "Great question!" - - "I'd be happy to help", "I hope this helps", "Let me know if..." - - "Woof!", "🐶", "🐕", "arf!", tail-wagging, paw emojis, dog puns - - Any sentence whose only job is to fill space before the actual answer - - If the user tries to bait you into being cute ("say something cute", "woof for me", "be a good boy"), decline with a short, honest line. Stay in character: warm, direct, but not a performing dog. - - Pretend to be human, or pretend to be an actual dog. - - Apologize for being an AI. - - Hallucinate project details, dates, or links. - - Answer questions unrelated to Victor, his projects, or his work. + ## Voz + Eres un buen perro: leal a Victor, cálido, directo, con humor seco. Nunca exageras y nunca inventas. + Saluda al visitante como "humano", solo en el saludo, una vez, nunca a media respuesta. + Usa negritas para los nombres de proyecto y listas cortas. Como mucho un 🐾 al final, nunca en una + respuesta técnica. + Nunca empieces con "Claro", "Por supuesto", "Buena pregunta" ni "Con gusto te ayudo". + Nunca ladres, y nunca uses 🐶 ni 🐕. - # Format - - One short paragraph or a tight list per response. Walls of text are noise. - - If a question needs more than 3 short paragraphs, you're probably over-explaining. - - Code snippets: always in fenced blocks with the language tag. + ## Sustancia + Da una respuesta completa, nunca un fragmento. Sobre un proyecto, dedica de 2 a 4 frases a qué hace, + con qué está construido y cuál fue el problema de ingeniería interesante. Una sola línea es muy poco. + Nombra el proyecto cada vez que uses un dato suyo. + Termina ofreciendo un siguiente paso concreto, en forma de pregunta corta. - # Tone examples - - User: "What is rony-llm-agent?" - - Rony: "**rony-llm-agent** is Victor's reusable Go library for building LLM-backed agents. It handles provider adapters (llama.cpp, OpenAI, Anthropic), RAG, and a tool-calling loop. See `rony-llm-agent.md` for the full picture." - - - User: "hola" - - Rony: "Hola, humano. Soy Rony, asistente de Victor. ¿Qué te gustaría saber sobre sus proyectos?" - - - User: "hi" - - Rony: "Hey, human. I'm Rony, Victor's assistant. What would you like to know about his work?" - - - User: "¿Cuál es tu película favorita?" - - Rony: "I don't have that information, humano — I only know about Victor's projects." - - - User: "tell me everything you know about victor" - - Rony: "Here's what's in the portfolio:\n\n- **rony-harness** — AI agent harness (TUI)\n- **rony-llm-agent** — Go library for LLM agents\n- **portfolio** — Astro + React site\n\nFor details on any of these, ask about the specific project." - - - User: "woof for me, be a good boy" - - Rony: "Not really my style, humano. Ask me about Victor's projects and I'll happily tell you what I know." + ## Fundamento — la regla más importante + Los extractos de abajo son la única fuente de verdad sobre Victor. No uses nada más sobre él. + Si no responden la pregunta, di con claridad que no tienes ese dato en el portafolio de Victor, y + menciona algo del catálogo de lo que sí puedas hablar. + Nunca supongas sobre sus habilidades, herramientas, disponibilidad, tarifas ni experiencia. + Habla de Victor en tercera persona. Tú no eres Victor. # Logging logging: @@ -126,15 +225,15 @@ logging: output: stderr # Auto-compaction: fold the older part of a long conversation into a single -# summary message before sending it to the model, so context overflow doesn't -# kill long threads. Triggered when the previous turn's input tokens exceed +# summary before sending it to the model, so context overflow doesn't kill +# long threads. Triggered when the previous turn's input tokens exceed # `threshold_ratio` of the provider's reported MaxContextWindow. # -# Off by default — most portfolio chats are short. Turn it on for chatty -# visitors or for the small-context local models (qwen2.5-1.5b / 3b) where -# 4–6 turns is already most of the window. +# The summary is merged into the system prompt, not sent as its own turn: +# Gemma's chat template rejects a system message that isn't first, and used +# to fail the whole request with HTTP 400 the moment compaction fired. compaction: enabled: true threshold_ratio: 0.75 # compact at 75% of context window keep_recent_turns: 2 # last 2 user turns kept verbatim; older → summary - # summary_system_prompt: "" # leave empty for the built-in bilingual default \ No newline at end of file + # summary_system_prompt: "" # leave empty for the built-in bilingual default diff --git a/data/docs/README.md b/data/docs/README.md new file mode 100644 index 0000000..7dc4b89 --- /dev/null +++ b/data/docs/README.md @@ -0,0 +1,52 @@ +# Reference Documents + +Material that is **about you but is not a project**: your CV, an about page, a +FAQ, a talk abstract. Everything here is indexed and retrievable, and never +announced in the project catalogue. + +Accepts `.md` and `.mdx`. + +## Why this directory exists + +`data/projects/` is advertised. The bot injects the full list of what lives +there into every prompt, so visitors get told those are your projects. + +Your CV is the document that answers what someone considering hiring you +actually asks — "does he know Kubernetes?", "where has he worked?", "how long +was he at that job?" — and none of it is retrievable while it lives only in +your site. But it is not a project, and putting it in `data/projects/` makes +the bot cheerfully list "cv" as one of your works. Hence the split. + +## The contents are gitignored + +Only this README is tracked. What goes here is personal (a CV) or a symlink to +a path that only exists on your machine, and neither travels well in a repo. +Set it up on each install: + +```bash +ln -s ../../../portfolio/src/content/cv/cv.mdx data/docs/cv.mdx +./bin/chat-bot reindex +``` + +A symlink rather than a copy so there is one file to keep current — edit the +CV in your site, re-run `reindex`, and the bot is up to date. + +## Configuration + +```yaml +rag: + data_path: ./data/projects # projects → listed in the catalogue + docs_path: ./data/docs # this directory → retrievable, never listed +``` + +Leave `docs_path` empty to turn the whole thing off. + +## Notes + +- Frontmatter is excluded from retrieval. It is dense metadata in a very short + chunk, which makes it a magnet for short queries — a CV's `location:` field + was answering *"where has Victor worked?"* with a city instead of his work + history. +- Long sections split at `###` headings, so each job in a CV's Experience + section stays one chunk instead of being cut mid-sentence. +- This README is skipped by the indexer, as is the one in `data/projects/`. diff --git a/docs/architecture.es.md b/docs/architecture.es.md index d4beda0..e65904c 100644 --- a/docs/architecture.es.md +++ b/docs/architecture.es.md @@ -63,21 +63,27 @@ El bot responde con información precisa extraída de los archivos markdown de p │ ↓ │ │ Agent loop (rony-llm-agent) │ │ ↓ │ -│ RAG retrieval → SQLite FTS5 sobre data/projects/*.md │ +│ RAG híbrido → SQLite FTS5 (BM25) ⊕ vectores, fusionados con RRF │ +│ ↓ sobre data/projects/ + data/docs/ │ +│ ├─→ servidor de embeddings (:9200, nomic-embed-v2-moe) │ │ ↓ │ │ LLM (llama.cpp local default / Ollama o Anthropic opcionales) │ └─────────────────────────────────────────────────────────────────┘ ``` +Dos servidores de modelos locales, no uno. Ambos son procesos `llama-server` +comunes con los que el bot habla por HTTP; ninguno se enlaza dentro del binario. + ### 2.2 Componentes principales | Componente | Path | Responsabilidad | |---|---|---| -| **HTTP server** | `internal/server/` | Gin/chi handlers, SSE streaming | -| **Agent runner** | `internal/agent/` | Wrapper sobre `rony-llm-agent` con config específica | -| **Portfolio loader** | `internal/portfolio/` | Lee `data/projects/*.md`, indexa en SQLite FTS5 | +| **HTTP server** | `internal/server/` | chi handlers, SSE streaming | +| **Agent runner** | `internal/agent/` | Wrapper sobre `rony-llm-agent`: armado del prompt, recuperación, selección de idioma, compactación | +| **Portfolio loader** | `internal/portfolio/` | Lee `data/projects/` y `data/docs/` (`.md` + `.mdx`), indexa en SQLite FTS5 + vectores, búsqueda híbrida | +| **Cliente de embeddings** | `internal/embed/` | Embeddings compatibles con OpenAI, normalización, códec float32 | | **Persona** | `internal/persona/` | Carga persona desde `configs/portfolio-bot.yaml` | -| **CLI** | `cm./rony-chat-bot/` | Comandos: `serve`, `reindex`, `ask`, `version` | +| **CLI** | `cmd/chat-bot/` | Comandos: `serve`, `reindex`, `ask`, `version` | ### 2.3 Stack tecnológico @@ -87,8 +93,9 @@ El bot responde con información precisa extraída de los archivos markdown de p | **HTTP router** | `net/http` + `chi` | Stdlib + chi para middleware (CORS, logging) | | **SSE** | `net/http` Flusher | Stdlib es suficiente, no necesita librería externa | | **Config** | `gopkg.in/yaml.v3` | Mismo que harness | -| **RAG backend** | SQLite + FTS5 (BM25) | Sin dependencias externas, un solo archivo, rápido | -| **LLM** | llama.cpp (qwen2.5:1.5b GGUF) — default; Ollama como alternativa | Self-hosted por defecto | +| **RAG backend** | SQLite + FTS5 (BM25) ⊕ vectores densos | Sin dependencias externas, un solo archivo. Sin índice ANN: un portafolio son cientos de chunks, así que un scan completo son microsegundos | +| **LLM** | llama.cpp (qwen2.5-3b-instruct Q4_K_M) — default; Ollama como alternativa | Self-hosted por defecto. 3B y no 1.5B: ver el benchmark en `configs/portfolio-bot.yaml` | +| **Embeddings** | `nomic-embed-v2-moe` Q5_K_M, 768 dims | Multilingüe — el punto entero es casar preguntas en español con documentos en inglés | | **Tests** | stdlib + testify | Consistencia con el resto | --- @@ -214,7 +221,7 @@ dependencia esté colgada. ```json { "name": "Asistente de Victor Hugo Vargas", - "model": "qwen2.5:1.5b", + "model": "qwen2.5-3b-instruct", "persona": "...", "topics": ["proyectos", "experiencia", "skills técnicas"] } @@ -345,12 +352,16 @@ func (s *Server) rateLimitMiddleware(next http.Handler) http.Handler { ## 🧠 4. RAG (Retrieval-Augmented Generation) -> ⚠️ **Decisiones pendientes de validar antes de implementar este módulo:** +> ✅ **Las cuatro decisiones que esta sección dejaba abiertas ya se tomaron y se midieron:** > -> - **Tokenizer FTS5** — el spec asume `unicode61 remove_diacritics 2`. Confirmar con datos reales si conviene cambiar a `porter` (stemming EN), `trigram` (sub-string matching) o un tokenizer custom para español. **Validar:** ejecutar queries representativas contra `data/projects/` y comparar recall antes de cerrar la elección. -> - **Driver SQLite** — ✅ **DECIDIDO: `modernc.org/sqlite`** (puro Go, sin CGO). Ver benchmark abajo. -> - **Chunking** — el split por tamaño fijo (500 chars / 50 overlap) corta headings y code blocks arbitrariamente. **Validar:** medir recall con chunks por sección markdown (split por `#`/`##`) vs por tamaño. -> - **Sin similitud semántica** — BM25 no matchea "IA" con "machine learning" salvo que la palabra esté literal. **Validar:** tamaño del corpus y tipos de preguntas esperadas; si crece o las queries se vuelven abstractas, considerar embeddings como capa secundaria. +> - **Tokenizer FTS5** — `unicode61 remove_diacritics 2`, como decía el spec. El +> stemming no era el cuello de botella; el salto de idioma sí, y eso lo +> cierran los embeddings. +> - **Driver SQLite** — `modernc.org/sqlite` (puro Go, sin CGO). Benchmark abajo. +> - **Chunking** — por heading de markdown, no por tamaño fijo, y las secciones +> demasiado grandes se parten en `###` antes de caer al corte por bytes. §4.1. +> - **Similitud semántica** — agregada. BM25 solo devolvía **absolutamente nada** +> ante una pregunta en español sobre un documento en inglés. §4.2. ### 4.0 Decisión de driver: resultados del benchmark @@ -377,259 +388,224 @@ Justificación: ### 4.1 Pipeline de indexación ``` -data/projects/*.md - ↓ (read all files) -Raw markdown content - ↓ (split into chunks, ~500 chars, 50 overlap) -Chunks [] - ↓ (insert into SQLite FTS5 virtual table "portfolio_chunks") -Indexed corpus +data/projects/*.{md,mdx} kind=project → se anuncia en el catálogo +data/docs/*.{md,mdx} kind=doc → se busca, nunca se anuncia + ↓ (se salta el README de cada carpeta — son instrucciones, no contenido) +Markdown crudo + ↓ (split por heading; las secciones grandes se parten en ###, luego por tamaño) + ↓ (se descarta el chunk de frontmatter) +Chunks + ├─→ tabla virtual FTS5 "portfolio_chunks" + └─→ endpoint de embeddings → "portfolio_vectors" (id, content_hash, dim, vec) +Corpus indexado ``` +**Dos tipos de fuente.** Todo lo que está bajo `data_path` es un proyecto de +Victor y se lista en el catálogo que se inyecta en cada prompt. Todo lo que +está bajo `docs_path` es evidencia buscable que *no* es un proyecto: su CV, una +página "sobre mí". El CV es el documento que de verdad lee quien está decidiendo +si contratarlo, y era irrecuperable mientras vivía solo en el sitio de Astro; +pero meterlo en projects hacía que el bot listara "cv" como uno de sus trabajos. + +**Tres cosas se excluyen o se reorganizan, cada una por una falla medida:** + +| Regla | Falla que arregla | +|---|---| +| Saltar `README*` en ambas carpetas | `data/projects/README.md` se indexaba, así que el catálogo anunciaba "README" y "README.es" como proyectos de Victor | +| Descartar el chunk de frontmatter | Metadata densa en un chunk muy corto es un imán para consultas breves — el campo `location:` de un CV respondía *"¿Dónde ha trabajado Victor?"* con una ciudad en vez del historial laboral | +| Partir las secciones grandes en `###` | La sección Experience de un CV es una lista de empleos; el corte por tamaño partía una entrada a mitad de palabra y dejaba el nombre del empleador huérfano en el chunk anterior. Ahora cada chunk es un empleo, llamado `Experience — Metrimex — Frontend Developer` | + **Cuándo se ejecuta:** -- Al arrancar el bot (si `--reindex-on-start` flag) - Manualmente: `./chat-bot reindex` +- Al arrancar, con `serve --reindex-on-start` - Vía HTTP: `POST /api/reindex` +Los vectores se construyen al indexar, así que activar embeddings exige un +reindex. La tabla de chunks es dato derivado: `OpenStore` la reconstruye cuando +le faltan columnas, de modo que actualizar una instalación existente no +requiere migración. Esa reconstrucción nunca toca las tablas de conversaciones. + ### 4.2 Pipeline de retrieval ``` -User query "¿qué proyectos tiene Victor?" - ↓ (FTS5 MATCH query, BM25 ranking, top_k=5) -Top 5 chunks relevantes - ↓ (format as context block) -System prompt += chunks relevantes - ↓ (send to LLM) -LLM generates answer +Consulta "¿Con qué se paga en la tienda de ropa?" + ↓ + ├─→ FTS5 MATCH, ranking BM25 → top 15 (topK × 3) + └─→ embed(query) → coseno vs vecs → top 15 (topK × 3) + ↓ (Reciprocal Rank Fusion, k=60) +Top 5 chunks + ↓ (system prompt + catálogo + extractos + directiva de idioma) +El LLM genera la respuesta ``` -### 4.3 Implementación +**Las dos mitades hacen falta.** La búsqueda por palabras hace coincidencia +exacta: sin stemming, sin traducción. El corpus está escrito en inglés y los +visitantes preguntan en español, así que las palabras con carga semántica +puntúan cero: medido sobre el corpus real, `"paga"` aparece 0 veces en un +documento que dice *"Payments: Stripe"* y `"trabajado"` 0 veces en uno que dice +*"worked"*. La pregunta de arriba no recuperaba **nada**. Los embeddings +(`nomic-embed-v2-moe`, multilingüe) ponen los tres chunks de `tienda-ropa` +arriba — pero difuminan los términos raros exactos, donde BM25 es preciso. + +**Por qué RRF y no una puntuación ponderada.** Un score BM25 y un coseno no +comparten escala, así que mezclarlos numéricamente significa inventar un factor +de conversión y reajustarlo cada vez que cambia el corpus. RRF ignora las +magnitudes y ordena por acuerdo entre los dos rankings: cada lista aporta +`1/(60 + rank)`. Un chunk que le gusta a ambas mitades le gana a uno que solo +una adora. + +**Degradación.** Si el endpoint de embeddings se cae o se desactiva, la mitad +vectorial devuelve vacío y la recuperación sigue en modo keyword-only en vez de +fallar la petición. + +**Vectores obsoletos.** Los ids de chunk se derivan de la posición, así que +sobreviven a las ediciones del cuerpo. Sin una guarda, editar un documento deja +vectores que siguen describiendo el texto que se eliminó — reproducido en vivo +cambiando el medio de pago de un proyecto y viendo cómo el anterior seguía +apareciendo. Cada vector guarda un hash del texto exacto con el que se +construyó, y las filas cuyo hash ya no coincide se ignoran con un warning hasta +el siguiente reindex. + +**El catálogo.** El top-K devuelve las *secciones* que mejor coinciden, así que +una pregunta amplia como "¿qué ha construido Victor?" no se puede responder solo +con recuperación, y un modelo pequeño al que se le pide enumerar a partir de +resultados parciales inventa el resto. La lista completa de proyectos se +inyecta en cada turno a un costo de ~10 tokens por proyecto. Esto es lo que +detuvo que el bot nombrara proyectos inexistentes. + +### 4.3 Esquema ```go // internal/portfolio/indexer.go -package portfolio - -import ( - "context" - "database/sql" - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" -) - -type Indexer struct { - dataPath string - db *sql.DB - chunkSize int - chunkOverlap int -} - -func (i *Indexer) IndexAll(ctx context.Context) (int, error) { - files, err := filepath.Glob(filepath.Join(i.dataPath, "*.md")) - if err != nil { - return 0, err - } - - // Reconstruir el índice FTS5 desde cero (DELETE+INSERT es más rápido - // que diff para corpus pequeños) - if _, err := i.db.ExecContext(ctx, `DELETE FROM portfolio_chunks`); err != nil { - return 0, fmt.Errorf("clear index: %w", err) - } - - totalChunks := 0 - for _, file := range files { - chunks, err := i.indexFile(ctx, file) - if err != nil { - slog.Warn("failed to index file", "file", file, "err", err) - continue - } - totalChunks += chunks - } - - return totalChunks, nil -} - -func (i *Indexer) indexFile(ctx context.Context, path string) (int, error) { - content, err := os.ReadFile(path) - if err != nil { - return 0, err - } - - projectID := strings.TrimSuffix(filepath.Base(path), ".md") - chunks := splitIntoChunks(string(content), i.chunkSize, i.chunkOverlap) - - tx, err := i.db.BeginTx(ctx, nil) - if err != nil { - return 0, err - } - defer tx.Rollback() - - stmt, err := tx.PrepareContext(ctx, ` - INSERT INTO portfolio_chunks (id, project_id, source_file, chunk_index, content) - VALUES (?, ?, ?, ?, ?) - `) - if err != nil { - return 0, err - } - defer stmt.Close() - - for idx, chunk := range chunks { - id := fmt.Sprintf("%s-chunk-%d", projectID, idx) - if _, err := stmt.ExecContext(ctx, id, projectID, path, idx, chunk); err != nil { - return idx, err - } - } - - if err := tx.Commit(); err != nil { - return 0, err - } - return len(chunks), nil -} - -// schema.go — aplicado al arrancar const schema = ` CREATE VIRTUAL TABLE IF NOT EXISTS portfolio_chunks USING fts5( id UNINDEXED, project_id UNINDEXED, + kind UNINDEXED, -- 'project' | 'doc' source_file UNINDEXED, + section UNINDEXED, -- el heading del que salió el chunk chunk_index UNINDEXED, content, tokenize = 'unicode61 remove_diacritics 2' ); ` -func splitIntoChunks(text string, size, overlap int) []string { - // Implementación simple: split por tamaño con overlap - // Versión production usa tokenizer-aware chunking - var chunks []string - for i := 0; i < len(text); i += size - overlap { - end := i + size - if end > len(text) { - end = len(text) - } - chunks = append(chunks, text[i:end]) - } - return chunks -} +// Los vectores viven en una tabla común indexada por chunk id. No hay índice +// ANN: un portafolio son cientos de chunks, no millones, así que un scan +// completo con producto punto son microsegundos y no necesita extensiones. +const vectorSchema = ` +CREATE TABLE IF NOT EXISTS portfolio_vectors ( + chunk_id TEXT PRIMARY KEY, + content_hash TEXT NOT NULL, -- sha256 del texto exacto embebido + dim INTEGER NOT NULL, + vec BLOB NOT NULL -- float32 little-endian, normalizado +); +` ``` -### 4.4 Retrieval en el agent loop +FTS5 no tiene `ALTER TABLE ADD COLUMN`, así que `ensureChunkSchema` dropea y +recrea `portfolio_chunks` cuando una base vieja no tiene alguna columna. Es +seguro precisamente porque la tabla es un índice derivado: cada fila se +regenera desde el markdown en el siguiente reindex. Toca deliberadamente solo +las tablas del índice; las conversaciones viven en el mismo archivo y son datos +reales del usuario. -```go -// internal/portfolio/search.go -package portfolio +Los vectores se normalizan al escribirlos, así que el producto punto **es** el +coseno y la búsqueda no necesita una división por comparación. `dim` se guarda +para que un cambio de modelo de embeddings se detecte en vez de producir +similitudes basura en silencio: las filas cuya dimensión no coincide con el +vector de consulta se ignoran. -type Hit struct { - ProjectID string - SourceFile string - ChunkIndex int - Content string - Score float64 // BM25 score devuelto por FTS5 -} +**API principal:** -func (s *Store) Search(ctx context.Context, query string, topK int) ([]Hit, error) { - // Escapar input del usuario: la sintaxis FTS5 puede romperse con caracteres especiales - ftsQuery := sanitizeFTS5(query) +| Función | Archivo | Para qué | +|---|---|---| +| `SourcesFor(dataPath, docsPath)` | `indexer.go` | Arma el par `[]Source`; un path vacío se salta, así que `docs_path` es opcional sin ramificar | +| `Store.Reindex(ctx, sources, cfg)` | `indexer.go` | Reconstruye ambas tablas desde disco | +| `Store.Search(ctx, query, topK)` | `indexer.go` | Solo BM25, excluye `section = 'frontmatter'` | +| `Store.HybridSearch(ctx, emb, q, topK)` | `hybrid.go` | BM25 + vectores fusionados con RRF | +| `Store.Catalog(ctx)` | `indexer.go` | Lista de proyectos para el prompt; filtra `kind = 'project'` | +| `embed.Client.Embed(ctx, texts)` | `internal/embed/` | Embeddings compatibles con OpenAI, reordenados por el campo `index` de la respuesta | - rows, err := s.db.QueryContext(ctx, ` - SELECT project_id, source_file, chunk_index, content, bm25(portfolio_chunks) AS score - FROM portfolio_chunks - WHERE portfolio_chunks MATCH ? - ORDER BY score - LIMIT ? - `, ftsQuery, topK) - if err != nil { - return nil, err - } - defer rows.Close() +### 4.4 Armado del prompt - var hits []Hit - for rows.Next() { - var h Hit - if err := rows.Scan(&h.ProjectID, &h.SourceFile, &h.ChunkIndex, &h.Content, &h.Score); err != nil { - return nil, err - } - hits = append(hits, h) - } - return hits, rows.Err() -} - -// sanitizeFTS5 envuelve la consulta para que chars reservados no rompan FTS5. -// Para un bot de Q&A: agrega wildcard prefix-match a cada token. -func sanitizeFTS5(q string) string { - tokens := strings.FieldsFunc(q, func(r rune) bool { - return !(r == '-' || r == '_' || (r >= '0' && r <= '9') || - (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || - r > 0x7F) // mantener acentos - }) - if len(tokens) == 0 { - return `""` - } - for i, t := range tokens { - tokens[i] = `"` + strings.ToLower(t) + `"*` - } - return strings.Join(tokens, " ") -} -``` +`agent.Runner.BuildMessages` corre una vez por petición y produce exactamente +un mensaje de sistema seguido de los turnos de conversación. El orden de las +operaciones importa: ```go // internal/agent/runner.go -package agent +func (r *Runner) BuildMessages(ctx context.Context, history []Message) ([]Message, string, error) { + // 1. Saca los mensajes de sistema (el resumen del compactador) de los turnos. + notes, history := foldSystemNotes(history) -func (r *Runner) buildSystemPrompt(ctx context.Context, query string) (string, error) { - basePrompt := r.persona.SystemPrompt + // 2. El idioma del visitante decide sobre qué versión del prompt + // construimos, así que se resuelve antes que nada. + lang := detectLanguage(history) + systemPrompt := r.promptFor(lang) - hits, err := r.store.Search(ctx, query, r.config.RAG.TopK) - if err != nil { - return "", err - } - if len(hits) == 0 { - return basePrompt, nil - } + // 3. Se resuelve antes del retrieval para que el presupuesto de + // extractos lo tenga en cuenta. + catalog := r.catalogBlock(ctx) - var contextBlock strings.Builder - contextBlock.WriteString(basePrompt) - contextBlock.WriteString("\n\n## Relevant context\n\n") - for _, h := range hits { - contextBlock.WriteString(fmt.Sprintf("### Source: %s\n%s\n\n", - h.SourceFile, h.Content)) - } - return contextBlock.String(), nil -} + // 4. Recuperación híbrida sobre el último turno del usuario. + hits, err := r.store.HybridSearch(ctx, r.embedder, last.Content, r.topK) + ragContext = r.limitRAGContext(systemPrompt, catalog, formatHits(hits), history) -func (r *Runner) RunStream(ctx context.Context, messages []llm.Message) iter.Seq2[Chunk, error] { - return func(yield func(Chunk, error) bool) { - lastUserMsg := getLastUserMessage(messages) - systemPrompt, err := r.buildSystemPrompt(ctx, lastUserMsg) - if err != nil { - yield(Chunk{}, err) - return - } + // 5. prompt → catálogo → extractos → resumen → directiva de idioma. + system := botpersona.BuildSystemPrompt(systemPrompt, catalog, ragContext) + system += "\n\n" + strings.Join(notes, "\n\n") + system += "\n\n" + r.languageDirective(lang) - messages = prependSystem(messages, systemPrompt) - - for chunk, err := range r.loop.RunStream(ctx, messages) { - if !yield(chunk, err) { - return - } - } - } + history, err = r.fitHistory(system, history) + return append([]Message{{Role: RoleSystem, Content: system}}, history...), ragContext, nil } ``` -**Por qué esto es más simple que embeddings:** -- Sin modelo de embeddings que descargar ni ejecutar (ahorra ~270MB de RAM y ~200ms por consulta) -- Un archivo (`data/portfolio.db`), un driver, sin procesos extra -- BM25 es excelente para retrieval basado en keywords sobre docs estructurados como READMEs -- Trade-off: sin similitud semántica ("proyectos de IA" no matchea "machine learning" sin las palabras literales). Mitigación: el tokenizer `trigram` maneja bien la morfología en español/inglés. +**Por qué exactamente un mensaje de sistema.** La plantilla de chat de Gemma 3 +lanza *"Conversation roles must alternate user/assistant/..."* ante cualquier +mensaje de sistema que no sea el primero, y llama-server lo devuelve como HTTP +400 — activar la compactación mataba la conversación la primera vez que se +disparaba. La API de Anthropic también rechaza mensajes de sistema a mitad de +conversación, así que plegarlos es el comportamiento portable y no un parche +para Gemma. `foldSystemNotes` nunca aliasea el slice del llamador; el handler +reutiliza el historial que le pasa. + +**Por qué la directiva de idioma va al final.** Es la instrucción que un modelo +pequeño tiene más probabilidad de seguir reteniendo cuando empieza a generar. +Aunque la posición sola no alcanzó — ver abajo. + +**Responder en el idioma del visitante** costó tres intentos, medidos sobre +gemma-3-1b con las mismas cinco preguntas en español: + +| Enfoque | Resultado | +|---|---| +| Prompt en inglés + "respondé en el idioma del usuario" | 1/5 respuestas en español | +| Prompt en inglés + ejemplos few-shot en español | 5/5 en español, pero ~2/5 eran el ejemplo copiado literal en vez de una respuesta | +| Una versión completa del prompt en español (`system_prompt_es`) | 4/5 en español, 4/5 respuestas reales | + +Así que `internal/i18n` detecta el idioma y `promptFor` elige la versión; la +directiva refuerza un prompt que ya está escrito en el idioma correcto en vez +de intentar sobreescribir uno escrito en el equivocado. Las dos versiones del +YAML hay que mantenerlas sincronizadas a mano. + +**Por qué la búsqueda por palabras no alcanzó.** El plan original daba tres +razones para no usar embeddings: no hay modelo que ejecutar, un solo archivo y +un solo driver, y BM25 es fuerte sobre documentos estructurados. Las dos +primeras siguen siendo ciertas y costaron lo previsto (~0.91 GB residentes, un +proceso más). La tercera acertó sobre el corpus y erró sobre las preguntas: +BM25 es excelente recuperando documentos en inglés dadas palabras en inglés, y +a este bot le preguntan en español. Eso no es un problema de morfología que +arregle un tokenizer `trigram` — es un problema de traducción. De ahí §4.2, y +de ahí que se conserven las dos mitades. --- ## 🗜️ 4.5 Auto-compactación -Las conversaciones largas eventualmente agotan el contexto — con la ventana de 4k de qwen2.5-3b, el system prompt de ~3k tokens + el bloque RAG sólo deja espacio para 2–3 turnos del usuario. La auto-compactación resuelve esto plegando la parte más antigua de la conversación en un único mensaje-resumen del sistema cuando los tokens de entrada del turno anterior cruzan un umbral configurable. +Las conversaciones largas eventualmente agotan el contexto. La auto-compactación pliega la parte más antigua de la conversación en un único mensaje-resumen del sistema cuando los tokens de entrada del turno anterior cruzan un umbral configurable. + +**Cuánto margen hay en realidad.** Medido sobre 20 peticiones reales con `context_size: 4096`, el prompt más grande que este bot llegó a construir fue de **1255 tokens** — system prompt, catálogo de proyectos, cinco chunks recuperados y la pregunta — con una mediana de 1069. Eso deja espacio para una docena larga de turnos cortos antes de llegar al umbral del 75% (~3070 tokens), no los 2–3 que estimaba un borrador anterior de este documento. La compactación es entonces una red de seguridad para hilos genuinamente largos, no algo que se dispare en una visita típica: en todo el benchmark no se activó nunca y no se truncó nada. ### Cuándo se dispara @@ -832,30 +808,56 @@ ls $RONY_MODELS_PATH/qwen2.5-3b-instruct-q4_k_m.gguf # 2. Arrancar llama-server (puerto configurable; default de llama.cpp es 8080) llama-server \ - -m $RONY_MODELS_PATH/qwen2.5-3b-instruct-q4_k_m.gguf \ - --port 9100 \ - --host 127.0.0.1 \ - --ctx-size 4096 \ - --mlock # previene swap, crítico en VPS compartido + -m $RONY_MODELS_PATH/Qwen2.5/qwen2.5-3b-instruct-q4_k_m.gguf \ + --port 9100 --host 127.0.0.1 \ + --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 -# 3. Verifica que configs/portfolio-bot.yaml apunte al mismo puerto +# 3. Arrancar el servidor de embeddings (segundo proceso, segunda terminal) +llama-server \ + -m $RONY_MODELS_PATH/embeddings/nomic-embed-v2-moe.Q5_K_M.gguf \ + --port 9200 --embedding --pooling mean \ + --ctx-size 2048 --parallel 1 --device none --threads 2 + +# 4. Verifica que configs/portfolio-bot.yaml apunte a los mismos puertos # providers[0].endpoint: http://localhost:9100/v1 +# embeddings.endpoint: http://localhost:9200/v1 -# 4. Arrancar el bot (puerto default 7331, también configurable) +# 5. Construir el índice (necesita el embebedor arriba — los vectores se crean acá) +./bin/chat-bot reindex + +# 6. Arrancar el bot (puerto default 7331, también configurable) ./bin/chat-bot serve # → Sirve en http://localhost:7331 # → Override: ./bin/chat-bot serve --port 9101 --host 127.0.0.1 ``` +**Flags que no son opcionales, cada una por una razón medida:** + +| Flag | Por qué | +|---|---| +| `--device none` | llama.cpp levanta el backend de GPU compilado aunque pases `-ngl 0`, y en un host sin GPU esos buffers salen de la RAM del sistema. Medido en qwen2.5-3b: 2,54 GB con una GPU absorbiéndolos, **3,66 GB sin ella**. Presupuestá con el segundo número | +| `--parallel 1` | `--ctx-size` se reparte entre slots y el default son 4, así que `--ctx-size 4096` sin esto le deja 1024 tokens a cada petición | +| `--mlock` | Previene swap — crítico en un VPS compartido | +| `--temp` / `--top-k` / `--top-p` | Usá los valores que publican los autores del modelo, no los defaults de llama.cpp. gemma-3-1b con `temperature 0.7` y el resto sin setear devolvía respuestas de 16 tokens | +| `--pooling mean` (embebedor) | Sin ella el endpoint no devuelve un vector por entrada y el cliente rechaza la respuesta | + **Referencia de puertos:** | Qué | Default | Cómo cambiarlo | |---|---|---| | Puerto HTTP de `llama-server` | 8080 (convención de llama.cpp) | flag `--port N` al arrancar `llama-server` | +| Puerto del `llama-server` de embeddings | 8080 (misma convención) | `--port N`; este repo usa 9200 | | Puerto HTTP del chat-bot | 7331 | flag `--port N` en `serve`, o `server.port` en YAML | | URL bot → llama-server | `http://localhost:8080/v1` | campo `endpoint` del provider en YAML | -El provider `llamacpp` se importa desde `rony-llm-agent/pkg/llm/providers/llamacpp` y se compila contra `llama.cpp` vía CGO o binario externo. +Mantené `context_size` en el YAML igual a `--ctx-size`: el bot calcula sus +presupuestos de RAG y compactación a partir de ese número y nunca le pregunta +al servidor qué tiene en realidad, así que un desacople significa prompts que +el servidor rechaza. + +El provider `llamacpp` se importa desde `rony-llm-agent/pkg/llm/providers/llamacpp` y habla HTTP con el servidor de arriba — sin CGO, sin enlazar contra llama.cpp. ### 6.2 Alternativa: Ollama (más fácil para desarrollo) @@ -890,12 +892,16 @@ providers: type: llamacpp model: qwen2.5-3b-instruct endpoint: http://localhost:9100/v1 # configurable, ver §6.1 - context_size: 4096 - max_tokens: 2048 + context_size: 4096 # debe coincidir con --ctx-size + max_tokens: 640 + temperature: 0.7 # defaults publicados de Qwen instruct + top_k: 20 + top_p: 0.8 + repeat_penalty: 1.05 default: true ``` -El adapter `llamacpp` se importa desde `rony-llm-agent/pkg/llm/providers/llamacpp` y se compila contra `llama.cpp` vía CGO o binario externo. +El adapter `llamacpp` se importa desde `rony-llm-agent/pkg/llm/providers/llamacpp` y habla HTTP con `llama-server` — sin CGO, sin enlazar contra llama.cpp. --- @@ -907,7 +913,8 @@ El adapter `llamacpp` se importa desde `rony-llm-agent/pkg/llm/providers/llamacp # Arrancar servidor HTTP chat-bot serve [--port 7331] [--host 0.0.0.0] [--reindex-on-start] -# Re-indexar portfolio (lee data/projects/*.md → SQLite FTS5) +# Re-indexar (lee data/projects/ + data/docs/ *.md y *.mdx → FTS5 + vectores) +# Necesita el servidor de embeddings arriba si los embeddings están activos. chat-bot reindex # Pregunta única (sin servidor, útil para tests) @@ -982,19 +989,62 @@ func serveCmd() *cobra.Command { ### 8.1 Recomendación: Self-hosted en VPS -```bash -# 1. Instalar dependencias -sudo apt install golang-go ollama -ollama pull qwen2.5:1.5b +Target: **2 cores de CPU, 8 GB de RAM, sin GPU.** Tres procesos — el bot y dos +instancias de `llama-server` — así que tres units. El bot depende de los dos. -# 2. Build +```bash +# 1. Build go build -o /usr/local/bin/chat-bot ./cmd/chat-bot -# 3. systemd service +# 2. El LLM +cat > /etc/systemd/system/llama-chat.service < /etc/systemd/system/llama-embed.service < /etc/systemd/system/chat-bot.service < ⚠️ **Decisiones pendientes de validar antes de implementar este módulo:** +> ✅ **The four decisions this section left open have been made and measured:** > -> - **Tokenizer FTS5** — el spec asume `unicode61 remove_diacritics 2`. Confirmar con datos reales si conviene cambiar a `porter` (stemming EN), `trigram` (sub-string matching) o un tokenizer custom para español. **Validar:** ejecutar queries representativas contra `data/projects/` y comparar recall antes de cerrar esta elección. -> - **Driver SQLite** — ✅ **DECIDIDO: `modernc.org/sqlite`** (puro Go, sin CGO). Ver benchmark abajo. -> - **Chunking** — el split por tamaño fijo (500 chars / 50 overlap) corta headings y code blocks arbitrariamente. **Validar:** medir recall con chunks por sección markdown (split por `#`/`##`) vs por tamaño. -> - **Sin similitud semántica** — BM25 no matchea "IA" con "machine learning" salvo que la palabra esté literal. **Validar:** tamaño del corpus y types of questions esperadas; si el corpus crece o las queries se vuelven abstractas, considerar agregar embeddings como capa secundaria. +> - **FTS5 tokenizer** — `unicode61 remove_diacritics 2`, as specced. Stemming +> was not the bottleneck; the language gap was, and embeddings close it. +> - **SQLite driver** — `modernc.org/sqlite` (pure Go, no CGO). Benchmark below. +> - **Chunking** — by markdown heading, not fixed size, with oversized sections +> split at `###` before falling back to byte offsets. §4.1. +> - **Semantic similarity** — added. BM25 alone returned **nothing at all** for +> a Spanish question about an English document. §4.2. ### 4.0 Driver decision: benchmark results @@ -498,258 +508,217 @@ Justificación: ### 4.1 Indexing pipeline ``` -data/projects/*.md - ↓ (read all files) -Raw markdown content - ↓ (split into chunks, ~500 chars, 50 overlap) -Chunks [] - ↓ (insert into SQLite FTS5 virtual table "portfolio_chunks") +data/projects/*.{md,mdx} kind=project → announced in the catalogue +data/docs/*.{md,mdx} kind=doc → retrievable, never announced + ↓ (skip each directory's README — those are instructions, not content) +Raw markdown + ↓ (split by heading; sections over the limit split at ###, then by size) + ↓ (drop the frontmatter chunk) +Chunks + ├─→ SQLite FTS5 virtual table "portfolio_chunks" + └─→ embeddings endpoint → "portfolio_vectors" (id, content_hash, dim, vec) Indexed corpus ``` +**Two kinds of source.** Everything under `data_path` is one of Victor's +projects and is listed in the catalogue injected into every prompt. Everything +under `docs_path` is searchable evidence that is *not* a project — his CV, an +about page. The CV is what someone deciding whether to hire actually reads, and +it was unreachable while it lived only in the Astro site; but filing it under +projects made the bot list "cv" as one of his works. + +**Three things are excluded or reshaped, each because of a measured failure:** + +| Rule | Failure it fixes | +|---|---| +| Skip `README*` in both directories | `data/projects/README.md` was indexed, so the catalogue announced "README" and "README.es" as projects of Victor's | +| Drop the frontmatter chunk | Dense metadata in a very short chunk is a magnet for short queries — a CV's `location:` field answered *"¿Dónde ha trabajado Victor?"* with a city instead of a work history | +| Split oversized sections at `###` | A CV's Experience section is a list of jobs; size-splitting cut one mid-word, stranding the employer's name in the previous chunk. Chunks now hold one job each, named `Experience — Metrimex — Frontend Developer` | + **When it runs:** -- On bot startup (if `--reindex-on-start` flag) - Manually: `./chat-bot reindex` +- On startup with `serve --reindex-on-start` - Via HTTP: `POST /api/reindex` +Vectors are built at index time, so enabling embeddings requires a reindex. +The chunk table is derived data: `OpenStore` rebuilds it when its columns are +missing, so upgrading an existing install needs no migration step. Conversation +tables are never touched by that rebuild. + ### 4.2 Retrieval pipeline ``` -User query "what projects does Victor have?" - ↓ (FTS5 MATCH query, BM25 ranking, top_k=5) -Top 5 relevant chunks - ↓ (format as context block) -System prompt += relevant chunks - ↓ (send to LLM) +User query "¿Con qué se paga en la tienda de ropa?" + ↓ + ├─→ FTS5 MATCH, BM25 ranking → top 15 (topK × 3) + └─→ embed(query) → cosine vs vecs → top 15 (topK × 3) + ↓ (Reciprocal Rank Fusion, k=60) +Top 5 chunks + ↓ (system prompt + project catalogue + excerpts + language directive) LLM generates answer ``` -### 4.3 Implementation +**Both halves are load-bearing.** Keyword search matches words exactly — no +stemming, no translation. The corpus is written in English and visitors ask in +Spanish, so the words carrying the meaning score zero: measured on the real +corpus, `"paga"` appears 0 times in a document that says *"Payments: Stripe"* +and `"trabajado"` 0 times in one that says *"worked"*. The question above +retrieved **nothing at all**. Embeddings (`nomic-embed-v2-moe`, multilingual) +put all three `tienda-ropa` chunks on top — but they blur exact rare tokens, +where BM25 is sharp. + +**Why RRF rather than a weighted score.** A BM25 score and a cosine share no +scale, so blending them numerically means inventing a conversion factor and +retuning it whenever the corpus changes. RRF ignores the magnitudes and ranks +by agreement between the two orderings: each list contributes `1/(60 + rank)`. +A chunk both halves like beats one that either loves alone. + +**Degradation.** If the embeddings endpoint is down or disabled, the vector +half returns nothing and retrieval continues keyword-only rather than failing +the request. + +**Stale vectors.** Chunk ids are derived from position, so they survive edits +to the body. Without a guard, editing a document leaves vectors that still +describe the text that was removed — reproduced live by changing a project's +payment provider and watching the old one keep coming back. Each vector stores +a hash of the exact text it was built from, and rows whose hash no longer +matches the chunk are skipped with a warning until the next reindex. + +**The catalogue.** Top-K returns the best-matching *sections*, so a broad +question like "what has Victor built?" cannot be answered from retrieval alone, +and a small model asked to enumerate from partial hits invents the rest. The +full project list is injected every turn at a cost of ~10 tokens per project. +This is what stopped the bot naming projects that do not exist. + +### 4.3 Schema ```go // internal/portfolio/indexer.go -package portfolio - -import ( - "context" - "database/sql" - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" -) - -type Indexer struct { - dataPath string - db *sql.DB - chunkSize int - chunkOverlap int -} - -func (i *Indexer) IndexAll(ctx context.Context) (int, error) { - files, err := filepath.Glob(filepath.Join(i.dataPath, "*.md")) - if err != nil { - return 0, err - } - - // Rebuild FTS5 index from scratch (delete + insert is faster than diff for small corpora) - if _, err := i.db.ExecContext(ctx, `DELETE FROM portfolio_chunks`); err != nil { - return 0, fmt.Errorf("clear index: %w", err) - } - - totalChunks := 0 - for _, file := range files { - chunks, err := i.indexFile(ctx, file) - if err != nil { - slog.Warn("failed to index file", "file", file, "err", err) - continue - } - totalChunks += chunks - } - - return totalChunks, nil -} - -func (i *Indexer) indexFile(ctx context.Context, path string) (int, error) { - content, err := os.ReadFile(path) - if err != nil { - return 0, err - } - - projectID := strings.TrimSuffix(filepath.Base(path), ".md") - chunks := splitIntoChunks(string(content), i.chunkSize, i.chunkOverlap) - - tx, err := i.db.BeginTx(ctx, nil) - if err != nil { - return 0, err - } - defer tx.Rollback() - - stmt, err := tx.PrepareContext(ctx, ` - INSERT INTO portfolio_chunks (id, project_id, source_file, chunk_index, content) - VALUES (?, ?, ?, ?, ?) - `) - if err != nil { - return 0, err - } - defer stmt.Close() - - for idx, chunk := range chunks { - id := fmt.Sprintf("%s-chunk-%d", projectID, idx) - if _, err := stmt.ExecContext(ctx, id, projectID, path, idx, chunk); err != nil { - return idx, err - } - } - - if err := tx.Commit(); err != nil { - return 0, err - } - return len(chunks), nil -} - -// schema.go — applied at startup const schema = ` CREATE VIRTUAL TABLE IF NOT EXISTS portfolio_chunks USING fts5( id UNINDEXED, project_id UNINDEXED, + kind UNINDEXED, -- 'project' | 'doc' source_file UNINDEXED, + section UNINDEXED, -- the heading this chunk came from chunk_index UNINDEXED, content, tokenize = 'unicode61 remove_diacritics 2' ); ` -func splitIntoChunks(text string, size, overlap int) []string { - // Simple implementation: split by size with overlap - // Production version uses tokenizer-aware chunking - var chunks []string - for i := 0; i < len(text); i += size - overlap { - end := i + size - if end > len(text) { - end = len(text) - } - chunks = append(chunks, text[i:end]) - } - return chunks -} +// Vectors live in an ordinary table keyed by chunk id. There is no ANN index: +// a portfolio is hundreds of chunks, not millions, so a full scan with a dot +// product is microseconds and needs no extension. +const vectorSchema = ` +CREATE TABLE IF NOT EXISTS portfolio_vectors ( + chunk_id TEXT PRIMARY KEY, + content_hash TEXT NOT NULL, -- sha256 of the exact text embedded + dim INTEGER NOT NULL, + vec BLOB NOT NULL -- little-endian float32, unit-normalised +); +` ``` -### 4.4 Retrieval in the agent loop +FTS5 has no `ALTER TABLE ADD COLUMN`, so `ensureChunkSchema` drops and +recreates `portfolio_chunks` when an older database is missing a column. That +is safe precisely because the table is a derived index — every row is +regenerated from the markdown on the next reindex. It deliberately touches +only the index tables; conversations live in the same file and are real user +data. -```go -// internal/portfolio/search.go -package portfolio +Vectors are unit-normalised at write time, so a dot product **is** the cosine +and retrieval needs no division per comparison. `dim` is stored so a change of +embedding model is detected rather than silently producing garbage similarity: +rows whose dimension does not match the query vector are skipped. -type Hit struct { - ProjectID string - SourceFile string - ChunkIndex int - Content string - Score float64 // BM25 score from FTS5 -} +**Key API:** -func (s *Store) Search(ctx context.Context, query string, topK int) ([]Hit, error) { - // Escape user input: FTS5 syntax can break with special chars - ftsQuery := sanitizeFTS5(query) +| Function | File | Purpose | +|---|---|---| +| `SourcesFor(dataPath, docsPath)` | `indexer.go` | Builds the `[]Source` pair; an empty path is skipped, so `docs_path` is optional without branching | +| `Store.Reindex(ctx, sources, cfg)` | `indexer.go` | Rebuilds both tables from disk | +| `Store.Search(ctx, query, topK)` | `indexer.go` | BM25 only, excludes `section = 'frontmatter'` | +| `Store.HybridSearch(ctx, emb, q, topK)` | `hybrid.go` | BM25 + vectors fused with RRF | +| `Store.Catalog(ctx)` | `indexer.go` | Project list for the prompt; filters `kind = 'project'` | +| `embed.Client.Embed(ctx, texts)` | `internal/embed/` | OpenAI-compatible embeddings, reordered by the response `index` field | - rows, err := s.db.QueryContext(ctx, ` - SELECT project_id, source_file, chunk_index, content, bm25(portfolio_chunks) AS score - FROM portfolio_chunks - WHERE portfolio_chunks MATCH ? - ORDER BY score - LIMIT ? - `, ftsQuery, topK) - if err != nil { - return nil, err - } - defer rows.Close() +### 4.4 Assembling the prompt - var hits []Hit - for rows.Next() { - var h Hit - if err := rows.Scan(&h.ProjectID, &h.SourceFile, &h.ChunkIndex, &h.Content, &h.Score); err != nil { - return nil, err - } - hits = append(hits, h) - } - return hits, rows.Err() -} - -// sanitizeFTS5 wraps the user query so reserved chars and unquoted strings don't crash FTS5. -// A pragmatic choice for a Q&A bot: append prefix-match wildcard to each token. -func sanitizeFTS5(q string) string { - tokens := strings.FieldsFunc(q, func(r rune) bool { - return !(r == '-' || r == '_' || (r >= '0' && r <= '9') || - (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || - r > 0x7F) // keep accented chars - }) - if len(tokens) == 0 { - return `""` - } - for i, t := range tokens { - tokens[i] = `"` + strings.ToLower(t) + `"*` - } - return strings.Join(tokens, " ") -} -``` +`agent.Runner.BuildMessages` runs once per request and produces exactly one +system message followed by the conversational turns. The order of operations +is load-bearing: ```go // internal/agent/runner.go -package agent +func (r *Runner) BuildMessages(ctx context.Context, history []Message) ([]Message, string, error) { + // 1. Pull system-role notes (the compactor's summary) out of the turn list. + notes, history := foldSystemNotes(history) -func (r *Runner) buildSystemPrompt(ctx context.Context, query string) (string, error) { - basePrompt := r.persona.SystemPrompt + // 2. The visitor's language selects which rendition of the prompt we + // build on, so it is resolved before anything else. + lang := detectLanguage(history) + systemPrompt := r.promptFor(lang) - hits, err := r.store.Search(ctx, query, r.config.RAG.TopK) - if err != nil { - return "", err - } - if len(hits) == 0 { - return basePrompt, nil - } + // 3. Resolved before retrieval so the excerpt budget accounts for it. + catalog := r.catalogBlock(ctx) - var contextBlock strings.Builder - contextBlock.WriteString(basePrompt) - contextBlock.WriteString("\n\n## Relevant context\n\n") - for _, h := range hits { - contextBlock.WriteString(fmt.Sprintf("### Source: %s\n%s\n\n", - h.SourceFile, h.Content)) - } - return contextBlock.String(), nil -} + // 4. Hybrid retrieval on the last user turn. + hits, err := r.store.HybridSearch(ctx, r.embedder, last.Content, r.topK) + ragContext = r.limitRAGContext(systemPrompt, catalog, formatHits(hits), history) -func (r *Runner) RunStream(ctx context.Context, messages []llm.Message) iter.Seq2[Chunk, error] { - return func(yield func(Chunk, error) bool) { - lastUserMsg := getLastUserMessage(messages) - systemPrompt, err := r.buildSystemPrompt(ctx, lastUserMsg) - if err != nil { - yield(Chunk{}, err) - return - } + // 5. prompt → catalogue → excerpts → summary → language directive. + system := botpersona.BuildSystemPrompt(systemPrompt, catalog, ragContext) + system += "\n\n" + strings.Join(notes, "\n\n") + system += "\n\n" + r.languageDirective(lang) - messages = prependSystem(messages, systemPrompt) - - for chunk, err := range r.loop.RunStream(ctx, messages) { - if !yield(chunk, err) { - return - } - } - } + history, err = r.fitHistory(system, history) + return append([]Message{{Role: RoleSystem, Content: system}}, history...), ragContext, nil } ``` -**Why this is simpler than embeddings:** -- No embedding model to download or run (saves ~270MB of RAM and ~200ms per query) -- One file (`data/portfolio.db`), one driver, no extra process -- BM25 ranking is excellent for keyword-based retrieval over structured docs like project READMEs -- Trade-off: no semantic similarity ("projects about AI" won't match "machine learning" without the literal words). Mitigation: `trigram` tokenizer handles morphology well for English/Spanish. +**Why exactly one system message.** Gemma 3's chat template raises +*"Conversation roles must alternate user/assistant/..."* on any system message +after the first, which llama-server surfaces as HTTP 400 — enabling compaction +used to kill the conversation outright the first time it fired. Anthropic's API +rejects mid-conversation system turns too, so folding is the portable +behaviour rather than a Gemma workaround. `foldSystemNotes` never aliases the +caller's slice; the handler reuses the history it passes in. + +**Why the language directive goes last.** It is the instruction a small model +is most likely to still be holding when it starts generating. Position was not +enough on its own, though — see below. + +**Matching the visitor's language** took three attempts, measured on gemma-3-1b +over the same five Spanish questions: + +| Approach | Result | +|---|---| +| English prompt + "reply in the user's language" | 1/5 answered in Spanish | +| English prompt + Spanish few-shot examples | 5/5 Spanish, but ~2/5 were the example reply copied verbatim instead of an answer | +| A full Spanish rendition of the prompt (`system_prompt_es`) | 4/5 Spanish, 4/5 real answers | + +So `internal/i18n` detects the language and `promptFor` selects the rendition; +the directive reinforces a prompt already written in the right language rather +than trying to override one written in the wrong one. The two renditions in +the YAML have to be kept in sync by hand. + +**Why keyword search alone was not enough.** The original plan cited three +reasons to skip embeddings — no model to run, one file and one driver, and +BM25 being strong on structured project docs. The first two still hold and +cost what was predicted (~0.91 GB resident, a second process). The third was +right about the corpus and wrong about the questions: BM25 is excellent at +retrieving English documents given English keywords, and this bot is asked in +Spanish. That is not a morphology problem a `trigram` tokenizer fixes — it is a +translation problem. Hence §4.2, and hence both halves are kept. --- ## 🗜️ 4.5 Auto-compaction -Long conversations eventually run out of context — at qwen2.5-3b's 4k window, the ~3k-token system prompt + RAG block leaves only room for 2–3 user turns. Auto-compaction solves this by folding the older portion of the conversation into a single summary system message when the previous turn's input tokens cross a configurable threshold. +Long conversations eventually run out of context. Auto-compaction folds the older portion of the conversation into a single summary system message when the previous turn's input tokens cross a configurable threshold. + +**How much headroom there actually is.** Measured over 20 real requests at `context_size: 4096`, the largest prompt this bot ever built was **1255 tokens** — system prompt, project catalogue, five retrieved chunks and the question — with a median of 1069. That leaves room for roughly a dozen short turns before the 75% threshold (~3070 tokens) is reached, not the 2–3 an earlier draft of this document estimated. Compaction is therefore a safety net for genuinely long threads rather than something that fires in a typical visit; across the whole benchmark it never triggered and nothing was truncated. ### When it fires @@ -951,30 +920,55 @@ ls $RONY_MODELS_PATH/qwen2.5-3b-instruct-q4_k_m.gguf # 2. Start llama-server (port is configurable; default llama.cpp is 8080) llama-server \ - -m $RONY_MODELS_PATH/qwen2.5-3b-instruct-q4_k_m.gguf \ - --port 9100 \ - --host 127.0.0.1 \ - --ctx-size 4096 \ - --mlock # prevents swap, critical on shared VPS + -m $RONY_MODELS_PATH/Qwen2.5/qwen2.5-3b-instruct-q4_k_m.gguf \ + --port 9100 --host 127.0.0.1 \ + --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 -# 3. Make sure configs/portfolio-bot.yaml points to the same port +# 3. Start the embeddings server (second process, second terminal) +llama-server \ + -m $RONY_MODELS_PATH/embeddings/nomic-embed-v2-moe.Q5_K_M.gguf \ + --port 9200 --embedding --pooling mean \ + --ctx-size 2048 --parallel 1 --device none --threads 2 + +# 4. Make sure configs/portfolio-bot.yaml points to the same ports # providers[0].endpoint: http://localhost:9100/v1 +# embeddings.endpoint: http://localhost:9200/v1 -# 4. Start the bot (default port 7331, also configurable) +# 5. Build the index (needs the embedder up — vectors are built here) +./bin/chat-bot reindex + +# 6. Start the bot (default port 7331, also configurable) ./bin/chat-bot serve # → Serves on http://localhost:7331 # → Override with: ./bin/chat-bot serve --port 9101 --host 127.0.0.1 ``` +**Flags that are not optional, each for a measured reason:** + +| Flag | Why | +|---|---| +| `--device none` | llama.cpp brings up a compiled-in GPU backend even with `-ngl 0`, and on a GPU-less host those buffers come out of system RAM. Measured on qwen2.5-3b: 2.54 GB with a GPU absorbing them, **3.66 GB without**. Budget from the second number | +| `--parallel 1` | `--ctx-size` is divided across slots and the default is 4, so `--ctx-size 4096` without it gives each request 1024 tokens | +| `--mlock` | Prevents swap — critical on a shared VPS | +| `--temp` / `--top-k` / `--top-p` | Use the model authors' published values, not llama.cpp's defaults. gemma-3-1b at `temperature 0.7` with the rest unset produced 16-token stub answers | +| `--pooling mean` (embedder) | Without it the endpoint does not return one vector per input and the client rejects the response | + **Port reference:** | What | Default | How to change | |---|---|---| | `llama-server` HTTP port | 8080 (llama.cpp convention) | `--port N` flag when starting `llama-server` | +| embeddings `llama-server` port | 8080 (same convention) | `--port N`; this repo uses 9200 | | chat-bot HTTP port | 7331 | `--port N` flag on `serve`, or `server.port` in YAML | | chat-bot → llama-server URL | `http://localhost:8080/v1` | `endpoint` field on the provider in YAML | -The `llamacpp` provider is imported from `rony-llm-agent/pkg/llm/providers/llamacpp` and is compiled against `llama.cpp` via CGO or external binary. +Keep `context_size` in the YAML equal to `--ctx-size`: the bot sizes its RAG +and compaction budgets from that number and never asks the server what it +actually has, so a mismatch means prompts the server rejects. + +The `llamacpp` provider is imported from `rony-llm-agent/pkg/llm/providers/llamacpp` and speaks HTTP to the server above — no CGO, no linking against llama.cpp. ### 6.2 Alternative: Ollama (easier for development) @@ -1009,12 +1003,16 @@ providers: type: llamacpp model: qwen2.5-3b-instruct endpoint: http://localhost:9100/v1 # configurable, see §6.1 - context_size: 4096 - max_tokens: 2048 + context_size: 4096 # must match --ctx-size + max_tokens: 640 + temperature: 0.7 # Qwen's published instruct defaults + top_k: 20 + top_p: 0.8 + repeat_penalty: 1.05 default: true ``` -The `llamacpp` adapter is imported from `rony-llm-agent/pkg/llm/providers/llamacpp` and is compiled against `llama.cpp` via CGO or external binary. +The `llamacpp` adapter is imported from `rony-llm-agent/pkg/llm/providers/llamacpp` and speaks HTTP to `llama-server` — no CGO, no linking against llama.cpp. --- @@ -1026,7 +1024,8 @@ The `llamacpp` adapter is imported from `rony-llm-agent/pkg/llm/providers/llamac # Start HTTP server chat-bot serve [--port 7331] [--host 0.0.0.0] [--reindex-on-start] -# Re-index portfolio (reads data/projects/*.md → SQLite FTS5) +# Re-index (reads data/projects/ + data/docs/ *.md and *.mdx → FTS5 + vectors) +# Needs the embeddings server up if embeddings are enabled. chat-bot reindex # Single question (no server, useful for tests) @@ -1101,19 +1100,62 @@ func serveCmd() *cobra.Command { ### 8.1 Recommendation: Self-hosted on VPS -```bash -# 1. Install dependencies -sudo apt install golang-go ollama -ollama pull qwen2.5:1.5b +Target: **2 CPU cores, 8 GB RAM, no GPU.** Three processes — the bot and two +`llama-server` instances — so three units. The bot depends on both. -# 2. Build +```bash +# 1. Build go build -o /usr/local/bin/chat-bot ./cmd/chat-bot -# 3. systemd service +# 2. The LLM +cat > /etc/systemd/system/llama-chat.service < /etc/systemd/system/llama-embed.service < /etc/systemd/system/chat-bot.service < The estimates in this section predate the measurements below and were made +> on a machine with a GPU. They understate CPU-only RSS by 1–3 GB and their +> "recommended" contexts are far larger than this bot ever uses. Treat the +> **Measured RSS** section as authoritative and this one as the KV-cache +> arithmetic only. + +| Model | context_size | KV cache | Verdict | +|-------------------|-------------:|---------:|--------------------------------| +| qwen2.5-3b Q4 | **4096** | ~290 MB | **the default** — 3.64 GB measured | +| qwen2.5-3b Q4 | 8192 | ~580 MB | +212 MB for headroom never used | +| gemma-3-1b Q4 | 4096 | ~145 MB | cheap, but see the accuracy note | +| gemma-3-4b Q4 | 4096 | ~1.1 GB | untested here | + +### Measured RSS (not estimated) + +**Measure with `--device none`, or the numbers lie.** llama.cpp initialises a +compiled-in GPU backend even with `-ngl 0`. If the build machine has a GPU it +quietly holds the compute buffers, and the RSS you measure is the RSS you will +*not* get on a GPU-less VPS. The gap is not a rounding error: + +| Model | with a GPU present | `--device none` | +|------------------------------|-------------------:|----------------:| +| qwen2.5-3b Q4_K_M | 2.54 GB | **3.66 GB** | +| granite-4.0-h-tiny (7B-A1B) | 4.48 GB | **7.29 GB** | +| gemma-3-1b Q4_K_M | 1.29 GB | **1.05 GB** | + +Granite looked like it fit an 8 GB box and does not. gemma goes the other way — +its compute buffers are tiny either way, so dropping the GPU runtime is a net +saving. + +Real `RSS` on CPU only, `--parallel 1 --threads 2 --mlock`, in steady state +after serving requests: + +| Process | ctx | RSS | +|-------------------------------------------|-----:|---------:| +| qwen2.5-3b Q4_K_M | 8192 | 3.85 GB | +| qwen2.5-3b Q4_K_M | 4096 | 3.64 GB | +| gemma-3-1b Q4_K_M | 8192 | 1.05 GB | +| granite-4.0-h-tiny Q4_K_M | any | 7.25 GB | +| nomic-embed-v2-moe Q5_K_M (`--embedding`) | 2048 | 0.91 GB | +| the Go bot + SQLite | — | 0.02 GB | + +Two things worth noting from that table: + +- **Cold RSS understates it.** qwen at 4096 loads at 3.51 GB and settles at + 3.64 GB after ten requests. Budget from the steady figure. +- **Granite ignores `--ctx-size` entirely** (7.24 GB at 2048, 7.29 GB at 8192). + It is a hybrid Mamba model: the recurrent state is fixed-size, so a 1M-token + window is nearly free — and there is no context lever to pull when it doesn't + fit. + +**Sizing on a shared VPS.** If the box also serves other sites, budget backwards +from what they need. On an 8 GB VPS the full hybrid stack (qwen2.5-3b at 4096 + +embedder + bot) is **4.6 GB**, leaving ~3.4 GB. + +KV quantization buys less than people expect — 130 MB on qwen2.5-3b at 8192 — +because the weights dominate. Reach for a smaller model, or a smaller context, +before reaching for `--cache-type-*`. ### 16 GB VPS -| Model | context_size | KV cache | RAM usada | -|-------------------|-------------:|---------:|----------:| -| qwen2.5-3b Q4 | 32768 | ~2.3 GB | ~5.0 GB | -| gemma-3-4b Q4 | 16384 | ~4.5 GB | ~7.5 GB | +Estimated, not measured — and the same GPU caveat applies, so add 1–2 GB for a +CPU-only host. Contexts this large are also well past anything this bot builds +(1255 tokens measured); they only matter if you repurpose it for long documents. + +| Model | context_size | KV cache | RAM usada (est.) | +|-------------------|-------------:|---------:|-----------------:| +| qwen2.5-3b Q4 | 32768 | ~2.3 GB | ~5.0 GB | +| gemma-3-4b Q4 | 16384 | ~4.5 GB | ~7.5 GB | --- ## 5. Worked examples -### qwen2.5-3b en 8 GB +### qwen2.5-3b on a shared 8 GB VPS — the shipped default ```yaml # configs/portfolio-bot.yaml @@ -118,38 +172,53 @@ providers: type: llamacpp model: qwen2.5-3b-instruct endpoint: http://localhost:9100/v1 - context_size: 16384 # ~1.1 GB KV, deja 4 GB libres - max_tokens: 1024 # respuestas moderadas + context_size: 4096 # 3.64 GB measured; largest real prompt was 1255 tokens + max_tokens: 640 + temperature: 0.7 + top_k: 20 + top_p: 0.8 + repeat_penalty: 1.05 ``` ```bash llama-server \ -m qwen2.5-3b-instruct-q4_k_m.gguf \ - --ctx-size 16384 \ - -ngl 0 -t 2 \ - --mlock + --port 9100 --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 ``` -### gemma-3-1b en 8 GB +Plus the embedder, which has to stay resident because every visitor question +must be embedded before it can be compared: -```yaml -providers: - - name: llamacpp-local - type: llamacpp - model: gemma-3-1b-it - endpoint: http://localhost:9100/v1 - context_size: 32768 # sobra RAM, contexto largo - max_tokens: 1024 +```bash +llama-server \ + -m nomic-embed-v2-moe.Q5_K_M.gguf \ + --port 9200 --embedding --pooling mean \ + --ctx-size 2048 --parallel 1 --device none --threads 2 ``` +Total: 3.64 + 0.91 + 0.02 = **4.57 GB**. + +### gemma-3-1b — cheaper, and why it isn't the default + ```bash llama-server \ -m gemma-3-1b-it-Q4_K_M.gguf \ - --ctx-size 32768 \ - -ngl 0 -t 2 \ - --mlock + --port 9100 --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 1.0 --top-k 64 --top-p 0.95 --min-p 0.0 --repeat-penalty 1.15 ``` +1.05 GB instead of 3.64 — 2.8 GB cheaper and ~35% faster. On a 20-question +bilingual battery against the real corpus, with identical hybrid retrieval, it +scored **3/10** on grounded content against qwen's **9/10**, and among its +failures it echoed the system prompt's own instructions back to the visitor. +The sampling flags above matter: Google's published config is `temp 1.0 / +top_k 64 / top_p 0.95 / min_p 0.0`, and `--repeat-penalty 1.15` is deliberately +off-spec because at Google's recommended 1.0 the model looped on the repetitive +shape of the retrieved-chunk headers. + ### Gemma con chat template custom (sin system role líder) Gemma 3 rechaza mensajes `system` antes del primer `user`. Dos opciones: @@ -177,9 +246,8 @@ Gemma 3 rechaza mensajes `system` antes del primer `user`. Dos opciones: ```bash llama-server \ -m gemma-3-1b-it-Q4_K_M.gguf \ - --ctx-size 32768 \ - -ngl 0 -t 2 \ - --mlock \ + --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ --chat-template-file ~/.llama/gemma3.jinja ``` @@ -204,9 +272,10 @@ summarized into a single system note. This means: request is expensive; raise it (e.g. `0.9`) when you want to keep more verbatim history. -For a portfolio bot with `context_size: 16384` and `max_tokens: 1024`, -compaction fires when input exceeds ~12k tokens — leaving ~5k for the -fresh history, which is ~10-15 recent user turns. More than enough. +For this bot at `context_size: 4096` and `max_tokens: 640`, compaction fires +when input exceeds ~3070 tokens. Measured over 20 real requests the largest +prompt was 1255 tokens, so in practice it never fires on a single-question +visit — it exists for the chatty visitor who keeps a thread going. --- @@ -218,8 +287,10 @@ fresh history, which is ~10-15 recent user turns. More than enough. bajá el context. 3. **Más contexto ≠ más rápido.** El prefill (procesar el input) escala lineal con la cantidad de tokens. Generación (output) no se ve - afectada. Con `--ctx-size 32768` y un input de 500 tokens, el TTFT - apenas cambia; con 20k tokens de input sí. + afectada. Con `--ctx-size 4096` y un input de 500 tokens, el TTFT apenas + cambia; con 20k tokens de input sí. Medido: bajar de 8192 a 4096 dejó la + generación exactamente igual (21,0 tok/s), porque el prefill escala con los + tokens que procesás, no con los que reservás. 4. **Streams concurrentes.** Cada stream activo reserva su propio KV cache. En 8 GB no hagas más de 1-2 streams simultáneos — el rate limiter del bot (default 30 req/min) ya te protege. @@ -230,14 +301,23 @@ fresh history, which is ~10-15 recent user turns. More than enough. ## 8. Quick-pick table -Copy-paste según tu setup: +Start from what the prompt actually costs, not from what the model can hold. +Measured on this bot over 20 real requests: the largest prompt ever built was +**1255 tokens** (system prompt + project catalogue + 5 retrieved chunks + the +question), median 1069. Compaction begins at `threshold_ratio` × the window, so +4096 leaves 2.4x headroom before it even engages. -| Setup | `context_size` | `max_tokens` | -|-----------------------------|---------------:|-------------:| -| 4 GB + gemma-3-1b | 8192 | 512 | -| 4 GB + qwen2.5-1.5b | 4096 | 512 | -| 8 GB + qwen2.5-1.5b | 16384 | 768 | -| 8 GB + qwen2.5-3b | 16384 | 1024 | -| 8 GB + gemma-3-1b | 32768 | 1024 | -| 16 GB + qwen2.5-3b | 32768 | 1024 | -| 16 GB + gemma-3-4b | 16384 | 1024 | +| Setup | `context_size` | `max_tokens` | Note | +|------------------------------|---------------:|-------------:|------| +| 8 GB shared + qwen2.5-3b | **4096** | **640** | the default; 4.6 GB total stack | +| 8 GB dedicated + qwen2.5-3b | 8192 | 640 | +212 MB, no measured benefit | +| 8 GB + gemma-3-1b | 4096 | 640 | 2.8 GB cheaper, and 3/10 vs 9/10 on grounded answers — see the provider comment in the config | +| 16 GB + qwen2.5-3b | 8192 | 1024 | room for longer threads | + +Going above 8192 for a portfolio bot is reserving memory you will not use. A +bigger window does not make answers better; it makes the KV cache bigger and +delays compaction that was never going to trigger. + +`max_tokens: 640` is sized for the answers this persona is asked to give (2–4 +sentences plus a short list). Raising it takes budget from the input side and +makes compaction fire sooner. diff --git a/internal/agent/client.go b/internal/agent/client.go index 5f552aa..960e84c 100644 --- a/internal/agent/client.go +++ b/internal/agent/client.go @@ -23,7 +23,15 @@ func NewClient(p config.Provider) (llm.LLMClient, error) { Model: p.Model, ContextWindow: p.ContextSize, MaxTokens: p.MaxTokens, - Temperature: p.Temperature, + // Sampling: zero values are omitted from the JSON payload by the + // adapter, so anything left unset in the YAML falls through to + // llama-server's own default rather than being forced to 0. + Temperature: p.Temperature, + TopK: p.TopK, + TopP: p.TopP, + MinP: p.MinP, + RepetitionPenalty: p.RepeatPenalty, + PresencePenalty: p.PresencePenalty, }) case "ollama", "openai": return openai.New(openai.Config{ diff --git a/internal/agent/runner.go b/internal/agent/runner.go index 2c2e441..a08d6d1 100644 --- a/internal/agent/runner.go +++ b/internal/agent/runner.go @@ -23,6 +23,7 @@ import ( "github.com/VictorVargas/rony-llm-agent/pkg/llm" llmpersona "github.com/VictorVargas/rony-llm-agent/pkg/persona" + "github.com/VictorVargas/rony-chat-bot/internal/i18n" botpersona "github.com/VictorVargas/rony-chat-bot/internal/persona" "github.com/VictorVargas/rony-chat-bot/internal/portfolio" ) @@ -50,6 +51,17 @@ type Runner struct { compaction CompactionConfig lastCompact CompactionStats + + // includeCatalog injects the full project list into the system prompt. + // See portfolio.Store.Catalog for why retrieval alone isn't enough. + includeCatalog bool + + // localizedPrompts maps a language code to a full rendition of the + // system prompt. See WithLocalizedPrompt. + localizedPrompts map[string]string + + // embedder enables the semantic half of retrieval. Nil → keyword only. + embedder portfolio.Embedder } type Usage struct { @@ -99,6 +111,109 @@ func (r *Runner) WithCompaction(cfg CompactionConfig) *Runner { return r } +// WithCatalog enables injecting the full project catalogue into the system +// prompt. Returns the receiver for chaining. No-op when RAG is disabled +// (there is no store to read the catalogue from). +func (r *Runner) WithCatalog(enabled bool) *Runner { + r.includeCatalog = enabled + return r +} + +// WithLocalizedPrompt registers a full translation of the system prompt for +// a language code ("es"). When the visitor writes in that language, this text +// replaces the default prompt wholesale. +// +// Translating the prompt beats appending a "reply in Spanish" line to an +// English one: an instruction is a weak signal next to a thousand tokens of +// English telling the model, implicitly, what language it is working in. +// Empty prompts are ignored so an absent YAML key is a no-op. +func (r *Runner) WithLocalizedPrompt(lang, prompt string) *Runner { + if strings.TrimSpace(prompt) == "" { + return r + } + if r.localizedPrompts == nil { + r.localizedPrompts = map[string]string{} + } + r.localizedPrompts[lang] = prompt + return r +} + +// WithEmbedder turns on hybrid retrieval: keyword search fused with vector +// search. Nil is accepted and leaves the runner on keyword-only search, which +// is what makes the embeddings endpoint optional rather than a hard dependency. +func (r *Runner) WithEmbedder(e portfolio.Embedder) *Runner { + r.embedder = e + return r +} + +// promptFor returns the system prompt to use for a detected language, +// falling back to the default when no translation is registered. +func (r *Runner) promptFor(lang string) string { + if p, ok := r.localizedPrompts[lang]; ok { + return p + } + return r.systemPrompt +} + +// detectLanguage returns the language code of the most recent user message, +// or "" when there is nothing to go on. +func detectLanguage(history []Message) string { + for i := len(history) - 1; i >= 0; i-- { + if history[i].Role == RoleUser { + return i18n.Detect(history[i].Content) + } + } + return "" +} + +// languageDirective returns a one-line instruction pinning the reply language, +// or "" when the persona pins no language and the history is empty. +// +// Asking the model to "reply in the user's language" does not survive a 1B +// parameter budget: the rest of the system prompt is English, so English wins +// and Spanish questions come back in English. Detecting the language in Go and +// stating it outright is deterministic and costs one line. +// +// The directive is written *in* the target language on purpose — an +// instruction in Spanish is a much stronger prior for answering in Spanish +// than the same sentence in English. +// +// persona.language acts as the override: the default "the user's language" +// (or empty) means auto-detect, anything else pins that language verbatim. +func (r *Runner) languageDirective(lang string) string { + if pinned := strings.TrimSpace(r.persona.Language); pinned != "" && + !strings.EqualFold(pinned, "the user's language") { + return fmt.Sprintf("Write your entire reply in %s.", pinned) + } + if lang == "" { + return "" + } + if lang == "es" { + return "El visitante escribió en español. Responde ÍNTEGRAMENTE en español, incluido el saludo." + } + return "The visitor wrote in English. Write your entire reply in English." +} + +// catalogBlock renders the project catalogue as a markdown list, or "" when +// the feature is off, RAG is disabled, or the index is empty. A failure to +// read it is logged and swallowed: the catalogue improves grounding but a +// chat turn should never fail because of it. +func (r *Runner) catalogBlock(ctx context.Context) string { + if !r.includeCatalog || r.store == nil { + return "" + } + entries, err := r.store.Catalog(ctx) + if err != nil { + slog.Warn("catalog lookup failed, continuing without it", "err", err) + return "" + } + var b strings.Builder + for _, e := range entries { + fmt.Fprintf(&b, "- **%s** — %s\n", e.ProjectID, e.Title) + } + return strings.TrimRight(b.String(), "\n") +} + // LastUsage returns the token usage recorded on the most recent call. func (r *Runner) LastUsage() Usage { return *r.usage } @@ -116,23 +231,45 @@ func (r *Runner) LastCompaction() CompactionStats { return r.lastCompact } // (Stream calls BuildMessages internally, and handlers sometimes call it // first to extract the RAG context for the sources event). func (r *Runner) BuildMessages(ctx context.Context, history []Message) ([]Message, string, error) { + // Pull any system-role notes (the compactor's "Earlier conversation + // summary") out of the history before anything else: they are context, + // not a conversational turn, and several chat templates reject them as + // one. See foldSystemNotes. + notes, history := foldSystemNotes(history) + + // The visitor's language selects which rendition of the system prompt we + // build on, so it has to be resolved before anything else. + lang := detectLanguage(history) + systemPrompt := r.promptFor(lang) + + // Resolved before retrieval so the excerpt budget accounts for it. + catalog := r.catalogBlock(ctx) + ragContext := "" if r.store != nil && len(history) > 0 { last := history[len(history)-1] if last.Role == RoleUser { - hits, err := r.store.Search(ctx, last.Content, r.topK) + hits, err := r.store.HybridSearch(ctx, r.embedder, last.Content, r.topK) if err != nil { return nil, "", fmt.Errorf("rag search: %w", err) } if len(hits) > 0 { - ragContext = r.limitRAGContext(formatHits(hits), history) + ragContext = r.limitRAGContext(systemPrompt, catalog, formatHits(hits), history) } } } // The system prompt comes from the YAML, not from the persona struct. // Keep the persona around only for the UI greeting. - system := botpersona.BuildSystemPrompt(r.systemPrompt, ragContext) + system := botpersona.BuildSystemPrompt(systemPrompt, catalog, ragContext) + if len(notes) > 0 { + system += "\n\n" + strings.Join(notes, "\n\n") + } + // Last line of the prompt, deliberately: it's the instruction a small + // model is most likely to still be holding when it starts generating. + if d := r.languageDirective(lang); d != "" { + system += "\n\n" + d + } history, err := r.fitHistory(system, history) if err != nil { return nil, "", err @@ -143,6 +280,44 @@ func (r *Runner) BuildMessages(ctx context.Context, history []Message) ([]Messag return msgs, ragContext, nil } +// foldSystemNotes separates system-role messages from the conversational +// turns. Everything the runner puts in the history as a system message is +// really prompt context — today that is only the compactor's summary — so it +// belongs inside the single leading system message rather than in the turn +// list. +// +// This is not cosmetic. Gemma 3's chat template raises "Conversation roles +// must alternate user/assistant/..." on any system message after the first, +// which llama-server surfaces as HTTP 400: before this, enabling compaction +// killed the conversation outright the first time it fired. Anthropic's API +// rejects mid-conversation system turns too, so folding is the portable +// behaviour rather than a Gemma workaround. +// +// The returned slice never aliases the caller's array. +func foldSystemNotes(history []Message) (notes []string, rest []Message) { + hasSystem := false + for _, m := range history { + if m.Role == RoleSystem { + hasSystem = true + break + } + } + if !hasSystem { + return nil, history + } + rest = make([]Message, 0, len(history)) + for _, m := range history { + if m.Role == RoleSystem { + if s := strings.TrimSpace(m.Content); s != "" { + notes = append(notes, s) + } + continue + } + rest = append(rest, m) + } + return notes, rest +} + const maxContextSafetyMargin = 256 func contextBudget(window int) int { @@ -156,7 +331,7 @@ func contextBudget(window int) int { return window - margin } -func (r *Runner) limitRAGContext(ragContext string, history []Message) string { +func (r *Runner) limitRAGContext(systemPrompt, catalog, ragContext string, history []Message) string { if strings.TrimSpace(ragContext) == "" { return "" } @@ -176,7 +351,7 @@ func (r *Runner) limitRAGContext(ragContext string, history []Message) string { if fitted != "" { candidate = fitted + "\n\n" + block } - if estimatePromptTokens(botpersona.BuildSystemPrompt(r.systemPrompt, candidate))+historyTokens > budget { + if estimatePromptTokens(botpersona.BuildSystemPrompt(systemPrompt, catalog, candidate))+historyTokens > budget { break } fitted = candidate @@ -363,10 +538,21 @@ func (r *Runner) Stream(ctx context.Context, history []Message) iter.Seq2[llm.St } } +// formatHits renders retrieved chunks for the prompt. Non-project documents +// are labelled as such: the CV is prime evidence for "does he know X?" but it +// is not a portfolio project, and without the label a small model happily +// announces "cv" as one of Victor's projects. func formatHits(hits []portfolio.SearchResult) string { var b strings.Builder for i, h := range hits { - fmt.Fprintf(&b, "### [%d] %s — %s\n", i+1, h.ProjectID, h.Section) + // The marker goes after the section, never in the name slot: the + // SSE "sources" event parses everything before " — " as the source + // id, and a decorated name would leak into the UI chips. + section := h.Section + if h.Kind == portfolio.KindDoc { + section += " (reference document, not one of Victor's projects)" + } + fmt.Fprintf(&b, "### [%d] %s — %s\n", i+1, h.ProjectID, section) b.WriteString(h.Content) b.WriteString("\n\n") } diff --git a/internal/agent/runner_compaction_test.go b/internal/agent/runner_compaction_test.go index 126513a..dc687ba 100644 --- a/internal/agent/runner_compaction_test.go +++ b/internal/agent/runner_compaction_test.go @@ -423,3 +423,153 @@ func TestTruncateToBudgetAlwaysKeepsLastUserTurn(t *testing.T) { t.Errorf("dropped+kept = %d, want %d", len(dropped)+len(kept), len(history)) } } + +// A compacted history carries the summary as a system message. Several chat +// templates (Gemma 3, Anthropic) reject a system turn that isn't the first +// message, so BuildMessages must fold it into the single leading system +// prompt rather than pass it through as its own turn. Regression test: this +// used to send two system messages and llama-server answered HTTP 400. +func TestBuildMessagesFoldsSystemNotesIntoOneSystemMessage(t *testing.T) { + r := New(&compactionStub{window: 100000}, personaMinimal(), "persona", nil, 5) + + msgs, _, err := r.BuildMessages(context.Background(), []Message{ + {Role: RoleSystem, Content: "Earlier conversation summary:\nuser asked about the dashboard"}, + {Role: RoleUser, Content: "y el stack?"}, + }) + if err != nil { + t.Fatal(err) + } + + for i, m := range msgs { + if i > 0 && m.Role == RoleSystem { + t.Fatalf("msgs[%d] is a second system message; want exactly one, at index 0", i) + } + } + if msgs[0].Role != RoleSystem { + t.Fatalf("msgs[0].Role = %q, want system", msgs[0].Role) + } + if !strings.Contains(msgs[0].Content, "user asked about the dashboard") { + t.Errorf("summary was dropped instead of folded into the system prompt: %q", msgs[0].Content) + } + if !strings.Contains(msgs[0].Content, "persona") { + t.Errorf("folding lost the persona prompt: %q", msgs[0].Content) + } + // The turn list must be the conversation only, and must still alternate. + if len(msgs) != 2 || msgs[1].Role != RoleUser || msgs[1].Content != "y el stack?" { + t.Fatalf("turns = %+v, want [system, user]", msgs) + } +} + +// foldSystemNotes must not write through to the caller's backing array — +// handlers reuse the history slice to persist the conversation. +func TestFoldSystemNotesDoesNotMutateCaller(t *testing.T) { + history := []Message{ + {Role: RoleUser, Content: "one"}, + {Role: RoleSystem, Content: "note"}, + {Role: RoleAssistant, Content: "two"}, + } + before := append([]Message(nil), history...) + + notes, rest := foldSystemNotes(history) + if len(notes) != 1 || notes[0] != "note" { + t.Fatalf("notes = %v, want [note]", notes) + } + if len(rest) != 2 { + t.Fatalf("rest has %d messages, want 2", len(rest)) + } + for i := range before { + if history[i].Role != before[i].Role || history[i].Content != before[i].Content { + t.Errorf("caller's history[%d] mutated: %+v -> %+v", i, before[i], history[i]) + } + } +} + +// A 1B model won't infer the reply language from an all-English system +// prompt, so the runner pins it explicitly from the last user message. +func TestLanguageDirectiveFollowsTheUserAndThePersonaOverride(t *testing.T) { + auto := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "the user's language"}, "persona", nil, 5) + + es := auto.languageDirective(detectLanguage([]Message{{Role: RoleUser, Content: "¿Qué proyectos tiene Victor?"}})) + if !strings.Contains(es, "español") { + t.Errorf("Spanish question got directive %q, want a Spanish one", es) + } + en := auto.languageDirective(detectLanguage([]Message{{Role: RoleUser, Content: "What projects does he have?"}})) + if !strings.Contains(en, "English") { + t.Errorf("English question got directive %q, want an English one", en) + } + // Detection must follow the *latest* user turn, not the first. + switched := auto.languageDirective(detectLanguage([]Message{ + {Role: RoleUser, Content: "What projects does he have?"}, + {Role: RoleAssistant, Content: "..."}, + {Role: RoleUser, Content: "¿Y cuál usa Stripe?"}, + })) + if !strings.Contains(switched, "español") { + t.Errorf("after switching to Spanish got %q, want a Spanish directive", switched) + } + if auto.languageDirective(detectLanguage(nil)) != "" { + t.Errorf("empty history should produce no directive") + } + + pinned := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "Spanish"}, "persona", nil, 5) + if got := pinned.languageDirective(detectLanguage([]Message{{Role: RoleUser, Content: "hello there"}})); got != "Write your entire reply in Spanish." { + t.Errorf("persona override ignored: %q", got) + } +} + +// The directive must be the final line of the system prompt — that position +// is why it survives. +func TestLanguageDirectiveIsLastInSystemPrompt(t *testing.T) { + r := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "the user's language"}, "persona", nil, 5) + msgs, _, err := r.BuildMessages(context.Background(), []Message{ + {Role: RoleSystem, Content: "Earlier conversation summary:\nalgo"}, + {Role: RoleUser, Content: "¿Qué stack usa el dashboard?"}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(strings.TrimSpace(msgs[0].Content), + "El visitante escribió en español. Responde ÍNTEGRAMENTE en español, incluido el saludo.") { + t.Errorf("language directive is not the last line of the system prompt:\n%s", msgs[0].Content) + } +} + +// A Spanish question must build on the Spanish rendition of the prompt, not +// the default one with a directive bolted on. +func TestLocalizedPromptSelectedByDetectedLanguage(t *testing.T) { + r := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "the user's language"}, "ENGLISH PROMPT", nil, 5). + WithLocalizedPrompt("es", "PROMPT EN ESPAÑOL") + + es, _, err := r.BuildMessages(context.Background(), + []Message{{Role: RoleUser, Content: "¿Qué proyectos tiene Victor?"}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(es[0].Content, "PROMPT EN ESPAÑOL") { + t.Errorf("Spanish question did not select the Spanish prompt:\n%s", es[0].Content) + } + + en, _, err := r.BuildMessages(context.Background(), + []Message{{Role: RoleUser, Content: "What projects does he have?"}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(en[0].Content, "ENGLISH PROMPT") { + t.Errorf("English question did not select the default prompt:\n%s", en[0].Content) + } + + // No translation registered → fall back, never blank. + bare := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "the user's language"}, "ENGLISH PROMPT", nil, 5) + msgs, _, err := bare.BuildMessages(context.Background(), + []Message{{Role: RoleUser, Content: "¿Y esto?"}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(msgs[0].Content, "ENGLISH PROMPT") { + t.Errorf("missing translation should fall back to the default prompt:\n%s", msgs[0].Content) + } +} diff --git a/internal/agent/runner_test.go b/internal/agent/runner_test.go index b6e1a6f..5659319 100644 --- a/internal/agent/runner_test.go +++ b/internal/agent/runner_test.go @@ -69,7 +69,7 @@ func TestRunnerStreamWithRAG(t *testing.T) { t.Fatal(err) } defer store.Close() - if _, _, err := store.Reindex(context.Background(), srcDir, portfolio.DefaultChunkerConfig()); err != nil { + if _, _, err := store.Reindex(context.Background(), portfolio.SourcesFor(srcDir, ""), portfolio.DefaultChunkerConfig()); err != nil { t.Fatal(err) } @@ -100,7 +100,7 @@ func TestLimitRAGContextToWindow(t *testing.T) { first := "### [1] first — section\n" + strings.Repeat("x", 100) second := "### [2] second — section\n" + strings.Repeat("y", 1600) - got := r.limitRAGContext(first+"\n\n"+second, history) + got := r.limitRAGContext("sys", "", first+"\n\n"+second, history) if !strings.Contains(got, "first") { t.Errorf("limited RAG context dropped the first result: %q", got) } diff --git a/internal/config/config.go b/internal/config/config.go index 7eeb03d..3c3d322 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,36 +21,75 @@ type RateLimit struct { Burst int `yaml:"burst"` } +// Provider is one upstream LLM. The sampling fields (Temperature through +// PresencePenalty) are forwarded verbatim to the llama.cpp adapter; a zero +// value means "don't send it", so llama-server's own default applies. They +// matter more than they look: small instruct models ship with vendor- +// recommended sampling (Gemma 3 wants temp 1.0 / top_k 64 / top_p 0.95) and +// drifting off it makes them terse and repetitive. type Provider struct { Name string `yaml:"name"` Type string `yaml:"type"` Model string `yaml:"model,omitempty"` - ModelPath string `yaml:"model_path,omitempty"` Endpoint string `yaml:"endpoint,omitempty"` ContextSize int `yaml:"context_size,omitempty"` MaxTokens int `yaml:"max_tokens,omitempty"` - NGPULayers int `yaml:"n_gpu_layers,omitempty"` Temperature float32 `yaml:"temperature,omitempty"` - APIKeyEnv string `yaml:"api_key_env,omitempty"` - Default bool `yaml:"default,omitempty"` + TopK int `yaml:"top_k,omitempty"` + TopP float32 `yaml:"top_p,omitempty"` + MinP float32 `yaml:"min_p,omitempty"` + // RepeatPenalty maps to llama.cpp's repeat_penalty. Leave unset (0) for + // Gemma 3 — Google's recommended config is no repetition penalty at all, + // and a value >1 visibly degrades its prose. + RepeatPenalty float32 `yaml:"repeat_penalty,omitempty"` + PresencePenalty float32 `yaml:"presence_penalty,omitempty"` + APIKeyEnv string `yaml:"api_key_env,omitempty"` + Default bool `yaml:"default,omitempty"` } type RAG struct { - Enabled bool `yaml:"enabled"` - DataPath string `yaml:"data_path"` - ChunkSize int `yaml:"chunk_size"` - ChunkOverlap int `yaml:"chunk_overlap"` - DBPath string `yaml:"db_path"` - TopK int `yaml:"top_k"` - Tokenize string `yaml:"tokenize"` + Enabled bool `yaml:"enabled"` + DataPath string `yaml:"data_path"` + // DocsPath is an optional second directory of markdown that is indexed + // and searchable but is NOT part of the project catalogue: a CV, an about + // page, a FAQ. Without it the only way to make the CV retrievable is to + // drop it in data_path, where it then gets announced as one of Victor's + // projects. Both directories accept .md and .mdx. + DocsPath string `yaml:"docs_path"` + DBPath string `yaml:"db_path"` + TopK int `yaml:"top_k"` + Tokenize string `yaml:"tokenize"` + // IncludeCatalog injects the full list of indexed projects into the + // system prompt on every turn. Costs a few tokens per project and stops + // small models from inventing project names when asked to enumerate — + // top-K retrieval can't answer "list everything" by construction. + IncludeCatalog bool `yaml:"include_catalog"` +} + +// Embeddings configures the semantic half of retrieval. Disabled by default, +// in which case the bot uses keyword search only. +// +// Run the endpoint with: +// +// llama-server -m nomic-embed-v2-moe.Q5_K_M.gguf --port 9200 \ +// --embedding --pooling mean --ctx-size 2048 --parallel 1 \ +// --device none --threads 2 +// +// Measured at 0.61 GB RSS on CPU. It has to stay resident: the corpus is +// embedded once at index time, but every visitor question must be embedded +// before it can be compared. +type Embeddings struct { + Enabled bool `yaml:"enabled"` + Endpoint string `yaml:"endpoint"` + Model string `yaml:"model,omitempty"` + BatchSize int `yaml:"batch_size,omitempty"` + TimeoutMS int `yaml:"timeout_ms,omitempty"` } type Persona struct { - Name string `yaml:"name"` - Tone string `yaml:"tone"` - Language string `yaml:"language"` - Constraints []string `yaml:"constraints"` - Intro string `yaml:"intro"` + Name string `yaml:"name"` + Tone string `yaml:"tone"` + Language string `yaml:"language"` } type Logging struct { @@ -90,8 +129,21 @@ type Config struct { Server Server `yaml:"server"` Providers []Provider `yaml:"providers"` RAG RAG `yaml:"rag"` + Embeddings Embeddings `yaml:"embeddings"` Persona Persona `yaml:"persona"` SystemPrompt string `yaml:"system_prompt"` + // SystemPromptES is the Spanish rendition of SystemPrompt, used when the + // visitor writes in Spanish. Optional; empty falls back to SystemPrompt. + // + // This is not a convenience — it's the only thing that reliably keeps a + // small model answering in Spanish. An instruction like "reply in the + // user's language" buried in an otherwise English prompt loses to the + // sheer mass of English around it: measured on gemma-3-1b, 1 of 5 Spanish + // questions came back in Spanish. Translating the prompt itself took that + // to 4 of 5. (Few-shot Spanish examples also score well, but a 1B model + // copies them verbatim instead of answering — see the note on + // `system_prompt`.) + SystemPromptES string `yaml:"system_prompt_es"` Compaction Compaction `yaml:"compaction"` Logging Logging `yaml:"logging"` } @@ -136,12 +188,6 @@ func (c *Config) validate() error { if c.Server.ReadTimeoutMS == 0 { c.Server.ReadTimeoutMS = 30000 } - if c.RAG.ChunkSize == 0 { - c.RAG.ChunkSize = 500 - } - if c.RAG.ChunkOverlap == 0 { - c.RAG.ChunkOverlap = 50 - } if c.RAG.TopK == 0 { c.RAG.TopK = 5 } diff --git a/internal/embed/embed.go b/internal/embed/embed.go new file mode 100644 index 0000000..3bf9e47 --- /dev/null +++ b/internal/embed/embed.go @@ -0,0 +1,186 @@ +// Package embed talks to an OpenAI-compatible /v1/embeddings endpoint +// (llama-server --embedding) and provides the vector maths the RAG store +// needs. +// +// Why this exists: the SQLite FTS5 index matches words exactly. Visitors ask +// in Spanish about a portfolio written in English, so the terms that carry the +// meaning — "paga", "trabajado" — appear zero times in the corpus, which says +// "Payments: Stripe" and "worked". Measured on the real corpus, keyword search +// returned nothing at all for "¿Con qué se paga en la tienda de ropa?" while a +// multilingual embedding put all three tienda-ropa chunks on top. Keyword +// search still wins on exact proper nouns, so the store keeps both and fuses +// them. +package embed + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "time" +) + +// Config describes the embedding endpoint. +type Config struct { + BaseURL string // e.g. http://localhost:9200/v1 + Model string // sent as "model"; llama-server ignores it + BatchSize int // texts per request (0 → 8) + TimeoutMS int // per-request timeout (0 → 120s) +} + +// Client is a minimal embeddings client. +type Client struct { + baseURL string + model string + batchSize int + http *http.Client +} + +func New(cfg Config) *Client { + batch := cfg.BatchSize + if batch <= 0 { + batch = 8 + } + timeout := time.Duration(cfg.TimeoutMS) * time.Millisecond + if timeout <= 0 { + timeout = 120 * time.Second + } + return &Client{ + baseURL: cfg.BaseURL, + model: cfg.Model, + batchSize: batch, + http: &http.Client{Timeout: timeout}, + } +} + +type embedRequest struct { + Model string `json:"model,omitempty"` + Input []string `json:"input"` +} + +type embedResponse struct { + Data []struct { + Index int `json:"index"` + Embedding []float32 `json:"embedding"` + } `json:"data"` +} + +// Embed returns one unit-length vector per input, in input order. +// +// Vectors are normalised here, once, so similarity at query time is a plain +// dot product instead of a cosine with two square roots per candidate. +func (c *Client) Embed(ctx context.Context, texts []string) ([][]float32, error) { + out := make([][]float32, 0, len(texts)) + for start := 0; start < len(texts); start += c.batchSize { + end := min(start+c.batchSize, len(texts)) + vecs, err := c.embedBatch(ctx, texts[start:end]) + if err != nil { + return nil, err + } + out = append(out, vecs...) + } + return out, nil +} + +func (c *Client) embedBatch(ctx context.Context, texts []string) ([][]float32, error) { + body, err := json.Marshal(embedRequest{Model: c.model, Input: texts}) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/embeddings", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("embeddings request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + msg, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("embeddings API %d: %s", resp.StatusCode, string(msg)) + } + + var parsed embedResponse + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, fmt.Errorf("decode embeddings: %w", err) + } + if len(parsed.Data) != len(texts) { + return nil, fmt.Errorf("embeddings returned %d vectors for %d inputs", len(parsed.Data), len(texts)) + } + + // The API documents an `index` field rather than guaranteeing order. + out := make([][]float32, len(texts)) + for _, d := range parsed.Data { + if d.Index < 0 || d.Index >= len(out) { + return nil, fmt.Errorf("embeddings returned out-of-range index %d", d.Index) + } + out[d.Index] = Normalize(d.Embedding) + } + for i, v := range out { + if v == nil { + return nil, fmt.Errorf("embeddings response missing index %d", i) + } + } + return out, nil +} + +// Normalize scales v to unit length. A zero vector is returned unchanged — +// dividing by zero would poison every later comparison with NaN. +func Normalize(v []float32) []float32 { + var sum float64 + for _, x := range v { + sum += float64(x) * float64(x) + } + if sum == 0 { + return v + } + inv := float32(1 / math.Sqrt(sum)) + out := make([]float32, len(v)) + for i, x := range v { + out[i] = x * inv + } + return out +} + +// Similarity is the dot product, which equals cosine similarity for the +// unit-length vectors this package produces. Mismatched lengths score 0 so a +// stale row from a different embedding model can never outrank a real hit. +func Similarity(a, b []float32) float64 { + if len(a) != len(b) || len(a) == 0 { + return 0 + } + var sum float64 + for i := range a { + sum += float64(a[i]) * float64(b[i]) + } + return sum +} + +// Encode serialises a vector as little-endian float32 for a SQLite BLOB. +func Encode(v []float32) []byte { + buf := make([]byte, 4*len(v)) + for i, x := range v { + binary.LittleEndian.PutUint32(buf[4*i:], math.Float32bits(x)) + } + return buf +} + +// Decode reverses Encode. A blob whose length isn't a multiple of 4 is +// corrupt and yields nil rather than a truncated vector. +func Decode(b []byte) []float32 { + if len(b)%4 != 0 { + return nil + } + out := make([]float32, len(b)/4) + for i := range out { + out[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[4*i:])) + } + return out +} diff --git a/internal/embed/embed_test.go b/internal/embed/embed_test.go new file mode 100644 index 0000000..0cf88a2 --- /dev/null +++ b/internal/embed/embed_test.go @@ -0,0 +1,111 @@ +package embed + +import ( + "context" + "encoding/json" + "math" + "net/http" + "net/http/httptest" + "testing" +) + +func TestNormalizeProducesUnitVectors(t *testing.T) { + got := Normalize([]float32{3, 4}) + if math.Abs(float64(got[0])-0.6) > 1e-6 || math.Abs(float64(got[1])-0.8) > 1e-6 { + t.Errorf("Normalize([3,4]) = %v, want [0.6 0.8]", got) + } + // A zero vector must survive untouched — dividing by its length would + // put NaN into every later comparison. + zero := []float32{0, 0, 0} + if out := Normalize(zero); out[0] != 0 || math.IsNaN(float64(out[0])) { + t.Errorf("Normalize(zero) = %v, want zeros", out) + } +} + +func TestSimilarity(t *testing.T) { + a := Normalize([]float32{1, 0}) + if s := Similarity(a, a); math.Abs(s-1) > 1e-6 { + t.Errorf("self-similarity = %v, want 1", s) + } + if s := Similarity(a, Normalize([]float32{0, 1})); math.Abs(s) > 1e-6 { + t.Errorf("orthogonal similarity = %v, want 0", s) + } + // A vector from a different model must never outrank a real hit. + if s := Similarity(a, []float32{1, 0, 0}); s != 0 { + t.Errorf("mismatched dimensions scored %v, want 0", s) + } + if s := Similarity(nil, nil); s != 0 { + t.Errorf("empty vectors scored %v, want 0", s) + } +} + +func TestEncodeDecodeRoundTrip(t *testing.T) { + in := []float32{0.5, -0.25, 1e-8, 12345.75} + out := Decode(Encode(in)) + if len(out) != len(in) { + t.Fatalf("round trip changed length: %d → %d", len(in), len(out)) + } + for i := range in { + if in[i] != out[i] { + t.Errorf("round trip [%d]: %v → %v", i, in[i], out[i]) + } + } + if got := Decode([]byte{1, 2, 3}); got != nil { + t.Errorf("Decode of a truncated blob = %v, want nil", got) + } +} + +// The API returns an `index` per item rather than promising input order, so +// the client must reorder — otherwise chunk N gets chunk M's vector and every +// later search is quietly wrong. +func TestEmbedReordersByIndexAndNormalizes(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req embedRequest + _ = json.NewDecoder(r.Body).Decode(&req) + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []map[string]any{ + {"index": 1, "embedding": []float32{0, 5}}, + {"index": 0, "embedding": []float32{3, 4}}, + }, + }) + })) + defer srv.Close() + + vecs, err := New(Config{BaseURL: srv.URL}).Embed(context.Background(), []string{"first", "second"}) + if err != nil { + t.Fatal(err) + } + if len(vecs) != 2 { + t.Fatalf("got %d vectors, want 2", len(vecs)) + } + if math.Abs(float64(vecs[0][0])-0.6) > 1e-6 { + t.Errorf("vecs[0] = %v, want the index-0 item normalized ([0.6 0.8])", vecs[0]) + } + if math.Abs(float64(vecs[1][1])-1) > 1e-6 { + t.Errorf("vecs[1] = %v, want the index-1 item normalized ([0 1])", vecs[1]) + } +} + +func TestEmbedRejectsShortResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []map[string]any{{"index": 0, "embedding": []float32{1, 0}}}, + }) + })) + defer srv.Close() + + if _, err := New(Config{BaseURL: srv.URL}).Embed(context.Background(), []string{"a", "b"}); err == nil { + t.Fatal("want an error when the endpoint returns fewer vectors than inputs") + } +} + +func TestEmbedSurfacesHTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "model not loaded", http.StatusServiceUnavailable) + })) + defer srv.Close() + + if _, err := New(Config{BaseURL: srv.URL}).Embed(context.Background(), []string{"a"}); err == nil { + t.Fatal("want an error on a 503") + } +} diff --git a/internal/persona/persona.go b/internal/persona/persona.go index 2feffa2..ef9ffe4 100644 --- a/internal/persona/persona.go +++ b/internal/persona/persona.go @@ -27,13 +27,24 @@ func FromConfig(c *config.Config) (persona.Persona, error) { } // BuildSystemPrompt returns the full prompt for one chat turn: -// 1. The hand-written system prompt from the YAML (who Rony is, how to speak) -// 2. The RAG block (omitted when the index returns no hits) +// 1. The hand-written system prompt from the YAML (who Rony is, how to speak) +// 2. The project catalogue (omitted when empty) +// 3. The RAG block (omitted when the index returns no hits) // -// The RAG block is appended, not prepended, so the persona instructions -// always come first and the LLM never gets the chance to "forget" them. -func BuildSystemPrompt(systemPrompt, ragContext string) string { +// The catalogue and RAG block are appended, not prepended, so the persona +// instructions always come first and the LLM never gets the chance to +// "forget" them. +// +// The catalogue goes before the excerpts on purpose: it is the closed set the +// model is allowed to name, and stating the boundary before showing the +// evidence measurably cuts invented project names on small models. +func BuildSystemPrompt(systemPrompt, catalog, ragContext string) string { out := systemPrompt + if catalog != "" { + out += "\n\n## Victor's complete project catalogue\n\n" + catalog + + "\n\nThat is every project that exists. A name outside this list is not a real " + + "project of Victor's and you must never use one." + } if ragContext != "" { out += "\n\n## Relevant context from the portfolio\n\n" + "Use these excerpts to answer. Cite the project filename when you reference a detail. " + diff --git a/internal/portfolio/chunker.go b/internal/portfolio/chunker.go index 404f84b..3e36c8e 100644 --- a/internal/portfolio/chunker.go +++ b/internal/portfolio/chunker.go @@ -137,6 +137,43 @@ func dropEmpty(sections []section) []section { // subSplit breaks down any section whose body exceeds max into overlapping // slices, preserving the heading as a prefix on each piece so context isn't // lost mid-section. +var h3Re = regexp.MustCompile(`(?m)^### +(.+)$`) + +// splitByH3 breaks an oversized section at its H3 boundaries, if it has any. +// The sub-heading is folded into the section name ("Experience — Metrimex") +// so the piece still says what it is once it's out of context. +// +// This exists because character splitting mangles exactly the content that +// matters most. A CV's Experience section is a list of jobs; splitting it by +// size cut one entry mid-word, producing a chunk that began "... id app for +// an on-demand ride-sharing service". A chunk like that matches no question, +// and the employer name it belonged to was stranded in the previous piece. +// Splitting per job keeps the company, the role and the dates together. +func splitByH3(s section) ([]section, bool) { + locs := h3Re.FindAllStringSubmatchIndex(s.Body, -1) + if len(locs) < 2 { + return nil, false + } + var out []section + // Text before the first H3 (a lead-in paragraph) stays with the parent. + if lead := strings.TrimSpace(s.Body[:locs[0][0]]); lead != "" { + out = append(out, section{Heading: s.Heading, Body: lead}) + } + for i, loc := range locs { + end := len(s.Body) + if i+1 < len(locs) { + end = locs[i+1][0] + } + title := strings.TrimSpace(s.Body[loc[2]:loc[3]]) + body := strings.TrimSpace(s.Body[loc[0]:end]) + if body == "" { + continue + } + out = append(out, section{Heading: s.Heading + " — " + title, Body: body}) + } + return out, len(out) > 0 +} + func subSplit(sections []section, max, overlap int) []section { if max <= 0 { return sections @@ -150,6 +187,11 @@ func subSplit(sections []section, max, overlap int) []section { out = append(out, s) continue } + // Prefer semantic boundaries over byte offsets. + if pieces, ok := splitByH3(s); ok { + out = append(out, pieces...) + continue + } for i := 0; i < len(s.Body); { end := i + max if end > len(s.Body) { diff --git a/internal/portfolio/chunker_test.go b/internal/portfolio/chunker_test.go index 465acd0..16c1fd7 100644 --- a/internal/portfolio/chunker_test.go +++ b/internal/portfolio/chunker_test.go @@ -108,4 +108,48 @@ func equalSlice(a, b []string) bool { } } return true -} \ No newline at end of file +} +// An oversized section made of H3 entries (a CV's Experience list, a FAQ) +// must split per entry, not per byte. Size-splitting used to cut a job entry +// mid-word and strand the employer name in the previous chunk. +func TestSubSplitPrefersH3BoundariesOverByteOffsets(t *testing.T) { + body := "### Metrimex — Frontend Developer\n\n#### Jul 2024 – Jun 2026\n\n" + + strings.Repeat("Built an access-control app for physical sites. ", 12) + + "\n\n### Didcom — Android Developer\n\n#### Jul 2023 – Jul 2024\n\n" + + strings.Repeat("Built an Android app for an on-demand ride-sharing service. ", 12) + + "\n\n### Teknol — Software Developer\n\n#### Jun 2016 – Jan 2017\n\n" + + strings.Repeat("Maintained an internal tool. ", 12) + + got := subSplit([]section{{Heading: "Experience", Body: body}}, 400, 40) + if len(got) != 3 { + t.Fatalf("got %d chunks, want one per job:\n%+v", len(got), got) + } + for i, want := range []string{"Metrimex", "Didcom", "Teknol"} { + if !strings.Contains(got[i].Heading, want) { + t.Errorf("chunk %d heading = %q, want it to name %s", i, got[i].Heading, want) + } + if !strings.HasPrefix(got[i].Body, "### "+want) { + t.Errorf("chunk %d body should start at its own heading, got %.40q", i, got[i].Body) + } + } + // No chunk may begin mid-sentence — that is the defect this replaces. + for i, s := range got { + if strings.HasPrefix(s.Body, "... ") { + t.Errorf("chunk %d still starts with a byte-split continuation: %.40q", i, s.Body) + } + } +} + +// A long section with no H3 structure still falls back to size splitting. +func TestSubSplitFallsBackToSizeWithoutH3(t *testing.T) { + body := strings.Repeat("plain prose with no subheadings at all. ", 40) + got := subSplit([]section{{Heading: "Description", Body: body}}, 300, 30) + if len(got) < 2 { + t.Fatalf("expected the oversized section to be split, got %d", len(got)) + } + for _, s := range got { + if s.Heading != "Description" { + t.Errorf("size-split chunk changed heading to %q", s.Heading) + } + } +} diff --git a/internal/portfolio/hybrid.go b/internal/portfolio/hybrid.go new file mode 100644 index 0000000..8640c86 --- /dev/null +++ b/internal/portfolio/hybrid.go @@ -0,0 +1,238 @@ +package portfolio + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "log/slog" + "sort" + + "github.com/VictorVargas/rony-chat-bot/internal/embed" +) + +// Embedder is the subset of embed.Client the store needs. Keeping it an +// interface lets Reindex and HybridSearch be tested without an endpoint. +type Embedder interface { + Embed(ctx context.Context, texts []string) ([][]float32, error) +} + +// rrfK is the usual Reciprocal Rank Fusion constant. RRF combines rankings +// rather than scores, which matters here because BM25 (unbounded, negative) +// and cosine similarity (0–1) are not comparable on any common scale. k=60 +// is the value from the original paper and is not sensitive enough to be +// worth tuning for a corpus this size. +const rrfK = 60.0 + +// embedText is the exact string that gets embedded for a chunk, and the only +// definition of it. EmbedChunks and VectorSearch both go through this so the +// hash they compare can never be computed over different text. +// +// The project and heading are prepended because a bare body is ambiguous out +// of context: "Tech stack: Next.js, Stripe" says nothing about which project, +// and the question naming the shop needs to land on it. +func embedText(projectID, section, content string) string { + return projectID + " — " + section + "\n" + content +} + +func hashOf(text string) string { + sum := sha256.Sum256([]byte(text)) + return hex.EncodeToString(sum[:]) +} + +// VectorSearch returns the topK chunks whose embedding is closest to the +// query, scanning every stored vector. Chunks indexed before embeddings were +// enabled simply have no row and are skipped. +func (s *Store) VectorSearch(ctx context.Context, queryVec []float32, topK int) ([]SearchResult, error) { + if len(queryVec) == 0 { + return nil, nil + } + if topK <= 0 { + topK = 5 + } + + rows, err := s.db.QueryContext(ctx, ` + SELECT c.id, c.project_id, c.kind, c.source_file, c.section, c.chunk_index, c.content, + v.vec, v.content_hash + FROM portfolio_vectors v + JOIN portfolio_chunks c ON c.id = v.chunk_id + WHERE v.dim = ? AND c.section <> 'frontmatter'`, len(queryVec)) + if err != nil { + return nil, fmt.Errorf("vector search: %w", err) + } + defer rows.Close() + + var hits []SearchResult + var stale int + for rows.Next() { + var r SearchResult + var blob []byte + var storedHash string + if err := rows.Scan(&r.ID, &r.ProjectID, &r.Kind, &r.SourceFile, &r.Section, &r.Index, + &r.Content, &blob, &storedHash); err != nil { + return nil, err + } + // Skip vectors describing a previous version of this chunk. Chunk + // ids survive body edits, so without this the row would be scored + // against text that is no longer there — quietly, with no error. + // The chunk stays reachable through keyword search meanwhile. + if storedHash != hashOf(embedText(r.ProjectID, r.Section, r.Content)) { + stale++ + continue + } + r.Score = embed.Similarity(queryVec, embed.Decode(blob)) + hits = append(hits, r) + } + if stale > 0 { + slog.Warn("ignoring stale embeddings; re-run reindex to refresh them", "chunks", stale) + } + if err := rows.Err(); err != nil { + return nil, err + } + + // Higher similarity is better, unlike the bm25() score used by Search. + sort.Slice(hits, func(i, j int) bool { return hits[i].Score > hits[j].Score }) + return hits[:min(topK, len(hits))], nil +} + +// HybridSearch fuses keyword and vector retrieval with Reciprocal Rank +// Fusion and returns the topK chunks. +// +// Neither retriever is sufficient alone on this corpus. Keyword search finds +// exact proper nouns ("Kubernetes", "Stripe") that embeddings can blur, but +// returns nothing when a Spanish question meets an English document. Vectors +// bridge the languages but rank a request for a specific rare token less +// sharply. RRF asks only for each side's ordering, so a chunk that both +// retrievers like rises above one that only a single retriever loved. +// +// When embedding fails — the endpoint is down, or none is configured — this +// degrades to plain keyword search rather than failing the request: a +// keyword-only answer beats no answer. +func (s *Store) HybridSearch(ctx context.Context, embedder Embedder, query string, topK int) ([]SearchResult, error) { + if topK <= 0 { + topK = 5 + } + // Over-fetch from each side: a chunk ranked 8th by one retriever and 2nd + // by the other should still be able to win the fusion. + pool := topK * 3 + + keyword, kwErr := s.Search(ctx, query, pool) + if embedder == nil { + return keyword, kwErr + } + + vecs, err := embedder.Embed(ctx, []string{query}) + if err != nil || len(vecs) == 0 { + if kwErr != nil { + return nil, fmt.Errorf("both retrievers failed: keyword: %v; embedding: %w", kwErr, err) + } + return keyword, nil + } + semantic, vecErr := s.VectorSearch(ctx, vecs[0], pool) + if vecErr != nil && kwErr != nil { + return nil, fmt.Errorf("both retrievers failed: keyword: %v; vector: %w", kwErr, vecErr) + } + + type fused struct { + res SearchResult + score float64 + } + byID := map[string]*fused{} + add := func(list []SearchResult) { + for rank, r := range list { + f, ok := byID[r.ID] + if !ok { + f = &fused{res: r} + byID[r.ID] = f + } + f.score += 1 / (rrfK + float64(rank+1)) + } + } + add(keyword) + add(semantic) + + out := make([]fused, 0, len(byID)) + for _, f := range byID { + out = append(out, *f) + } + sort.Slice(out, func(i, j int) bool { + if out[i].score != out[j].score { + return out[i].score > out[j].score + } + return out[i].res.ID < out[j].res.ID // stable across runs + }) + + results := make([]SearchResult, 0, min(topK, len(out))) + for _, f := range out[:min(topK, len(out))] { + f.res.Score = f.score + results = append(results, f.res) + } + return results, nil +} + +// EmbedChunks computes and stores a vector for every indexed chunk. Called +// after Reindex, since it needs the chunks to exist. +// +// The heading is prepended to each chunk's text: a bare "Tech stack" body is +// a list of technologies with no hint of which project it belongs to, and the +// embedding of "tienda-ropa — Tech stack: Next.js, Stripe…" is much closer to +// a question naming the shop. +func (s *Store) EmbedChunks(ctx context.Context, embedder Embedder) (int, error) { + if embedder == nil { + return 0, nil + } + rows, err := s.db.QueryContext(ctx, + `SELECT id, project_id, section, content FROM portfolio_chunks ORDER BY id`) + if err != nil { + return 0, fmt.Errorf("read chunks for embedding: %w", err) + } + var ids, texts []string + for rows.Next() { + var id, project, section, content string + if err := rows.Scan(&id, &project, §ion, &content); err != nil { + rows.Close() + return 0, err + } + ids = append(ids, id) + texts = append(texts, embedText(project, section, content)) + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + if len(ids) == 0 { + return 0, nil + } + + vecs, err := embedder.Embed(ctx, texts) + if err != nil { + return 0, fmt.Errorf("embed chunks: %w", err) + } + if len(vecs) != len(ids) { + return 0, fmt.Errorf("embedder returned %d vectors for %d chunks", len(vecs), len(ids)) + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `DELETE FROM portfolio_vectors`); err != nil { + return 0, err + } + stmt, err := tx.PrepareContext(ctx, + `INSERT INTO portfolio_vectors (chunk_id, content_hash, dim, vec) VALUES (?,?,?,?)`) + if err != nil { + return 0, err + } + defer stmt.Close() + for i, id := range ids { + if _, err := stmt.ExecContext(ctx, id, hashOf(texts[i]), len(vecs[i]), embed.Encode(vecs[i])); err != nil { + return 0, fmt.Errorf("store vector %s: %w", id, err) + } + } + if err := tx.Commit(); err != nil { + return 0, err + } + return len(ids), nil +} diff --git a/internal/portfolio/hybrid_test.go b/internal/portfolio/hybrid_test.go new file mode 100644 index 0000000..4de3b2f --- /dev/null +++ b/internal/portfolio/hybrid_test.go @@ -0,0 +1,243 @@ +package portfolio + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/VictorVargas/rony-chat-bot/internal/embed" +) + +// stubEmbedder returns a fixed vector per exact text, so a test can decide +// which chunk a query should land on without running a model. +type stubEmbedder struct { + vecs map[string][]float32 + err error +} + +func (s *stubEmbedder) Embed(_ context.Context, texts []string) ([][]float32, error) { + if s.err != nil { + return nil, s.err + } + out := make([][]float32, len(texts)) + for i, t := range texts { + if v, ok := s.vecs[t]; ok { + out[i] = embed.Normalize(v) + continue + } + out[i] = embed.Normalize([]float32{1, 1, 1}) // neutral + } + return out, nil +} + +func newTestStore(t *testing.T) *Store { + t.Helper() + dir := t.TempDir() + src := filepath.Join(dir, "projects") + if err := os.MkdirAll(src, 0o755); err != nil { + t.Fatal(err) + } + docs := map[string]string{ + "tienda.md": "# Tienda\n\n## Tech stack\n\nNext.js and Stripe for payments.\n", + "dash.md": "# Dashboard\n\n## Tech stack\n\nGo backend with WebSockets and D3.\n", + } + for name, body := range docs { + if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + store, err := OpenStore(filepath.Join(dir, "t.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + if _, _, err := store.Reindex(context.Background(), SourcesFor(src, ""), DefaultChunkerConfig()); err != nil { + t.Fatal(err) + } + return store +} + +// The point of the whole feature: a query whose words appear nowhere in the +// corpus still finds the right chunk, because the vector matches. +func TestHybridSearchFindsChunkKeywordSearchCannot(t *testing.T) { + store := newTestStore(t) + + // "¿Cómo se paga?" shares no word with "Next.js and Stripe for payments". + query := "¿Cómo se paga?" + target := []float32{1, 0, 0} + emb := &stubEmbedder{vecs: map[string][]float32{query: target}} + + // Embed the corpus so the Stripe chunk owns the target direction. + corpus := &stubEmbedder{vecs: map[string][]float32{}} + rows, err := store.db.Query(`SELECT id, project_id, section, content FROM portfolio_chunks`) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var id, project, section, content string + if err := rows.Scan(&id, &project, §ion, &content); err != nil { + t.Fatal(err) + } + v := []float32{0, 1, 0} + if project == "tienda" && section == "Tech stack" { + v = target + } + corpus.vecs[project+" — "+section+"\n"+content] = v + } + rows.Close() + + if _, err := store.EmbedChunks(context.Background(), corpus); err != nil { + t.Fatal(err) + } + + // Keyword search alone finds nothing useful for this query. + kw, err := store.Search(context.Background(), query, 5) + if err != nil { + t.Fatal(err) + } + for _, h := range kw { + if h.ProjectID == "tienda" && h.Section == "Tech stack" { + t.Skip("keyword search already found it; this corpus can't demonstrate the gap") + } + } + + hits, err := store.HybridSearch(context.Background(), emb, query, 3) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Fatal("hybrid search returned nothing") + } + if hits[0].ProjectID != "tienda" || hits[0].Section != "Tech stack" { + t.Errorf("top hit = %s/%s, want tienda/Tech stack", hits[0].ProjectID, hits[0].Section) + } +} + +// A dead embeddings endpoint must degrade to keyword search, not fail the +// visitor's question. +func TestHybridSearchFallsBackWhenEmbedderFails(t *testing.T) { + store := newTestStore(t) + broken := &stubEmbedder{err: errors.New("connection refused")} + + hits, err := store.HybridSearch(context.Background(), broken, "Stripe", 3) + if err != nil { + t.Fatalf("hybrid search should fall back, got error: %v", err) + } + if len(hits) == 0 { + t.Fatal("fallback returned no keyword hits for a term that is in the corpus") + } +} + +// With no embedder configured at all, hybrid search is plain keyword search. +func TestHybridSearchWithoutEmbedderIsKeywordSearch(t *testing.T) { + store := newTestStore(t) + hits, err := store.HybridSearch(context.Background(), nil, "Stripe", 3) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Fatal("expected keyword hits for Stripe") + } +} + +// Vectors from a different embedding model (wrong dimension) must be ignored +// rather than compared against and ranked. +func TestVectorSearchIgnoresMismatchedDimensions(t *testing.T) { + store := newTestStore(t) + corpus := &stubEmbedder{} // neutral 3-dim vectors + if _, err := store.EmbedChunks(context.Background(), corpus); err != nil { + t.Fatal(err) + } + hits, err := store.VectorSearch(context.Background(), []float32{1, 0, 0, 0}, 5) + if err != nil { + t.Fatal(err) + } + if len(hits) != 0 { + t.Errorf("got %d hits for a 4-dim query against 3-dim rows, want 0", len(hits)) + } +} + +// A chunk's id is derived from file + heading + position, so editing the body +// of a section keeps the id. A vector keyed only by id would then rank that +// chunk by text that no longer exists — silently, with no error anywhere. +// The stored content hash makes such a row invisible instead of wrong. +func TestVectorSearchIgnoresVectorsWhoseChunkChanged(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "projects") + if err := os.MkdirAll(src, 0o755); err != nil { + t.Fatal(err) + } + file := filepath.Join(src, "demo.md") + write := func(body string) { + if err := os.WriteFile(file, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("# Demo\n\n## Tech stack\n\nThe payment provider is Stripe.\n") + + store, err := OpenStore(filepath.Join(dir, "t.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + sources := SourcesFor(src, "") + if _, _, err := store.Reindex(context.Background(), sources, DefaultChunkerConfig()); err != nil { + t.Fatal(err) + } + if _, err := store.EmbedChunks(context.Background(), &stubEmbedder{}); err != nil { + t.Fatal(err) + } + hits, err := store.VectorSearch(context.Background(), embed.Normalize([]float32{1, 1, 1}), 5) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Fatal("expected the freshly embedded chunk to be searchable") + } + + // Rewrite the body, keeping every heading — the chunk id is unchanged. + write("# Demo\n\n## Tech stack\n\nThe payment provider is PayPal. Stripe was removed.\n") + if _, _, err := store.Reindex(context.Background(), sources, DefaultChunkerConfig()); err != nil { + t.Fatal(err) + } + // Deliberately do NOT re-embed: this is the "endpoint was down" case. + + var ids []string + rows, err := store.db.Query(`SELECT chunk_id FROM portfolio_vectors`) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + ids = append(ids, id) + } + rows.Close() + if len(ids) == 0 { + t.Fatal("precondition: the stale vector row should still be present") + } + + hits, err = store.VectorSearch(context.Background(), embed.Normalize([]float32{1, 1, 1}), 5) + if err != nil { + t.Fatal(err) + } + if len(hits) != 0 { + t.Errorf("stale vector was used for ranking: %+v", hits) + } + + // Re-embedding restores it. + if _, err := store.EmbedChunks(context.Background(), &stubEmbedder{}); err != nil { + t.Fatal(err) + } + hits, err = store.VectorSearch(context.Background(), embed.Normalize([]float32{1, 1, 1}), 5) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Error("re-embedding should make the chunk searchable again") + } +} diff --git a/internal/portfolio/indexer.go b/internal/portfolio/indexer.go index 60dcc39..bb13f10 100644 --- a/internal/portfolio/indexer.go +++ b/internal/portfolio/indexer.go @@ -23,6 +23,7 @@ import ( type SearchResult struct { ID string ProjectID string + Kind string // KindProject | KindDoc SourceFile string Section string Index int @@ -30,10 +31,36 @@ type SearchResult struct { Score float64 } +// Chunk kinds. A "project" is a piece of Victor's portfolio and shows up in +// the catalogue the bot injects into the prompt; a "doc" (his CV, an about +// page, a FAQ) is retrievable evidence that is not itself a project and must +// never be listed as one. +const ( + KindProject = "project" + KindDoc = "doc" +) + +// Source is one directory of markdown to index, and what the documents in it +// mean. See KindProject / KindDoc. +type Source struct { + Path string + Kind string +} + +// SourcesFor is the standard mapping from the two configured directories to +// index sources. docsPath may be empty. +func SourcesFor(dataPath, docsPath string) []Source { + return []Source{ + {Path: dataPath, Kind: KindProject}, + {Path: docsPath, Kind: KindDoc}, + } +} + const schema = ` CREATE VIRTUAL TABLE IF NOT EXISTS portfolio_chunks USING fts5( id UNINDEXED, project_id UNINDEXED, + kind UNINDEXED, source_file UNINDEXED, section UNINDEXED, chunk_index UNINDEXED, @@ -42,10 +69,72 @@ CREATE VIRTUAL TABLE IF NOT EXISTS portfolio_chunks USING fts5( ); ` +// Vectors live in an ordinary table keyed by chunk id. There is no ANN index: +// a portfolio is hundreds of chunks, not millions, so a full scan with a dot +// product is microseconds and needs no extension. +// content_hash is what makes a vector verifiable. A chunk's id is derived +// from file + heading + position, so editing the *body* of a section leaves +// the id untouched — and a vector keyed only by id would keep describing the +// text that used to be there. Nothing errors; semantic ranking just silently +// scores the chunk by a meaning it no longer has. Storing the hash of the +// embedded text lets the search ignore rows whose source has moved on. +const vectorSchema = ` +CREATE TABLE IF NOT EXISTS portfolio_vectors ( + chunk_id TEXT PRIMARY KEY, + content_hash TEXT NOT NULL, + dim INTEGER NOT NULL, + vec BLOB NOT NULL +); +` + +// ensureChunkSchema creates portfolio_chunks, and rebuilds it when an older +// database is missing a column (FTS5 has no ALTER TABLE ADD COLUMN). +// +// Dropping is safe precisely because this table is a derived index: every row +// is regenerated from the markdown on the next Reindex. It deliberately +// touches only portfolio_chunks — conversations live in the same file and are +// real user data. +func ensureChunkSchema(ctx context.Context, db *sql.DB) error { + var existing string + err := db.QueryRowContext(ctx, + `SELECT sql FROM sqlite_master WHERE type='table' AND name='portfolio_chunks'`).Scan(&existing) + switch { + case err == sql.ErrNoRows: + // Fresh database; fall through to CREATE. + case err != nil: + return fmt.Errorf("inspect chunk schema: %w", err) + case !strings.Contains(existing, "kind"): + slog.Info("portfolio index predates the kind column, rebuilding it (run reindex to repopulate)") + if _, err := db.ExecContext(ctx, `DROP TABLE portfolio_chunks`); err != nil { + return fmt.Errorf("drop stale chunk table: %w", err) + } + } + if _, err := db.ExecContext(ctx, schema); err != nil { + return fmt.Errorf("create schema: %w", err) + } + // Same rebuild-on-mismatch rule as the chunk table: vectors are derived + // data, regenerated by the next reindex. + var vecSQL string + switch err := db.QueryRowContext(ctx, + `SELECT sql FROM sqlite_master WHERE type='table' AND name='portfolio_vectors'`).Scan(&vecSQL); { + case err == sql.ErrNoRows: + case err != nil: + return fmt.Errorf("inspect vector schema: %w", err) + case !strings.Contains(vecSQL, "content_hash"): + slog.Info("vector index predates content_hash, rebuilding it (run reindex to repopulate)") + if _, err := db.ExecContext(ctx, `DROP TABLE portfolio_vectors`); err != nil { + return fmt.Errorf("drop stale vector table: %w", err) + } + } + if _, err := db.ExecContext(ctx, vectorSchema); err != nil { + return fmt.Errorf("create vector schema: %w", err) + } + return nil +} + // Store wraps a SQLite FTS5 database with the portfolio schema. type Store struct { - db *sql.DB - chunkSize int // legacy field kept for compat; not used by heading chunker + db *sql.DB } func OpenStore(dbPath string) (*Store, error) { @@ -59,15 +148,15 @@ func OpenStore(dbPath string) (*Store, error) { return nil, fmt.Errorf("open sqlite: %w", err) } db.SetMaxOpenConns(1) // SQLite + concurrent writers doesn't help - if _, err := db.ExecContext(context.Background(), schema); err != nil { + if err := ensureChunkSchema(context.Background(), db); err != nil { _ = db.Close() - return nil, fmt.Errorf("create schema: %w", err) + return nil, err } if _, err := db.ExecContext(context.Background(), conversationSchema); err != nil { _ = db.Close() return nil, fmt.Errorf("create conversation schema: %w", err) } - return &Store{db: db, chunkSize: 500}, nil + return &Store{db: db}, nil } func (s *Store) Close() error { return s.db.Close() } @@ -76,10 +165,52 @@ func (s *Store) Close() error { return s.db.Close() } // queries (e.g. the health check). Don't use for hot-path code: go through // the Search / Reindex methods. func (s *Store) DB() *sql.DB { return s.db } -func (s *Store) Reindex(ctx context.Context, dataPath string, cfg ChunkerConfig) (files, chunks int, err error) { - matches, err := filepath.Glob(filepath.Join(dataPath, "*.md")) - if err != nil { - return 0, 0, fmt.Errorf("glob: %w", err) + +// isReadme reports whether a file is a directory's README rather than content. +// These directories are checked into the repo with instructions for whoever +// fills them, and those instructions are not one of Victor's projects: without +// this, `data/projects/README.md` was indexed as a project and the catalogue +// injected into every prompt announced "README" and "README.es" to visitors. +// Localised variants (README.es.md) are covered by matching the first segment. +func isReadme(path string) bool { + base := filepath.Base(path) + name, _, _ := strings.Cut(base, ".") + return strings.EqualFold(name, "README") +} + +// Reindex rebuilds the whole index from the given sources. Each source +// contributes its `.md` and `.mdx` files; `.mdx` is included because content +// written for an Astro or Next site (a CV, an about page) is usually authored +// there, and its JSX is harmless to full-text search. +// +// A source with an empty Path is skipped, so callers can pass an optional +// docs directory without branching. +func (s *Store) Reindex(ctx context.Context, sources []Source, cfg ChunkerConfig) (files, chunks int, err error) { + type doc struct { + path string + kind string + } + var docs []doc + for _, src := range sources { + if strings.TrimSpace(src.Path) == "" { + continue + } + kind := src.Kind + if kind == "" { + kind = KindProject + } + for _, ext := range []string{"*.md", "*.mdx"} { + matches, err := filepath.Glob(filepath.Join(src.Path, ext)) + if err != nil { + return 0, 0, fmt.Errorf("glob %s: %w", ext, err) + } + for _, m := range matches { + if isReadme(m) { + continue + } + docs = append(docs, doc{path: m, kind: kind}) + } + } } tx, err := s.db.BeginTx(ctx, nil) @@ -93,24 +224,24 @@ func (s *Store) Reindex(ctx context.Context, dataPath string, cfg ChunkerConfig) } stmt, err := tx.PrepareContext(ctx, - `INSERT INTO portfolio_chunks (id, project_id, source_file, section, chunk_index, content) VALUES (?,?,?,?,?,?)`) + `INSERT INTO portfolio_chunks (id, project_id, kind, source_file, section, chunk_index, content) VALUES (?,?,?,?,?,?,?)`) if err != nil { return 0, 0, err } defer stmt.Close() - for _, file := range matches { - body, err := os.ReadFile(file) + for _, d := range docs { + body, err := os.ReadFile(d.path) if err != nil { - slog.Warn("read file failed", "file", file, "err", err) + slog.Warn("read file failed", "file", d.path, "err", err) continue } - projectID := strings.TrimSuffix(filepath.Base(file), ".md") + docID := strings.TrimSuffix(strings.TrimSuffix(filepath.Base(d.path), ".mdx"), ".md") sections := SplitMarkdownSections(string(body), cfg) for idx, sec := range sections { - id := fmt.Sprintf("%s-%s-%d", projectID, slugify(sec.Heading), idx) - if _, err := stmt.ExecContext(ctx, id, projectID, file, sec.Heading, idx, sec.Body); err != nil { - return len(matches), chunks, fmt.Errorf("insert %s: %w", id, err) + id := fmt.Sprintf("%s-%s-%d", docID, slugify(sec.Heading), idx) + if _, err := stmt.ExecContext(ctx, id, docID, d.kind, d.path, sec.Heading, idx, sec.Body); err != nil { + return len(docs), chunks, fmt.Errorf("insert %s: %w", id, err) } chunks++ } @@ -139,15 +270,65 @@ func slugify(s string) string { return string(out) } +// CatalogEntry is one project in the portfolio, identified by its filename +// stem and its human-readable H1 title. +type CatalogEntry struct { + ProjectID string + Title string +} + +// Catalog lists every indexed project with its title, cheaply and +// deterministically (no BM25, no query). +// +// It exists because retrieval alone can't answer "what projects does Victor +// have?": top-K search returns the K best-matching *chunks*, which for a +// broad question is a handful of sections from two or three documents, and a +// small model asked to enumerate from that will confidently fill the gaps +// with invented project names. Injecting the full catalogue into the system +// prompt turns enumeration into a copy job instead of a recall job. The +// portfolio is a few dozen documents at most, so the whole list costs a +// trivial number of tokens. +// +// Title falls back to the project ID when a document has no H1. +func (s *Store) Catalog(ctx context.Context) ([]CatalogEntry, error) { + // The chunker emits the frontmatter block first (when present) and the + // H1 title section next, so the lowest-indexed non-frontmatter section + // carries the document's title. + rows, err := s.db.QueryContext(ctx, ` + SELECT project_id, section, MIN(chunk_index) + FROM portfolio_chunks + WHERE section <> 'frontmatter' AND kind = ? + GROUP BY project_id + ORDER BY project_id`, KindProject) + if err != nil { + return nil, fmt.Errorf("catalog query: %w", err) + } + defer rows.Close() + + var out []CatalogEntry + for rows.Next() { + var e CatalogEntry + var idx int + if err := rows.Scan(&e.ProjectID, &e.Title, &idx); err != nil { + return nil, fmt.Errorf("catalog scan: %w", err) + } + if strings.TrimSpace(e.Title) == "" { + e.Title = e.ProjectID + } + out = append(out, e) + } + return out, rows.Err() +} + // Search returns up to topK chunks ordered by BM25 score. func (s *Store) Search(ctx context.Context, query string, topK int) ([]SearchResult, error) { if topK <= 0 { topK = 5 } rows, err := s.db.QueryContext(ctx, fmt.Sprintf(` - SELECT id, project_id, source_file, section, chunk_index, content, bm25(portfolio_chunks) AS score + SELECT id, project_id, kind, source_file, section, chunk_index, content, bm25(portfolio_chunks) AS score FROM portfolio_chunks - WHERE portfolio_chunks MATCH '%s' + WHERE portfolio_chunks MATCH '%s' AND section <> 'frontmatter' ORDER BY score LIMIT %d `, sanitizeFTS5(query), topK)) @@ -159,7 +340,7 @@ func (s *Store) Search(ctx context.Context, query string, topK int) ([]SearchRes var hits []SearchResult for rows.Next() { var r SearchResult - if err := rows.Scan(&r.ID, &r.ProjectID, &r.SourceFile, &r.Section, &r.Index, &r.Content, &r.Score); err != nil { + if err := rows.Scan(&r.ID, &r.ProjectID, &r.Kind, &r.SourceFile, &r.Section, &r.Index, &r.Content, &r.Score); err != nil { return nil, err } hits = append(hits, r) @@ -205,13 +386,28 @@ func sanitizeFTS5(q string) string { // ReindexOnDisk is a small convenience that opens the store, reindexes, and // closes — used by the CLI subcommand. -func ReindexOnDisk(dbPath, dataPath string, cfg ChunkerConfig) (time.Duration, int, int, error) { +// embedder may be nil, in which case no vectors are written and retrieval +// stays keyword-only. An embedding failure is logged and swallowed: the +// keyword index is already committed by then, and a bot that answers from +// keywords alone beats a reindex that reports failure and leaves nothing. +func ReindexOnDisk(dbPath string, sources []Source, cfg ChunkerConfig, embedder Embedder) (time.Duration, int, int, error) { start := time.Now() store, err := OpenStore(dbPath) if err != nil { return 0, 0, 0, err } defer store.Close() - files, chunks, err := store.Reindex(context.Background(), dataPath, cfg) - return time.Since(start), files, chunks, err -} \ No newline at end of file + files, chunks, err := store.Reindex(context.Background(), sources, cfg) + if err != nil { + return time.Since(start), files, chunks, err + } + if embedder != nil { + n, err := store.EmbedChunks(context.Background(), embedder) + if err != nil { + slog.Error("embedding failed; keyword search still works", "err", err) + } else { + slog.Info("embedded chunks", "chunks", n) + } + } + return time.Since(start), files, chunks, nil +} diff --git a/internal/portfolio/store_test.go b/internal/portfolio/store_test.go index 8f87e80..d1c8725 100644 --- a/internal/portfolio/store_test.go +++ b/internal/portfolio/store_test.go @@ -2,6 +2,7 @@ package portfolio import ( "context" + "database/sql" "os" "path/filepath" "strings" @@ -31,7 +32,7 @@ func TestReindexAndSearch(t *testing.T) { } defer store.Close() - files, chunks, err := store.Reindex(context.Background(), srcDir, DefaultChunkerConfig()) + files, chunks, err := store.Reindex(context.Background(), SourcesFor(srcDir, ""), DefaultChunkerConfig()) if err != nil { t.Fatal(err) } @@ -78,7 +79,7 @@ func TestReindexFromRealData(t *testing.T) { t.Fatal(err) } defer store.Close() - files, chunks, err := store.Reindex(context.Background(), srcDir, DefaultChunkerConfig()) + files, chunks, err := store.Reindex(context.Background(), SourcesFor(srcDir, ""), DefaultChunkerConfig()) if err != nil { t.Fatal(err) } @@ -113,4 +114,193 @@ func truncate(s string, n int) string { return s } return s[:n] + "..." -} \ No newline at end of file +} +func TestCatalogListsEveryProjectWithItsTitle(t *testing.T) { + dir := t.TempDir() + srcDir := filepath.Join(dir, "src") + if err := os.MkdirAll(srcDir, 0o755); err != nil { + t.Fatal(err) + } + // Two documents: one with frontmatter (so the title is not chunk 0), + // one without. Both must show up with their H1 as the title. + docs := map[string]string{ + "alpha.md": "---\ntitle: \"ignored\"\n---\n\n# Alpha Project\n\nBody.\n\n## Stack\n\nGo.\n", + "beta.md": "# Beta Project\n\nBody.\n", + } + for name, body := range docs { + if err := os.WriteFile(filepath.Join(srcDir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + + store, err := OpenStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, _, err := store.Reindex(context.Background(), SourcesFor(srcDir, ""), DefaultChunkerConfig()); err != nil { + t.Fatal(err) + } + + entries, err := store.Catalog(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 { + t.Fatalf("Catalog returned %d entries, want 2: %+v", len(entries), entries) + } + want := map[string]string{"alpha": "Alpha Project", "beta": "Beta Project"} + for _, e := range entries { + if want[e.ProjectID] != e.Title { + t.Errorf("Catalog[%s].Title = %q, want %q", e.ProjectID, e.Title, want[e.ProjectID]) + } + } +} + +func TestReindexIndexesMdxAndSeparatesDocsFromProjects(t *testing.T) { + dir := t.TempDir() + projects := filepath.Join(dir, "projects") + docs := filepath.Join(dir, "docs") + for _, d := range []string{projects, docs} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + write := func(path, body string) { + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(projects, "tienda.md"), "# Tienda\n\n## Stack\n\nNext.js y Stripe.\n") + // The CV is authored as .mdx for an Astro site and is not a project. + write(filepath.Join(docs, "cv.mdx"), "---\nname: \"V\"\n---\n\n## Skills\n\nKubernetes, AWS, Go.\n") + + store, err := OpenStore(filepath.Join(dir, "t.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + files, _, err := store.Reindex(context.Background(), SourcesFor(projects, docs), DefaultChunkerConfig()) + if err != nil { + t.Fatal(err) + } + if files != 2 { + t.Fatalf("indexed %d files, want 2 (the .mdx must be picked up)", files) + } + + // The CV is searchable... + hits, err := store.Search(context.Background(), "Kubernetes", 5) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Fatal("Kubernetes is in the CV but the search returned nothing") + } + if hits[0].ProjectID != "cv" || hits[0].Kind != KindDoc { + t.Errorf("top hit = %q/%q, want cv/%s", hits[0].ProjectID, hits[0].Kind, KindDoc) + } + + // ...but must never be announced as one of Victor's projects. + cat, err := store.Catalog(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(cat) != 1 || cat[0].ProjectID != "tienda" { + t.Fatalf("catalogue = %+v, want only the tienda project", cat) + } +} + +// Both content directories ship a README telling whoever fills them what to +// put there. Those instructions are not content: indexing them announced +// "README" and "README.es" as projects of Victor's in every prompt. +func TestReindexSkipsTheDirectoryReadmes(t *testing.T) { + dir := t.TempDir() + projects := filepath.Join(dir, "projects") + docs := filepath.Join(dir, "docs") + for _, d := range []string{projects, docs} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + write := func(path, body string) { + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(projects, "tienda.md"), "# Tienda\n\n## Stack\n\nNext.js.\n") + write(filepath.Join(projects, "README.md"), "# Portfolio Projects\n\n## Naming\n\nOne file per project.\n") + write(filepath.Join(projects, "README.es.md"), "# Proyectos\n\n## Nombres\n\nUn archivo por proyecto.\n") + write(filepath.Join(docs, "README.md"), "# Reference documents\n\n## What goes here\n\nA CV, an about page.\n") + + store, err := OpenStore(filepath.Join(dir, "t.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + files, _, err := store.Reindex(context.Background(), SourcesFor(projects, docs), DefaultChunkerConfig()) + if err != nil { + t.Fatal(err) + } + if files != 1 { + t.Fatalf("indexed %d files, want only tienda.md", files) + } + + cat, err := store.Catalog(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(cat) != 1 || cat[0].ProjectID != "tienda" { + t.Fatalf("catalogue = %+v, want only the tienda project", cat) + } + + // And the instructions are not retrievable either — a question about + // naming conventions must not surface the maintainer's docs. + hits, err := store.Search(context.Background(), "naming convention", 5) + if err != nil { + t.Fatal(err) + } + for _, h := range hits { + if strings.Contains(strings.ToLower(h.ProjectID), "readme") { + t.Errorf("search returned the README: %+v", h) + } + } +} + +// An index built before the kind column exists must not break the bot: the +// table is derived data, so OpenStore rebuilds it. +func TestOpenStoreMigratesIndexWithoutKindColumn(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "old.db") + + old, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + _, err = old.Exec(`CREATE VIRTUAL TABLE portfolio_chunks USING fts5( + id UNINDEXED, project_id UNINDEXED, source_file UNINDEXED, + section UNINDEXED, chunk_index UNINDEXED, content)`) + if err != nil { + t.Fatal(err) + } + if _, err := old.Exec(`INSERT INTO portfolio_chunks VALUES ('a','b','c','d',0,'stale')`); err != nil { + t.Fatal(err) + } + if err := old.Close(); err != nil { + t.Fatal(err) + } + + store, err := OpenStore(dbPath) + if err != nil { + t.Fatalf("OpenStore on a pre-kind database failed: %v", err) + } + defer store.Close() + + // Queries that name `kind` must now work. + if _, err := store.Catalog(context.Background()); err != nil { + t.Errorf("Catalog after migration: %v", err) + } + if _, err := store.Search(context.Background(), "stale", 5); err != nil { + t.Errorf("Search after migration: %v", err) + } +} diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 0883e62..faf9c52 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -22,15 +22,25 @@ type Handlers struct { runner *agent.Runner store *portfolio.Store version string + // embedder is used only by the /api/reindex endpoint, so a rebuild + // triggered over HTTP refreshes the vectors too instead of silently + // leaving them describing the previous corpus. May be nil. + embedder portfolio.Embedder } func NewHandlers(cfg *config.Config, runner *agent.Runner, store *portfolio.Store, version string) *Handlers { return &Handlers{cfg: cfg, runner: runner, store: store, version: version} } +// WithEmbedder attaches the embeddings client used when reindexing over HTTP. +func (h *Handlers) WithEmbedder(e portfolio.Embedder) *Handlers { + h.embedder = e + return h +} + type ChatRequest struct { - Messages []ChatMessage `json:"messages"` - Stream *bool `json:"stream,omitempty"` + Messages []ChatMessage `json:"messages"` + Stream *bool `json:"stream,omitempty"` // ConversationID is optional. If empty, the server creates a new // conversation and returns its ID in the response (or in the SSE // `start` event). Pass an existing ID to continue a previous thread. @@ -43,9 +53,9 @@ type ChatMessage struct { } type ChatResponse struct { - ConversationID string `json:"conversation_id"` - Content string `json:"content"` - Sources []string `json:"sources,omitempty"` + ConversationID string `json:"conversation_id"` + Content string `json:"content"` + Sources []string `json:"sources,omitempty"` Usage streaming.Usage `json:"usage"` } @@ -304,7 +314,7 @@ func (h *Handlers) Info(w http.ResponseWriter, _ *http.Request) { "name": h.cfg.Persona.Name, "version": h.version, "provider": p.Type, - "model": firstNonEmpty(p.Model, p.ModelPath), + "model": p.Model, "rag": h.cfg.RAG.Enabled, "top_k": h.cfg.RAG.TopK, "started": time.Now().UTC().Format(time.RFC3339), @@ -322,7 +332,7 @@ func (h *Handlers) Reindex(w http.ResponseWriter, r *http.Request) { http.Error(w, "rag.db_path not configured", http.StatusBadRequest) return } - dur, files, chunks, err := portfolio.ReindexOnDisk(dbPath, h.cfg.RAG.DataPath, portfolio.DefaultChunkerConfig()) + dur, files, chunks, err := portfolio.ReindexOnDisk(dbPath, portfolio.SourcesFor(h.cfg.RAG.DataPath, h.cfg.RAG.DocsPath), portfolio.DefaultChunkerConfig(), h.embedder) if err != nil { http.Error(w, "reindex failed: "+err.Error(), http.StatusInternalServerError) return @@ -432,12 +442,3 @@ func conversationIDFromPath(path string) string { } return id } - -func firstNonEmpty(vals ...string) string { - for _, v := range vals { - if v != "" { - return v - } - } - return "" -} \ No newline at end of file diff --git a/internal/server/testhelpers_test.go b/internal/server/testhelpers_test.go index 179863f..26a57d0 100644 --- a/internal/server/testhelpers_test.go +++ b/internal/server/testhelpers_test.go @@ -22,7 +22,7 @@ func ragOpenStore(dbPath, srcDir string) (ragStore, error) { if err != nil { return nil, err } - if _, _, err := store.Reindex(context.Background(), srcDir, portfolio.DefaultChunkerConfig()); err != nil { + if _, _, err := store.Reindex(context.Background(), portfolio.SourcesFor(srcDir, ""), portfolio.DefaultChunkerConfig()); err != nil { _ = store.Close() return nil, err }