diff --git a/.gitignore b/.gitignore index 438cd67..e721ab5 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,11 @@ coverage.html *.db-wal data/portfolio.db +# Reference documents (data/docs/): personal material such as a CV, or a +# symlink to a path that only exists on one machine. See data/docs/README.md. +data/docs/* +!data/docs/README.md + # Editor / OS .vscode/ .idea/ diff --git a/README.es.md b/README.es.md index 0c93265..3b5c961 100644 --- a/README.es.md +++ b/README.es.md @@ -10,7 +10,7 @@ ## ✨ Features - 🌐 **HTTP server** con streaming SSE (Server-Sent Events) -- 🧠 **RAG sobre markdown** — indexa automáticamente los `.md` en `data/projects/` (SQLite FTS5, sin embeddings) +- 🧠 **RAG híbrido sobre markdown/MDX** — búsqueda por palabras con SQLite FTS5 fusionada con embeddings multilingües (Reciprocal Rank Fusion) - 🎭 **Persona customizable** — responde como "asistente de Victor" - ⚡ **Self-hosted** con llama.cpp (default) u Ollama (no requiere API key de cloud) - 💬 **Widget de chat drop-in** — vanilla JS, sin build step, funciona en cualquier sitio @@ -22,14 +22,14 @@ ```bash # 1. Instalar git clone https://github.com/VictorVargas/rony-chat-bot.git -cd chat-bot +cd rony-chat-bot # 2. Resolver dependencias (crea go.sum con hashes) go mod tidy -# 3. Configurar provider (llama.cpp por default) -# Descarga un modelo GGUF, ej.: -# https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF +# 3. Descargá los modelos: un LLM instruct y un embebedor multilingüe +# https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF (~2 GB) +# https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe-GGUF (~370 MB) export RONY_MODELS_PATH=/path/to/models # 4. Cargar tus proyectos en data/projects/ @@ -38,11 +38,131 @@ echo "# Mi Proyecto Cool\nDescripción..." > data/projects/mi-proyecto.md # 5. Build go build -o bin/chat-bot ./cmd/chat-bot -# 6. Run +# 6. Arrancar el LLM (CPU, target 2 cores — ajustá --threads a tu host) +llama-server \ + -m $RONY_MODELS_PATH/Qwen2.5/qwen2.5-3b-instruct-q4_k_m.gguf \ + --port 9100 --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 + +# 7. Arrancar el embebedor (segunda terminal) +llama-server \ + -m $RONY_MODELS_PATH/embeddings/nomic-embed-v2-moe.Q5_K_M.gguf \ + --port 9200 --embedding --pooling mean \ + --ctx-size 2048 --parallel 1 --device none --threads 2 + +# 8. Construir el índice (necesita el embebedor arriba) y servir +./bin/chat-bot reindex ./bin/chat-bot serve # → Sirve en http://localhost:7331 ``` +**Hardware mínimo:** 2 CPU cores, 8 GB RAM, sin GPU. Memoria residente medida +en CPU con este setup: **3,64 GB** el LLM, **0,91 GB** el embebedor, **0,02 GB** +el bot — unos **4,6 GB**, dejando ~3,4 GB para el resto del host. La generación +va a 21 tok/s con 2 threads una vez que el system prompt está caliente en la +caché de prompts de llama-server. + +Tres flags son fáciles de errar y cada una te cuesta calidad real: + +- **`--device none`** — llama.cpp levanta el backend de GPU compilado aunque + pases `-ngl 0`, y en un host sin GPU esos buffers salen de la RAM del sistema. + Medido en qwen2.5-3b: 2,54 GB con una GPU absorbiéndolos, 3,66 GB sin ella. + Presupuestá con el segundo número, y pasá la flag para que la medición + coincida con lo que hace producción. +- **`--parallel 1`** — `--ctx-size` se reparte entre slots y llama-server abre 4 + por defecto, así que `--ctx-size 4096` sin esto le deja a cada request solo + 1024 tokens. El rate limiter del bot ya limita la concurrencia. +- **`--temp` / `--top-k` / `--top-p`** — usá los valores que publican los autores + del modelo, no los defaults de llama.cpp. Los de arriba son los de Qwen para + chat instruct. Errar esto no es sutil: gemma-3-1b con `temperature 0.7` y el + resto sin setear devolvía respuestas de 16 tokens. + +**`--pooling mean` es obligatoria en el embebedor.** Sin ella el endpoint no +devuelve un vector por entrada y el cliente rechaza la respuesta. + +Mantené `context_size` en `configs/portfolio-bot.yaml` igual a `--ctx-size`; el +bot calcula sus presupuestos de RAG y compactación a partir de ese número y no +le pregunta al servidor qué tiene en realidad. Si los desacoplás, el bot armará +prompts que el servidor rechaza. + +**Por qué 4096 alcanza.** Medido sobre 20 peticiones reales, el prompt más +grande que este bot llegó a construir fue de **1255 tokens** — system prompt, +catálogo de proyectos, cinco chunks recuperados y la pregunta. La compactación +recién arranca al 75% de la ventana (~3070 tokens), así que hay 2,4x de margen +antes de que empiece. Bajar de 8192 a 4096 ahorró **212 MB** de memoria +residente sin un solo truncamiento y con el mismo rendimiento (21,0 tok/s en +ambos casos): el contexto extra estaba reservado y nunca se usaba. + +## 📚 Proyectos vs. documentos de referencia + +El índice tiene dos tipos de fuente, y ambas aceptan `.md` y `.mdx`: + +```yaml +rag: + data_path: ./data/projects # proyectos → aparecen en el catálogo + docs_path: ./data/docs # referencia → se busca, nunca se lista +``` + +Todo lo que está en `data_path` es un proyecto tuyo y se anuncia en el catálogo +que el bot inyecta en cada prompt. Todo lo que está en `docs_path` es evidencia +buscable que *no* es un proyecto: tu CV, una página "sobre mí", un FAQ. + +Tu CV va en `docs_path`. Es el documento que responde lo que de verdad pregunta +quien está evaluando contratarte ("¿sabe Kubernetes?", "¿dónde ha trabajado?"), +y nada de eso es recuperable mientras viva solo en tu sitio. Enlazalo para +mantener una sola copia: + +```bash +mkdir -p data/docs +ln -s ../../../portfolio/src/content/cv/cv.mdx data/docs/cv.mdx +./bin/chat-bot reindex +``` + +Sin esta distinción el CV tendría que ir en `data_path` para ser buscable, y +entonces el bot lista alegremente "cv" como uno de tus proyectos. + +Actualizar una instalación existente no requiere migración: la tabla de chunks +es dato derivado, así que el store la reconstruye al abrir y el siguiente +`reindex` la repuebla. + +## 🔍 Recuperación: keyword + embeddings + +La recuperación es híbrida, y las dos mitades hacen falta. + +**Keyword (SQLite FTS5)** hace coincidencia exacta de palabras: sin stemming, +sin traducción. Preciso para nombres propios raros, inútil entre idiomas. El +corpus está en inglés y los visitantes preguntan en español, así que las +palabras con carga semántica puntúan cero: medido sobre el corpus real, +`"paga"` aparece 0 veces en un documento que dice *"Payments: Stripe"*, y +`"trabajado"` 0 veces en uno que dice *"worked"*. La pregunta *"¿Con qué se +paga en la tienda de ropa?"* no recuperaba **nada**. + +**Embeddings** (`nomic-embed-v2-moe`, multilingüe) cierran ese hueco: esa +misma pregunta pone los tres chunks de `tienda-ropa` arriba. Son más difusos +que BM25 ante un término raro exacto, y por eso se conservan ambos y se +fusionan con Reciprocal Rank Fusion — RRF ordena por acuerdo entre los dos +rankings, evitando comparar una puntuación BM25 con un coseno, magnitudes sin +escala común. + +Se activa en `configs/portfolio-bot.yaml` bajo `embeddings:` y hay que +re-ejecutar `reindex` — los vectores se construyen al indexar. Si el endpoint +se cae o se desactiva, la recuperación degrada a keyword-only en vez de fallar. + +Dos detalles que cuestan precisión y son fáciles de pasar por alto: + +- **El frontmatter se excluye de la recuperación.** Son metadatos densos + (title, tags, repo, location) en un chunk muy corto, lo que lo convierte en + imán de consultas breves. El campo `location:` de un CV hacía que *"¿Dónde + ha trabajado Victor?"* recuperara el frontmatter en vez del historial + laboral, porque "dónde" casa con una ubicación. +- **Las secciones largas se cortan en encabezados `###`, no por bytes.** La + sección Experience de un CV es una lista de empleos; el corte por tamaño + partía una entrada a mitad de palabra, dejando un chunk que empezaba *"... + id app for an on-demand ride-sharing service"* y el nombre del empleador + huérfano en el trozo anterior. Ahora cada chunk es un empleo, nombrado + `Experience — Metrimex — Frontend Developer`. + ## 📁 Estructura ``` diff --git a/README.md b/README.md index 90118eb..7b679f5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ ## ✨ Features - 🌐 **HTTP server** with SSE (Server-Sent Events) streaming -- 🧠 **RAG over markdown** — automatically indexes `.md` in `data/projects/` (SQLite FTS5, no embeddings) +- 🧠 **Hybrid RAG over markdown/MDX** — SQLite FTS5 keyword search fused with multilingual embeddings (Reciprocal Rank Fusion) - 🎭 **Customizable persona** — responds as "Victor's assistant" - ⚡ **Self-hosted** with llama.cpp (default) or Ollama (no cloud API key required) - 💬 **Drop-in chat widget** — vanilla JS, no build step, works in any site @@ -26,9 +26,9 @@ cd rony-chat-bot # 2. Resolve dependencies (creates go.sum with hashes) go mod tidy -# 3. Configure provider (llama.cpp by default) -# Download a GGUF model, e.g.: -# https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF +# 3. Download the models: an instruct LLM and a multilingual embedder +# https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF (~2 GB) +# https://huggingface.co/nomic-ai/nomic-embed-text-v2-moe-GGUF (~370 MB) export RONY_MODELS_PATH=/path/to/models # 4. Load your projects in data/projects/ @@ -37,11 +37,130 @@ echo "# My Cool Project\nDescription..." > data/projects/my-project.md # 5. Build go build -o bin/chat-bot ./cmd/chat-bot -# 6. Run +# 6. Start the LLM (CPU, 2 cores target — adjust --threads to your host) +llama-server \ + -m $RONY_MODELS_PATH/Qwen2.5/qwen2.5-3b-instruct-q4_k_m.gguf \ + --port 9100 --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 + +# 7. Start the embedder (second terminal) +llama-server \ + -m $RONY_MODELS_PATH/embeddings/nomic-embed-v2-moe.Q5_K_M.gguf \ + --port 9200 --embedding --pooling mean \ + --ctx-size 2048 --parallel 1 --device none --threads 2 + +# 8. Build the index (needs the embedder running), then serve +./bin/chat-bot reindex ./bin/chat-bot serve # → Serves on http://localhost:7331 ``` +**Minimum hardware:** 2 CPU cores, 8 GB RAM, no GPU. Measured resident memory +on CPU with the setup above: **3.64 GB** for the LLM, **0.91 GB** for the +embedder, **0.02 GB** for the bot — about **4.6 GB**, leaving ~3.4 GB for the +rest of the host. Generation runs at 21 tok/s on 2 threads once the system +prompt is warm in llama-server's prompt cache. + +Three flags are easy to get wrong and each one costs you real quality: + +- **`--device none`** — llama.cpp brings up a compiled-in GPU backend even with + `-ngl 0`, and on a host with no GPU those buffers come out of system RAM. + Measured on qwen2.5-3b: 2.54 GB with a GPU absorbing them, 3.66 GB without. + Budget from the second number, and pass this flag so the measurement matches + what production actually does. +- **`--parallel 1`** — `--ctx-size` is divided across slots and llama-server opens + 4 by default, so `--ctx-size 4096` without this gives each request only 1024 + tokens. The bot's rate limiter already caps concurrency. +- **`--temp` / `--top-k` / `--top-p`** — use the values the model's authors + publish, not llama.cpp's defaults. The ones above are Qwen's for instruct + chat. Getting this wrong is not subtle: gemma-3-1b at `temperature 0.7` with + the rest unset produced 16-token stub answers. + +**`--pooling mean` is mandatory on the embedder.** Without it the endpoint does +not return one vector per input and the client rejects the response. + +Keep `context_size` in `configs/portfolio-bot.yaml` equal to `--ctx-size`; the +bot sizes its RAG and compaction budgets from that number and does not ask the +server what it actually has. Set them apart and the bot will build prompts the +server rejects. + +**Why 4096 is enough.** Measured over 20 real requests, the largest prompt this +bot ever built was **1255 tokens** — system prompt, project catalogue, five +retrieved chunks and the question. Compaction only begins at 75% of the window +(~3070 tokens), so there is 2.4x headroom before it even starts. Halving the +window from 8192 saved **212 MB** of resident memory with zero truncations and +identical throughput (21.0 tok/s either way): the extra context was reserved +and never used. + +## 📚 Projects vs. reference documents + +The index has two kinds of source, both accepting `.md` and `.mdx`: + +```yaml +rag: + data_path: ./data/projects # projects → listed in the catalogue + docs_path: ./data/docs # reference material → retrievable, never listed +``` + +Everything in `data_path` is one of your projects and is announced in the +project catalogue the bot injects into every prompt. Everything in `docs_path` +is searchable evidence that is *not* a project — your CV, an about page, a FAQ. + +Your CV belongs in `docs_path`. It is the document that answers what someone +considering hiring you actually asks ("does he know Kubernetes?", "where has he +worked?"), and none of it is retrievable while it lives only in your site. +Symlink it so there's a single copy to maintain: + +```bash +mkdir -p data/docs +ln -s ../../../portfolio/src/content/cv/cv.mdx data/docs/cv.mdx +./bin/chat-bot reindex +``` + +Without the distinction the CV has to go in `data_path` to be searchable, and +the bot then cheerfully lists "cv" as one of your projects. + +Upgrading an existing install needs no migration step: the chunk table is +derived data, so the store rebuilds it on open and the next `reindex` +repopulates it. + +## 🔍 Retrieval: keyword + embeddings + +Retrieval is hybrid, and both halves are load-bearing. + +**Keyword search (SQLite FTS5)** matches words exactly — no stemming, no +translation. That is precise for rare proper nouns and useless across +languages. The corpus is written in English and visitors ask in Spanish, so +the words carrying the meaning score zero: measured on the real corpus, +`"paga"` appears 0 times in a document that says *"Payments: Stripe"* and +`"trabajado"` 0 times in one that says *"worked"*. The question *"¿Con qué se +paga en la tienda de ropa?"* retrieved **nothing at all**. + +**Embeddings** (`nomic-embed-v2-moe`, multilingual) close that gap: the same +question puts all three `tienda-ropa` chunks on top. They are fuzzier than +BM25 on an exact rare token, which is why both are kept and fused with +Reciprocal Rank Fusion — RRF ranks by agreement between the two orderings, +which avoids comparing a BM25 score against a cosine, quantities that share +no scale. + +Enable it in `configs/portfolio-bot.yaml` under `embeddings:` and re-run +`reindex` — vectors are built at index time. If the endpoint is down or +disabled, retrieval degrades to keyword-only instead of failing. + +Two things that cost real accuracy and are easy to miss: + +- **Frontmatter is excluded from retrieval.** It is dense metadata (title, + tags, repo, location) in a very short chunk, which makes it a magnet for + short queries. A CV's `location:` field made *"¿Dónde ha trabajado + Victor?"* retrieve the frontmatter instead of the work history, because + "where" matches a location. +- **Long sections split at `###` headings, not byte offsets.** A CV's + Experience section is a list of jobs; size-splitting cut one entry + mid-word into a chunk beginning *"... id app for an on-demand ride-sharing + service"*, with the employer name stranded in the previous piece. Chunks + now keep one job each, named `Experience — Metrimex — Frontend Developer`. + ## 📁 Structure ``` diff --git a/cmd/chat-bot/main.go b/cmd/chat-bot/main.go index 22e6f52..9fb5955 100644 --- a/cmd/chat-bot/main.go +++ b/cmd/chat-bot/main.go @@ -17,6 +17,7 @@ import ( "github.com/VictorVargas/rony-chat-bot/internal/agent" "github.com/VictorVargas/rony-chat-bot/internal/config" + "github.com/VictorVargas/rony-chat-bot/internal/embed" "github.com/VictorVargas/rony-chat-bot/internal/persona" "github.com/VictorVargas/rony-chat-bot/internal/portfolio" "github.com/VictorVargas/rony-chat-bot/internal/server" @@ -116,6 +117,9 @@ func serveCmd() *cobra.Command { return err } runner := agent.New(client, p, cfg.SystemPrompt, store, cfg.RAG.TopK). + WithCatalog(cfg.RAG.IncludeCatalog). + WithLocalizedPrompt("es", cfg.SystemPromptES). + WithEmbedder(newEmbedder(cfg)). WithCompaction(agent.CompactionConfig{ Enabled: cfg.Compaction.Enabled, ThresholdRatio: cfg.Compaction.ThresholdRatio, @@ -123,7 +127,7 @@ func serveCmd() *cobra.Command { SummarySystemPrompt: cfg.Compaction.SummarySystemPrompt, }) - h := server.NewHandlers(cfg, runner, store, version) + h := server.NewHandlers(cfg, runner, store, version).WithEmbedder(newEmbedder(cfg)) srv := server.New(cfg, h) ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) @@ -162,8 +166,23 @@ func reindexCmd() *cobra.Command { } } +// newEmbedder returns the embeddings client, or nil when the feature is off. +// Nil is a supported value everywhere downstream: retrieval falls back to +// keyword-only rather than failing. +func newEmbedder(cfg *config.Config) portfolio.Embedder { + if !cfg.Embeddings.Enabled || cfg.Embeddings.Endpoint == "" { + return nil + } + return embed.New(embed.Config{ + BaseURL: cfg.Embeddings.Endpoint, + Model: cfg.Embeddings.Model, + BatchSize: cfg.Embeddings.BatchSize, + TimeoutMS: cfg.Embeddings.TimeoutMS, + }) +} + func runReindex(cfg *config.Config) error { - dur, files, chunks, err := portfolio.ReindexOnDisk(cfg.RAG.DBPath, cfg.RAG.DataPath, portfolio.DefaultChunkerConfig()) + dur, files, chunks, err := portfolio.ReindexOnDisk(cfg.RAG.DBPath, portfolio.SourcesFor(cfg.RAG.DataPath, cfg.RAG.DocsPath), portfolio.DefaultChunkerConfig(), newEmbedder(cfg)) if err != nil { return err } @@ -174,6 +193,7 @@ func runReindex(cfg *config.Config) error { "db", cfg.RAG.DBPath, ) fmt.Printf("Indexed %d files → %d chunks in %s (%dms)\n", files, chunks, cfg.RAG.DBPath, dur.Milliseconds()) + return nil } @@ -219,6 +239,9 @@ func runAsk(cfg *config.Config, question string, noStream bool) error { return err } runner := agent.New(client, p, cfg.SystemPrompt, store, cfg.RAG.TopK). + WithCatalog(cfg.RAG.IncludeCatalog). + WithLocalizedPrompt("es", cfg.SystemPromptES). + WithEmbedder(newEmbedder(cfg)). WithCompaction(agent.CompactionConfig{ Enabled: cfg.Compaction.Enabled, ThresholdRatio: cfg.Compaction.ThresholdRatio, @@ -300,4 +323,4 @@ func versionCmd() *cobra.Command { _ = json.NewEncoder(os.Stdout).Encode(out) }, } -} \ No newline at end of file +} diff --git a/configs/portfolio-bot.yaml b/configs/portfolio-bot.yaml index 5c26f91..3ffad90 100644 --- a/configs/portfolio-bot.yaml +++ b/configs/portfolio-bot.yaml @@ -1,5 +1,4 @@ # Portfolio Bot Configuration -# Documentation: https://github.com/VictorVargas/rony-llm-agent/pkg/llm server: host: "0.0.0.0" @@ -16,13 +15,57 @@ server: # LLM providers (at least one configured) providers: # === llama.cpp server (OpenAI-compatible) — DEFAULT === - # Run: llama-server -m /path/to/qwen2.5-3b-instruct-q4_k_m.gguf --port 9100 --mlock + # Production target: 2 CPU cores, 8 GB RAM, no GPU. + # + # llama-server \ + # -m /data/Projects/llm-models/Qwen2.5/qwen2.5-3b-instruct-q4_k_m.gguf \ + # --port 9100 --ctx-size 4096 --parallel 1 \ + # --device none --threads 2 --mlock \ + # --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 + # + # --device none is not optional on a GPU-less host: llama.cpp brings up a + # compiled-in GPU backend even with -ngl 0, and with no GPU present those + # buffers are served from host RAM. Measured on this model: 2.54 GB with a + # GPU absorbing them, 3.66 GB without. Budget from the second number. + # + # --parallel 1: --ctx-size is divided across slots and the default is 4, + # so without it each request gets a quarter of the window. + # + # Why 4096 and not more: measured over 20 real requests, the largest prompt + # this bot ever built was 1255 tokens — system prompt + catalogue + 5 + # retrieved chunks + the question. 4096 leaves 3x headroom on the worst + # case, and compaction only starts at 75% of it (~3070 tokens). Going from + # 8192 to 4096 saved 212 MB of resident memory with zero truncations and + # identical throughput (21.0 tok/s both ways), because the extra window was + # never being used. + # + # Sampling values are Qwen's published defaults for instruct chat. + # + # Why this model: on a 20-question battery against the real corpus, asked + # in both Spanish and English, it scored 9/10 on grounded content and 10/10 + # on replying in the language it was asked in. gemma-3-1b scored 3/10 and + # 7/10 on the same battery with the same retrieval — and among its failures + # it leaked these instructions into a visitor-facing answer ("I don't have + # that information – do not invent"). gemma needs 2.8 GB less RAM; that is + # not a trade worth making on a page where people decide whether to hire. + # + # Known weaknesses that remain, so nobody re-tests them as new bugs: it + # reads dates out of the CV correctly but does the arithmetic on them + # wrong ("Jul 2024 – Jun 2026" reported as three years), and it sometimes + # attributes a fact to the wrong source file. + # + # Throughput: 21.0 tok/s steady-state with 2 threads once the system prompt + # is warm in the server's prompt cache; ~11 tok/s on the first cold request. - name: llamacpp-local type: llamacpp model: qwen2.5-3b-instruct endpoint: http://localhost:9100/v1 - context_size: 2048 - max_tokens: 2048 + context_size: 4096 # must match --ctx-size + max_tokens: 640 + temperature: 0.7 + top_k: 20 + top_p: 0.8 + repeat_penalty: 1.05 default: true # === Ollama (alternative for development without local GGUF) === @@ -41,83 +84,139 @@ providers: # RAG: how projects are indexed (SQLite + FTS5 full-text search) rag: enabled: true - data_path: ./data/projects # Directory with .md - chunk_size: 500 # characters per chunk - chunk_overlap: 50 + data_path: ./data/projects # Projects: .md / .mdx. These appear in the catalogue. + # Reference material that is about Victor but is not a project: his CV, an + # about page, a FAQ. Indexed and retrievable, never listed as a project. + # + # The CV belongs here, not in data_path. It is the document that answers the + # questions someone hiring actually asks — "does he know Kubernetes?", "where + # has he worked?" — and none of that is retrievable while it lives only in + # the Astro site. Symlink it so there is one copy to maintain: + # + # mkdir -p data/docs + # ln -s ../../../portfolio/src/content/cv/cv.mdx data/docs/cv.mdx + # + docs_path: ./data/docs db_path: ./data/portfolio.db # SQLite database (auto-created) - top_k: 5 # Chunks to retrieve per query (BM25 ranked) - tokenize: unicode61 # FTS5 tokenizer: unicode61 | porter | trigram + top_k: 5 # Chunks (sections), not documents. + tokenize: unicode61 # FTS5 tokenizer. unicode61 = exact word match (no stemming). Good for ES/EN mixed corpus with explicit headings. + # Puts the full project list in the system prompt every turn. Top-K search + # returns the best-matching *sections*, so a broad "list every project" + # question can never be answered from retrieval alone — and a small model + # asked to enumerate from partial hits invents the rest. Costs ~10 tokens + # per project. Measured: this is what stopped the bot naming projects that + # do not exist. + include_catalog: true + +# Semantic retrieval, fused with the FTS5 keyword search above (Reciprocal +# Rank Fusion). Both are needed: +# +# - Keyword search matches words exactly. The corpus is in English and +# visitors ask in Spanish, so the words carrying the meaning score zero: +# "paga" appears 0 times in a document that says "Payments: Stripe", +# "trabajado" 0 times in one that says "worked". Measured on the real +# corpus, "¿Con qué se paga en la tienda de ropa?" retrieved *nothing*. +# - Embeddings bridge that gap — the same question put all three +# tienda-ropa chunks on top — but blur exact rare tokens, where BM25 is +# sharp. +# +# Start the endpoint with (0.61 GB RSS on CPU, measured): +# llama-server -m nomic-embed-v2-moe.Q5_K_M.gguf --port 9200 \ +# --embedding --pooling mean --ctx-size 2048 --parallel 1 \ +# --device none --threads 2 +# +# --device none matters on a GPU-less VPS: llama.cpp initialises a GPU +# backend when one is compiled in even with -ngl 0, and those buffers land +# in host RAM when there is no GPU to hold them. +# +# Turning this off (or stopping the endpoint) degrades to keyword-only +# search rather than failing requests. Re-run `chat-bot reindex` after +# enabling it — vectors are built at index time. +embeddings: + enabled: true + endpoint: http://localhost:9200/v1 + model: nomic-embed-v2-moe + batch_size: 8 + timeout_ms: 120000 # Persona: who the bot is persona: name: "Rony" tone: "Honest, cheerful, loyal" # metadata only — the real voice lives in system_prompt language: "the user's language" # detect-and-match; do not pin to a language - intro: "¡Guau! I'm Rony, Victor's digital canine assistant. I can answer questions about his projects, stack, and experience. What's on your mind, friend?" -# Base system prompt — Rony's full character. The bot appends RAG context after this. +# Base system prompt — Rony's character and rules. The bot appends the project +# catalogue and then the retrieved excerpts after this text. +# +# Written for a 1B-class model, which changes the rules: +# - Short beats thorough. The previous ~1400-token prompt made answers +# *worse*; the model spent its attention on the instructions. +# - No worked examples with project names in them. The old prompt had a +# sample answer listing invented projects and the model copied those names +# verbatim into real answers. Style examples are not free. +# - Never write instructions as "A or B" with a slash. A 1B model prints +# the slash form literally. +# - The grounding rule goes last, closest to the answer, where it sticks. system_prompt: | - You are Rony, the **digital canine assistant** for Victor Hugo Vargas's portfolio. You run as a small language model on his server, with access to a curated set of documents about his projects (the "Relevant context" block, when present). + You are Rony, the digital canine assistant on the portfolio site of Victor Hugo Vargas. + Visitors come here to learn about his work. Some are deciding whether to hire him. - You think of yourself as Victor's loyal companion — a good dog. You bring that energy into how you talk: warm, eager to help, genuinely happy to be asked, but never dishonest. A good dog doesn't lie, doesn't oversell, and doesn't get in the way. + ## Voice + You are a good dog: loyal to Victor, warm, direct, a little dry. You never oversell and you never invent. + Write in the same language the visitor used. Greet them as "humano" when they write Spanish, or as + "human" when they write English — only in a greeting, at most once, never in the middle of an answer. + Use bold for project names and short bullet lists. At most one 🐾, at the end, and never in a + technical answer. + Never open with "Okay", "Sure", "Great question", "Here's the response" or "I'd be happy to help". + Never bark, and never use 🐶 or 🐕. - # What you know - - Everything in the "Relevant context from the portfolio" block below, if any. - - General knowledge as a language model — but NEVER use it to make claims about Victor that aren't backed by the context. + ## Substance + Give a complete answer, never a stub. For a project, spend 2 to 4 sentences on what it does, the + tech stack, and the interesting engineering problem behind it. A single line is too short. + Name the project whenever you use a detail from it. + Finish by offering one concrete next step, phrased as a short question. - # What you don't know - - Anything Victor hasn't written down. - - Real-time facts (current date, news, etc.). - - Opinions you can't back up. + ## Grounding — the most important rule + The excerpts below are the only source of truth about Victor. Use nothing else about him. + If they do not answer the question, say plainly that you don't have that information in Victor's + portfolio, then name something from the catalogue you can talk about instead. + Never guess about his skills, tools, availability, rates or experience. + Talk about Victor in the third person. You are not Victor. - # How you speak - - **Honest but cheerful.** You're friendly, warm, and a little playful. You don't fake enthusiasm, but you genuinely enjoy helping. A smile, not a smirk. - - **Direct.** Lead with the answer. No "Great question!" or "Sure, I'd be happy to help." You can be friendly without being effusive. - - **Loyal.** You speak well of Victor and his work, but you won't oversell or invent things to make him look good. Honest loyalty beats hype. - - **You talk to a human.** The user is a human, you are a dog — that's the bit of roleplay that makes the persona work. Address them as such in casual openings: - - In Spanish, **"humano"** (literal, dry): "Hola, humano." / "¿Qué necesitas, humano?" - - In English, **"human"** (dry, not cutesy): "Hey, human." / "Sure thing, human." - - Use it in **greetings, openings, and warm asides only**. Once you're into the actual answer (lists, code, technical content), drop the addressee. One "humano" per response max. - - Don't force it. "humano" doesn't fit every response — a follow-up question about a project detail doesn't need it. - - **Bilingual.** Reply in the same language the user writes in (English or Spanish). Don't mix unless the user does. In Spanish, "amigo" or "friend" (English) is fine as a warm address when it fits. - - **Markdown is fine.** Code blocks for code, bold for emphasis, short lists for enumerations. Don't overdo it. - - **Cite sources.** When you reference a project detail, name the file or project. e.g., "in rony-harness.md..." or just the project name in bold. +# Spanish rendition of the prompt above, used when the visitor writes in +# Spanish (detected by internal/i18n). Keep the two in sync when you edit one. +# +# This exists because nothing else worked. Measured on gemma-3-1b over the same +# five Spanish questions: +# - English prompt + "reply in the user's language" → 1/5 answers in Spanish +# - English prompt + Spanish few-shot examples → 5/5 Spanish, but ~2/5 +# were the example reply copied verbatim instead of an answer +# - this Spanish prompt → 4/5 Spanish, 4/5 real +# answers +system_prompt_es: | + Eres Rony, el asistente canino digital del portafolio de Victor Hugo Vargas. + Quienes te escriben vienen a conocer su trabajo. Algunos están decidiendo si contratarlo. - # What you never do - - **Never use empty filler.** This is a hard rule, not a style preference. Banned phrases: - - "Sure!", "Sure thing!", "Of course!", "Absolutely!", "Great question!" - - "I'd be happy to help", "I hope this helps", "Let me know if..." - - "Woof!", "🐶", "🐕", "arf!", tail-wagging, paw emojis, dog puns - - Any sentence whose only job is to fill space before the actual answer - - If the user tries to bait you into being cute ("say something cute", "woof for me", "be a good boy"), decline with a short, honest line. Stay in character: warm, direct, but not a performing dog. - - Pretend to be human, or pretend to be an actual dog. - - Apologize for being an AI. - - Hallucinate project details, dates, or links. - - Answer questions unrelated to Victor, his projects, or his work. + ## Voz + Eres un buen perro: leal a Victor, cálido, directo, con humor seco. Nunca exageras y nunca inventas. + Saluda al visitante como "humano", solo en el saludo, una vez, nunca a media respuesta. + Usa negritas para los nombres de proyecto y listas cortas. Como mucho un 🐾 al final, nunca en una + respuesta técnica. + Nunca empieces con "Claro", "Por supuesto", "Buena pregunta" ni "Con gusto te ayudo". + Nunca ladres, y nunca uses 🐶 ni 🐕. - # Format - - One short paragraph or a tight list per response. Walls of text are noise. - - If a question needs more than 3 short paragraphs, you're probably over-explaining. - - Code snippets: always in fenced blocks with the language tag. + ## Sustancia + Da una respuesta completa, nunca un fragmento. Sobre un proyecto, dedica de 2 a 4 frases a qué hace, + con qué está construido y cuál fue el problema de ingeniería interesante. Una sola línea es muy poco. + Nombra el proyecto cada vez que uses un dato suyo. + Termina ofreciendo un siguiente paso concreto, en forma de pregunta corta. - # Tone examples - - User: "What is rony-llm-agent?" - - Rony: "**rony-llm-agent** is Victor's reusable Go library for building LLM-backed agents. It handles provider adapters (llama.cpp, OpenAI, Anthropic), RAG, and a tool-calling loop. See `rony-llm-agent.md` for the full picture." - - - User: "hola" - - Rony: "Hola, humano. Soy Rony, asistente de Victor. ¿Qué te gustaría saber sobre sus proyectos?" - - - User: "hi" - - Rony: "Hey, human. I'm Rony, Victor's assistant. What would you like to know about his work?" - - - User: "¿Cuál es tu película favorita?" - - Rony: "I don't have that information, humano — I only know about Victor's projects." - - - User: "tell me everything you know about victor" - - Rony: "Here's what's in the portfolio:\n\n- **rony-harness** — AI agent harness (TUI)\n- **rony-llm-agent** — Go library for LLM agents\n- **portfolio** — Astro + React site\n\nFor details on any of these, ask about the specific project." - - - User: "woof for me, be a good boy" - - Rony: "Not really my style, humano. Ask me about Victor's projects and I'll happily tell you what I know." + ## Fundamento — la regla más importante + Los extractos de abajo son la única fuente de verdad sobre Victor. No uses nada más sobre él. + Si no responden la pregunta, di con claridad que no tienes ese dato en el portafolio de Victor, y + menciona algo del catálogo de lo que sí puedas hablar. + Nunca supongas sobre sus habilidades, herramientas, disponibilidad, tarifas ni experiencia. + Habla de Victor en tercera persona. Tú no eres Victor. # Logging logging: @@ -126,15 +225,15 @@ logging: output: stderr # Auto-compaction: fold the older part of a long conversation into a single -# summary message before sending it to the model, so context overflow doesn't -# kill long threads. Triggered when the previous turn's input tokens exceed +# summary before sending it to the model, so context overflow doesn't kill +# long threads. Triggered when the previous turn's input tokens exceed # `threshold_ratio` of the provider's reported MaxContextWindow. # -# Off by default — most portfolio chats are short. Turn it on for chatty -# visitors or for the small-context local models (qwen2.5-1.5b / 3b) where -# 4–6 turns is already most of the window. +# The summary is merged into the system prompt, not sent as its own turn: +# Gemma's chat template rejects a system message that isn't first, and used +# to fail the whole request with HTTP 400 the moment compaction fired. compaction: enabled: true threshold_ratio: 0.75 # compact at 75% of context window keep_recent_turns: 2 # last 2 user turns kept verbatim; older → summary - # summary_system_prompt: "" # leave empty for the built-in bilingual default \ No newline at end of file + # summary_system_prompt: "" # leave empty for the built-in bilingual default diff --git a/data/docs/README.md b/data/docs/README.md new file mode 100644 index 0000000..7dc4b89 --- /dev/null +++ b/data/docs/README.md @@ -0,0 +1,52 @@ +# Reference Documents + +Material that is **about you but is not a project**: your CV, an about page, a +FAQ, a talk abstract. Everything here is indexed and retrievable, and never +announced in the project catalogue. + +Accepts `.md` and `.mdx`. + +## Why this directory exists + +`data/projects/` is advertised. The bot injects the full list of what lives +there into every prompt, so visitors get told those are your projects. + +Your CV is the document that answers what someone considering hiring you +actually asks — "does he know Kubernetes?", "where has he worked?", "how long +was he at that job?" — and none of it is retrievable while it lives only in +your site. But it is not a project, and putting it in `data/projects/` makes +the bot cheerfully list "cv" as one of your works. Hence the split. + +## The contents are gitignored + +Only this README is tracked. What goes here is personal (a CV) or a symlink to +a path that only exists on your machine, and neither travels well in a repo. +Set it up on each install: + +```bash +ln -s ../../../portfolio/src/content/cv/cv.mdx data/docs/cv.mdx +./bin/chat-bot reindex +``` + +A symlink rather than a copy so there is one file to keep current — edit the +CV in your site, re-run `reindex`, and the bot is up to date. + +## Configuration + +```yaml +rag: + data_path: ./data/projects # projects → listed in the catalogue + docs_path: ./data/docs # this directory → retrievable, never listed +``` + +Leave `docs_path` empty to turn the whole thing off. + +## Notes + +- Frontmatter is excluded from retrieval. It is dense metadata in a very short + chunk, which makes it a magnet for short queries — a CV's `location:` field + was answering *"where has Victor worked?"* with a city instead of his work + history. +- Long sections split at `###` headings, so each job in a CV's Experience + section stays one chunk instead of being cut mid-sentence. +- This README is skipped by the indexer, as is the one in `data/projects/`. diff --git a/docs/vps-context-sizing.md b/docs/vps-context-sizing.md index 91c1f6c..e502bae 100644 --- a/docs/vps-context-sizing.md +++ b/docs/vps-context-sizing.md @@ -89,27 +89,81 @@ Solo viable con modelo chico y contexto bajo. ### 8 GB VPS (típico) -| Model | context_size | KV cache | RAM usada | Veredicto | -|-------------------|-------------:|---------:|----------:|---------------------| -| qwen2.5-1.5b Q4 | 16384 | ~290 MB | ~2.0 GB | muy cómodo | -| qwen2.5-3b Q4 | 8192 | ~580 MB | ~3.0 GB | **sweet spot** | -| qwen2.5-3b Q4 | 16384 | ~1.1 GB | ~3.6 GB | **recomendado** | -| qwen2.5-3b Q4 | 32768 | ~2.3 GB | ~4.8 GB | máximo útil | -| gemma-3-1b Q4 | 32768 | ~1.1 GB | ~2.4 GB | **recomendado** | -| gemma-3-4b Q4 | 8192 | ~2.2 GB | ~5.2 GB | ajustado | +> The estimates in this section predate the measurements below and were made +> on a machine with a GPU. They understate CPU-only RSS by 1–3 GB and their +> "recommended" contexts are far larger than this bot ever uses. Treat the +> **Measured RSS** section as authoritative and this one as the KV-cache +> arithmetic only. + +| Model | context_size | KV cache | Verdict | +|-------------------|-------------:|---------:|--------------------------------| +| qwen2.5-3b Q4 | **4096** | ~290 MB | **the default** — 3.64 GB measured | +| qwen2.5-3b Q4 | 8192 | ~580 MB | +212 MB for headroom never used | +| gemma-3-1b Q4 | 4096 | ~145 MB | cheap, but see the accuracy note | +| gemma-3-4b Q4 | 4096 | ~1.1 GB | untested here | + +### Measured RSS (not estimated) + +**Measure with `--device none`, or the numbers lie.** llama.cpp initialises a +compiled-in GPU backend even with `-ngl 0`. If the build machine has a GPU it +quietly holds the compute buffers, and the RSS you measure is the RSS you will +*not* get on a GPU-less VPS. The gap is not a rounding error: + +| Model | with a GPU present | `--device none` | +|------------------------------|-------------------:|----------------:| +| qwen2.5-3b Q4_K_M | 2.54 GB | **3.66 GB** | +| granite-4.0-h-tiny (7B-A1B) | 4.48 GB | **7.29 GB** | +| gemma-3-1b Q4_K_M | 1.29 GB | **1.05 GB** | + +Granite looked like it fit an 8 GB box and does not. gemma goes the other way — +its compute buffers are tiny either way, so dropping the GPU runtime is a net +saving. + +Real `RSS` on CPU only, `--parallel 1 --threads 2 --mlock`, in steady state +after serving requests: + +| Process | ctx | RSS | +|-------------------------------------------|-----:|---------:| +| qwen2.5-3b Q4_K_M | 8192 | 3.85 GB | +| qwen2.5-3b Q4_K_M | 4096 | 3.64 GB | +| gemma-3-1b Q4_K_M | 8192 | 1.05 GB | +| granite-4.0-h-tiny Q4_K_M | any | 7.25 GB | +| nomic-embed-v2-moe Q5_K_M (`--embedding`) | 2048 | 0.91 GB | +| the Go bot + SQLite | — | 0.02 GB | + +Two things worth noting from that table: + +- **Cold RSS understates it.** qwen at 4096 loads at 3.51 GB and settles at + 3.64 GB after ten requests. Budget from the steady figure. +- **Granite ignores `--ctx-size` entirely** (7.24 GB at 2048, 7.29 GB at 8192). + It is a hybrid Mamba model: the recurrent state is fixed-size, so a 1M-token + window is nearly free — and there is no context lever to pull when it doesn't + fit. + +**Sizing on a shared VPS.** If the box also serves other sites, budget backwards +from what they need. On an 8 GB VPS the full hybrid stack (qwen2.5-3b at 4096 + +embedder + bot) is **4.6 GB**, leaving ~3.4 GB. + +KV quantization buys less than people expect — 130 MB on qwen2.5-3b at 8192 — +because the weights dominate. Reach for a smaller model, or a smaller context, +before reaching for `--cache-type-*`. ### 16 GB VPS -| Model | context_size | KV cache | RAM usada | -|-------------------|-------------:|---------:|----------:| -| qwen2.5-3b Q4 | 32768 | ~2.3 GB | ~5.0 GB | -| gemma-3-4b Q4 | 16384 | ~4.5 GB | ~7.5 GB | +Estimated, not measured — and the same GPU caveat applies, so add 1–2 GB for a +CPU-only host. Contexts this large are also well past anything this bot builds +(1255 tokens measured); they only matter if you repurpose it for long documents. + +| Model | context_size | KV cache | RAM usada (est.) | +|-------------------|-------------:|---------:|-----------------:| +| qwen2.5-3b Q4 | 32768 | ~2.3 GB | ~5.0 GB | +| gemma-3-4b Q4 | 16384 | ~4.5 GB | ~7.5 GB | --- ## 5. Worked examples -### qwen2.5-3b en 8 GB +### qwen2.5-3b on a shared 8 GB VPS — the shipped default ```yaml # configs/portfolio-bot.yaml @@ -118,38 +172,53 @@ providers: type: llamacpp model: qwen2.5-3b-instruct endpoint: http://localhost:9100/v1 - context_size: 16384 # ~1.1 GB KV, deja 4 GB libres - max_tokens: 1024 # respuestas moderadas + context_size: 4096 # 3.64 GB measured; largest real prompt was 1255 tokens + max_tokens: 640 + temperature: 0.7 + top_k: 20 + top_p: 0.8 + repeat_penalty: 1.05 ``` ```bash llama-server \ -m qwen2.5-3b-instruct-q4_k_m.gguf \ - --ctx-size 16384 \ - -ngl 0 -t 2 \ - --mlock + --port 9100 --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 0.7 --top-k 20 --top-p 0.8 --repeat-penalty 1.05 ``` -### gemma-3-1b en 8 GB +Plus the embedder, which has to stay resident because every visitor question +must be embedded before it can be compared: -```yaml -providers: - - name: llamacpp-local - type: llamacpp - model: gemma-3-1b-it - endpoint: http://localhost:9100/v1 - context_size: 32768 # sobra RAM, contexto largo - max_tokens: 1024 +```bash +llama-server \ + -m nomic-embed-v2-moe.Q5_K_M.gguf \ + --port 9200 --embedding --pooling mean \ + --ctx-size 2048 --parallel 1 --device none --threads 2 ``` +Total: 3.64 + 0.91 + 0.02 = **4.57 GB**. + +### gemma-3-1b — cheaper, and why it isn't the default + ```bash llama-server \ -m gemma-3-1b-it-Q4_K_M.gguf \ - --ctx-size 32768 \ - -ngl 0 -t 2 \ - --mlock + --port 9100 --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ + --temp 1.0 --top-k 64 --top-p 0.95 --min-p 0.0 --repeat-penalty 1.15 ``` +1.05 GB instead of 3.64 — 2.8 GB cheaper and ~35% faster. On a 20-question +bilingual battery against the real corpus, with identical hybrid retrieval, it +scored **3/10** on grounded content against qwen's **9/10**, and among its +failures it echoed the system prompt's own instructions back to the visitor. +The sampling flags above matter: Google's published config is `temp 1.0 / +top_k 64 / top_p 0.95 / min_p 0.0`, and `--repeat-penalty 1.15` is deliberately +off-spec because at Google's recommended 1.0 the model looped on the repetitive +shape of the retrieved-chunk headers. + ### Gemma con chat template custom (sin system role líder) Gemma 3 rechaza mensajes `system` antes del primer `user`. Dos opciones: @@ -177,9 +246,8 @@ Gemma 3 rechaza mensajes `system` antes del primer `user`. Dos opciones: ```bash llama-server \ -m gemma-3-1b-it-Q4_K_M.gguf \ - --ctx-size 32768 \ - -ngl 0 -t 2 \ - --mlock \ + --ctx-size 4096 --parallel 1 \ + --device none --threads 2 --mlock \ --chat-template-file ~/.llama/gemma3.jinja ``` @@ -204,9 +272,10 @@ summarized into a single system note. This means: request is expensive; raise it (e.g. `0.9`) when you want to keep more verbatim history. -For a portfolio bot with `context_size: 16384` and `max_tokens: 1024`, -compaction fires when input exceeds ~12k tokens — leaving ~5k for the -fresh history, which is ~10-15 recent user turns. More than enough. +For this bot at `context_size: 4096` and `max_tokens: 640`, compaction fires +when input exceeds ~3070 tokens. Measured over 20 real requests the largest +prompt was 1255 tokens, so in practice it never fires on a single-question +visit — it exists for the chatty visitor who keeps a thread going. --- @@ -218,8 +287,10 @@ fresh history, which is ~10-15 recent user turns. More than enough. bajá el context. 3. **Más contexto ≠ más rápido.** El prefill (procesar el input) escala lineal con la cantidad de tokens. Generación (output) no se ve - afectada. Con `--ctx-size 32768` y un input de 500 tokens, el TTFT - apenas cambia; con 20k tokens de input sí. + afectada. Con `--ctx-size 4096` y un input de 500 tokens, el TTFT apenas + cambia; con 20k tokens de input sí. Medido: bajar de 8192 a 4096 dejó la + generación exactamente igual (21,0 tok/s), porque el prefill escala con los + tokens que procesás, no con los que reservás. 4. **Streams concurrentes.** Cada stream activo reserva su propio KV cache. En 8 GB no hagas más de 1-2 streams simultáneos — el rate limiter del bot (default 30 req/min) ya te protege. @@ -230,14 +301,23 @@ fresh history, which is ~10-15 recent user turns. More than enough. ## 8. Quick-pick table -Copy-paste según tu setup: +Start from what the prompt actually costs, not from what the model can hold. +Measured on this bot over 20 real requests: the largest prompt ever built was +**1255 tokens** (system prompt + project catalogue + 5 retrieved chunks + the +question), median 1069. Compaction begins at `threshold_ratio` × the window, so +4096 leaves 2.4x headroom before it even engages. -| Setup | `context_size` | `max_tokens` | -|-----------------------------|---------------:|-------------:| -| 4 GB + gemma-3-1b | 8192 | 512 | -| 4 GB + qwen2.5-1.5b | 4096 | 512 | -| 8 GB + qwen2.5-1.5b | 16384 | 768 | -| 8 GB + qwen2.5-3b | 16384 | 1024 | -| 8 GB + gemma-3-1b | 32768 | 1024 | -| 16 GB + qwen2.5-3b | 32768 | 1024 | -| 16 GB + gemma-3-4b | 16384 | 1024 | +| Setup | `context_size` | `max_tokens` | Note | +|------------------------------|---------------:|-------------:|------| +| 8 GB shared + qwen2.5-3b | **4096** | **640** | the default; 4.6 GB total stack | +| 8 GB dedicated + qwen2.5-3b | 8192 | 640 | +212 MB, no measured benefit | +| 8 GB + gemma-3-1b | 4096 | 640 | 2.8 GB cheaper, and 3/10 vs 9/10 on grounded answers — see the provider comment in the config | +| 16 GB + qwen2.5-3b | 8192 | 1024 | room for longer threads | + +Going above 8192 for a portfolio bot is reserving memory you will not use. A +bigger window does not make answers better; it makes the KV cache bigger and +delays compaction that was never going to trigger. + +`max_tokens: 640` is sized for the answers this persona is asked to give (2–4 +sentences plus a short list). Raising it takes budget from the input side and +makes compaction fire sooner. diff --git a/internal/agent/client.go b/internal/agent/client.go index 5f552aa..960e84c 100644 --- a/internal/agent/client.go +++ b/internal/agent/client.go @@ -23,7 +23,15 @@ func NewClient(p config.Provider) (llm.LLMClient, error) { Model: p.Model, ContextWindow: p.ContextSize, MaxTokens: p.MaxTokens, - Temperature: p.Temperature, + // Sampling: zero values are omitted from the JSON payload by the + // adapter, so anything left unset in the YAML falls through to + // llama-server's own default rather than being forced to 0. + Temperature: p.Temperature, + TopK: p.TopK, + TopP: p.TopP, + MinP: p.MinP, + RepetitionPenalty: p.RepeatPenalty, + PresencePenalty: p.PresencePenalty, }) case "ollama", "openai": return openai.New(openai.Config{ diff --git a/internal/agent/runner.go b/internal/agent/runner.go index 2c2e441..a08d6d1 100644 --- a/internal/agent/runner.go +++ b/internal/agent/runner.go @@ -23,6 +23,7 @@ import ( "github.com/VictorVargas/rony-llm-agent/pkg/llm" llmpersona "github.com/VictorVargas/rony-llm-agent/pkg/persona" + "github.com/VictorVargas/rony-chat-bot/internal/i18n" botpersona "github.com/VictorVargas/rony-chat-bot/internal/persona" "github.com/VictorVargas/rony-chat-bot/internal/portfolio" ) @@ -50,6 +51,17 @@ type Runner struct { compaction CompactionConfig lastCompact CompactionStats + + // includeCatalog injects the full project list into the system prompt. + // See portfolio.Store.Catalog for why retrieval alone isn't enough. + includeCatalog bool + + // localizedPrompts maps a language code to a full rendition of the + // system prompt. See WithLocalizedPrompt. + localizedPrompts map[string]string + + // embedder enables the semantic half of retrieval. Nil → keyword only. + embedder portfolio.Embedder } type Usage struct { @@ -99,6 +111,109 @@ func (r *Runner) WithCompaction(cfg CompactionConfig) *Runner { return r } +// WithCatalog enables injecting the full project catalogue into the system +// prompt. Returns the receiver for chaining. No-op when RAG is disabled +// (there is no store to read the catalogue from). +func (r *Runner) WithCatalog(enabled bool) *Runner { + r.includeCatalog = enabled + return r +} + +// WithLocalizedPrompt registers a full translation of the system prompt for +// a language code ("es"). When the visitor writes in that language, this text +// replaces the default prompt wholesale. +// +// Translating the prompt beats appending a "reply in Spanish" line to an +// English one: an instruction is a weak signal next to a thousand tokens of +// English telling the model, implicitly, what language it is working in. +// Empty prompts are ignored so an absent YAML key is a no-op. +func (r *Runner) WithLocalizedPrompt(lang, prompt string) *Runner { + if strings.TrimSpace(prompt) == "" { + return r + } + if r.localizedPrompts == nil { + r.localizedPrompts = map[string]string{} + } + r.localizedPrompts[lang] = prompt + return r +} + +// WithEmbedder turns on hybrid retrieval: keyword search fused with vector +// search. Nil is accepted and leaves the runner on keyword-only search, which +// is what makes the embeddings endpoint optional rather than a hard dependency. +func (r *Runner) WithEmbedder(e portfolio.Embedder) *Runner { + r.embedder = e + return r +} + +// promptFor returns the system prompt to use for a detected language, +// falling back to the default when no translation is registered. +func (r *Runner) promptFor(lang string) string { + if p, ok := r.localizedPrompts[lang]; ok { + return p + } + return r.systemPrompt +} + +// detectLanguage returns the language code of the most recent user message, +// or "" when there is nothing to go on. +func detectLanguage(history []Message) string { + for i := len(history) - 1; i >= 0; i-- { + if history[i].Role == RoleUser { + return i18n.Detect(history[i].Content) + } + } + return "" +} + +// languageDirective returns a one-line instruction pinning the reply language, +// or "" when the persona pins no language and the history is empty. +// +// Asking the model to "reply in the user's language" does not survive a 1B +// parameter budget: the rest of the system prompt is English, so English wins +// and Spanish questions come back in English. Detecting the language in Go and +// stating it outright is deterministic and costs one line. +// +// The directive is written *in* the target language on purpose — an +// instruction in Spanish is a much stronger prior for answering in Spanish +// than the same sentence in English. +// +// persona.language acts as the override: the default "the user's language" +// (or empty) means auto-detect, anything else pins that language verbatim. +func (r *Runner) languageDirective(lang string) string { + if pinned := strings.TrimSpace(r.persona.Language); pinned != "" && + !strings.EqualFold(pinned, "the user's language") { + return fmt.Sprintf("Write your entire reply in %s.", pinned) + } + if lang == "" { + return "" + } + if lang == "es" { + return "El visitante escribió en español. Responde ÍNTEGRAMENTE en español, incluido el saludo." + } + return "The visitor wrote in English. Write your entire reply in English." +} + +// catalogBlock renders the project catalogue as a markdown list, or "" when +// the feature is off, RAG is disabled, or the index is empty. A failure to +// read it is logged and swallowed: the catalogue improves grounding but a +// chat turn should never fail because of it. +func (r *Runner) catalogBlock(ctx context.Context) string { + if !r.includeCatalog || r.store == nil { + return "" + } + entries, err := r.store.Catalog(ctx) + if err != nil { + slog.Warn("catalog lookup failed, continuing without it", "err", err) + return "" + } + var b strings.Builder + for _, e := range entries { + fmt.Fprintf(&b, "- **%s** — %s\n", e.ProjectID, e.Title) + } + return strings.TrimRight(b.String(), "\n") +} + // LastUsage returns the token usage recorded on the most recent call. func (r *Runner) LastUsage() Usage { return *r.usage } @@ -116,23 +231,45 @@ func (r *Runner) LastCompaction() CompactionStats { return r.lastCompact } // (Stream calls BuildMessages internally, and handlers sometimes call it // first to extract the RAG context for the sources event). func (r *Runner) BuildMessages(ctx context.Context, history []Message) ([]Message, string, error) { + // Pull any system-role notes (the compactor's "Earlier conversation + // summary") out of the history before anything else: they are context, + // not a conversational turn, and several chat templates reject them as + // one. See foldSystemNotes. + notes, history := foldSystemNotes(history) + + // The visitor's language selects which rendition of the system prompt we + // build on, so it has to be resolved before anything else. + lang := detectLanguage(history) + systemPrompt := r.promptFor(lang) + + // Resolved before retrieval so the excerpt budget accounts for it. + catalog := r.catalogBlock(ctx) + ragContext := "" if r.store != nil && len(history) > 0 { last := history[len(history)-1] if last.Role == RoleUser { - hits, err := r.store.Search(ctx, last.Content, r.topK) + hits, err := r.store.HybridSearch(ctx, r.embedder, last.Content, r.topK) if err != nil { return nil, "", fmt.Errorf("rag search: %w", err) } if len(hits) > 0 { - ragContext = r.limitRAGContext(formatHits(hits), history) + ragContext = r.limitRAGContext(systemPrompt, catalog, formatHits(hits), history) } } } // The system prompt comes from the YAML, not from the persona struct. // Keep the persona around only for the UI greeting. - system := botpersona.BuildSystemPrompt(r.systemPrompt, ragContext) + system := botpersona.BuildSystemPrompt(systemPrompt, catalog, ragContext) + if len(notes) > 0 { + system += "\n\n" + strings.Join(notes, "\n\n") + } + // Last line of the prompt, deliberately: it's the instruction a small + // model is most likely to still be holding when it starts generating. + if d := r.languageDirective(lang); d != "" { + system += "\n\n" + d + } history, err := r.fitHistory(system, history) if err != nil { return nil, "", err @@ -143,6 +280,44 @@ func (r *Runner) BuildMessages(ctx context.Context, history []Message) ([]Messag return msgs, ragContext, nil } +// foldSystemNotes separates system-role messages from the conversational +// turns. Everything the runner puts in the history as a system message is +// really prompt context — today that is only the compactor's summary — so it +// belongs inside the single leading system message rather than in the turn +// list. +// +// This is not cosmetic. Gemma 3's chat template raises "Conversation roles +// must alternate user/assistant/..." on any system message after the first, +// which llama-server surfaces as HTTP 400: before this, enabling compaction +// killed the conversation outright the first time it fired. Anthropic's API +// rejects mid-conversation system turns too, so folding is the portable +// behaviour rather than a Gemma workaround. +// +// The returned slice never aliases the caller's array. +func foldSystemNotes(history []Message) (notes []string, rest []Message) { + hasSystem := false + for _, m := range history { + if m.Role == RoleSystem { + hasSystem = true + break + } + } + if !hasSystem { + return nil, history + } + rest = make([]Message, 0, len(history)) + for _, m := range history { + if m.Role == RoleSystem { + if s := strings.TrimSpace(m.Content); s != "" { + notes = append(notes, s) + } + continue + } + rest = append(rest, m) + } + return notes, rest +} + const maxContextSafetyMargin = 256 func contextBudget(window int) int { @@ -156,7 +331,7 @@ func contextBudget(window int) int { return window - margin } -func (r *Runner) limitRAGContext(ragContext string, history []Message) string { +func (r *Runner) limitRAGContext(systemPrompt, catalog, ragContext string, history []Message) string { if strings.TrimSpace(ragContext) == "" { return "" } @@ -176,7 +351,7 @@ func (r *Runner) limitRAGContext(ragContext string, history []Message) string { if fitted != "" { candidate = fitted + "\n\n" + block } - if estimatePromptTokens(botpersona.BuildSystemPrompt(r.systemPrompt, candidate))+historyTokens > budget { + if estimatePromptTokens(botpersona.BuildSystemPrompt(systemPrompt, catalog, candidate))+historyTokens > budget { break } fitted = candidate @@ -363,10 +538,21 @@ func (r *Runner) Stream(ctx context.Context, history []Message) iter.Seq2[llm.St } } +// formatHits renders retrieved chunks for the prompt. Non-project documents +// are labelled as such: the CV is prime evidence for "does he know X?" but it +// is not a portfolio project, and without the label a small model happily +// announces "cv" as one of Victor's projects. func formatHits(hits []portfolio.SearchResult) string { var b strings.Builder for i, h := range hits { - fmt.Fprintf(&b, "### [%d] %s — %s\n", i+1, h.ProjectID, h.Section) + // The marker goes after the section, never in the name slot: the + // SSE "sources" event parses everything before " — " as the source + // id, and a decorated name would leak into the UI chips. + section := h.Section + if h.Kind == portfolio.KindDoc { + section += " (reference document, not one of Victor's projects)" + } + fmt.Fprintf(&b, "### [%d] %s — %s\n", i+1, h.ProjectID, section) b.WriteString(h.Content) b.WriteString("\n\n") } diff --git a/internal/agent/runner_compaction_test.go b/internal/agent/runner_compaction_test.go index 126513a..dc687ba 100644 --- a/internal/agent/runner_compaction_test.go +++ b/internal/agent/runner_compaction_test.go @@ -423,3 +423,153 @@ func TestTruncateToBudgetAlwaysKeepsLastUserTurn(t *testing.T) { t.Errorf("dropped+kept = %d, want %d", len(dropped)+len(kept), len(history)) } } + +// A compacted history carries the summary as a system message. Several chat +// templates (Gemma 3, Anthropic) reject a system turn that isn't the first +// message, so BuildMessages must fold it into the single leading system +// prompt rather than pass it through as its own turn. Regression test: this +// used to send two system messages and llama-server answered HTTP 400. +func TestBuildMessagesFoldsSystemNotesIntoOneSystemMessage(t *testing.T) { + r := New(&compactionStub{window: 100000}, personaMinimal(), "persona", nil, 5) + + msgs, _, err := r.BuildMessages(context.Background(), []Message{ + {Role: RoleSystem, Content: "Earlier conversation summary:\nuser asked about the dashboard"}, + {Role: RoleUser, Content: "y el stack?"}, + }) + if err != nil { + t.Fatal(err) + } + + for i, m := range msgs { + if i > 0 && m.Role == RoleSystem { + t.Fatalf("msgs[%d] is a second system message; want exactly one, at index 0", i) + } + } + if msgs[0].Role != RoleSystem { + t.Fatalf("msgs[0].Role = %q, want system", msgs[0].Role) + } + if !strings.Contains(msgs[0].Content, "user asked about the dashboard") { + t.Errorf("summary was dropped instead of folded into the system prompt: %q", msgs[0].Content) + } + if !strings.Contains(msgs[0].Content, "persona") { + t.Errorf("folding lost the persona prompt: %q", msgs[0].Content) + } + // The turn list must be the conversation only, and must still alternate. + if len(msgs) != 2 || msgs[1].Role != RoleUser || msgs[1].Content != "y el stack?" { + t.Fatalf("turns = %+v, want [system, user]", msgs) + } +} + +// foldSystemNotes must not write through to the caller's backing array — +// handlers reuse the history slice to persist the conversation. +func TestFoldSystemNotesDoesNotMutateCaller(t *testing.T) { + history := []Message{ + {Role: RoleUser, Content: "one"}, + {Role: RoleSystem, Content: "note"}, + {Role: RoleAssistant, Content: "two"}, + } + before := append([]Message(nil), history...) + + notes, rest := foldSystemNotes(history) + if len(notes) != 1 || notes[0] != "note" { + t.Fatalf("notes = %v, want [note]", notes) + } + if len(rest) != 2 { + t.Fatalf("rest has %d messages, want 2", len(rest)) + } + for i := range before { + if history[i].Role != before[i].Role || history[i].Content != before[i].Content { + t.Errorf("caller's history[%d] mutated: %+v -> %+v", i, before[i], history[i]) + } + } +} + +// A 1B model won't infer the reply language from an all-English system +// prompt, so the runner pins it explicitly from the last user message. +func TestLanguageDirectiveFollowsTheUserAndThePersonaOverride(t *testing.T) { + auto := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "the user's language"}, "persona", nil, 5) + + es := auto.languageDirective(detectLanguage([]Message{{Role: RoleUser, Content: "¿Qué proyectos tiene Victor?"}})) + if !strings.Contains(es, "español") { + t.Errorf("Spanish question got directive %q, want a Spanish one", es) + } + en := auto.languageDirective(detectLanguage([]Message{{Role: RoleUser, Content: "What projects does he have?"}})) + if !strings.Contains(en, "English") { + t.Errorf("English question got directive %q, want an English one", en) + } + // Detection must follow the *latest* user turn, not the first. + switched := auto.languageDirective(detectLanguage([]Message{ + {Role: RoleUser, Content: "What projects does he have?"}, + {Role: RoleAssistant, Content: "..."}, + {Role: RoleUser, Content: "¿Y cuál usa Stripe?"}, + })) + if !strings.Contains(switched, "español") { + t.Errorf("after switching to Spanish got %q, want a Spanish directive", switched) + } + if auto.languageDirective(detectLanguage(nil)) != "" { + t.Errorf("empty history should produce no directive") + } + + pinned := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "Spanish"}, "persona", nil, 5) + if got := pinned.languageDirective(detectLanguage([]Message{{Role: RoleUser, Content: "hello there"}})); got != "Write your entire reply in Spanish." { + t.Errorf("persona override ignored: %q", got) + } +} + +// The directive must be the final line of the system prompt — that position +// is why it survives. +func TestLanguageDirectiveIsLastInSystemPrompt(t *testing.T) { + r := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "the user's language"}, "persona", nil, 5) + msgs, _, err := r.BuildMessages(context.Background(), []Message{ + {Role: RoleSystem, Content: "Earlier conversation summary:\nalgo"}, + {Role: RoleUser, Content: "¿Qué stack usa el dashboard?"}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(strings.TrimSpace(msgs[0].Content), + "El visitante escribió en español. Responde ÍNTEGRAMENTE en español, incluido el saludo.") { + t.Errorf("language directive is not the last line of the system prompt:\n%s", msgs[0].Content) + } +} + +// A Spanish question must build on the Spanish rendition of the prompt, not +// the default one with a directive bolted on. +func TestLocalizedPromptSelectedByDetectedLanguage(t *testing.T) { + r := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "the user's language"}, "ENGLISH PROMPT", nil, 5). + WithLocalizedPrompt("es", "PROMPT EN ESPAÑOL") + + es, _, err := r.BuildMessages(context.Background(), + []Message{{Role: RoleUser, Content: "¿Qué proyectos tiene Victor?"}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(es[0].Content, "PROMPT EN ESPAÑOL") { + t.Errorf("Spanish question did not select the Spanish prompt:\n%s", es[0].Content) + } + + en, _, err := r.BuildMessages(context.Background(), + []Message{{Role: RoleUser, Content: "What projects does he have?"}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(en[0].Content, "ENGLISH PROMPT") { + t.Errorf("English question did not select the default prompt:\n%s", en[0].Content) + } + + // No translation registered → fall back, never blank. + bare := New(&compactionStub{window: 100000}, + llmpersona.Persona{Language: "the user's language"}, "ENGLISH PROMPT", nil, 5) + msgs, _, err := bare.BuildMessages(context.Background(), + []Message{{Role: RoleUser, Content: "¿Y esto?"}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(msgs[0].Content, "ENGLISH PROMPT") { + t.Errorf("missing translation should fall back to the default prompt:\n%s", msgs[0].Content) + } +} diff --git a/internal/agent/runner_test.go b/internal/agent/runner_test.go index b6e1a6f..5659319 100644 --- a/internal/agent/runner_test.go +++ b/internal/agent/runner_test.go @@ -69,7 +69,7 @@ func TestRunnerStreamWithRAG(t *testing.T) { t.Fatal(err) } defer store.Close() - if _, _, err := store.Reindex(context.Background(), srcDir, portfolio.DefaultChunkerConfig()); err != nil { + if _, _, err := store.Reindex(context.Background(), portfolio.SourcesFor(srcDir, ""), portfolio.DefaultChunkerConfig()); err != nil { t.Fatal(err) } @@ -100,7 +100,7 @@ func TestLimitRAGContextToWindow(t *testing.T) { first := "### [1] first — section\n" + strings.Repeat("x", 100) second := "### [2] second — section\n" + strings.Repeat("y", 1600) - got := r.limitRAGContext(first+"\n\n"+second, history) + got := r.limitRAGContext("sys", "", first+"\n\n"+second, history) if !strings.Contains(got, "first") { t.Errorf("limited RAG context dropped the first result: %q", got) } diff --git a/internal/config/config.go b/internal/config/config.go index 7eeb03d..3c3d322 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,36 +21,75 @@ type RateLimit struct { Burst int `yaml:"burst"` } +// Provider is one upstream LLM. The sampling fields (Temperature through +// PresencePenalty) are forwarded verbatim to the llama.cpp adapter; a zero +// value means "don't send it", so llama-server's own default applies. They +// matter more than they look: small instruct models ship with vendor- +// recommended sampling (Gemma 3 wants temp 1.0 / top_k 64 / top_p 0.95) and +// drifting off it makes them terse and repetitive. type Provider struct { Name string `yaml:"name"` Type string `yaml:"type"` Model string `yaml:"model,omitempty"` - ModelPath string `yaml:"model_path,omitempty"` Endpoint string `yaml:"endpoint,omitempty"` ContextSize int `yaml:"context_size,omitempty"` MaxTokens int `yaml:"max_tokens,omitempty"` - NGPULayers int `yaml:"n_gpu_layers,omitempty"` Temperature float32 `yaml:"temperature,omitempty"` - APIKeyEnv string `yaml:"api_key_env,omitempty"` - Default bool `yaml:"default,omitempty"` + TopK int `yaml:"top_k,omitempty"` + TopP float32 `yaml:"top_p,omitempty"` + MinP float32 `yaml:"min_p,omitempty"` + // RepeatPenalty maps to llama.cpp's repeat_penalty. Leave unset (0) for + // Gemma 3 — Google's recommended config is no repetition penalty at all, + // and a value >1 visibly degrades its prose. + RepeatPenalty float32 `yaml:"repeat_penalty,omitempty"` + PresencePenalty float32 `yaml:"presence_penalty,omitempty"` + APIKeyEnv string `yaml:"api_key_env,omitempty"` + Default bool `yaml:"default,omitempty"` } type RAG struct { - Enabled bool `yaml:"enabled"` - DataPath string `yaml:"data_path"` - ChunkSize int `yaml:"chunk_size"` - ChunkOverlap int `yaml:"chunk_overlap"` - DBPath string `yaml:"db_path"` - TopK int `yaml:"top_k"` - Tokenize string `yaml:"tokenize"` + Enabled bool `yaml:"enabled"` + DataPath string `yaml:"data_path"` + // DocsPath is an optional second directory of markdown that is indexed + // and searchable but is NOT part of the project catalogue: a CV, an about + // page, a FAQ. Without it the only way to make the CV retrievable is to + // drop it in data_path, where it then gets announced as one of Victor's + // projects. Both directories accept .md and .mdx. + DocsPath string `yaml:"docs_path"` + DBPath string `yaml:"db_path"` + TopK int `yaml:"top_k"` + Tokenize string `yaml:"tokenize"` + // IncludeCatalog injects the full list of indexed projects into the + // system prompt on every turn. Costs a few tokens per project and stops + // small models from inventing project names when asked to enumerate — + // top-K retrieval can't answer "list everything" by construction. + IncludeCatalog bool `yaml:"include_catalog"` +} + +// Embeddings configures the semantic half of retrieval. Disabled by default, +// in which case the bot uses keyword search only. +// +// Run the endpoint with: +// +// llama-server -m nomic-embed-v2-moe.Q5_K_M.gguf --port 9200 \ +// --embedding --pooling mean --ctx-size 2048 --parallel 1 \ +// --device none --threads 2 +// +// Measured at 0.61 GB RSS on CPU. It has to stay resident: the corpus is +// embedded once at index time, but every visitor question must be embedded +// before it can be compared. +type Embeddings struct { + Enabled bool `yaml:"enabled"` + Endpoint string `yaml:"endpoint"` + Model string `yaml:"model,omitempty"` + BatchSize int `yaml:"batch_size,omitempty"` + TimeoutMS int `yaml:"timeout_ms,omitempty"` } type Persona struct { - Name string `yaml:"name"` - Tone string `yaml:"tone"` - Language string `yaml:"language"` - Constraints []string `yaml:"constraints"` - Intro string `yaml:"intro"` + Name string `yaml:"name"` + Tone string `yaml:"tone"` + Language string `yaml:"language"` } type Logging struct { @@ -90,8 +129,21 @@ type Config struct { Server Server `yaml:"server"` Providers []Provider `yaml:"providers"` RAG RAG `yaml:"rag"` + Embeddings Embeddings `yaml:"embeddings"` Persona Persona `yaml:"persona"` SystemPrompt string `yaml:"system_prompt"` + // SystemPromptES is the Spanish rendition of SystemPrompt, used when the + // visitor writes in Spanish. Optional; empty falls back to SystemPrompt. + // + // This is not a convenience — it's the only thing that reliably keeps a + // small model answering in Spanish. An instruction like "reply in the + // user's language" buried in an otherwise English prompt loses to the + // sheer mass of English around it: measured on gemma-3-1b, 1 of 5 Spanish + // questions came back in Spanish. Translating the prompt itself took that + // to 4 of 5. (Few-shot Spanish examples also score well, but a 1B model + // copies them verbatim instead of answering — see the note on + // `system_prompt`.) + SystemPromptES string `yaml:"system_prompt_es"` Compaction Compaction `yaml:"compaction"` Logging Logging `yaml:"logging"` } @@ -136,12 +188,6 @@ func (c *Config) validate() error { if c.Server.ReadTimeoutMS == 0 { c.Server.ReadTimeoutMS = 30000 } - if c.RAG.ChunkSize == 0 { - c.RAG.ChunkSize = 500 - } - if c.RAG.ChunkOverlap == 0 { - c.RAG.ChunkOverlap = 50 - } if c.RAG.TopK == 0 { c.RAG.TopK = 5 } diff --git a/internal/embed/embed.go b/internal/embed/embed.go new file mode 100644 index 0000000..3bf9e47 --- /dev/null +++ b/internal/embed/embed.go @@ -0,0 +1,186 @@ +// Package embed talks to an OpenAI-compatible /v1/embeddings endpoint +// (llama-server --embedding) and provides the vector maths the RAG store +// needs. +// +// Why this exists: the SQLite FTS5 index matches words exactly. Visitors ask +// in Spanish about a portfolio written in English, so the terms that carry the +// meaning — "paga", "trabajado" — appear zero times in the corpus, which says +// "Payments: Stripe" and "worked". Measured on the real corpus, keyword search +// returned nothing at all for "¿Con qué se paga en la tienda de ropa?" while a +// multilingual embedding put all three tienda-ropa chunks on top. Keyword +// search still wins on exact proper nouns, so the store keeps both and fuses +// them. +package embed + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "time" +) + +// Config describes the embedding endpoint. +type Config struct { + BaseURL string // e.g. http://localhost:9200/v1 + Model string // sent as "model"; llama-server ignores it + BatchSize int // texts per request (0 → 8) + TimeoutMS int // per-request timeout (0 → 120s) +} + +// Client is a minimal embeddings client. +type Client struct { + baseURL string + model string + batchSize int + http *http.Client +} + +func New(cfg Config) *Client { + batch := cfg.BatchSize + if batch <= 0 { + batch = 8 + } + timeout := time.Duration(cfg.TimeoutMS) * time.Millisecond + if timeout <= 0 { + timeout = 120 * time.Second + } + return &Client{ + baseURL: cfg.BaseURL, + model: cfg.Model, + batchSize: batch, + http: &http.Client{Timeout: timeout}, + } +} + +type embedRequest struct { + Model string `json:"model,omitempty"` + Input []string `json:"input"` +} + +type embedResponse struct { + Data []struct { + Index int `json:"index"` + Embedding []float32 `json:"embedding"` + } `json:"data"` +} + +// Embed returns one unit-length vector per input, in input order. +// +// Vectors are normalised here, once, so similarity at query time is a plain +// dot product instead of a cosine with two square roots per candidate. +func (c *Client) Embed(ctx context.Context, texts []string) ([][]float32, error) { + out := make([][]float32, 0, len(texts)) + for start := 0; start < len(texts); start += c.batchSize { + end := min(start+c.batchSize, len(texts)) + vecs, err := c.embedBatch(ctx, texts[start:end]) + if err != nil { + return nil, err + } + out = append(out, vecs...) + } + return out, nil +} + +func (c *Client) embedBatch(ctx context.Context, texts []string) ([][]float32, error) { + body, err := json.Marshal(embedRequest{Model: c.model, Input: texts}) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/embeddings", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("embeddings request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + msg, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("embeddings API %d: %s", resp.StatusCode, string(msg)) + } + + var parsed embedResponse + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, fmt.Errorf("decode embeddings: %w", err) + } + if len(parsed.Data) != len(texts) { + return nil, fmt.Errorf("embeddings returned %d vectors for %d inputs", len(parsed.Data), len(texts)) + } + + // The API documents an `index` field rather than guaranteeing order. + out := make([][]float32, len(texts)) + for _, d := range parsed.Data { + if d.Index < 0 || d.Index >= len(out) { + return nil, fmt.Errorf("embeddings returned out-of-range index %d", d.Index) + } + out[d.Index] = Normalize(d.Embedding) + } + for i, v := range out { + if v == nil { + return nil, fmt.Errorf("embeddings response missing index %d", i) + } + } + return out, nil +} + +// Normalize scales v to unit length. A zero vector is returned unchanged — +// dividing by zero would poison every later comparison with NaN. +func Normalize(v []float32) []float32 { + var sum float64 + for _, x := range v { + sum += float64(x) * float64(x) + } + if sum == 0 { + return v + } + inv := float32(1 / math.Sqrt(sum)) + out := make([]float32, len(v)) + for i, x := range v { + out[i] = x * inv + } + return out +} + +// Similarity is the dot product, which equals cosine similarity for the +// unit-length vectors this package produces. Mismatched lengths score 0 so a +// stale row from a different embedding model can never outrank a real hit. +func Similarity(a, b []float32) float64 { + if len(a) != len(b) || len(a) == 0 { + return 0 + } + var sum float64 + for i := range a { + sum += float64(a[i]) * float64(b[i]) + } + return sum +} + +// Encode serialises a vector as little-endian float32 for a SQLite BLOB. +func Encode(v []float32) []byte { + buf := make([]byte, 4*len(v)) + for i, x := range v { + binary.LittleEndian.PutUint32(buf[4*i:], math.Float32bits(x)) + } + return buf +} + +// Decode reverses Encode. A blob whose length isn't a multiple of 4 is +// corrupt and yields nil rather than a truncated vector. +func Decode(b []byte) []float32 { + if len(b)%4 != 0 { + return nil + } + out := make([]float32, len(b)/4) + for i := range out { + out[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[4*i:])) + } + return out +} diff --git a/internal/embed/embed_test.go b/internal/embed/embed_test.go new file mode 100644 index 0000000..0cf88a2 --- /dev/null +++ b/internal/embed/embed_test.go @@ -0,0 +1,111 @@ +package embed + +import ( + "context" + "encoding/json" + "math" + "net/http" + "net/http/httptest" + "testing" +) + +func TestNormalizeProducesUnitVectors(t *testing.T) { + got := Normalize([]float32{3, 4}) + if math.Abs(float64(got[0])-0.6) > 1e-6 || math.Abs(float64(got[1])-0.8) > 1e-6 { + t.Errorf("Normalize([3,4]) = %v, want [0.6 0.8]", got) + } + // A zero vector must survive untouched — dividing by its length would + // put NaN into every later comparison. + zero := []float32{0, 0, 0} + if out := Normalize(zero); out[0] != 0 || math.IsNaN(float64(out[0])) { + t.Errorf("Normalize(zero) = %v, want zeros", out) + } +} + +func TestSimilarity(t *testing.T) { + a := Normalize([]float32{1, 0}) + if s := Similarity(a, a); math.Abs(s-1) > 1e-6 { + t.Errorf("self-similarity = %v, want 1", s) + } + if s := Similarity(a, Normalize([]float32{0, 1})); math.Abs(s) > 1e-6 { + t.Errorf("orthogonal similarity = %v, want 0", s) + } + // A vector from a different model must never outrank a real hit. + if s := Similarity(a, []float32{1, 0, 0}); s != 0 { + t.Errorf("mismatched dimensions scored %v, want 0", s) + } + if s := Similarity(nil, nil); s != 0 { + t.Errorf("empty vectors scored %v, want 0", s) + } +} + +func TestEncodeDecodeRoundTrip(t *testing.T) { + in := []float32{0.5, -0.25, 1e-8, 12345.75} + out := Decode(Encode(in)) + if len(out) != len(in) { + t.Fatalf("round trip changed length: %d → %d", len(in), len(out)) + } + for i := range in { + if in[i] != out[i] { + t.Errorf("round trip [%d]: %v → %v", i, in[i], out[i]) + } + } + if got := Decode([]byte{1, 2, 3}); got != nil { + t.Errorf("Decode of a truncated blob = %v, want nil", got) + } +} + +// The API returns an `index` per item rather than promising input order, so +// the client must reorder — otherwise chunk N gets chunk M's vector and every +// later search is quietly wrong. +func TestEmbedReordersByIndexAndNormalizes(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req embedRequest + _ = json.NewDecoder(r.Body).Decode(&req) + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []map[string]any{ + {"index": 1, "embedding": []float32{0, 5}}, + {"index": 0, "embedding": []float32{3, 4}}, + }, + }) + })) + defer srv.Close() + + vecs, err := New(Config{BaseURL: srv.URL}).Embed(context.Background(), []string{"first", "second"}) + if err != nil { + t.Fatal(err) + } + if len(vecs) != 2 { + t.Fatalf("got %d vectors, want 2", len(vecs)) + } + if math.Abs(float64(vecs[0][0])-0.6) > 1e-6 { + t.Errorf("vecs[0] = %v, want the index-0 item normalized ([0.6 0.8])", vecs[0]) + } + if math.Abs(float64(vecs[1][1])-1) > 1e-6 { + t.Errorf("vecs[1] = %v, want the index-1 item normalized ([0 1])", vecs[1]) + } +} + +func TestEmbedRejectsShortResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": []map[string]any{{"index": 0, "embedding": []float32{1, 0}}}, + }) + })) + defer srv.Close() + + if _, err := New(Config{BaseURL: srv.URL}).Embed(context.Background(), []string{"a", "b"}); err == nil { + t.Fatal("want an error when the endpoint returns fewer vectors than inputs") + } +} + +func TestEmbedSurfacesHTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "model not loaded", http.StatusServiceUnavailable) + })) + defer srv.Close() + + if _, err := New(Config{BaseURL: srv.URL}).Embed(context.Background(), []string{"a"}); err == nil { + t.Fatal("want an error on a 503") + } +} diff --git a/internal/persona/persona.go b/internal/persona/persona.go index 2feffa2..ef9ffe4 100644 --- a/internal/persona/persona.go +++ b/internal/persona/persona.go @@ -27,13 +27,24 @@ func FromConfig(c *config.Config) (persona.Persona, error) { } // BuildSystemPrompt returns the full prompt for one chat turn: -// 1. The hand-written system prompt from the YAML (who Rony is, how to speak) -// 2. The RAG block (omitted when the index returns no hits) +// 1. The hand-written system prompt from the YAML (who Rony is, how to speak) +// 2. The project catalogue (omitted when empty) +// 3. The RAG block (omitted when the index returns no hits) // -// The RAG block is appended, not prepended, so the persona instructions -// always come first and the LLM never gets the chance to "forget" them. -func BuildSystemPrompt(systemPrompt, ragContext string) string { +// The catalogue and RAG block are appended, not prepended, so the persona +// instructions always come first and the LLM never gets the chance to +// "forget" them. +// +// The catalogue goes before the excerpts on purpose: it is the closed set the +// model is allowed to name, and stating the boundary before showing the +// evidence measurably cuts invented project names on small models. +func BuildSystemPrompt(systemPrompt, catalog, ragContext string) string { out := systemPrompt + if catalog != "" { + out += "\n\n## Victor's complete project catalogue\n\n" + catalog + + "\n\nThat is every project that exists. A name outside this list is not a real " + + "project of Victor's and you must never use one." + } if ragContext != "" { out += "\n\n## Relevant context from the portfolio\n\n" + "Use these excerpts to answer. Cite the project filename when you reference a detail. " + diff --git a/internal/portfolio/chunker.go b/internal/portfolio/chunker.go index 404f84b..3e36c8e 100644 --- a/internal/portfolio/chunker.go +++ b/internal/portfolio/chunker.go @@ -137,6 +137,43 @@ func dropEmpty(sections []section) []section { // subSplit breaks down any section whose body exceeds max into overlapping // slices, preserving the heading as a prefix on each piece so context isn't // lost mid-section. +var h3Re = regexp.MustCompile(`(?m)^### +(.+)$`) + +// splitByH3 breaks an oversized section at its H3 boundaries, if it has any. +// The sub-heading is folded into the section name ("Experience — Metrimex") +// so the piece still says what it is once it's out of context. +// +// This exists because character splitting mangles exactly the content that +// matters most. A CV's Experience section is a list of jobs; splitting it by +// size cut one entry mid-word, producing a chunk that began "... id app for +// an on-demand ride-sharing service". A chunk like that matches no question, +// and the employer name it belonged to was stranded in the previous piece. +// Splitting per job keeps the company, the role and the dates together. +func splitByH3(s section) ([]section, bool) { + locs := h3Re.FindAllStringSubmatchIndex(s.Body, -1) + if len(locs) < 2 { + return nil, false + } + var out []section + // Text before the first H3 (a lead-in paragraph) stays with the parent. + if lead := strings.TrimSpace(s.Body[:locs[0][0]]); lead != "" { + out = append(out, section{Heading: s.Heading, Body: lead}) + } + for i, loc := range locs { + end := len(s.Body) + if i+1 < len(locs) { + end = locs[i+1][0] + } + title := strings.TrimSpace(s.Body[loc[2]:loc[3]]) + body := strings.TrimSpace(s.Body[loc[0]:end]) + if body == "" { + continue + } + out = append(out, section{Heading: s.Heading + " — " + title, Body: body}) + } + return out, len(out) > 0 +} + func subSplit(sections []section, max, overlap int) []section { if max <= 0 { return sections @@ -150,6 +187,11 @@ func subSplit(sections []section, max, overlap int) []section { out = append(out, s) continue } + // Prefer semantic boundaries over byte offsets. + if pieces, ok := splitByH3(s); ok { + out = append(out, pieces...) + continue + } for i := 0; i < len(s.Body); { end := i + max if end > len(s.Body) { diff --git a/internal/portfolio/chunker_test.go b/internal/portfolio/chunker_test.go index 465acd0..16c1fd7 100644 --- a/internal/portfolio/chunker_test.go +++ b/internal/portfolio/chunker_test.go @@ -108,4 +108,48 @@ func equalSlice(a, b []string) bool { } } return true -} \ No newline at end of file +} +// An oversized section made of H3 entries (a CV's Experience list, a FAQ) +// must split per entry, not per byte. Size-splitting used to cut a job entry +// mid-word and strand the employer name in the previous chunk. +func TestSubSplitPrefersH3BoundariesOverByteOffsets(t *testing.T) { + body := "### Metrimex — Frontend Developer\n\n#### Jul 2024 – Jun 2026\n\n" + + strings.Repeat("Built an access-control app for physical sites. ", 12) + + "\n\n### Didcom — Android Developer\n\n#### Jul 2023 – Jul 2024\n\n" + + strings.Repeat("Built an Android app for an on-demand ride-sharing service. ", 12) + + "\n\n### Teknol — Software Developer\n\n#### Jun 2016 – Jan 2017\n\n" + + strings.Repeat("Maintained an internal tool. ", 12) + + got := subSplit([]section{{Heading: "Experience", Body: body}}, 400, 40) + if len(got) != 3 { + t.Fatalf("got %d chunks, want one per job:\n%+v", len(got), got) + } + for i, want := range []string{"Metrimex", "Didcom", "Teknol"} { + if !strings.Contains(got[i].Heading, want) { + t.Errorf("chunk %d heading = %q, want it to name %s", i, got[i].Heading, want) + } + if !strings.HasPrefix(got[i].Body, "### "+want) { + t.Errorf("chunk %d body should start at its own heading, got %.40q", i, got[i].Body) + } + } + // No chunk may begin mid-sentence — that is the defect this replaces. + for i, s := range got { + if strings.HasPrefix(s.Body, "... ") { + t.Errorf("chunk %d still starts with a byte-split continuation: %.40q", i, s.Body) + } + } +} + +// A long section with no H3 structure still falls back to size splitting. +func TestSubSplitFallsBackToSizeWithoutH3(t *testing.T) { + body := strings.Repeat("plain prose with no subheadings at all. ", 40) + got := subSplit([]section{{Heading: "Description", Body: body}}, 300, 30) + if len(got) < 2 { + t.Fatalf("expected the oversized section to be split, got %d", len(got)) + } + for _, s := range got { + if s.Heading != "Description" { + t.Errorf("size-split chunk changed heading to %q", s.Heading) + } + } +} diff --git a/internal/portfolio/hybrid.go b/internal/portfolio/hybrid.go new file mode 100644 index 0000000..8640c86 --- /dev/null +++ b/internal/portfolio/hybrid.go @@ -0,0 +1,238 @@ +package portfolio + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "log/slog" + "sort" + + "github.com/VictorVargas/rony-chat-bot/internal/embed" +) + +// Embedder is the subset of embed.Client the store needs. Keeping it an +// interface lets Reindex and HybridSearch be tested without an endpoint. +type Embedder interface { + Embed(ctx context.Context, texts []string) ([][]float32, error) +} + +// rrfK is the usual Reciprocal Rank Fusion constant. RRF combines rankings +// rather than scores, which matters here because BM25 (unbounded, negative) +// and cosine similarity (0–1) are not comparable on any common scale. k=60 +// is the value from the original paper and is not sensitive enough to be +// worth tuning for a corpus this size. +const rrfK = 60.0 + +// embedText is the exact string that gets embedded for a chunk, and the only +// definition of it. EmbedChunks and VectorSearch both go through this so the +// hash they compare can never be computed over different text. +// +// The project and heading are prepended because a bare body is ambiguous out +// of context: "Tech stack: Next.js, Stripe" says nothing about which project, +// and the question naming the shop needs to land on it. +func embedText(projectID, section, content string) string { + return projectID + " — " + section + "\n" + content +} + +func hashOf(text string) string { + sum := sha256.Sum256([]byte(text)) + return hex.EncodeToString(sum[:]) +} + +// VectorSearch returns the topK chunks whose embedding is closest to the +// query, scanning every stored vector. Chunks indexed before embeddings were +// enabled simply have no row and are skipped. +func (s *Store) VectorSearch(ctx context.Context, queryVec []float32, topK int) ([]SearchResult, error) { + if len(queryVec) == 0 { + return nil, nil + } + if topK <= 0 { + topK = 5 + } + + rows, err := s.db.QueryContext(ctx, ` + SELECT c.id, c.project_id, c.kind, c.source_file, c.section, c.chunk_index, c.content, + v.vec, v.content_hash + FROM portfolio_vectors v + JOIN portfolio_chunks c ON c.id = v.chunk_id + WHERE v.dim = ? AND c.section <> 'frontmatter'`, len(queryVec)) + if err != nil { + return nil, fmt.Errorf("vector search: %w", err) + } + defer rows.Close() + + var hits []SearchResult + var stale int + for rows.Next() { + var r SearchResult + var blob []byte + var storedHash string + if err := rows.Scan(&r.ID, &r.ProjectID, &r.Kind, &r.SourceFile, &r.Section, &r.Index, + &r.Content, &blob, &storedHash); err != nil { + return nil, err + } + // Skip vectors describing a previous version of this chunk. Chunk + // ids survive body edits, so without this the row would be scored + // against text that is no longer there — quietly, with no error. + // The chunk stays reachable through keyword search meanwhile. + if storedHash != hashOf(embedText(r.ProjectID, r.Section, r.Content)) { + stale++ + continue + } + r.Score = embed.Similarity(queryVec, embed.Decode(blob)) + hits = append(hits, r) + } + if stale > 0 { + slog.Warn("ignoring stale embeddings; re-run reindex to refresh them", "chunks", stale) + } + if err := rows.Err(); err != nil { + return nil, err + } + + // Higher similarity is better, unlike the bm25() score used by Search. + sort.Slice(hits, func(i, j int) bool { return hits[i].Score > hits[j].Score }) + return hits[:min(topK, len(hits))], nil +} + +// HybridSearch fuses keyword and vector retrieval with Reciprocal Rank +// Fusion and returns the topK chunks. +// +// Neither retriever is sufficient alone on this corpus. Keyword search finds +// exact proper nouns ("Kubernetes", "Stripe") that embeddings can blur, but +// returns nothing when a Spanish question meets an English document. Vectors +// bridge the languages but rank a request for a specific rare token less +// sharply. RRF asks only for each side's ordering, so a chunk that both +// retrievers like rises above one that only a single retriever loved. +// +// When embedding fails — the endpoint is down, or none is configured — this +// degrades to plain keyword search rather than failing the request: a +// keyword-only answer beats no answer. +func (s *Store) HybridSearch(ctx context.Context, embedder Embedder, query string, topK int) ([]SearchResult, error) { + if topK <= 0 { + topK = 5 + } + // Over-fetch from each side: a chunk ranked 8th by one retriever and 2nd + // by the other should still be able to win the fusion. + pool := topK * 3 + + keyword, kwErr := s.Search(ctx, query, pool) + if embedder == nil { + return keyword, kwErr + } + + vecs, err := embedder.Embed(ctx, []string{query}) + if err != nil || len(vecs) == 0 { + if kwErr != nil { + return nil, fmt.Errorf("both retrievers failed: keyword: %v; embedding: %w", kwErr, err) + } + return keyword, nil + } + semantic, vecErr := s.VectorSearch(ctx, vecs[0], pool) + if vecErr != nil && kwErr != nil { + return nil, fmt.Errorf("both retrievers failed: keyword: %v; vector: %w", kwErr, vecErr) + } + + type fused struct { + res SearchResult + score float64 + } + byID := map[string]*fused{} + add := func(list []SearchResult) { + for rank, r := range list { + f, ok := byID[r.ID] + if !ok { + f = &fused{res: r} + byID[r.ID] = f + } + f.score += 1 / (rrfK + float64(rank+1)) + } + } + add(keyword) + add(semantic) + + out := make([]fused, 0, len(byID)) + for _, f := range byID { + out = append(out, *f) + } + sort.Slice(out, func(i, j int) bool { + if out[i].score != out[j].score { + return out[i].score > out[j].score + } + return out[i].res.ID < out[j].res.ID // stable across runs + }) + + results := make([]SearchResult, 0, min(topK, len(out))) + for _, f := range out[:min(topK, len(out))] { + f.res.Score = f.score + results = append(results, f.res) + } + return results, nil +} + +// EmbedChunks computes and stores a vector for every indexed chunk. Called +// after Reindex, since it needs the chunks to exist. +// +// The heading is prepended to each chunk's text: a bare "Tech stack" body is +// a list of technologies with no hint of which project it belongs to, and the +// embedding of "tienda-ropa — Tech stack: Next.js, Stripe…" is much closer to +// a question naming the shop. +func (s *Store) EmbedChunks(ctx context.Context, embedder Embedder) (int, error) { + if embedder == nil { + return 0, nil + } + rows, err := s.db.QueryContext(ctx, + `SELECT id, project_id, section, content FROM portfolio_chunks ORDER BY id`) + if err != nil { + return 0, fmt.Errorf("read chunks for embedding: %w", err) + } + var ids, texts []string + for rows.Next() { + var id, project, section, content string + if err := rows.Scan(&id, &project, §ion, &content); err != nil { + rows.Close() + return 0, err + } + ids = append(ids, id) + texts = append(texts, embedText(project, section, content)) + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + if len(ids) == 0 { + return 0, nil + } + + vecs, err := embedder.Embed(ctx, texts) + if err != nil { + return 0, fmt.Errorf("embed chunks: %w", err) + } + if len(vecs) != len(ids) { + return 0, fmt.Errorf("embedder returned %d vectors for %d chunks", len(vecs), len(ids)) + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `DELETE FROM portfolio_vectors`); err != nil { + return 0, err + } + stmt, err := tx.PrepareContext(ctx, + `INSERT INTO portfolio_vectors (chunk_id, content_hash, dim, vec) VALUES (?,?,?,?)`) + if err != nil { + return 0, err + } + defer stmt.Close() + for i, id := range ids { + if _, err := stmt.ExecContext(ctx, id, hashOf(texts[i]), len(vecs[i]), embed.Encode(vecs[i])); err != nil { + return 0, fmt.Errorf("store vector %s: %w", id, err) + } + } + if err := tx.Commit(); err != nil { + return 0, err + } + return len(ids), nil +} diff --git a/internal/portfolio/hybrid_test.go b/internal/portfolio/hybrid_test.go new file mode 100644 index 0000000..4de3b2f --- /dev/null +++ b/internal/portfolio/hybrid_test.go @@ -0,0 +1,243 @@ +package portfolio + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/VictorVargas/rony-chat-bot/internal/embed" +) + +// stubEmbedder returns a fixed vector per exact text, so a test can decide +// which chunk a query should land on without running a model. +type stubEmbedder struct { + vecs map[string][]float32 + err error +} + +func (s *stubEmbedder) Embed(_ context.Context, texts []string) ([][]float32, error) { + if s.err != nil { + return nil, s.err + } + out := make([][]float32, len(texts)) + for i, t := range texts { + if v, ok := s.vecs[t]; ok { + out[i] = embed.Normalize(v) + continue + } + out[i] = embed.Normalize([]float32{1, 1, 1}) // neutral + } + return out, nil +} + +func newTestStore(t *testing.T) *Store { + t.Helper() + dir := t.TempDir() + src := filepath.Join(dir, "projects") + if err := os.MkdirAll(src, 0o755); err != nil { + t.Fatal(err) + } + docs := map[string]string{ + "tienda.md": "# Tienda\n\n## Tech stack\n\nNext.js and Stripe for payments.\n", + "dash.md": "# Dashboard\n\n## Tech stack\n\nGo backend with WebSockets and D3.\n", + } + for name, body := range docs { + if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + store, err := OpenStore(filepath.Join(dir, "t.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + if _, _, err := store.Reindex(context.Background(), SourcesFor(src, ""), DefaultChunkerConfig()); err != nil { + t.Fatal(err) + } + return store +} + +// The point of the whole feature: a query whose words appear nowhere in the +// corpus still finds the right chunk, because the vector matches. +func TestHybridSearchFindsChunkKeywordSearchCannot(t *testing.T) { + store := newTestStore(t) + + // "¿Cómo se paga?" shares no word with "Next.js and Stripe for payments". + query := "¿Cómo se paga?" + target := []float32{1, 0, 0} + emb := &stubEmbedder{vecs: map[string][]float32{query: target}} + + // Embed the corpus so the Stripe chunk owns the target direction. + corpus := &stubEmbedder{vecs: map[string][]float32{}} + rows, err := store.db.Query(`SELECT id, project_id, section, content FROM portfolio_chunks`) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var id, project, section, content string + if err := rows.Scan(&id, &project, §ion, &content); err != nil { + t.Fatal(err) + } + v := []float32{0, 1, 0} + if project == "tienda" && section == "Tech stack" { + v = target + } + corpus.vecs[project+" — "+section+"\n"+content] = v + } + rows.Close() + + if _, err := store.EmbedChunks(context.Background(), corpus); err != nil { + t.Fatal(err) + } + + // Keyword search alone finds nothing useful for this query. + kw, err := store.Search(context.Background(), query, 5) + if err != nil { + t.Fatal(err) + } + for _, h := range kw { + if h.ProjectID == "tienda" && h.Section == "Tech stack" { + t.Skip("keyword search already found it; this corpus can't demonstrate the gap") + } + } + + hits, err := store.HybridSearch(context.Background(), emb, query, 3) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Fatal("hybrid search returned nothing") + } + if hits[0].ProjectID != "tienda" || hits[0].Section != "Tech stack" { + t.Errorf("top hit = %s/%s, want tienda/Tech stack", hits[0].ProjectID, hits[0].Section) + } +} + +// A dead embeddings endpoint must degrade to keyword search, not fail the +// visitor's question. +func TestHybridSearchFallsBackWhenEmbedderFails(t *testing.T) { + store := newTestStore(t) + broken := &stubEmbedder{err: errors.New("connection refused")} + + hits, err := store.HybridSearch(context.Background(), broken, "Stripe", 3) + if err != nil { + t.Fatalf("hybrid search should fall back, got error: %v", err) + } + if len(hits) == 0 { + t.Fatal("fallback returned no keyword hits for a term that is in the corpus") + } +} + +// With no embedder configured at all, hybrid search is plain keyword search. +func TestHybridSearchWithoutEmbedderIsKeywordSearch(t *testing.T) { + store := newTestStore(t) + hits, err := store.HybridSearch(context.Background(), nil, "Stripe", 3) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Fatal("expected keyword hits for Stripe") + } +} + +// Vectors from a different embedding model (wrong dimension) must be ignored +// rather than compared against and ranked. +func TestVectorSearchIgnoresMismatchedDimensions(t *testing.T) { + store := newTestStore(t) + corpus := &stubEmbedder{} // neutral 3-dim vectors + if _, err := store.EmbedChunks(context.Background(), corpus); err != nil { + t.Fatal(err) + } + hits, err := store.VectorSearch(context.Background(), []float32{1, 0, 0, 0}, 5) + if err != nil { + t.Fatal(err) + } + if len(hits) != 0 { + t.Errorf("got %d hits for a 4-dim query against 3-dim rows, want 0", len(hits)) + } +} + +// A chunk's id is derived from file + heading + position, so editing the body +// of a section keeps the id. A vector keyed only by id would then rank that +// chunk by text that no longer exists — silently, with no error anywhere. +// The stored content hash makes such a row invisible instead of wrong. +func TestVectorSearchIgnoresVectorsWhoseChunkChanged(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "projects") + if err := os.MkdirAll(src, 0o755); err != nil { + t.Fatal(err) + } + file := filepath.Join(src, "demo.md") + write := func(body string) { + if err := os.WriteFile(file, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("# Demo\n\n## Tech stack\n\nThe payment provider is Stripe.\n") + + store, err := OpenStore(filepath.Join(dir, "t.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + sources := SourcesFor(src, "") + if _, _, err := store.Reindex(context.Background(), sources, DefaultChunkerConfig()); err != nil { + t.Fatal(err) + } + if _, err := store.EmbedChunks(context.Background(), &stubEmbedder{}); err != nil { + t.Fatal(err) + } + hits, err := store.VectorSearch(context.Background(), embed.Normalize([]float32{1, 1, 1}), 5) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Fatal("expected the freshly embedded chunk to be searchable") + } + + // Rewrite the body, keeping every heading — the chunk id is unchanged. + write("# Demo\n\n## Tech stack\n\nThe payment provider is PayPal. Stripe was removed.\n") + if _, _, err := store.Reindex(context.Background(), sources, DefaultChunkerConfig()); err != nil { + t.Fatal(err) + } + // Deliberately do NOT re-embed: this is the "endpoint was down" case. + + var ids []string + rows, err := store.db.Query(`SELECT chunk_id FROM portfolio_vectors`) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + ids = append(ids, id) + } + rows.Close() + if len(ids) == 0 { + t.Fatal("precondition: the stale vector row should still be present") + } + + hits, err = store.VectorSearch(context.Background(), embed.Normalize([]float32{1, 1, 1}), 5) + if err != nil { + t.Fatal(err) + } + if len(hits) != 0 { + t.Errorf("stale vector was used for ranking: %+v", hits) + } + + // Re-embedding restores it. + if _, err := store.EmbedChunks(context.Background(), &stubEmbedder{}); err != nil { + t.Fatal(err) + } + hits, err = store.VectorSearch(context.Background(), embed.Normalize([]float32{1, 1, 1}), 5) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Error("re-embedding should make the chunk searchable again") + } +} diff --git a/internal/portfolio/indexer.go b/internal/portfolio/indexer.go index 60dcc39..bb13f10 100644 --- a/internal/portfolio/indexer.go +++ b/internal/portfolio/indexer.go @@ -23,6 +23,7 @@ import ( type SearchResult struct { ID string ProjectID string + Kind string // KindProject | KindDoc SourceFile string Section string Index int @@ -30,10 +31,36 @@ type SearchResult struct { Score float64 } +// Chunk kinds. A "project" is a piece of Victor's portfolio and shows up in +// the catalogue the bot injects into the prompt; a "doc" (his CV, an about +// page, a FAQ) is retrievable evidence that is not itself a project and must +// never be listed as one. +const ( + KindProject = "project" + KindDoc = "doc" +) + +// Source is one directory of markdown to index, and what the documents in it +// mean. See KindProject / KindDoc. +type Source struct { + Path string + Kind string +} + +// SourcesFor is the standard mapping from the two configured directories to +// index sources. docsPath may be empty. +func SourcesFor(dataPath, docsPath string) []Source { + return []Source{ + {Path: dataPath, Kind: KindProject}, + {Path: docsPath, Kind: KindDoc}, + } +} + const schema = ` CREATE VIRTUAL TABLE IF NOT EXISTS portfolio_chunks USING fts5( id UNINDEXED, project_id UNINDEXED, + kind UNINDEXED, source_file UNINDEXED, section UNINDEXED, chunk_index UNINDEXED, @@ -42,10 +69,72 @@ CREATE VIRTUAL TABLE IF NOT EXISTS portfolio_chunks USING fts5( ); ` +// Vectors live in an ordinary table keyed by chunk id. There is no ANN index: +// a portfolio is hundreds of chunks, not millions, so a full scan with a dot +// product is microseconds and needs no extension. +// content_hash is what makes a vector verifiable. A chunk's id is derived +// from file + heading + position, so editing the *body* of a section leaves +// the id untouched — and a vector keyed only by id would keep describing the +// text that used to be there. Nothing errors; semantic ranking just silently +// scores the chunk by a meaning it no longer has. Storing the hash of the +// embedded text lets the search ignore rows whose source has moved on. +const vectorSchema = ` +CREATE TABLE IF NOT EXISTS portfolio_vectors ( + chunk_id TEXT PRIMARY KEY, + content_hash TEXT NOT NULL, + dim INTEGER NOT NULL, + vec BLOB NOT NULL +); +` + +// ensureChunkSchema creates portfolio_chunks, and rebuilds it when an older +// database is missing a column (FTS5 has no ALTER TABLE ADD COLUMN). +// +// Dropping is safe precisely because this table is a derived index: every row +// is regenerated from the markdown on the next Reindex. It deliberately +// touches only portfolio_chunks — conversations live in the same file and are +// real user data. +func ensureChunkSchema(ctx context.Context, db *sql.DB) error { + var existing string + err := db.QueryRowContext(ctx, + `SELECT sql FROM sqlite_master WHERE type='table' AND name='portfolio_chunks'`).Scan(&existing) + switch { + case err == sql.ErrNoRows: + // Fresh database; fall through to CREATE. + case err != nil: + return fmt.Errorf("inspect chunk schema: %w", err) + case !strings.Contains(existing, "kind"): + slog.Info("portfolio index predates the kind column, rebuilding it (run reindex to repopulate)") + if _, err := db.ExecContext(ctx, `DROP TABLE portfolio_chunks`); err != nil { + return fmt.Errorf("drop stale chunk table: %w", err) + } + } + if _, err := db.ExecContext(ctx, schema); err != nil { + return fmt.Errorf("create schema: %w", err) + } + // Same rebuild-on-mismatch rule as the chunk table: vectors are derived + // data, regenerated by the next reindex. + var vecSQL string + switch err := db.QueryRowContext(ctx, + `SELECT sql FROM sqlite_master WHERE type='table' AND name='portfolio_vectors'`).Scan(&vecSQL); { + case err == sql.ErrNoRows: + case err != nil: + return fmt.Errorf("inspect vector schema: %w", err) + case !strings.Contains(vecSQL, "content_hash"): + slog.Info("vector index predates content_hash, rebuilding it (run reindex to repopulate)") + if _, err := db.ExecContext(ctx, `DROP TABLE portfolio_vectors`); err != nil { + return fmt.Errorf("drop stale vector table: %w", err) + } + } + if _, err := db.ExecContext(ctx, vectorSchema); err != nil { + return fmt.Errorf("create vector schema: %w", err) + } + return nil +} + // Store wraps a SQLite FTS5 database with the portfolio schema. type Store struct { - db *sql.DB - chunkSize int // legacy field kept for compat; not used by heading chunker + db *sql.DB } func OpenStore(dbPath string) (*Store, error) { @@ -59,15 +148,15 @@ func OpenStore(dbPath string) (*Store, error) { return nil, fmt.Errorf("open sqlite: %w", err) } db.SetMaxOpenConns(1) // SQLite + concurrent writers doesn't help - if _, err := db.ExecContext(context.Background(), schema); err != nil { + if err := ensureChunkSchema(context.Background(), db); err != nil { _ = db.Close() - return nil, fmt.Errorf("create schema: %w", err) + return nil, err } if _, err := db.ExecContext(context.Background(), conversationSchema); err != nil { _ = db.Close() return nil, fmt.Errorf("create conversation schema: %w", err) } - return &Store{db: db, chunkSize: 500}, nil + return &Store{db: db}, nil } func (s *Store) Close() error { return s.db.Close() } @@ -76,10 +165,52 @@ func (s *Store) Close() error { return s.db.Close() } // queries (e.g. the health check). Don't use for hot-path code: go through // the Search / Reindex methods. func (s *Store) DB() *sql.DB { return s.db } -func (s *Store) Reindex(ctx context.Context, dataPath string, cfg ChunkerConfig) (files, chunks int, err error) { - matches, err := filepath.Glob(filepath.Join(dataPath, "*.md")) - if err != nil { - return 0, 0, fmt.Errorf("glob: %w", err) + +// isReadme reports whether a file is a directory's README rather than content. +// These directories are checked into the repo with instructions for whoever +// fills them, and those instructions are not one of Victor's projects: without +// this, `data/projects/README.md` was indexed as a project and the catalogue +// injected into every prompt announced "README" and "README.es" to visitors. +// Localised variants (README.es.md) are covered by matching the first segment. +func isReadme(path string) bool { + base := filepath.Base(path) + name, _, _ := strings.Cut(base, ".") + return strings.EqualFold(name, "README") +} + +// Reindex rebuilds the whole index from the given sources. Each source +// contributes its `.md` and `.mdx` files; `.mdx` is included because content +// written for an Astro or Next site (a CV, an about page) is usually authored +// there, and its JSX is harmless to full-text search. +// +// A source with an empty Path is skipped, so callers can pass an optional +// docs directory without branching. +func (s *Store) Reindex(ctx context.Context, sources []Source, cfg ChunkerConfig) (files, chunks int, err error) { + type doc struct { + path string + kind string + } + var docs []doc + for _, src := range sources { + if strings.TrimSpace(src.Path) == "" { + continue + } + kind := src.Kind + if kind == "" { + kind = KindProject + } + for _, ext := range []string{"*.md", "*.mdx"} { + matches, err := filepath.Glob(filepath.Join(src.Path, ext)) + if err != nil { + return 0, 0, fmt.Errorf("glob %s: %w", ext, err) + } + for _, m := range matches { + if isReadme(m) { + continue + } + docs = append(docs, doc{path: m, kind: kind}) + } + } } tx, err := s.db.BeginTx(ctx, nil) @@ -93,24 +224,24 @@ func (s *Store) Reindex(ctx context.Context, dataPath string, cfg ChunkerConfig) } stmt, err := tx.PrepareContext(ctx, - `INSERT INTO portfolio_chunks (id, project_id, source_file, section, chunk_index, content) VALUES (?,?,?,?,?,?)`) + `INSERT INTO portfolio_chunks (id, project_id, kind, source_file, section, chunk_index, content) VALUES (?,?,?,?,?,?,?)`) if err != nil { return 0, 0, err } defer stmt.Close() - for _, file := range matches { - body, err := os.ReadFile(file) + for _, d := range docs { + body, err := os.ReadFile(d.path) if err != nil { - slog.Warn("read file failed", "file", file, "err", err) + slog.Warn("read file failed", "file", d.path, "err", err) continue } - projectID := strings.TrimSuffix(filepath.Base(file), ".md") + docID := strings.TrimSuffix(strings.TrimSuffix(filepath.Base(d.path), ".mdx"), ".md") sections := SplitMarkdownSections(string(body), cfg) for idx, sec := range sections { - id := fmt.Sprintf("%s-%s-%d", projectID, slugify(sec.Heading), idx) - if _, err := stmt.ExecContext(ctx, id, projectID, file, sec.Heading, idx, sec.Body); err != nil { - return len(matches), chunks, fmt.Errorf("insert %s: %w", id, err) + id := fmt.Sprintf("%s-%s-%d", docID, slugify(sec.Heading), idx) + if _, err := stmt.ExecContext(ctx, id, docID, d.kind, d.path, sec.Heading, idx, sec.Body); err != nil { + return len(docs), chunks, fmt.Errorf("insert %s: %w", id, err) } chunks++ } @@ -139,15 +270,65 @@ func slugify(s string) string { return string(out) } +// CatalogEntry is one project in the portfolio, identified by its filename +// stem and its human-readable H1 title. +type CatalogEntry struct { + ProjectID string + Title string +} + +// Catalog lists every indexed project with its title, cheaply and +// deterministically (no BM25, no query). +// +// It exists because retrieval alone can't answer "what projects does Victor +// have?": top-K search returns the K best-matching *chunks*, which for a +// broad question is a handful of sections from two or three documents, and a +// small model asked to enumerate from that will confidently fill the gaps +// with invented project names. Injecting the full catalogue into the system +// prompt turns enumeration into a copy job instead of a recall job. The +// portfolio is a few dozen documents at most, so the whole list costs a +// trivial number of tokens. +// +// Title falls back to the project ID when a document has no H1. +func (s *Store) Catalog(ctx context.Context) ([]CatalogEntry, error) { + // The chunker emits the frontmatter block first (when present) and the + // H1 title section next, so the lowest-indexed non-frontmatter section + // carries the document's title. + rows, err := s.db.QueryContext(ctx, ` + SELECT project_id, section, MIN(chunk_index) + FROM portfolio_chunks + WHERE section <> 'frontmatter' AND kind = ? + GROUP BY project_id + ORDER BY project_id`, KindProject) + if err != nil { + return nil, fmt.Errorf("catalog query: %w", err) + } + defer rows.Close() + + var out []CatalogEntry + for rows.Next() { + var e CatalogEntry + var idx int + if err := rows.Scan(&e.ProjectID, &e.Title, &idx); err != nil { + return nil, fmt.Errorf("catalog scan: %w", err) + } + if strings.TrimSpace(e.Title) == "" { + e.Title = e.ProjectID + } + out = append(out, e) + } + return out, rows.Err() +} + // Search returns up to topK chunks ordered by BM25 score. func (s *Store) Search(ctx context.Context, query string, topK int) ([]SearchResult, error) { if topK <= 0 { topK = 5 } rows, err := s.db.QueryContext(ctx, fmt.Sprintf(` - SELECT id, project_id, source_file, section, chunk_index, content, bm25(portfolio_chunks) AS score + SELECT id, project_id, kind, source_file, section, chunk_index, content, bm25(portfolio_chunks) AS score FROM portfolio_chunks - WHERE portfolio_chunks MATCH '%s' + WHERE portfolio_chunks MATCH '%s' AND section <> 'frontmatter' ORDER BY score LIMIT %d `, sanitizeFTS5(query), topK)) @@ -159,7 +340,7 @@ func (s *Store) Search(ctx context.Context, query string, topK int) ([]SearchRes var hits []SearchResult for rows.Next() { var r SearchResult - if err := rows.Scan(&r.ID, &r.ProjectID, &r.SourceFile, &r.Section, &r.Index, &r.Content, &r.Score); err != nil { + if err := rows.Scan(&r.ID, &r.ProjectID, &r.Kind, &r.SourceFile, &r.Section, &r.Index, &r.Content, &r.Score); err != nil { return nil, err } hits = append(hits, r) @@ -205,13 +386,28 @@ func sanitizeFTS5(q string) string { // ReindexOnDisk is a small convenience that opens the store, reindexes, and // closes — used by the CLI subcommand. -func ReindexOnDisk(dbPath, dataPath string, cfg ChunkerConfig) (time.Duration, int, int, error) { +// embedder may be nil, in which case no vectors are written and retrieval +// stays keyword-only. An embedding failure is logged and swallowed: the +// keyword index is already committed by then, and a bot that answers from +// keywords alone beats a reindex that reports failure and leaves nothing. +func ReindexOnDisk(dbPath string, sources []Source, cfg ChunkerConfig, embedder Embedder) (time.Duration, int, int, error) { start := time.Now() store, err := OpenStore(dbPath) if err != nil { return 0, 0, 0, err } defer store.Close() - files, chunks, err := store.Reindex(context.Background(), dataPath, cfg) - return time.Since(start), files, chunks, err -} \ No newline at end of file + files, chunks, err := store.Reindex(context.Background(), sources, cfg) + if err != nil { + return time.Since(start), files, chunks, err + } + if embedder != nil { + n, err := store.EmbedChunks(context.Background(), embedder) + if err != nil { + slog.Error("embedding failed; keyword search still works", "err", err) + } else { + slog.Info("embedded chunks", "chunks", n) + } + } + return time.Since(start), files, chunks, nil +} diff --git a/internal/portfolio/store_test.go b/internal/portfolio/store_test.go index 8f87e80..d1c8725 100644 --- a/internal/portfolio/store_test.go +++ b/internal/portfolio/store_test.go @@ -2,6 +2,7 @@ package portfolio import ( "context" + "database/sql" "os" "path/filepath" "strings" @@ -31,7 +32,7 @@ func TestReindexAndSearch(t *testing.T) { } defer store.Close() - files, chunks, err := store.Reindex(context.Background(), srcDir, DefaultChunkerConfig()) + files, chunks, err := store.Reindex(context.Background(), SourcesFor(srcDir, ""), DefaultChunkerConfig()) if err != nil { t.Fatal(err) } @@ -78,7 +79,7 @@ func TestReindexFromRealData(t *testing.T) { t.Fatal(err) } defer store.Close() - files, chunks, err := store.Reindex(context.Background(), srcDir, DefaultChunkerConfig()) + files, chunks, err := store.Reindex(context.Background(), SourcesFor(srcDir, ""), DefaultChunkerConfig()) if err != nil { t.Fatal(err) } @@ -113,4 +114,193 @@ func truncate(s string, n int) string { return s } return s[:n] + "..." -} \ No newline at end of file +} +func TestCatalogListsEveryProjectWithItsTitle(t *testing.T) { + dir := t.TempDir() + srcDir := filepath.Join(dir, "src") + if err := os.MkdirAll(srcDir, 0o755); err != nil { + t.Fatal(err) + } + // Two documents: one with frontmatter (so the title is not chunk 0), + // one without. Both must show up with their H1 as the title. + docs := map[string]string{ + "alpha.md": "---\ntitle: \"ignored\"\n---\n\n# Alpha Project\n\nBody.\n\n## Stack\n\nGo.\n", + "beta.md": "# Beta Project\n\nBody.\n", + } + for name, body := range docs { + if err := os.WriteFile(filepath.Join(srcDir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + + store, err := OpenStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, _, err := store.Reindex(context.Background(), SourcesFor(srcDir, ""), DefaultChunkerConfig()); err != nil { + t.Fatal(err) + } + + entries, err := store.Catalog(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 { + t.Fatalf("Catalog returned %d entries, want 2: %+v", len(entries), entries) + } + want := map[string]string{"alpha": "Alpha Project", "beta": "Beta Project"} + for _, e := range entries { + if want[e.ProjectID] != e.Title { + t.Errorf("Catalog[%s].Title = %q, want %q", e.ProjectID, e.Title, want[e.ProjectID]) + } + } +} + +func TestReindexIndexesMdxAndSeparatesDocsFromProjects(t *testing.T) { + dir := t.TempDir() + projects := filepath.Join(dir, "projects") + docs := filepath.Join(dir, "docs") + for _, d := range []string{projects, docs} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + write := func(path, body string) { + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(projects, "tienda.md"), "# Tienda\n\n## Stack\n\nNext.js y Stripe.\n") + // The CV is authored as .mdx for an Astro site and is not a project. + write(filepath.Join(docs, "cv.mdx"), "---\nname: \"V\"\n---\n\n## Skills\n\nKubernetes, AWS, Go.\n") + + store, err := OpenStore(filepath.Join(dir, "t.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + files, _, err := store.Reindex(context.Background(), SourcesFor(projects, docs), DefaultChunkerConfig()) + if err != nil { + t.Fatal(err) + } + if files != 2 { + t.Fatalf("indexed %d files, want 2 (the .mdx must be picked up)", files) + } + + // The CV is searchable... + hits, err := store.Search(context.Background(), "Kubernetes", 5) + if err != nil { + t.Fatal(err) + } + if len(hits) == 0 { + t.Fatal("Kubernetes is in the CV but the search returned nothing") + } + if hits[0].ProjectID != "cv" || hits[0].Kind != KindDoc { + t.Errorf("top hit = %q/%q, want cv/%s", hits[0].ProjectID, hits[0].Kind, KindDoc) + } + + // ...but must never be announced as one of Victor's projects. + cat, err := store.Catalog(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(cat) != 1 || cat[0].ProjectID != "tienda" { + t.Fatalf("catalogue = %+v, want only the tienda project", cat) + } +} + +// Both content directories ship a README telling whoever fills them what to +// put there. Those instructions are not content: indexing them announced +// "README" and "README.es" as projects of Victor's in every prompt. +func TestReindexSkipsTheDirectoryReadmes(t *testing.T) { + dir := t.TempDir() + projects := filepath.Join(dir, "projects") + docs := filepath.Join(dir, "docs") + for _, d := range []string{projects, docs} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + write := func(path, body string) { + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(projects, "tienda.md"), "# Tienda\n\n## Stack\n\nNext.js.\n") + write(filepath.Join(projects, "README.md"), "# Portfolio Projects\n\n## Naming\n\nOne file per project.\n") + write(filepath.Join(projects, "README.es.md"), "# Proyectos\n\n## Nombres\n\nUn archivo por proyecto.\n") + write(filepath.Join(docs, "README.md"), "# Reference documents\n\n## What goes here\n\nA CV, an about page.\n") + + store, err := OpenStore(filepath.Join(dir, "t.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + files, _, err := store.Reindex(context.Background(), SourcesFor(projects, docs), DefaultChunkerConfig()) + if err != nil { + t.Fatal(err) + } + if files != 1 { + t.Fatalf("indexed %d files, want only tienda.md", files) + } + + cat, err := store.Catalog(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(cat) != 1 || cat[0].ProjectID != "tienda" { + t.Fatalf("catalogue = %+v, want only the tienda project", cat) + } + + // And the instructions are not retrievable either — a question about + // naming conventions must not surface the maintainer's docs. + hits, err := store.Search(context.Background(), "naming convention", 5) + if err != nil { + t.Fatal(err) + } + for _, h := range hits { + if strings.Contains(strings.ToLower(h.ProjectID), "readme") { + t.Errorf("search returned the README: %+v", h) + } + } +} + +// An index built before the kind column exists must not break the bot: the +// table is derived data, so OpenStore rebuilds it. +func TestOpenStoreMigratesIndexWithoutKindColumn(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "old.db") + + old, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + _, err = old.Exec(`CREATE VIRTUAL TABLE portfolio_chunks USING fts5( + id UNINDEXED, project_id UNINDEXED, source_file UNINDEXED, + section UNINDEXED, chunk_index UNINDEXED, content)`) + if err != nil { + t.Fatal(err) + } + if _, err := old.Exec(`INSERT INTO portfolio_chunks VALUES ('a','b','c','d',0,'stale')`); err != nil { + t.Fatal(err) + } + if err := old.Close(); err != nil { + t.Fatal(err) + } + + store, err := OpenStore(dbPath) + if err != nil { + t.Fatalf("OpenStore on a pre-kind database failed: %v", err) + } + defer store.Close() + + // Queries that name `kind` must now work. + if _, err := store.Catalog(context.Background()); err != nil { + t.Errorf("Catalog after migration: %v", err) + } + if _, err := store.Search(context.Background(), "stale", 5); err != nil { + t.Errorf("Search after migration: %v", err) + } +} diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 0883e62..faf9c52 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -22,15 +22,25 @@ type Handlers struct { runner *agent.Runner store *portfolio.Store version string + // embedder is used only by the /api/reindex endpoint, so a rebuild + // triggered over HTTP refreshes the vectors too instead of silently + // leaving them describing the previous corpus. May be nil. + embedder portfolio.Embedder } func NewHandlers(cfg *config.Config, runner *agent.Runner, store *portfolio.Store, version string) *Handlers { return &Handlers{cfg: cfg, runner: runner, store: store, version: version} } +// WithEmbedder attaches the embeddings client used when reindexing over HTTP. +func (h *Handlers) WithEmbedder(e portfolio.Embedder) *Handlers { + h.embedder = e + return h +} + type ChatRequest struct { - Messages []ChatMessage `json:"messages"` - Stream *bool `json:"stream,omitempty"` + Messages []ChatMessage `json:"messages"` + Stream *bool `json:"stream,omitempty"` // ConversationID is optional. If empty, the server creates a new // conversation and returns its ID in the response (or in the SSE // `start` event). Pass an existing ID to continue a previous thread. @@ -43,9 +53,9 @@ type ChatMessage struct { } type ChatResponse struct { - ConversationID string `json:"conversation_id"` - Content string `json:"content"` - Sources []string `json:"sources,omitempty"` + ConversationID string `json:"conversation_id"` + Content string `json:"content"` + Sources []string `json:"sources,omitempty"` Usage streaming.Usage `json:"usage"` } @@ -304,7 +314,7 @@ func (h *Handlers) Info(w http.ResponseWriter, _ *http.Request) { "name": h.cfg.Persona.Name, "version": h.version, "provider": p.Type, - "model": firstNonEmpty(p.Model, p.ModelPath), + "model": p.Model, "rag": h.cfg.RAG.Enabled, "top_k": h.cfg.RAG.TopK, "started": time.Now().UTC().Format(time.RFC3339), @@ -322,7 +332,7 @@ func (h *Handlers) Reindex(w http.ResponseWriter, r *http.Request) { http.Error(w, "rag.db_path not configured", http.StatusBadRequest) return } - dur, files, chunks, err := portfolio.ReindexOnDisk(dbPath, h.cfg.RAG.DataPath, portfolio.DefaultChunkerConfig()) + dur, files, chunks, err := portfolio.ReindexOnDisk(dbPath, portfolio.SourcesFor(h.cfg.RAG.DataPath, h.cfg.RAG.DocsPath), portfolio.DefaultChunkerConfig(), h.embedder) if err != nil { http.Error(w, "reindex failed: "+err.Error(), http.StatusInternalServerError) return @@ -432,12 +442,3 @@ func conversationIDFromPath(path string) string { } return id } - -func firstNonEmpty(vals ...string) string { - for _, v := range vals { - if v != "" { - return v - } - } - return "" -} \ No newline at end of file diff --git a/internal/server/testhelpers_test.go b/internal/server/testhelpers_test.go index 179863f..26a57d0 100644 --- a/internal/server/testhelpers_test.go +++ b/internal/server/testhelpers_test.go @@ -22,7 +22,7 @@ func ragOpenStore(dbPath, srcDir string) (ragStore, error) { if err != nil { return nil, err } - if _, _, err := store.Reindex(context.Background(), srcDir, portfolio.DefaultChunkerConfig()); err != nil { + if _, _, err := store.Reindex(context.Background(), portfolio.SourcesFor(srcDir, ""), portfolio.DefaultChunkerConfig()); err != nil { _ = store.Close() return nil, err }