package portfolio import ( "context" "database/sql" "os" "path/filepath" "strings" "testing" ) func TestReindexAndSearch(t *testing.T) { // Use a temp dir so we don't pollute data/projects dir := t.TempDir() srcDir := filepath.Join(dir, "src") if err := os.MkdirAll(srcDir, 0o755); err != nil { t.Fatal(err) } // Copy the real example-project.md data, err := os.ReadFile("../../data/projects/example-project.md") if err != nil { t.Skip("example-project.md not found, skipping integration test:", err) } if err := os.WriteFile(filepath.Join(srcDir, "example-project.md"), data, 0o644); err != nil { t.Fatal(err) } dbPath := filepath.Join(dir, "test.db") store, err := OpenStore(dbPath) if err != nil { t.Fatal(err) } defer store.Close() files, chunks, err := store.Reindex(context.Background(), SourcesFor(srcDir, ""), DefaultChunkerConfig()) if err != nil { t.Fatal(err) } if files != 1 { t.Errorf("files = %d, want 1", files) } if chunks < 4 { t.Errorf("chunks = %d, want >= 4 (frontmatter + 4 H2 sections)", chunks) } // Search for something specific to the file hits, err := store.Search(context.Background(), "Go SQLite framework", 5) if err != nil { t.Fatal(err) } if len(hits) == 0 { t.Error("expected at least one hit for 'Go SQLite framework'") } t.Logf("Search hits: %d", len(hits)) for i, h := range hits { t.Logf(" [%d] project=%s section=%s score=%.3f body=%q", i, h.ProjectID, h.Section, h.Score, truncate(h.Content, 80)) } // No results for nonsense hits, err = store.Search(context.Background(), "asdfqwerty12345", 5) if err != nil { t.Fatal(err) } if len(hits) != 0 { t.Errorf("expected 0 hits for nonsense query, got %d", len(hits)) } } func TestReindexFromRealData(t *testing.T) { // Index the real data/projects/ to sanity-check on real data srcDir := "../../data/projects" if _, err := os.Stat(srcDir); err != nil { t.Skip("data/projects not found:", err) } dir := t.TempDir() dbPath := filepath.Join(dir, "real.db") store, err := OpenStore(dbPath) if err != nil { t.Fatal(err) } defer store.Close() files, chunks, err := store.Reindex(context.Background(), SourcesFor(srcDir, ""), DefaultChunkerConfig()) if err != nil { t.Fatal(err) } t.Logf("Indexed %d files, %d chunks", files, chunks) if files == 0 { t.Fatal("no files indexed") } // Print what we got sections := map[string]int{} _, _ = store.Search(context.Background(), "rony", 100) // noop // Walk DB to see sections — easier: just re-read with a fresh load _ = sections // Verify queries work for _, q := range []string{"rony-llm-agent", "Qwen", "agent harness", "tech stack", "deploy"} { hits, err := store.Search(context.Background(), q, 3) if err != nil { t.Fatal(err) } t.Logf("query %q → %d hits", q, len(hits)) for _, h := range hits { if !strings.Contains(strings.ToLower(h.Content), strings.ToLower(strings.SplitN(q, "-", 2)[0])) { // FTS5 might do prefix matches; this is a soft assertion } } } } func truncate(s string, n int) string { s = strings.ReplaceAll(s, "\n", " ") if len(s) <= n { return s } return s[:n] + "..." } func TestCatalogListsEveryProjectWithItsTitle(t *testing.T) { dir := t.TempDir() srcDir := filepath.Join(dir, "src") if err := os.MkdirAll(srcDir, 0o755); err != nil { t.Fatal(err) } // Two documents: one with frontmatter (so the title is not chunk 0), // one without. Both must show up with their H1 as the title. docs := map[string]string{ "alpha.md": "---\ntitle: \"ignored\"\n---\n\n# Alpha Project\n\nBody.\n\n## Stack\n\nGo.\n", "beta.md": "# Beta Project\n\nBody.\n", } for name, body := range docs { if err := os.WriteFile(filepath.Join(srcDir, name), []byte(body), 0o644); err != nil { t.Fatal(err) } } store, err := OpenStore(filepath.Join(dir, "test.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, _, err := store.Reindex(context.Background(), SourcesFor(srcDir, ""), DefaultChunkerConfig()); err != nil { t.Fatal(err) } entries, err := store.Catalog(context.Background()) if err != nil { t.Fatal(err) } if len(entries) != 2 { t.Fatalf("Catalog returned %d entries, want 2: %+v", len(entries), entries) } want := map[string]string{"alpha": "Alpha Project", "beta": "Beta Project"} for _, e := range entries { if want[e.ProjectID] != e.Title { t.Errorf("Catalog[%s].Title = %q, want %q", e.ProjectID, e.Title, want[e.ProjectID]) } } } func TestReindexIndexesMdxAndSeparatesDocsFromProjects(t *testing.T) { dir := t.TempDir() projects := filepath.Join(dir, "projects") docs := filepath.Join(dir, "docs") for _, d := range []string{projects, docs} { if err := os.MkdirAll(d, 0o755); err != nil { t.Fatal(err) } } write := func(path, body string) { if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } } write(filepath.Join(projects, "tienda.md"), "# Tienda\n\n## Stack\n\nNext.js y Stripe.\n") // The CV is authored as .mdx for an Astro site and is not a project. write(filepath.Join(docs, "cv.mdx"), "---\nname: \"V\"\n---\n\n## Skills\n\nKubernetes, AWS, Go.\n") store, err := OpenStore(filepath.Join(dir, "t.db")) if err != nil { t.Fatal(err) } defer store.Close() files, _, err := store.Reindex(context.Background(), SourcesFor(projects, docs), DefaultChunkerConfig()) if err != nil { t.Fatal(err) } if files != 2 { t.Fatalf("indexed %d files, want 2 (the .mdx must be picked up)", files) } // The CV is searchable... hits, err := store.Search(context.Background(), "Kubernetes", 5) if err != nil { t.Fatal(err) } if len(hits) == 0 { t.Fatal("Kubernetes is in the CV but the search returned nothing") } if hits[0].ProjectID != "cv" || hits[0].Kind != KindDoc { t.Errorf("top hit = %q/%q, want cv/%s", hits[0].ProjectID, hits[0].Kind, KindDoc) } // ...but must never be announced as one of Victor's projects. cat, err := store.Catalog(context.Background()) if err != nil { t.Fatal(err) } if len(cat) != 1 || cat[0].ProjectID != "tienda" { t.Fatalf("catalogue = %+v, want only the tienda project", cat) } } // Both content directories ship a README telling whoever fills them what to // put there. Those instructions are not content: indexing them announced // "README" and "README.es" as projects of Victor's in every prompt. func TestReindexSkipsTheDirectoryReadmes(t *testing.T) { dir := t.TempDir() projects := filepath.Join(dir, "projects") docs := filepath.Join(dir, "docs") for _, d := range []string{projects, docs} { if err := os.MkdirAll(d, 0o755); err != nil { t.Fatal(err) } } write := func(path, body string) { if err := os.WriteFile(path, []byte(body), 0o644); err != nil { t.Fatal(err) } } write(filepath.Join(projects, "tienda.md"), "# Tienda\n\n## Stack\n\nNext.js.\n") write(filepath.Join(projects, "README.md"), "# Portfolio Projects\n\n## Naming\n\nOne file per project.\n") write(filepath.Join(projects, "README.es.md"), "# Proyectos\n\n## Nombres\n\nUn archivo por proyecto.\n") write(filepath.Join(docs, "README.md"), "# Reference documents\n\n## What goes here\n\nA CV, an about page.\n") store, err := OpenStore(filepath.Join(dir, "t.db")) if err != nil { t.Fatal(err) } defer store.Close() files, _, err := store.Reindex(context.Background(), SourcesFor(projects, docs), DefaultChunkerConfig()) if err != nil { t.Fatal(err) } if files != 1 { t.Fatalf("indexed %d files, want only tienda.md", files) } cat, err := store.Catalog(context.Background()) if err != nil { t.Fatal(err) } if len(cat) != 1 || cat[0].ProjectID != "tienda" { t.Fatalf("catalogue = %+v, want only the tienda project", cat) } // And the instructions are not retrievable either — a question about // naming conventions must not surface the maintainer's docs. hits, err := store.Search(context.Background(), "naming convention", 5) if err != nil { t.Fatal(err) } for _, h := range hits { if strings.Contains(strings.ToLower(h.ProjectID), "readme") { t.Errorf("search returned the README: %+v", h) } } } // An index built before the kind column exists must not break the bot: the // table is derived data, so OpenStore rebuilds it. func TestOpenStoreMigratesIndexWithoutKindColumn(t *testing.T) { dbPath := filepath.Join(t.TempDir(), "old.db") old, err := sql.Open("sqlite", dbPath) if err != nil { t.Fatal(err) } _, err = old.Exec(`CREATE VIRTUAL TABLE portfolio_chunks USING fts5( id UNINDEXED, project_id UNINDEXED, source_file UNINDEXED, section UNINDEXED, chunk_index UNINDEXED, content)`) if err != nil { t.Fatal(err) } if _, err := old.Exec(`INSERT INTO portfolio_chunks VALUES ('a','b','c','d',0,'stale')`); err != nil { t.Fatal(err) } if err := old.Close(); err != nil { t.Fatal(err) } store, err := OpenStore(dbPath) if err != nil { t.Fatalf("OpenStore on a pre-kind database failed: %v", err) } defer store.Close() // Queries that name `kind` must now work. if _, err := store.Catalog(context.Background()); err != nil { t.Errorf("Catalog after migration: %v", err) } if _, err := store.Search(context.Background(), "stale", 5); err != nil { t.Errorf("Search after migration: %v", err) } }