f33708534a
Initial implementation of the bot: - cmd/chat-bot: CLI entrypoint (serve, reindex, ask, version) - internal/agent: LLM provider client + agent runner with RAG injection - internal/config: YAML config loader (providers, RAG, persona, server) - internal/i18n: response-language detection (EN/ES) - internal/persona: persona system prompt assembly from YAML - internal/portfolio: heading-based chunker + SQLite FTS5 indexer - internal/server: chi router with /api/chat (SSE), /api/health, /api/info, /api/reindex, middleware (RequestID, Logging, CORS, RateLimit) - internal/streaming: SSE protocol helpers (start, chunk, sources, done, error) - web/: drop-in vanilla-JS chat widget (no build, no deps) + demo + README - bench/: reproducible driver benchmark (modernc vs mattn SQLite) - configs/portfolio-bot.yaml: llama.cpp default provider, SQLite RAG, canine persona - docs/architecture.md / .es.md: aligned with SQLite FTS5 + llama.cpp decisions - data/projects/README*.md: project data documentation - README.md / .es.md: updated for current implementation All tests pass (go test ./...). Bot is functional end-to-end with the configured LLM provider.
91 lines
No EOL
2.5 KiB
Go
91 lines
No EOL
2.5 KiB
Go
package i18n
|
|
|
|
import (
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
var (
|
|
esStopwords = map[string]bool{
|
|
"el": true, "la": true, "los": true, "las": true, "de": true,
|
|
"que": true, "y": true, "en": true, "un": true, "una": true,
|
|
"es": true, "se": true, "no": true, "con": true, "para": true,
|
|
"por": true, "su": true, "del": true, "al": true, "lo": true,
|
|
"qué": true, "cómo": true, "dónde": true, "cuándo": true,
|
|
"cuál": true, "cuáles": true, "quién": true, "habla": true,
|
|
"tienes": true, "dime": true, "háblame": true, "sobre": true,
|
|
"más": true, "pero": true, "como": true, "este": true, "esta": true,
|
|
"estos": true, "estas": true, "ese": true, "esa": true, "aquel": true,
|
|
"muy": true, "sin": true, "hay": true, "sí": true, "yo": true,
|
|
"tú": true, "él": true, "ella": true, "nosotros": true,
|
|
}
|
|
enStopwords = map[string]bool{
|
|
"the": true, "is": true, "are": true, "of": true, "and": true,
|
|
"in": true, "to": true, "a": true, "an": true, "for": true,
|
|
"with": true, "on": true, "by": true, "from": true, "what": true,
|
|
"which": true, "who": true, "how": true, "when": true, "where": true,
|
|
"tell": true, "about": true, "do": true, "does": true, "can": true,
|
|
"you": true, "your": true, "have": true, "has": true, "i": true,
|
|
"we": true, "they": true, "this": true, "that": true, "these": true,
|
|
"those": true, "be": true, "been": true, "will": true, "would": true,
|
|
"should": true, "could": true, "my": true, "our": true, "their": true,
|
|
}
|
|
)
|
|
|
|
// Detect returns "es" or "en" based on lightweight heuristics. Good enough
|
|
// to choose the response language for a portfolio chatbot; the LLM (once
|
|
// wired in) is the final authority.
|
|
//
|
|
// Heuristic:
|
|
// - Spanish diacritics or ¿/¡ → +N Spanish markers
|
|
// - Tokenize and count stopwords in each language
|
|
// - Whichever side wins; tie → English
|
|
func Detect(text string) string {
|
|
if text == "" {
|
|
return "en"
|
|
}
|
|
|
|
markers := 0
|
|
for _, c := range text {
|
|
switch c {
|
|
case '¿', '¡':
|
|
markers += 2
|
|
case 'ñ':
|
|
markers += 2
|
|
case 'á', 'é', 'í', 'ó', 'ú', 'ü':
|
|
markers++
|
|
}
|
|
}
|
|
|
|
es, en := 0, 0
|
|
for _, tok := range tokenize(text) {
|
|
if esStopwords[tok] {
|
|
es++
|
|
}
|
|
if enStopwords[tok] {
|
|
en++
|
|
}
|
|
}
|
|
|
|
switch {
|
|
case markers >= 2:
|
|
return "es"
|
|
case es > en:
|
|
return "es"
|
|
case en > es:
|
|
return "en"
|
|
default:
|
|
return "en"
|
|
}
|
|
}
|
|
|
|
func tokenize(s string) []string {
|
|
s = strings.ToLower(s)
|
|
f := func(c rune) bool {
|
|
if unicode.IsLetter(c) || unicode.IsDigit(c) {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
return strings.FieldsFunc(s, f)
|
|
} |