From 4d4e99c2384b7a9150acf013526bd1ad2a4b4b66 Mon Sep 17 00:00:00 2001 From: Alfredo Medrano Date: Fri, 1 May 2026 19:41:33 -0600 Subject: [PATCH 1/4] refactor: reorganize project into packages following Go best practices - Move all files from src/ to proper package structure - Create cmd/codeindex for CLI entry point - Create internal packages: config, daemon, embeddings, index, search, types, utils - Apply DRY principle: remove duplicate code - Remove duplicate MatchSegments (consolidate in utils) - Remove unnecessary FilepathToSlash wrapper - Remove redundant match.go entirely (use utils.MatchPattern) - Remove redundant fileHash wrapper (use FileHash directly) - Fix all import cycles in test files - Fix all test imports and undefined symbols - All tests passing (6 packages) --- {src => cmd/codeindex}/main.go | 87 ++++++++--------- {src => cmd/codeindex}/main_test.go | 13 ++- {src => internal/config}/config.go | 79 ++++++++------- {src => internal/config}/config_test.go | 62 ++++++------ {src => internal/daemon}/cli_e2e_test.go | 14 +-- {src => internal/daemon}/daemon.go | 63 ++++++------ .../daemon/proc_unix.go | 2 +- .../daemon/proc_windows.go | 2 +- {src => internal/daemon}/registry.go | 6 +- {src => internal/embeddings}/embeddings.go | 20 ++-- .../embeddings}/embeddings_test.go | 34 +++---- {src => internal/embeddings}/llamacpp_test.go | 30 +++--- {src => internal/index}/chunk.go | 22 +++-- {src => internal/index}/chunk_test.go | 6 +- {src => internal/index}/index.go | 96 ++++++++++--------- {src => internal/index}/index_test.go | 48 ++++++---- {src => internal/index}/scan.go | 23 +++-- {src => internal/index}/scan_test.go | 20 ++-- {src => internal/index}/store.go | 22 +++-- {src => internal/index}/store_test.go | 8 +- {src => internal/search}/hybrid.go | 25 +++-- {src => internal/search}/hybrid_test.go | 43 +++++---- {src => internal/search}/match_test.go | 16 ++-- {src => internal/types}/types.go | 2 +- internal/utils/utils.go | 66 +++++++++++++ src/match.go | 44 --------- 26 files changed, 477 insertions(+), 376 deletions(-) rename {src => cmd/codeindex}/main.go (90%) rename {src => cmd/codeindex}/main_test.go (79%) rename {src => internal/config}/config.go (82%) rename {src => internal/config}/config_test.go (84%) rename {src => internal/daemon}/cli_e2e_test.go (88%) rename {src => internal/daemon}/daemon.go (88%) rename src/daemon_proc_unix.go => internal/daemon/proc_unix.go (91%) rename src/daemon_proc_windows.go => internal/daemon/proc_windows.go (90%) rename {src => internal/daemon}/registry.go (97%) rename {src => internal/embeddings}/embeddings.go (95%) rename {src => internal/embeddings}/embeddings_test.go (91%) rename {src => internal/embeddings}/llamacpp_test.go (86%) rename {src => internal/index}/chunk.go (83%) rename {src => internal/index}/chunk_test.go (99%) rename {src => internal/index}/index.go (71%) rename {src => internal/index}/index_test.go (72%) rename {src => internal/index}/scan.go (80%) rename {src => internal/index}/scan_test.go (83%) rename {src => internal/index}/store.go (67%) rename {src => internal/index}/store_test.go (89%) rename {src => internal/search}/hybrid.go (69%) rename {src => internal/search}/hybrid_test.go (57%) rename {src => internal/search}/match_test.go (52%) rename {src => internal/types}/types.go (98%) create mode 100644 internal/utils/utils.go delete mode 100644 src/match.go diff --git a/src/main.go b/cmd/codeindex/main.go similarity index 90% rename from src/main.go rename to cmd/codeindex/main.go index 7f277e2..49c076e 100644 --- a/src/main.go +++ b/cmd/codeindex/main.go @@ -13,6 +13,13 @@ import ( "strings" "sync" "time" + + "github.com/QuinsZouls/code-index/internal/config" + "github.com/QuinsZouls/code-index/internal/daemon" + "github.com/QuinsZouls/code-index/internal/embeddings" + "github.com/QuinsZouls/code-index/internal/index" + "github.com/QuinsZouls/code-index/internal/types" + "github.com/QuinsZouls/code-index/internal/utils" ) const appVersion = "0.1.6" @@ -38,7 +45,7 @@ func main() { case "clear": runClear(os.Args[2:]) case "daemon": - runDaemon(os.Args[2:]) + daemon.RunDaemon(os.Args[2:]) case "onboard": runOnboard(os.Args[2:]) default: @@ -57,7 +64,7 @@ func findProjectRoot(start string) (string, error) { return "", err } for { - if _, err := os.Stat(settingsPath(cur)); err == nil { + if _, err := os.Stat(config.SettingsPath(cur)); err == nil { return cur, nil } parent := filepath.Dir(cur) @@ -77,7 +84,7 @@ func runInit(args []string) { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - if _, err := initProject(root); err != nil { + if _, err := config.InitProject(root); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -94,12 +101,12 @@ func runIndex(args []string) { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - cfg, err := loadConfig(root) + cfg, err := config.LoadConfig(root) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - indexer, err := newIndexer(root, cfg) + indexer, err := index.NewIndexer(root, cfg) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -110,7 +117,7 @@ func runIndex(args []string) { printer := newProgressPrinter(os.Stdout, os.Stderr, "Indexing") defer printer.Stop() var indexed int - indexer.progressFn = func(p IndexProgress) { + indexer.ProgressFn = func(p types.IndexProgress) { if *verbose { printer.Emit(p) } @@ -151,17 +158,17 @@ func runSearch(args []string) { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - indexFile := indexPath(root) + indexFile := config.IndexPath(root) if _, err := os.Stat(indexFile); errors.Is(err, os.ErrNotExist) { fmt.Fprintln(os.Stderr, "No index found. Run 'codeindex init' first to initialize the project.") os.Exit(1) } - cfg, err := loadConfig(root) + cfg, err := config.LoadConfig(root) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - indexer, err := newIndexer(root, cfg) + indexer, err := index.NewIndexer(root, cfg) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -178,7 +185,7 @@ func runSearch(args []string) { if fetchLimit < 50 { fetchLimit = 50 } - results, err := indexer.Search(ctx, SearchOptions{ + results, err := indexer.Search(ctx, index.SearchOptions{ Query: query, Limit: fetchLimit, Offset: *offset, @@ -190,7 +197,7 @@ func runSearch(args []string) { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - filtered := make([]SearchResult, 0, len(results)) + filtered := make([]types.SearchResult, 0, len(results)) for _, r := range results { if r.Score >= cfg.ScoreThreshold { filtered = append(filtered, r) @@ -210,7 +217,7 @@ func runSearch(args []string) { return } for i, r := range filtered { - content := readChunkContent(root, r.FilePath, r.StartLine, r.EndLine) + content := utils.ReadChunkContent(root, r.FilePath, r.StartLine, r.EndLine) fmt.Printf("\n--- Result %d (score: %.3f) ---\n", i+1, r.Score) fmt.Printf("File: %s:%d-%d [%s]\n", r.FilePath, r.StartLine, r.EndLine, r.Language) fmt.Println(content) @@ -226,17 +233,17 @@ func runStatus(args []string) { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - indexFile := indexPath(root) + indexFile := config.IndexPath(root) if _, err := os.Stat(indexFile); errors.Is(err, os.ErrNotExist) { fmt.Fprintln(os.Stderr, "No index found. Run 'codeindex init' first to initialize the project.") os.Exit(1) } - cfg, err := loadConfig(root) + cfg, err := config.LoadConfig(root) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - indexer, err := newIndexer(root, cfg) + indexer, err := index.NewIndexer(root, cfg) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -270,22 +277,6 @@ func splitCSV(s string) []string { return out } -func readChunkContent(projectRoot, relPath string, startLine, endLine int) string { - absPath := filepath.Join(projectRoot, relPath) - data, err := os.ReadFile(absPath) - if err != nil { - return fmt.Sprintf("[file unavailable: %v]", err) - } - lines := strings.Split(string(data), "\n") - if startLine < 1 || startLine > len(lines) { - return "[line range invalid]" - } - if endLine > len(lines) { - endLine = len(lines) - } - return strings.Join(lines[startLine-1:endLine], "\n") -} - type progressPrinter struct { out *os.File err *os.File @@ -324,7 +315,7 @@ func (p *progressPrinter) spin() { } } -func (p *progressPrinter) Emit(progress IndexProgress) { +func (p *progressPrinter) Emit(progress types.IndexProgress) { if progress.File == "" { return } @@ -377,7 +368,7 @@ func runDoctor(args []string) { hasErrors = true } else { fmt.Printf("[OK] Home directory: %s\n", home) - globalPath := filepath.Join(home, settingsDirName, "default_settings.json") + globalPath := filepath.Join(home, config.SettingsDirName, "default_settings.json") if data, err := os.ReadFile(globalPath); err != nil { if errors.Is(err, os.ErrNotExist) { fmt.Printf("[WARN] Global settings: not found at %s\n", globalPath) @@ -387,16 +378,16 @@ func runDoctor(args []string) { } } else { fmt.Printf("[OK] Global settings: %s\n", globalPath) - var cfg Config + var cfg config.Config if err := json.Unmarshal(data, &cfg); err != nil { fmt.Printf("[FAIL] Global settings parse: %v\n", err) hasErrors = true } else { - cfg.normalize() + cfg.Normalize() fmt.Printf(" Provider: %s\n", cfg.Embedding.Provider) fmt.Printf(" Model: %s\n", cfg.Embedding.Model) fmt.Printf(" Base URL: %s\n", cfg.Embedding.BaseURL) - key := apiKey(cfg.Embedding) + key := config.APIKey(cfg.Embedding) if key == "" && cfg.Embedding.Provider != "ollama" && cfg.Embedding.Provider != "lmstudio" && cfg.Embedding.Provider != "llamacpp" { fmt.Printf("[FAIL] API key: not set (env: %s)\n", cfg.Embedding.APIKeyEnv) hasErrors = true @@ -405,7 +396,7 @@ func runDoctor(args []string) { } else { fmt.Printf("[OK] API key: not required for %s\n", cfg.Embedding.Provider) } - provider, err := newEmbeddingProvider(cfg.Embedding) + provider, err := embeddings.NewEmbeddingProvider(cfg.Embedding) if err != nil { fmt.Printf("[FAIL] Provider init: %v\n", err) hasErrors = true @@ -427,7 +418,7 @@ func runDoctor(args []string) { } } } - projectSettings := settingsPath(root) + projectSettings := config.SettingsPath(root) if data, err := os.ReadFile(projectSettings); err != nil { if errors.Is(err, os.ErrNotExist) { fmt.Printf("[WARN] Project settings: not found at %s\n", projectSettings) @@ -437,13 +428,13 @@ func runDoctor(args []string) { } } else { fmt.Printf("[OK] Project settings: %s\n", projectSettings) - var cfg Config + var cfg config.Config if err := json.Unmarshal(data, &cfg); err != nil { fmt.Printf("[FAIL] Project settings parse: %v\n", err) hasErrors = true } } - indexFile := indexPath(root) + indexFile := config.IndexPath(root) if _, err := os.Stat(indexFile); err != nil { if errors.Is(err, os.ErrNotExist) { fmt.Printf("[WARN] Index file: not found at %s\n", indexFile) @@ -471,7 +462,7 @@ func runClear(args []string) { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - indexFile := indexPath(root) + indexFile := config.IndexPath(root) if _, err := os.Stat(indexFile); err != nil { if errors.Is(err, os.ErrNotExist) { fmt.Println("no index to clear") @@ -511,7 +502,7 @@ func runOnboard(args []string) { apiKeyEnvFlag := fs.String("api-key-env", "", "environment variable name for API key") _ = fs.Parse(args) - cfg, _ := loadUserDefaultConfig() + cfg, _ := config.LoadUserDefaultConfig() if *providerFlag != "" || *modelFlag != "" || *baseURLFlag != "" || *apiKeyEnvFlag != "" { if *providerFlag != "" { cfg.Embedding.Provider = *providerFlag @@ -532,12 +523,12 @@ func runOnboard(args []string) { if *apiKeyEnvFlag != "" { cfg.Embedding.APIKeyEnv = *apiKeyEnvFlag } - cfg.normalize() - if err := saveUserDefaultConfig(cfg); err != nil { + cfg.Normalize() + if err := config.SaveUserDefaultConfig(cfg); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - path, _ := userDefaultConfigPath() + path, _ := config.UserDefaultConfigPath() fmt.Printf("saved to %s\n", path) fmt.Printf("provider: %s\n", cfg.Embedding.Provider) fmt.Printf("model: %s\n", cfg.Embedding.Model) @@ -605,7 +596,7 @@ func runOnboard(args []string) { } else { cfg.Embedding.APIKeyEnv = "" } - cfg.normalize() + cfg.Normalize() fmt.Println() fmt.Println("Configuration:") @@ -625,12 +616,12 @@ func runOnboard(args []string) { return } - if err := saveUserDefaultConfig(cfg); err != nil { + if err := config.SaveUserDefaultConfig(cfg); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - path, _ := userDefaultConfigPath() + path, _ := config.UserDefaultConfigPath() fmt.Printf("saved to %s\n", path) fmt.Println() fmt.Println("You can now run 'codeindex init' in your projects to use these defaults.") diff --git a/src/main_test.go b/cmd/codeindex/main_test.go similarity index 79% rename from src/main_test.go rename to cmd/codeindex/main_test.go index e771dab..73884ef 100644 --- a/src/main_test.go +++ b/cmd/codeindex/main_test.go @@ -6,6 +6,9 @@ import ( "reflect" "strings" "testing" + + "github.com/QuinsZouls/code-index/internal/config" + "github.com/QuinsZouls/code-index/internal/utils" ) func TestSplitCSV(t *testing.T) { @@ -18,10 +21,10 @@ func TestSplitCSV(t *testing.T) { func TestFindProjectRoot(t *testing.T) { root := t.TempDir() - if err := os.MkdirAll(filepath.Join(root, settingsDirName), 0o755); err != nil { + if err := os.MkdirAll(filepath.Join(root, config.SettingsDirName), 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(settingsPath(root), []byte("{}"), 0o644); err != nil { + if err := os.WriteFile(config.SettingsPath(root), []byte("{}"), 0o644); err != nil { t.Fatal(err) } nested := filepath.Join(root, "nested", "deeper") @@ -69,7 +72,7 @@ func TestReadChunkContent(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := readChunkContent(root, "test.go", tt.startLine, tt.endLine) + got := utils.ReadChunkContent(root, "test.go", tt.startLine, tt.endLine) if got != tt.want { t.Fatalf("readChunkContent() = %q, want %q", got, tt.want) } @@ -77,14 +80,14 @@ func TestReadChunkContent(t *testing.T) { } t.Run("missing file", func(t *testing.T) { - got := readChunkContent(root, "missing.go", 1, 5) + got := utils.ReadChunkContent(root, "missing.go", 1, 5) if !strings.Contains(got, "file unavailable") { t.Fatalf("expected error message, got %q", got) } }) t.Run("invalid start line", func(t *testing.T) { - got := readChunkContent(root, "test.go", 0, 5) + got := utils.ReadChunkContent(root, "test.go", 0, 5) if got != "[line range invalid]" { t.Fatalf("expected invalid range message, got %q", got) } diff --git a/src/config.go b/internal/config/config.go similarity index 82% rename from src/config.go rename to internal/config/config.go index b180c81..ce279e7 100644 --- a/src/config.go +++ b/internal/config/config.go @@ -1,4 +1,4 @@ -package main +package config import ( "encoding/json" @@ -10,8 +10,8 @@ import ( ) const ( - settingsDirName = ".codeindex" - settingsFileName = "settings.json" + SettingsDirName = ".codeindex" + SettingsFileName = "settings.json" indexFileName = "index.gob" configVersion = 1 ) @@ -110,7 +110,7 @@ func defaultConfig() Config { } } -func (c *Config) normalize() { +func (c *Config) Normalize() { if c.Version == 0 { c.Version = configVersion } @@ -161,10 +161,10 @@ func (c *Config) normalize() { c.VectorWeight = 0.7 c.KeywordWeight = 0.3 } - c.Embedding.normalize() + c.Embedding.Normalize() } -func (e *EmbeddingConfig) normalize() { +func (e *EmbeddingConfig) Normalize() { e.Provider = strings.ToLower(strings.TrimSpace(e.Provider)) if e.Provider == "" { e.Provider = "openai" @@ -221,8 +221,8 @@ func (e *EmbeddingConfig) normalize() { } } -func loadConfig(projectRoot string) (Config, error) { - path := settingsPath(projectRoot) +func LoadConfig(projectRoot string) (Config, error) { + path := SettingsPath(projectRoot) data, err := os.ReadFile(path) if err != nil { if errors.Is(err, os.ErrNotExist) { @@ -235,13 +235,13 @@ func loadConfig(projectRoot string) (Config, error) { if err := json.Unmarshal(data, &cfg); err != nil { return Config{}, fmt.Errorf("parse config %s: %w", path, err) } - cfg.normalize() + cfg.Normalize() return cfg, nil } -func saveConfig(projectRoot string, cfg Config) error { - cfg.normalize() - if err := os.MkdirAll(settingsDir(projectRoot), 0o755); err != nil { +func SaveConfig(projectRoot string, cfg Config) error { + cfg.Normalize() + if err := os.MkdirAll(SettingsDir(projectRoot), 0o755); err != nil { return err } data, err := json.MarshalIndent(cfg, "", " ") @@ -249,27 +249,27 @@ func saveConfig(projectRoot string, cfg Config) error { return err } data = append(data, '\n') - return os.WriteFile(settingsPath(projectRoot), data, 0o644) + return os.WriteFile(SettingsPath(projectRoot), data, 0o644) } -func initProject(projectRoot string) (Config, error) { - cfg, _ := loadUserDefaultConfig() - if err := saveConfig(projectRoot, cfg); err != nil { +func InitProject(projectRoot string) (Config, error) { + cfg, _ := LoadUserDefaultConfig() + if err := SaveConfig(projectRoot, cfg); err != nil { return Config{}, err } - if err := ensureGitignore(projectRoot); err != nil { + if err := EnsureGitignore(projectRoot); err != nil { return Config{}, err } return cfg, nil } -func loadUserDefaultConfig() (Config, error) { +func LoadUserDefaultConfig() (Config, error) { home, err := os.UserHomeDir() if err != nil { cfg := defaultConfig() return cfg, nil } - path := filepath.Join(home, settingsDirName, "default_settings.json") + path := filepath.Join(home, SettingsDirName, "default_settings.json") data, err := os.ReadFile(path) if err != nil { if errors.Is(err, os.ErrNotExist) { @@ -282,21 +282,21 @@ func loadUserDefaultConfig() (Config, error) { if err := json.Unmarshal(data, &cfg); err != nil { return Config{}, fmt.Errorf("parse user default config %s: %w", path, err) } - cfg.normalize() + cfg.Normalize() return cfg, nil } -func saveUserDefaultConfig(cfg Config) error { +func SaveUserDefaultConfig(cfg Config) error { home, err := os.UserHomeDir() if err != nil { return err } - dir := filepath.Join(home, settingsDirName) + dir := filepath.Join(home, SettingsDirName) if err := os.MkdirAll(dir, 0o755); err != nil { return err } path := filepath.Join(dir, "default_settings.json") - cfg.normalize() + cfg.Normalize() data, err := json.MarshalIndent(cfg, "", " ") if err != nil { return err @@ -305,32 +305,32 @@ func saveUserDefaultConfig(cfg Config) error { return os.WriteFile(path, data, 0o644) } -func userDefaultConfigPath() (string, error) { +func UserDefaultConfigPath() (string, error) { home, err := os.UserHomeDir() if err != nil { return "", err } - return filepath.Join(home, settingsDirName, "default_settings.json"), nil + return filepath.Join(home, SettingsDirName, "default_settings.json"), nil } -func settingsDir(projectRoot string) string { - return filepath.Join(projectRoot, settingsDirName) +func SettingsDir(projectRoot string) string { + return filepath.Join(projectRoot, SettingsDirName) } -func settingsPath(projectRoot string) string { - return filepath.Join(settingsDir(projectRoot), settingsFileName) +func SettingsPath(projectRoot string) string { + return filepath.Join(SettingsDir(projectRoot), SettingsFileName) } -func indexPath(projectRoot string) string { - return filepath.Join(settingsDir(projectRoot), indexFileName) +func IndexPath(projectRoot string) string { + return filepath.Join(SettingsDir(projectRoot), indexFileName) } -func ensureGitignore(projectRoot string) error { +func EnsureGitignore(projectRoot string) error { if _, err := os.Stat(filepath.Join(projectRoot, ".git")); err != nil { return nil } path := filepath.Join(projectRoot, ".gitignore") - entry := "/" + settingsDirName + "/" + entry := "/" + SettingsDirName + "/" content, err := os.ReadFile(path) if err != nil && !errors.Is(err, os.ErrNotExist) { return err @@ -349,7 +349,7 @@ func ensureGitignore(projectRoot string) error { return os.WriteFile(path, content, 0o644) } -func apiKey(cfg EmbeddingConfig) string { +func APIKey(cfg EmbeddingConfig) string { if cfg.APIKey != "" { return cfg.APIKey } @@ -370,7 +370,16 @@ func apiKey(cfg EmbeddingConfig) string { } } -func (c Config) embeddingSignature() string { +func (c Config) EmbeddingSignature() string { b, _ := json.Marshal(c.Embedding) return string(b) } + +func containsString(values []string, target string) bool { + for _, v := range values { + if v == target { + return true + } + } + return false +} diff --git a/src/config_test.go b/internal/config/config_test.go similarity index 84% rename from src/config_test.go rename to internal/config/config_test.go index 49ede0a..6b041fe 100644 --- a/src/config_test.go +++ b/internal/config/config_test.go @@ -1,4 +1,4 @@ -package main +package config import ( "encoding/json" @@ -11,7 +11,7 @@ import ( func TestDefaultConfigNormalize(t *testing.T) { cfg := defaultConfig() - cfg.normalize() + cfg.Normalize() if cfg.Embedding.Provider != "openai" { t.Fatalf("provider = %q, want openai", cfg.Embedding.Provider) } @@ -33,7 +33,7 @@ func TestConfigSupportsIndexerTuning(t *testing.T) { cfg := defaultConfig() cfg.WorkerCount = 12 cfg.CheckpointEvery = 25 - cfg.normalize() + cfg.Normalize() if cfg.WorkerCount != 12 || cfg.CheckpointEvery != 25 { t.Fatalf("tuning values lost: %#v", cfg) } @@ -46,10 +46,10 @@ func TestSaveLoadConfigRoundTrip(t *testing.T) { cfg.ExcludePatterns = []string{"**/tmp"} cfg.LanguageOverrides = map[string]string{"inc": "php"} cfg.Embedding = EmbeddingConfig{Provider: "openai-compatible", Model: "embed-1", BaseURL: "https://example.com/v1", APIKeyEnv: "MY_KEY"} - if err := saveConfig(root, cfg); err != nil { + if err := SaveConfig(root, cfg); err != nil { t.Fatal(err) } - loaded, err := loadConfig(root) + loaded, err := LoadConfig(root) if err != nil { t.Fatal(err) } @@ -73,7 +73,7 @@ func TestInitProjectCreatesGitignore(t *testing.T) { t.Fatal(err) } home := t.TempDir() - if err := os.MkdirAll(filepath.Join(home, settingsDirName), 0o755); err != nil { + if err := os.MkdirAll(filepath.Join(home, SettingsDirName), 0o755); err != nil { t.Fatal(err) } t.Setenv("HOME", home) @@ -83,13 +83,13 @@ func TestInitProjectCreatesGitignore(t *testing.T) { if err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(home, settingsDirName, "default_settings.json"), append(data, '\n'), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(home, SettingsDirName, "default_settings.json"), append(data, '\n'), 0o644); err != nil { t.Fatal(err) } - if _, err := initProject(root); err != nil { + if _, err := InitProject(root); err != nil { t.Fatal(err) } - projectCfg, err := loadConfig(root) + projectCfg, err := LoadConfig(root) if err != nil { t.Fatal(err) } @@ -113,10 +113,10 @@ func TestInitProjectIgnoresMissingUserDefaults(t *testing.T) { } home := t.TempDir() t.Setenv("HOME", home) - if _, err := initProject(root); err != nil { + if _, err := InitProject(root); err != nil { t.Fatal(err) } - projectCfg, err := loadConfig(root) + projectCfg, err := LoadConfig(root) if err != nil { t.Fatal(err) } @@ -127,17 +127,17 @@ func TestInitProjectIgnoresMissingUserDefaults(t *testing.T) { func TestAPIKeyResolution(t *testing.T) { t.Setenv("OPENAI_API_KEY", "from-env") - if got := apiKey(EmbeddingConfig{Provider: "openai"}); got != "from-env" { - t.Fatalf("apiKey() = %q, want from-env", got) + if got := APIKey(EmbeddingConfig{Provider: "openai"}); got != "from-env" { + t.Fatalf("APIKey() = %q, want from-env", got) } - if got := apiKey(EmbeddingConfig{Provider: "openai", APIKey: "explicit"}); got != "explicit" { - t.Fatalf("apiKey() = %q, want explicit", got) + if got := APIKey(EmbeddingConfig{Provider: "openai", APIKey: "explicit"}); got != "explicit" { + t.Fatalf("APIKey() = %q, want explicit", got) } } func TestEmbeddingConfigRateLimit(t *testing.T) { cfg := EmbeddingConfig{RateLimit: 10} - cfg.normalize() + cfg.Normalize() if cfg.RateLimit != 10 { t.Fatalf("RateLimit = %d, want 10", cfg.RateLimit) } @@ -145,7 +145,7 @@ func TestEmbeddingConfigRateLimit(t *testing.T) { func TestEmbeddingConfigTimeoutDefault(t *testing.T) { cfg := EmbeddingConfig{} - cfg.normalize() + cfg.Normalize() if cfg.Timeout != "60s" { t.Fatalf("Timeout = %q, want 60s", cfg.Timeout) } @@ -153,7 +153,7 @@ func TestEmbeddingConfigTimeoutDefault(t *testing.T) { func TestEmbeddingConfigTimeoutCustom(t *testing.T) { cfg := EmbeddingConfig{Timeout: "30s"} - cfg.normalize() + cfg.Normalize() if cfg.Timeout != "30s" { t.Fatalf("Timeout = %q, want 30s", cfg.Timeout) } @@ -164,10 +164,10 @@ func TestConfigWithRateLimitRoundTrip(t *testing.T) { cfg := defaultConfig() cfg.Embedding.RateLimit = 15 cfg.Embedding.Timeout = "45s" - if err := saveConfig(root, cfg); err != nil { + if err := SaveConfig(root, cfg); err != nil { t.Fatal(err) } - loaded, err := loadConfig(root) + loaded, err := LoadConfig(root) if err != nil { t.Fatal(err) } @@ -181,7 +181,7 @@ func TestConfigWithRateLimitRoundTrip(t *testing.T) { func TestEmbeddingConfigRetryDefaults(t *testing.T) { cfg := EmbeddingConfig{} - cfg.normalize() + cfg.Normalize() if cfg.MaxRetries != 0 { t.Fatalf("MaxRetries = %d, want 0 (disabled by default)", cfg.MaxRetries) } @@ -199,10 +199,10 @@ func TestConfigWithRetryRoundTrip(t *testing.T) { cfg.Embedding.MaxRetries = 3 cfg.Embedding.RetryInitialDelay = "2s" cfg.Embedding.RetryMaxDelay = "20s" - if err := saveConfig(root, cfg); err != nil { + if err := SaveConfig(root, cfg); err != nil { t.Fatal(err) } - loaded, err := loadConfig(root) + loaded, err := LoadConfig(root) if err != nil { t.Fatal(err) } @@ -223,10 +223,10 @@ func TestSaveUserDefaultConfig(t *testing.T) { cfg := defaultConfig() cfg.Embedding.Provider = "ollama" cfg.Embedding.Model = "nomic-embed-text" - if err := saveUserDefaultConfig(cfg); err != nil { + if err := SaveUserDefaultConfig(cfg); err != nil { t.Fatal(err) } - path := filepath.Join(home, settingsDirName, "default_settings.json") + path := filepath.Join(home, SettingsDirName, "default_settings.json") data, err := os.ReadFile(path) if err != nil { t.Fatal(err) @@ -246,23 +246,23 @@ func TestSaveUserDefaultConfig(t *testing.T) { func TestSaveUserDefaultConfigUpdatesExisting(t *testing.T) { home := t.TempDir() t.Setenv("HOME", home) - if err := os.MkdirAll(filepath.Join(home, settingsDirName), 0o755); err != nil { + if err := os.MkdirAll(filepath.Join(home, SettingsDirName), 0o755); err != nil { t.Fatal(err) } existing := defaultConfig() existing.Embedding.Provider = "openai" existing.Embedding.Model = "text-embedding-3-large" data, _ := json.MarshalIndent(existing, "", " ") - if err := os.WriteFile(filepath.Join(home, settingsDirName, "default_settings.json"), append(data, '\n'), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(home, SettingsDirName, "default_settings.json"), append(data, '\n'), 0o644); err != nil { t.Fatal(err) } cfg := defaultConfig() cfg.Embedding.Provider = "mistral" cfg.Embedding.Model = "mistral-embed" - if err := saveUserDefaultConfig(cfg); err != nil { + if err := SaveUserDefaultConfig(cfg); err != nil { t.Fatal(err) } - loaded, err := loadUserDefaultConfig() + loaded, err := LoadUserDefaultConfig() if err != nil { t.Fatal(err) } @@ -277,11 +277,11 @@ func TestSaveUserDefaultConfigUpdatesExisting(t *testing.T) { func TestUserDefaultConfigPath(t *testing.T) { home := t.TempDir() t.Setenv("HOME", home) - path, err := userDefaultConfigPath() + path, err := UserDefaultConfigPath() if err != nil { t.Fatal(err) } - expected := filepath.Join(home, settingsDirName, "default_settings.json") + expected := filepath.Join(home, SettingsDirName, "default_settings.json") if path != expected { t.Fatalf("path = %q, want %q", path, expected) } diff --git a/src/cli_e2e_test.go b/internal/daemon/cli_e2e_test.go similarity index 88% rename from src/cli_e2e_test.go rename to internal/daemon/cli_e2e_test.go index f2a2378..c8bca71 100644 --- a/src/cli_e2e_test.go +++ b/internal/daemon/cli_e2e_test.go @@ -1,4 +1,4 @@ -package main +package daemon import ( "encoding/json" @@ -10,6 +10,8 @@ import ( "runtime" "strings" "testing" + + "github.com/QuinsZouls/code-index/internal/config" ) func TestCLIE2E(t *testing.T) { @@ -54,7 +56,7 @@ func TestCLIE2E(t *testing.T) { if runtime.GOOS == "windows" { bin += ".exe" } - build := exec.Command("go", "build", "-o", bin, "./src") + build := exec.Command("go", "build", "-o", bin, "../cmd/codeindex") build.Dir = projectRoot(t) if out, err := build.CombinedOutput(); err != nil { t.Fatalf("go build failed: %v\n%s", err, out) @@ -74,17 +76,17 @@ func TestCLIE2E(t *testing.T) { if out := run("init", "-path", root); !strings.Contains(out, "initialized:") { t.Fatalf("init output = %q", out) } - settings := filepath.Join(root, settingsDirName, settingsFileName) + settings := filepath.Join(root, config.SettingsDirName, config.SettingsFileName) data, err := os.ReadFile(settings) if err != nil { t.Fatal(err) } - var cfg Config + var cfg config.Config if err := json.Unmarshal(data, &cfg); err != nil { t.Fatal(err) } - cfg.Embedding = EmbeddingConfig{Provider: "openai-compatible", Model: "fake", BaseURL: server.URL, APIKey: "test"} - if err := saveConfig(root, cfg); err != nil { + cfg.Embedding = config.EmbeddingConfig{Provider: "openai-compatible", Model: "fake", BaseURL: server.URL, APIKey: "test"} + if err := config.SaveConfig(root, cfg); err != nil { t.Fatal(err) } diff --git a/src/daemon.go b/internal/daemon/daemon.go similarity index 88% rename from src/daemon.go rename to internal/daemon/daemon.go index d2246f5..6ce5162 100644 --- a/src/daemon.go +++ b/internal/daemon/daemon.go @@ -1,4 +1,4 @@ -package main +package daemon import ( "context" @@ -12,6 +12,10 @@ import ( "sync" "syscall" "time" + + "github.com/QuinsZouls/code-index/internal/config" + "github.com/QuinsZouls/code-index/internal/index" + "github.com/QuinsZouls/code-index/internal/types" ) type FileChangeType int @@ -23,8 +27,8 @@ const ( type Daemon struct { projectRoot string - cfg Config - indexer *Indexer + cfg config.Config + indexer *index.Indexer interval time.Duration debounce time.Duration verbose bool @@ -34,7 +38,7 @@ type Daemon struct { releaseLock func() } -func runDaemon(args []string) { +func RunDaemon(args []string) { if len(args) < 1 { printDaemonUsage() os.Exit(1) @@ -92,12 +96,13 @@ func runDaemonStart(args []string) { return } - cfg, err := loadConfig(root) +cfg, err := config.LoadConfig(root) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } _ = cfg + _ = index.NewIndexer lockPath := lockFilePath(root) data, err := os.ReadFile(lockPath) @@ -111,7 +116,7 @@ func runDaemonStart(args []string) { } } - indexFile := indexPath(root) + indexFile := config.IndexPath(root) if _, err := os.Stat(indexFile); os.IsNotExist(err) { fmt.Fprintln(os.Stderr, "No index found. Run 'codeindex index' first.") os.Exit(1) @@ -153,13 +158,12 @@ func runDaemonStart(args []string) { } func runDaemonProcess(root string, interval, debounce time.Duration, verbose bool) { - cfg, err := loadConfig(root) + cfg, err := config.LoadConfig(root) if err != nil { - fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err) + fmt.Fprintln(os.Stderr, err) os.Exit(1) } - - indexer, err := newIndexer(root, cfg) + indexer, err := index.NewIndexer(root, cfg) if err != nil { fmt.Fprintf(os.Stderr, "failed to create indexer: %v\n", err) os.Exit(1) @@ -306,9 +310,9 @@ func runDaemonStatus(args []string) { fmt.Printf("Debounce: %s\n", info.Debounce) if status == "running" { - cfg, err := loadConfig(root) + cfg, err := config.LoadConfig(root) if err == nil { - indexer, err := newIndexer(root, cfg) + indexer, err := index.NewIndexer(root, cfg) if err == nil { s := indexer.Status() fmt.Printf("Files: %d\n", s.Files) @@ -377,7 +381,7 @@ func (d *Daemon) Stop() { func (d *Daemon) scan() map[string]FileChangeType { changes := make(map[string]FileChangeType) - files, err := walkFiles(d.projectRoot, d.cfg) + files, err := index.WalkFiles(d.projectRoot, d.cfg) if err != nil { return changes } @@ -394,7 +398,7 @@ func (d *Daemon) scan() map[string]FileChangeType { continue } - prevState, exists := d.indexer.index.Files[rel] + prevState, exists := d.indexer.IndexData().Files[rel] if !exists { changes[rel] = FileModified continue @@ -407,7 +411,7 @@ func (d *Daemon) scan() map[string]FileChangeType { } } - for rel := range d.indexer.index.Files { + for rel := range d.indexer.IndexData().Files { if _, exists := fileSet[rel]; !exists { changes[rel] = FileDeleted } @@ -439,8 +443,8 @@ func (d *Daemon) processBatch() { if len(deleted) > 0 { for _, path := range deleted { - delete(d.indexer.index.Files, path) - delete(d.indexer.index.ChunksByFile, path) + delete(d.indexer.IndexData().Files, path) + delete(d.indexer.IndexData().ChunksByFile, path) if d.verbose { fmt.Printf("[~] %s (deleted)\n", path) } @@ -466,7 +470,7 @@ func (d *Daemon) processBatch() { continue } - chunks := d.indexer.fileChunks(rel, string(data)) + chunks := d.indexer.FileChunks(rel, string(data)) if len(chunks) == 0 { continue } @@ -476,7 +480,7 @@ func (d *Daemon) processBatch() { texts = append(texts, ch.Content) } - vecs, err := d.indexer.provider.Embed(ctx, texts) + vecs, err := d.indexer.Provider.Embed(ctx, texts) if err != nil { if d.verbose { fmt.Printf("[!] %s: embed failed: %v\n", rel, err) @@ -484,28 +488,29 @@ func (d *Daemon) processBatch() { continue } - records := make([]ChunkRecord, 0, len(chunks)) - lang := d.indexer.languageFor(rel) + records := make([]types.ChunkRecord, 0, len(chunks)) + lang := d.indexer.LanguageFor(rel) for idx, ch := range chunks { - records = append(records, ChunkRecord{ + records = append(records, types.ChunkRecord{ FilePath: rel, Language: lang, StartLine: ch.StartLine, EndLine: ch.EndLine, Embedding: vecs[idx], - ChunkHash: fileHash([]byte(ch.Content)), + ChunkHash: index.FileHash([]byte(ch.Content)), }) } - hash := fileHash(data) - _, existed := d.indexer.index.Files[rel] - d.indexer.index.Files[rel] = FileState{ + hash := index.FileHash(data) + idxData := d.indexer.IndexData() + _, existed := idxData.Files[rel] + idxData.Files[rel] = types.FileState{ Hash: hash, ChunkCount: len(records), Size: info.Size(), ModTimeUnixNano: info.ModTime().UTC().UnixNano(), } - d.indexer.index.ChunksByFile[rel] = records + idxData.ChunksByFile[rel] = records if d.verbose { if !existed { @@ -517,8 +522,8 @@ func (d *Daemon) processBatch() { } } - if len(modified) > 0 || len(deleted) > 0 { - if err := saveIndex(indexPath(d.projectRoot), d.indexer.index); err != nil { +if len(modified) > 0 || len(deleted) > 0 { + if err := index.SaveIndex(config.IndexPath(d.projectRoot), d.indexer.IndexData()); err != nil { if d.verbose { fmt.Printf("[!] failed to save index: %v\n", err) } diff --git a/src/daemon_proc_unix.go b/internal/daemon/proc_unix.go similarity index 91% rename from src/daemon_proc_unix.go rename to internal/daemon/proc_unix.go index d1c48f5..005e004 100644 --- a/src/daemon_proc_unix.go +++ b/internal/daemon/proc_unix.go @@ -1,6 +1,6 @@ //go:build !windows -package main +package daemon import ( "os/exec" diff --git a/src/daemon_proc_windows.go b/internal/daemon/proc_windows.go similarity index 90% rename from src/daemon_proc_windows.go rename to internal/daemon/proc_windows.go index fed9eaf..c5cdf74 100644 --- a/src/daemon_proc_windows.go +++ b/internal/daemon/proc_windows.go @@ -1,6 +1,6 @@ //go:build windows -package main +package daemon import ( "os/exec" diff --git a/src/registry.go b/internal/daemon/registry.go similarity index 97% rename from src/registry.go rename to internal/daemon/registry.go index b8d8e61..fb4b481 100644 --- a/src/registry.go +++ b/internal/daemon/registry.go @@ -1,4 +1,4 @@ -package main +package daemon import ( "encoding/json" @@ -10,6 +10,8 @@ import ( "strconv" "syscall" "time" + + "github.com/QuinsZouls/code-index/internal/index" ) type DaemonInfo struct { @@ -33,7 +35,7 @@ func registryPath() (string, error) { } func lockFilePath(projectRoot string) string { - hash := fileHash([]byte(projectRoot)) + hash := index.FileHash([]byte(projectRoot)) return filepath.Join(os.TempDir(), "codeindex-"+hash[:16]+".lock") } diff --git a/src/embeddings.go b/internal/embeddings/embeddings.go similarity index 95% rename from src/embeddings.go rename to internal/embeddings/embeddings.go index 736f3d7..5192466 100644 --- a/src/embeddings.go +++ b/internal/embeddings/embeddings.go @@ -1,4 +1,4 @@ -package main +package embeddings import ( "bytes" @@ -13,14 +13,16 @@ import ( "strings" "sync" "time" + + "github.com/QuinsZouls/code-index/internal/config" ) type EmbeddingProvider interface { Embed(ctx context.Context, texts []string) ([][]float32, error) } -func newEmbeddingProvider(cfg EmbeddingConfig) (EmbeddingProvider, error) { - cfg.normalize() +func NewEmbeddingProvider(cfg config.EmbeddingConfig) (EmbeddingProvider, error) { + cfg.Normalize() switch cfg.Provider { case "openai", "openai-compatible", "openrouter", "mistral", "lmstudio", "llamacpp": return newOpenAICompatibleProvider(cfg), nil @@ -80,7 +82,7 @@ type retryConfig struct { currentDelay time.Duration } -func newRetryConfig(cfg EmbeddingConfig) *retryConfig { +func newRetryConfig(cfg config.EmbeddingConfig) *retryConfig { maxRetries := cfg.MaxRetries if maxRetries <= 0 { return nil @@ -188,7 +190,7 @@ type openAICompatibleProvider struct { retryCfg *retryConfig } -func newOpenAICompatibleProvider(cfg EmbeddingConfig) *openAICompatibleProvider { +func newOpenAICompatibleProvider(cfg config.EmbeddingConfig) *openAICompatibleProvider { timeout, _ := time.ParseDuration(cfg.Timeout) if timeout == 0 { timeout = 60 * time.Second @@ -196,7 +198,7 @@ func newOpenAICompatibleProvider(cfg EmbeddingConfig) *openAICompatibleProvider return &openAICompatibleProvider{ baseURL: strings.TrimRight(cfg.BaseURL, "/"), model: cfg.Model, - apiKey: apiKey(cfg), + apiKey: config.APIKey(cfg), headers: cfg.Headers, client: &http.Client{Timeout: timeout}, limiter: newSimpleRateLimiter(cfg.RateLimit), @@ -276,7 +278,7 @@ type geminiProvider struct { retryCfg *retryConfig } -func newGeminiProvider(cfg EmbeddingConfig) *geminiProvider { +func newGeminiProvider(cfg config.EmbeddingConfig) *geminiProvider { timeout, _ := time.ParseDuration(cfg.Timeout) if timeout == 0 { timeout = 60 * time.Second @@ -284,7 +286,7 @@ func newGeminiProvider(cfg EmbeddingConfig) *geminiProvider { return &geminiProvider{ baseURL: strings.TrimRight(cfg.BaseURL, "/"), model: cfg.Model, - apiKey: apiKey(cfg), + apiKey: config.APIKey(cfg), client: &http.Client{Timeout: timeout}, limiter: newSimpleRateLimiter(cfg.RateLimit), retryCfg: newRetryConfig(cfg), @@ -364,7 +366,7 @@ type ollamaProvider struct { retryCfg *retryConfig } -func newOllamaProvider(cfg EmbeddingConfig) *ollamaProvider { +func newOllamaProvider(cfg config.EmbeddingConfig) *ollamaProvider { timeout, _ := time.ParseDuration(cfg.Timeout) if timeout == 0 { timeout = 60 * time.Second diff --git a/src/embeddings_test.go b/internal/embeddings/embeddings_test.go similarity index 91% rename from src/embeddings_test.go rename to internal/embeddings/embeddings_test.go index c3741f5..b0a2790 100644 --- a/src/embeddings_test.go +++ b/internal/embeddings/embeddings_test.go @@ -1,4 +1,4 @@ -package main +package embeddings import ( "context" @@ -10,6 +10,8 @@ import ( "reflect" "testing" "time" + + "github.com/QuinsZouls/code-index/internal/config" ) func approxEqual(a, b float32) bool { @@ -51,7 +53,7 @@ func TestOpenAICompatibleProviderEmbed(t *testing.T) { })) defer server.Close() - p := newOpenAICompatibleProvider(EmbeddingConfig{BaseURL: server.URL, Model: "embed-1", APIKey: "secret"}) + p := newOpenAICompatibleProvider(config.EmbeddingConfig{BaseURL: server.URL, Model: "embed-1", APIKey: "secret"}) vecs, err := p.Embed(context.Background(), []string{"hello", "world"}) if err != nil { t.Fatal(err) @@ -83,7 +85,7 @@ func TestGeminiProviderEmbed(t *testing.T) { })) defer server.Close() - p := newGeminiProvider(EmbeddingConfig{BaseURL: server.URL, Model: "gem-embed"}) + p := newGeminiProvider(config.EmbeddingConfig{BaseURL: server.URL, Model: "gem-embed"}) vecs, err := p.Embed(context.Background(), []string{"query"}) if err != nil { t.Fatal(err) @@ -112,7 +114,7 @@ func TestOllamaProviderEmbed(t *testing.T) { })) defer server.Close() - p := newOllamaProvider(EmbeddingConfig{BaseURL: server.URL, Model: "ollama-embed"}) + p := newOllamaProvider(config.EmbeddingConfig{BaseURL: server.URL, Model: "ollama-embed"}) vecs, err := p.Embed(context.Background(), []string{"hello"}) if err != nil { t.Fatal(err) @@ -123,7 +125,7 @@ func TestOllamaProviderEmbed(t *testing.T) { } func TestUnsupportedEmbeddingProvider(t *testing.T) { - if _, err := newEmbeddingProvider(EmbeddingConfig{Provider: "nope"}); err == nil { + if _, err := NewEmbeddingProvider(config.EmbeddingConfig{Provider: "nope"}); err == nil { t.Fatal("expected error") } } @@ -175,7 +177,7 @@ func TestOpenAICompatibleProviderWithRateLimit(t *testing.T) { })) defer server.Close() - p := newOpenAICompatibleProvider(EmbeddingConfig{ + p := newOpenAICompatibleProvider(config.EmbeddingConfig{ BaseURL: server.URL, Model: "test", RateLimit: 5, @@ -205,7 +207,7 @@ func TestOpenAICompatibleProviderWithTimeout(t *testing.T) { })) defer server.Close() - p := newOpenAICompatibleProvider(EmbeddingConfig{ + p := newOpenAICompatibleProvider(config.EmbeddingConfig{ BaseURL: server.URL, Model: "test", Timeout: "100ms", @@ -224,7 +226,7 @@ func TestGeminiProviderWithRateLimit(t *testing.T) { })) defer server.Close() - p := newGeminiProvider(EmbeddingConfig{ + p := newGeminiProvider(config.EmbeddingConfig{ BaseURL: server.URL, Model: "test", RateLimit: 5, @@ -251,7 +253,7 @@ func TestOllamaProviderWithRateLimit(t *testing.T) { })) defer server.Close() - p := newOllamaProvider(EmbeddingConfig{ + p := newOllamaProvider(config.EmbeddingConfig{ BaseURL: server.URL, Model: "test", RateLimit: 5, @@ -271,14 +273,14 @@ func TestOllamaProviderWithRateLimit(t *testing.T) { } func TestRetryConfigNil(t *testing.T) { - cfg := newRetryConfig(EmbeddingConfig{MaxRetries: 0}) + cfg := newRetryConfig(config.EmbeddingConfig{MaxRetries: 0}) if cfg != nil { t.Fatal("expected nil retry config for max_retries=0") } } func TestRetryConfigBasic(t *testing.T) { - cfg := newRetryConfig(EmbeddingConfig{ + cfg := newRetryConfig(config.EmbeddingConfig{ MaxRetries: 3, RetryInitialDelay: "1s", RetryMaxDelay: "10s", @@ -298,7 +300,7 @@ func TestRetryConfigBasic(t *testing.T) { } func TestRetryConfigExponentialBackoff(t *testing.T) { - cfg := newRetryConfig(EmbeddingConfig{ + cfg := newRetryConfig(config.EmbeddingConfig{ MaxRetries: 5, RetryInitialDelay: "1s", RetryMaxDelay: "16s", @@ -323,7 +325,7 @@ func TestRetryConfigExponentialBackoff(t *testing.T) { } func TestRetryConfigMaxDelay(t *testing.T) { - cfg := newRetryConfig(EmbeddingConfig{ + cfg := newRetryConfig(config.EmbeddingConfig{ MaxRetries: 5, RetryInitialDelay: "1s", RetryMaxDelay: "4s", @@ -375,7 +377,7 @@ func TestIsRetryableError(t *testing.T) { } func TestRetryWithBackoffSuccess(t *testing.T) { - cfg := newRetryConfig(EmbeddingConfig{ + cfg := newRetryConfig(config.EmbeddingConfig{ MaxRetries: 3, RetryInitialDelay: "100ms", RetryMaxDelay: "1s", @@ -397,7 +399,7 @@ func TestRetryWithBackoffSuccess(t *testing.T) { } func TestRetryWithBackoffMaxRetries(t *testing.T) { - cfg := newRetryConfig(EmbeddingConfig{ + cfg := newRetryConfig(config.EmbeddingConfig{ MaxRetries: 2, RetryInitialDelay: "50ms", RetryMaxDelay: "200ms", @@ -429,7 +431,7 @@ func TestOpenAICompatibleProviderWithRetry(t *testing.T) { })) defer server.Close() - p := newOpenAICompatibleProvider(EmbeddingConfig{ + p := newOpenAICompatibleProvider(config.EmbeddingConfig{ BaseURL: server.URL, Model: "test", MaxRetries: 3, diff --git a/src/llamacpp_test.go b/internal/embeddings/llamacpp_test.go similarity index 86% rename from src/llamacpp_test.go rename to internal/embeddings/llamacpp_test.go index 72a1c0c..51f089f 100644 --- a/src/llamacpp_test.go +++ b/internal/embeddings/llamacpp_test.go @@ -1,4 +1,4 @@ -package main +package embeddings import ( "context" @@ -6,11 +6,13 @@ import ( "net/http" "net/http/httptest" "testing" + + "github.com/QuinsZouls/code-index/internal/config" ) func TestLlamacppConfigNormalization(t *testing.T) { - cfg := EmbeddingConfig{Provider: "llamacpp"} - cfg.normalize() + cfg := config.EmbeddingConfig{Provider: "llamacpp"} + cfg.Normalize() if cfg.BaseURL != "http://localhost:8080/v1" { t.Errorf("expected BaseURL http://localhost:8080/v1, got %s", cfg.BaseURL) @@ -21,11 +23,11 @@ func TestLlamacppConfigNormalization(t *testing.T) { } func TestLlamacppConfigCustomURL(t *testing.T) { - cfg := EmbeddingConfig{ + cfg := config.EmbeddingConfig{ Provider: "llamacpp", BaseURL: "http://custom-host:9999/v1", } - cfg.normalize() + cfg.Normalize() if cfg.BaseURL != "http://custom-host:9999/v1" { t.Errorf("expected custom BaseURL, got %s", cfg.BaseURL) @@ -33,14 +35,14 @@ func TestLlamacppConfigCustomURL(t *testing.T) { } func TestLlamacppProviderCreation(t *testing.T) { - cfg := EmbeddingConfig{ + cfg := config.EmbeddingConfig{ Provider: "llamacpp", Model: "test-model", BaseURL: "http://localhost:8080/v1", } - cfg.normalize() + cfg.Normalize() - provider, err := newEmbeddingProvider(cfg) + provider, err := NewEmbeddingProvider(cfg) if err != nil { t.Fatalf("failed to create provider: %v", err) } @@ -86,14 +88,14 @@ func TestLlamacppEmbeddingRequest(t *testing.T) { })) defer server.Close() - cfg := EmbeddingConfig{ + cfg := config.EmbeddingConfig{ Provider: "llamacpp", Model: "test-model", BaseURL: server.URL + "/v1", } - cfg.normalize() + cfg.Normalize() - provider, err := newEmbeddingProvider(cfg) + provider, err := NewEmbeddingProvider(cfg) if err != nil { t.Fatalf("failed to create provider: %v", err) } @@ -121,14 +123,14 @@ func TestLlamacppEmbeddingError(t *testing.T) { })) defer server.Close() - cfg := EmbeddingConfig{ + cfg := config.EmbeddingConfig{ Provider: "llamacpp", Model: "test-model", BaseURL: server.URL + "/v1", } - cfg.normalize() + cfg.Normalize() - provider, err := newEmbeddingProvider(cfg) + provider, err := NewEmbeddingProvider(cfg) if err != nil { t.Fatalf("failed to create provider: %v", err) } diff --git a/src/chunk.go b/internal/index/chunk.go similarity index 83% rename from src/chunk.go rename to internal/index/chunk.go index d1a21ad..f43c05f 100644 --- a/src/chunk.go +++ b/internal/index/chunk.go @@ -1,8 +1,12 @@ -package main +package index -import "strings" +import ( + "strings" -func chunkText(text string, maxLines, overlap, contextSize int) []Chunk { + "github.com/QuinsZouls/code-index/internal/types" +) + +func chunkText(text string, maxLines, overlap, contextSize int) []types.Chunk { if maxLines <= 0 { maxLines = 120 } @@ -21,13 +25,13 @@ func chunkText(text string, maxLines, overlap, contextSize int) []Chunk { } if len(lines) <= maxLines { - return []Chunk{{Content: text, StartLine: 1, EndLine: len(lines)}} + return []types.Chunk{{Content: text, StartLine: 1, EndLine: len(lines)}} } step := maxLines - overlap if step <= 0 { step = maxLines } - chunks := make([]Chunk, 0, (len(lines)/step)+1) + chunks := make([]types.Chunk, 0, (len(lines)/step)+1) for start := 0; start < len(lines); start += step { end := start + maxLines if end > len(lines) { @@ -38,7 +42,7 @@ func chunkText(text string, maxLines, overlap, contextSize int) []Chunk { break } content := strings.Join(chunkLines, "\n") - chunks = append(chunks, Chunk{Content: content, StartLine: start + 1, EndLine: end}) + chunks = append(chunks, types.Chunk{Content: content, StartLine: start + 1, EndLine: end}) if end == len(lines) { break } @@ -48,13 +52,13 @@ func chunkText(text string, maxLines, overlap, contextSize int) []Chunk { // chunkByContextSize splits text into chunks that fit within contextSize characters, // respecting line boundaries when possible. -func chunkByContextSize(text string, contextSize, overlap int) []Chunk { +func chunkByContextSize(text string, contextSize, overlap int) []types.Chunk { lines := strings.Split(text, "\n") if len(lines) == 0 { return nil } - var chunks []Chunk + var chunks []types.Chunk startLine := 0 for startLine < len(lines) { @@ -78,7 +82,7 @@ func chunkByContextSize(text string, contextSize, overlap int) []Chunk { } content := strings.Join(lines[chunkStart:endLine], "\n") - chunks = append(chunks, Chunk{ + chunks = append(chunks, types.Chunk{ Content: content, StartLine: chunkStart + 1, EndLine: endLine, diff --git a/src/chunk_test.go b/internal/index/chunk_test.go similarity index 99% rename from src/chunk_test.go rename to internal/index/chunk_test.go index cde7fc8..6a99a0c 100644 --- a/src/chunk_test.go +++ b/internal/index/chunk_test.go @@ -1,6 +1,8 @@ -package main +package index -import "testing" +import ( + "testing" +) func TestChunkTextSingleChunk(t *testing.T) { got := chunkText("a\nb\nc", 10, 2, 0) diff --git a/src/index.go b/internal/index/index.go similarity index 71% rename from src/index.go rename to internal/index/index.go index aa64aa9..1c7da85 100644 --- a/src/index.go +++ b/internal/index/index.go @@ -1,4 +1,4 @@ -package main +package index import ( "context" @@ -11,6 +11,12 @@ import ( "strings" "sync" "time" + + "github.com/QuinsZouls/code-index/internal/config" + "github.com/QuinsZouls/code-index/internal/embeddings" + "github.com/QuinsZouls/code-index/internal/search" + "github.com/QuinsZouls/code-index/internal/types" + "github.com/QuinsZouls/code-index/internal/utils" ) type SearchOptions struct { @@ -24,34 +30,34 @@ type SearchOptions struct { type Indexer struct { projectRoot string - cfg Config - provider EmbeddingProvider + cfg config.Config + Provider embeddings.EmbeddingProvider index *IndexData - progressFn func(IndexProgress) + ProgressFn func(types.IndexProgress) } -func newIndexer(projectRoot string, cfg Config) (*Indexer, error) { - provider, err := newEmbeddingProvider(cfg.Embedding) +func NewIndexer(projectRoot string, cfg config.Config) (*Indexer, error) { + provider, err := embeddings.NewEmbeddingProvider(cfg.Embedding) if err != nil { return nil, err } - idx, err := loadIndex(indexPath(projectRoot)) + idx, err := loadIndex(config.IndexPath(projectRoot)) if err != nil { return nil, err } - sig := cfg.embeddingSignature() + sig := cfg.EmbeddingSignature() if idx == nil || idx.EmbeddingSignature != sig { idx = newIndexData(sig) } - return &Indexer{projectRoot: projectRoot, cfg: cfg, provider: provider, index: idx}, nil + return &Indexer{projectRoot: projectRoot, cfg: cfg, Provider: provider, index: idx}, nil } func (i *Indexer) Index(ctx context.Context) error { - files, err := walkFiles(i.projectRoot, i.cfg) + files, err := WalkFiles(i.projectRoot, i.cfg) if err != nil { return err } - currentSig := i.cfg.embeddingSignature() + currentSig := i.cfg.EmbeddingSignature() checkpointEvery := i.cfg.CheckpointEvery if checkpointEvery <= 0 { checkpointEvery = len(files) / 50 @@ -62,7 +68,7 @@ func (i *Indexer) Index(ctx context.Context) error { checkpointEvery = 100 } } - prevFiles := make(map[string]FileState, len(i.index.Files)) + prevFiles := make(map[string]types.FileState, len(i.index.Files)) for path, state := range i.index.Files { prevFiles[path] = state } @@ -77,7 +83,7 @@ func (i *Indexer) Index(ctx context.Context) error { size int64 modNano int64 kind string - records []ChunkRecord + records []types.ChunkRecord skipped bool err error } @@ -106,7 +112,7 @@ func (i *Indexer) Index(ctx context.Context) error { if !force && pendingWrites < checkpointEvery && time.Since(lastFlush) < 3*time.Second { return nil } - if err := saveIndex(indexPath(i.projectRoot), i.index); err != nil { + if err := saveIndex(config.IndexPath(i.projectRoot), i.index); err != nil { return err } pendingWrites = 0 @@ -140,27 +146,27 @@ func (i *Indexer) Index(ctx context.Context) error { results <- fileResult{rel: job.rel, err: err} continue } - hash := fileHash(data) - chunks := i.fileChunks(job.rel, string(data)) + hash := FileHash(data) + chunks := i.FileChunks(job.rel, string(data)) texts := make([]string, 0, len(chunks)) for _, ch := range chunks { texts = append(texts, ch.Content) } - vecs, err := i.provider.Embed(ctx, texts) + vecs, err := i.Provider.Embed(ctx, texts) if err != nil { results <- fileResult{rel: job.rel, err: fmt.Errorf("embed %s: %w", job.rel, err)} continue } - records := make([]ChunkRecord, 0, len(chunks)) - lang := i.languageFor(job.rel) + records := make([]types.ChunkRecord, 0, len(chunks)) + lang := i.LanguageFor(job.rel) for idx, ch := range chunks { - records = append(records, ChunkRecord{ + records = append(records, types.ChunkRecord{ FilePath: job.rel, Language: lang, StartLine: ch.StartLine, EndLine: ch.EndLine, Embedding: vecs[idx], - ChunkHash: fileHash([]byte(ch.Content)), + ChunkHash: FileHash([]byte(ch.Content)), }) } kind := "modified" @@ -176,8 +182,8 @@ func (i *Indexer) Index(ctx context.Context) error { } go func() { for idx, rel := range files { - if i.progressFn != nil { - i.progressFn(IndexProgress{Current: idx + 1, Total: len(files), File: rel, Action: "queued"}) + if i.ProgressFn != nil { + i.ProgressFn(types.IndexProgress{Current: idx + 1, Total: len(files), File: rel, Action: "queued"}) } jobs <- fileJob{rel: rel, idx: idx} } @@ -195,12 +201,12 @@ func (i *Indexer) Index(ctx context.Context) error { } seen[result.rel] = struct{}{} if result.skipped { - if i.progressFn != nil { - i.progressFn(IndexProgress{File: result.rel, Action: "skipped", Kind: result.kind}) + if i.ProgressFn != nil { + i.ProgressFn(types.IndexProgress{File: result.rel, Action: "skipped", Kind: result.kind}) } continue } - i.index.Files[result.rel] = FileState{Hash: result.hash, ChunkCount: len(result.records), Size: result.size, ModTimeUnixNano: result.modNano} + i.index.Files[result.rel] = types.FileState{Hash: result.hash, ChunkCount: len(result.records), Size: result.size, ModTimeUnixNano: result.modNano} i.index.ChunksByFile[result.rel] = result.records pendingWrites++ if err := flushIndex(false); err != nil { @@ -209,8 +215,8 @@ func (i *Indexer) Index(ctx context.Context) error { } continue } - if i.progressFn != nil { - i.progressFn(IndexProgress{File: result.rel, Action: "indexed", Kind: result.kind, Chunks: len(result.records)}) + if i.ProgressFn != nil { + i.ProgressFn(types.IndexProgress{File: result.rel, Action: "indexed", Kind: result.kind, Chunks: len(result.records)}) } } for file := range i.index.Files { @@ -219,18 +225,18 @@ func (i *Indexer) Index(ctx context.Context) error { delete(i.index.ChunksByFile, file) } } - i.index.EmbeddingSignature = i.cfg.embeddingSignature() + i.index.EmbeddingSignature = i.cfg.EmbeddingSignature() if err := flushIndex(true); err != nil && firstErr == nil { firstErr = err } return firstErr } -func (i *Indexer) fileChunks(relPath, content string) []Chunk { +func (i *Indexer) FileChunks(relPath, content string) []types.Chunk { return chunkText(content, i.cfg.ChunkSize, i.cfg.ChunkOverlap, i.cfg.ContextSize) } -func (i *Indexer) languageFor(relPath string) string { +func (i *Indexer) LanguageFor(relPath string) string { ext := strings.ToLower(filepath.Ext(relPath)) if lang, ok := i.cfg.LanguageOverrides[strings.TrimPrefix(ext, ".")]; ok { return lang @@ -261,7 +267,7 @@ func (i *Indexer) languageFor(relPath string) string { } } -func (i *Indexer) Search(ctx context.Context, opts SearchOptions) ([]SearchResult, error) { +func (i *Indexer) Search(ctx context.Context, opts SearchOptions) ([]types.SearchResult, error) { if i.index == nil { return nil, errors.New("index not loaded") } @@ -274,7 +280,7 @@ func (i *Indexer) Search(ctx context.Context, opts SearchOptions) ([]SearchResul useHybrid := i.cfg.HybridSearch || opts.UseHybrid - queryVecs, err := i.provider.Embed(ctx, []string{opts.Query}) + queryVecs, err := i.Provider.Embed(ctx, []string{opts.Query}) if err != nil { return nil, err } @@ -283,15 +289,15 @@ func (i *Indexer) Search(ctx context.Context, opts SearchOptions) ([]SearchResul } queryVec := queryVecs[0] - var hybridScorer *HybridScorer + var hybridScorer *search.HybridScorer var queryTerms []string if useHybrid { - queryTerms = extractQueryTerms(opts.Query) - hybridScorer = newHybridScorer(i.cfg.VectorWeight, i.cfg.KeywordWeight, i.index.ChunksByFile, i.projectRoot) + queryTerms = search.ExtractQueryTerms(opts.Query) + hybridScorer = search.NewHybridScorer(i.cfg.VectorWeight, i.cfg.KeywordWeight, i.index.ChunksByFile, i.projectRoot) } - results := make([]SearchResult, 0, 32) + results := make([]types.SearchResult, 0, 32) for _, chunks := range i.index.ChunksByFile { for _, ch := range chunks { if len(opts.Languages) > 0 && !containsString(opts.Languages, ch.Language) { @@ -302,14 +308,14 @@ func (i *Indexer) Search(ctx context.Context, opts SearchOptions) ([]SearchResul } vectorScore := cosine(queryVec, ch.Embedding) - var finalScore float64 + var finalScore float64 if useHybrid && hybridScorer != nil { - finalScore = hybridScorer.combineScores(vectorScore, ch.FilePath, ch.StartLine, ch.EndLine, queryTerms) + finalScore = hybridScorer.CombineScores(vectorScore, ch.FilePath, ch.StartLine, ch.EndLine, queryTerms) } else { finalScore = vectorScore } - results = append(results, SearchResult{ + results = append(results, types.SearchResult{ FilePath: ch.FilePath, Language: ch.Language, StartLine: ch.StartLine, @@ -329,8 +335,8 @@ func (i *Indexer) Search(ctx context.Context, opts SearchOptions) ([]SearchResul return results, nil } -func (i *Indexer) Status() Status { - status := Status{Langs: map[string]int{}} +func (i *Indexer) Status() types.Status { + status := types.Status{Langs: map[string]int{}} if i.index == nil { return status } @@ -344,6 +350,10 @@ func (i *Indexer) Status() Status { return status } +func (i *Indexer) IndexData() *IndexData { + return i.index +} + func containsString(values []string, target string) bool { for _, v := range values { if v == target { @@ -355,7 +365,7 @@ func containsString(values []string, target string) bool { func matchesAnyGlob(patterns []string, relPath string) bool { for _, pattern := range patterns { - if matchPattern(pattern, relPath) { + if utils.MatchPattern(pattern, relPath) { return true } } diff --git a/src/index_test.go b/internal/index/index_test.go similarity index 72% rename from src/index_test.go rename to internal/index/index_test.go index 48a2774..700684f 100644 --- a/src/index_test.go +++ b/internal/index/index_test.go @@ -1,4 +1,4 @@ -package main +package index import ( "context" @@ -6,12 +6,14 @@ import ( "path/filepath" "reflect" "testing" + + "github.com/QuinsZouls/code-index/internal/config" ) type stubEmbeddingProvider struct { calls int texts []string - vecs map[string][]float32 + vects map[string][]float32 } func (s *stubEmbeddingProvider) Embed(ctx context.Context, texts []string) ([][]float32, error) { @@ -19,11 +21,11 @@ func (s *stubEmbeddingProvider) Embed(ctx context.Context, texts []string) ([][] s.texts = append(s.texts, texts...) out := make([][]float32, len(texts)) for i, text := range texts { - vec, ok := s.vecs[text] + vect, ok := s.vects[text] if !ok { - vec = []float32{0} + vect = []float32{0} } - out[i] = append([]float32(nil), vec...) + out[i] = append([]float32(nil), vect...) } return out, nil } @@ -33,15 +35,16 @@ func TestIndexerIndexSearchAndSkipUnchanged(t *testing.T) { if err := os.WriteFile(filepath.Join(root, "hello.go"), []byte("alpha"), 0o644); err != nil { t.Fatal(err) } - cfg := defaultConfig() + cfg := config.Config{} + cfg.Normalize() cfg.IncludePatterns = []string{"**/*.go"} - cfg.Embedding = EmbeddingConfig{Provider: "openai-compatible", Model: "fake"} - provider := &stubEmbeddingProvider{vecs: map[string][]float32{"alpha": {1, 0}}} + cfg.Embedding = config.EmbeddingConfig{Provider: "openai-compatible", Model: "fake"} + provider := &stubEmbeddingProvider{vects: map[string][]float32{"alpha": {1, 0}}} indexer := &Indexer{ projectRoot: root, cfg: cfg, - provider: provider, - index: newIndexData(cfg.embeddingSignature()), + Provider: provider, + index: newIndexData(cfg.EmbeddingSignature()), } if err := indexer.Index(context.Background()); err != nil { t.Fatal(err) @@ -59,11 +62,11 @@ func TestIndexerIndexSearchAndSkipUnchanged(t *testing.T) { if firstState := indexer.index.Files["hello.go"]; firstState.Size != firstStat.Size() || firstState.ModTimeUnixNano == 0 { t.Fatalf("file state = %#v", firstState) } - if _, err := os.Stat(indexPath(root)); err != nil { + if _, err := os.Stat(config.IndexPath(root)); err != nil { t.Fatal(err) } - searchProvider := &stubEmbeddingProvider{vecs: map[string][]float32{"alpha": {1, 0}}} - indexer.provider = searchProvider + searchProvider := &stubEmbeddingProvider{vects: map[string][]float32{"alpha": {1, 0}}} + indexer.Provider = searchProvider results, err := indexer.Search(context.Background(), SearchOptions{Query: "alpha", Limit: 5}) if err != nil { t.Fatal(err) @@ -74,7 +77,7 @@ func TestIndexerIndexSearchAndSkipUnchanged(t *testing.T) { if results[0].Score < 0.99 { t.Fatalf("score = %v, want close to 1", results[0].Score) } - indexer.provider = provider + indexer.Provider = provider if err := indexer.Index(context.Background()); err != nil { t.Fatal(err) } @@ -96,9 +99,11 @@ func TestIndexerIndexSearchAndSkipUnchanged(t *testing.T) { } func TestIndexerHelpers(t *testing.T) { - idx := &Indexer{cfg: defaultConfig()} - if got := idx.languageFor("src/main.go"); got != "go" { - t.Fatalf("languageFor() = %q", got) + cfg := config.Config{} + cfg.Normalize() + idx := &Indexer{cfg: cfg} + if got := idx.LanguageFor("src/main.go"); got != "go" { + t.Fatalf("LanguageFor() = %q", got) } if !containsString([]string{"go", "python"}, "python") { t.Fatal("containsString() should be true") @@ -119,13 +124,14 @@ func TestIndexerRespectsConfigTuning(t *testing.T) { if err := os.WriteFile(filepath.Join(root, "a.go"), []byte("alpha"), 0o644); err != nil { t.Fatal(err) } - cfg := defaultConfig() + cfg := config.Config{} + cfg.Normalize() cfg.IncludePatterns = []string{"**/*.go"} cfg.WorkerCount = 1 cfg.CheckpointEvery = 1 - cfg.Embedding = EmbeddingConfig{Provider: "openai-compatible", Model: "fake"} - provider := &stubEmbeddingProvider{vecs: map[string][]float32{"alpha": {1, 0}}} - indexer := &Indexer{projectRoot: root, cfg: cfg, provider: provider, index: newIndexData(cfg.embeddingSignature())} + cfg.Embedding = config.EmbeddingConfig{Provider: "openai-compatible", Model: "fake"} + provider := &stubEmbeddingProvider{vects: map[string][]float32{"alpha": {1, 0}}} + indexer := &Indexer{projectRoot: root, cfg: cfg, Provider: provider, index: newIndexData(cfg.EmbeddingSignature())} if err := indexer.Index(context.Background()); err != nil { t.Fatal(err) } diff --git a/src/scan.go b/internal/index/scan.go similarity index 80% rename from src/scan.go rename to internal/index/scan.go index be7aad7..381a20f 100644 --- a/src/scan.go +++ b/internal/index/scan.go @@ -1,4 +1,4 @@ -package main +package index import ( "crypto/sha256" @@ -7,6 +7,9 @@ import ( "path/filepath" "sort" "strings" + + "github.com/QuinsZouls/code-index/internal/config" + "github.com/QuinsZouls/code-index/internal/utils" ) var ignoredDirs = map[string]struct{}{ @@ -50,7 +53,7 @@ func compileGitignorePattern(dirRel, line string) string { if line == "" { return "" } - dirRel = filepathToSlash(dirRel) + dirRel = utils.FilepathToSlash(dirRel) prefix := "" if dirRel != "" { prefix = dirRel + "/" @@ -70,8 +73,8 @@ func compileGitignorePattern(dirRel, line string) string { return prefix + "**/" + line } -func shouldExclude(relPath string, isDir bool, cfg Config, gitignorePatterns []string) bool { - relPath = filepathToSlash(relPath) +func shouldExclude(relPath string, isDir bool, cfg config.Config, gitignorePatterns []string) bool { + relPath = utils.FilepathToSlash(relPath) base := filepath.Base(relPath) if isDir { if _, ok := ignoredDirs[base]; ok { @@ -79,31 +82,31 @@ func shouldExclude(relPath string, isDir bool, cfg Config, gitignorePatterns []s } } for _, pattern := range append(append([]string{}, cfg.ExcludePatterns...), gitignorePatterns...) { - if matchPattern(pattern, relPath) { + if utils.MatchPattern(pattern, relPath) { return true } } return false } -func shouldInclude(relPath string, cfg Config) bool { +func shouldInclude(relPath string, cfg config.Config) bool { if len(cfg.IncludePatterns) == 0 { return true } for _, pattern := range cfg.IncludePatterns { - if matchPattern(pattern, relPath) { + if utils.MatchPattern(pattern, relPath) { return true } } return false } -func fileHash(b []byte) string { +func FileHash(b []byte) string { sum := sha256.Sum256(b) return hex.EncodeToString(sum[:]) } -func walkFiles(projectRoot string, cfg Config) ([]string, error) { +func WalkFiles(projectRoot string, cfg config.Config) ([]string, error) { var files []string if err := walkFilesDir(projectRoot, projectRoot, "", cfg, nil, &files); err != nil { return nil, err @@ -112,7 +115,7 @@ func walkFiles(projectRoot string, cfg Config) ([]string, error) { return files, nil } -func walkFilesDir(projectRoot, dirAbs, dirRel string, cfg Config, inheritedPatterns []string, files *[]string) error { +func walkFilesDir(projectRoot, dirAbs, dirRel string, cfg config.Config, inheritedPatterns []string, files *[]string) error { patterns := append([]string{}, inheritedPatterns...) patterns = append(patterns, collectGitignorePatterns(dirAbs, dirRel)...) entries, err := os.ReadDir(dirAbs) diff --git a/src/scan_test.go b/internal/index/scan_test.go similarity index 83% rename from src/scan_test.go rename to internal/index/scan_test.go index 4e5d40d..91e5809 100644 --- a/src/scan_test.go +++ b/internal/index/scan_test.go @@ -1,10 +1,12 @@ -package main +package index import ( "os" "path/filepath" "reflect" "testing" + + "github.com/QuinsZouls/code-index/internal/config" ) func TestWalkFilesHonorsIncludesAndIgnores(t *testing.T) { @@ -37,13 +39,16 @@ func TestWalkFilesHonorsIncludesAndIgnores(t *testing.T) { t.Fatal(err) } } - got, err := walkFiles(root, defaultConfig()) + cfg := config.Config{} + cfg.Normalize() + cfg.IncludePatterns = []string{"**/*.go", "**/*.md"} + got, err := WalkFiles(root, cfg) if err != nil { t.Fatal(err) } want := []string{"main.go", filepath.Join("sub", "readme.md")} if !reflect.DeepEqual(got, want) { - t.Fatalf("walkFiles() = %#v, want %#v", got, want) + t.Fatalf("WalkFiles() = %#v, want %#v", got, want) } } @@ -61,18 +66,21 @@ func TestWalkFilesHonorsNestedGitignore(t *testing.T) { if err := os.WriteFile(filepath.Join(root, "repo", "nested", "skip.go"), []byte("package main"), 0o644); err != nil { t.Fatal(err) } - got, err := walkFiles(filepath.Join(root, "repo"), defaultConfig()) + cfg := config.Config{} + cfg.Normalize() + got, err := WalkFiles(filepath.Join(root, "repo"), cfg) if err != nil { t.Fatal(err) } want := []string{"keep.go"} if !reflect.DeepEqual(got, want) { - t.Fatalf("walkFiles() = %#v, want %#v", got, want) + t.Fatalf("WalkFiles() = %#v, want %#v", got, want) } } func TestMatchExcludeHelper(t *testing.T) { - cfg := defaultConfig() + cfg := config.Config{} + cfg.Normalize() if !shouldExclude(filepath.Join("a", ".codeindex"), true, cfg, nil) { t.Fatal("expected .codeindex directory to be excluded") } diff --git a/src/store.go b/internal/index/store.go similarity index 67% rename from src/store.go rename to internal/index/store.go index 9aada00..dfbf38f 100644 --- a/src/store.go +++ b/internal/index/store.go @@ -1,25 +1,27 @@ -package main +package index import ( "encoding/gob" "errors" "os" "path/filepath" + + "github.com/QuinsZouls/code-index/internal/types" ) type IndexData struct { Version int `json:"version"` EmbeddingSignature string `json:"embedding_signature"` - Files map[string]FileState `json:"files"` - ChunksByFile map[string][]ChunkRecord `json:"chunks_by_file"` + Files map[string]types.FileState `json:"files"` + ChunksByFile map[string][]types.ChunkRecord `json:"chunks_by_file"` } func newIndexData(signature string) *IndexData { return &IndexData{ Version: 1, EmbeddingSignature: signature, - Files: map[string]FileState{}, - ChunksByFile: map[string][]ChunkRecord{}, + Files: map[string]types.FileState{}, + ChunksByFile: map[string][]types.ChunkRecord{}, } } @@ -37,10 +39,10 @@ func loadIndex(path string) (*IndexData, error) { return nil, err } if data.Files == nil { - data.Files = map[string]FileState{} + data.Files = map[string]types.FileState{} } if data.ChunksByFile == nil { - data.ChunksByFile = map[string][]ChunkRecord{} + data.ChunksByFile = map[string][]types.ChunkRecord{} } if data.Version != 1 { return nil, nil @@ -48,7 +50,7 @@ func loadIndex(path string) (*IndexData, error) { return &data, nil } -func saveIndex(path string, data *IndexData) error { +func SaveIndex(path string, data *IndexData) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } @@ -60,3 +62,7 @@ func saveIndex(path string, data *IndexData) error { enc := gob.NewEncoder(file) return enc.Encode(data) } + +func saveIndex(path string, data *IndexData) error { + return SaveIndex(path, data) +} diff --git a/src/store_test.go b/internal/index/store_test.go similarity index 89% rename from src/store_test.go rename to internal/index/store_test.go index a36f5ed..1d47c3f 100644 --- a/src/store_test.go +++ b/internal/index/store_test.go @@ -1,9 +1,11 @@ -package main +package index import ( "path/filepath" "reflect" "testing" + + "github.com/QuinsZouls/code-index/internal/types" ) func TestIndexStoreRoundTrip(t *testing.T) { @@ -11,10 +13,10 @@ func TestIndexStoreRoundTrip(t *testing.T) { original := &IndexData{ Version: 1, EmbeddingSignature: "sig", - Files: map[string]FileState{ + Files: map[string]types.FileState{ "a.go": {Hash: "hash", ChunkCount: 1}, }, - ChunksByFile: map[string][]ChunkRecord{ + ChunksByFile: map[string][]types.ChunkRecord{ "a.go": { { FilePath: "a.go", diff --git a/src/hybrid.go b/internal/search/hybrid.go similarity index 69% rename from src/hybrid.go rename to internal/search/hybrid.go index 07b86f4..7f94b64 100644 --- a/src/hybrid.go +++ b/internal/search/hybrid.go @@ -1,8 +1,11 @@ -package main +package search import ( "regexp" "strings" + + "github.com/QuinsZouls/code-index/internal/types" + "github.com/QuinsZouls/code-index/internal/utils" ) type HybridScorer struct { @@ -13,7 +16,7 @@ type HybridScorer struct { projectRoot string } -func newHybridScorer(vectorWeight, keywordWeight float64, chunksByFile map[string][]ChunkRecord, projectRoot string) *HybridScorer { +func NewHybridScorer(vectorWeight, keywordWeight float64, chunksByFile map[string][]types.ChunkRecord, projectRoot string) *HybridScorer { if vectorWeight < 0 { vectorWeight = 0 } @@ -34,7 +37,7 @@ func newHybridScorer(vectorWeight, keywordWeight float64, chunksByFile map[strin for _, chunks := range chunksByFile { totalDocs += len(chunks) for _, ch := range chunks { - content := readChunkContent(projectRoot, ch.FilePath, ch.StartLine, ch.EndLine) + content := utils.ReadChunkContent(projectRoot, ch.FilePath, ch.StartLine, ch.EndLine) terms := tokenize(content) seen := make(map[string]bool) for _, term := range terms { @@ -55,15 +58,20 @@ func newHybridScorer(vectorWeight, keywordWeight float64, chunksByFile map[strin } } -func (h *HybridScorer) combineScores(vectorScore float64, filePath string, startLine, endLine int, queryTerms []string) float64 { +func (h *HybridScorer) CombineScores(vectorScore float64, filePath string, startLine, endLine int, queryTerms []string) float64 { if h.keywordWeight == 0 { return vectorScore } - content := readChunkContent(h.projectRoot, filePath, startLine, endLine) + content := utils.ReadChunkContent(h.projectRoot, filePath, startLine, endLine) keywordScore := h.tfidfScore(content, queryTerms) return h.vectorWeight*vectorScore + h.keywordWeight*keywordScore } +// TFIDFScore computes the TF-IDF keyword score for the given content and query terms. +func (h *HybridScorer) TFIDFScore(content string, queryTerms []string) float64 { + return h.tfidfScore(content, queryTerms) +} + func (h *HybridScorer) tfidfScore(content string, queryTerms []string) float64 { if len(queryTerms) == 0 || h.totalDocs == 0 { return 0 @@ -106,6 +114,11 @@ func tokenize(text string) []string { return terms } -func extractQueryTerms(query string) []string { +// Tokenize splits text into lowercase tokens. +func Tokenize(text string) []string { + return tokenize(text) +} + +func ExtractQueryTerms(query string) []string { return tokenize(query) } diff --git a/src/hybrid_test.go b/internal/search/hybrid_test.go similarity index 57% rename from src/hybrid_test.go rename to internal/search/hybrid_test.go index f3554d2..08b8cfe 100644 --- a/src/hybrid_test.go +++ b/internal/search/hybrid_test.go @@ -1,7 +1,10 @@ -package main +package search_test import ( "testing" + + "github.com/QuinsZouls/code-index/internal/search" + "github.com/QuinsZouls/code-index/internal/types" ) func TestTokenize(t *testing.T) { @@ -17,14 +20,14 @@ func TestTokenize(t *testing.T) { } for _, tt := range tests { - got := tokenize(tt.input) + got := search.Tokenize(tt.input) if len(got) != len(tt.expected) { - t.Errorf("tokenize(%q) = %v, want %v", tt.input, got, tt.expected) + t.Errorf("Tokenize(%q) = %v, want %v", tt.input, got, tt.expected) continue } for i, v := range got { if v != tt.expected[i] { - t.Errorf("tokenize(%q)[%d] = %q, want %q", tt.input, i, v, tt.expected[i]) + t.Errorf("Tokenize(%q)[%d] = %q, want %q", tt.input, i, v, tt.expected[i]) } } } @@ -32,74 +35,74 @@ func TestTokenize(t *testing.T) { func TestExtractQueryTerms(t *testing.T) { query := "database connection pooling" - terms := extractQueryTerms(query) + terms := search.ExtractQueryTerms(query) expected := []string{"database", "connection", "pooling"} if len(terms) != len(expected) { - t.Fatalf("extractQueryTerms(%q) = %v, want %v", query, terms, expected) + t.Fatalf("ExtractQueryTerms(%q) = %v, want %v", query, terms, expected) } for i, term := range terms { if term != expected[i] { - t.Errorf("extractQueryTerms(%q)[%d] = %q, want %q", query, i, term, expected[i]) + t.Errorf("ExtractQueryTerms(%q)[%d] = %q, want %q", query, i, term, expected[i]) } } } func TestHybridScorerTFIDF(t *testing.T) { projectRoot := t.TempDir() - chunks := map[string][]ChunkRecord{ + chunks := map[string][]types.ChunkRecord{ "test.go": { {FilePath: "test.go", StartLine: 1, EndLine: 10, Language: "go"}, }, } - scorer := newHybridScorer(0.7, 0.3, chunks, projectRoot) + scorer := search.NewHybridScorer(0.7, 0.3, chunks, projectRoot) content := "package main\n\nfunc main() {\n\tfmt.Println(\"hello world\")\n}" queryTerms := []string{"main", "func"} - score := scorer.tfidfScore(content, queryTerms) + score := scorer.TFIDFScore(content, queryTerms) if score < 0 || score > 1 { - t.Errorf("tfidfScore() = %v, want between 0 and 1", score) + t.Errorf("TFIDFScore() = %v, want between 0 and 1", score) } } func TestHybridScorerCombineScores(t *testing.T) { projectRoot := t.TempDir() - chunks := map[string][]ChunkRecord{ + chunks := map[string][]types.ChunkRecord{ "test.go": { {FilePath: "test.go", StartLine: 1, EndLine: 10, Language: "go"}, }, } - scorer := newHybridScorer(0.7, 0.3, chunks, projectRoot) + scorer := search.NewHybridScorer(0.7, 0.3, chunks, projectRoot) vectorScore := 0.8 queryTerms := []string{"main"} - combined := scorer.combineScores(vectorScore, "test.go", 1, 10, queryTerms) + combined := scorer.CombineScores(vectorScore, "test.go", 1, 10, queryTerms) if combined < 0 || combined > 1 { - t.Errorf("combineScores() = %v, want between 0 and 1", combined) + t.Errorf("CombineScores() = %v, want between 0 and 1", combined) } if combined < vectorScore*0.7 { - t.Errorf("combineScores() = %v, should be at least 0.7 * vectorScore", combined) + t.Errorf("CombineScores() = %v, should be at least 0.7 * vectorScore", combined) } } func TestHybridScorerZeroKeywordWeight(t *testing.T) { projectRoot := t.TempDir() - chunks := map[string][]ChunkRecord{} + chunks := map[string][]types.ChunkRecord{} - scorer := newHybridScorer(1.0, 0.0, chunks, projectRoot) + scorer := search.NewHybridScorer(1.0, 0.0, chunks, projectRoot) vectorScore := 0.8 - combined := scorer.combineScores(vectorScore, "test.go", 1, 10, []string{"query"}) + combined := scorer.CombineScores(vectorScore, "test.go", 1, 10, []string{"query"}) if combined != vectorScore { - t.Errorf("combineScores() with zero keyword weight = %v, want %v", combined, vectorScore) + t.Errorf("CombineScores() with zero keyword weight = %v, want %v", combined, vectorScore) } } diff --git a/src/match_test.go b/internal/search/match_test.go similarity index 52% rename from src/match_test.go rename to internal/search/match_test.go index df63786..e49606e 100644 --- a/src/match_test.go +++ b/internal/search/match_test.go @@ -1,6 +1,10 @@ -package main +package search_test -import "testing" +import ( + "testing" + + "github.com/QuinsZouls/code-index/internal/utils" +) func TestMatchPattern(t *testing.T) { cases := []struct { @@ -14,14 +18,14 @@ func TestMatchPattern(t *testing.T) { {"src/*.go", "src/pkg/main.go", false}, } for _, tc := range cases { - if got := matchPattern(tc.pattern, tc.path); got != tc.want { - t.Fatalf("matchPattern(%q, %q) = %v, want %v", tc.pattern, tc.path, got, tc.want) + if got := utils.MatchPattern(tc.pattern, tc.path); got != tc.want { + t.Fatalf("MatchPattern(%q, %q) = %v, want %v", tc.pattern, tc.path, got, tc.want) } } } func TestFilepathToSlash(t *testing.T) { - if got := filepathToSlash("a\\b\\c"); got != "a/b/c" { - t.Fatalf("filepathToSlash() = %q, want a/b/c", got) + if got := utils.FilepathToSlash("a\\b\\c"); got != "a/b/c" { + t.Fatalf("FilepathToSlash() = %q, want a/b/c", got) } } diff --git a/src/types.go b/internal/types/types.go similarity index 98% rename from src/types.go rename to internal/types/types.go index 56298fb..ab38062 100644 --- a/src/types.go +++ b/internal/types/types.go @@ -1,4 +1,4 @@ -package main +package types type Chunk struct { Content string diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..b1bc781 --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,66 @@ +package utils + +import ( + "os" + "path" + "strings" +) + +// ReadChunkContent reads file content by line range +func ReadChunkContent(projectRoot, relPath string, startLine, endLine int) string { + absPath := projectRoot + "/" + relPath + data, err := os.ReadFile(absPath) + if err != nil { + return "[file unavailable: " + err.Error() + "]" + } + lines := strings.Split(string(data), "\n") + if startLine < 1 || startLine > len(lines) { + return "[line range invalid]" + } + if endLine > len(lines) { + endLine = len(lines) + } + return strings.Join(lines[startLine-1:endLine], "\n") +} + +// MatchPattern matches a pattern against a path +func MatchPattern(pattern, relPath string) bool { + pattern = strings.TrimSpace(pattern) + pattern = strings.ReplaceAll(pattern, "\\", "/") + relPath = strings.ReplaceAll(relPath, "\\", "/") + if pattern == "" { + return false + } + return MatchSegments(strings.Split(pattern, "/"), strings.Split(relPath, "/")) +} + +// MatchSegments matches path segments +func MatchSegments(patternSegs, pathSegs []string) bool { + if len(patternSegs) == 0 { + return len(pathSegs) == 0 + } + if patternSegs[0] == "**" { + if len(patternSegs) == 1 { + return true + } + for i := 0; i <= len(pathSegs); i++ { + if MatchSegments(patternSegs[1:], pathSegs[i:]) { + return true + } + } + return false + } + if len(pathSegs) == 0 { + return false + } + ok, err := path.Match(patternSegs[0], pathSegs[0]) + if err != nil || !ok { + return false + } + return MatchSegments(patternSegs[1:], pathSegs[1:]) +} + +// FilepathToSlash converts Windows path separators to Unix separators +func FilepathToSlash(s string) string { + return strings.ReplaceAll(s, "\\", "/") +} diff --git a/src/match.go b/src/match.go deleted file mode 100644 index e0f182a..0000000 --- a/src/match.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "path" - "strings" -) - -func matchPattern(pattern, relPath string) bool { - pattern = strings.TrimSpace(filepathToSlash(pattern)) - relPath = filepathToSlash(relPath) - if pattern == "" { - return false - } - return matchSegments(strings.Split(pattern, "/"), strings.Split(relPath, "/")) -} - -func matchSegments(patternSegs, pathSegs []string) bool { - if len(patternSegs) == 0 { - return len(pathSegs) == 0 - } - if patternSegs[0] == "**" { - if len(patternSegs) == 1 { - return true - } - for i := 0; i <= len(pathSegs); i++ { - if matchSegments(patternSegs[1:], pathSegs[i:]) { - return true - } - } - return false - } - if len(pathSegs) == 0 { - return false - } - ok, err := path.Match(patternSegs[0], pathSegs[0]) - if err != nil || !ok { - return false - } - return matchSegments(patternSegs[1:], pathSegs[1:]) -} - -func filepathToSlash(s string) string { - return strings.ReplaceAll(s, "\\", "/") -} From 76e850c35ac3d9c8c3323b3b0a3110bd4477b6e0 Mon Sep 17 00:00:00 2001 From: Alfredo Medrano Date: Sat, 2 May 2026 11:33:40 -0600 Subject: [PATCH 2/4] fix: remove dead code, fix formatting and filepath.Join usage --- cmd/codeindex/main.go | 14 +++++++------- internal/daemon/daemon.go | 11 +---------- internal/daemon/proc_unix.go | 2 +- internal/daemon/proc_windows.go | 2 +- internal/index/chunk_test.go | 2 +- internal/index/index.go | 2 +- internal/index/store.go | 4 ++-- internal/utils/utils.go | 3 ++- 8 files changed, 16 insertions(+), 24 deletions(-) diff --git a/cmd/codeindex/main.go b/cmd/codeindex/main.go index 49c076e..c25f018 100644 --- a/cmd/codeindex/main.go +++ b/cmd/codeindex/main.go @@ -484,13 +484,13 @@ var providerDefaults = map[string]struct { keyEnv string needsKey bool }{ - "openai": {"text-embedding-3-small", "https://api.openai.com/v1", "OPENAI_API_KEY", true}, - "ollama": {"nomic-embed-text", "http://localhost:11434", "", false}, - "openrouter": {"openai/text-embedding-3-small", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY", true}, - "mistral": {"mistral-embed", "https://api.mistral.ai/v1", "MISTRAL_API_KEY", true}, - "gemini": {"text-embedding-004", "https://generativelanguage.googleapis.com/v1beta", "GEMINI_API_KEY", true}, - "lmstudio": {"text-embedding-nomic-embed-text-v1.5", "http://localhost:1234/v1", "", false}, - "llamacpp": {"local-model", "http://localhost:8080/v1", "", false}, + "openai": {"text-embedding-3-small", "https://api.openai.com/v1", "OPENAI_API_KEY", true}, + "ollama": {"nomic-embed-text", "http://localhost:11434", "", false}, + "openrouter": {"openai/text-embedding-3-small", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY", true}, + "mistral": {"mistral-embed", "https://api.mistral.ai/v1", "MISTRAL_API_KEY", true}, + "gemini": {"text-embedding-004", "https://generativelanguage.googleapis.com/v1beta", "GEMINI_API_KEY", true}, + "lmstudio": {"text-embedding-nomic-embed-text-v1.5", "http://localhost:1234/v1", "", false}, + "llamacpp": {"local-model", "http://localhost:8080/v1", "", false}, "openai-compatible": {"", "", "OPENAI_API_KEY", true}, } diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 6ce5162..739697c 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -96,14 +96,6 @@ func runDaemonStart(args []string) { return } -cfg, err := config.LoadConfig(root) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - _ = cfg - _ = index.NewIndexer - lockPath := lockFilePath(root) data, err := os.ReadFile(lockPath) if err == nil { @@ -521,8 +513,7 @@ func (d *Daemon) processBatch() { } } } - -if len(modified) > 0 || len(deleted) > 0 { + if len(modified) > 0 || len(deleted) > 0 { if err := index.SaveIndex(config.IndexPath(d.projectRoot), d.indexer.IndexData()); err != nil { if d.verbose { fmt.Printf("[!] failed to save index: %v\n", err) diff --git a/internal/daemon/proc_unix.go b/internal/daemon/proc_unix.go index 005e004..59e8ae3 100644 --- a/internal/daemon/proc_unix.go +++ b/internal/daemon/proc_unix.go @@ -9,4 +9,4 @@ import ( func configureDaemonCmd(cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} -} \ No newline at end of file +} diff --git a/internal/daemon/proc_windows.go b/internal/daemon/proc_windows.go index c5cdf74..b84a620 100644 --- a/internal/daemon/proc_windows.go +++ b/internal/daemon/proc_windows.go @@ -9,4 +9,4 @@ import ( func configureDaemonCmd(cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{} -} \ No newline at end of file +} diff --git a/internal/index/chunk_test.go b/internal/index/chunk_test.go index 6a99a0c..c29607e 100644 --- a/internal/index/chunk_test.go +++ b/internal/index/chunk_test.go @@ -147,4 +147,4 @@ func TestChunkByContextSize(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/internal/index/index.go b/internal/index/index.go index 1c7da85..6503336 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -308,7 +308,7 @@ func (i *Indexer) Search(ctx context.Context, opts SearchOptions) ([]types.Searc } vectorScore := cosine(queryVec, ch.Embedding) - var finalScore float64 + var finalScore float64 if useHybrid && hybridScorer != nil { finalScore = hybridScorer.CombineScores(vectorScore, ch.FilePath, ch.StartLine, ch.EndLine, queryTerms) } else { diff --git a/internal/index/store.go b/internal/index/store.go index dfbf38f..d286299 100644 --- a/internal/index/store.go +++ b/internal/index/store.go @@ -10,8 +10,8 @@ import ( ) type IndexData struct { - Version int `json:"version"` - EmbeddingSignature string `json:"embedding_signature"` + Version int `json:"version"` + EmbeddingSignature string `json:"embedding_signature"` Files map[string]types.FileState `json:"files"` ChunksByFile map[string][]types.ChunkRecord `json:"chunks_by_file"` } diff --git a/internal/utils/utils.go b/internal/utils/utils.go index b1bc781..6ad9299 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -3,12 +3,13 @@ package utils import ( "os" "path" + "path/filepath" "strings" ) // ReadChunkContent reads file content by line range func ReadChunkContent(projectRoot, relPath string, startLine, endLine int) string { - absPath := projectRoot + "/" + relPath + absPath := filepath.Join(projectRoot, relPath) data, err := os.ReadFile(absPath) if err != nil { return "[file unavailable: " + err.Error() + "]" From 58bedcf7baca66ea2a2b462552be55e2dbdc2fd6 Mon Sep 17 00:00:00 2001 From: Alfredo Medrano Date: Sat, 2 May 2026 11:41:03 -0600 Subject: [PATCH 3/4] Add release workflow and CI configuration --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a1fb900 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,32 @@ +name: ci + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + check-and-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.26" + + - name: Check formatting + run: | + unformatted=$(gofmt -l ./cmd/ ./internal/) + if [ -n "$unformatted" ]; then + echo "The following files are not formatted with gofmt:" + echo "$unformatted" + exit 1 + fi + + - name: Run tests + run: go test ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 58181ae..2c22426 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,7 +50,7 @@ jobs: shell: bash run: | mkdir -p dist - GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X 'main.appVersion=${{ github.ref_name }}'" -o "dist/codeindex-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.ext }}" ./src + GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X 'main.appVersion=${{ github.ref_name }}'" -o "dist/codeindex-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.ext }}" ./cmd/codeindex/ - name: Package Unix release if: matrix.goos != 'windows' From b2c92f9c553a86106ad0698c650d60828e466282 Mon Sep 17 00:00:00 2001 From: Alfredo Medrano Date: Wed, 13 May 2026 14:32:10 -0600 Subject: [PATCH 4/4] refactor: update embedding input handling and improve error reporting - Modify the processBatch method to use EmbeddingInputForChunk for preparing text inputs for embedding. - Introduce embeddingResponseBodySnippet to enhance error messages with response body snippets. - Add unit tests for EmbeddingInputForChunk to ensure correct handling of various input cases. --- internal/daemon/daemon.go | 2 +- internal/embeddings/embeddings.go | 18 +++++++++++++++--- internal/index/chunk.go | 11 +++++++++++ internal/index/chunk_test.go | 19 +++++++++++++++++++ internal/index/index.go | 2 +- 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 739697c..c62a3d0 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -469,7 +469,7 @@ func (d *Daemon) processBatch() { texts := make([]string, 0, len(chunks)) for _, ch := range chunks { - texts = append(texts, ch.Content) + texts = append(texts, index.EmbeddingInputForChunk(ch.Content)) } vecs, err := d.indexer.Provider.Embed(ctx, texts) diff --git a/internal/embeddings/embeddings.go b/internal/embeddings/embeddings.go index 5192466..785f479 100644 --- a/internal/embeddings/embeddings.go +++ b/internal/embeddings/embeddings.go @@ -228,6 +228,14 @@ func (p *openAICompatibleProvider) Embed(ctx context.Context, texts []string) ([ return vecs, nil } +func embeddingResponseBodySnippet(b []byte, max int) string { + s := strings.TrimSpace(string(b)) + if len(s) > max { + return s[:max] + "..." + } + return s +} + func (p *openAICompatibleProvider) doRequest(ctx context.Context, texts []string) ([][]float32, error) { body := map[string]any{"model": p.model, "input": texts} data, _ := json.Marshal(body) @@ -251,16 +259,20 @@ func (p *openAICompatibleProvider) doRequest(ctx context.Context, texts []string b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) return nil, fmt.Errorf("embedding API error: %s: %s", resp.Status, strings.TrimSpace(string(b))) } + rawBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } var out struct { Data []struct { Embedding []float32 `json:"embedding"` } `json:"data"` } - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return nil, err + if err := json.Unmarshal(rawBody, &out); err != nil { + return nil, fmt.Errorf("embedding response decode: %w (body: %s)", err, embeddingResponseBodySnippet(rawBody, 512)) } if len(out.Data) != len(texts) { - return nil, fmt.Errorf("embedding response count mismatch: got %d want %d", len(out.Data), len(texts)) + return nil, fmt.Errorf("embedding response count mismatch: got %d want %d (body: %s)", len(out.Data), len(texts), embeddingResponseBodySnippet(rawBody, 512)) } vecs := make([][]float32, len(out.Data)) for i := range out.Data { diff --git a/internal/index/chunk.go b/internal/index/chunk.go index f43c05f..f2bf0e1 100644 --- a/internal/index/chunk.go +++ b/internal/index/chunk.go @@ -50,6 +50,17 @@ func chunkText(text string, maxLines, overlap, contextSize int) []types.Chunk { return chunks } +// EmbeddingInputForChunk returns text suitable for embedding APIs that reject +// empty input (e.g. Perplexity pplx-embed via OpenRouter). The original chunk +// content is unchanged for storage and hashing; only the value sent to Embed +// should pass through this function. +func EmbeddingInputForChunk(content string) string { + if strings.TrimSpace(content) == "" { + return " " + } + return content +} + // chunkByContextSize splits text into chunks that fit within contextSize characters, // respecting line boundaries when possible. func chunkByContextSize(text string, contextSize, overlap int) []types.Chunk { diff --git a/internal/index/chunk_test.go b/internal/index/chunk_test.go index c29607e..716b478 100644 --- a/internal/index/chunk_test.go +++ b/internal/index/chunk_test.go @@ -148,3 +148,22 @@ func TestChunkByContextSize(t *testing.T) { }) } } + +func TestEmbeddingInputForChunk(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"", " "}, + {" ", " "}, + {"\n\t\n", " "}, + {"hello", "hello"}, + {" hi ", " hi "}, + } + for _, tt := range tests { + got := EmbeddingInputForChunk(tt.in) + if got != tt.want { + t.Errorf("EmbeddingInputForChunk(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} diff --git a/internal/index/index.go b/internal/index/index.go index 6503336..4081711 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -150,7 +150,7 @@ func (i *Indexer) Index(ctx context.Context) error { chunks := i.FileChunks(job.rel, string(data)) texts := make([]string, 0, len(chunks)) for _, ch := range chunks { - texts = append(texts, ch.Content) + texts = append(texts, EmbeddingInputForChunk(ch.Content)) } vecs, err := i.Provider.Embed(ctx, texts) if err != nil {