diff --git a/Makefile b/Makefile
index a6bb687..b44f692 100644
--- a/Makefile
+++ b/Makefile
@@ -1,21 +1,18 @@
-.PHONY: dev build run clean web build-web
+.PHONY: dev build run clean web
dev:
@$(MAKE) -s build
- @./tmp/crawler
+ @./tmp/crawler crawl
build:
- go build -o ./tmp/crawler ./cmd/crawler/main.go
-
-build-web:
- go build -o ./tmp/web ./cmd/web/main.go
+ go build -o ./tmp/crawler ./cmd/crawler/
run:
- ./tmp/crawler
+ ./tmp/crawler crawl
web:
- @$(MAKE) -s build-web
- @./tmp/web
+ @$(MAKE) -s build
+ @./tmp/crawler web
clean:
rm -rf ./tmp
diff --git a/README.md b/README.md
index 6c423ff..a7200ca 100644
--- a/README.md
+++ b/README.md
@@ -22,33 +22,33 @@ crowlr works in a similar way: it visits multiple pages at once using a pool of
```mermaid
flowchart TB
- Start([Seeds]):::start -->|enqueue| Queue
+ Binary([crowlr]):::binary
- Queue[Frontier
- BFS queue with per-host queues
- thread-safe with mutex
- deduplicates via seen set
- crawl limit terminates program]
+ Binary -->|crawl| Seeds
+ Binary -->|web| Search
- Queue -->|dequeue| Fetch
+ Seeds([seed URLs]) -->|push| Frontier
- Fetch[Fetch URLs
- concurrent worker pool
- respects robots.txt
- per-host politeness delay]
+ Frontier[Frontier
- per-host BFS queues
- seen-URL dedup
- polite pop with per-host delay]
- Fetch --> Parse
+ Frontier -->|next eligible URL| Workers
- Parse[Parse Page
- validate HTML content-type
- extract links from a tags
- extract text content and title
- normalize URLs]
+ Workers[Worker Pool
- concurrent fetchers
- robots.txt + sitemap support
- configurable count and delay]
- Parse -->|new URLs| Queue
- Parse -->|store| DB
+ Workers -->|HTML| Extract
- DB[(PostgreSQL
- url, title, content, html
- status code, outlinks
- tsvector full-text index
- weighted: title > url > content)]
+ Extract[Extract
- title from title tag
- outlinks from anchor hrefs
- resolve and normalize URLs]
- DB -->|full-text search| Search
+ Extract -->|new URLs| Frontier
+ Extract -->|page| DB
- Search[Search UI
- HTMX
- ts_rank_cd ranking
- highlighted snippets]
+ DB[(PostgreSQL
- url, title, html, outlinks
- tsvector full-text index)]
- Seen[Seen Set
- normalized URL dedup
- thread-safe with mutex]
+ DB -->|full-text search| Search
- Parse -.->|check| Seen
- Queue -.->|check| Seen
+ Search[Search UI
- HTMX
- ts_rank_cd ranking
- highlighted snippets]
- classDef start stroke:#666,stroke-width:2px
+ classDef binary stroke:#666,stroke-width:2px
```
## Features
@@ -86,6 +86,10 @@ make dev
# Run search UI (separate terminal)
make web
# Open http://localhost:8080
+
+# Or use the binary directly
+./tmp/crawler crawl
+./tmp/crawler web --port 9000
```
## Configuration
@@ -99,6 +103,7 @@ See `config.example.toml` for all options.
| `crawler.crawl_limit` | Max pages to crawl | `1000` |
| `crawler.user_agent` | User-Agent header | - |
| `politeness.delay` | Min delay between requests to same host | `1s` |
+| `politeness.fetch_timeout` | Max duration for an individual fetch | `10s` |
| `logging.level` | Log level (debug, info, warn, error) | `info` |
| `logging.format` | Log format (text, json) | `json` |
@@ -106,12 +111,9 @@ See `config.example.toml` for all options.
```
cmd/
- crawler/ # crawler binary
- web/ # search UI binary
+ crawler/ # single binary — `crawl` and `web` subcommands
pkg/
- crawler/ # coordinator, workers, stats
- process/ # HTML parsing, text extraction, normalization, robots.txt
- storage/ # PostgreSQL with migrations and full-text search
+ crawler/ # frontier, workers, postgres store, full-text search
config/ # TOML configuration
logger/ # structured logging (bunyan-compatible)
```
diff --git a/cmd/crawler/crawl.go b/cmd/crawler/crawl.go
new file mode 100644
index 0000000..965169d
--- /dev/null
+++ b/cmd/crawler/crawl.go
@@ -0,0 +1,138 @@
+package main
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/devraulu/crowlr/pkg/config"
+ "github.com/devraulu/crowlr/pkg/logger"
+ "github.com/devraulu/crowlr/pkg/crawler"
+ _ "github.com/lib/pq"
+ "github.com/spf13/cobra"
+)
+
+var crawlCmd = &cobra.Command{
+ Use: "crawl",
+ Short: "Start crawling from seed URLs",
+ RunE: runCrawl,
+}
+
+var (
+ flagSeeds string
+ flagWorkers int
+ flagDelay string
+ flagFetchTimeout string
+ flagLimit int
+ flagUserAgent string
+ flagLogLevel string
+ flagLogFormat string
+)
+
+func init() {
+ crawlCmd.Flags().StringVarP(&flagSeeds, "seeds", "s", "", "seeds file (overrides config)")
+ crawlCmd.Flags().IntVarP(&flagWorkers, "workers", "w", 0, "number of workers (overrides config)")
+ crawlCmd.Flags().StringVarP(&flagDelay, "delay", "d", "", "politeness delay, e.g. 500ms (overrides config)")
+ crawlCmd.Flags().StringVar(&flagFetchTimeout, "fetch-timeout", "", "fetch timeout, e.g. 10s (overrides config)")
+ crawlCmd.Flags().IntVarP(&flagLimit, "limit", "l", -1, "max pages to crawl, 0 = unlimited (overrides config)")
+ crawlCmd.Flags().StringVar(&flagUserAgent, "user-agent", "", "user agent string (overrides config)")
+ crawlCmd.Flags().StringVar(&flagLogLevel, "log-level", "", "log level: debug, info, warn, error (overrides config)")
+ crawlCmd.Flags().StringVar(&flagLogFormat, "log-format", "", "log format: text, json (overrides config)")
+ rootCmd.AddCommand(crawlCmd)
+}
+
+func runCrawl(cmd *cobra.Command, _ []string) error {
+ cfg, err := config.Load(cfgFile)
+ if err != nil {
+ return fmt.Errorf("failed to load config: %w", err)
+ }
+
+ if cmd.Flags().Changed("log-level") {
+ cfg.Logging.Level = flagLogLevel
+ }
+ if cmd.Flags().Changed("log-format") {
+ cfg.Logging.Format = flagLogFormat
+ }
+ logger.InitLogger(cfg)
+
+ if cmd.Flags().Changed("seeds") {
+ cfg.Crawler.SeedsFile = flagSeeds
+ }
+ if cmd.Flags().Changed("workers") {
+ cfg.Crawler.Workers = flagWorkers
+ }
+ if cmd.Flags().Changed("delay") {
+ cfg.Politeness.Delay = flagDelay
+ }
+ if cmd.Flags().Changed("fetch-timeout") {
+ cfg.Politeness.FetchTimeout = flagFetchTimeout
+ }
+ if cmd.Flags().Changed("limit") {
+ cfg.Crawler.CrawlLimit = flagLimit
+ }
+ if cmd.Flags().Changed("user-agent") {
+ cfg.Crawler.UserAgent = flagUserAgent
+ }
+ if err := cfg.Validate(); err != nil {
+ return fmt.Errorf("invalid config: %w", err)
+ }
+
+ seeds, err := crawler.LoadSeedsFile(cfg.Crawler.SeedsFile)
+ if err != nil {
+ return fmt.Errorf("failed to load seeds: %w", err)
+ }
+
+ db, err := sql.Open("postgres", cfg.DSN)
+ if err != nil {
+ return fmt.Errorf("failed to open database: %w", err)
+ }
+ defer db.Close()
+
+ if err := db.Ping(); err != nil {
+ return fmt.Errorf("failed to ping database: %w", err)
+ }
+ slog.Debug("database connected")
+
+ slog.Debug("running database migrations")
+ if err := crawler.RunMigrations(db); err != nil {
+ return fmt.Errorf("failed to run migrations: %w", err)
+ }
+ slog.Debug("database migrations complete")
+
+ store := crawler.NewPostgresStore(db)
+
+ client := &http.Client{}
+ c := crawler.NewCrawler(crawler.NewHTTPFetcher(client),
+ store,
+ crawler.WithUserAgent(cfg.Crawler.UserAgent),
+ crawler.WithCrawlDelay(cfg.Politeness.GetDelay()),
+ crawler.WithWorkers(cfg.Crawler.Workers),
+ crawler.WithCrawlLimit(cfg.Crawler.CrawlLimit),
+ crawler.WithFetchTimeout(cfg.Politeness.GetFetchTimeout()),
+ )
+
+ ctx, stop := context.WithCancel(context.Background())
+ defer stop()
+
+ sig := make(chan os.Signal, 1)
+ signal.Notify(sig, syscall.SIGINT, syscall.SIGQUIT)
+ go func() {
+ select {
+ case s := <-sig:
+ slog.Info("received signal, shutting down", slog.String("signal", s.String()))
+ stop()
+ case <-ctx.Done():
+ }
+ }()
+
+ if err := c.Run(ctx, seeds); err != nil && !errors.Is(err, context.Canceled) {
+ return err
+ }
+ return nil
+}
diff --git a/cmd/crawler/crawler_acceptance_test.go b/cmd/crawler/crawler_acceptance_test.go
new file mode 100644
index 0000000..5e0640b
--- /dev/null
+++ b/cmd/crawler/crawler_acceptance_test.go
@@ -0,0 +1,95 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "testing"
+
+ "github.com/devraulu/crowlr/pkg/crawler"
+)
+
+func TestCrawlerStoresPagesEndToEnd(t *testing.T) {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, `
+ Page A
+ Page B
+ `)
+ })
+ mux.HandleFunc("/page-a", func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, `Page A
`)
+ })
+ mux.HandleFunc("/page-b", func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, `Page B
`)
+ })
+
+ app := newTestApp(t, mux)
+
+ seed, err := crawler.NewLink(app.server.URL + "/")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ c := crawler.NewCrawler(
+ crawler.NewHTTPFetcher(&http.Client{}),
+ app.store,
+ crawler.WithWorkers(1),
+ )
+
+ if err := c.Run(t.Context(), []crawler.Link{seed}); err != nil {
+ t.Fatal(err)
+ }
+
+ type row struct {
+ url string
+ statusCode int
+ outlinks []string
+ }
+
+ rows, err := app.db.QueryContext(t.Context(),
+ `SELECT url, status_code, outlinks FROM pages ORDER BY url`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer rows.Close()
+
+ var pages []row
+ for rows.Next() {
+ var p row
+ var rawOutlinks []byte
+ if err := rows.Scan(&p.url, &p.statusCode, &rawOutlinks); err != nil {
+ t.Fatal(err)
+ }
+ json.Unmarshal(rawOutlinks, &p.outlinks)
+ pages = append(pages, p)
+ }
+ if err := rows.Err(); err != nil {
+ t.Fatal(err)
+ }
+
+ base := app.server.URL
+
+ want := []row{
+ {url: base + "/", statusCode: 200, outlinks: []string{base + "/page-a", base + "/page-b"}},
+ {url: base + "/page-a", statusCode: 200, outlinks: []string{}},
+ {url: base + "/page-b", statusCode: 200, outlinks: []string{}},
+ }
+
+ if len(pages) != len(want) {
+ t.Fatalf("expected %d pages, got %d: %v", len(want), len(pages), pages)
+ }
+
+ for i, w := range want {
+ got := pages[i]
+ if got.url != w.url {
+ t.Errorf("[%d] url: got %q, want %q", i, got.url, w.url)
+ }
+ if got.statusCode != w.statusCode {
+ t.Errorf("[%d] status_code: got %d, want %d", i, got.statusCode, w.statusCode)
+ }
+ if len(got.outlinks) != len(w.outlinks) {
+ t.Errorf("[%d] outlinks: got %v, want %v", i, got.outlinks, w.outlinks)
+ }
+ }
+}
diff --git a/cmd/crawler/main.go b/cmd/crawler/main.go
index c2acf13..736ef31 100644
--- a/cmd/crawler/main.go
+++ b/cmd/crawler/main.go
@@ -1,81 +1,5 @@
package main
-import (
- "context"
- "database/sql"
- "log/slog"
- "os"
- "os/signal"
- "sync"
- "syscall"
-
- _ "github.com/lib/pq"
-
- frontier "github.com/devraulu/crowlr/pkg"
- "github.com/devraulu/crowlr/pkg/config"
- "github.com/devraulu/crowlr/pkg/crawler"
- "github.com/devraulu/crowlr/pkg/logger"
- "github.com/devraulu/crowlr/pkg/storage"
-)
-
func main() {
- var err error
-
- cfg, err := config.Load("config.toml")
- if err != nil {
- slog.Error("fatal: couldn't load config", slog.Any("err", err))
- os.Exit(1)
- }
-
- logger.InitLogger(cfg)
-
- f := frontier.NewFrontier()
-
- if err := frontier.LoadSeeds(cfg.Crawler.SeedsFile, f); err != nil {
- slog.Error("fatal: couldn't load seeds", slog.Any("err", err))
- os.Exit(1)
- }
-
- pool, err := sql.Open("postgres", cfg.DSN)
- if err != nil {
- slog.Error("fatal: couldn't open database", slog.Any("err", err))
- os.Exit(1)
- }
-
- defer pool.Close()
-
- if err := storage.RunMigrations(pool); err != nil {
- slog.Error("fatal: failed to run migrations", "err", err)
- os.Exit(1)
- }
-
- store := storage.NewPostgresStorage(pool)
-
- c := crawler.New(cfg, f, store)
-
- ctx, stop := context.WithCancel(context.Background())
- defer stop()
-
- var wg sync.WaitGroup
-
- appSignal := make(chan os.Signal, 1)
- signal.Notify(appSignal, syscall.SIGINT, syscall.SIGQUIT)
-
- wg.Add(1)
- go func() {
- defer wg.Done()
- c.Start(ctx)
- stop()
- }()
-
- select {
- case s := <-appSignal:
- slog.Info("received system signal", slog.String("signal", s.String()))
- stop()
- case <-ctx.Done():
- slog.Info("context done, stopping")
- }
-
- wg.Wait()
- slog.Info("shutdown complete")
+ Execute()
}
diff --git a/cmd/crawler/root.go b/cmd/crawler/root.go
new file mode 100644
index 0000000..fd7727b
--- /dev/null
+++ b/cmd/crawler/root.go
@@ -0,0 +1,25 @@
+package main
+
+import (
+ "os"
+
+ "github.com/spf13/cobra"
+)
+
+var cfgFile string
+
+var rootCmd = &cobra.Command{
+ Use: "crowlr",
+ Short: "A fast, polite web crawler",
+ SilenceUsage: true,
+}
+
+func init() {
+ rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "config.toml", "config file path")
+}
+
+func Execute() {
+ if err := rootCmd.Execute(); err != nil {
+ os.Exit(1)
+ }
+}
diff --git a/cmd/web/static/fonts/geistmono_variable_wght.ttf b/cmd/crawler/static/fonts/geistmono_variable_wght.ttf
similarity index 100%
rename from cmd/web/static/fonts/geistmono_variable_wght.ttf
rename to cmd/crawler/static/fonts/geistmono_variable_wght.ttf
diff --git a/cmd/web/static/styles.css b/cmd/crawler/static/styles.css
similarity index 100%
rename from cmd/web/static/styles.css
rename to cmd/crawler/static/styles.css
diff --git a/cmd/web/templates/index.html b/cmd/crawler/templates/index.html
similarity index 100%
rename from cmd/web/templates/index.html
rename to cmd/crawler/templates/index.html
diff --git a/cmd/web/templates/results.html b/cmd/crawler/templates/results.html
similarity index 100%
rename from cmd/web/templates/results.html
rename to cmd/crawler/templates/results.html
diff --git a/cmd/crawler/testhelper_test.go b/cmd/crawler/testhelper_test.go
new file mode 100644
index 0000000..96fa8d5
--- /dev/null
+++ b/cmd/crawler/testhelper_test.go
@@ -0,0 +1,125 @@
+package main
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "log"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ _ "github.com/lib/pq"
+ "github.com/testcontainers/testcontainers-go"
+ "github.com/testcontainers/testcontainers-go/modules/postgres"
+
+ "github.com/devraulu/crowlr/pkg/crawler"
+)
+
+var pgContainer *postgres.PostgresContainer
+var logSink testWriter
+
+func TestMain(m *testing.M) {
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logSink, nil)))
+
+ ctx := context.Background()
+
+ var err error
+ pgContainer, err = postgres.Run(ctx, "postgres:16-alpine",
+ postgres.WithDatabase("postgres"),
+ postgres.WithUsername("crowlr"),
+ postgres.WithPassword("password"),
+ postgres.BasicWaitStrategies(),
+ )
+ if err != nil {
+ log.Fatalf("start postgres container: %v", err)
+ }
+
+ code := m.Run()
+
+ if err := testcontainers.TerminateContainer(pgContainer); err != nil {
+ log.Printf("terminate postgres container: %v", err)
+ }
+
+ os.Exit(code)
+}
+
+type testApp struct {
+ store *crawler.PostgresStore
+ db *sql.DB
+ server *httptest.Server
+}
+
+func newTestApp(t *testing.T, handler http.Handler) *testApp {
+ t.Helper()
+ ctx := context.Background()
+
+ baseConnStr, err := pgContainer.ConnectionString(ctx, "sslmode=disable")
+ if err != nil {
+ t.Fatalf("get connection string: %v", err)
+ }
+
+ adminDB, err := sql.Open("postgres", baseConnStr)
+ if err != nil {
+ t.Fatalf("open admin db: %v", err)
+ }
+ defer adminDB.Close()
+
+ dbName := fmt.Sprintf("test_%d", time.Now().UnixNano())
+ if _, err := adminDB.ExecContext(ctx, fmt.Sprintf(`CREATE DATABASE "%s"`, dbName)); err != nil {
+ t.Fatalf("create test db: %v", err)
+ }
+
+ u, _ := url.Parse(baseConnStr)
+ u.Path = "/" + dbName
+ db, err := sql.Open("postgres", u.String())
+ if err != nil {
+ t.Fatalf("open test db: %v", err)
+ }
+ t.Cleanup(func() { db.Close() })
+
+ if err := crawler.RunMigrations(db); err != nil {
+ t.Fatalf("run migrations: %v", err)
+ }
+
+ server := httptest.NewServer(handler)
+ t.Cleanup(server.Close)
+
+ return &testApp{
+ store: crawler.NewPostgresStore(db),
+ db: db,
+ server: server,
+ }
+}
+
+type testWriter struct {
+ mu sync.Mutex
+ t testing.TB
+}
+
+func (w *testWriter) Write(p []byte) (int, error) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.t != nil {
+ w.t.Log(strings.TrimRight(string(p), "\n"))
+ }
+ return len(p), nil
+}
+
+func setLogger(t testing.TB) {
+ t.Helper()
+ logSink.mu.Lock()
+ logSink.t = t
+ logSink.mu.Unlock()
+ t.Cleanup(func() {
+ logSink.mu.Lock()
+ logSink.t = nil
+ logSink.mu.Unlock()
+ })
+}
diff --git a/cmd/crawler/web.go b/cmd/crawler/web.go
new file mode 100644
index 0000000..4ea333b
--- /dev/null
+++ b/cmd/crawler/web.go
@@ -0,0 +1,135 @@
+package main
+
+import (
+ "context"
+ "database/sql"
+ "embed"
+ "fmt"
+ "html/template"
+ "io/fs"
+ "log/slog"
+ "net/http"
+
+ _ "github.com/lib/pq"
+ "github.com/spf13/cobra"
+
+ "github.com/devraulu/crowlr/pkg/config"
+ "github.com/devraulu/crowlr/pkg/logger"
+ "github.com/devraulu/crowlr/pkg/crawler"
+)
+
+var webCmd = &cobra.Command{
+ Use: "web",
+ Short: "Start the web search interface",
+ RunE: runWeb,
+}
+
+var flagPort int
+
+func init() {
+ webCmd.Flags().IntVarP(&flagPort, "port", "p", 8080, "port to listen on")
+ rootCmd.AddCommand(webCmd)
+}
+
+//go:embed templates/*
+var templates embed.FS
+
+//go:embed static/*
+var staticFiles embed.FS
+
+type SearchResults struct {
+ Results []crawler.SearchResult
+ Count int
+ Query string
+}
+
+var tmpl *template.Template
+
+func runWeb(cmd *cobra.Command, args []string) error {
+ cfg, err := config.Load(cfgFile)
+ if err != nil {
+ return fmt.Errorf("failed to load config: %w", err)
+ }
+
+ logger.InitLogger(cfg)
+
+ db, err := sql.Open("postgres", cfg.DSN)
+ if err != nil {
+ return fmt.Errorf("failed to open database: %w", err)
+ }
+ defer db.Close()
+
+ if err := db.Ping(); err != nil {
+ return fmt.Errorf("failed to ping database: %w", err)
+ }
+ slog.Debug("database connected")
+
+ store := crawler.NewPostgresStore(db)
+
+ funcMap := template.FuncMap{
+ "safeHTML": func(s string) template.HTML { return template.HTML(s) },
+ }
+ tmpl = template.Must(template.New("").Funcs(funcMap).ParseFS(templates, "templates/*.html"))
+
+ mux := http.NewServeMux()
+ mux.HandleFunc("/", handleIndex)
+ mux.HandleFunc("/search", handleSearch(store))
+
+ staticFS, _ := fs.Sub(staticFiles, "static")
+ mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(staticFS))))
+
+ addr := fmt.Sprintf(":%d", flagPort)
+ slog.Info("starting web server", "addr", addr)
+
+ if err := http.ListenAndServe(addr, mux); err != nil {
+ return fmt.Errorf("server failed: %w", err)
+ }
+
+ return nil
+}
+
+func handleIndex(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/" {
+ http.NotFound(w, r)
+ return
+ }
+ slog.Info("request", "method", r.Method, "path", r.URL.Path)
+ if err := tmpl.ExecuteTemplate(w, "index.html", nil); err != nil {
+ slog.Error("failed to render index", slog.Any("err", err))
+ http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+ }
+}
+
+func handleSearch(store *crawler.PostgresStore) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ query := r.URL.Query().Get("q")
+ if query == "" {
+ if err := tmpl.ExecuteTemplate(w, "results.html", nil); err != nil {
+ slog.Error("failed to render results", slog.Any("err", err))
+ http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+ }
+ return
+ }
+
+ slog.Info("search", slog.String("query", query))
+
+ searchResponse, err := store.Search(context.Background(), query, 500)
+ if err != nil {
+ slog.Error("search failed", slog.String("query", query), slog.Any("err", err))
+ http.Error(w, "Search failed", http.StatusInternalServerError)
+ return
+ }
+
+ searchResults := SearchResults{
+ Results: searchResponse.Results,
+ Count: searchResponse.TotalCount,
+ Query: query,
+ }
+
+ slog.Info("search complete", slog.String("query", query), slog.Int("results", len(searchResponse.Results)), slog.Int("total", searchResponse.TotalCount))
+ if err := tmpl.ExecuteTemplate(w, "results.html", searchResults); err != nil {
+ slog.Error("failed to render results", slog.Any("err", err))
+ http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+ }
+ }
+}
diff --git a/cmd/web/main.go b/cmd/web/main.go
deleted file mode 100644
index f016b57..0000000
--- a/cmd/web/main.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package main
-
-import (
- "context"
- "database/sql"
- "embed"
- "html/template"
- "io/fs"
- "log"
- "log/slog"
- "net/http"
-
- _ "github.com/lib/pq"
-
- "github.com/devraulu/crowlr/pkg/config"
- "github.com/devraulu/crowlr/pkg/logger"
- "github.com/devraulu/crowlr/pkg/storage"
-)
-
-//go:embed templates/*
-var templates embed.FS
-
-//go:embed static/*
-var staticFiles embed.FS
-
-type SearchResults struct {
- Results []storage.SearchResult
- Count int
- Query string
-}
-
-var tmpl *template.Template
-
-func main() {
- cfg, err := config.Load("config.toml")
- if err != nil {
- log.Fatal(err)
- }
-
- logger.InitLogger(cfg)
-
- db, err := sql.Open("postgres", cfg.DSN)
- if err != nil {
- log.Fatal(err)
- }
- defer db.Close()
-
- store := storage.NewPostgresStorage(db)
-
- funcMap := template.FuncMap{
- "safeHTML": func(s string) template.HTML { return template.HTML(s) },
- }
- tmpl = template.Must(template.New("").Funcs(funcMap).ParseFS(templates, "templates/*.html"))
-
- http.HandleFunc("/", handleIndex)
- http.HandleFunc("/search", handleSearch(store))
-
- staticFS, _ := fs.Sub(staticFiles, "static")
- http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(staticFS))))
-
- addr := ":8080"
- slog.Info("starting web server", "addr", addr)
- log.Fatal(http.ListenAndServe(addr, nil))
-}
-
-func handleIndex(w http.ResponseWriter, r *http.Request) {
- slog.Info("request", "method", r.Method, "path", r.URL.Path)
- tmpl.ExecuteTemplate(w, "index.html", nil)
-}
-
-func handleSearch(store *storage.PostgresStorage) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- query := r.URL.Query().Get("q")
- if query == "" {
- tmpl.ExecuteTemplate(w, "results.html", nil)
- return
- }
-
- slog.Info("search", slog.String("query", query))
-
- searchResponse, err := store.Search(context.Background(), query, 500)
- if err != nil {
- slog.Error("search failed", slog.String("query", query), slog.Any("err", err))
- http.Error(w, "Search failed", http.StatusInternalServerError)
- return
- }
-
- searchResults := SearchResults{
- Results: searchResponse.Results,
- Count: searchResponse.TotalCount,
- Query: query,
- }
-
- slog.Info("search complete", slog.String("query", query), slog.Int("results", len(searchResponse.Results)), slog.Int("total", searchResponse.TotalCount))
- tmpl.ExecuteTemplate(w, "results.html", searchResults)
- }
-}
diff --git a/config.example.toml b/config.example.toml
index cadbb49..0d85712 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -9,7 +9,7 @@ workers = 8
[politeness]
delay = "1s"
-robots_timeout = "10s"
+fetch_timeout = "10s"
[logging]
level = "info" # debug, info, warn, error
diff --git a/crawler b/crawler
new file mode 100755
index 0000000..4bf1ea5
Binary files /dev/null and b/crawler differ
diff --git a/go.mod b/go.mod
index 0e8972c..48a57fd 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module github.com/devraulu/crowlr
-go 1.24.5
+go 1.25.0
require (
github.com/PuerkitoBio/purell v1.2.1
@@ -8,16 +8,65 @@ require (
github.com/golang-migrate/migrate/v4 v4.19.1
github.com/lib/pq v1.10.9
github.com/pelletier/go-toml/v2 v2.2.4
- golang.org/x/net v0.47.0
+ github.com/spf13/cobra v1.10.2
+ github.com/testcontainers/testcontainers-go v0.42.0
+ github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
+ golang.org/x/net v0.49.0
)
require (
+ dario.cat/mergo v1.0.2 // indirect
+ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
+ github.com/Microsoft/go-winio v0.6.2 // indirect
+ github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/containerd/errdefs v1.0.0 // indirect
+ github.com/containerd/errdefs/pkg v0.3.0 // indirect
+ github.com/containerd/log v0.1.0 // indirect
+ github.com/containerd/platforms v0.2.1 // indirect
+ github.com/cpuguy83/dockercfg v0.3.2 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/distribution/reference v0.6.0 // indirect
github.com/docker/docker v28.5.1+incompatible // indirect
github.com/docker/go-connections v0.6.0 // indirect
+ github.com/docker/go-units v0.5.0 // indirect
+ github.com/ebitengine/purego v0.10.0 // indirect
+ github.com/felixge/httpsnoop v1.0.4 // indirect
+ github.com/go-logr/logr v1.4.3 // indirect
+ github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/go-ole/go-ole v1.2.6 // indirect
+ github.com/google/uuid v1.6.0 // indirect
+ github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/klauspost/compress v1.18.5 // indirect
+ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+ github.com/magiconair/properties v1.8.10 // indirect
+ github.com/moby/docker-image-spec v1.3.1 // indirect
+ github.com/moby/go-archive v0.2.0 // indirect
+ github.com/moby/moby/api v1.54.1 // indirect
+ github.com/moby/moby/client v0.4.0 // indirect
+ github.com/moby/patternmatcher v0.6.1 // indirect
+ github.com/moby/sys/sequential v0.6.0 // indirect
+ github.com/moby/sys/user v0.4.0 // indirect
+ github.com/moby/sys/userns v0.1.0 // indirect
+ github.com/moby/term v0.5.2 // indirect
+ github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
+ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+ github.com/shirou/gopsutil/v4 v4.26.3 // indirect
+ github.com/sirupsen/logrus v1.9.4 // indirect
+ github.com/spf13/pflag v1.0.9 // indirect
github.com/stretchr/testify v1.11.1 // indirect
+ github.com/tklauser/go-sysconf v0.3.16 // indirect
+ github.com/tklauser/numcpus v0.11.0 // indirect
+ github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
- go.opentelemetry.io/otel/metric v1.38.0 // indirect
- go.opentelemetry.io/otel/trace v1.38.0 // indirect
- golang.org/x/text v0.32.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+ go.opentelemetry.io/otel v1.41.0 // indirect
+ go.opentelemetry.io/otel/metric v1.41.0 // indirect
+ go.opentelemetry.io/otel/trace v1.41.0 // indirect
+ golang.org/x/crypto v0.48.0 // indirect
+ golang.org/x/sys v0.42.0 // indirect
+ golang.org/x/text v0.34.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index cd27789..ba8ea91 100644
--- a/go.sum
+++ b/go.sum
@@ -1,15 +1,32 @@
-github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
-github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
+dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/PuerkitoBio/purell v1.2.1 h1:QsZ4TjvwiMpat6gBCBxEQI0rcS9ehtkKtSpiUnd9N28=
github.com/PuerkitoBio/purell v1.2.1/go.mod h1:ZwHcC/82TOaovDi//J/804umJFFmbOHPngi8iYYv/Eo=
github.com/benjaminestes/robots v2.0.4+incompatible h1:SGr/APXpUcozEAzmN8WAkTJ1VLzOQNggj3KQ99gMs5E=
github.com/benjaminestes/robots v2.0.4+incompatible/go.mod h1:UDP8zkjT11SFzRbWuGilsONCRUNJtd6IUgAAliGOMKM=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
+github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
+github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
+github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
@@ -22,20 +39,66 @@ github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pM
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
+github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8=
+github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
+github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
+github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
-github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
-github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
+github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
+github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4=
+github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
+github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw=
+github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g=
+github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U=
+github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
+github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
+github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
+github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
+github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
+github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
+github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
+github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -48,23 +111,70 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
+github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
+github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
+github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
+github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
+github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo=
+github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs=
+github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
+github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
+github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
-go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
-go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
-go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
-go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
-go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
-go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
-golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
+go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
+go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
+go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
+go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
+go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
+go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
+go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis=
+go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4=
+go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
+go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
+golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
+gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
+pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=
+pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 45976a5..5c16043 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -1,6 +1,7 @@
package config
import (
+ "fmt"
"os"
"time"
@@ -22,8 +23,11 @@ type CrawlerConfig struct {
}
type PolitenessConfig struct {
- Delay string `toml:"delay"`
- RobotsTimeout string `toml:"robots_timeout"`
+ Delay string `toml:"delay"`
+ FetchTimeout string `toml:"fetch_timeout"`
+
+ delay time.Duration
+ fetchTimeout time.Duration
}
type LoggingConfig struct {
@@ -40,6 +44,7 @@ func Load(path string) (*Config, error) {
var cfg Config
cfg.Crawler.SeedsFile = "seeds.txt"
cfg.Politeness.Delay = "1s"
+ cfg.Politeness.FetchTimeout = "10s"
cfg.Logging.Format = "text"
cfg.Logging.Level = "info"
@@ -48,13 +53,48 @@ func Load(path string) (*Config, error) {
return nil, err
}
+ if err := cfg.Validate(); err != nil {
+ return nil, err
+ }
+
return &cfg, nil
}
+func (c *Config) Validate() error {
+ if err := c.Politeness.Validate(); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (c *PolitenessConfig) Validate() error {
+ delay, err := parseDuration("politeness.delay", c.Delay)
+ if err != nil {
+ return err
+ }
+
+ fetchTimeout, err := parseDuration("politeness.fetch_timeout", c.FetchTimeout)
+ if err != nil {
+ return err
+ }
+
+ c.delay = delay
+ c.fetchTimeout = fetchTimeout
+ return nil
+}
+
func (c *PolitenessConfig) GetDelay() time.Duration {
- d, err := time.ParseDuration(c.Delay)
+ return c.delay
+}
+
+func (c *PolitenessConfig) GetFetchTimeout() time.Duration {
+ return c.fetchTimeout
+}
+
+func parseDuration(name, value string) (time.Duration, error) {
+ d, err := time.ParseDuration(value)
if err != nil {
- return 1 * time.Second // Fallback
+ return 0, fmt.Errorf("invalid %s %q: %w", name, value, err)
}
- return d
+ return d, nil
}
diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
new file mode 100644
index 0000000..7f1b9d7
--- /dev/null
+++ b/pkg/config/config_test.go
@@ -0,0 +1,58 @@
+package config
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestLoadParsesPolitenessDurations(t *testing.T) {
+ path := writeConfig(t, `
+dsn = "postgres://example"
+
+[politeness]
+delay = "250ms"
+fetch_timeout = "3s"
+`)
+
+ cfg, err := Load(path)
+ if err != nil {
+ t.Fatalf("Load returned error: %v", err)
+ }
+
+ if got := cfg.Politeness.GetDelay(); got != 250*time.Millisecond {
+ t.Fatalf("delay = %v, want 250ms", got)
+ }
+ if got := cfg.Politeness.GetFetchTimeout(); got != 3*time.Second {
+ t.Fatalf("fetch timeout = %v, want 3s", got)
+ }
+}
+
+func TestLoadRejectsInvalidPolitenessDuration(t *testing.T) {
+ path := writeConfig(t, `
+dsn = "postgres://example"
+
+[politeness]
+delay = "ten seconds"
+`)
+
+ _, err := Load(path)
+ if err == nil {
+ t.Fatal("Load returned nil error")
+ }
+ if !strings.Contains(err.Error(), "politeness.delay") {
+ t.Fatalf("error = %q, want field name", err.Error())
+ }
+}
+
+func writeConfig(t *testing.T, content string) string {
+ t.Helper()
+
+ path := filepath.Join(t.TempDir(), "config.toml")
+ if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
+ t.Fatalf("write config: %v", err)
+ }
+ return path
+}
diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go
index 4ed72ac..a599818 100644
--- a/pkg/crawler/crawler.go
+++ b/pkg/crawler/crawler.go
@@ -1,182 +1,482 @@
package crawler
import (
+ "bytes"
"context"
+ "encoding/xml"
+ "errors"
+ "io"
"log/slog"
+ "net/http"
+ "net/url"
+ "runtime"
+ "strings"
"time"
"github.com/benjaminestes/robots"
- frontier "github.com/devraulu/crowlr/pkg"
- "github.com/devraulu/crowlr/pkg/config"
- "github.com/devraulu/crowlr/pkg/process"
- "github.com/devraulu/crowlr/pkg/storage"
+ "golang.org/x/net/html"
)
-type CrawlStats struct {
- StartTime time.Time
- PagesProcessed int
- PagesErrored int
- PagesSkipped int
+type Fetcher interface {
+ Fetch(*http.Request) (*http.Response, error)
}
-func (s *CrawlStats) Elapsed() time.Duration {
- return time.Since(s.StartTime)
+type HTTPFetcher struct {
+ client *http.Client
}
-func (s *CrawlStats) PagesPerSecond() float64 {
- elapsed := s.Elapsed().Seconds()
- if elapsed == 0 {
- return 0
+func NewHTTPFetcher(client *http.Client) *HTTPFetcher {
+ return &HTTPFetcher{client: client}
+}
+
+func (f *HTTPFetcher) Fetch(req *http.Request) (*http.Response, error) {
+ return f.client.Do(req)
+}
+
+func (c *Crawler) fetch(ctx context.Context, rawURL string) (*http.Response, error) {
+ req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
+ if err != nil {
+ return nil, err
}
- return float64(s.PagesProcessed) / elapsed
+ if c.userAgent != "" {
+ req.Header.Set("User-Agent", c.userAgent)
+ }
+ return c.fetcher.Fetch(req)
}
-type Crawler struct {
- cfg *config.Config
- frontier *frontier.Frontier
- store storage.Storage
- robotsCache map[string]*robots.Robots
- Stats CrawlStats
+type Page struct {
+ URL string
+ RawURL string
+ Referrer string
+ StatusCode int
+ HTML string
+ Outlinks []string
+ FetchedAt time.Time
+ Title string
}
-type Job struct {
- referrer string
- rawUrl string
- url string
+type PageSaver interface {
+ SavePage(ctx context.Context, p Page) error
}
-func New(cfg *config.Config, f *frontier.Frontier, s storage.Storage) *Crawler {
- return &Crawler{
- cfg: cfg,
- frontier: f,
- store: s,
- robotsCache: make(map[string]*robots.Robots),
- }
+type NullStore struct{}
+
+func (NullStore) SavePage(_ context.Context, _ Page) error { return nil }
+
+type CrawlerOption func(*Crawler)
+
+func WithCrawlDelay(d time.Duration) CrawlerOption {
+ return func(c *Crawler) { c.crawlDelay = d }
+}
+
+func WithUserAgent(ua string) CrawlerOption {
+ return func(c *Crawler) { c.userAgent = ua }
+}
+
+func WithWorkers(n int) CrawlerOption {
+ return func(c *Crawler) { c.workers = n }
+}
+
+func WithCrawlLimit(n int) CrawlerOption {
+ return func(c *Crawler) { c.crawlLimit = n }
}
-func (c *Crawler) Start(ctx context.Context) {
- c.Stats.StartTime = time.Now()
+func WithFetchTimeout(d time.Duration) CrawlerOption {
+ return func(c *Crawler) { c.fetchTimeout = d }
+}
- jobs := make(chan frontier.Candidate, c.cfg.Crawler.Workers)
- results := make(chan CrawlResult, c.cfg.Crawler.Workers)
+func NewCrawler(fetcher Fetcher, store PageSaver, opts ...CrawlerOption) *Crawler {
+ crawler := &Crawler{
+ fetcher: fetcher,
+ store: store,
+ frontier: NewFrontier(),
+ workers: runtime.NumCPU(),
+ hosts: make(map[string]time.Time),
+ robotsCache: map[string]*robots.Robots{},
+ stats: NewStats(),
+ }
- for i := 0; i < c.cfg.Crawler.Workers; i++ {
- go c.worker(ctx, i, jobs, results)
+ for _, opt := range opts {
+ opt(crawler)
}
+ return crawler
+}
+
+type Crawler struct {
+ fetcher Fetcher
+ frontier *Frontier
+ workers int
+ crawlLimit int
+ crawlDelay time.Duration
+ fetchTimeout time.Duration
+ robotsCache map[string]*robots.Robots
+ hosts map[string]time.Time
+ userAgent string
+ store PageSaver
+ stats *Stats
+}
- c.coordinator(ctx, jobs, results)
+func (c *Crawler) Run(ctx context.Context, seeds []Link) error {
+ if len(seeds) == 0 {
+ return errors.New("no seeds provided")
+ }
- slog.Info("crawl complete",
- slog.Int("processed", c.Stats.PagesProcessed),
- slog.Int("errored", c.Stats.PagesErrored),
- slog.Int("skipped", c.Stats.PagesSkipped),
- slog.Duration("elapsed", c.Stats.Elapsed()),
- slog.Float64("pages_per_sec", c.Stats.PagesPerSecond()),
+ c.loadSeeds(seeds)
+ slog.Info("crawler starting",
+ slog.Int("workers", c.workers),
+ slog.Int("seeds", len(seeds)),
+ slog.Int("crawl_limit", c.crawlLimit),
+ slog.Duration("crawl_delay", c.crawlDelay),
+ slog.String("user_agent", c.userAgent),
)
-}
-func (c *Crawler) coordinator(ctx context.Context, jobs chan<- frontier.Candidate, results <-chan CrawlResult) {
- activeWorkers := 0
- for {
- if c.cfg.Crawler.CrawlLimit > 0 && c.Stats.PagesProcessed >= c.cfg.Crawler.CrawlLimit {
- for activeWorkers > 0 {
- select {
- case res := <-results:
- activeWorkers--
- c.processResult(ctx, res)
- case <-ctx.Done():
- return
+ jobs := make(chan Link, c.workers)
+ out := make(chan []Link, c.workers)
+ var retryTimer <-chan time.Time
+
+ for range c.workers {
+ go func() {
+ for link := range jobs {
+ fetchCtx := ctx
+ if c.fetchTimeout > 0 {
+ var cancel context.CancelFunc
+ fetchCtx, cancel = context.WithTimeout(ctx, c.fetchTimeout)
+ defer cancel()
+ }
+ result := c.visit(fetchCtx, link)
+ if result.Err != nil {
+ c.stats.RecordError()
+ slog.Error("visit failed", slog.String("url", link.Normalized), slog.Any("err", result.Err))
+ out <- nil
+ continue
+ }
+
+ if result.Body == nil {
+ c.stats.RecordVisit(link.Host, result.StatusCode, result.Duration, result.LastModified, 0)
+ out <- nil
+ continue
+ }
+
+ c.stats.RecordVisit(link.Host, result.StatusCode, result.Duration, result.LastModified, len(result.Outlinks))
+ slog.Info("visited", slog.String("url", link.Normalized), slog.Int("status", result.StatusCode), slog.Int("outlinks", len(result.Outlinks)))
+ outstrs := make([]string, len(result.Outlinks))
+ for i, ol := range result.Outlinks {
+ outstrs[i] = ol.Normalized
+ }
+
+ if err := c.store.SavePage(ctx, Page{
+ URL: link.Normalized,
+ RawURL: link.Original,
+ Referrer: link.Referrer,
+ StatusCode: result.StatusCode,
+ HTML: string(result.Body),
+ Outlinks: outstrs,
+ FetchedAt: time.Now(),
+ Title: result.Title,
+ }); err != nil {
+ slog.Warn("failed to save page", slog.String("url", link.Normalized), slog.Any("err", err))
}
+ out <- result.Outlinks
}
- return
- }
+ }()
+ }
- var candidate *frontier.Candidate
- var jobsChan chan<- frontier.Candidate
- var waitTime time.Duration
+ var pending, pagesProcessed int
+ for {
+ limitReached := c.crawlLimit > 0 && pagesProcessed >= c.crawlLimit
- if c.frontier.Len() > 0 {
- candidate, waitTime = c.frontier.Pop(c.cfg.Politeness.GetDelay())
- if candidate == nil {
- slog.Info("frontier empty and no active workers. mission complete.")
- return
+ if (c.frontier.Len() == 0 || limitReached) && pending == 0 {
+ if limitReached {
+ slog.Info("crawl limit reached", slog.Int("limit", c.crawlLimit))
+ } else {
+ slog.Info("crawl complete")
}
- if candidate.Normalized != "" {
- r := process.CheckRobots(candidate.Normalized, c.robotsCache)
- if r != nil && !r.Test(c.cfg.Crawler.UserAgent, candidate.Normalized) {
- slog.Info("robots.txt disallowed", slog.Any("candidate", candidate))
- continue
- }
+ c.stats.Log(c.frontier.Len())
+ close(jobs)
+ return nil
+ }
- jobsChan = jobs
- } else if waitTime > 0 {
- time.Sleep(waitTime)
- continue
+ var (
+ jobCh chan<- Link
+ next Link
+ )
+
+ if c.frontier.Len() > 0 && !limitReached {
+ link := c.frontier.PopElegible(c.politeness)
+ if link != nil && c.robotsCheck(ctx, link) {
+ c.hosts[link.Host] = time.Now()
+ jobCh = jobs
+ next = *link
+ } else {
+ retryTimer = time.After(c.crawlDelay)
}
- } else if activeWorkers == 0 {
- slog.Info("frontier empty and no active workers. mission complete.")
- return
}
- if candidate != nil {
- select {
- case jobsChan <- *candidate:
- activeWorkers++
- slog.Info("job dispatched", slog.Any("candidate", candidate), slog.Int("active_workers", activeWorkers), slog.Int("seen", c.frontier.Len()))
+ select {
+ case <-ctx.Done():
+ close(jobs)
+ return ctx.Err()
+
+ case jobCh <- next:
+ pending++
- case res := <-results:
- activeWorkers--
- c.processResult(ctx, res)
+ case <-retryTimer:
+ continue
- case <-ctx.Done():
- return
+ case outlinks := <-out:
+ pending--
+ if outlinks != nil {
+ pagesProcessed++
}
- } else {
- select {
- case res := <-results:
- activeWorkers--
- c.processResult(ctx, res)
- case <-ctx.Done():
- return
+ for _, ol := range outlinks {
+ if !c.frontier.Push(ol) {
+ slog.Debug("frontier duplicate, skipping", slog.String("url", ol.Normalized))
+ continue
+ }
}
}
}
}
-func (c *Crawler) processResult(ctx context.Context, res CrawlResult) {
- if res.Error != nil {
- c.Stats.PagesErrored++
- slog.Error("crawl failed", slog.String("url", res.URL), slog.Any("err", res.Error))
+func (c *Crawler) politeness(link Link) bool {
+ if t, ok := c.hosts[link.Host]; ok {
+ if time.Since(t) < c.crawlDelay {
+ return false
+ }
+ }
+
+ return true
+}
+
+func (c *Crawler) loadSeeds(seeds []Link) {
+ for _, link := range seeds {
+ c.frontier.Push(link)
+ }
+}
+
+func (c *Crawler) robotsCheck(ctx context.Context, link *Link) bool {
+ robotsURL, err := robots.Locate(link.Normalized)
+ if err != nil {
+ // shouldn't fail because NewLink already fails if URL is invalid
+ return true
+ }
+
+ var r *robots.Robots
+ if cached, ok := c.robotsCache[robotsURL]; ok {
+ r = cached
+ } else {
+ r = c.fetchRobots(ctx, robotsURL)
+ c.robotsCache[robotsURL] = r
+ if r != nil {
+ slog.Debug("fetched robots.txt", slog.String("url", robotsURL))
+ c.sitemaps(ctx, r)
+ }
+ }
+
+ if r != nil && !r.Test(c.userAgent, link.Normalized) {
+ slog.Debug("disallowed by robots.txt", slog.String("url", link.Normalized))
+ return false
+ }
+
+ return true
+}
+
+func (c *Crawler) fetchRobots(ctx context.Context, robotsURL string) *robots.Robots {
+ resp, err := c.fetch(ctx, robotsURL)
+ if err != nil || resp.StatusCode != http.StatusOK {
+ return nil
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil || len(body) == 0 {
+ return nil
+ }
+
+ r, err := robots.From(resp.StatusCode, bytes.NewReader(body))
+ if err != nil {
+ return nil
+ }
+ return r
+}
+
+func (c *Crawler) sitemaps(ctx context.Context, r *robots.Robots) {
+ for _, sitemapURL := range r.Sitemaps() {
+ c.fetchSitemap(ctx, sitemapURL)
+ }
+}
+
+func (c *Crawler) fetchSitemap(ctx context.Context, sitemapURL string) {
+ slog.Debug("fetching sitemap", slog.String("url", sitemapURL))
+ link, err := NewLink(sitemapURL)
+ if err != nil || !c.robotsCheck(ctx, &link) {
return
}
- if res.PageData == nil {
- c.Stats.PagesSkipped++
+ resp, err := c.fetch(ctx, sitemapURL)
+ if err != nil || resp.StatusCode != http.StatusOK {
return
}
+ defer resp.Body.Close()
- c.Stats.PagesProcessed++
- slog.Info("crawl success",
- slog.String("url", res.URL),
- slog.Int("outlinks", len(res.Outlinks)),
- slog.Int("processed", c.Stats.PagesProcessed),
- slog.Float64("pages_per_sec", c.Stats.PagesPerSecond()),
- )
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return
+ }
+
+ var index struct {
+ Sitemaps []struct {
+ Loc string `xml:"loc"`
+ } `xml:"sitemap"`
+ }
+ if err := xml.Unmarshal(body, &index); err == nil && len(index.Sitemaps) > 0 {
+ for _, s := range index.Sitemaps {
+ c.fetchSitemap(ctx, s.Loc)
+ }
+ return
+ }
+
+ var urlset struct {
+ URLs []struct {
+ Loc string `xml:"loc"`
+ } `xml:"url"`
+ }
+ if err := xml.Unmarshal(body, &urlset); err != nil {
+ return
+ }
+ for _, u := range urlset.URLs {
+ if l, err := NewLink(u.Loc); err == nil {
+ c.frontier.Push(l)
+ slog.Debug("sitemap url queued", slog.String("url", u.Loc))
+ }
+ }
+}
- if err := c.store.SavePage(ctx, *res.PageData); err != nil {
- c.Stats.PagesErrored++
- slog.Error("failed to save page", slog.String("url", res.URL), slog.Any("err", err))
+type VisitResult struct {
+ StatusCode int
+ Body []byte
+ Outlinks []Link
+ Title string
+ Duration time.Duration
+ LastModified *time.Time
+ Err error
+}
+
+func (c *Crawler) visit(ctx context.Context, link Link) VisitResult {
+ start := time.Now()
+ resp, err := c.fetch(ctx, link.Normalized)
+ dur := time.Since(start)
+ if err != nil {
+ return VisitResult{Err: err, Duration: dur}
}
+ defer resp.Body.Close()
- for _, link := range res.Outlinks {
- if c.cfg.Crawler.CrawlLimit > 0 && c.Stats.PagesProcessed >= c.cfg.Crawler.CrawlLimit {
- slog.Info("crawl limit reached, stopping outlink push",
- slog.Int("processed", c.Stats.PagesProcessed),
- slog.Int("queued", c.frontier.Len()),
- slog.Int("limit", c.cfg.Crawler.CrawlLimit),
- )
- break
+ var lastMod *time.Time
+ if raw := resp.Header.Get("Last-Modified"); raw != "" {
+ if t, err := http.ParseTime(raw); err == nil {
+ lastMod = &t
}
- c.frontier.Push(link.Normalized, link.Original, res.URL)
}
+
+ ct := resp.Header.Get("Content-Type")
+ if ct != "" && !strings.Contains(strings.ToLower(ct), "text/html") {
+ slog.Debug("skipping non-HTML response", slog.String("url", link.Normalized), slog.String("content_type", ct))
+ return VisitResult{StatusCode: resp.StatusCode, Duration: dur, LastModified: lastMod}
+ }
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return VisitResult{Err: err, Duration: dur}
+ }
+
+ title, outlinks, err := extract(bytes.NewReader(body), link)
+ if err != nil {
+ return VisitResult{Err: err, Duration: dur}
+ }
+
+ return VisitResult{
+ StatusCode: resp.StatusCode,
+ Body: body,
+ Outlinks: outlinks,
+ Title: title,
+ Duration: dur,
+ LastModified: lastMod,
+ }
+}
+
+func extract(r io.Reader, referrer Link) (string, []Link, error) {
+ base, _ := url.Parse(referrer.Normalized)
+ z := html.NewTokenizer(r)
+ var links []Link
+ var title string
+ var inTitle bool
+
+ for {
+ tt := z.Next()
+ switch tt {
+ case html.ErrorToken:
+ if z.Err() == io.EOF {
+ return title, links, nil
+ }
+ return title, links, z.Err()
+
+ case html.TextToken:
+ if inTitle {
+ title = string(z.Text())
+ inTitle = false
+ }
+
+ case html.EndTagToken:
+ name, _ := z.TagName()
+ if string(name) == "title" {
+ inTitle = false
+ }
+
+ case html.StartTagToken, html.SelfClosingTagToken:
+ name, hasAttr := z.TagName()
+ tag := string(name)
+
+ if tag == "title" {
+ inTitle = true
+ }
+
+ if !hasAttr {
+ continue
+ }
+
+ for {
+ k, v, more := z.TagAttr()
+ if tag == "base" && string(k) == "href" {
+ if newBase, err := base.Parse(string(v)); err == nil {
+ base = newBase
+ }
+ }
+ if tag == "a" && string(k) == "href" {
+ nl, err := NewLink(resolve(string(v), base), WithReferrer(referrer.Normalized))
+ if err == nil {
+ links = append(links, nl)
+ }
+ }
+ if !more {
+ break
+ }
+ }
+ }
+ }
+}
+
+func resolve(ref string, base *url.URL) string {
+ u, err := url.Parse(ref)
+ if err != nil {
+ return ""
+ }
+
+ abs := base.ResolveReference(u)
+
+ scheme := strings.ToLower(abs.Scheme)
+ if scheme != "http" && scheme != "https" {
+ return ""
+ }
+
+ return abs.String()
}
diff --git a/pkg/crawler/crawler_test.go b/pkg/crawler/crawler_test.go
new file mode 100644
index 0000000..1c5a455
--- /dev/null
+++ b/pkg/crawler/crawler_test.go
@@ -0,0 +1,588 @@
+package crawler
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+func TestCrawlerVisitsLink(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/foo": "",
+ "https://example.com/bar": "",
+ }}
+ link1, _ := NewLink("https://example.com/foo", WithReferrer("https://example.com/bar"))
+ link2, _ := NewLink("https://example.com/bar")
+
+ NewCrawler(fetcher, NullStore{}).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertTotalVisits(t, 2)
+}
+
+func TestCrawlerFindsNewLinks(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/foo": htmlBody("Title", "https://domain.com/discovered-1", "https://domain.com/discovered-2"),
+ "https://example.com/bar": "",
+ "https://domain.com/discovered-1": "",
+ "https://domain.com/discovered-2": "",
+ }}
+ link1, _ := NewLink("https://example.com/foo")
+ link2, _ := NewLink("https://example.com/bar")
+
+ NewCrawler(fetcher, NullStore{}).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertTotalVisits(t, 4)
+}
+
+func TestCrawlerFailsOnEmptySeeds(t *testing.T) {
+ err := NewCrawler(&mockFetcher{}, NullStore{}).Run(t.Context(), []Link{})
+ if err == nil {
+ t.Fatal("expected error, but got nil")
+ }
+}
+
+func TestCrawlerDoesNotCrawlDuplicates(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/foo": "",
+ }}
+ link1, _ := NewLink("https://example.com/foo")
+ link2, _ := NewLink("https://example.com:80/foO")
+
+ NewCrawler(fetcher, NullStore{}).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertTotalVisits(t, 1)
+}
+
+func TestShouldNotFetchTheSameHostBeforeDelay(t *testing.T) {
+ const delay = 50 * time.Millisecond
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/foo": "",
+ "https://example.com/bar": "",
+ }}
+ link1, _ := NewLink("https://example.com/foo")
+ link2, _ := NewLink("https://example.com/bar")
+
+ NewCrawler(fetcher, NullStore{}, WithCrawlDelay(delay)).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertTotalVisits(t, 2)
+ fetcher.assertVisitGap(t, "https://example.com/foo", "https://example.com/bar", delay)
+}
+
+func TestCrawlerShouldFetchNextAvailableHost(t *testing.T) {
+ const delay = 50 * time.Millisecond
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/foo": "",
+ "https://example.com/bar": "",
+ "https://baz.com/": "",
+ }}
+ link1, _ := NewLink("https://example.com/foo")
+ link2, _ := NewLink("https://example.com/bar")
+ link3, _ := NewLink("https://baz.com/")
+
+ NewCrawler(fetcher, NullStore{}, WithCrawlDelay(delay)).Run(t.Context(), []Link{link1, link2, link3})
+
+ fetcher.assertTotalVisits(t, 3)
+ fetcher.assertVisitedBefore(t, "https://baz.com/", "https://example.com/bar")
+}
+
+func TestCrawlerShouldRespectRobotsTxt(t *testing.T) {
+ t.Run("robots.txt should be fetched", func(t *testing.T) {
+ robotsURL := "https://domain.com/robots.txt"
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://domain.com/foo": "",
+ "https://domain.com/cat": "",
+ robotsURL: "",
+ }}
+ link1, _ := NewLink("https://domain.com/foo")
+ link2, _ := NewLink("https://domain.com/cat")
+
+ NewCrawler(fetcher, NullStore{}).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertVisited(t, robotsURL)
+ })
+
+ t.Run("disallowed by robots.txt", func(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://domain.com/foo": "",
+ "https://domain.com/cat": "",
+ "https://domain.com/robots.txt": robotsTxt("*", "/cat", "", ""),
+ }}
+ link1, _ := NewLink("https://domain.com/foo")
+ link2, _ := NewLink("https://domain.com/cat")
+
+ NewCrawler(fetcher, NullStore{}).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertNotVisited(t, "https://domain.com/cat")
+ })
+
+ t.Run("allowed by robots.txt", func(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://domain.com/foo": "",
+ "https://domain.com/cat": "",
+ "https://domain.com/robots.txt": robotsTxt("*", "", "/cat", ""),
+ }}
+ link1, _ := NewLink("https://domain.com/foo")
+ link2, _ := NewLink("https://domain.com/cat")
+
+ NewCrawler(fetcher, NullStore{}).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertVisited(t, "https://domain.com/cat")
+ })
+
+ t.Run("disallowed user agent is not visited", func(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://domain.com/foo": "",
+ "https://domain.com/cat": "",
+ "https://domain.com/robots.txt": robotsTxt("BadBot", "/", "", ""),
+ }}
+ link1, _ := NewLink("https://domain.com/foo")
+ link2, _ := NewLink("https://domain.com/cat")
+
+ NewCrawler(fetcher, NullStore{}, WithUserAgent("BadBot")).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertNotVisited(t, "https://domain.com/foo")
+ fetcher.assertNotVisited(t, "https://domain.com/cat")
+ })
+
+ t.Run("robots.txt is fetched once per host", func(t *testing.T) {
+ robotsURL := "https://domain.com/robots.txt"
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://domain.com/foo": "",
+ "https://domain.com/cat": "",
+ robotsURL: robotsTxt("*", "", "", ""),
+ }}
+ link1, _ := NewLink("https://domain.com/foo")
+ link2, _ := NewLink("https://domain.com/cat")
+
+ NewCrawler(fetcher, NullStore{}).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertHitCount(t, robotsURL, 1)
+ })
+}
+
+func TestSitemapLinksShouldBeCrawled(t *testing.T) {
+ sitemapURL := "https://domain.com/sitemap.xml"
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://domain.com/foo": "",
+ "https://domain.com/cat": "",
+ "https://domain.com/robots.txt": robotsTxt("*", "", "/cat", sitemapURL),
+ sitemapURL: sampleXML,
+ "http://www.example.net/?id=who": "",
+ "http://www.example.net/?id=what": "",
+ "http://www.example.net/?id=how": "",
+ }}
+ link1, _ := NewLink("https://domain.com/foo")
+ link2, _ := NewLink("https://domain.com/cat")
+
+ NewCrawler(fetcher, NullStore{}).Run(t.Context(), []Link{link1, link2})
+
+ fetcher.assertVisited(t, sitemapURL)
+ fetcher.assertVisited(t, "http://www.example.net/?id=who")
+}
+
+func TestCrawlerFetchesConcurrently(t *testing.T) {
+ const fetchDelay = 100 * time.Millisecond
+ fetcher := &mockFetcher{
+ fetchDelay: fetchDelay,
+ pages: map[string]string{
+ "https://example.com/robots.txt": "",
+ "https://example.com/1": "",
+ "https://example.com/2": "",
+ "https://example.com/3": "",
+ "https://example.com/4": "",
+ },
+ }
+ links := make([]Link, 4)
+ for i := range 4 {
+ links[i], _ = NewLink(fmt.Sprintf("https://example.com/%d", i+1))
+ }
+
+ start := time.Now()
+ NewCrawler(fetcher, NullStore{}, WithWorkers(4)).Run(t.Context(), links)
+ elapsed := time.Since(start)
+
+ // robots.txt is fetched once (sequential, fetchDelay), then 4 pages fetch in parallel (~fetchDelay).
+ // sequential would be robots + 4*pages = 5*fetchDelay.
+ if elapsed > 3*fetchDelay {
+ t.Errorf("expected concurrent fetching, took %v (want < %v)", elapsed, 3*fetchDelay)
+ }
+}
+
+func TestCrawlerOnlyFetchesHTML(t *testing.T) {
+ fetcher := &mockFetcher{
+ pages: map[string]string{
+ "https://example.com/page.html": "",
+ "https://example.com/file.pdf": "%PDF-1.4 binary content",
+ },
+ responseHeaders: map[string]http.Header{
+ "https://example.com/file.pdf": {"Content-Type": []string{"application/pdf"}},
+ },
+ }
+ store := &mockStore{}
+ htmlLink, _ := NewLink("https://example.com/page.html")
+ pdfLink, _ := NewLink("https://example.com/file.pdf")
+
+ NewCrawler(fetcher, store).Run(t.Context(), []Link{htmlLink, pdfLink})
+
+ store.assertSaved(t, "https://example.com/page.html")
+ store.assertNotSaved(t, "https://example.com/file.pdf")
+}
+
+func TestCrawlerTimesOutOnSlowResponses(t *testing.T) {
+ const fetchDelay = 500 * time.Millisecond
+ const timeout = 100 * time.Millisecond
+
+ fetcher := &mockFetcher{
+ fetchDelay: fetchDelay,
+ pages: map[string]string{"https://example.com/slow": ""},
+ }
+ link, _ := NewLink("https://example.com/slow")
+
+ start := time.Now()
+ NewCrawler(fetcher, NullStore{}, WithFetchTimeout(timeout)).Run(t.Context(), []Link{link})
+ elapsed := time.Since(start)
+
+ if elapsed > fetchDelay {
+ t.Errorf("fetch was not cancelled by timeout: took %v, timeout was %v", elapsed, timeout)
+ }
+}
+
+func TestCrawlerHasSpecifiedUserAgent(t *testing.T) {
+ const ua = "crowlr-test/1.0"
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/": "",
+ }}
+ link, _ := NewLink("https://example.com/")
+
+ NewCrawler(fetcher, NullStore{}, WithUserAgent(ua)).Run(t.Context(), []Link{link})
+
+ fetcher.mu.Lock()
+ got := fetcher.lastHeaders.Get("User-Agent")
+ fetcher.mu.Unlock()
+
+ if got != ua {
+ t.Errorf("expected User-Agent %q, got %q", ua, got)
+ }
+}
+
+func TestVisitReturnsTitle(t *testing.T) {
+ title := "Example Title"
+ link, _ := NewLink("https://example.com/")
+ fetcher := &mockFetcher{
+ pages: map[string]string{
+ link.Normalized: htmlBody(title, ""),
+ },
+ }
+
+ c := NewCrawler(fetcher, NullStore{})
+ result := c.visit(t.Context(), link)
+ if result.Title != title {
+ t.Errorf("wanted result title to be %s, but got %s", title, result.Title)
+ }
+}
+
+func TestVisitReturnsDuration(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/": "",
+ }}
+ c := NewCrawler(fetcher, NullStore{})
+ link, _ := NewLink("https://example.com/")
+
+ result := c.visit(t.Context(), link)
+
+ if result.Duration <= 0 {
+ t.Errorf("expected positive Duration, got %v", result.Duration)
+ }
+}
+
+func TestVisitParsesLastModifiedHeader(t *testing.T) {
+ mod := time.Date(2024, 1, 15, 12, 0, 0, 0, time.UTC)
+ fetcher := &mockFetcher{
+ pages: map[string]string{"https://example.com/": ""},
+ responseHeaders: map[string]http.Header{
+ "https://example.com/": {"Last-Modified": []string{mod.Format(http.TimeFormat)}},
+ },
+ }
+ c := NewCrawler(fetcher, NullStore{})
+ link, _ := NewLink("https://example.com/")
+
+ result := c.visit(t.Context(), link)
+
+ if result.LastModified == nil {
+ t.Fatal("expected LastModified to be set, got nil")
+ }
+ if !result.LastModified.Equal(mod) {
+ t.Errorf("expected LastModified=%v, got %v", mod, *result.LastModified)
+ }
+}
+
+func TestVisitReturnsNilLastModifiedWhenHeaderAbsent(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/": "",
+ }}
+ c := NewCrawler(fetcher, NullStore{})
+ link, _ := NewLink("https://example.com/")
+
+ result := c.visit(t.Context(), link)
+
+ if result.LastModified != nil {
+ t.Errorf("expected LastModified to be nil, got %v", result.LastModified)
+ }
+}
+
+func TestCrawlerPopulatesStatsAfterRun(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/foo": "",
+ "https://example.com/bar": "",
+ }}
+ link1, _ := NewLink("https://example.com/foo")
+ link2, _ := NewLink("https://example.com/bar")
+ c := NewCrawler(fetcher, NullStore{})
+
+ c.Run(t.Context(), []Link{link1, link2})
+
+ c.stats.mu.Lock()
+ defer c.stats.mu.Unlock()
+ if c.stats.fetchDurCount != 2 {
+ t.Errorf("expected 2 pages visited in stats, got %d", c.stats.fetchDurCount)
+ }
+ if c.stats.statusCodes[200] != 2 {
+ t.Errorf("expected 2 status-200 in stats, got %d", c.stats.statusCodes[200])
+ }
+ if len(c.stats.uniqueDomains) != 1 {
+ t.Errorf("expected 1 unique domain, got %d", len(c.stats.uniqueDomains))
+ }
+}
+
+func TestCrawlerRecordsErrorsInStats(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/ok": "",
+ "https://example.com/bad": "", // will time out
+ }}
+ fetcher.errorURLs = map[string]bool{"https://example.com/bad": true}
+ link1, _ := NewLink("https://example.com/ok")
+ link2, _ := NewLink("https://example.com/bad")
+ c := NewCrawler(fetcher, NullStore{})
+
+ c.Run(t.Context(), []Link{link1, link2})
+
+ c.stats.mu.Lock()
+ defer c.stats.mu.Unlock()
+ if c.stats.errors != 1 {
+ t.Errorf("expected 1 error in stats, got %d", c.stats.errors)
+ }
+}
+
+func TestCrawlerStoresResults(t *testing.T) {
+ fetcher := &mockFetcher{pages: map[string]string{
+ "https://example.com/foo": htmlBody("https://example.com/bar"),
+ "https://example.com/bar": "",
+ }}
+ store := &mockStore{}
+ link1, _ := NewLink("https://example.com/foo")
+ link2, _ := NewLink("https://example.com/bar")
+
+ NewCrawler(fetcher, store).Run(t.Context(), []Link{link1, link2})
+
+ store.assertSaved(t, "https://example.com/foo")
+ store.assertSaved(t, "https://example.com/bar")
+}
+
+type mockFetcher struct {
+ pages map[string]string
+ responseHeaders map[string]http.Header
+ errorURLs map[string]bool
+ fetchDelay time.Duration
+ visited map[string]time.Time
+ hits []string
+ lastHeaders http.Header
+ mu sync.Mutex
+}
+
+func (m *mockFetcher) Fetch(req *http.Request) (*http.Response, error) {
+ url := req.URL.String()
+ m.mu.Lock()
+ _, exists := m.pages[url]
+ isError := m.errorURLs[url]
+ m.mu.Unlock()
+
+ if !exists {
+ return nil, fmt.Errorf("no such page: %s", url)
+ }
+ if isError {
+ return nil, fmt.Errorf("simulated fetch error: %s", url)
+ }
+
+ if m.fetchDelay > 0 {
+ select {
+ case <-time.After(m.fetchDelay):
+ case <-req.Context().Done():
+ return nil, req.Context().Err()
+ }
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.visited == nil {
+ m.visited = make(map[string]time.Time)
+ }
+ m.visited[url] = time.Now()
+ m.hits = append(m.hits, url)
+ m.lastHeaders = req.Header
+ respHeader := http.Header{}
+ if m.responseHeaders != nil {
+ if h, ok := m.responseHeaders[url]; ok {
+ respHeader = h
+ }
+ }
+ return &http.Response{
+ StatusCode: 200,
+ Header: respHeader,
+ Body: io.NopCloser(strings.NewReader(m.pages[url])),
+ }, nil
+}
+
+func (m *mockFetcher) assertVisited(t *testing.T, url string) {
+ t.Helper()
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if _, ok := m.visited[url]; !ok {
+ t.Errorf("expected %s to be visited, but it wasn't", url)
+ }
+}
+
+func (m *mockFetcher) assertNotVisited(t *testing.T, url string) {
+ t.Helper()
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if ts, ok := m.visited[url]; ok {
+ t.Errorf("expected %s to not be visited, but it was at %s", url, ts)
+ }
+}
+
+func (m *mockFetcher) assertTotalVisits(t *testing.T, want int) {
+ t.Helper()
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if got := len(m.visited); got != want {
+ t.Errorf("expected %d visits, got %d", want, got)
+ }
+}
+
+func (m *mockFetcher) assertHitCount(t *testing.T, url string, want int) {
+ t.Helper()
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ got := 0
+ for _, h := range m.hits {
+ if h == url {
+ got++
+ }
+ }
+ if got != want {
+ t.Errorf("%s: expected %d hits, got %d", url, want, got)
+ }
+}
+
+func (m *mockFetcher) assertVisitGap(t *testing.T, url1, url2 string, min time.Duration) {
+ t.Helper()
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ gap := m.visited[url2].Sub(m.visited[url1])
+ if gap < 0 {
+ gap = -gap
+ }
+ if gap < min {
+ t.Errorf("gap between %s and %s was %v, want >= %v", url1, url2, gap, min)
+ }
+}
+
+func (m *mockFetcher) assertVisitedBefore(t *testing.T, url1, url2 string) {
+ t.Helper()
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ t1, ok1 := m.visited[url1]
+ t2, ok2 := m.visited[url2]
+ if !ok1 || !ok2 {
+ t.Fatalf("one of %s or %s was not visited", url1, url2)
+ }
+ if !t1.Before(t2) {
+ t.Errorf("expected %s to be visited before %s", url1, url2)
+ }
+}
+
+// mockStore
+
+type mockStore struct {
+ mu sync.Mutex
+ saved []Page
+}
+
+func (m *mockStore) SavePage(_ context.Context, p Page) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.saved = append(m.saved, p)
+ return nil
+}
+
+func (m *mockStore) assertSaved(t *testing.T, url string) {
+ t.Helper()
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ for _, p := range m.saved {
+ if p.URL == url {
+ return
+ }
+ }
+ t.Errorf("expected page %s to be saved, but it wasn't", url)
+}
+
+func (m *mockStore) assertNotSaved(t *testing.T, url string) {
+ t.Helper()
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ for _, p := range m.saved {
+ if p.URL == url {
+ t.Errorf("expected page %s to not be saved, but it was", url)
+ return
+ }
+ }
+}
+
+// helpers
+
+func htmlBody(title string, links ...string) string {
+ var sb strings.Builder
+ sb.WriteString("")
+ sb.WriteString("" + title + "")
+ for _, l := range links {
+ sb.WriteString(`link`)
+ }
+ sb.WriteString("")
+ return sb.String()
+}
+
+func robotsTxt(userAgent, disallow, allow, sitemap string) string {
+ return fmt.Sprintf("User-agent: %s\nDisallow: %s\nAllow: %s\nSitemap: %s", userAgent, disallow, allow, sitemap)
+}
+
+var sampleXML = `
+
+
+ http://www.example.net/?id=who
+ 2009-09-22
+
+
+ http://www.example.net/?id=what
+ 2009-09-22
+
+
+ http://www.example.net/?id=how
+ 2009-09-22
+
+
+`
diff --git a/pkg/crawler/frontier.go b/pkg/crawler/frontier.go
new file mode 100644
index 0000000..9025c89
--- /dev/null
+++ b/pkg/crawler/frontier.go
@@ -0,0 +1,71 @@
+package crawler
+
+import "sync"
+
+type Frontier struct {
+ queue []Link
+ seen map[string]struct{}
+ mu sync.Mutex
+}
+
+func NewFrontier() *Frontier {
+ return &Frontier{
+ queue: []Link{},
+ seen: map[string]struct{}{},
+ }
+}
+
+func (f *Frontier) Push(link Link) bool {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if _, ok := f.seen[link.Normalized]; ok {
+ return false
+ }
+ f.queue = append(f.queue, link)
+ f.seen[link.Normalized] = struct{}{}
+ return true
+}
+
+func (f *Frontier) Pop() *Link {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if len(f.queue) < 1 {
+ return nil
+ }
+ first := f.queue[0]
+ f.queue = f.queue[1:]
+
+ return &first
+}
+
+func (f *Frontier) Len() int {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return len(f.queue)
+}
+
+func (f *Frontier) Peek() *Link {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if len(f.queue) < 1 {
+ return nil
+ }
+ first := f.queue[0]
+ return &first
+}
+
+func (f *Frontier) PopElegible(isElegible func(Link) bool) *Link {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+
+ if len(f.queue) < 1 {
+ return nil
+ }
+ for i, link := range f.queue {
+ if isElegible(link) {
+ f.queue = append(f.queue[:i], f.queue[i+1:]...)
+ return &link
+ }
+ }
+ return nil
+}
diff --git a/pkg/crawler/frontier_test.go b/pkg/crawler/frontier_test.go
new file mode 100644
index 0000000..537f778
--- /dev/null
+++ b/pkg/crawler/frontier_test.go
@@ -0,0 +1,100 @@
+package crawler
+
+import (
+ "fmt"
+ "sync"
+ "testing"
+)
+
+var exampleLink, _ = NewLink("link-1")
+
+func TestFrontier(t *testing.T) {
+ t.Run("should push and pop", func(t *testing.T) {
+ f := NewFrontier()
+ want := exampleLink
+ f.Push(want)
+
+ got := f.Pop()
+
+ if *got != want {
+ t.Fatalf("got %v, but wanted %v", got, want)
+ return
+ }
+ })
+
+ t.Run("popped should be removed", func(t *testing.T) {
+ f := NewFrontier()
+ want := exampleLink
+ f.Push(want)
+ f.Pop()
+
+ if f.Len() != 0 {
+ t.Fatalf("got %v, but wanted %v", f.Len(), 0)
+ }
+ })
+
+ t.Run("peek shouldn't remove", func(t *testing.T) {
+ f := NewFrontier()
+ want := exampleLink
+ f.Push(want)
+ got := f.Peek()
+
+ if *got != want {
+ t.Fatalf("got %v, but wanted %v", got, want)
+ }
+
+ if f.Len() != 1 {
+ t.Fatalf("got %v, but wanted %v", f.Len(), 1)
+ }
+ })
+
+ t.Run("pop elegible should return if match", func(t *testing.T) {
+ f := NewFrontier()
+ f.Push(exampleLink)
+
+ isElegible := func(link Link) bool {
+ return link.Normalized == exampleLink.Normalized
+ }
+ got := f.PopElegible(isElegible)
+ if got == nil {
+ t.Fatal("expected elegible link, but got nil")
+ }
+
+ if *got != exampleLink {
+ t.Fatalf("got %v, but wanted %v", got, exampleLink)
+ }
+ })
+}
+
+func TestFrontierNoDuplicates(t *testing.T) {
+ f := NewFrontier()
+
+ links := []Link{
+ exampleLink, exampleLink,
+ }
+ for _, link := range links {
+ f.Push(link)
+ }
+
+ got := f.Len()
+ want := 1
+
+ if got != want {
+ t.Fatalf("got %v, but wanted %v", got, want)
+ }
+}
+
+func TestFrontierShouldSync(t *testing.T) {
+ f := NewFrontier()
+
+ wg := sync.WaitGroup{}
+ for i := range 10 {
+ wg.Add(1)
+ wg.Go(func() {
+ defer wg.Done()
+ newLink, _ := NewLink(fmt.Sprintf("link-%d", i))
+ f.Push(newLink)
+ })
+ }
+ wg.Wait()
+}
diff --git a/pkg/crawler/link.go b/pkg/crawler/link.go
new file mode 100644
index 0000000..08f5940
--- /dev/null
+++ b/pkg/crawler/link.go
@@ -0,0 +1,72 @@
+package crawler
+
+import (
+ "net/url"
+ "strings"
+
+ "github.com/PuerkitoBio/purell"
+)
+
+type Link struct {
+ Original string
+ Referrer string
+ Normalized string
+ Host string
+}
+
+type Option func(*Link)
+
+func WithReferrer(referrer string) Option {
+ return func(l *Link) {
+ l.Referrer = referrer
+ }
+}
+
+func NewLink(original string, opts ...Option) (Link, error) {
+ l := &Link{
+ Original: original,
+ }
+
+ for _, opt := range opts {
+ opt(l)
+ }
+
+ normalized, err := normalizeURL(original)
+ if err != nil {
+ return Link{}, err
+ }
+ l.Normalized = normalized
+
+ host, err := getHost(normalized)
+ if err != nil {
+ return Link{}, err
+ }
+ l.Host = host
+
+ return *l, nil
+}
+
+func normalizeURL(url string) (string, error) {
+ flags := purell.FlagLowercaseScheme |
+ purell.FlagLowercaseHost |
+ purell.FlagRemoveDefaultPort |
+ purell.FlagRemoveFragment |
+ purell.FlagDecodeUnnecessaryEscapes |
+ purell.FlagSortQuery |
+ purell.FlagRemoveDuplicateSlashes |
+ purell.FlagRemoveDotSegments
+
+ return purell.NormalizeURLString(url, flags)
+}
+
+func getHost(str string) (string, error) {
+ u, err := url.Parse(str)
+ if err != nil {
+ return "", err
+ }
+ return strings.ToLower(u.Hostname()), nil
+}
+
+func (l Link) String() string {
+ return l.Normalized
+}
diff --git a/pkg/crawler/link_test.go b/pkg/crawler/link_test.go
new file mode 100644
index 0000000..93f6a18
--- /dev/null
+++ b/pkg/crawler/link_test.go
@@ -0,0 +1,95 @@
+package crawler
+
+import "testing"
+
+func TestNewLink(t *testing.T) {
+ l, _ := NewLink("http://example.com/a", WithReferrer("http://example.com/"))
+
+ testCases := []struct {
+ desc string
+ got string
+ want string
+ }{
+ {
+ desc: "original",
+ got: l.Original,
+ want: "http://example.com/a",
+ },
+ {
+ desc: "referrer",
+ got: l.Referrer,
+ want: "http://example.com/",
+ },
+ {
+ desc: "normalized",
+ got: l.Normalized,
+ want: "http://example.com/a",
+ },
+ {
+ desc: "host",
+ got: l.Host,
+ want: "example.com",
+ },
+ }
+ for _, c := range testCases {
+ t.Run(c.desc, func(t *testing.T) {
+ if c.got != c.want {
+ t.Fatalf("got %v, but wanted %v", c.got, c.want)
+ }
+ })
+ }
+}
+
+func TestNewLinkShouldErrorOnBadURL(t *testing.T) {
+ _, err := NewLink("not an url :(")
+
+ if err == nil {
+ t.Fatalf("got no error, but wanted one")
+ }
+}
+
+func TestLinkShouldNormalizeLinks(t *testing.T) {
+ testCases := []struct {
+ desc string
+ input string
+ want string
+ }{
+ {
+ desc: "Remove default port",
+ input: "http://example.com:80/",
+ want: "http://example.com/",
+ },
+
+ {
+ desc: "Remove dot-segments",
+ input: "http://example.com/foo/./bar/baz/../qux",
+ want: "http://example.com/foo/bar/qux",
+ },
+
+ {
+ desc: "Decode unreserved characters",
+ input: "http://example.com/%7Efoo",
+ want: "http://example.com/~foo",
+ },
+
+ {
+ desc: "Scheme and host to lowercase",
+ input: "HTTP://User@Example.COM/Foo",
+ want: "http://User@example.com/Foo",
+ },
+ }
+
+ for _, c := range testCases {
+ t.Run(c.desc, func(t *testing.T) {
+ f, err := NewLink(c.input)
+ if err != nil {
+ t.Fatalf("got error %v", err)
+ }
+
+ got := f.Normalized
+ if got != c.want {
+ t.Fatalf("got %v but wanted %v", got, c.want)
+ }
+ })
+ }
+}
diff --git a/pkg/crawler/migrations/000001_init.down.sql b/pkg/crawler/migrations/000001_init.down.sql
new file mode 100644
index 0000000..97c364c
--- /dev/null
+++ b/pkg/crawler/migrations/000001_init.down.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS pages;
diff --git a/pkg/crawler/migrations/000001_init.up.sql b/pkg/crawler/migrations/000001_init.up.sql
new file mode 100644
index 0000000..71fd558
--- /dev/null
+++ b/pkg/crawler/migrations/000001_init.up.sql
@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS pages (
+ id SERIAL PRIMARY KEY,
+ url TEXT UNIQUE NOT NULL,
+ raw_url TEXT NOT NULL,
+ title TEXT NOT NULL,
+ referrer TEXT,
+ status_code INTEGER,
+ html TEXT,
+ outlinks JSONB,
+ fetched_at TIMESTAMP WITH TIME ZONE,
+ last_modified TIMESTAMP WITH TIME ZONE
+);
diff --git a/pkg/storage/migrations/000004_add_textsearch.up.sql b/pkg/crawler/migrations/000002_add_textsearch.down.sql
similarity index 53%
rename from pkg/storage/migrations/000004_add_textsearch.up.sql
rename to pkg/crawler/migrations/000002_add_textsearch.down.sql
index c38c240..11c1552 100644
--- a/pkg/storage/migrations/000004_add_textsearch.up.sql
+++ b/pkg/crawler/migrations/000002_add_textsearch.down.sql
@@ -1,9 +1,5 @@
-ALTER TABLE pages
- ADD COLUMN textsearch tsvector
- GENERATED ALWAYS AS (
- setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
+ALTER TABLE PAGES ADD COLUMN textsearch tsvector GENERATED ALWAYS AS (setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
setweight(to_tsvector('english', coalesce(url, '')), 'B') ||
- setweight(to_tsvector('english', coalesce(content, '')), 'C')
- ) STORED;
+ setweight(to_tsvector('english', coalesce(content, '')), 'C')) STORED;
CREATE INDEX textsearch_idx ON pages USING GIN (textsearch);
diff --git a/pkg/storage/migrations/000004_add_textsearch.down.sql b/pkg/crawler/migrations/000002_add_textsearch.up.sql
similarity index 100%
rename from pkg/storage/migrations/000004_add_textsearch.down.sql
rename to pkg/crawler/migrations/000002_add_textsearch.up.sql
index 7c7937a..2235851 100644
--- a/pkg/storage/migrations/000004_add_textsearch.down.sql
+++ b/pkg/crawler/migrations/000002_add_textsearch.up.sql
@@ -1,2 +1,2 @@
-DROP INDEX IF EXISTS textsearch_idx;
ALTER TABLE pages DROP COLUMN IF EXISTS textsearch;
+DROP INDEX IF EXISTS textsearch_idx;
diff --git a/pkg/storage/postgres.go b/pkg/crawler/postgres_store.go
similarity index 54%
rename from pkg/storage/postgres.go
rename to pkg/crawler/postgres_store.go
index b9cdcb6..6ff4bab 100644
--- a/pkg/storage/postgres.go
+++ b/pkg/crawler/postgres_store.go
@@ -1,4 +1,4 @@
-package storage
+package crawler
import (
"context"
@@ -7,56 +7,40 @@ import (
"log/slog"
)
-type PostgresStorage struct {
+type PostgresStore struct {
db *sql.DB
}
-func NewPostgresStorage(db *sql.DB) *PostgresStorage {
- return &PostgresStorage{db: db}
+type SearchResult struct {
+ URL string
+ Title string
+ Snippet string
+ Rank float64
}
-func (s *PostgresStorage) SavePage(ctx context.Context, p Page) error {
- jsonOutlinks, err := json.Marshal(p.Outlinks)
- if err != nil {
- return err
- }
-
- var id int
- err = s.db.QueryRowContext(ctx, `
- INSERT INTO pages (url, normalized_url, timestamp, title, content, html, status_code, outlinks, last_modified, referrer)
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
- RETURNING id`,
- p.RawURL, p.URL, p.Timestamp, p.Title, p.Content, p.HTML, p.StatusCode, jsonOutlinks, p.LastModified, p.Referrer,
- ).Scan(&id)
-
- if err != nil {
- return err
- }
-
- slog.Info("saved page", "id", id)
- return nil
+type SearchResponse struct {
+ Results []SearchResult
+ TotalCount int
}
-func (s *PostgresStorage) SaveSitemap(ctx context.Context, sm Sitemap) error {
- var id int
- err := s.db.QueryRowContext(ctx, `
- INSERT INTO sitemaps (url, last_checked, status_code, content)
- VALUES ($1, $2, $3, $4)
- ON CONFLICT (url) DO UPDATE
- SET last_checked = EXCLUDED.last_checked, status_code = EXCLUDED.status_code, content = EXCLUDED.content
- RETURNING id`,
- sm.URL, sm.LastChecked, sm.StatusCode, sm.Content,
- ).Scan(&id)
+func NewPostgresStore(db *sql.DB) *PostgresStore {
+ return &PostgresStore{db: db}
+}
+func (s *PostgresStore) SavePage(ctx context.Context, p Page) error {
+ outlinks, err := json.Marshal(p.Outlinks)
if err != nil {
return err
}
-
- slog.Info("saved sitemap", "id", id)
- return nil
+ _, err = s.db.ExecContext(ctx, `
+ INSERT INTO pages (url, raw_url, title, referrer, status_code, html, outlinks, fetched_at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
+ p.URL, p.RawURL, p.Title, p.Referrer, p.StatusCode, p.HTML, outlinks, p.FetchedAt,
+ )
+ return err
}
-func (s *PostgresStorage) Search(ctx context.Context, query string, limit int) (SearchResponse, error) {
+func (s *PostgresStore) Search(ctx context.Context, query string, limit int) (SearchResponse, error) {
slog.Debug("search query", "query", query, "limit", limit)
// Get total count first
@@ -112,7 +96,3 @@ func (s *PostgresStorage) Search(ctx context.Context, query string, limit int) (
TotalCount: totalCount,
}, nil
}
-
-func (s *PostgresStorage) Close() error {
- return s.db.Close()
-}
diff --git a/pkg/crawler/seeds.go b/pkg/crawler/seeds.go
new file mode 100644
index 0000000..4198d98
--- /dev/null
+++ b/pkg/crawler/seeds.go
@@ -0,0 +1,46 @@
+package crawler
+
+import (
+ "bufio"
+ "errors"
+ "log/slog"
+ "os"
+ "strings"
+)
+
+var ErrNoSeeds = errors.New("no seeds loaded")
+
+func LoadSeedsFile(path string) ([]Link, error) {
+ slog.Info("loading seeds", slog.String("path", path))
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ var links []Link
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ raw := strings.TrimSpace(scanner.Text())
+ if raw == "" || strings.HasPrefix(raw, "#") {
+ continue
+ }
+ link, err := NewLink(raw)
+ if err != nil {
+ slog.Warn("invalid seed url, skipping", slog.String("url", raw), slog.Any("err", err))
+ continue
+ }
+ links = append(links, link)
+ }
+
+ if err := scanner.Err(); err != nil {
+ return nil, err
+ }
+
+ if len(links) == 0 {
+ return nil, ErrNoSeeds
+ }
+
+ slog.Info("seeds loaded", slog.Int("count", len(links)))
+ return links, nil
+}
diff --git a/pkg/crawler/stats.go b/pkg/crawler/stats.go
new file mode 100644
index 0000000..db4ec35
--- /dev/null
+++ b/pkg/crawler/stats.go
@@ -0,0 +1,104 @@
+package crawler
+
+import (
+ "log/slog"
+ "sync"
+ "time"
+)
+
+type Stats struct {
+ mu sync.Mutex
+ startTime time.Time
+ statusCodes map[int]int
+ uniqueDomains map[string]struct{}
+ totalOutlinks int
+ errors int
+ fetchDurSum time.Duration
+ fetchDurCount int
+ lastModSum time.Duration
+ lastModCount int
+}
+
+func NewStats() *Stats {
+ return &Stats{
+ startTime: time.Now(),
+ statusCodes: make(map[int]int),
+ uniqueDomains: make(map[string]struct{}),
+ }
+}
+
+func (s *Stats) RecordVisit(host string, status int, duration time.Duration, lastModified *time.Time, outlinks int) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.statusCodes[status]++
+ s.uniqueDomains[host] = struct{}{}
+ s.totalOutlinks += outlinks
+ s.fetchDurSum += duration
+ s.fetchDurCount++
+ if lastModified != nil {
+ age := time.Since(*lastModified)
+ if age > 0 {
+ s.lastModSum += age
+ s.lastModCount++
+ }
+ }
+}
+
+func (s *Stats) RecordError() {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.errors++
+}
+
+func (s *Stats) Log(frontierRemaining int) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ elapsed := time.Since(s.startTime).Round(time.Millisecond)
+
+ var pagesPerSec float64
+ if secs := elapsed.Seconds(); secs > 0 {
+ pagesPerSec = float64(s.fetchDurCount) / secs
+ }
+
+ var avgFetchMs int64
+ if s.fetchDurCount > 0 {
+ avgFetchMs = s.fetchDurSum.Milliseconds() / int64(s.fetchDurCount)
+ }
+
+ var status2xx, status3xx, status4xx, status5xx int
+ for code, count := range s.statusCodes {
+ switch code / 100 {
+ case 2:
+ status2xx += count
+ case 3:
+ status3xx += count
+ case 4:
+ status4xx += count
+ case 5:
+ status5xx += count
+ }
+ }
+
+ attrs := []any{
+ slog.String("elapsed", elapsed.String()),
+ slog.Int("pages_visited", s.fetchDurCount),
+ slog.Float64("pages_per_sec", pagesPerSec),
+ slog.Int("errors", s.errors),
+ slog.Int("unique_domains", len(s.uniqueDomains)),
+ slog.Int("total_outlinks", s.totalOutlinks),
+ slog.Int("frontier_remaining", frontierRemaining),
+ slog.Int("status_2xx", status2xx),
+ slog.Int("status_3xx", status3xx),
+ slog.Int("status_4xx", status4xx),
+ slog.Int("status_5xx", status5xx),
+ slog.Int64("avg_fetch_ms", avgFetchMs),
+ }
+
+ if s.lastModCount > 0 {
+ avgAge := s.lastModSum / time.Duration(s.lastModCount)
+ attrs = append(attrs, slog.String("avg_content_age", avgAge.Round(time.Minute).String()))
+ }
+
+ slog.Info("crawl stats", attrs...)
+}
diff --git a/pkg/crawler/stats_test.go b/pkg/crawler/stats_test.go
new file mode 100644
index 0000000..10d1437
--- /dev/null
+++ b/pkg/crawler/stats_test.go
@@ -0,0 +1,151 @@
+package crawler
+
+import (
+ "context"
+ "log/slog"
+ "sync"
+ "testing"
+ "time"
+)
+
+func TestStatsRecordVisitAccumulatesStatusCodes(t *testing.T) {
+ s := NewStats()
+ s.RecordVisit("example.com", 200, time.Millisecond, nil, 5)
+ s.RecordVisit("example.com", 200, time.Millisecond, nil, 3)
+ s.RecordVisit("example.com", 404, time.Millisecond, nil, 0)
+
+ if s.statusCodes[200] != 2 {
+ t.Errorf("expected 2 200s, got %d", s.statusCodes[200])
+ }
+ if s.statusCodes[404] != 1 {
+ t.Errorf("expected 1 404, got %d", s.statusCodes[404])
+ }
+}
+
+func TestStatsRecordVisitAccumulatesDuration(t *testing.T) {
+ s := NewStats()
+ s.RecordVisit("example.com", 200, 100*time.Millisecond, nil, 0)
+ s.RecordVisit("example.com", 200, 200*time.Millisecond, nil, 0)
+
+ if s.fetchDurCount != 2 {
+ t.Errorf("expected fetchDurCount=2, got %d", s.fetchDurCount)
+ }
+ if s.fetchDurSum != 300*time.Millisecond {
+ t.Errorf("expected fetchDurSum=300ms, got %v", s.fetchDurSum)
+ }
+}
+
+func TestStatsRecordVisitDeduplicatesDomains(t *testing.T) {
+ s := NewStats()
+ s.RecordVisit("example.com", 200, time.Millisecond, nil, 0)
+ s.RecordVisit("example.com", 200, time.Millisecond, nil, 0)
+ s.RecordVisit("other.com", 200, time.Millisecond, nil, 0)
+
+ if got := len(s.uniqueDomains); got != 2 {
+ t.Errorf("expected 2 unique domains, got %d", got)
+ }
+}
+
+func TestStatsRecordVisitTracksLastModifiedWhenPresent(t *testing.T) {
+ s := NewStats()
+ mod := time.Now().Add(-24 * time.Hour)
+ s.RecordVisit("example.com", 200, time.Millisecond, &mod, 0)
+
+ if s.lastModCount != 1 {
+ t.Errorf("expected lastModCount=1, got %d", s.lastModCount)
+ }
+ if s.lastModSum <= 0 {
+ t.Errorf("expected positive lastModSum, got %v", s.lastModSum)
+ }
+}
+
+func TestStatsRecordVisitSkipsLastModifiedWhenAbsent(t *testing.T) {
+ s := NewStats()
+ s.RecordVisit("example.com", 200, time.Millisecond, nil, 0)
+
+ if s.lastModCount != 0 {
+ t.Errorf("expected lastModCount=0, got %d", s.lastModCount)
+ }
+}
+
+func TestStatsRecordErrorIncrementsErrors(t *testing.T) {
+ s := NewStats()
+ s.RecordError()
+ s.RecordError()
+
+ if s.errors != 2 {
+ t.Errorf("expected errors=2, got %d", s.errors)
+ }
+}
+
+func TestStatsLogOmitsContentAgeWhenNoLastModified(t *testing.T) {
+ s := NewStats()
+ s.RecordVisit("example.com", 200, time.Millisecond, nil, 0)
+
+ rec := captureStatsLog(t, s)
+
+ rec.Attrs(func(a slog.Attr) bool {
+ if a.Key == "avg_content_age" {
+ t.Error("expected avg_content_age to be omitted when no Last-Modified headers seen")
+ }
+ return true
+ })
+}
+
+func TestStatsLogIncludesContentAgeWhenLastModifiedPresent(t *testing.T) {
+ s := NewStats()
+ mod := time.Now().Add(-48 * time.Hour)
+ s.RecordVisit("example.com", 200, time.Millisecond, &mod, 0)
+
+ rec := captureStatsLog(t, s)
+
+ var found bool
+ rec.Attrs(func(a slog.Attr) bool {
+ if a.Key == "avg_content_age" {
+ found = true
+ }
+ return true
+ })
+ if !found {
+ t.Error("expected avg_content_age in 'crawl stats' log when Last-Modified headers seen")
+ }
+}
+
+// captureStatsLog calls s.Log and returns the captured "crawl stats" slog.Record.
+func captureStatsLog(t *testing.T, s *Stats) slog.Record {
+ t.Helper()
+ var mu sync.Mutex
+ var records []slog.Record
+ h := &captureHandler{capture: func(r slog.Record) {
+ mu.Lock()
+ records = append(records, r)
+ mu.Unlock()
+ }}
+ orig := slog.Default()
+ slog.SetDefault(slog.New(h))
+ defer slog.SetDefault(orig)
+
+ s.Log(0)
+
+ mu.Lock()
+ defer mu.Unlock()
+ for _, r := range records {
+ if r.Message == "crawl stats" {
+ return r
+ }
+ }
+ t.Fatal("expected 'crawl stats' log record but none was emitted")
+ return slog.Record{}
+}
+
+type captureHandler struct {
+ capture func(slog.Record)
+}
+
+func (h *captureHandler) Enabled(_ context.Context, _ slog.Level) bool { return true }
+func (h *captureHandler) Handle(_ context.Context, r slog.Record) error {
+ h.capture(r)
+ return nil
+}
+func (h *captureHandler) WithAttrs(_ []slog.Attr) slog.Handler { return h }
+func (h *captureHandler) WithGroup(_ string) slog.Handler { return h }
diff --git a/pkg/storage/migrations.go b/pkg/crawler/store_migrations.go
similarity index 55%
rename from pkg/storage/migrations.go
rename to pkg/crawler/store_migrations.go
index b2289de..24c243f 100644
--- a/pkg/storage/migrations.go
+++ b/pkg/crawler/store_migrations.go
@@ -1,42 +1,37 @@
-package storage
+package crawler
import (
"database/sql"
"embed"
"errors"
"fmt"
- "log/slog"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/postgres"
"github.com/golang-migrate/migrate/v4/source/iofs"
)
-//go:embed all:migrations/*.sql
+//go:embed migrations/*.sql
var migrationsFS embed.FS
func RunMigrations(db *sql.DB) error {
sourceDriver, err := iofs.New(migrationsFS, "migrations")
if err != nil {
- return fmt.Errorf("could not create source driver: %w", err)
+ return fmt.Errorf("create source driver: %w", err)
}
driver, err := postgres.WithInstance(db, &postgres.Config{})
if err != nil {
- return fmt.Errorf("could not create migration driver: %w", err)
+ return fmt.Errorf("create migration driver: %w", err)
}
- m, err := migrate.NewWithInstance(
- "iofs", sourceDriver,
- "postgres", driver)
+ m, err := migrate.NewWithInstance("iofs", sourceDriver, "postgres", driver)
if err != nil {
- return fmt.Errorf("could not create migrate instance: %w", err)
+ return fmt.Errorf("create migrate instance: %w", err)
}
if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
- return fmt.Errorf("could not run up migrations: %w", err)
+ return fmt.Errorf("run migrations: %w", err)
}
-
- slog.Info("migrations ran successfully")
return nil
}
diff --git a/pkg/crawler/testmain_test.go b/pkg/crawler/testmain_test.go
new file mode 100644
index 0000000..0d8b570
--- /dev/null
+++ b/pkg/crawler/testmain_test.go
@@ -0,0 +1,42 @@
+package crawler
+
+import (
+ "log/slog"
+ "os"
+ "strings"
+ "sync"
+ "testing"
+)
+
+var logSink testWriter
+
+func TestMain(m *testing.M) {
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logSink, nil)))
+ os.Exit(m.Run())
+}
+
+type testWriter struct {
+ mu sync.Mutex
+ t testing.TB
+}
+
+func (w *testWriter) Write(p []byte) (int, error) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.t != nil {
+ w.t.Log(strings.TrimRight(string(p), "\n"))
+ }
+ return len(p), nil
+}
+
+func setLogger(t testing.TB) {
+ t.Helper()
+ logSink.mu.Lock()
+ logSink.t = t
+ logSink.mu.Unlock()
+ t.Cleanup(func() {
+ logSink.mu.Lock()
+ logSink.t = nil
+ logSink.mu.Unlock()
+ })
+}
diff --git a/pkg/crawler/types.go b/pkg/crawler/types.go
deleted file mode 100644
index 0dfc895..0000000
--- a/pkg/crawler/types.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package crawler
-
-import "github.com/devraulu/crowlr/pkg/storage"
-
-type Outlink struct {
- Normalized string
- Original string
-}
-
-type CrawlResult struct {
- URL string
- Error error
- PageData *storage.Page
- Outlinks []Outlink
-}
diff --git a/pkg/crawler/worker.go b/pkg/crawler/worker.go
deleted file mode 100644
index e20bc4c..0000000
--- a/pkg/crawler/worker.go
+++ /dev/null
@@ -1,138 +0,0 @@
-package crawler
-
-import (
- "bytes"
- "context"
- "io"
- "log/slog"
- "net/http"
- "strings"
- "time"
-
- frontier "github.com/devraulu/crowlr/pkg"
- "github.com/devraulu/crowlr/pkg/process"
- "github.com/devraulu/crowlr/pkg/storage"
-)
-
-func (c *Crawler) worker(ctx context.Context, id int, jobs <-chan frontier.Candidate, results chan<- CrawlResult) {
- slog.Info("worker started", "id", id)
- for {
- select {
- case <-ctx.Done():
- return
- case job, ok := <-jobs:
- if !ok {
- return
- }
- slog.Debug("worker received job", slog.Int("id", id), slog.Any("job", job))
- results <- c.fetchAndProcess(job)
- }
- }
-}
-
-func (c *Crawler) fetchAndProcess(job frontier.Candidate) CrawlResult {
- res := CrawlResult{
- URL: job.Normalized,
- }
-
- req, err := http.NewRequest("GET", job.Normalized, nil)
- if err != nil {
- res.Error = err
- return res
- }
-
- req.Header.Add("Accept", "text/html")
- req.Header.Add("User-Agent", c.cfg.Crawler.UserAgent)
-
- client := &http.Client{
- Timeout: time.Second * 10,
- }
-
- resp, err := client.Do(req)
- if err != nil {
- res.Error = err
- return res
- }
- defer resp.Body.Close()
-
- if !validateHTMLContentTypeHeader(resp, "text/html") {
- return res
- }
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- res.Error = err
- return res
- }
-
- if !validateBodyContentType(body, "text/html") {
- return res
- }
-
- extracted, err := process.ExtractLinks(bytes.NewReader(body), job.Normalized)
- if err != nil {
- res.Error = err
- return res
- }
-
- var outlinks []Outlink
-
- for _, absolute := range extracted.Outlinks {
- normalized, err := process.Normalize(absolute)
- if err == nil {
- outlinks = append(outlinks, Outlink{
- Normalized: normalized,
- Original: absolute,
- })
- }
- }
-
- res.Outlinks = outlinks
-
- textContent, err := process.ExtractText(bytes.NewReader(body))
- if err != nil {
- slog.Warn("failed to extract text",
- slog.String("url", job.Normalized),
- slog.Any("err", err))
-
- textContent = ""
- // continue even if text extraction fails
- }
-
- var lastMod *time.Time
- if lm := resp.Header.Get("Last-Modified"); lm != "" {
- if t, err := http.ParseTime(lm); err == nil {
- lastMod = &t
- }
- }
-
- storageOutlinks := make([]string, len(outlinks))
- for i, o := range outlinks {
- storageOutlinks[i] = o.Normalized
- }
-
- res.PageData = &storage.Page{
- Referrer: job.Referrer,
- RawURL: job.Original,
- URL: job.Normalized,
- Timestamp: time.Now(),
- LastModified: lastMod,
- Title: extracted.Title,
- Content: textContent,
- HTML: string(body),
- StatusCode: resp.StatusCode,
- Outlinks: storageOutlinks,
- }
-
- return res
-}
-
-func validateHTMLContentTypeHeader(resp *http.Response, contentType string) bool {
- header := resp.Header.Get("Content-Type")
-
- return strings.Contains(strings.ToLower(header), contentType)
-}
-
-func validateBodyContentType(body []byte, contentType string) bool {
- return strings.HasPrefix(http.DetectContentType(body), contentType)
-}
diff --git a/pkg/frontier.go b/pkg/frontier.go
deleted file mode 100644
index 842a893..0000000
--- a/pkg/frontier.go
+++ /dev/null
@@ -1,132 +0,0 @@
-package frontier
-
-import (
- "log/slog"
- netUrl "net/url"
- "strings"
- "sync"
- "time"
-)
-
-type Candidate struct {
- Original string
- Normalized string
- Referrer string
-}
-
-type HostQueue struct {
- Host string
- URLs []string
- NextVisit time.Time
-}
-
-type SeenRecord struct {
- OriginalURL string
- Referrer string
-}
-
-type Frontier struct {
- mu sync.Mutex
- queues map[string]*HostQueue
- seen map[string]SeenRecord
-}
-
-func NewFrontier() *Frontier {
- return &Frontier{
- queues: make(map[string]*HostQueue),
- seen: make(map[string]SeenRecord),
- }
-}
-
-func (f *Frontier) Push(normalizedURL, originalURL, referrer string) {
- f.mu.Lock()
- defer f.mu.Unlock()
-
- if _, ok := f.seen[normalizedURL]; ok {
- slog.Info("frontier duplicate, skipping", slog.String("url", normalizedURL), slog.String("original_url", originalURL))
- return
- }
-
- host, err := getHost(normalizedURL)
- if err != nil {
- slog.Error("frontier bad url", slog.String("url", normalizedURL), slog.Any("err", err))
- return
- }
-
- f.seen[normalizedURL] = SeenRecord{
- OriginalURL: originalURL,
- Referrer: referrer,
- }
-
- hq, ok := f.queues[host]
- if !ok {
- hq = &HostQueue{
- Host: host,
- }
- f.queues[host] = hq
- }
-
- hq.URLs = append(hq.URLs, normalizedURL)
- slog.Debug("frontier push", slog.String("host", host), slog.String("url", normalizedURL), slog.Int("queue_len", len(hq.URLs)))
-}
-
-func (f *Frontier) Pop(defaultDelay time.Duration) (*Candidate, time.Duration) {
- f.mu.Lock()
- defer f.mu.Unlock()
-
- if len(f.queues) == 0 {
- return nil, 0
- }
-
- now := time.Now()
- var minWait time.Duration = -1
-
- for host, hq := range f.queues {
- if len(hq.URLs) == 0 {
- delete(f.queues, host)
- continue
- }
-
- if now.After(hq.NextVisit) {
- // found a ready host!
- url := hq.URLs[0]
- hq.URLs = hq.URLs[1:]
- hq.NextVisit = now.Add(defaultDelay)
-
- slog.Info("next candidate", slog.String("host", host), slog.String("url", url))
- seen := f.seen[url]
- referrer := seen.Referrer
-
- return &Candidate{
- Original: seen.OriginalURL,
- Normalized: url,
- Referrer: referrer,
- }, 0
- }
-
- wait := hq.NextVisit.Sub(now)
- if minWait == -1 || wait < minWait {
- minWait = wait
- }
- }
-
- return nil, minWait
-}
-
-func (f *Frontier) Len() int {
- f.mu.Lock()
- defer f.mu.Unlock()
- count := 0
- for _, hq := range f.queues {
- count += len(hq.URLs)
- }
- return count
-}
-
-func getHost(str string) (string, error) {
- u, err := netUrl.Parse(str)
- if err != nil {
- return "", err
- }
- return strings.ToLower(u.Hostname()), nil
-}
diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go
index 243cc04..16dc529 100644
--- a/pkg/logger/logger.go
+++ b/pkg/logger/logger.go
@@ -3,6 +3,7 @@ package logger
import (
"log/slog"
"os"
+ "strings"
"github.com/devraulu/crowlr/pkg/config"
)
@@ -15,7 +16,7 @@ func InitLogger(cfg *config.Config) {
var handler slog.Handler
opts := &slog.HandlerOptions{
- Level: slog.LevelInfo,
+ Level: parseLevel(cfg.Logging.Level),
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
if a.Key == slog.LevelKey {
// Only use bunyan levels if JSON
@@ -42,6 +43,19 @@ func InitLogger(cfg *config.Config) {
slog.SetDefault(logger)
}
+func parseLevel(s string) slog.Level {
+ switch strings.ToLower(s) {
+ case "debug":
+ return slog.LevelDebug
+ case "warn":
+ return slog.LevelWarn
+ case "error":
+ return slog.LevelError
+ default:
+ return slog.LevelInfo
+ }
+}
+
func bunyanLevel(level slog.Level) int {
switch {
case level >= slog.LevelError:
diff --git a/pkg/process/extract.go b/pkg/process/extract.go
deleted file mode 100644
index 4690270..0000000
--- a/pkg/process/extract.go
+++ /dev/null
@@ -1,111 +0,0 @@
-package process
-
-import (
- "io"
- "net/url"
- "strings"
-
- "golang.org/x/net/html"
-)
-
-type extractionResult struct {
- Outlinks []string
- Title string
-}
-
-func ExtractLinks(body io.Reader, baseURL string) (*extractionResult, error) {
- doc, err := html.Parse(body)
- if err != nil {
- return nil, err
- }
-
- base, err := url.Parse(baseURL)
- if err != nil {
- return nil, err
- }
-
- if newBaseStr := findBase(doc); newBaseStr != "" {
- if newBase, err := base.Parse(newBaseStr); err == nil {
- base = newBase
- }
- }
-
- links := extractAndResolve(doc, base)
- title := extractTitle(doc)
-
- return &extractionResult{
- Outlinks: links,
- Title: title,
- }, nil
-}
-
-func findBase(n *html.Node) string {
- if n.Type == html.ElementNode && n.Data == "base" {
- for _, attr := range n.Attr {
- if attr.Key == "href" {
- return attr.Val
- }
- }
- }
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- if res := findBase(c); res != "" {
- return res
- }
- }
- return ""
-}
-
-func extractAndResolve(n *html.Node, base *url.URL) []string {
- var links []string
- if n.Type == html.ElementNode && n.Data == "a" {
- for _, attr := range n.Attr {
- if attr.Key == "href" {
- val := strings.TrimSpace(attr.Val)
- if val == "" {
- continue
- }
-
- resolved := resolve(val, base)
- if resolved != "" {
- links = append(links, resolved)
- }
- break
- }
- }
- }
-
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- links = append(links, extractAndResolve(c, base)...)
- }
- return links
-}
-
-func resolve(ref string, base *url.URL) string {
- u, err := url.Parse(ref)
- if err != nil {
- return ""
- }
-
- abs := base.ResolveReference(u)
-
- scheme := strings.ToLower(abs.Scheme)
- if scheme != "http" && scheme != "https" {
- return ""
- }
-
- return abs.String()
-}
-
-func extractTitle(n *html.Node) string {
- if n.Type == html.ElementNode && n.Data == "title" {
- if n.FirstChild != nil {
- return n.FirstChild.Data
- }
- }
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- if t := extractTitle(c); t != "" {
- return t
- }
- }
- return ""
-}
diff --git a/pkg/process/normalize.go b/pkg/process/normalize.go
deleted file mode 100644
index fa64272..0000000
--- a/pkg/process/normalize.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package process
-
-import "github.com/PuerkitoBio/purell"
-
-func Normalize(url string) (string, error) {
- flags := purell.FlagLowercaseScheme |
- purell.FlagLowercaseHost |
- purell.FlagRemoveDefaultPort |
- purell.FlagRemoveFragment |
- purell.FlagDecodeUnnecessaryEscapes |
- purell.FlagSortQuery |
- purell.FlagRemoveDuplicateSlashes |
- purell.FlagRemoveDotSegments
-
- return purell.NormalizeURLString(url, flags)
-}
diff --git a/pkg/process/robots.go b/pkg/process/robots.go
deleted file mode 100644
index 4b0b86c..0000000
--- a/pkg/process/robots.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package process
-
-import (
- "bytes"
- "io"
- "log/slog"
- "net/http"
-
- "github.com/benjaminestes/robots"
-)
-
-func CheckRobots(url string, cache map[string]*robots.Robots) *robots.Robots {
- defer func() {
- if r := recover(); r != nil {
- slog.Warn("panic in robots.txt parsing, assuming allowed", slog.String("url", url), slog.Any("panic", r))
- }
- }()
-
- robotsURL, err := robots.Locate(url)
- if err != nil {
- return nil
- }
-
- if r, ok := cache[robotsURL]; ok {
- return r
- }
-
- r, err := getRobots(robotsURL)
- if err != nil {
- slog.Warn("failed to fetch robots.txt", slog.String("url", robotsURL), slog.Any("err", err))
- cache[robotsURL] = nil
- return nil
- }
-
- cache[robotsURL] = r
- return r
-}
-
-func getRobots(url string) (*robots.Robots, error) {
- resp, err := http.Get(url)
- if err != nil {
- return nil, err
- }
-
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, err
- }
-
- slog.Debug("robots.txt response",
- slog.String("url", url),
- slog.Int("status_code", resp.StatusCode),
- slog.Int("body_length", len(body)),
- slog.String("body_preview", string(body[:min(len(body), 200)])),
- )
-
- return robots.From(resp.StatusCode, bytes.NewReader(body))
-}
diff --git a/pkg/process/text.go b/pkg/process/text.go
deleted file mode 100644
index abbdada..0000000
--- a/pkg/process/text.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package process
-
-import (
- "io"
- "strings"
-
- "golang.org/x/net/html"
-)
-
-func ExtractText(body io.Reader) (string, error) {
- doc, err := html.Parse(body)
- if err != nil {
- return "", err
- }
-
- var sb strings.Builder
- extractTextNodes(doc, &sb)
-
- text := sb.String()
- text = strings.Join(strings.Fields(text), " ")
- return strings.TrimSpace(text), nil
-}
-
-func extractTextNodes(n *html.Node, sb *strings.Builder) {
- if n.Type == html.ElementNode {
- switch n.Data {
- case "script", "style", "noscript", "iframe", "svg":
- return
- }
- }
-
- if n.Type == html.TextNode {
- sb.WriteString(n.Data)
- sb.WriteString(" ")
- }
-
- for c := n.FirstChild; c != nil; c = c.NextSibling {
- extractTextNodes(c, sb)
- }
-}
diff --git a/pkg/seeds.go b/pkg/seeds.go
deleted file mode 100644
index 93e47d7..0000000
--- a/pkg/seeds.go
+++ /dev/null
@@ -1,45 +0,0 @@
-package frontier
-
-import (
- "bufio"
- "errors"
- "log/slog"
- "os"
-
- "github.com/devraulu/crowlr/pkg/process"
-)
-
-var (
- ErrNoSeeds = errors.New("no seeds loaded")
-)
-
-func LoadSeeds(path string, f *Frontier) error {
- slog.Info("loading seeds", "path", path)
- file, err := os.Open(path)
- if err != nil {
- return err
- }
- defer file.Close()
-
- scanner := bufio.NewScanner(file)
- for scanner.Scan() {
- url := scanner.Text()
- normalized, err := process.Normalize(url)
- if err != nil {
- slog.Error("couldn't normalize seed", slog.String("seed", url), slog.Any("err", err))
- continue
- }
- f.Push(normalized, url, "")
- }
-
- if err := scanner.Err(); err != nil {
- return err
- }
-
- if f.Len() == 0 {
- return ErrNoSeeds
- }
-
- slog.Info("loaded seeds", "count", f.Len())
- return nil
-}
diff --git a/pkg/storage/migrations/000001_init_schema.down.sql b/pkg/storage/migrations/000001_init_schema.down.sql
deleted file mode 100644
index fda7bb9..0000000
--- a/pkg/storage/migrations/000001_init_schema.down.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS sitemaps;
-DROP TABLE IF EXISTS pages;
diff --git a/pkg/storage/migrations/000001_init_schema.up.sql b/pkg/storage/migrations/000001_init_schema.up.sql
deleted file mode 100644
index 5934813..0000000
--- a/pkg/storage/migrations/000001_init_schema.up.sql
+++ /dev/null
@@ -1,19 +0,0 @@
-CREATE TABLE IF NOT EXISTS pages (
- id SERIAL PRIMARY KEY,
- url TEXT NOT NULL,
- normalized_url TEXT NOT NULL,
- timestamp TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
- title TEXT,
- content TEXT,
- html TEXT,
- status_code INTEGER,
- outlinks JSONB
-);
-
-CREATE TABLE IF NOT EXISTS sitemaps (
- id SERIAL PRIMARY KEY,
- url TEXT NOT NULL UNIQUE,
- last_checked TIMESTAMPTZ,
- status_code INTEGER,
- content TEXT
-);
diff --git a/pkg/storage/migrations/000002_add_last_modified.down.sql b/pkg/storage/migrations/000002_add_last_modified.down.sql
deleted file mode 100644
index 142d356..0000000
--- a/pkg/storage/migrations/000002_add_last_modified.down.sql
+++ /dev/null
@@ -1 +0,0 @@
-ALTER TABLE pages DROP COLUMN last_modified;
diff --git a/pkg/storage/migrations/000002_add_last_modified.up.sql b/pkg/storage/migrations/000002_add_last_modified.up.sql
deleted file mode 100644
index 87b2183..0000000
--- a/pkg/storage/migrations/000002_add_last_modified.up.sql
+++ /dev/null
@@ -1 +0,0 @@
-ALTER TABLE pages ADD COLUMN last_modified TIMESTAMP WITH TIME ZONE;
diff --git a/pkg/storage/migrations/000003_add_referrer.down.sql b/pkg/storage/migrations/000003_add_referrer.down.sql
deleted file mode 100644
index 061b8e6..0000000
--- a/pkg/storage/migrations/000003_add_referrer.down.sql
+++ /dev/null
@@ -1 +0,0 @@
-ALTER TABLE pages DROP COLUMN referrer;
diff --git a/pkg/storage/migrations/000003_add_referrer.up.sql b/pkg/storage/migrations/000003_add_referrer.up.sql
deleted file mode 100644
index 7819e91..0000000
--- a/pkg/storage/migrations/000003_add_referrer.up.sql
+++ /dev/null
@@ -1 +0,0 @@
-ALTER TABLE pages ADD COLUMN referrer TEXT;
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
deleted file mode 100644
index a9932c7..0000000
--- a/pkg/storage/storage.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package storage
-
-import (
- "context"
- "time"
-)
-
-type Page struct {
- ID string
- Referrer string
- RawURL string
- URL string
- Timestamp time.Time
- LastModified *time.Time
- Title string
- Content string
- HTML string
- StatusCode int
- Outlinks []string
-}
-
-type Sitemap struct {
- URL string
- LastChecked time.Time
- StatusCode int
- Content string
-}
-
-type SearchResult struct {
- URL string
- Title string
- Snippet string
- Rank float64
-}
-
-type SearchResponse struct {
- Results []SearchResult
- TotalCount int
-}
-
-type Storage interface {
- SavePage(ctx context.Context, p Page) error
- SaveSitemap(ctx context.Context, s Sitemap) error
- Search(ctx context.Context, query string, limit int) (SearchResponse, error)
- Close() error
-}