introduce v2

Makefile

@@ -1,21 +1,18 @@
-.PHONY: dev build run clean web build-web
+.PHONY: dev build run clean web
 
 dev:
 	@$(MAKE) -s build
-	@./tmp/crawler
+	@./tmp/crawler crawl
 
 build:
-	go build -o ./tmp/crawler ./cmd/crawler/main.go
-
-build-web:
-	go build -o ./tmp/web ./cmd/web/main.go
+	go build -o ./tmp/crawler ./cmd/crawler/
 
 run:
-	./tmp/crawler
+	./tmp/crawler crawl
 
 web:
-	@$(MAKE) -s build-web
-	@./tmp/web
+	@$(MAKE) -s build
+	@./tmp/crawler web
 
 clean:
 	rm -rf ./tmp

README.md

@@ -22,33 +22,33 @@ crowlr works in a similar way: it visits multiple pages at once using a pool of
 
 ```mermaid
 flowchart TB
-    Start([Seeds]):::start -->|enqueue| Queue
+    Binary([crowlr]):::binary
 
-    Queue[Frontier<br/><br/>- BFS queue with per-host queues<br/>- thread-safe with mutex<br/>- deduplicates via seen set<br/>- crawl limit terminates program]
+    Binary -->|crawl| Seeds
+    Binary -->|web| Search
 
-    Queue -->|dequeue| Fetch
+    Seeds([seed URLs]) -->|push| Frontier
 
-    Fetch[Fetch URLs<br/><br/>- concurrent worker pool<br/>- respects robots.txt<br/>- per-host politeness delay]
+    Frontier[Frontier<br/><br/>- per-host BFS queues<br/>- seen-URL dedup<br/>- polite pop with per-host delay]
 
-    Fetch --> Parse
+    Frontier -->|next eligible URL| Workers
 
-    Parse[Parse Page<br/><br/>- validate HTML content-type<br/>- extract links from a tags<br/>- extract text content and title<br/>- normalize URLs]
+    Workers[Worker Pool<br/><br/>- concurrent fetchers<br/>- robots.txt + sitemap support<br/>- configurable count and delay]
 
-    Parse -->|new URLs| Queue
-    Parse -->|store| DB
+    Workers -->|HTML| Extract
 
-    DB[(PostgreSQL<br/><br/>- url, title, content, html<br/>- status code, outlinks<br/>- tsvector full-text index<br/>- weighted: title > url > content)]
+    Extract[Extract<br/><br/>- title from title tag<br/>- outlinks from anchor hrefs<br/>- resolve and normalize URLs]
 
-    DB -->|full-text search| Search
+    Extract -->|new URLs| Frontier
+    Extract -->|page| DB
 
-    Search[Search UI<br/><br/>- HTMX<br/>- ts_rank_cd ranking<br/>- highlighted snippets]
+    DB[(PostgreSQL<br/><br/>- url, title, html, outlinks<br/>- tsvector full-text index)]
 
-    Seen[Seen Set<br/><br/>- normalized URL dedup<br/>- thread-safe with mutex]
+    DB -->|full-text search| Search
 
-    Parse -.->|check| Seen
-    Queue -.->|check| Seen
+    Search[Search UI<br/><br/>- HTMX<br/>- ts_rank_cd ranking<br/>- highlighted snippets]
 
-    classDef start stroke:#666,stroke-width:2px
+    classDef binary stroke:#666,stroke-width:2px
 ```
 
 ## Features
@@ -86,6 +86,10 @@ make dev
 # Run search UI (separate terminal)
 make web
 # Open http://localhost:8080
+
+# Or use the binary directly
+./tmp/crawler crawl
+./tmp/crawler web --port 9000
 ```
 
 ## Configuration
@@ -99,19 +103,17 @@ See `config.example.toml` for all options.
 | `crawler.crawl_limit` | Max pages to crawl | `1000` |
 | `crawler.user_agent` | User-Agent header | - |
 | `politeness.delay` | Min delay between requests to same host | `1s` |
+| `politeness.fetch_timeout` | Max duration for an individual fetch | `10s` |
 | `logging.level` | Log level (debug, info, warn, error) | `info` |
 | `logging.format` | Log format (text, json) | `json` |
 
 ## Project Structure
 
 ```
 cmd/
-  crawler/    # crawler binary
-  web/        # search UI binary
+  crawler/    # single binary — `crawl` and `web` subcommands
 pkg/
-  crawler/    # coordinator, workers, stats
-  process/    # HTML parsing, text extraction, normalization, robots.txt
-  storage/    # PostgreSQL with migrations and full-text search
+  crawler/    # frontier, workers, postgres store, full-text search
   config/     # TOML configuration
   logger/     # structured logging (bunyan-compatible)
 ```

cmd/crawler/crawl.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/devraulu/crowlr/pkg/config"
+	"github.com/devraulu/crowlr/pkg/logger"
+	"github.com/devraulu/crowlr/pkg/crawler"
+	_ "github.com/lib/pq"
+	"github.com/spf13/cobra"
+)
+
+var crawlCmd = &cobra.Command{
+	Use:   "crawl",
+	Short: "Start crawling from seed URLs",
+	RunE:  runCrawl,
+}
+
+var (
+	flagSeeds        string
+	flagWorkers      int
+	flagDelay        string
+	flagFetchTimeout string
+	flagLimit        int
+	flagUserAgent    string
+	flagLogLevel     string
+	flagLogFormat    string
+)
+
+func init() {
+	crawlCmd.Flags().StringVarP(&flagSeeds, "seeds", "s", "", "seeds file (overrides config)")
+	crawlCmd.Flags().IntVarP(&flagWorkers, "workers", "w", 0, "number of workers (overrides config)")
+	crawlCmd.Flags().StringVarP(&flagDelay, "delay", "d", "", "politeness delay, e.g. 500ms (overrides config)")
+	crawlCmd.Flags().StringVar(&flagFetchTimeout, "fetch-timeout", "", "fetch timeout, e.g. 10s (overrides config)")
+	crawlCmd.Flags().IntVarP(&flagLimit, "limit", "l", -1, "max pages to crawl, 0 = unlimited (overrides config)")
+	crawlCmd.Flags().StringVar(&flagUserAgent, "user-agent", "", "user agent string (overrides config)")
+	crawlCmd.Flags().StringVar(&flagLogLevel, "log-level", "", "log level: debug, info, warn, error (overrides config)")
+	crawlCmd.Flags().StringVar(&flagLogFormat, "log-format", "", "log format: text, json (overrides config)")
+	rootCmd.AddCommand(crawlCmd)
+}
+
+func runCrawl(cmd *cobra.Command, _ []string) error {
+	cfg, err := config.Load(cfgFile)
+	if err != nil {
+		return fmt.Errorf("failed to load config: %w", err)
+	}
+
+	if cmd.Flags().Changed("log-level") {
+		cfg.Logging.Level = flagLogLevel
+	}
+	if cmd.Flags().Changed("log-format") {
+		cfg.Logging.Format = flagLogFormat
+	}
+	logger.InitLogger(cfg)
+
+	if cmd.Flags().Changed("seeds") {
+		cfg.Crawler.SeedsFile = flagSeeds
+	}
+	if cmd.Flags().Changed("workers") {
+		cfg.Crawler.Workers = flagWorkers
+	}
+	if cmd.Flags().Changed("delay") {
+		cfg.Politeness.Delay = flagDelay
+	}
+	if cmd.Flags().Changed("fetch-timeout") {
+		cfg.Politeness.FetchTimeout = flagFetchTimeout
+	}
+	if cmd.Flags().Changed("limit") {
+		cfg.Crawler.CrawlLimit = flagLimit
+	}
+	if cmd.Flags().Changed("user-agent") {
+		cfg.Crawler.UserAgent = flagUserAgent
+	}
+	if err := cfg.Validate(); err != nil {
+		return fmt.Errorf("invalid config: %w", err)
+	}
+
+	seeds, err := crawler.LoadSeedsFile(cfg.Crawler.SeedsFile)
+	if err != nil {
+		return fmt.Errorf("failed to load seeds: %w", err)
+	}
+
+	db, err := sql.Open("postgres", cfg.DSN)
+	if err != nil {
+		return fmt.Errorf("failed to open database: %w", err)
+	}
+	defer db.Close()
+
+	if err := db.Ping(); err != nil {
+		return fmt.Errorf("failed to ping database: %w", err)
+	}
+	slog.Debug("database connected")
+
+	slog.Debug("running database migrations")
+	if err := crawler.RunMigrations(db); err != nil {
+		return fmt.Errorf("failed to run migrations: %w", err)
+	}
+	slog.Debug("database migrations complete")
+
+	store := crawler.NewPostgresStore(db)
+
+	client := &http.Client{}
+	c := crawler.NewCrawler(crawler.NewHTTPFetcher(client),
+		store,
+		crawler.WithUserAgent(cfg.Crawler.UserAgent),
+		crawler.WithCrawlDelay(cfg.Politeness.GetDelay()),
+		crawler.WithWorkers(cfg.Crawler.Workers),
+		crawler.WithCrawlLimit(cfg.Crawler.CrawlLimit),
+		crawler.WithFetchTimeout(cfg.Politeness.GetFetchTimeout()),
+	)
+
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGINT, syscall.SIGQUIT)
+	go func() {
+		select {
+		case s := <-sig:
+			slog.Info("received signal, shutting down", slog.String("signal", s.String()))
+			stop()
+		case <-ctx.Done():
+		}
+	}()
+
+	if err := c.Run(ctx, seeds); err != nil && !errors.Is(err, context.Canceled) {
+		return err
+	}
+	return nil
+}

cmd/crawler/crawler_acceptance_test.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/devraulu/crowlr/pkg/crawler"
+)
+
+func TestCrawlerStoresPagesEndToEnd(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `<html><body>
+			<a href="/page-a">Page A</a>
+			<a href="/page-b">Page B</a>
+		</body></html>`)
+	})
+	mux.HandleFunc("/page-a", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `<html><body><h1>Page A</h1></body></html>`)
+	})
+	mux.HandleFunc("/page-b", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `<html><body><h1>Page B</h1></body></html>`)
+	})
+
+	app := newTestApp(t, mux)
+
+	seed, err := crawler.NewLink(app.server.URL + "/")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := crawler.NewCrawler(
+		crawler.NewHTTPFetcher(&http.Client{}),
+		app.store,
+		crawler.WithWorkers(1),
+	)
+
+	if err := c.Run(t.Context(), []crawler.Link{seed}); err != nil {
+		t.Fatal(err)
+	}
+
+	type row struct {
+		url        string
+		statusCode int
+		outlinks   []string
+	}
+
+	rows, err := app.db.QueryContext(t.Context(),
+		`SELECT url, status_code, outlinks FROM pages ORDER BY url`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+
+	var pages []row
+	for rows.Next() {
+		var p row
+		var rawOutlinks []byte
+		if err := rows.Scan(&p.url, &p.statusCode, &rawOutlinks); err != nil {
+			t.Fatal(err)
+		}
+		json.Unmarshal(rawOutlinks, &p.outlinks)
+		pages = append(pages, p)
+	}
+	if err := rows.Err(); err != nil {
+		t.Fatal(err)
+	}
+
+	base := app.server.URL
+
+	want := []row{
+		{url: base + "/", statusCode: 200, outlinks: []string{base + "/page-a", base + "/page-b"}},
+		{url: base + "/page-a", statusCode: 200, outlinks: []string{}},
+		{url: base + "/page-b", statusCode: 200, outlinks: []string{}},
+	}
+
+	if len(pages) != len(want) {
+		t.Fatalf("expected %d pages, got %d: %v", len(want), len(pages), pages)
+	}
+
+	for i, w := range want {
+		got := pages[i]
+		if got.url != w.url {
+			t.Errorf("[%d] url: got %q, want %q", i, got.url, w.url)
+		}
+		if got.statusCode != w.statusCode {
+			t.Errorf("[%d] status_code: got %d, want %d", i, got.statusCode, w.statusCode)
+		}
+		if len(got.outlinks) != len(w.outlinks) {
+			t.Errorf("[%d] outlinks: got %v, want %v", i, got.outlinks, w.outlinks)
+		}
+	}
+}