diff --git a/README.md b/README.md
index 7c34de6..f96f224 100644
--- a/README.md
+++ b/README.md
@@ -142,6 +142,151 @@ The library handles DNS resolution differently depending on the connection type:
**Important for Privacy**: When using `socks5h://` or other remote DNS proxies, your local DNS servers will not see any queries for the target domains, maintaining better privacy and anonymity.
+
+### Metrics and Observability
+
+`gowarc` provides a `StatsRegistry` interface that allows you to integrate your own metrics collection system (Prometheus, Datadog, etc.). The library tracks various metrics including data written, deduplication statistics, and proxy usage.
+
+#### Using the StatsRegistry Interface
+
+The `StatsRegistry` interface can be found in [`stats.go`](stats.go). To implement your own metrics collection:
+
+```go
+// Implement the StatsRegistry interface
+type MyPrometheusRegistry struct {
+ // Your Prometheus registry fields
+}
+
+func (r *MyPrometheusRegistry) RegisterCounter(name, help string, labelNames []string) warc.Counter {
+ // Return a Counter that wraps your Prometheus counter
+ // The Counter interface requires WithLabels() method for dimensional metrics
+}
+
+func (r *MyPrometheusRegistry) RegisterGauge(name, help string, labelNames []string) warc.Gauge {
+ // Return a Gauge that wraps your Prometheus gauge
+}
+
+func (r *MyPrometheusRegistry) RegisterHistogram(name, help string, buckets []int64, labelNames []string) warc.Histogram {
+ // Return a Histogram that wraps your Prometheus histogram
+}
+
+// Pass your registry to the HTTP client
+clientSettings := warc.HTTPClientSettings{
+ StatsRegistry: &MyPrometheusRegistry{},
+ // ... other settings
+}
+```
+
+#### Available Metrics
+
+The library tracks the following metrics:
+
+- **`total_data_written`**: Total bytes written to WARC files
+- **`local_deduped_bytes_total`**: Bytes saved through local deduplication
+- **`local_deduped_total`**: Number of records deduplicated locally
+- **`doppelganger_deduped_bytes_total`**: Bytes saved through Doppelganger deduplication
+- **`doppelganger_deduped_total`**: Number of records deduplicated via Doppelganger
+- **`cdx_deduped_bytes_total`**: Bytes saved through CDX deduplication
+- **`cdx_deduped_total`**: Number of records deduplicated via CDX
+- **`proxy_requests_total`**: Total requests through each proxy (with `proxy` label)
+- **`proxy_errors_total`**: Total errors for each proxy (with `proxy` label)
+- **`proxy_last_used_nanoseconds`**: Last usage timestamp for each proxy (with `proxy` label)
+
+#### Label Support
+
+Metrics support Prometheus-style labels for dimensional data:
+
+```go
+// Register a counter with label dimensions
+counter := registry.RegisterCounter("http_requests_total", "Total HTTP requests", []string{"method", "status"})
+
+// Record metrics with specific label values
+counter.WithLabels(warc.Labels{"method": "GET", "status": "200"}).Inc()
+counter.WithLabels(warc.Labels{"method": "POST", "status": "201"}).Add(5)
+
+// Each unique label combination creates a separate metric series
+```
+
+**Interface Details**: See the complete interface contract in [`stats.go`](stats.go) for full implementation requirements.
+
+### Logging
+
+`gowarc` provides a `LogBackend` interface that allows you to integrate your logging solution (slog, zap, logrus, etc.). The library logs key events including connection establishment, DNS resolution, proxy selection, TLS handshakes, WARC file operations, and errors.
+
+#### Using the LogBackend Interface
+
+The `LogBackend` interface can be found in [`logging.go`](logging.go). The interface matches `slog.Logger` method signatures for easy integration:
+
+```go
+// Example: Wrapping slog.Logger to implement LogBackend
+type SlogAdapter struct {
+ logger *slog.Logger
+}
+
+func (s *SlogAdapter) Debug(msg string, args ...any) {
+ s.logger.Debug(msg, args...)
+}
+
+func (s *SlogAdapter) Info(msg string, args ...any) {
+ s.logger.Info(msg, args...)
+}
+
+func (s *SlogAdapter) Warn(msg string, args ...any) {
+ s.logger.Warn(msg, args...)
+}
+
+func (s *SlogAdapter) Error(msg string, args ...any) {
+ s.logger.Error(msg, args...)
+}
+
+func (s *SlogAdapter) Log(ctx context.Context, level slog.Level, msg string, args ...any) {
+ s.logger.Log(ctx, level, msg, args...)
+}
+
+// Configure with your logger
+handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})
+logger := slog.New(handler)
+
+clientSettings := warc.HTTPClientSettings{
+ LogBackend: &SlogAdapter{logger: logger},
+ // ... other settings
+}
+```
+
+#### Log Events
+
+The library logs structured events with contextual key-value pairs:
+
+**Connection & Network:**
+- Proxy selection and connection status
+- Direct connection establishment
+- DNS resolution results and failures
+- TLS handshake success and failures
+
+**WARC Operations:**
+- WARC record writing and file rotation
+- Data written and compression events
+- File creation and closure
+
+**Errors:**
+- Connection failures (proxy and direct)
+- DNS resolution errors
+- TLS handshake failures
+- WARC write errors
+
+#### Log Levels
+
+- **Debug**: Verbose operational details (DNS lookups, successful connections, record writes)
+- **Info**: Important state changes (file rotation, new WARC files)
+- **Warn**: Recoverable issues and fallbacks
+- **Error**: Failures and exceptions (connection errors, DNS failures, write errors)
+
+Users control which log levels are recorded by configuring their logger implementation's level threshold.
+
+**Note**: The `LogBackend` interface is intended to eventually replace the `ErrChan` error reporting mechanism. For now, both are maintained for backward compatibility.
+
+**Interface Details**: See the complete interface contract in [`logging.go`](logging.go) for full implementation requirements.
+
## CLI Tools
In addition to the Go library, gowarc provides several command-line utilities for working with WARC files:
diff --git a/client.go b/client.go
index 9591e55..8c60275 100644
--- a/client.go
+++ b/client.go
@@ -4,7 +4,6 @@ import (
"net/http"
"os"
"sync"
- "sync/atomic"
"time"
)
@@ -13,9 +12,52 @@ type Error struct {
Func string
}
+// ProxyNetwork defines the network layer (IPv4/IPv6) a proxy can support
+type ProxyNetwork int
+
+const (
+ // ProxyNetworkUnset is the zero value and must not be used - forces explicit selection
+ ProxyNetworkUnset ProxyNetwork = iota
+ // ProxyNetworkAny means the proxy can be used for both IPv4 and IPv6 connections
+ ProxyNetworkAny
+ // ProxyNetworkIPv4 means the proxy should only be used for IPv4 connections
+ ProxyNetworkIPv4
+ // ProxyNetworkIPv6 means the proxy should only be used for IPv6 connections
+ ProxyNetworkIPv6
+)
+
+// ProxyType defines the infrastructure type of a proxy
+type ProxyType int
+
+const (
+ // ProxyTypeAny means the proxy can be used for any type of request
+ ProxyTypeAny ProxyType = iota
+ // ProxyTypeMobile means the proxy uses mobile network infrastructure
+ ProxyTypeMobile
+ // ProxyTypeResidential means the proxy uses residential IP addresses
+ ProxyTypeResidential
+ // ProxyTypeDatacenter means the proxy uses datacenter infrastructure
+ ProxyTypeDatacenter
+)
+
+// ProxyConfig defines the configuration for a single proxy
+type ProxyConfig struct {
+ // URL is the proxy URL (e.g., "socks5://proxy.example.com:1080")
+ URL string
+ // Network specifies if this proxy supports IPv4, IPv6, or both
+ Network ProxyNetwork
+ // Type specifies the infrastructure type (Mobile, Residential, Datacenter, or Any)
+ Type ProxyType
+ // AllowedDomains is a list of glob patterns for domains this proxy should handle
+ // Examples: "*.example.com", "api.*.org"
+ // If empty, the proxy can be used for any domain
+ AllowedDomains []string
+}
+
type HTTPClientSettings struct {
RotatorSettings *RotatorSettings
- Proxy string
+ Proxies []ProxyConfig
+ AllowDirectFallback bool
TempDir string
DiscardHook DiscardHook
DNSServers []string
@@ -39,13 +81,14 @@ type HTTPClientSettings struct {
DisableIPv6 bool
IPv6AnyIP bool
DigestAlgorithm DigestAlgorithm
+ StatsRegistry StatsRegistry
+ LogBackend LogBackend
}
type CustomHTTPClient struct {
interfacesWatcherStop chan bool
WaitGroup *WaitGroupWithCount
dedupeHashTable *sync.Map
- ErrChan chan *Error
WARCWriter chan *RecordBatch
interfacesWatcherStarted chan bool
http.Client
@@ -64,15 +107,9 @@ type CustomHTTPClient struct {
// If set to <= 0, the default value is DefaultMaxRAMUsageFraction.
MaxRAMUsageFraction float64
randomLocalIP bool
- DataTotal *atomic.Int64
-
- CDXDedupeTotalBytes *atomic.Int64
- DoppelgangerDedupeTotalBytes *atomic.Int64
- LocalDedupeTotalBytes *atomic.Int64
- CDXDedupeTotal *atomic.Int64
- DoppelgangerDedupeTotal *atomic.Int64
- LocalDedupeTotal *atomic.Int64
+ statsRegistry StatsRegistry
+ logBackend LogBackend
}
func (c *CustomHTTPClient) Close() error {
@@ -91,7 +128,6 @@ func (c *CustomHTTPClient) Close() error {
}
wg.Wait()
- close(c.ErrChan)
if c.randomLocalIP {
c.interfacesWatcherStop <- true
@@ -106,16 +142,24 @@ func (c *CustomHTTPClient) Close() error {
func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient *CustomHTTPClient, err error) {
httpClient = new(CustomHTTPClient)
- // Initialize counters
- httpClient.DataTotal = &DataTotal
-
- httpClient.CDXDedupeTotalBytes = &CDXDedupeTotalBytes
- httpClient.DoppelgangerDedupeTotalBytes = &DoppelgangerDedupeTotalBytes
- httpClient.LocalDedupeTotalBytes = &LocalDedupeTotalBytes
+ // Initialize stats registry
+ if HTTPClientSettings.StatsRegistry != nil {
+ httpClient.statsRegistry = HTTPClientSettings.StatsRegistry
+ HTTPClientSettings.RotatorSettings.StatsRegistry = HTTPClientSettings.StatsRegistry
+ } else {
+ localStatsRegistry := newLocalRegistry()
+ httpClient.statsRegistry = localStatsRegistry
+ HTTPClientSettings.RotatorSettings.StatsRegistry = localStatsRegistry
+ }
- httpClient.CDXDedupeTotal = &CDXDedupeTotal
- httpClient.DoppelgangerDedupeTotal = &DoppelgangerDedupeTotal
- httpClient.LocalDedupeTotal = &LocalDedupeTotal
+ // Initialize log backend
+ if HTTPClientSettings.LogBackend != nil {
+ httpClient.logBackend = HTTPClientSettings.LogBackend
+ HTTPClientSettings.RotatorSettings.LogBackend = HTTPClientSettings.LogBackend
+ } else {
+ httpClient.logBackend = &noopLogger{}
+ HTTPClientSettings.RotatorSettings.LogBackend = &noopLogger{}
+ }
// Configure random local IP
httpClient.randomLocalIP = HTTPClientSettings.RandomLocalIP
@@ -142,9 +186,6 @@ func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient
// Set a hook to determine if we should discard a response
httpClient.DiscardHook = HTTPClientSettings.DiscardHook
- // Create an error channel for sending WARC errors through
- httpClient.ErrChan = make(chan *Error)
-
// Toggle verification of certificates
// InsecureSkipVerify expects the opposite of the verifyCerts flag, as such we flip it.
httpClient.verifyCerts = !HTTPClientSettings.VerifyCerts
@@ -216,7 +257,7 @@ func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient
httpClient.ConnReadDeadline = HTTPClientSettings.ConnReadDeadline
// Configure custom dialer / transport
- customDialer, err := newCustomDialer(httpClient, HTTPClientSettings.Proxy, HTTPClientSettings.DialTimeout, HTTPClientSettings.DNSRecordsTTL, HTTPClientSettings.DNSResolutionTimeout, HTTPClientSettings.DNSCacheSize, HTTPClientSettings.DNSServers, HTTPClientSettings.DNSConcurrency, HTTPClientSettings.DisableIPv4, HTTPClientSettings.DisableIPv6)
+ customDialer, err := newCustomDialer(httpClient, HTTPClientSettings.Proxies, HTTPClientSettings.AllowDirectFallback, HTTPClientSettings.DialTimeout, HTTPClientSettings.DNSRecordsTTL, HTTPClientSettings.DNSResolutionTimeout, HTTPClientSettings.DNSCacheSize, HTTPClientSettings.DNSServers, HTTPClientSettings.DNSConcurrency, HTTPClientSettings.DisableIPv4, HTTPClientSettings.DisableIPv6)
if err != nil {
return nil, err
}
diff --git a/client_test.go b/client_test.go
index 99ef81e..560100d 100644
--- a/client_test.go
+++ b/client_test.go
@@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"io"
+ "log/slog"
"math/big"
"net"
"net/http"
@@ -39,6 +40,8 @@ func defaultRotatorSettings(t *testing.T) *RotatorSettings {
err error
)
+ rotatorSettings.StatsRegistry = &localRegistry{}
+ rotatorSettings.LogBackend = &noopLogger{}
rotatorSettings.Prefix = "TEST"
rotatorSettings.OutputDirectory, err = os.MkdirTemp("", "warc-tests-")
if err != nil {
@@ -105,19 +108,6 @@ func sumRecordContentLengths(path string) (int64, error) {
return total, nil
}
-// Helper function used in many tests
-func drainErrChan(t *testing.T, errChan chan *Error) func() {
- var wg sync.WaitGroup
- wg.Add(1)
- go func() {
- defer wg.Done()
- for err := range errChan {
- t.Errorf("Error writing to WARC: %s", err.Err.Error())
- }
- }()
- return func() { wg.Wait() }
-}
-
func newTestImageServer(t testing.TB, st int) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fileBytes, err := os.ReadFile(path.Join("testdata", "image.svg"))
@@ -160,9 +150,6 @@ func TestHTTPClient(t *testing.T) {
err error
)
- // Reset counter to 0
- DataTotal.Store(0)
-
// init test HTTP endpoint
server := newTestImageServer(t, http.StatusOK)
defer server.Close()
@@ -172,7 +159,6 @@ func TestHTTPClient(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL+"/testdata/image.svg", nil)
if err != nil {
@@ -188,7 +174,6 @@ func TestHTTPClient(t *testing.T) {
io.Copy(io.Discard, resp.Body)
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -207,7 +192,7 @@ func TestHTTPClient(t *testing.T) {
}
// verify that the remote dedupe count is correct
- dataTotal := httpClient.DataTotal.Load()
+ dataTotal := httpClient.statsRegistry.RegisterCounter(totalDataWritten, totalDataWrittenHelp, nil).WithLabels(nil).Get()
if dataTotal != expectedPayloadBytes {
t.Fatalf("total bytes downloaded mismatch, expected %d got %d", expectedPayloadBytes, dataTotal)
}
@@ -223,22 +208,18 @@ func TestHTTPClientRequestFailing(t *testing.T) {
server := newTestImageServer(t, http.StatusOK)
defer server.Close()
+ // init test logger to capture errors
+ testLog := NewTestLogger()
+
// init the HTTP client responsible for recording HTTP(s) requests / responses
- httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{RotatorSettings: rotatorSettings})
+ httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
+ RotatorSettings: rotatorSettings,
+ LogBackend: testLog,
+ })
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- errCh := make(chan *Error, 1)
- var errChWg sync.WaitGroup
- errChWg.Add(1)
- go func() {
- defer errChWg.Done()
- for err := range httpClient.ErrChan {
- errCh <- err
- }
- }()
-
// Prepare some dummy data and configure our error injector
data := []byte("this is some test data")
erc := &errorReadCloser{data: data, readBefore: 10} // allow 10 bytes before error
@@ -249,33 +230,21 @@ func TestHTTPClientRequestFailing(t *testing.T) {
}
_, err = httpClient.Do(req)
- if err == nil {
- select {
- case recv := <-errCh:
- if recv == nil {
- t.Fatal("expected error via ErrChan but channel closed without value")
- }
- case <-time.After(2 * time.Second):
- t.Fatal("expected error on Do or via ErrChan, got none")
- }
- } else {
- t.Logf("got expected error: %v", err)
+ if err == nil || !strings.Contains(err.Error(), "injected read error") {
+ t.Fatal("expected \"injected read error\" error, got nil")
}
httpClient.Close()
- errChWg.Wait()
- close(errCh)
}
func TestHTTPClientConnReadDeadline(t *testing.T) {
var (
rotatorSettings = defaultRotatorSettings(t)
- errWg sync.WaitGroup
err error
)
// 1) Set up a test server that sends its response slowly, in chunks
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
@@ -301,14 +270,6 @@ func TestHTTPClientConnReadDeadline(t *testing.T) {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- // Read any WARC-writing errors
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for range httpClient.ErrChan {
- }
- }()
-
// 3) Create a request
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -335,7 +296,6 @@ func TestHTTPClientConnReadDeadline(t *testing.T) {
func TestHTTPClientContextCancellation(t *testing.T) {
var (
rotatorSettings = defaultRotatorSettings(t)
- errWg sync.WaitGroup
err error
)
@@ -363,15 +323,6 @@ func TestHTTPClientContextCancellation(t *testing.T) {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- // Read any WARC-writing errors
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for _ = range httpClient.ErrChan {
- // t.Errorf("Error writing to WARC: %s", e.Err.Error())
- }
- }()
-
// 3) Create a request with a cancellable context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -424,11 +375,12 @@ func TestHTTPClientWithFeedbackChan(t *testing.T) {
defer server.Close()
// init the HTTP client responsible for recording HTTP(s) requests / responses
- httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{RotatorSettings: rotatorSettings})
+ httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
+ RotatorSettings: rotatorSettings,
+ })
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL+"/testdata/image.svg", nil)
if err != nil {
@@ -449,7 +401,6 @@ func TestHTTPClientWithFeedbackChan(t *testing.T) {
<-feedbackCh
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -499,6 +450,9 @@ func TestHTTPClientTLSHandshakeTimeout(t *testing.T) {
serverURL := "https://" + ln.Addr().String()
+ // 4) Set up a test logger to capture errors
+ testLog := NewTestLogger()
+
// 5) Create the WARC-writing HTTP client
// The critical part here is enforcing the handshake timeout.
// (Exact field names may differ based on your library.)
@@ -506,11 +460,11 @@ func TestHTTPClientTLSHandshakeTimeout(t *testing.T) {
RotatorSettings: rotatorSettings,
TLSHandshakeTimeout: 1 * time.Second, // <--- The key line
VerifyCerts: true, // or "VerifyCerts: false" depending on your lib
+ LogBackend: testLog,
})
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %v", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
// 6) Attempt the GET, which should fail due to TLS handshake delay
req, err := http.NewRequest("GET", serverURL, nil)
@@ -529,8 +483,25 @@ func TestHTTPClientTLSHandshakeTimeout(t *testing.T) {
t.Logf("Got expected error: %v", err)
}
+ // Check if an error is logged with the expected message
+ gotErr := false
+ logEntries := make([]LogEntry, 0)
+ if testLog.HasLevel(slog.LevelError) {
+ for _, entry := range testLog.Entries() {
+ if entry.Level == slog.LevelError && strings.Contains(entry.Message, "TLS handshake failed") && strings.Contains(entry.Args[3].(error).Error(), "context deadline exceeded") {
+ gotErr = true
+ } else if entry.Level == slog.LevelError {
+ t.Logf("Unexpected log entry: %v", entry)
+ }
+ logEntries = append(logEntries, entry)
+ }
+ }
+
+ if !gotErr {
+ t.Fatalf("Expected TLS handshake failed error in logs, got %v", logEntries)
+ }
+
httpClient.Close()
- waitForErrors()
<-doneChan // Wait for the server goroutine to exit
}
@@ -538,7 +509,6 @@ func TestHTTPClientTLSHandshakeTimeout(t *testing.T) {
func TestHTTPClientServerClosingConnection(t *testing.T) {
var (
rotatorSettings = defaultRotatorSettings(t)
- errWg sync.WaitGroup
err error
)
@@ -573,19 +543,13 @@ func TestHTTPClientServerClosingConnection(t *testing.T) {
defer server.Close()
// init the HTTP client responsible for recording HTTP(s) requests / responses
- httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{RotatorSettings: rotatorSettings})
+ httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
+ RotatorSettings: rotatorSettings,
+ })
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for _ = range httpClient.ErrChan {
- // We expect an error here, so we don't need to log it
- }
- }()
-
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
t.Fatal(err)
@@ -618,14 +582,16 @@ func TestHTTPClientDNSFailure(t *testing.T) {
err error
)
+ testLog := NewTestLogger()
+
// Initialize the WARC-writing HTTP client
httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
RotatorSettings: rotatorSettings,
+ LogBackend: testLog,
})
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
// Use a guaranteed-nonresolvable domain
req, err := http.NewRequest("GET", "http://should-not-resolve.example.invalid", nil)
@@ -644,8 +610,23 @@ func TestHTTPClientDNSFailure(t *testing.T) {
t.Logf("Got expected DNS error: %v", err)
}
+ // Verify the error was logged
+ gotErr := false
+ logEntries := make([]LogEntry, 0)
+ if testLog.HasLevel(slog.LevelError) {
+ for _, entry := range testLog.Entries() {
+ if entry.Level == slog.LevelError && strings.Contains(entry.Message, "DNS resolution failed") && strings.Contains(entry.Args[3].(error).Error(), "failed to resolve DNS: A error: no TYPE=A record found, AAAA error: no TYPE=AAAA record found") {
+ gotErr = true
+ }
+ logEntries = append(logEntries, entry)
+ }
+ }
+
+ if !gotErr {
+ t.Fatalf("Expected DNS resolution failed error in logs, got %v", logEntries)
+ }
+
httpClient.Close()
- waitForErrors()
}
func TestHTTPClientWithProxy(t *testing.T) {
@@ -688,11 +669,16 @@ func TestHTTPClientWithProxy(t *testing.T) {
// init the HTTP client responsible for recording HTTP(s) requests / responses
httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
RotatorSettings: rotatorSettings,
- Proxy: fmt.Sprintf("socks5://%s", proxyAddr)})
+ Proxies: []ProxyConfig{
+ {
+ URL: fmt.Sprintf("socks5://%s", proxyAddr),
+ Network: ProxyNetworkAny,
+ Type: ProxyTypeAny,
+ },
+ }})
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -708,7 +694,6 @@ func TestHTTPClientWithProxy(t *testing.T) {
io.Copy(io.Discard, resp.Body)
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -732,11 +717,12 @@ func TestHTTPClientConcurrent(t *testing.T) {
defer server.Close()
// init the HTTP client responsible for recording HTTP(s) requests / responses
- httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{RotatorSettings: rotatorSettings})
+ httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
+ RotatorSettings: rotatorSettings,
+ })
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
wg.Add(concurrency)
for i := 0; i < concurrency; i++ {
@@ -746,13 +732,11 @@ func TestHTTPClientConcurrent(t *testing.T) {
req, err := http.NewRequest("GET", server.URL, nil)
req.Close = true
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
resp, err := httpClient.Do(req)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
defer resp.Body.Close()
@@ -765,7 +749,6 @@ func TestHTTPClientConcurrent(t *testing.T) {
wg.Wait()
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -790,11 +773,12 @@ func TestHTTPClientMultiWARCWriters(t *testing.T) {
defer server.Close()
// init the HTTP client responsible for recording HTTP(s) requests / responses
- httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{RotatorSettings: rotatorSettings})
+ httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
+ RotatorSettings: rotatorSettings,
+ })
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
wg.Add(concurrency)
for i := 0; i < concurrency; i++ {
@@ -804,13 +788,11 @@ func TestHTTPClientMultiWARCWriters(t *testing.T) {
req, err := http.NewRequest("GET", server.URL, nil)
req.Close = true
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
resp, err := httpClient.Do(req)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
defer resp.Body.Close()
@@ -823,7 +805,6 @@ func TestHTTPClientMultiWARCWriters(t *testing.T) {
wg.Wait()
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -846,10 +827,6 @@ func TestHTTPClientLocalDedupe(t *testing.T) {
err error
)
- // Reset counter to 0
- LocalDedupeTotal.Store(0)
- LocalDedupeTotalBytes.Store(0)
-
// init test HTTP endpoint
server := newTestImageServer(t, http.StatusOK)
defer server.Close()
@@ -864,7 +841,6 @@ func TestHTTPClientLocalDedupe(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
for i := 0; i < 2; i++ {
req, err := http.NewRequest("GET", server.URL, nil)
@@ -884,7 +860,6 @@ func TestHTTPClientLocalDedupe(t *testing.T) {
}
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -897,18 +872,13 @@ func TestHTTPClientLocalDedupe(t *testing.T) {
}
// verify that the local dedupe count is correct
- if LocalDedupeTotalBytes.Load() != 26872 {
- t.Fatalf("local dedupe total bytes mismatch, expected: 26872 got: %d", LocalDedupeTotalBytes.Load())
- }
-
- // Ensure that HTTP client results work correctly as well
- if httpClient.LocalDedupeTotalBytes.Load() != 26872 {
- t.Fatalf("local dedupe total bytes mismatch, expected: 26872 got: %d", httpClient.LocalDedupeTotalBytes.Load())
+ if httpClient.statsRegistry.RegisterCounter(localDedupedBytesTotal, localDedupedBytesTotalHelp, nil).WithLabels(nil).Get() != 26872 {
+ t.Fatalf("local dedupe total bytes mismatch, expected: 26872 got: %d", httpClient.statsRegistry.RegisterCounter(localDedupedBytesTotal, localDedupedBytesTotalHelp, nil).WithLabels(nil).Get())
}
// 1 is expected due to requiring one request to enter into the table.
- if httpClient.LocalDedupeTotal.Load() != 1 {
- t.Fatalf("local dedupe total mismatch, expected: 1 got: %d", httpClient.LocalDedupeTotal.Load())
+ if httpClient.statsRegistry.RegisterCounter(localDedupedTotal, localDedupedTotalHelp, nil).WithLabels(nil).Get() != 1 {
+ t.Fatalf("local dedupe total mismatch, expected: 1 got: %d", httpClient.statsRegistry.RegisterCounter(localDedupedTotal, localDedupedTotalHelp, nil).WithLabels(nil).Get())
}
}
@@ -922,11 +892,7 @@ func TestHTTPClientRemoteDedupe(t *testing.T) {
// init test HTTP endpoint
mux := http.NewServeMux()
- // Reset counter to 0
- CDXDedupeTotal.Store(0)
- CDXDedupeTotalBytes.Store(0)
-
- mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
fileBytes, err := os.ReadFile(path.Join("testdata", "image.svg"))
if err != nil {
t.Fatal(err)
@@ -937,7 +903,7 @@ func TestHTTPClientRemoteDedupe(t *testing.T) {
w.Write(fileBytes)
})
- mux.HandleFunc(dedupePath, func(w http.ResponseWriter, r *http.Request) {
+ mux.HandleFunc(dedupePath, func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/plain;charset=UTF-8")
w.WriteHeader(http.StatusOK)
w.Write([]byte(dedupeResp))
@@ -958,7 +924,6 @@ func TestHTTPClientRemoteDedupe(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
for i := 0; i < 4; i++ {
req, err := http.NewRequest("GET", server.URL, nil)
@@ -978,7 +943,6 @@ func TestHTTPClientRemoteDedupe(t *testing.T) {
}
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -990,18 +954,13 @@ func TestHTTPClientRemoteDedupe(t *testing.T) {
testFileRevisitVailidity(t, path, "2022-03-20T00:25:18Z", "sha1:UIRWL5DFIPQ4MX3D3GFHM2HCVU3TZ6I3", false)
}
- // verify that the remote dedupe count is correct
- if CDXDedupeTotalBytes.Load() != 107488 {
- t.Fatalf("remote dedupe total bytes mismatch, expected: 107488 got: %d", CDXDedupeTotalBytes.Load())
- }
-
- // Ensure that HTTP client results work correctly as well
- if httpClient.CDXDedupeTotalBytes.Load() != 107488 {
- t.Fatalf("remote dedupe total bytes mismatch, expected: 107488 got: %d", httpClient.CDXDedupeTotalBytes.Load())
+ // verify that the CDX dedupe count is correct
+ if httpClient.statsRegistry.RegisterCounter(cdxDedupedBytesTotal, cdxDedupedBytesTotalHelp, nil).WithLabels(nil).Get() != 107488 {
+ t.Fatalf("CDX dedupe total bytes mismatch, expected: 26872 got: %d", httpClient.statsRegistry.RegisterCounter(cdxDedupedBytesTotal, cdxDedupedBytesTotalHelp, nil).WithLabels(nil).Get())
}
- if httpClient.CDXDedupeTotal.Load() != 4 {
- t.Fatalf("remote dedupe total mismatch, expected: 4 got: %d", httpClient.CDXDedupeTotal.Load())
+ if httpClient.statsRegistry.RegisterCounter(cdxDedupedTotal, cdxDedupedTotalHelp, nil).WithLabels(nil).Get() != 4 {
+ t.Fatalf("CDX dedupe total mismatch, expected: 1 got: %d", httpClient.statsRegistry.RegisterCounter(cdxDedupedTotal, cdxDedupedTotalHelp, nil).WithLabels(nil).Get())
}
}
@@ -1010,17 +969,12 @@ func TestHTTPClientDoppelgangerDedupe(t *testing.T) {
dedupePath = "/api/records/UIRWL5DFIPQ4MX3D3GFHM2HCVU3TZ6I3"
dedupeResp = "{\"id\":\"UIRWL5DFIPQ4MX3D3GFHM2HCVU3TZ6I3\",\"uri\":\"https://upload.wikimedia.org/wikipedia/commons/5/55/Blason_ville_fr_Sarlat-la-Can%C3%A9da_%28Dordogne%29.svg\",\"date\":20220320002518}"
rotatorSettings = defaultRotatorSettings(t)
- errWg sync.WaitGroup
err error
)
// init test HTTP endpoint
mux := http.NewServeMux()
- // Reset counter to 0
- DoppelgangerDedupeTotal.Store(0)
- DoppelgangerDedupeTotalBytes.Store(0)
-
- mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) {
fileBytes, err := os.ReadFile(path.Join("testdata", "image.svg"))
if err != nil {
t.Fatal(err)
@@ -1031,7 +985,7 @@ func TestHTTPClientDoppelgangerDedupe(t *testing.T) {
w.Write(fileBytes)
})
- mux.HandleFunc(dedupePath, func(w http.ResponseWriter, r *http.Request) {
+ mux.HandleFunc(dedupePath, func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(dedupeResp))
@@ -1054,14 +1008,6 @@ func TestHTTPClientDoppelgangerDedupe(t *testing.T) {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for err := range httpClient.ErrChan {
- t.Errorf("Error writing to WARC: %s", err.Err.Error())
- }
- }()
-
for i := 0; i < 4; i++ {
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -1089,18 +1035,13 @@ func TestHTTPClientDoppelgangerDedupe(t *testing.T) {
testFileRevisitVailidity(t, path, "2022-03-20T00:25:18Z", "sha1:UIRWL5DFIPQ4MX3D3GFHM2HCVU3TZ6I3", false)
}
- // verify that the remote dedupe count is correct
- if DoppelgangerDedupeTotalBytes.Load() != 107488 {
- t.Fatalf("remote dedupe total bytes mismatch, expected: 107488 got: %d", DoppelgangerDedupeTotalBytes.Load())
+ // verify that the Doppelganger count is correct
+ if httpClient.statsRegistry.RegisterCounter(doppelgangerDedupedBytesTotal, doppelgangerDedupedBytesTotalHelp, nil).WithLabels(nil).Get() != 107488 {
+ t.Fatalf("Doppelganger total bytes mismatch, expected: 26872 got: %d", httpClient.statsRegistry.RegisterCounter(doppelgangerDedupedBytesTotal, doppelgangerDedupedBytesTotalHelp, nil).WithLabels(nil).Get())
}
- // Ensure that HTTP client results work correctly as well
- if httpClient.DoppelgangerDedupeTotalBytes.Load() != 107488 {
- t.Fatalf("remote dedupe total bytes mismatch, expected: 107488 got: %d", httpClient.DoppelgangerDedupeTotalBytes.Load())
- }
-
- if httpClient.DoppelgangerDedupeTotal.Load() != 4 {
- t.Fatalf("remote dedupe total mismatch, expected: 4 got: %d", httpClient.DoppelgangerDedupeTotal.Load())
+ if httpClient.statsRegistry.RegisterCounter(doppelgangerDedupedTotal, doppelgangerDedupedTotalHelp, nil).WithLabels(nil).Get() != 4 {
+ t.Fatalf("Doppelganger total mismatch, expected: 1 got: %d", httpClient.statsRegistry.RegisterCounter(doppelgangerDedupedTotal, doppelgangerDedupedTotalHelp, nil).WithLabels(nil).Get())
}
}
@@ -1110,10 +1051,6 @@ func TestHTTPClientDedupeEmptyPayload(t *testing.T) {
err error
)
- // Reset counter to 0
- LocalDedupeTotal.Store(0)
- LocalDedupeTotalBytes.Store(0)
-
// init test HTTP endpoint
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Empty. This is intentional to mirror 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ.
@@ -1134,7 +1071,6 @@ func TestHTTPClientDedupeEmptyPayload(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
for i := 0; i < 2; i++ {
req, err := http.NewRequest("GET", server.URL, nil)
@@ -1154,7 +1090,6 @@ func TestHTTPClientDedupeEmptyPayload(t *testing.T) {
}
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -1167,24 +1102,18 @@ func TestHTTPClientDedupeEmptyPayload(t *testing.T) {
}
// verify that the local dedupe count is correct
- if LocalDedupeTotalBytes.Load() != 0 {
- t.Fatalf("local dedupe total bytes mismatch, expected: 0 got: %d", LocalDedupeTotalBytes.Load())
- }
-
- // Ensure that HTTP client results work correctly as well
- if httpClient.LocalDedupeTotalBytes.Load() != 0 {
- t.Fatalf("local dedupe total bytes mismatch, expected: 0 got: %d", httpClient.LocalDedupeTotalBytes.Load())
+ if httpClient.statsRegistry.RegisterCounter(localDedupedBytesTotal, localDedupedBytesTotalHelp, nil).WithLabels(nil).Get() != 0 {
+ t.Fatalf("local dedupe total bytes mismatch, expected: 26872 got: %d", httpClient.statsRegistry.RegisterCounter(localDedupedBytesTotal, localDedupedBytesTotalHelp, nil).WithLabels(nil).Get())
}
- if httpClient.LocalDedupeTotal.Load() != 0 {
- t.Fatalf("local dedupe total mismatch, expected: 0 got: %d", httpClient.LocalDedupeTotal.Load())
+ if httpClient.statsRegistry.RegisterCounter(localDedupedTotal, localDedupedTotalHelp, nil).WithLabels(nil).Get() != 0 {
+ t.Fatalf("local dedupe total mismatch, expected: 1 got: %d", httpClient.statsRegistry.RegisterCounter(localDedupedTotal, localDedupedTotalHelp, nil).WithLabels(nil).Get())
}
}
func TestHTTPClientDiscardHook(t *testing.T) {
var (
rotatorSettings = defaultRotatorSettings(t)
- errWg sync.WaitGroup
err error
)
@@ -1194,9 +1123,12 @@ func TestHTTPClientDiscardHook(t *testing.T) {
server := newTestImageServer(t, http.StatusTooManyRequests)
defer server.Close()
+ testLog := NewTestLogger()
+
// init the HTTP client responsible for recording HTTP(s) requests / responses
httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
RotatorSettings: rotatorSettings,
+ LogBackend: testLog,
// Set up a discard hook to discard 429 responses
DiscardHook: func(resp *http.Response) (bool, string) {
if resp.StatusCode != http.StatusTooManyRequests {
@@ -1210,25 +1142,6 @@ func TestHTTPClientDiscardHook(t *testing.T) {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for err := range httpClient.ErrChan {
- // validate 429 filtering as well as error reporting by url
- discardErr, ok := err.Err.(*DiscardHookError)
- if !ok {
- t.Errorf("Expected DiscardHookError, got: %T, error: %v", err.Err, err)
- continue
- }
- if discardErr.URL != server.URL+"/" {
- t.Errorf("Expected URL %s, got: %s", server.URL+"/", discardErr.URL)
- }
- if discardErr.Reason != expectedReason {
- t.Errorf("Expected Reason %s, got: %s", expectedReason, discardErr.Reason)
- }
- }
- }()
-
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
t.Fatal(err)
@@ -1244,6 +1157,22 @@ func TestHTTPClientDiscardHook(t *testing.T) {
httpClient.Close()
+ // Verify the error was logged
+ gotErr := false
+ logEntries := make([]LogEntry, 0)
+ if testLog.HasLevel(slog.LevelError) {
+ for _, entry := range testLog.Entries() {
+ if entry.Level == slog.LevelError && strings.Contains(entry.Message, "error reading from connection") && strings.Contains(entry.Args[3].(*DiscardHookError).Error(), "response was blocked by DiscardHook") && strings.Contains(entry.Args[3].(*DiscardHookError).Error(), "429 response") {
+ gotErr = true
+ }
+ logEntries = append(logEntries, entry)
+ }
+ }
+
+ if !gotErr {
+ t.Fatalf("Expected \"error reading from connection\" error in logs, got %v", logEntries)
+ }
+
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
t.Fatal(err)
@@ -1279,7 +1208,6 @@ func TestHTTPClientPayloadLargerThan2MB(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -1295,7 +1223,6 @@ func TestHTTPClientPayloadLargerThan2MB(t *testing.T) {
io.Copy(io.Discard, resp.Body)
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -1334,7 +1261,6 @@ func TestConcurrentHTTPClientPayloadLargerThan2MB(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
wg.Add(concurrency)
for i := 0; i < concurrency; i++ {
@@ -1344,13 +1270,11 @@ func TestConcurrentHTTPClientPayloadLargerThan2MB(t *testing.T) {
req, err := http.NewRequest("GET", server.URL, nil)
req.Close = true
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
resp, err := httpClient.Do(req)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
@@ -1363,7 +1287,6 @@ func TestConcurrentHTTPClientPayloadLargerThan2MB(t *testing.T) {
wg.Wait()
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -1395,7 +1318,6 @@ func TestHTTPClientWithSelfSignedCertificate(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -1411,7 +1333,6 @@ func TestHTTPClientWithSelfSignedCertificate(t *testing.T) {
io.Copy(io.Discard, resp.Body)
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -1448,7 +1369,6 @@ func TestWARCWritingWithDisallowedCertificate(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -1467,7 +1387,6 @@ func TestWARCWritingWithDisallowedCertificate(t *testing.T) {
}
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -1495,7 +1414,6 @@ func TestHTTPClientFullOnDisk(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -1511,7 +1429,6 @@ func TestHTTPClientFullOnDisk(t *testing.T) {
io.Copy(io.Discard, resp.Body)
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -1526,7 +1443,6 @@ func TestHTTPClientFullOnDisk(t *testing.T) {
func TestHTTPClientWithoutIoCopy(t *testing.T) {
var (
rotatorSettings = defaultRotatorSettings(t)
- errWg sync.WaitGroup
err error
)
@@ -1537,23 +1453,17 @@ func TestHTTPClientWithoutIoCopy(t *testing.T) {
server := newTestImageServer(t, http.StatusOK)
defer server.Close()
+ testLog := NewTestLogger()
+
// init the HTTP client responsible for recording HTTP(s) requests / responses
- httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{RotatorSettings: rotatorSettings})
+ httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
+ RotatorSettings: rotatorSettings,
+ LogBackend: testLog,
+ })
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for err := range httpClient.ErrChan {
- // validate 429 filtering as well as error reporting by url
- if strings.Contains(err.Err.Error(), "SHA1 ran into an unrecoverable error url") {
- t.Errorf("Error writing to WARC: %s", err.Err.Error())
- }
- }
- }()
-
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
t.Fatal(err)
@@ -1569,6 +1479,21 @@ func TestHTTPClientWithoutIoCopy(t *testing.T) {
httpClient.Close()
+ gotErr := false
+ logEntries := make([]LogEntry, 0)
+ if testLog.HasLevel(slog.LevelError) {
+ for _, entry := range testLog.Entries() {
+ if entry.Level == slog.LevelError && strings.Contains(entry.Message, "error reading from connection") && strings.Contains(entry.Args[3].(error).Error(), "readResponse: payload digest calculation failed: unexpected EOF") {
+ gotErr = true
+ }
+ logEntries = append(logEntries, entry)
+ }
+ }
+
+ if !gotErr {
+ t.Fatalf("Expected \"error reading from connection\" error in logs, got %v", logEntries)
+ }
+
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
t.Fatal(err)
@@ -1598,7 +1523,6 @@ func TestHTTPClientWithoutChunkEncoding(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -1614,7 +1538,6 @@ func TestHTTPClientWithoutChunkEncoding(t *testing.T) {
io.Copy(io.Discard, resp.Body)
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -1642,7 +1565,6 @@ func TestHTTPClientWithZStandard(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -1658,7 +1580,6 @@ func TestHTTPClientWithZStandard(t *testing.T) {
io.Copy(io.Discard, resp.Body)
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -1688,7 +1609,6 @@ func TestHTTPClientWithZStandardDictionary(t *testing.T) {
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- waitForErrors := drainErrChan(t, httpClient.ErrChan)
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
@@ -1704,7 +1624,6 @@ func TestHTTPClientWithZStandardDictionary(t *testing.T) {
io.Copy(io.Discard, resp.Body)
httpClient.Close()
- waitForErrors()
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
@@ -1763,9 +1682,12 @@ func TestHTTPClientWithIPv4Disabled(t *testing.T) {
rotatorSettings := defaultRotatorSettings(t)
+ testLog := NewTestLogger()
+
httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
RotatorSettings: rotatorSettings,
DisableIPv4: true,
+ LogBackend: testLog,
})
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
@@ -1791,6 +1713,21 @@ func TestHTTPClientWithIPv4Disabled(t *testing.T) {
httpClient.Close()
+ gotErr := false
+ logEntries := make([]LogEntry, 0)
+ if testLog.HasLevel(slog.LevelError) {
+ for _, entry := range testLog.Entries() {
+ if entry.Level == slog.LevelError && strings.Contains(entry.Message, "direct connection failed") && strings.Contains(entry.Args[3].(error).Error(), "dial tcp6: address 127.0.0.1: no suitable address found") {
+ gotErr = true
+ }
+ logEntries = append(logEntries, entry)
+ }
+ }
+
+ if !gotErr {
+ t.Fatalf("Expected \"direct connection failed\" error in logs, got %v", logEntries)
+ }
+
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
t.Fatal(err)
@@ -1810,9 +1747,12 @@ func TestHTTPClientWithIPv6Disabled(t *testing.T) {
rotatorSettings := defaultRotatorSettings(t)
+ testLog := NewTestLogger()
+
httpClient, err := NewWARCWritingHTTPClient(HTTPClientSettings{
RotatorSettings: rotatorSettings,
DisableIPv6: true,
+ LogBackend: testLog,
})
if err != nil {
t.Fatalf("Unable to init WARC writing HTTP client: %s", err)
@@ -1838,6 +1778,21 @@ func TestHTTPClientWithIPv6Disabled(t *testing.T) {
httpClient.Close()
+ gotErr := false
+ logEntries := make([]LogEntry, 0)
+ if testLog.HasLevel(slog.LevelError) {
+ for _, entry := range testLog.Entries() {
+ if entry.Level == slog.LevelError && strings.Contains(entry.Message, "direct connection failed") && strings.Contains(entry.Args[3].(error).Error(), "dial tcp4: address ::1: no suitable address found") {
+ gotErr = true
+ }
+ logEntries = append(logEntries, entry)
+ }
+ }
+
+ if !gotErr {
+ t.Fatalf("Expected \"direct connection failed\" error in logs, got %v", logEntries)
+ }
+
files, err := filepath.Glob(rotatorSettings.OutputDirectory + "/*")
if err != nil {
t.Fatal(err)
@@ -1853,7 +1808,6 @@ func BenchmarkConcurrentUnder2MB(b *testing.B) {
var (
rotatorSettings = defaultBenchmarkRotatorSettings(b)
wg sync.WaitGroup
- errWg sync.WaitGroup
err error
)
@@ -1876,14 +1830,6 @@ func BenchmarkConcurrentUnder2MB(b *testing.B) {
b.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for err := range httpClient.ErrChan {
- b.Errorf("Error writing to WARC: %s", err.Err.Error())
- }
- }()
-
wg.Add(b.N)
for n := 0; n < b.N; n++ {
go func() {
@@ -1891,13 +1837,11 @@ func BenchmarkConcurrentUnder2MB(b *testing.B) {
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
resp, err := httpClient.Do(req)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
defer resp.Body.Close()
@@ -1914,7 +1858,6 @@ func BenchmarkConcurrentUnder2MBZStandard(b *testing.B) {
var (
rotatorSettings = defaultBenchmarkRotatorSettings(b)
wg sync.WaitGroup
- errWg sync.WaitGroup
err error
)
rotatorSettings.Compression = "ZSTD"
@@ -1938,14 +1881,6 @@ func BenchmarkConcurrentUnder2MBZStandard(b *testing.B) {
b.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for err := range httpClient.ErrChan {
- b.Errorf("Error writing to WARC: %s", err.Err.Error())
- }
- }()
-
wg.Add(b.N)
for n := 0; n < b.N; n++ {
go func() {
@@ -1953,13 +1888,11 @@ func BenchmarkConcurrentUnder2MBZStandard(b *testing.B) {
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
resp, err := httpClient.Do(req)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
defer resp.Body.Close()
@@ -1976,7 +1909,6 @@ func BenchmarkConcurrentOver2MB(b *testing.B) {
var (
rotatorSettings = defaultBenchmarkRotatorSettings(b)
wg sync.WaitGroup
- errWg sync.WaitGroup
err error
)
@@ -1999,14 +1931,6 @@ func BenchmarkConcurrentOver2MB(b *testing.B) {
b.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for err := range httpClient.ErrChan {
- b.Errorf("Error writing to WARC: %s", err.Err.Error())
- }
- }()
-
wg.Add(b.N)
for n := 0; n < b.N; n++ {
go func() {
@@ -2014,13 +1938,11 @@ func BenchmarkConcurrentOver2MB(b *testing.B) {
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
resp, err := httpClient.Do(req)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
defer resp.Body.Close()
@@ -2037,7 +1959,6 @@ func BenchmarkConcurrentOver2MBZStandard(b *testing.B) {
var (
rotatorSettings = defaultBenchmarkRotatorSettings(b)
wg sync.WaitGroup
- errWg sync.WaitGroup
err error
)
rotatorSettings.Compression = "ZSTD"
@@ -2061,14 +1982,6 @@ func BenchmarkConcurrentOver2MBZStandard(b *testing.B) {
b.Fatalf("Unable to init WARC writing HTTP client: %s", err)
}
- errWg.Add(1)
- go func() {
- defer errWg.Done()
- for err := range httpClient.ErrChan {
- b.Errorf("Error writing to WARC: %s", err.Err.Error())
- }
- }()
-
wg.Add(b.N)
for n := 0; n < b.N; n++ {
go func() {
@@ -2076,13 +1989,11 @@ func BenchmarkConcurrentOver2MBZStandard(b *testing.B) {
req, err := http.NewRequest("GET", server.URL, nil)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
resp, err := httpClient.Do(req)
if err != nil {
- httpClient.ErrChan <- &Error{Err: err}
return
}
defer resp.Body.Close()
diff --git a/dedupe.go b/dedupe.go
index ba1b36b..c93cb94 100644
--- a/dedupe.go
+++ b/dedupe.go
@@ -51,7 +51,7 @@ func (d *customDialer) checkLocalRevisit(digest string) revisitRecord {
func checkCDXRevisit(CDXURL string, digest string, targetURI string, cookie string) (revisitRecord, error) {
// CDX expects no hash header. For now we need to strip it.
digest = strings.SplitN(digest, ":", 2)[1]
-
+
req, err := http.NewRequest("GET", CDXURL+"/web/timemap/cdx?url="+url.QueryEscape(targetURI)+"&limit=-1", nil)
if err != nil {
return revisitRecord{}, err
@@ -95,7 +95,7 @@ func checkCDXRevisit(CDXURL string, digest string, targetURI string, cookie stri
func checkDoppelgangerRevisit(DoppelgangerHost string, digest string, targetURI string) (revisitRecord, error) {
// Doppelganger is not expecting a hash header either but this will all be rewritten ... shortly...
digest = strings.SplitN(digest, ":", 2)[1]
-
+
req, err := http.NewRequest("GET", DoppelgangerHost+"/api/records/"+digest+"?uri="+targetURI, nil)
if err != nil {
return revisitRecord{}, err
diff --git a/dialer.go b/dialer.go
index c642cc5..8108330 100644
--- a/dialer.go
+++ b/dialer.go
@@ -9,6 +9,7 @@ import (
"net"
"net/http"
"net/url"
+ "path/filepath"
"slices"
"strconv"
"strings"
@@ -39,6 +40,12 @@ const (
// This is used internally to retrieve the wrapped connection for advanced use cases.
// Use WithWrappedConnection() helper function for convenience.
ContextKeyWrappedConn contextKey = "wrappedConn"
+
+ // ContextKeyProxyType is the context key for requesting a specific proxy type.
+ // External callers (like Zeno) can set this to request a proxy of a specific type
+ // (Mobile, Residential, or Datacenter).
+ // Use WithProxyType() helper function for convenience.
+ ContextKeyProxyType contextKey = "proxyType"
)
// WithFeedbackChannel adds a feedback channel to the request context.
@@ -62,23 +69,51 @@ func WithWrappedConnection(ctx context.Context, wrappedConnChan chan *CustomConn
return context.WithValue(ctx, ContextKeyWrappedConn, wrappedConnChan)
}
+// WithProxyType adds a proxy type preference to the request context.
+// When set, the proxy selector will prefer proxies of the specified type
+// (Mobile, Residential, or Datacenter).
+// This is typically used by external callers like Zeno when they need a specific proxy type.
+//
+// Example:
+//
+// req = req.WithContext(warc.WithProxyType(req.Context(), warc.ProxyTypeMobile))
+func WithProxyType(ctx context.Context, proxyType ProxyType) context.Context {
+ return context.WithValue(ctx, ContextKeyProxyType, proxyType)
+}
+
// dnsExchanger is an interface for DNS clients that can exchange messages
type dnsExchanger interface {
ExchangeContext(ctx context.Context, m *dns.Msg, address string) (r *dns.Msg, rtt time.Duration, err error)
}
+// proxyDialerInfo holds information about a configured proxy
+type proxyDialerInfo struct {
+ dialer proxy.ContextDialer
+ needsHostname bool // true if proxy requires hostname (socks5h, http), false if can use IP (socks5)
+ proxyNetwork ProxyNetwork
+ proxyType ProxyType
+ allowedDomains []string // glob patterns
+ url string
+ name string
+ stats StatsRegistry
+}
+
type customDialer struct {
- proxyDialer proxy.ContextDialer
- proxyNeedsHostname bool // true if proxy requires hostname (socks5h, http), false if can use IP (socks5)
- client *CustomHTTPClient
- DNSConfig *dns.ClientConfig
- DNSClient dnsExchanger
- DNSRecords *otter.Cache[string, net.IP]
+ proxyDialers []proxyDialerInfo
+ proxyRoundRobinIndex atomic.Uint32
+ allowDirectFallback bool
+ client *CustomHTTPClient
+ DNSConfig *dns.ClientConfig
+ DNSClient dnsExchanger
+ DNSRecords *otter.Cache[string, net.IP]
net.Dialer
disableIPv4 bool
disableIPv6 bool
dnsConcurrency int
dnsRoundRobinIndex atomic.Uint32
+
+ stats StatsRegistry
+ logBackend LogBackend
}
var emptyPayloadDigests = []string{
@@ -88,14 +123,18 @@ var emptyPayloadDigests = []string{
"blake3:af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
}
-func newCustomDialer(httpClient *CustomHTTPClient, proxyURL string, DialTimeout, DNSRecordsTTL, DNSResolutionTimeout time.Duration, DNSCacheSize int, DNSServers []string, DNSConcurrency int, disableIPv4, disableIPv6 bool) (d *customDialer, err error) {
+func newCustomDialer(httpClient *CustomHTTPClient, proxies []ProxyConfig, allowDirectFallback bool, DialTimeout, DNSRecordsTTL, DNSResolutionTimeout time.Duration, DNSCacheSize int, DNSServers []string, DNSConcurrency int, disableIPv4, disableIPv6 bool) (d *customDialer, err error) {
d = new(customDialer)
+ d.stats = httpClient.statsRegistry
+ d.logBackend = httpClient.logBackend
+
d.Timeout = DialTimeout
d.client = httpClient
d.disableIPv4 = disableIPv4
d.disableIPv6 = disableIPv6
d.dnsConcurrency = DNSConcurrency
+ d.allowDirectFallback = allowDirectFallback
DNScache, err := otter.MustBuilder[string, net.IP](DNSCacheSize).
// CollectStats(). // Uncomment this line to enable stats collection, can be useful later on
@@ -121,29 +160,163 @@ func newCustomDialer(httpClient *CustomHTTPClient, proxyURL string, DialTimeout,
Timeout: DNSResolutionTimeout,
}
- if proxyURL != "" {
- u, err := url.Parse(proxyURL)
+ // Initialize all proxies
+ for _, proxyConfig := range proxies {
+ if proxyConfig.URL == "" {
+ continue
+ }
+
+ // Validate that Network is explicitly set
+ if proxyConfig.Network == ProxyNetworkUnset {
+ return nil, fmt.Errorf("proxy %s: Network must be explicitly set to ProxyNetworkAny, ProxyNetworkIPv4, or ProxyNetworkIPv6", proxyConfig.URL)
+ }
+
+ u, err := url.Parse(proxyConfig.URL)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to parse proxy URL %s: %w", proxyConfig.URL, err)
}
var proxyDialer proxy.Dialer
if proxyDialer, err = proxy.FromURL(u, d); err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to create proxy from URL %s: %w", proxyConfig.URL, err)
}
- d.proxyDialer = proxyDialer.(proxy.ContextDialer)
-
// Determine if this proxy requires hostname (remote DNS) or can use IP (local DNS)
// Proxies with remote DNS: socks5h, socks4a, http, https
// Proxies with local DNS: socks5, socks4
- d.proxyNeedsHostname = u.Scheme == "socks5h" || u.Scheme == "socks4a" ||
+ needsHostname := u.Scheme == "socks5h" || u.Scheme == "socks4a" ||
u.Scheme == "http" || u.Scheme == "https"
+
+ d.proxyDialers = append(d.proxyDialers, proxyDialerInfo{
+ dialer: proxyDialer.(proxy.ContextDialer),
+ needsHostname: needsHostname,
+ proxyNetwork: proxyConfig.Network,
+ proxyType: proxyConfig.Type,
+ allowedDomains: proxyConfig.AllowedDomains,
+ url: proxyConfig.URL,
+ name: proxyName(u),
+ stats: httpClient.statsRegistry,
+ })
}
return d, nil
}
+// selectProxy selects an appropriate proxy based on network type, domain, and context flags.
+// Returns nil if no proxy is available or should be used (direct connection).
+// Returns an error if proxies exist but none match the requirements and direct fallback is disabled.
+func (d *customDialer) selectProxy(ctx context.Context, network, address string) (*proxyDialerInfo, error) {
+ // No proxies configured, use direct connection
+ if len(d.proxyDialers) == 0 {
+ return nil, nil
+ }
+
+ // Extract hostname from address for domain matching
+ hostname, _, err := net.SplitHostPort(address)
+ if err != nil {
+ // If no port, treat the whole address as hostname
+ hostname = address
+ }
+
+ // Check if a specific proxy type is requested via context
+ var requestedProxyType *ProxyType
+ if ctx.Value(ContextKeyProxyType) != nil {
+ if val, ok := ctx.Value(ContextKeyProxyType).(ProxyType); ok {
+ requestedProxyType = &val
+ }
+ }
+
+ // Filter eligible proxies
+ var eligible []*proxyDialerInfo
+ for i := range d.proxyDialers {
+ proxy := &d.proxyDialers[i]
+
+ // Filter by proxy type (Mobile, Residential, Datacenter)
+ // If a specific type is requested, only use matching proxies
+ // If no type is requested, only use ProxyTypeAny proxies
+ if requestedProxyType != nil {
+ // Specific type requested: only match that exact type
+ if proxy.proxyType != *requestedProxyType {
+ continue
+ }
+ } else {
+ // No type requested: only use ProxyTypeAny proxies
+ if proxy.proxyType != ProxyTypeAny {
+ continue
+ }
+ }
+
+ // Filter by network type (IPv4/IPv6)
+ switch proxy.proxyNetwork {
+ case ProxyNetworkIPv4:
+ if strings.HasSuffix(network, "6") {
+ continue // Skip IPv6 networks
+ }
+ case ProxyNetworkIPv6:
+ if strings.HasSuffix(network, "4") {
+ continue // Skip IPv4 networks
+ }
+ case ProxyNetworkAny:
+ // Always eligible regardless of network type
+ }
+
+ // Filter by domain patterns
+ if len(proxy.allowedDomains) > 0 {
+ matched := false
+ for _, pattern := range proxy.allowedDomains {
+ // Use filepath.Match for glob pattern matching
+ // Note: filepath.Match doesn't support '**' but supports '*' and '?'
+ if match, _ := filepath.Match(pattern, hostname); match {
+ matched = true
+ break
+ }
+ // Also check if pattern matches a subdomain pattern
+ // For example, "*.example.com" should match "api.example.com"
+ if strings.HasPrefix(pattern, "*.") {
+ suffix := pattern[1:] // Remove the leading '*'
+ if strings.HasSuffix(hostname, suffix) || hostname == suffix[1:] {
+ matched = true
+ break
+ }
+ }
+ }
+ if !matched {
+ continue // Skip if domain doesn't match any pattern
+ }
+ }
+
+ eligible = append(eligible, proxy)
+ }
+
+ // No eligible proxies found
+ if len(eligible) == 0 {
+ if d.allowDirectFallback {
+ d.logBackend.Debug("no eligible proxies found, using direct connection", "network", network, "address", address)
+ return nil, nil // Use direct connection
+ }
+ proxyTypeStr := "any"
+ if requestedProxyType != nil {
+ proxyTypeStr = fmt.Sprintf("%v", *requestedProxyType)
+ }
+ d.logBackend.Error("no eligible proxies found and direct fallback disabled", "network", network, "address", address, "proxyType", proxyTypeStr)
+ return nil, fmt.Errorf("no eligible proxies found for network=%s, address=%s, proxyType=%s and direct fallback is disabled", network, address, proxyTypeStr)
+ }
+
+ // Round-robin selection among eligible proxies
+ startIdx := int(d.proxyRoundRobinIndex.Add(1)-1) % len(eligible)
+ selectedProxy := eligible[startIdx]
+
+ // Update proxy statistics
+ if selectedProxy.stats != nil {
+ selectedProxy.stats.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": selectedProxy.name}).Add(1)
+ selectedProxy.stats.RegisterGauge(proxyLastUsedNanoseconds, proxyLastUsedNanosecondsHelp, []string{"proxy"}).WithLabels(Labels{"proxy": selectedProxy.name}).Set(time.Now().UnixNano())
+ }
+
+ d.logBackend.Debug("proxy selected", "proxy", selectedProxy.name, "network", network, "address", address)
+
+ return selectedProxy, nil
+}
+
type CustomConnection struct {
net.Conn
io.Reader
@@ -235,10 +408,16 @@ func (d *customDialer) CustomDialContext(ctx context.Context, network, address s
return nil, errors.New("no supported network type available")
}
+ // Select appropriate proxy based on context, network type, and domain
+ selectedProxy, err := d.selectProxy(ctx, network, address)
+ if err != nil {
+ return nil, err
+ }
+
var dialAddr string
var IP net.IP
- if d.proxyDialer != nil && d.proxyNeedsHostname {
+ if selectedProxy != nil && selectedProxy.needsHostname {
// Remote DNS proxy (socks5h, socks4a, http, https)
// Skip DNS archiving to avoid privacy leak and ensure accuracy.
// The proxy will handle DNS resolution on its end, and we don't want to:
@@ -250,6 +429,7 @@ func (d *customDialer) CustomDialContext(ctx context.Context, network, address s
// Archive DNS and use resolved IP
IP, _, err = d.archiveDNS(ctx, address)
if err != nil {
+ d.logBackend.Error("DNS resolution failed", "address", address, "error", err)
return nil, err
}
@@ -261,10 +441,19 @@ func (d *customDialer) CustomDialContext(ctx context.Context, network, address s
}
dialAddr = net.JoinHostPort(IP.String(), port)
+ d.logBackend.Debug("DNS resolved", "address", address, "ip", IP.String())
}
- if d.proxyDialer != nil {
- conn, err = d.proxyDialer.DialContext(ctx, network, dialAddr)
+ if selectedProxy != nil {
+ conn, err = selectedProxy.dialer.DialContext(ctx, network, dialAddr)
+ if err != nil {
+ d.logBackend.Error("proxy connection failed", "proxy", selectedProxy.name, "address", dialAddr, "error", err)
+ if selectedProxy.stats != nil {
+ selectedProxy.stats.RegisterCounter(proxyErrorsTotal, proxyErrorsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": selectedProxy.name}).Add(1)
+ }
+ } else {
+ d.logBackend.Debug("connection established via proxy", "proxy", selectedProxy.name, "address", dialAddr)
+ }
} else {
if d.client.randomLocalIP {
localAddr := getLocalAddr(network, IP)
@@ -279,6 +468,11 @@ func (d *customDialer) CustomDialContext(ctx context.Context, network, address s
}
conn, err = d.DialContext(ctx, network, dialAddr)
+ if err != nil {
+ d.logBackend.Error("direct connection failed", "address", dialAddr, "error", err)
+ } else {
+ d.logBackend.Debug("direct connection established", "address", dialAddr)
+ }
}
if err != nil {
@@ -299,11 +493,16 @@ func (d *customDialer) CustomDialTLSContext(ctx context.Context, network, addres
return nil, errors.New("no supported network type available")
}
+ // Select appropriate proxy based on context, network type, and domain
+ selectedProxy, err := d.selectProxy(ctx, network, address)
+ if err != nil {
+ return nil, err
+ }
+
var dialAddr string
var IP net.IP
- var err error
- if d.proxyDialer != nil && d.proxyNeedsHostname {
+ if selectedProxy != nil && selectedProxy.needsHostname {
// Remote DNS proxy (socks5h, socks4a, http, https)
// Skip DNS archiving to avoid privacy leak and ensure accuracy.
// The proxy will handle DNS resolution on its end, and we don't want to:
@@ -315,6 +514,7 @@ func (d *customDialer) CustomDialTLSContext(ctx context.Context, network, addres
// Archive DNS and use resolved IP
IP, _, err = d.archiveDNS(ctx, address)
if err != nil {
+ d.logBackend.Error("DNS resolution failed for TLS connection", "address", address, "error", err)
return nil, err
}
@@ -326,12 +526,16 @@ func (d *customDialer) CustomDialTLSContext(ctx context.Context, network, addres
}
dialAddr = net.JoinHostPort(IP.String(), port)
+ d.logBackend.Debug("DNS resolved for TLS connection", "address", address, "ip", IP.String())
}
var plainConn net.Conn
- if d.proxyDialer != nil {
- plainConn, err = d.proxyDialer.DialContext(ctx, network, dialAddr)
+ if selectedProxy != nil {
+ plainConn, err = selectedProxy.dialer.DialContext(ctx, network, dialAddr)
+ if err != nil && selectedProxy.stats != nil {
+ selectedProxy.stats.RegisterCounter(proxyErrorsTotal, proxyErrorsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": selectedProxy.name}).Add(1)
+ }
} else {
if d.client.randomLocalIP {
localAddr := getLocalAddr(network, IP)
@@ -367,6 +571,15 @@ func (d *customDialer) CustomDialTLSContext(ctx context.Context, network, addres
defer cancel()
if err := tlsConn.HandshakeContext(handshakeCtx); err != nil {
+ // Track TLS handshake errors for proxy connections
+ if selectedProxy != nil {
+ d.logBackend.Error("TLS handshake failed via proxy", "proxy", selectedProxy.name, "address", address, "error", err)
+ if selectedProxy.stats != nil {
+ selectedProxy.stats.RegisterCounter(proxyErrorsTotal, proxyErrorsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": selectedProxy.name}).Add(1)
+ }
+ } else {
+ d.logBackend.Error("TLS handshake failed", "address", address, "error", err)
+ }
closeErr := plainConn.Close()
if closeErr != nil {
return nil, fmt.Errorf("CustomDialTLS: TLS handshake failed and closing plain connection failed: %s", closeErr.Error())
@@ -374,6 +587,12 @@ func (d *customDialer) CustomDialTLSContext(ctx context.Context, network, addres
return nil, fmt.Errorf("CustomDialTLS: TLS handshake failed: %w", err)
}
+ if selectedProxy != nil {
+ d.logBackend.Debug("TLS connection established via proxy", "proxy", selectedProxy.name, "address", address)
+ } else {
+ d.logBackend.Debug("TLS connection established", "address", address)
+ }
+
return d.wrapConnection(ctx, tlsConn, "https"), nil
}
@@ -448,17 +667,11 @@ func (d *customDialer) writeWARCFromConnection(ctx context.Context, reqPipe, res
close(recordChan)
if readErr != nil {
- d.client.ErrChan <- &Error{
- Err: readErr,
- Func: "writeWARCFromConnection",
- }
+ d.logBackend.Error("error reading from connection", "func", "writeWARCFromConnection", "error", readErr)
for record := range recordChan {
if closeErr := record.Content.Close(); closeErr != nil {
- d.client.ErrChan <- &Error{
- Err: closeErr,
- Func: "writeWARCFromConnection",
- }
+ d.logBackend.Error("error closing record content", "func", "writeWARCFromConnection", "error", closeErr)
}
}
@@ -477,14 +690,11 @@ func (d *customDialer) writeWARCFromConnection(ctx context.Context, reqPipe, res
if len(batch.Records) != 2 {
err.Err = errors.New("warc: there was an unspecified problem creating one of the WARC records")
- d.client.ErrChan <- err
+ d.logBackend.Error("failed to create WARC records", "func", err.Func, "error", err.Err)
for _, record := range batch.Records {
if closeErr := record.Content.Close(); closeErr != nil {
- d.client.ErrChan <- &Error{
- Err: closeErr,
- Func: "writeWARCFromConnection",
- }
+ d.logBackend.Error("error closing record content", "func", "writeWARCFromConnection", "error", closeErr)
}
}
@@ -511,7 +721,7 @@ func (d *customDialer) writeWARCFromConnection(ctx context.Context, reqPipe, res
case <-ctx.Done():
return
default:
- if d.proxyDialer == nil {
+ if len(d.proxyDialers) == 0 {
switch addr := conn.RemoteAddr().(type) {
case *net.TCPAddr:
IP := addr.IP.String()
@@ -530,19 +740,13 @@ func (d *customDialer) writeWARCFromConnection(ctx context.Context, reqPipe, res
r.Header.Set("WARC-Target-URI", warcTargetURI)
if _, seekErr := r.Content.Seek(0, 0); seekErr != nil {
- d.client.ErrChan <- &Error{
- Err: seekErr,
- Func: "writeWARCFromConnection",
- }
+ d.logBackend.Error("error seeking record content", "func", "writeWARCFromConnection", "error", seekErr)
return
}
digest, err := GetDigest(r.Content, d.client.DigestAlgorithm)
if err != nil {
- d.client.ErrChan <- &Error{
- Err: err,
- Func: "writeWARCFromConnection",
- }
+ d.logBackend.Error("error calculating digest", "func", "writeWARCFromConnection", "error", err)
return
}
@@ -553,10 +757,7 @@ func (d *customDialer) writeWARCFromConnection(ctx context.Context, reqPipe, res
if r.Header.Get("WARC-Type") == "response" && !slices.Contains(emptyPayloadDigests, r.Header.Get("WARC-Payload-Digest")) {
captureTime, timeConversionErr := time.Parse(time.RFC3339, batch.CaptureTime)
if timeConversionErr != nil {
- d.client.ErrChan <- &Error{
- Err: timeConversionErr,
- Func: "writeWARCFromConnection.timeConversionErr",
- }
+ d.logBackend.Error("error parsing capture time", "func", "writeWARCFromConnection", "error", timeConversionErr)
return
}
d.client.dedupeHashTable.Store(r.Header.Get("WARC-Payload-Digest"), revisitRecord{
@@ -645,8 +846,8 @@ func (d *customDialer) readResponse(ctx context.Context, respPipe *io.PipeReader
if d.client.dedupeOptions.LocalDedupe {
revisit = d.checkLocalRevisit(payloadDigest)
if revisit.targetURI != "" {
- LocalDedupeTotalBytes.Add(int64(revisit.size))
- LocalDedupeTotal.Add(1)
+ d.stats.RegisterCounter(localDedupedBytesTotal, localDedupedBytesTotalHelp, nil).WithLabels(nil).Add(int64(revisit.size))
+ d.stats.RegisterCounter(localDedupedTotal, localDedupedTotalHelp, nil).WithLabels(nil).Add(1)
}
}
@@ -655,8 +856,8 @@ func (d *customDialer) readResponse(ctx context.Context, respPipe *io.PipeReader
if d.client.dedupeOptions.DoppelgangerDedupe && d.client.DigestAlgorithm == SHA1 && revisit.targetURI == "" {
revisit, _ = checkDoppelgangerRevisit(d.client.dedupeOptions.DoppelgangerHost, payloadDigest, warcTargetURI)
if revisit.targetURI != "" {
- DoppelgangerDedupeTotalBytes.Add(bytesCopied)
- DoppelgangerDedupeTotal.Add(1)
+ d.stats.RegisterCounter(doppelgangerDedupedBytesTotal, doppelgangerDedupedBytesTotalHelp, nil).WithLabels(nil).Add(bytesCopied)
+ d.stats.RegisterCounter(doppelgangerDedupedTotal, doppelgangerDedupedTotalHelp, nil).WithLabels(nil).Add(1)
}
}
@@ -664,8 +865,8 @@ func (d *customDialer) readResponse(ctx context.Context, respPipe *io.PipeReader
if d.client.dedupeOptions.CDXDedupe && d.client.DigestAlgorithm == SHA1 && revisit.targetURI == "" {
revisit, _ = checkCDXRevisit(d.client.dedupeOptions.CDXURL, payloadDigest, warcTargetURI, d.client.dedupeOptions.CDXCookie)
if revisit.targetURI != "" {
- CDXDedupeTotalBytes.Add(bytesCopied)
- CDXDedupeTotal.Add(1)
+ d.stats.RegisterCounter(cdxDedupedBytesTotal, cdxDedupedBytesTotalHelp, nil).WithLabels(nil).Add(bytesCopied)
+ d.stats.RegisterCounter(cdxDedupedTotal, cdxDedupedTotalHelp, nil).WithLabels(nil).Add(1)
}
}
}
diff --git a/dialer_test.go b/dialer_test.go
index 147668b..0baff06 100644
--- a/dialer_test.go
+++ b/dialer_test.go
@@ -2,9 +2,11 @@ package warc
import (
"bytes"
+ "context"
"io"
"strings"
"testing"
+ "time"
)
func TestGetNetworkType(t *testing.T) {
@@ -165,3 +167,713 @@ func TestFindEndOfHeadersOffset(t *testing.T) {
})
}
}
+
+func TestProxySelection(t *testing.T) {
+ t.Run("NoProxies", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{},
+ logBackend: &noopLogger{},
+ }
+ proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy != nil {
+ t.Error("expected nil proxy when no proxies configured")
+ }
+ })
+
+ t.Run("IPv4ProxyWithIPv4Network", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkIPv4,
+ proxyType: ProxyTypeAny,
+ url: "socks5://ipv4-proxy:1080",
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+ proxy, err := d.selectProxy(context.Background(), "tcp4", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil {
+ t.Error("expected proxy for IPv4 network with IPv4 proxy")
+ }
+ if proxy != nil && proxy.url != "socks5://ipv4-proxy:1080" {
+ t.Errorf("expected socks5://ipv4-proxy:1080, got %s", proxy.url)
+ }
+ })
+
+ t.Run("IPv4ProxyWithIPv6Network", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkIPv4,
+ proxyType: ProxyTypeAny,
+ url: "socks5://ipv4-proxy:1080",
+ },
+ },
+ logBackend: &noopLogger{},
+ allowDirectFallback: true,
+ }
+ proxy, err := d.selectProxy(context.Background(), "tcp6", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy != nil {
+ t.Error("expected nil proxy for IPv6 network with IPv4 proxy")
+ }
+ })
+
+ t.Run("IPv6ProxyWithIPv6Network", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkIPv6,
+ proxyType: ProxyTypeAny,
+ url: "socks5://ipv6-proxy:1080",
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+ proxy, err := d.selectProxy(context.Background(), "tcp6", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil {
+ t.Error("expected proxy for IPv6 network with IPv6 proxy")
+ }
+ })
+
+ t.Run("DomainFiltering", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ allowedDomains: []string{"*.example.com"},
+ url: "socks5://domain-proxy:1080",
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+
+ // Should match subdomain
+ proxy, err := d.selectProxy(context.Background(), "tcp", "api.example.com:443")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil {
+ t.Error("expected proxy for matching domain")
+ }
+
+ // Should match base domain
+ d.allowDirectFallback = true
+ proxy, err = d.selectProxy(context.Background(), "tcp", "example.com:443")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil {
+ t.Error("expected proxy for base domain")
+ }
+
+ // Should not match different domain
+ proxy, err = d.selectProxy(context.Background(), "tcp", "other.com:443")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy != nil {
+ t.Error("expected nil proxy for non-matching domain")
+ }
+ })
+
+ t.Run("ProxyTypeSelection", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://any-proxy:1080",
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeMobile,
+ url: "socks5://mobile-proxy:1080",
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeResidential,
+ url: "socks5://residential-proxy:1080",
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+
+ // Without proxy type context, should use ProxyTypeAny proxy
+ proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.url != "socks5://any-proxy:1080" {
+ t.Error("expected any-proxy without proxy type context")
+ }
+
+ // With mobile proxy type context, should use mobile proxy
+ ctx := WithProxyType(context.Background(), ProxyTypeMobile)
+ proxy, err = d.selectProxy(ctx, "tcp", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.url != "socks5://mobile-proxy:1080" {
+ t.Error("expected mobile-proxy with mobile proxy type context")
+ }
+
+ // With residential proxy type context, should use residential proxy
+ ctx = WithProxyType(context.Background(), ProxyTypeResidential)
+ proxy, err = d.selectProxy(ctx, "tcp", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.url != "socks5://residential-proxy:1080" {
+ t.Error("expected residential-proxy with residential proxy type context")
+ }
+ })
+
+ t.Run("RoundRobinSelection", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy1:1080",
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy2:1080",
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy3:1080",
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+
+ // Expected order for 3 complete cycles (9 selections)
+ expectedOrder := []string{
+ "socks5://proxy1:1080",
+ "socks5://proxy2:1080",
+ "socks5://proxy3:1080",
+ "socks5://proxy1:1080",
+ "socks5://proxy2:1080",
+ "socks5://proxy3:1080",
+ "socks5://proxy1:1080",
+ "socks5://proxy2:1080",
+ "socks5://proxy3:1080",
+ }
+
+ // Select proxies 9 times and verify sequential round-robin order
+ for i := 0; i < 9; i++ {
+ proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80")
+ if err != nil {
+ t.Errorf("iteration %d: unexpected error: %v", i, err)
+ }
+ if proxy == nil {
+ t.Errorf("iteration %d: expected proxy, got nil", i)
+ } else if proxy.url != expectedOrder[i] {
+ t.Errorf("iteration %d: expected %s, got %s", i, expectedOrder[i], proxy.url)
+ }
+ }
+ })
+
+ t.Run("NoEligibleProxiesWithFallback", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkIPv6,
+ proxyType: ProxyTypeAny,
+ url: "socks5://ipv6-proxy:1080",
+ },
+ },
+ logBackend: &noopLogger{},
+ allowDirectFallback: true,
+ }
+
+ // IPv4 network with IPv6 proxy should use direct connection
+ proxy, err := d.selectProxy(context.Background(), "tcp4", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy != nil {
+ t.Error("expected nil proxy with direct fallback")
+ }
+ })
+
+ t.Run("NoEligibleProxiesWithoutFallback", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkIPv6,
+ proxyType: ProxyTypeAny,
+ url: "socks5://ipv6-proxy:1080",
+ },
+ },
+ logBackend: &noopLogger{},
+ allowDirectFallback: false,
+ }
+
+ // IPv4 network with IPv6 proxy and no fallback should error
+ proxy, err := d.selectProxy(context.Background(), "tcp4", "example.com:80")
+ if err == nil {
+ t.Error("expected error when no eligible proxies and no fallback")
+ }
+ if proxy != nil {
+ t.Error("expected nil proxy")
+ }
+ })
+
+ t.Run("ComplexFiltering", func(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkIPv4,
+ proxyType: ProxyTypeAny,
+ allowedDomains: []string{"*.api.example.com"},
+ url: "socks5://api-ipv4-proxy:1080",
+ },
+ {
+ proxyNetwork: ProxyNetworkIPv6,
+ proxyType: ProxyTypeAny,
+ allowedDomains: []string{"*.media.example.com"},
+ url: "socks5://media-ipv6-proxy:1080",
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeMobile,
+ url: "socks5://mobile-proxy:1080",
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeResidential,
+ url: "socks5://residential-proxy:1080",
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+
+ // Test IPv4 API domain
+ proxy, err := d.selectProxy(context.Background(), "tcp4", "service.api.example.com:443")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.url != "socks5://api-ipv4-proxy:1080" {
+ t.Error("expected api-ipv4-proxy for IPv4 API domain")
+ }
+
+ // Test IPv6 media domain
+ proxy, err = d.selectProxy(context.Background(), "tcp6", "cdn.media.example.com:443")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil {
+ t.Error("expected proxy for IPv6 media domain, got nil")
+ } else if proxy.url != "socks5://media-ipv6-proxy:1080" {
+ t.Errorf("expected media-ipv6-proxy for IPv6 media domain, got %s", proxy.url)
+ }
+
+ // Test mobile proxy type context
+ ctx := WithProxyType(context.Background(), ProxyTypeMobile)
+ proxy, err = d.selectProxy(ctx, "tcp", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.url != "socks5://mobile-proxy:1080" {
+ t.Error("expected mobile-proxy with mobile proxy type context")
+ }
+
+ // Test residential proxy type context
+ ctx = WithProxyType(context.Background(), ProxyTypeResidential)
+ proxy, err = d.selectProxy(ctx, "tcp", "example.com:80")
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.url != "socks5://residential-proxy:1080" {
+ t.Error("expected residential-proxy with residential proxy type context")
+ }
+ })
+}
+
+// TestProxyStatsMetricNames tests that proxy metrics use labels to distinguish between proxies
+func TestProxyStatsMetricNames(t *testing.T) {
+ registry := newLocalRegistry()
+
+ tests := []struct {
+ name string
+ proxyName string
+ }{
+ {
+ name: "simple proxy",
+ proxyName: "example_com_8080",
+ },
+ {
+ name: "IPv4 proxy",
+ proxyName: "192_168_1_1_3128",
+ },
+ {
+ name: "IPv6 proxy",
+ proxyName: "2001_db8__1_8080",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Register counters for this proxy using labels
+ requestsCounter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": tt.proxyName})
+ errorsCounter := registry.RegisterCounter(proxyErrorsTotal, proxyErrorsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": tt.proxyName})
+ lastUsedGauge := registry.RegisterGauge(proxyLastUsedNanoseconds, proxyLastUsedNanosecondsHelp, []string{"proxy"}).WithLabels(Labels{"proxy": tt.proxyName})
+
+ // Verify counters start at 0
+ if requestsCounter.Get() != 0 {
+ t.Errorf("Expected requests counter to start at 0, got %d", requestsCounter.Get())
+ }
+ if errorsCounter.Get() != 0 {
+ t.Errorf("Expected errors counter to start at 0, got %d", errorsCounter.Get())
+ }
+
+ // Increment counters
+ requestsCounter.Add(5)
+ errorsCounter.Add(2)
+ lastUsedGauge.Set(123456789)
+
+ // Verify values are independent per proxy
+ if requestsCounter.Get() != 5 {
+ t.Errorf("Expected requests counter to be 5, got %d", requestsCounter.Get())
+ }
+ if errorsCounter.Get() != 2 {
+ t.Errorf("Expected errors counter to be 2, got %d", errorsCounter.Get())
+ }
+ if lastUsedGauge.Get() != 123456789 {
+ t.Errorf("Expected last used gauge to be 123456789, got %d", lastUsedGauge.Get())
+ }
+ })
+ }
+
+ // Verify that different proxy labels create independent counters
+ proxy1Counter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": "example_com_8080"})
+ proxy2Counter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": "192_168_1_1_3128"})
+
+ if proxy1Counter.Get() != 5 {
+ t.Errorf("Expected proxy1 counter to be 5 (from earlier test), got %d", proxy1Counter.Get())
+ }
+ if proxy2Counter.Get() != 5 {
+ t.Errorf("Expected proxy2 counter to be 5 (from earlier test), got %d", proxy2Counter.Get())
+ }
+}
+
+// TestProxyStatsRequestCount tests that proxy request counts are incremented correctly
+func TestProxyStatsRequestCount(t *testing.T) {
+ registry := newLocalRegistry()
+
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy1:1080",
+ name: "proxy1_1080",
+ stats: registry,
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy2:1080",
+ name: "proxy2_1080",
+ stats: registry,
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+
+ // Select proxies multiple times and verify request counts
+ for i := 0; i < 5; i++ {
+ proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80")
+ if err != nil {
+ t.Fatalf("iteration %d: unexpected error: %v", i, err)
+ }
+ if proxy == nil {
+ t.Fatalf("iteration %d: expected proxy, got nil", i)
+ }
+ }
+
+ // Verify both proxies were used (round-robin)
+ // With 5 selections: proxy1 should be used 3 times, proxy2 should be used 2 times
+ proxy1Counter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": "proxy1_1080"})
+ proxy2Counter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": "proxy2_1080"})
+
+ if proxy1Counter.Get() != 3 {
+ t.Errorf("Expected proxy1 request count 3, got %d", proxy1Counter.Get())
+ }
+ if proxy2Counter.Get() != 2 {
+ t.Errorf("Expected proxy2 request count 2, got %d", proxy2Counter.Get())
+ }
+}
+
+// TestProxyStatsLastUsed tests that proxy last used timestamps are updated
+func TestProxyStatsLastUsed(t *testing.T) {
+ registry := newLocalRegistry()
+
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy:1080",
+ name: "proxy_1080",
+ stats: registry,
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+
+ // Record time before selection
+ timeBefore := time.Now().UnixNano()
+
+ // Select proxy
+ proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if proxy == nil {
+ t.Fatal("expected proxy, got nil")
+ }
+
+ // Record time after selection
+ timeAfter := time.Now().UnixNano()
+
+ // Verify last used timestamp is within expected range
+ lastUsedGauge := registry.RegisterGauge(proxyLastUsedNanoseconds, proxyLastUsedNanosecondsHelp, []string{"proxy"}).WithLabels(Labels{"proxy": "proxy_1080"})
+ lastUsed := lastUsedGauge.Get()
+
+ if lastUsed < timeBefore || lastUsed > timeAfter {
+ t.Errorf("Expected last used timestamp between %d and %d, got %d", timeBefore, timeAfter, lastUsed)
+ }
+
+ // Wait a bit and select again
+ time.Sleep(10 * time.Millisecond)
+ timeBeforeSecond := time.Now().UnixNano()
+
+ proxy, err = d.selectProxy(context.Background(), "tcp", "example.com:80")
+ if err != nil {
+ t.Fatalf("unexpected error on second selection: %v", err)
+ }
+ if proxy == nil {
+ t.Fatal("expected proxy on second selection, got nil")
+ }
+
+ timeAfterSecond := time.Now().UnixNano()
+
+ // Verify last used timestamp was updated
+ lastUsedSecond := lastUsedGauge.Get()
+ if lastUsedSecond < timeBeforeSecond || lastUsedSecond > timeAfterSecond {
+ t.Errorf("Expected updated last used timestamp between %d and %d, got %d", timeBeforeSecond, timeAfterSecond, lastUsedSecond)
+ }
+ if lastUsedSecond <= lastUsed {
+ t.Errorf("Expected last used timestamp to be updated, but %d <= %d", lastUsedSecond, lastUsed)
+ }
+}
+
+// TestProxyStatsWithNilRegistry tests that proxy selection works when stats registry is nil
+func TestProxyStatsWithNilRegistry(t *testing.T) {
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy:1080",
+ name: "proxy_1080",
+ stats: nil, // No stats registry
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+
+ // Should not panic when stats is nil
+ proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if proxy == nil {
+ t.Fatal("expected proxy, got nil")
+ }
+}
+
+// TestProxyStatsMultipleProxiesRoundRobin tests that stats are correctly tracked across multiple proxies in round-robin
+func TestProxyStatsMultipleProxiesRoundRobin(t *testing.T) {
+ registry := newLocalRegistry()
+
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy1:1080",
+ name: "proxy1",
+ stats: registry,
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy2:1080",
+ name: "proxy2",
+ stats: registry,
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ url: "socks5://proxy3:1080",
+ name: "proxy3",
+ stats: registry,
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+
+ // Select proxies 12 times (4 complete round-robin cycles)
+ for i := 0; i < 12; i++ {
+ proxy, err := d.selectProxy(context.Background(), "tcp", "example.com:80")
+ if err != nil {
+ t.Fatalf("iteration %d: unexpected error: %v", i, err)
+ }
+ if proxy == nil {
+ t.Fatalf("iteration %d: expected proxy, got nil", i)
+ }
+ }
+
+ // Verify each proxy was used exactly 4 times
+ for i := 1; i <= 3; i++ {
+ proxyName := "proxy" + string(rune('0'+i))
+ counter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": proxyName})
+
+ expectedCount := int64(4)
+ if counter.Get() != expectedCount {
+ t.Errorf("Expected %s request count %d, got %d", proxyName, expectedCount, counter.Get())
+ }
+ }
+}
+
+// TestProxyStatsWithDomainFiltering tests that stats are only updated for eligible proxies
+func TestProxyStatsWithDomainFiltering(t *testing.T) {
+ registry := newLocalRegistry()
+
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ allowedDomains: []string{"*.example.com"},
+ url: "socks5://example-proxy:1080",
+ name: "example_proxy",
+ stats: registry,
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeAny,
+ allowedDomains: []string{"*.test.com"},
+ url: "socks5://test-proxy:1080",
+ name: "test_proxy",
+ stats: registry,
+ },
+ },
+ logBackend: &noopLogger{},
+ allowDirectFallback: true,
+ }
+
+ // Select proxy for example.com domain - should use example-proxy
+ proxy, err := d.selectProxy(context.Background(), "tcp", "api.example.com:443")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.name != "example_proxy" {
+ t.Fatal("expected example-proxy")
+ }
+
+ // Select proxy for test.com domain - should use test-proxy
+ proxy, err = d.selectProxy(context.Background(), "tcp", "api.test.com:443")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.name != "test_proxy" {
+ t.Fatal("expected test-proxy")
+ }
+
+ // Verify stats
+ exampleCounter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": "example_proxy"})
+ testCounter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": "test_proxy"})
+
+ if exampleCounter.Get() != 1 {
+ t.Errorf("Expected example_proxy request count 1, got %d", exampleCounter.Get())
+ }
+ if testCounter.Get() != 1 {
+ t.Errorf("Expected test_proxy request count 1, got %d", testCounter.Get())
+ }
+}
+
+// TestProxyStatsProxyTypeFiltering tests that stats work correctly with proxy type filtering
+func TestProxyStatsProxyTypeFiltering(t *testing.T) {
+ registry := newLocalRegistry()
+
+ d := &customDialer{
+ proxyDialers: []proxyDialerInfo{
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeMobile,
+ url: "socks5://mobile-proxy:1080",
+ name: "mobile_proxy",
+ stats: registry,
+ },
+ {
+ proxyNetwork: ProxyNetworkAny,
+ proxyType: ProxyTypeResidential,
+ url: "socks5://residential-proxy:1080",
+ name: "residential_proxy",
+ stats: registry,
+ },
+ },
+ logBackend: &noopLogger{},
+ }
+
+ // Select mobile proxy
+ ctx := WithProxyType(context.Background(), ProxyTypeMobile)
+ proxy, err := d.selectProxy(ctx, "tcp", "example.com:80")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.proxyType != ProxyTypeMobile {
+ t.Fatal("expected mobile proxy")
+ }
+
+ // Select residential proxy
+ ctx = WithProxyType(context.Background(), ProxyTypeResidential)
+ proxy, err = d.selectProxy(ctx, "tcp", "example.com:80")
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if proxy == nil || proxy.proxyType != ProxyTypeResidential {
+ t.Fatal("expected residential proxy")
+ }
+
+ // Verify stats
+ mobileCounter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": "mobile_proxy"})
+ residentialCounter := registry.RegisterCounter(proxyRequestsTotal, proxyRequestsTotalHelp, []string{"proxy"}).WithLabels(Labels{"proxy": "residential_proxy"})
+
+ if mobileCounter.Get() != 1 {
+ t.Errorf("Expected mobile_proxy request count 1, got %d", mobileCounter.Get())
+ }
+ if residentialCounter.Get() != 1 {
+ t.Errorf("Expected residential_proxy request count 1, got %d", residentialCounter.Get())
+ }
+}
diff --git a/doc/README.md b/doc/README.md
new file mode 100644
index 0000000..339ddf2
--- /dev/null
+++ b/doc/README.md
@@ -0,0 +1,234 @@
+# gowarc Architecture Documentation
+
+This directory contains comprehensive architectural documentation for the gowarc library, a sophisticated web archiving system that captures HTTP/HTTPS traffic and stores it in WARC format.
+
+## Documentation Overview
+
+### 📋 [Architecture Overview](architecture-overview.md)
+**Start here** for a high-level understanding of the system.
+
+**Contents:**
+- High-level component architecture diagram
+- Core data structures and their relationships
+- Component responsibilities
+- Threading model and concurrency patterns
+- Configuration flow
+- Metrics collection points
+- Logging events catalog
+
+**Best for:**
+- New contributors understanding the system
+- Architects evaluating the design
+- Developers planning new features
+
+---
+
+### 🔄 [Request Flow](request-flow.md)
+**Deep dive** into a complete request lifecycle.
+
+**Example traced:** HTTPS request via IPv6 residential proxy → WARC file
+
+**Contents:**
+- Complete 67-step sequence diagram
+- Detailed breakdown of each phase:
+ - Request initiation
+ - Network type selection
+ - Proxy selection algorithm
+ - DNS resolution
+ - Connection establishment (TLS handshake)
+ - Connection wrapping for capture
+ - Request/response parsing
+ - Deduplication logic
+ - Batch assembly
+ - WARC file writing
+- Final WARC file structure
+- Performance characteristics
+
+**Best for:**
+- Understanding data flow through the system
+- Debugging request issues
+- Learning how proxies and DNS work
+- Understanding WARC format generation
+
+---
+
+### 🔧 [Component Interactions](component-interactions.md)
+**Subsystem-level** architecture and interfaces.
+
+**Contents:**
+- 7 major subsystems:
+ 1. Client Subsystem
+ 2. Network Subsystem
+ 3. Capture Subsystem
+ 4. Deduplication Subsystem
+ 5. WARC Writing Subsystem
+ 6. Observability Subsystem
+ 7. Storage Subsystem
+- Detailed interaction diagrams for each
+- Inter-subsystem communication patterns
+- Synchronization primitives used
+- Context propagation
+- Advanced features:
+ - Synchronous WARC writes
+ - Connection inspection
+ - Random local IP
+ - Custom TLS fingerprinting
+- Error handling patterns
+- Performance optimizations
+
+**Best for:**
+- Understanding specific subsystems
+- Modifying existing components
+- Adding new features
+- Performance tuning
+
+---
+
+## Quick Reference
+
+### Key Components by File
+
+| Component | File | Description |
+|-----------|------|-------------|
+| CustomHTTPClient | `client.go` | Main entry point, extends http.Client |
+| customDialer | `dialer.go` | Connection establishment, proxy selection, DNS |
+| customTransport | `dialer.go` | HTTP transport with compression handling |
+| CustomConnection | `dialer.go` | Wrapped connection for byte capture |
+| RecordBatch | `write.go` | Groups related WARC records |
+| Writer | `write.go` | WARC file writer with compression |
+| recordWriter | `warc.go` | Goroutine that writes to files |
+| DNS resolver | `dns.go` | Concurrent DNS lookup with caching |
+| SpooledTempFile | `utils.go` | RAM/disk hybrid temp storage |
+| StatsRegistry | `stats.go` | Metrics interface |
+| LogBackend | `logging.go` | Logging interface |
+
+### Key Data Flows
+
+```
+1. HTTP Request Flow:
+ App → Client → Transport → Dialer → DNS/Proxy → Connection → Server
+
+2. Response Capture Flow:
+ Server → Connection → TeeReader → App
+ └→ Pipe → Parser → Batch → Writer → Disk
+
+3. Deduplication Flow:
+ Response → Digest → Local Cache ──(miss)→ Doppelganger API ──(miss)→ CDX API
+ └(hit)→ Revisit Record
+```
+
+### Important Patterns
+
+1. **Goroutine Coordination**
+ - WaitGroup for active requests
+ - Buffered channels for async communication
+ - Feedback channels for sync operations
+
+2. **Resource Management**
+ - SpooledTempFile for memory efficiency
+ - Connection pooling disabled (clean boundaries)
+ - File rotation based on size
+
+3. **Observability**
+ - Structured logging at all layers
+ - Metrics for data written, dedupe stats, proxy usage
+ - Optional custom stats/log backends
+
+4. **Fault Tolerance**
+ - Proxy fallback to direct connection
+ - Graceful error handling (log and continue)
+ - DNS failure handling
+
+## Diagram Notation
+
+All diagrams use Mermaid syntax and follow these conventions:
+
+**Colors:**
+- 🔵 Blue (`#e3f2fd`) - Client/Transport layer
+- 🟡 Yellow (`#fff9c4`) - Stats/Logging/Observability
+- 🟠 Orange (`#fff4e1`) - Network/Dialer layer
+- 🔴 Red (`#ffe1e1`) - Connection/Capture layer
+- 🟢 Green (`#e1ffe1`) - WARC Writing/Batch layer
+- 🟣 Purple (`#f0e1ff`) - Writer/Compression layer
+
+**Arrows:**
+- `→` Solid: Synchronous call/data flow
+- `-.→` Dashed: Async/optional interaction
+- `-->>` Return: Function return or response
+
+**Shapes:**
+- Rectangle: Component/process
+- Cylinder: Data storage
+- Diamond: Decision point
+- Circle: Start/end state
+
+## How to Use This Documentation
+
+### For New Contributors:
+1. Read [Architecture Overview](architecture-overview.md) first
+2. Follow an example in [Request Flow](request-flow.md)
+3. Deep dive into relevant subsystems in [Component Interactions](component-interactions.md)
+
+### For Bug Fixing:
+1. Identify the subsystem in [Component Interactions](component-interactions.md)
+2. Trace the flow in [Request Flow](request-flow.md)
+3. Check metrics/logging in [Architecture Overview](architecture-overview.md)
+
+### For Feature Development:
+1. Understand affected subsystems in [Component Interactions](component-interactions.md)
+2. Review interfaces in [Architecture Overview](architecture-overview.md)
+3. Plan data flow using [Request Flow](request-flow.md) as reference
+
+### For Performance Tuning:
+1. Check "Performance Characteristics" in [Request Flow](request-flow.md)
+2. Review "Performance Optimizations" in [Component Interactions](component-interactions.md)
+3. Identify metrics in [Architecture Overview](architecture-overview.md)
+
+## Viewing Mermaid Diagrams
+
+The documentation uses Mermaid for diagrams. To view them:
+
+**GitHub:** Renders natively in `.md` files
+
+**VS Code:** Install "Markdown Preview Mermaid Support" extension
+
+**Command Line:**
+```bash
+# Install mermaid CLI
+npm install -g @mermaid-js/mermaid-cli
+
+# Convert to PNG
+mmdc -i doc/architecture-overview.md -o doc/architecture-overview.png
+```
+
+**Online:** Copy diagram code to https://mermaid.live/
+
+## Contributing to Documentation
+
+When modifying the library:
+
+1. **Update diagrams** if component interactions change
+2. **Add new subsystems** to component-interactions.md
+3. **Document new metrics/logs** in architecture-overview.md
+4. **Trace new features** in request-flow.md if they affect the main path
+
+Keep diagrams:
+- ✅ Up-to-date with code
+- ✅ Consistent in style/notation
+- ✅ Focused on one concept per diagram
+- ✅ Annotated with file/line references
+
+## Additional Resources
+
+- **Main README:** `../README.md` - Usage examples and API documentation
+- **Test files:** `../*_test.go` - Concrete usage examples
+- **Godoc:** Run `godoc -http=:6060` and visit http://localhost:6060/pkg/
+
+## Questions or Issues?
+
+If you find the documentation unclear or incorrect:
+1. Open an issue describing the confusion
+2. Suggest improvements or corrections
+3. Submit a PR with documentation fixes
+
+Good documentation is essential for a complex system like gowarc. Your feedback helps improve it!
diff --git a/doc/architecture-overview.md b/doc/architecture-overview.md
new file mode 100644
index 0000000..b6e9937
--- /dev/null
+++ b/doc/architecture-overview.md
@@ -0,0 +1,355 @@
+# Go WARC Library - Architecture Overview
+
+This document provides a comprehensive overview of the gowarc library architecture and component interactions.
+
+## High-Level Component Architecture
+
+```mermaid
+graph TB
+ subgraph "User Application"
+ APP[Application Code]
+ end
+
+ subgraph "HTTP Client Layer"
+ CLIENT[CustomHTTPClient
- http.Client wrapper
- Dedup hash table
- WARC writer channel
- Stats/Logging]
+ TRANSPORT[customTransport
- Force gzip header
- Decompress responses]
+ DIALER[customDialer
- Proxy selection
- DNS resolution
- Connection wrapping]
+ end
+
+ subgraph "Network Layer"
+ DNS[DNS Resolver
- Concurrent lookup
- TTL cache
- WARC recording]
+ PROXY[Proxy Selection
- Type filtering
- Network filtering
- Domain filtering
- Round-robin LB]
+ CONN[CustomConnection
- Bidirectional capture
- TeeReader/MultiWriter
- Read deadline]
+ end
+
+ subgraph "Capture Layer"
+ PIPES[IO Pipes
- Request pipe
- Response pipe]
+ PARSE[HTTP Parser
- Request/Response
- Extract metadata
- Calculate digests]
+ DEDUPE[Deduplication
- Local hash table
- Doppelganger API
- CDX API]
+ end
+
+ subgraph "WARC Writing Pipeline"
+ BATCH[RecordBatch Channel
- Buffered channel
- Multiple writers]
+ WRITER[recordWriter Goroutines
- Pool of writers
- File rotation
- Compression]
+ DISK[WARC Files
- .warc.gz/.warc.zst
- Auto-rotation
- Sequential naming]
+ end
+
+ subgraph "Supporting Systems"
+ STATS[Stats Registry
- Counters
- Gauges
- Histograms]
+ LOG[Log Backend
- slog interface
- Debug/Info/Warn/Error]
+ TEMP[Temp File System
- SpooledTempFile
- RAM → Disk threshold]
+ end
+
+ APP --> CLIENT
+ CLIENT --> TRANSPORT
+ TRANSPORT --> DIALER
+
+ DIALER --> DNS
+ DIALER --> PROXY
+ DIALER --> CONN
+
+ CONN --> PIPES
+ PIPES --> PARSE
+ PARSE --> DEDUPE
+
+ DEDUPE --> BATCH
+ BATCH --> WRITER
+ WRITER --> DISK
+
+ CLIENT -.-> STATS
+ CLIENT -.-> LOG
+ DIALER -.-> STATS
+ DIALER -.-> LOG
+ PARSE -.-> TEMP
+ WRITER -.-> STATS
+ WRITER -.-> LOG
+
+ style CLIENT fill:#e1f5ff
+ style DIALER fill:#fff4e1
+ style CONN fill:#ffe1e1
+ style BATCH fill:#e1ffe1
+ style WRITER fill:#f0e1ff
+```
+
+## Core Data Structures
+
+```mermaid
+classDiagram
+ class CustomHTTPClient {
+ +http.Client
+ +WARCWriter chan RecordBatch
+ +dedupeHashTable sync.Map
+ +WaitGroup
+ +statsRegistry StatsRegistry
+ +logBackend LogBackend
+ +TempDir string
+ +DigestAlgorithm
+ +Do(req) Response
+ +Close() error
+ }
+
+ class customDialer {
+ +proxyDialers []proxyDialerInfo
+ +proxyRoundRobinIndex atomic.Uint32
+ +DNSRecords Cache
+ +allowDirectFallback bool
+ +disableIPv4 bool
+ +disableIPv6 bool
+ +CustomDialContext(ctx, network, addr) Conn
+ +CustomDialTLSContext(ctx, network, addr) Conn
+ }
+
+ class CustomConnection {
+ +net.Conn
+ +io.Reader (TeeReader)
+ +io.Writer (MultiWriter)
+ +closers []PipeWriter
+ +connReadDeadline Duration
+ +Read(b) int
+ +Write(b) int
+ +Close() error
+ }
+
+ class RecordBatch {
+ +Records []Record
+ +CaptureTime string
+ +FeedbackChan chan
+ }
+
+ class Record {
+ +Header map~string~string
+ +Content ReadWriteSeekCloser
+ +Version string
+ +Offset int64
+ +Size int64
+ }
+
+ class Writer {
+ +FileWriter bufio.Writer
+ +GZIPWriter GzipWriter
+ +ZSTDWriter Encoder
+ +FileName string
+ +Compression string
+ +WriteRecord(record) error
+ }
+
+ class ProxyConfig {
+ +URL string
+ +Network ProxyNetwork
+ +Type ProxyType
+ +AllowedDomains []string
+ }
+
+ CustomHTTPClient --> customDialer : uses
+ CustomHTTPClient --> RecordBatch : sends
+ customDialer --> CustomConnection : creates
+ customDialer --> ProxyConfig : configures
+ CustomConnection --> RecordBatch : captured as
+ RecordBatch --> Record : contains
+ Writer --> Record : writes
+```
+
+## Component Responsibilities
+
+### CustomHTTPClient
+- **Purpose**: Main entry point for making HTTP requests with WARC recording
+- **Key Features**:
+ - Extends standard `http.Client`
+ - Manages deduplication hash table
+ - Coordinates WARC writer goroutines
+ - Tracks active requests with WaitGroup
+ - Integrates stats and logging
+
+### customDialer
+- **Purpose**: Establishes network connections with proxy support and DNS resolution
+- **Key Features**:
+ - Proxy selection and filtering (type, network, domain)
+ - Concurrent DNS resolution with caching
+ - Connection wrapping for byte capture
+ - TLS handshake with custom fingerprinting
+ - Load balancing across proxies
+
+### CustomConnection
+- **Purpose**: Transparent connection wrapper that captures all traffic
+- **Key Features**:
+ - Bidirectional byte capture using io.Pipe
+ - TeeReader for response interception
+ - MultiWriter for request interception
+ - Configurable read deadlines
+ - Automatic goroutine launch for WARC creation
+
+### RecordBatch & Record
+- **Purpose**: Represent HTTP request/response pairs for archival
+- **Key Features**:
+ - Groups related records (request + response)
+ - Shared capture timestamp
+ - Optional feedback channel for synchronous writes
+ - SpooledTempFile content for memory efficiency
+
+### Writer & RotatorSettings
+- **Purpose**: Write WARC records to compressed files with rotation
+- **Key Features**:
+ - Multi-threaded writer pool
+ - Automatic file rotation based on size
+ - GZIP/ZSTD compression support
+ - Dictionary compression for ZSTD
+ - Sequential file naming with collision avoidance
+
+## Threading Model
+
+```mermaid
+sequenceDiagram
+ participant App as Application
+ participant Client as CustomHTTPClient
+ participant Dialer as customDialer
+ participant Conn as CustomConnection
+ participant WG as writeWARCFromConnection
+ participant Writers as recordWriter Pool
+ participant Disk as WARC Files
+
+ App->>Client: Do(request)
+ Client->>Dialer: DialContext/DialTLSContext
+ Dialer->>Conn: Create wrapped connection
+
+ activate Conn
+ Conn->>WG: Launch goroutine
+ activate WG
+ Conn-->>Dialer: Return connection
+
+ Dialer-->>Client: Return connection
+ Client->>App: Return response
+
+ Note over App,Conn: App reads response body
+
+ WG->>WG: Parse request in goroutine
+ WG->>WG: Parse response in goroutine
+ WG->>WG: Calculate digests
+ WG->>WG: Check deduplication
+ WG->>Client: Send RecordBatch to channel
+ deactivate WG
+ deactivate Conn
+
+ Client->>Writers: RecordBatch via channel
+
+ activate Writers
+ Writers->>Writers: Check file size, rotate if needed
+ Writers->>Disk: Write compressed records
+ Writers-->>Client: Signal via FeedbackChan (optional)
+ deactivate Writers
+```
+
+## Configuration Flow
+
+```mermaid
+graph LR
+ subgraph "HTTPClientSettings"
+ RS[RotatorSettings]
+ PROXIES[Proxies Config]
+ DNS_CFG[DNS Config]
+ DEDUPE[Dedupe Options]
+ TIMEOUTS[Timeouts]
+ FLAGS[Feature Flags]
+ STATS_CFG[Stats Registry]
+ LOG_CFG[Log Backend]
+ end
+
+ subgraph "Initialization"
+ INIT[NewWARCWritingHTTPClient]
+ end
+
+ subgraph "Created Components"
+ CLIENT[CustomHTTPClient]
+ DIALER_INST[customDialer]
+ TRANSPORT_INST[customTransport]
+ ROTATOR[recordWriter Pool]
+ end
+
+ RS --> INIT
+ PROXIES --> INIT
+ DNS_CFG --> INIT
+ DEDUPE --> INIT
+ TIMEOUTS --> INIT
+ FLAGS --> INIT
+ STATS_CFG --> INIT
+ LOG_CFG --> INIT
+
+ INIT --> CLIENT
+ INIT --> DIALER_INST
+ INIT --> TRANSPORT_INST
+ INIT --> ROTATOR
+
+ CLIENT --> DIALER_INST
+ CLIENT --> TRANSPORT_INST
+ CLIENT --> ROTATOR
+
+ style INIT fill:#ffdddd
+ style CLIENT fill:#ddffdd
+```
+
+## Metrics Collection Points
+
+```mermaid
+graph TB
+ subgraph "Request Metrics"
+ PROXY_REQ[proxy_requests_total
Label: proxy name]
+ PROXY_ERR[proxy_errors_total
Label: proxy name]
+ PROXY_LAST[proxy_last_used_nanoseconds
Label: proxy name]
+ end
+
+ subgraph "Deduplication Metrics"
+ LOCAL_COUNT[local_deduped_total]
+ LOCAL_BYTES[local_deduped_bytes_total]
+ DOPPEL_COUNT[doppelganger_deduped_total]
+ DOPPEL_BYTES[doppelganger_deduped_bytes_total]
+ CDX_COUNT[cdx_deduped_total]
+ CDX_BYTES[cdx_deduped_bytes_total]
+ end
+
+ subgraph "Writing Metrics"
+ TOTAL_WRITTEN[total_data_written]
+ end
+
+ subgraph "StatsRegistry"
+ REGISTRY[User-provided or
Local Registry]
+ end
+
+ PROXY_REQ --> REGISTRY
+ PROXY_ERR --> REGISTRY
+ PROXY_LAST --> REGISTRY
+ LOCAL_COUNT --> REGISTRY
+ LOCAL_BYTES --> REGISTRY
+ DOPPEL_COUNT --> REGISTRY
+ DOPPEL_BYTES --> REGISTRY
+ CDX_COUNT --> REGISTRY
+ CDX_BYTES --> REGISTRY
+ TOTAL_WRITTEN --> REGISTRY
+
+ style REGISTRY fill:#ffffdd
+```
+
+## Logging Events
+
+The library emits structured logs at various levels:
+
+### Debug Level
+- Proxy selection decisions
+- DNS resolution results
+- Connection establishment details
+- TLS handshake success
+
+### Info Level
+- WARC file creation
+- WARC file rotation
+- WARC writer shutdown
+
+### Error Level
+- DNS resolution failures
+- Proxy connection failures
+- Direct connection failures
+- TLS handshake failures
+- WARC record writing errors
+- Digest calculation errors
+- HTTP parsing errors
+- DiscardHook rejections
+
+All log events include contextual key-value pairs for filtering and debugging.
diff --git a/doc/component-interactions.md b/doc/component-interactions.md
new file mode 100644
index 0000000..92c5070
--- /dev/null
+++ b/doc/component-interactions.md
@@ -0,0 +1,1019 @@
+# Component Interactions and Subsystems
+
+This document breaks down the gowarc library into logical subsystems and shows how they interact with each other.
+
+## Subsystem Architecture
+
+```mermaid
+graph TB
+ subgraph "Client Subsystem"
+ direction TB
+ CLIENT[CustomHTTPClient]
+ SETTINGS[HTTPClientSettings]
+ TRANSPORT[customTransport]
+ SETTINGS --> CLIENT
+ CLIENT --> TRANSPORT
+ end
+
+ subgraph "Network Subsystem"
+ direction TB
+ DIALER[customDialer]
+ PROXY[Proxy Selector]
+ DNS[DNS Resolver]
+ CONN[CustomConnection]
+ DIALER --> PROXY
+ DIALER --> DNS
+ DIALER --> CONN
+ end
+
+ subgraph "Capture Subsystem"
+ direction TB
+ PIPES[IO Pipes]
+ PARSER[HTTP Parser]
+ METADATA[Metadata Extractor]
+ PIPES --> PARSER
+ PARSER --> METADATA
+ end
+
+ subgraph "Deduplication Subsystem"
+ direction TB
+ LOCAL[Local Hash Table]
+ DOPPEL[Doppelganger API]
+ CDX[CDX API]
+ DEDUPE_LOGIC[Dedupe Logic]
+ DEDUPE_LOGIC --> LOCAL
+ DEDUPE_LOGIC --> DOPPEL
+ DEDUPE_LOGIC --> CDX
+ end
+
+ subgraph "WARC Writing Subsystem"
+ direction TB
+ BATCH[RecordBatch Channel]
+ ROTATOR[File Rotator]
+ COMPRESS[Compression]
+ WRITER[File Writer]
+ BATCH --> ROTATOR
+ ROTATOR --> COMPRESS
+ COMPRESS --> WRITER
+ end
+
+ subgraph "Observability Subsystem"
+ direction TB
+ STATS[Stats Registry]
+ LOG[Log Backend]
+ end
+
+ subgraph "Storage Subsystem"
+ direction TB
+ SPOOL[SpooledTempFile]
+ DISK[Disk I/O]
+ SPOOL --> DISK
+ end
+
+ TRANSPORT --> DIALER
+ CONN --> PIPES
+ METADATA --> DEDUPE_LOGIC
+ DEDUPE_LOGIC --> BATCH
+
+ CLIENT -.-> STATS
+ CLIENT -.-> LOG
+ DIALER -.-> STATS
+ DIALER -.-> LOG
+ DEDUPE_LOGIC -.-> STATS
+ ROTATOR -.-> STATS
+ ROTATOR -.-> LOG
+
+ PARSER -.-> SPOOL
+ BATCH -.-> SPOOL
+
+ style CLIENT fill:#e3f2fd
+ style DIALER fill:#fff3e0
+ style PIPES fill:#fce4ec
+ style DEDUPE_LOGIC fill:#f3e5f5
+ style BATCH fill:#e8f5e9
+ style STATS fill:#fff9c4
+ style SPOOL fill:#ffe0b2
+```
+
+## Subsystem Details
+
+### 1. Client Subsystem
+
+**Purpose:** Entry point and coordination of all other subsystems
+
+**Components:**
+- `CustomHTTPClient` - Main client that extends `http.Client`
+- `HTTPClientSettings` - Configuration structure
+- `customTransport` - Custom HTTP transport layer
+
+**Key Interactions:**
+
+```mermaid
+sequenceDiagram
+ participant User
+ participant Client as CustomHTTPClient
+ participant Settings as HTTPClientSettings
+ participant Transport as customTransport
+
+ User->>Settings: Create configuration
+ User->>Client: NewWARCWritingHTTPClient(settings)
+ activate Client
+ Client->>Client: Initialize stats registry
+ Client->>Client: Initialize log backend
+ Client->>Client: Create deduplication table
+ Client->>Client: Setup WARC writer pool
+ Client->>Transport: Create custom transport
+ Transport-->>Client: Return configured transport
+ Client-->>User: Return client
+ deactivate Client
+
+ User->>Client: Do(request)
+ Client->>Transport: RoundTrip(request)
+ Transport-->>Client: Return response
+ Client-->>User: Return response
+```
+
+**Responsibilities:**
+- Initialize all subsystems
+- Manage deduplication hash table
+- Coordinate WARC writer goroutines
+- Track active requests via WaitGroup
+- Provide public API surface
+
+### 2. Network Subsystem
+
+**Purpose:** Establish network connections with proxy and DNS support
+
+**Components:**
+- `customDialer` - Connection establishment logic
+- Proxy selection algorithm
+- DNS resolution with caching
+- `CustomConnection` - Wrapped connection
+
+**Proxy Selection Logic:**
+
+```mermaid
+flowchart TD
+ START[selectProxy called]
+
+ CTX_TYPE{Context specifies
ProxyType?}
+ START --> CTX_TYPE
+
+ CTX_TYPE -->|Yes| FILTER_TYPE[Filter: Keep only
matching ProxyType]
+ CTX_TYPE -->|No| FILTER_ANY[Filter: Keep only
ProxyTypeAny]
+
+ FILTER_TYPE --> FILTER_NET
+ FILTER_ANY --> FILTER_NET
+
+ FILTER_NET[Filter: Network
IPv4/IPv6 compatibility]
+
+ FILTER_NET --> FILTER_DOM[Filter: Domain
glob patterns]
+
+ FILTER_DOM --> CHECK_EMPTY{Any proxies
remaining?}
+
+ CHECK_EMPTY -->|Yes| ROUND_ROBIN[Round-robin select
atomic counter]
+ CHECK_EMPTY -->|No| CHECK_FALLBACK{Direct fallback
enabled?}
+
+ CHECK_FALLBACK -->|Yes| RETURN_NIL[Return nil
Use direct connection]
+ CHECK_FALLBACK -->|No| RETURN_ERR[Return error]
+
+ ROUND_ROBIN --> UPDATE_STATS[Update metrics:
proxy_requests_total
proxy_last_used]
+
+ UPDATE_STATS --> RETURN_PROXY[Return selected proxy]
+
+ style FILTER_TYPE fill:#ffebee
+ style FILTER_NET fill:#fff3e0
+ style FILTER_DOM fill:#e8f5e9
+ style ROUND_ROBIN fill:#e3f2fd
+ style UPDATE_STATS fill:#fff9c4
+```
+
+**DNS Resolution Flow:**
+
+```mermaid
+flowchart TD
+ START[archiveDNS called]
+
+ CACHE_CHECK{In cache?}
+ START --> CACHE_CHECK
+
+ CACHE_CHECK -->|Yes| RETURN_CACHED[Return cached IP]
+ CACHE_CHECK -->|No| CONCURRENT[concurrentDNSLookup]
+
+ CONCURRENT --> WORKER_POOL[Create worker pool
size = dnsConcurrency]
+
+ WORKER_POOL --> ROUND_ROBIN[Round-robin
distribute servers]
+
+ ROUND_ROBIN --> SPAWN[Spawn workers]
+
+ SPAWN --> QUERY_A[Query A records
IPv4]
+ SPAWN --> QUERY_AAAA[Query AAAA records
IPv6]
+
+ QUERY_A --> BOTH_FOUND{Both A and AAAA
found?}
+ QUERY_AAAA --> BOTH_FOUND
+
+ BOTH_FOUND -->|Yes| CANCEL[Cancel remaining
queries early]
+ BOTH_FOUND -->|No| CONTINUE[Continue querying]
+
+ CANCEL --> SELECT_IP
+ CONTINUE --> SELECT_IP[Select IP:
Prefer IPv6 if available]
+
+ SELECT_IP --> CACHE_STORE[Store in cache
with TTL]
+
+ CACHE_STORE --> WRITE_WARC[Write DNS response
to WARC]
+
+ WRITE_WARC --> RETURN[Return IP]
+
+ style CACHE_CHECK fill:#e8f5e9
+ style WORKER_POOL fill:#e3f2fd
+ style QUERY_A fill:#ffebee
+ style QUERY_AAAA fill:#f3e5f5
+ style CACHE_STORE fill:#fff9c4
+```
+
+**Connection Wrapping:**
+
+```mermaid
+graph LR
+ subgraph "Real Network"
+ SERVER[Server]
+ end
+
+ subgraph "CustomConnection"
+ READER[TeeReader]
+ WRITER[MultiWriter]
+ end
+
+ subgraph "Application"
+ APP[App Read/Write]
+ end
+
+ subgraph "Capture Pipes"
+ REQ_PIPE[Request Pipe]
+ RESP_PIPE[Response Pipe]
+ end
+
+ subgraph "WARC Goroutine"
+ WARC[writeWARCFromConnection]
+ end
+
+ APP -->|Write request| WRITER
+ WRITER -->|Original| SERVER
+ WRITER -->|Copy| REQ_PIPE
+ REQ_PIPE --> WARC
+
+ SERVER -->|Response| READER
+ READER -->|Original| APP
+ READER -->|Copy| RESP_PIPE
+ RESP_PIPE --> WARC
+
+ style READER fill:#e3f2fd
+ style WRITER fill:#ffebee
+ style WARC fill:#e8f5e9
+```
+
+### 3. Capture Subsystem
+
+**Purpose:** Parse and extract metadata from HTTP traffic
+
+**Components:**
+- IO pipes for request/response capture
+- HTTP request/response parsers
+- Metadata extraction (URI, headers, timing)
+
+**Request/Response Processing:**
+
+```mermaid
+stateDiagram-v2
+ [*] --> CreatePipes: wrapConnection()
+ CreatePipes --> LaunchGoroutine: spawn writeWARCFromConnection
+ LaunchGoroutine --> SpawnParsers: create io.Pipes
+
+ state SpawnParsers {
+ [*] --> ReqParser: go readRequest()
+ [*] --> RespParser: go readResponse()
+ }
+
+ state ReqParser {
+ [*] --> CreateReqRecord
+ CreateReqRecord --> CopyBytes: io.Copy to SpooledTempFile
+ CopyBytes --> ParseHTTP: http.ReadRequest
+ ParseHTTP --> ExtractURI: Parse request line + Host
+ ExtractURI --> SendURI: channel → response parser
+ SendURI --> SetHeaders: WARC headers
+ SetHeaders --> SendRecord: channel → assembler
+ }
+
+ state RespParser {
+ [*] --> WaitURI: Receive from request parser
+ WaitURI --> CreateRespRecord
+ CreateRespRecord --> CopyRespBytes: io.Copy to SpooledTempFile
+ CopyRespBytes --> ParseResp: http.ReadResponse
+ ParseResp --> CheckDiscard: DiscardHook?
+ CheckDiscard --> CalcPayloadDigest: Hash response body
+ CalcPayloadDigest --> CheckDedupe: Query dedup systems
+ CheckDedupe --> CalcBlockDigest: Hash entire record
+ CalcBlockDigest --> SendRespRecord: channel → assembler
+ }
+
+ SpawnParsers --> WaitBoth: sync.WaitGroup
+ WaitBoth --> AssembleBatch: Create RecordBatch
+ AssembleBatch --> SetMetadata: UUIDs, cross-refs, etc.
+ SetMetadata --> SendToChannel: client.WARCWriter
+ SendToChannel --> [*]
+```
+
+**Metadata Extraction Points:**
+
+| Metadata | Source | Location |
+|----------|--------|----------|
+| WARC-Target-URI | HTTP request line + Host header | client.go:679 |
+| WARC-Date | Capture timestamp | warc.go:165 |
+| WARC-Record-ID | UUID generation | client.go:735 |
+| WARC-Concurrent-To | Cross-reference to paired record | client.go:740 |
+| WARC-IP-Address | conn.RemoteAddr() | dialer.go:765 |
+| WARC-Payload-Digest | Hash of response body | client.go:693 |
+| WARC-Block-Digest | Hash of entire record | client.go:720 |
+| Content-Type | HTTP response header | client.go:656 |
+| Content-Length | Record content size | warc.go:169 |
+
+### 4. Deduplication Subsystem
+
+**Purpose:** Avoid storing duplicate content
+
+**Components:**
+- Local hash table (`sync.Map`)
+- Doppelganger API client
+- CDX API client
+
+**Deduplication Decision Tree:**
+
+```mermaid
+flowchart TD
+ START[Response parsed]
+
+ SIZE_CHECK{Payload size >=
threshold?}
+ START --> SIZE_CHECK
+
+ SIZE_CHECK -->|No| SKIP[Skip deduplication
Store full record]
+ SIZE_CHECK -->|Yes| EMPTY_CHECK
+
+ EMPTY_CHECK{Is empty payload?
Known empty digest}
+ EMPTY_CHECK -->|Yes| SKIP
+ EMPTY_CHECK -->|No| LOCAL_CHECK
+
+ LOCAL_CHECK[Check local
dedupeHashTable]
+
+ LOCAL_CHECK --> LOCAL_FOUND{Found locally?}
+
+ LOCAL_FOUND -->|Yes| CREATE_REVISIT[Create revisit record
WARC-Type: revisit
WARC-Refers-To: UUID
WARC-Truncated: length]
+ LOCAL_FOUND -->|No| DOPPEL_ENABLED
+
+ DOPPEL_ENABLED{Doppelganger
enabled?}
+
+ DOPPEL_ENABLED -->|Yes| DOPPEL_QUERY[HTTP GET to
doppelganger API]
+ DOPPEL_ENABLED -->|No| CDX_ENABLED
+
+ DOPPEL_QUERY --> DOPPEL_FOUND{Found?}
+
+ DOPPEL_FOUND -->|Yes| CREATE_DOPPEL_REVISIT[Create revisit record
WARC-Refers-To-Target-URI
WARC-Refers-To-Date]
+ DOPPEL_FOUND -->|No| CDX_ENABLED
+
+ CDX_ENABLED{CDX enabled?}
+
+ CDX_ENABLED -->|Yes| CDX_QUERY[HTTP GET to
CDX API]
+ CDX_ENABLED -->|No| STORE_FULL
+
+ CDX_QUERY --> CDX_FOUND{Found?}
+
+ CDX_FOUND -->|Yes| CREATE_CDX_REVISIT[Create revisit record
WARC-Refers-To-Target-URI
WARC-Refers-To-Date]
+ CDX_FOUND -->|No| STORE_FULL
+
+ STORE_FULL[Store full record
Add to dedupeHashTable]
+
+ CREATE_REVISIT --> UPDATE_LOCAL_STATS[Increment:
local_deduped_total
local_deduped_bytes_total]
+
+ CREATE_DOPPEL_REVISIT --> UPDATE_DOPPEL_STATS[Increment:
doppelganger_deduped_total
doppelganger_deduped_bytes_total]
+
+ CREATE_CDX_REVISIT --> UPDATE_CDX_STATS[Increment:
cdx_deduped_total
cdx_deduped_bytes_total]
+
+ UPDATE_LOCAL_STATS --> END[Send to WARC writer]
+ UPDATE_DOPPEL_STATS --> END
+ UPDATE_CDX_STATS --> END
+ STORE_FULL --> END
+ SKIP --> END
+
+ style LOCAL_CHECK fill:#e8f5e9
+ style DOPPEL_QUERY fill:#e3f2fd
+ style CDX_QUERY fill:#f3e5f5
+ style CREATE_REVISIT fill:#fff9c4
+ style UPDATE_LOCAL_STATS fill:#ffebee
+```
+
+**Revisit Record Structure:**
+
+```
+Normal Response Record:
+WARC-Type: response
+WARC-Record-ID:
+WARC-Target-URI: https://example.com/page
+WARC-Payload-Digest: sha1:ABC123...
+Content-Length: 50000
+[Full HTTP response with body]
+
+↓ Becomes (if duplicate found) ↓
+
+Revisit Record:
+WARC-Type: revisit
+WARC-Record-ID:
+WARC-Target-URI: https://example.com/page
+WARC-Refers-To-Target-URI: https://example.com/page
+WARC-Refers-To-Date: 2025-01-20T10:30:00Z
+WARC-Refers-To: (if local)
+WARC-Payload-Digest: sha1:ABC123...
+WARC-Truncated: length
+WARC-Profile: http://netpreserve.org/warc/1.1/revisit/identical-payload-digest
+Content-Length: 250
+[HTTP response headers only, body truncated]
+```
+
+### 5. WARC Writing Subsystem
+
+**Purpose:** Persist records to disk with compression and rotation
+
+**Components:**
+- RecordBatch channel (buffered)
+- Pool of recordWriter goroutines
+- File rotation logic
+- Compression (GZIP/ZSTD)
+
+**Writer Pool Architecture:**
+
+```mermaid
+graph TD
+ subgraph "Client"
+ CH[WARCWriter Channel
buffered: 1000]
+ end
+
+ subgraph "Writer Pool"
+ W1[Writer Goroutine 1]
+ W2[Writer Goroutine 2]
+ W3[Writer Goroutine 3]
+ W4[Writer Goroutine 4]
+ end
+
+ subgraph "WARC Files"
+ F1[WARC-...-00001.warc.gz]
+ F2[WARC-...-00002.warc.gz]
+ F3[WARC-...-00003.warc.gz]
+ F4[WARC-...-00004.warc.gz]
+ end
+
+ CH -->|RecordBatch| W1
+ CH -->|RecordBatch| W2
+ CH -->|RecordBatch| W3
+ CH -->|RecordBatch| W4
+
+ W1 -->|Write, rotate| F1
+ W2 -->|Write, rotate| F2
+ W3 -->|Write, rotate| F3
+ W4 -->|Write, rotate| F4
+
+ style CH fill:#e8f5e9
+ style W1 fill:#e3f2fd
+ style W2 fill:#e3f2fd
+ style W3 fill:#e3f2fd
+ style W4 fill:#e3f2fd
+```
+
+**File Rotation State Machine:**
+
+```mermaid
+stateDiagram-v2
+ [*] --> Initialize
+ Initialize --> GenerateFilename: Serial counter
+ GenerateFilename --> CheckExists: File exists?
+ CheckExists --> IncrementSerial: Yes
+ IncrementSerial --> GenerateFilename
+ CheckExists --> CreateFile: No
+ CreateFile --> AddOpenSuffix: Add ".open" suffix
+ AddOpenSuffix --> OpenWriter: Open file for writing
+ OpenWriter --> WriteWarcinfo: Write warcinfo record
+ WriteWarcinfo --> WaitingForBatch
+
+ WaitingForBatch --> CheckSize: RecordBatch received
+
+ state CheckSize <>
+ CheckSize --> WriteBatch: Size OK
+ CheckSize --> Rotate: Size exceeded
+
+ Rotate --> Flush: Flush buffers
+ Flush --> CloseCompressor: Close gzip/zstd
+ CloseCompressor --> CloseFile
+ CloseFile --> Rename: Remove ".open" suffix
+ Rename --> GenerateFilename
+
+ WriteBatch --> WriteRecords: For each record
+ WriteRecords --> UpdateMetrics: total_data_written
+ UpdateMetrics --> SignalFeedback: Optional feedback
+ SignalFeedback --> WaitingForBatch
+
+ WaitingForBatch --> Shutdown: Channel closed
+ Shutdown --> Flush
+ Shutdown --> [*]
+```
+
+**Compression Pipeline:**
+
+```mermaid
+graph LR
+ subgraph "Record Content"
+ CONTENT[SpooledTempFile
Uncompressed data]
+ end
+
+ subgraph "Writer Layers"
+ RECORD_BUF[Record bufio.Writer
4KB buffer]
+ COMPRESSOR[GZIP/ZSTD Encoder
Compression block]
+ FILE_BUF[File bufio.Writer
64KB buffer]
+ FILE[os.File
Disk I/O]
+ end
+
+ CONTENT -->|io.Copy| RECORD_BUF
+ RECORD_BUF -->|Flush per record| COMPRESSOR
+ COMPRESSOR -->|Compressed chunks| FILE_BUF
+ FILE_BUF -->|Periodic flush| FILE
+
+ style CONTENT fill:#e3f2fd
+ style COMPRESSOR fill:#fff9c4
+ style FILE fill:#e8f5e9
+```
+
+### 6. Observability Subsystem
+
+**Purpose:** Provide metrics and logging for monitoring and debugging
+
+**Components:**
+- StatsRegistry interface (Counters, Gauges, Histograms)
+- LogBackend interface (Debug, Info, Warn, Error)
+
+**Metrics Collection Points:**
+
+```mermaid
+graph TB
+ subgraph "Data Path Metrics"
+ M1[total_data_written
Type: Counter
Location: Writer]
+ end
+
+ subgraph "Deduplication Metrics"
+ M2[local_deduped_total
Type: Counter
Location: Dedupe]
+ M3[local_deduped_bytes_total
Type: Counter
Location: Dedupe]
+ M4[doppelganger_deduped_total
Type: Counter
Location: Dedupe]
+ M5[doppelganger_deduped_bytes_total
Type: Counter
Location: Dedupe]
+ M6[cdx_deduped_total
Type: Counter
Location: Dedupe]
+ M7[cdx_deduped_bytes_total
Type: Counter
Location: Dedupe]
+ end
+
+ subgraph "Proxy Metrics"
+ M8[proxy_requests_total
Type: Counter
Labels: proxy
Location: Dialer]
+ M9[proxy_errors_total
Type: Counter
Labels: proxy
Location: Dialer]
+ M10[proxy_last_used_nanoseconds
Type: Gauge
Labels: proxy
Location: Dialer]
+ end
+
+ subgraph "StatsRegistry Implementation"
+ IMPL[User-provided or
Local Registry]
+ end
+
+ M1 --> IMPL
+ M2 --> IMPL
+ M3 --> IMPL
+ M4 --> IMPL
+ M5 --> IMPL
+ M6 --> IMPL
+ M7 --> IMPL
+ M8 --> IMPL
+ M9 --> IMPL
+ M10 --> IMPL
+
+ style M1 fill:#e3f2fd
+ style M2 fill:#fff9c4
+ style M8 fill:#ffebee
+ style IMPL fill:#e8f5e9
+```
+
+**Logging Event Hierarchy:**
+
+```mermaid
+graph TB
+ subgraph "Debug Events"
+ D1[Proxy selected
dialer.go:293]
+ D2[DNS resolved
dns.go:44]
+ D3[Connection established
dialer.go:473]
+ D4[TLS connection established
dialer.go:594]
+ D5[WARC record written
write.go:145]
+ end
+
+ subgraph "Info Events"
+ I1[WARC file created
warc.go:132]
+ I2[WARC file rotation
warc.go:152]
+ I3[WARC writer shutdown
warc.go:224]
+ end
+
+ subgraph "Error Events"
+ E1[DNS resolution failed
dns.go:52]
+ E2[Proxy connection failed
dialer.go:456]
+ E3[Direct connection failed
dialer.go:483]
+ E4[TLS handshake failed
dialer.go:575]
+ E5[WARC record write failed
write.go:139]
+ E6[DiscardHook rejection
client.go:671]
+ E7[Digest calculation failed
client.go:705]
+ end
+
+ subgraph "LogBackend"
+ LOG[User-provided or
No-op Logger]
+ end
+
+ D1 --> LOG
+ D2 --> LOG
+ D3 --> LOG
+ D4 --> LOG
+ D5 --> LOG
+ I1 --> LOG
+ I2 --> LOG
+ I3 --> LOG
+ E1 --> LOG
+ E2 --> LOG
+ E3 --> LOG
+ E4 --> LOG
+ E5 --> LOG
+ E6 --> LOG
+ E7 --> LOG
+
+ style D1 fill:#e3f2fd
+ style I1 fill:#e8f5e9
+ style E1 fill:#ffebee
+ style LOG fill:#fff9c4
+```
+
+### 7. Storage Subsystem
+
+**Purpose:** Efficient temporary storage with RAM/disk threshold
+
+**Component:**
+- `SpooledTempFile` - Automatic RAM → disk spillover
+
+**SpooledTempFile State Machine:**
+
+```mermaid
+stateDiagram-v2
+ [*] --> InMemory: Create()
+
+ state InMemory {
+ [*] --> BytesBuffer
+ BytesBuffer --> CheckSize: Write()
+ CheckSize --> BytesBuffer: Size < threshold
+ }
+
+ CheckSize --> SpillToDisk: Size >= threshold
+
+ state SpillToDisk {
+ [*] --> CreateTempFile: os.CreateTemp()
+ CreateTempFile --> CopyBuffer: io.Copy(file, buffer)
+ CopyBuffer --> FreeBuffer: buffer = nil (GC)
+ }
+
+ SpillToDisk --> OnDisk
+
+ state OnDisk {
+ [*] --> FileOps
+ FileOps --> FileOps: Read/Write/Seek
+ }
+
+ OnDisk --> Cleanup: Close()
+ InMemory --> Cleanup: Close()
+
+ Cleanup --> RemoveFile: os.Remove() if on disk
+ RemoveFile --> [*]
+
+ note right of InMemory
+ Threshold = MaxRAMUsageFraction
+ of system RAM or FullOnDisk=true
+ forces immediate disk spooling
+ end note
+```
+
+**Memory Pressure Management:**
+
+```mermaid
+flowchart TD
+ START[Record content being written]
+
+ CHECK_MODE{FullOnDisk
enabled?}
+ START --> CHECK_MODE
+
+ CHECK_MODE -->|Yes| DISK[Immediately create
temp file on disk]
+ CHECK_MODE -->|No| RAM_CHECK
+
+ RAM_CHECK[Calculate threshold:
sysRAM * MaxRAMUsageFraction]
+
+ RAM_CHECK --> WRITE_MEM[Write to in-memory
bytes.Buffer]
+
+ WRITE_MEM --> SIZE_CHECK{Buffer size >
threshold?}
+
+ SIZE_CHECK -->|No| WRITE_MEM
+ SIZE_CHECK -->|Yes| SPILL[Spill to disk:
1. Create temp file
2. Copy buffer
3. Free memory]
+
+ SPILL --> WRITE_DISK
+ DISK --> WRITE_DISK[Write to disk file]
+
+ WRITE_DISK --> COMPLETE[Content complete]
+
+ style RAM_CHECK fill:#e3f2fd
+ style WRITE_MEM fill:#e8f5e9
+ style SPILL fill:#fff9c4
+ style WRITE_DISK fill:#ffebee
+```
+
+## Inter-Subsystem Communication
+
+### Message Passing
+
+```mermaid
+graph LR
+ subgraph "Goroutines"
+ APP[Application]
+ DIAL[Dialer]
+ WRAP[writeWARCFromConnection]
+ REQ[readRequest]
+ RESP[readResponse]
+ WRITE[recordWriter Pool]
+ end
+
+ subgraph "Channels"
+ TARGET[targetURI chan]
+ RECORD[record chan]
+ BATCH[RecordBatch chan]
+ FEEDBACK[feedback chan]
+ end
+
+ APP -->|synchronous| DIAL
+ DIAL -->|spawn| WRAP
+ WRAP -->|spawn| REQ
+ WRAP -->|spawn| RESP
+
+ REQ -->|async| TARGET
+ TARGET -->|async| RESP
+
+ REQ -->|async| RECORD
+ RESP -->|async| RECORD
+
+ RECORD -->|collect| WRAP
+ WRAP -->|async| BATCH
+
+ BATCH -->|load balanced| WRITE
+ WRITE -->|optional| FEEDBACK
+ FEEDBACK -->|sync| APP
+
+ style TARGET fill:#e3f2fd
+ style RECORD fill:#fff9c4
+ style BATCH fill:#e8f5e9
+ style FEEDBACK fill:#ffebee
+```
+
+### Synchronization Primitives
+
+| Primitive | Usage | Location |
+|-----------|-------|----------|
+| `sync.WaitGroup` | Wait for all active requests | CustomHTTPClient.WaitGroup |
+| `sync.Map` | Thread-safe deduplication table | CustomHTTPClient.dedupeHashTable |
+| `atomic.Uint32` | Lock-free proxy round-robin | customDialer.proxyRoundRobinIndex |
+| `sync.Once` | One-time read deadline set | CustomConnection.firstRead |
+| `sync.Mutex` | Protect testLogger entries | testLogger.mu |
+| Buffered channels | Async batch submission | WARCWriter (buffer: 1000) |
+| Unbuffered channels | Sync metadata passing | targetURICh, recordChan |
+| Optional feedback chan | Synchronous WARC write | RecordBatch.FeedbackChan |
+
+### Context Propagation
+
+```mermaid
+graph TD
+ START[Application creates context]
+
+ START --> CTX1[context.Background]
+
+ CTX1 --> ADD_TIMEOUT[Add timeout:
ctx, cancel = context.WithTimeout]
+
+ ADD_TIMEOUT --> ADD_PROXY[Add proxy type:
warc.WithProxyType]
+
+ ADD_PROXY --> ADD_FEEDBACK[Add feedback channel:
warc.WithFeedbackChannel]
+
+ ADD_FEEDBACK --> ADD_CONN[Add connection channel:
warc.WithWrappedConnection]
+
+ ADD_CONN --> USE_CTX[req = req.WithContext(ctx)]
+
+ USE_CTX --> PROP1[client.Do
→ transport.RoundTrip]
+
+ PROP1 --> PROP2[→ dialer.CustomDialContext]
+
+ PROP2 --> PROP3[→ wrapConnection]
+
+ PROP3 --> PROP4[→ writeWARCFromConnection]
+
+ PROP4 --> EXTRACT[Extract context values:
- ProxyType
- FeedbackChan
- ConnChan]
+
+ EXTRACT --> USE[Use values to control:
- Proxy selection
- Synchronous writes
- Connection inspection]
+
+ style ADD_TIMEOUT fill:#ffebee
+ style ADD_PROXY fill:#e3f2fd
+ style ADD_FEEDBACK fill:#e8f5e9
+ style EXTRACT fill:#fff9c4
+```
+
+## Advanced Features
+
+### 1. Synchronous WARC Writes
+
+**Default (Async):**
+```go
+resp, _ := client.Do(req)
+io.Copy(io.Discard, resp.Body)
+resp.Body.Close()
+// WARC writing happens in background
+// Returns immediately
+```
+
+**Synchronous:**
+```go
+feedbackChan := make(chan struct{})
+ctx := warc.WithFeedbackChannel(context.Background(), feedbackChan)
+req = req.WithContext(ctx)
+
+resp, _ := client.Do(req)
+io.Copy(io.Discard, resp.Body)
+resp.Body.Close()
+
+<-feedbackChan // Blocks until WARC written to disk
+```
+
+### 2. Connection Inspection
+
+```go
+connChan := make(chan *warc.CustomConnection, 1)
+ctx := warc.WithWrappedConnection(context.Background(), connChan)
+req = req.WithContext(ctx)
+
+resp, _ := client.Do(req)
+
+// Access wrapped connection
+wrappedConn := <-connChan
+realConn := wrappedConn.Conn // Underlying net.Conn
+// Can inspect TLS state, remote address, etc.
+```
+
+### 3. Random Local IP (IPv6)
+
+**Purpose:** Avoid rate limiting by varying source IP
+
+```go
+client, _ := warc.NewWARCWritingHTTPClient(warc.HTTPClientSettings{
+ RandomLocalIP: true,
+ IPv6AnyIP: true, // Use ::/64 prefix from available interfaces
+})
+```
+
+**How it works:**
+1. Goroutine watches network interfaces (polls every 10s)
+2. Detects IPv6 addresses with /64 prefix
+3. For each request:
+ - Generates random IPv6 in detected subnets
+ - Sets as `LocalAddr` on dialer
+ - Each request uses different source IP
+
+### 4. Custom TLS Fingerprinting
+
+Uses `refraction-networking/utls` to mimic real browsers:
+
+```go
+func getCustomTLSSpec() utls.ClientHelloID {
+ return utls.HelloChrome_120 // Looks like Chrome 120
+}
+```
+
+Avoids TLS-based blocking/fingerprinting.
+
+## Error Handling Patterns
+
+### Network Errors
+
+```go
+// Proxy connection failed
+if err != nil {
+ logBackend.Error("proxy connection failed",
+ "proxy", selectedProxy.name,
+ "address", address,
+ "error", err)
+ statsRegistry.Counter("proxy_errors_total",
+ label.String("proxy", selectedProxy.name)).Inc()
+
+ // Try direct connection if allowed
+ if allowDirectFallback {
+ return directConnection()
+ }
+ return nil, err
+}
+```
+
+### WARC Writing Errors
+
+```go
+if err := writer.WriteRecord(record); err != nil {
+ logBackend.Error("failed to write WARC record content",
+ "file", writer.FileName,
+ "error", err)
+ // Continue to next record (don't crash)
+}
+```
+
+### Graceful Degradation
+
+- DNS failure → Return error (don't archive)
+- Proxy failure → Try direct (if allowed)
+- Dedupe API failure → Store full record
+- Digest calculation error → Log and skip digest
+- DiscardHook error → Log and keep record
+
+## Performance Optimizations
+
+### 1. Concurrent DNS Queries
+
+```go
+// Sequential (dnsConcurrency = 1)
+for _, server := range dnsServers {
+ result, err := query(server)
+ if err == nil { return result }
+}
+
+// Parallel (dnsConcurrency = -1)
+results := make(chan result, len(dnsServers))
+for _, server := range dnsServers {
+ go func(s string) {
+ results <- query(s)
+ }(server)
+}
+// Early termination when both A and AAAA found
+```
+
+### 2. Writer Pool Parallelism
+
+```go
+// Single writer (WARCWriterPoolSize = 1)
+// Bottleneck: One file open at a time
+
+// Multiple writers (WARCWriterPoolSize = 4)
+// Each writer has its own file
+// 4x throughput (if not I/O bound)
+```
+
+### 3. Buffered Channels
+
+```go
+// Unbuffered (bad)
+WARCWriter = make(chan *RecordBatch)
+// Producer blocks until consumer ready
+
+// Buffered (good)
+WARCWriter = make(chan *RecordBatch, 1000)
+// Producer continues until buffer full
+// Smooths burst traffic
+```
+
+### 4. Atomic Operations
+
+```go
+// Lock-based (slower)
+mu.Lock()
+index = (index + 1) % len(proxies)
+mu.Unlock()
+
+// Atomic (faster)
+index := d.proxyRoundRobinIndex.Add(1) % uint32(len(proxies))
+```
+
+## Summary
+
+The gowarc library is built on several key architectural principles:
+
+1. **Separation of Concerns**: Each subsystem has a clear responsibility
+2. **Concurrent Design**: Goroutines + channels for parallelism
+3. **Pluggable Interfaces**: Stats, logging, deduplication can be customized
+4. **Resource Efficiency**: Spooled temp files, connection pooling, caching
+5. **Observability**: Comprehensive metrics and structured logging
+6. **Fault Tolerance**: Graceful error handling, fallback mechanisms
+7. **Performance**: Parallel DNS, writer pools, atomic operations
+
+Any software engineer reviewing these diagrams should understand:
+- How requests flow from application to WARC file
+- Where proxies and DNS resolution fit in
+- How byte capture works (pipes + goroutines)
+- When and how deduplication occurs
+- The role of each subsystem
+- How to customize behavior via interfaces
diff --git a/doc/quick-reference.txt b/doc/quick-reference.txt
new file mode 100644
index 0000000..383c213
--- /dev/null
+++ b/doc/quick-reference.txt
@@ -0,0 +1,639 @@
+┌─────────────────────────────────────────────────────────────────────────────────┐
+│ GOWARC LIBRARY - QUICK REFERENCE │
+│ Complete HTTPS Request Flow Example │
+│ (IPv6 Residential Proxy → WARC File) │
+└─────────────────────────────────────────────────────────────────────────────────┘
+
+═══════════════════════════════════════════════════════════════════════════════════
+PHASE 1: REQUEST INITIATION
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌────────────────┐
+│ Application │
+│ │ req := http.NewRequest("GET", "https://example.com/page", nil)
+│ │ resp, err := client.Do(req)
+└───────┬────────┘
+ │
+ ▼
+┌────────────────────────┐
+│ CustomHTTPClient │
+│ - http.Client wrapper │
+│ - Dedup hash table │ Add "Accept-Encoding: gzip"
+│ - WARC writer channel │ Force compression
+└───────┬────────────────┘
+ │
+ ▼
+┌────────────────────────┐
+│ customTransport │
+│ - RoundTrip() │ Intercept request
+│ - Decompress response │ Delegate to http.Transport
+└───────┬────────────────┘
+ │
+ ▼
+
+═══════════════════════════════════════════════════════════════════════════════════
+PHASE 2: PROXY SELECTION & DNS
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌────────────────────────────────────────────────────────────────────┐
+│ customDialer │
+│ │
+│ Step 1: Network Type Selection │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ getNetworkType("tcp") │ │
+│ │ - disableIPv4=true, disableIPv6=false │ │
+│ │ → Returns "tcp6" (force IPv6) │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ Step 2: Proxy Selection │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ selectProxy(ctx, "tcp6", "example.com:443") │ │
+│ │ │ │
+│ │ Configured proxies: │ │
+│ │ ├─ socks5://proxy.datacenter.net [Type=Datacenter] │ │
+│ │ ├─ socks5://proxy.residential.net [Type=Residential] ✓ │ │
+│ │ └─ socks5://proxy.ipv4only.net [Network=IPv4] │ │
+│ │ │ │
+│ │ Filter 1: ProxyType │ │
+│ │ Context requests Residential → Keep only Residential │ │
+│ │ ✓ proxy.residential.net │ │
+│ │ ✗ proxy.datacenter.net │ │
+│ │ │ │
+│ │ Filter 2: Network (IPv6 needed) │ │
+│ │ ✓ proxy.residential.net (Network=Any) │ │
+│ │ ✗ proxy.ipv4only.net (Network=IPv4) │ │
+│ │ │ │
+│ │ Filter 3: Domain (example.com) │ │
+│ │ ✓ AllowedDomains: ["*.example.com"] matches │ │
+│ │ │ │
+│ │ Round-robin: Select from 1 eligible proxy │ │
+│ │ → Selected: proxy.residential.net:1080 │ │
+│ │ │ │
+│ │ Metrics: │ │
+│ │ proxy_requests_total{proxy="proxy.residential.net"} += 1 │ │
+│ │ proxy_last_used_nanoseconds{proxy="..."} = now() │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ Step 3: DNS Resolution │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ Proxy scheme: socks5h → REMOTE DNS │ │
+│ │ Skip local DNS, send hostname to proxy │ │
+│ │ (Proxy will resolve "example.com" on its network) │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+└────────────────────────────────────────────────────────────────────┘
+ │
+ ▼
+
+═══════════════════════════════════════════════════════════════════════════════════
+PHASE 3: CONNECTION ESTABLISHMENT
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌────────────────────────────────────────────────────────────────────┐
+│ Connect to Proxy │
+│ │
+│ proxyDialer.DialContext(ctx, "tcp6", "example.com:443") │
+│ │
+│ SOCKS5 Handshake: │
+│ ──────────────────────────────────────────────────────────────── │
+│ Client → Proxy: [0x05, 0x01, 0x00] (version, methods) │
+│ Proxy → Client: [0x05, 0x00] (accepted) │
+│ Client → Proxy: [0x05, 0x01, 0x00, 0x03, "example.com", 443] │
+│ (CONNECT to domain) │
+│ Proxy → Client: [0x05, 0x00, ...] (success) │
+│ │
+│ → Plain TCP connection ready │
+└───────┬────────────────────────────────────────────────────────────┘
+ │
+ ▼
+┌────────────────────────────────────────────────────────────────────┐
+│ TLS Handshake │
+│ │
+│ tlsConn = utls.UClient(plainConn, tlsConfig) │
+│ tlsConn.ApplyPreset(HelloChrome_120) // Browser fingerprinting │
+│ err = tlsConn.HandshakeContext(ctx, 10s timeout) │
+│ │
+│ TLS 1.3 Handshake: │
+│ ──────────────────────────────────────────────────────────────── │
+│ ClientHello → [via proxy] → Server │
+│ ServerHello ← [via proxy] ← Server │
+│ Certificate ← [via proxy] ← Server │
+│ ... (standard TLS 1.3 handshake) │
+│ │
+│ → Encrypted TLS connection ready │
+│ │
+│ Log: "TLS connection established via proxy" │
+└───────┬────────────────────────────────────────────────────────────┘
+ │
+ ▼
+
+═══════════════════════════════════════════════════════════════════════════════════
+PHASE 4: CONNECTION WRAPPING FOR CAPTURE
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌────────────────────────────────────────────────────────────────────┐
+│ wrapConnection(ctx, tlsConn, "https") │
+│ │
+│ Create bidirectional capture pipes: │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ reqReader, reqWriter := io.Pipe() // Request capture │ │
+│ │ respReader, respWriter := io.Pipe() // Response capture │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ Launch WARC recording goroutine: │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ go writeWARCFromConnection( │ │
+│ │ ctx, reqReader, respReader, "https", tlsConn) │ │
+│ │ │ │
+│ │ This goroutine will: │ │
+│ │ 1. Read HTTP request from reqReader pipe │ │
+│ │ 2. Read HTTP response from respReader pipe │ │
+│ │ 3. Parse both into WARC records │ │
+│ │ 4. Send to WARC writer channel │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ Create CustomConnection: │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ CustomConnection { │ │
+│ │ Conn: tlsConn, │ │
+│ │ Reader: io.TeeReader(tlsConn, respWriter), │ │
+│ │ └─> Splits reads: app + pipe │ │
+│ │ Writer: io.MultiWriter(reqWriter, tlsConn), │ │
+│ │ └─> Duplicates writes: pipe + network │ │
+│ │ } │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ Data Flow Visualization: │
+│ │
+│ Application WRITE (HTTP Request): │
+│ ──────────────────────────────────────────────────────────────── │
+│ App │
+│ │ │
+│ ▼ │
+│ MultiWriter │
+│ ├──────────────────┬─────────────────┐ │
+│ │ │ │ │
+│ ▼ ▼ ▼ │
+│ reqWriter (pipe) tlsConn To WARC Goroutine │
+│ │ │
+│ ▼ │
+│ Proxy → Server │
+│ │
+│ Server RESPONSE: │
+│ ──────────────────────────────────────────────────────────────── │
+│ Server → Proxy │
+│ │ │
+│ ▼ │
+│ tlsConn │
+│ │ │
+│ ▼ │
+│ TeeReader │
+│ ├────────────────┬────────────────┐ │
+│ │ │ │ │
+│ ▼ ▼ ▼ │
+│ App respWriter To WARC Goroutine │
+│ (pipe) │
+│ │
+└───────┬────────────────────────────────────────────────────────────┘
+ │
+ ▼
+
+═══════════════════════════════════════════════════════════════════════════════════
+PHASE 5: HTTP REQUEST/RESPONSE (Application Layer)
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌────────────────────────────────────────────────────────────────────┐
+│ Application writes HTTP request │
+│ ──────────────────────────────────────────────────────────────── │
+│ GET /page HTTP/1.1 │
+│ Host: example.com │
+│ User-Agent: ... │
+│ Accept-Encoding: gzip │
+│ [empty line] │
+│ │
+│ → Bytes flow through MultiWriter to BOTH: │
+│ 1. Real TLS connection (to server) │
+│ 2. reqWriter pipe (to WARC goroutine) │
+└────────────────────────────────────────────────────────────────────┘
+ │
+ ▼
+┌────────────────────────────────────────────────────────────────────┐
+│ Server sends HTTP response │
+│ ──────────────────────────────────────────────────────────────── │
+│ HTTP/1.1 200 OK │
+│ Content-Type: text/html │
+│ Content-Encoding: gzip │
+│ Content-Length: 5432 │
+│ [empty line] │
+│ [gzipped HTML body] │
+│ │
+│ → Bytes flow through TeeReader to BOTH: │
+│ 1. Application (io.Copy to destination) │
+│ 2. respWriter pipe (to WARC goroutine) │
+└────────────────────────────────────────────────────────────────────┘
+
+═══════════════════════════════════════════════════════════════════════════════════
+PHASE 6: WARC RECORD CREATION (Background Goroutines)
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌────────────────────────────────────────────────────────────────────┐
+│ writeWARCFromConnection Goroutine │
+│ │
+│ Spawns 2 sub-goroutines: │
+│ │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ [1] readRequest(reqReader) │ │
+│ │ │ │
+│ │ • Create Record with SpooledTempFile │ │
+│ │ • io.Copy all bytes from reqReader → Record.Content │ │
+│ │ • Parse HTTP request (http.ReadRequest) │ │
+│ │ • Extract: │ │
+│ │ - Request line: "GET /page HTTP/1.1" │ │
+│ │ - Host header: "example.com" │ │
+│ │ - Construct URI: "https://example.com/page" │ │
+│ │ • Send targetURI → response goroutine (via channel) │ │
+│ │ • Set WARC headers: │ │
+│ │ - WARC-Type: request │ │
+│ │ - WARC-Target-URI: https://example.com/page │ │
+│ │ - Content-Type: application/http;msgtype=request │ │
+│ │ • Send record → batch assembler (via channel) │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ [2] readResponse(respReader) │ │
+│ │ │ │
+│ │ • Receive targetURI from request goroutine │ │
+│ │ • Create Record with SpooledTempFile │ │
+│ │ • io.Copy all bytes from respReader → Record.Content │ │
+│ │ • Parse HTTP response (http.ReadResponse) │ │
+│ │ • Check DiscardHook(response): │ │
+│ │ - Can filter by status code, headers, etc. │ │
+│ │ - Example: Discard 429 rate limit responses │ │
+│ │ • Calculate payload digest: │ │
+│ │ - Skip HTTP headers │ │
+│ │ - SHA1 hash of response body │ │
+│ │ - Result: "sha1:ABC123..." │ │
+│ │ • Set WARC-Payload-Digest header │ │
+│ │ • Send record → batch assembler (via channel) │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ Wait for both goroutines to complete │
+│ Assemble RecordBatch │
+└───────┬────────────────────────────────────────────────────────────┘
+ │
+ ▼
+
+═══════════════════════════════════════════════════════════════════════════════════
+PHASE 7: DEDUPLICATION
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌────────────────────────────────────────────────────────────────────┐
+│ Deduplication Check │
+│ │
+│ payloadDigest = "sha1:ABC123..." │
+│ │
+│ Step 1: Local Deduplication │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ existing, found := dedupeHashTable.Load(payloadDigest) │ │
+│ │ │ │
+│ │ if found { │ │
+│ │ // Payload seen before in this session │ │
+│ │ Convert to REVISIT record: │ │
+│ │ WARC-Type: revisit │ │
+│ │ WARC-Refers-To: │ │
+│ │ WARC-Refers-To-Target-URI: https://... │ │
+│ │ WARC-Refers-To-Date: 2025-01-25T12:30:00Z │ │
+│ │ WARC-Truncated: length │ │
+│ │ Content: [HTTP headers only, body removed] │ │
+│ │ │ │
+│ │ Metrics: │ │
+│ │ local_deduped_total += 1 │ │
+│ │ local_deduped_bytes_total += original_size │ │
+│ │ │ │
+│ │ → Skip external checks │ │
+│ │ } │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ Step 2: Doppelganger API (if configured) │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ if DoppelgangerHost != "" && !foundLocally { │ │
+│ │ url := DoppelgangerHost + "/api/records/" + │ │
+│ │ payloadDigest + "?uri=" + targetURI │ │
+│ │ │ │
+│ │ resp := http.Get(url) │ │
+│ │ if resp.StatusCode == 200 { │ │
+│ │ // Found in external service │ │
+│ │ Parse: {"id": "...", "uri": "...", "date": "..."} │ │
+│ │ Convert to REVISIT record │ │
+│ │ │ │
+│ │ Metrics: │ │
+│ │ doppelganger_deduped_total += 1 │ │
+│ │ doppelganger_deduped_bytes_total += size │ │
+│ │ } │ │
+│ │ } │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ Step 3: CDX API (if configured) │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ if CDXDedupeServer != "" && !foundYet { │ │
+│ │ url := CDXDedupeServer + "/web/timemap/cdx?" + │ │
+│ │ "url=" + targetURI + "&limit=-1" │ │
+│ │ │ │
+│ │ resp := http.Get(url) │ │
+│ │ Parse CDX line: "com,example)/page 20250101... sha1:..." │ │
+│ │ │ │
+│ │ if digest matches payloadDigest { │ │
+│ │ Convert to REVISIT record │ │
+│ │ │ │
+│ │ Metrics: │ │
+│ │ cdx_deduped_total += 1 │ │
+│ │ cdx_deduped_bytes_total += size │ │
+│ │ } │ │
+│ │ } │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+│ │
+│ If NOT found anywhere: │
+│ ┌──────────────────────────────────────────────────────────────┐ │
+│ │ Store full response record │ │
+│ │ Add to dedupeHashTable for future local checks: │ │
+│ │ Key: payloadDigest │ │
+│ │ Value: {recordID, targetURI, date, size} │ │
+│ └──────────────────────────────────────────────────────────────┘ │
+└───────┬────────────────────────────────────────────────────────────┘
+ │
+ ▼
+
+═══════════════════════════════════════════════════════════════════════════════════
+PHASE 8: BATCH FINALIZATION
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌────────────────────────────────────────────────────────────────────┐
+│ RecordBatch Assembly │
+│ │
+│ batch := &RecordBatch{ │
+│ Records: [requestRecord, responseRecord], │
+│ CaptureTime: time.Now().UTC().Format(RFC3339Nano), │
+│ FeedbackChan: nil, // or user-provided channel for sync │
+│ } │
+│ │
+│ Generate UUIDs: │
+│ ──────────────────────────────────────────────────────────────── │
+│ reqUUID = "550e8400-e29b-41d4-a716-446655440000" │
+│ respUUID = "6ba7b810-9dad-11d1-80b4-00c04fd430c8" │
+│ │
+│ Set Record IDs: │
+│ ──────────────────────────────────────────────────────────────── │
+│ requestRecord.Header["WARC-Record-ID"] = │
+│ "" │
+│ responseRecord.Header["WARC-Record-ID"] = │
+│ "" │
+│ │
+│ Cross-reference records: │
+│ ──────────────────────────────────────────────────────────────── │
+│ requestRecord.Header["WARC-Concurrent-To"] = │
+│ "" │
+│ responseRecord.Header["WARC-Concurrent-To"] = │
+│ "" │
+│ │
+│ Set Target URI: │
+│ ──────────────────────────────────────────────────────────────── │
+│ requestRecord.Header["WARC-Target-URI"] = │
+│ "https://example.com/page" │
+│ responseRecord.Header["WARC-Target-URI"] = │
+│ "https://example.com/page" │
+│ │
+│ IP Address (skipped when using proxy): │
+│ ──────────────────────────────────────────────────────────────── │
+│ // WARC-IP-Address NOT set │
+│ // (We connected via proxy, don't know real server IP) │
+│ │
+│ Calculate Block Digest: │
+│ ──────────────────────────────────────────────────────────────── │
+│ blockDigest = SHA1(entire record content including headers) │
+│ responseRecord.Header["WARC-Block-Digest"] = blockDigest │
+│ │
+│ Send to WARC writer: │
+│ ──────────────────────────────────────────────────────────────── │
+│ client.WARCWriter <- batch │
+│ │
+│ (Optional) Wait for write confirmation: │
+│ ──────────────────────────────────────────────────────────────── │
+│ if batch.FeedbackChan != nil { │
+│ <-batch.FeedbackChan // Block until written │
+│ } │
+└───────┬────────────────────────────────────────────────────────────┘
+ │
+ ▼
+
+═══════════════════════════════════════════════════════════════════════════════════
+PHASE 9: WARC FILE WRITING
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌────────────────────────────────────────────────────────────────────┐
+│ recordWriter Goroutine (one of pool) │
+│ │
+│ Current state: │
+│ currentFile = "WARC-20250125122030-00001-hostname.warc.gz.open"│
+│ currentSize = 45.2 MB │
+│ sizeLimit = 100 MB │
+│ │
+│ for batch := range WARCWriter { │
+│ │
+│ ┌────────────────────────────────────────────────────────────┐ │
+│ │ Check file size │ │
+│ │ if currentSize >= sizeLimit { │ │
+│ │ // Rotate to new file │ │
+│ │ 1. Flush all buffers │ │
+│ │ 2. Close GZIP writer │ │
+│ │ 3. Close file │ │
+│ │ 4. Rename: remove ".open" suffix │ │
+│ │ "...-00001.warc.gz.open" → "...-00001.warc.gz" │ │
+│ │ 5. Increment serial: 00002 │ │
+│ │ 6. Create new file with ".open" suffix │ │
+│ │ 7. Write warcinfo record to new file │ │
+│ │ 8. Log: "WARC file rotation" │ │
+│ │ } │ │
+│ └────────────────────────────────────────────────────────────┘ │
+│ │
+│ For each record in batch.Records { │
+│ │
+│ ┌──────────────────────────────────────────────────────────┐ │
+│ │ Set timestamp │ │
+│ │ record.Header["WARC-Date"] = batch.CaptureTime │ │
+│ │ → "2025-01-25T12:30:45.123456789Z" │ │
+│ │ │ │
+│ │ Reference warcinfo │ │
+│ │ record.Header["WARC-Warcinfo-ID"] = warcinfoID │ │
+│ │ │ │
+│ │ Calculate content length │ │
+│ │ record.Header["Content-Length"] = record.Content.Size() │ │
+│ └──────────────────────────────────────────────────────────┘ │
+│ │
+│ ┌──────────────────────────────────────────────────────────┐ │
+│ │ WriteRecord(record) │ │
+│ │ │ │
+│ │ Write to file: │ │
+│ │ ┌──────────────────────────────────────────────────────┐ │ │
+│ │ │ WARC/1.1 │ │ │
+│ │ │ WARC-Type: response │ │ │
+│ │ │ WARC-Record-ID: │ │ │
+│ │ │ WARC-Concurrent-To: │ │ │
+│ │ │ WARC-Target-URI: https://example.com/page │ │ │
+│ │ │ WARC-Date: 2025-01-25T12:30:45.123456789Z │ │ │
+│ │ │ WARC-Payload-Digest: sha1:ABC123... │ │ │
+│ │ │ WARC-Block-Digest: sha1:DEF456... │ │ │
+│ │ │ WARC-Warcinfo-ID: │ │ │
+│ │ │ Content-Type: application/http;msgtype=response │ │ │
+│ │ │ Content-Length: 5678 │ │ │
+│ │ │ │ │ │
+│ │ │ HTTP/1.1 200 OK │ │ │
+│ │ │ Content-Type: text/html │ │ │
+│ │ │ Content-Encoding: gzip │ │ │
+│ │ │ │ │ │
+│ │ │ [gzipped HTML body bytes...] │ │ │
+│ │ │ │ │ │
+│ │ │ │ │ │
+│ │ └──────────────────────────────────────────────────────┘ │ │
+│ │ │ │
+│ │ Compression pipeline: │ │
+│ │ Record → bufio.Writer → gzip.Writer → file │ │
+│ │ │ │
+│ │ Flush record-level buffers │ │
+│ │ Log: "WARC record written" │ │
+│ └──────────────────────────────────────────────────────────┘ │
+│ │
+│ Metrics: │
+│ total_data_written += bytes_written │
+│ } │
+│ │
+│ Flush file-level buffers │
+│ │
+│ if batch.FeedbackChan != nil { │
+│ batch.FeedbackChan <- struct{}{} // Signal completion │
+│ } │
+│ } │
+│ │
+│ // Channel closed - shutdown │
+│ Flush all buffers │
+│ Close GZIP writer │
+│ Close file │
+│ Rename: remove ".open" suffix │
+│ Log: "WARC writer shutting down cleanly" │
+│ done <- true │
+└────────────────────────────────────────────────────────────────────┘
+
+═══════════════════════════════════════════════════════════════════════════════════
+FINAL RESULT: WARC FILE ON DISK
+═══════════════════════════════════════════════════════════════════════════════════
+
+File: /output/WARC-20250125122030-00001-hostname.warc.gz
+Size: 47.3 MB (compressed)
+
+Contents (conceptual, actual file is gzip-compressed):
+
+┌────────────────────────────────────────────────────────────────────┐
+│ RECORD 1: warcinfo │
+├────────────────────────────────────────────────────────────────────┤
+│ WARC/1.1 │
+│ WARC-Type: warcinfo │
+│ WARC-Date: 2025-01-25T12:20:30.000000000Z │
+│ WARC-Record-ID: │
+│ Content-Type: application/warc-fields │
+│ Content-Length: 123 │
+│ │
+│ software: gowarc/1.0.0 │
+│ format: WARC File Format 1.1 │
+│ conformsTo: http://iipc.github.io/warc-specifications/... │
+│ │
+│ │
+└────────────────────────────────────────────────────────────────────┘
+
+┌────────────────────────────────────────────────────────────────────┐
+│ RECORD 2: request │
+├────────────────────────────────────────────────────────────────────┤
+│ WARC/1.1 │
+│ WARC-Type: request │
+│ WARC-Date: 2025-01-25T12:30:45.123456789Z │
+│ WARC-Record-ID: │
+│ WARC-Concurrent-To: │
+│ WARC-Target-URI: https://example.com/page │
+│ WARC-Warcinfo-ID: │
+│ WARC-Block-Digest: sha1:XYZ789... │
+│ Content-Type: application/http;msgtype=request │
+│ Content-Length: 234 │
+│ │
+│ GET /page HTTP/1.1 │
+│ Host: example.com │
+│ User-Agent: Go-http-client/1.1 │
+│ Accept-Encoding: gzip │
+│ │
+│ │
+└────────────────────────────────────────────────────────────────────┘
+
+┌────────────────────────────────────────────────────────────────────┐
+│ RECORD 3: response │
+├────────────────────────────────────────────────────────────────────┤
+│ WARC/1.1 │
+│ WARC-Type: response │
+│ WARC-Date: 2025-01-25T12:30:45.123456789Z │
+│ WARC-Record-ID: │
+│ WARC-Concurrent-To: │
+│ WARC-Target-URI: https://example.com/page │
+│ WARC-Warcinfo-ID: │
+│ WARC-Payload-Digest: sha1:ABC123... │
+│ WARC-Block-Digest: sha1:DEF456... │
+│ Content-Type: application/http;msgtype=response │
+│ Content-Length: 5678 │
+│ │
+│ HTTP/1.1 200 OK │
+│ Date: Sat, 25 Jan 2025 12:30:45 GMT │
+│ Content-Type: text/html; charset=UTF-8 │
+│ Content-Encoding: gzip │
+│ Content-Length: 5432 │
+│ │
+│ [5432 bytes of gzipped HTML content...] │
+│ │
+│ │
+└────────────────────────────────────────────────────────────────────┘
+
+... (more request/response pairs) ...
+
+═══════════════════════════════════════════════════════════════════════════════════
+OBSERVABILITY THROUGHOUT THE FLOW
+═══════════════════════════════════════════════════════════════════════════════════
+
+METRICS COLLECTED:
+──────────────────
+ proxy_requests_total{proxy="proxy.residential.net:1080"} = 1
+ proxy_last_used_nanoseconds{proxy="proxy.residential.net:1080"} = 1706140800000
+ total_data_written = 5678 bytes
+ (local_deduped_total = 0, doppelganger_deduped_total = 0, etc.)
+
+LOGS EMITTED:
+─────────────
+ [DEBUG] Proxy selected: proxy=proxy.residential.net:1080 network=tcp6
+ [DEBUG] Connection established via proxy: proxy=proxy.residential.net:1080
+ [DEBUG] TLS connection established via proxy: proxy=proxy.residential.net:1080
+ [INFO] WARC record written: file=WARC-...-00001.warc.gz bytes=5678
+ [DEBUG] WARC record written: recordID=
+
+═══════════════════════════════════════════════════════════════════════════════════
+KEY COMPONENTS SUMMARY
+═══════════════════════════════════════════════════════════════════════════════════
+
+┌─────────────────────┬──────────────────┬────────────────────────────────────┐
+│ Component │ File │ Responsibility │
+├─────────────────────┼──────────────────┼────────────────────────────────────┤
+│ CustomHTTPClient │ client.go │ Main entry point, coordination │
+│ customDialer │ dialer.go │ Proxy/DNS/connection setup │
+│ CustomConnection │ dialer.go │ Bidirectional byte capture │
+│ DNS resolver │ dns.go │ Concurrent DNS with caching │
+│ Proxy selector │ dialer.go:208 │ Filter & round-robin proxies │
+│ SpooledTempFile │ utils.go │ RAM→disk temp storage │
+│ RecordBatch │ write.go │ Group related records │
+│ recordWriter │ warc.go:100 │ File writing & rotation │
+│ Writer │ write.go:50 │ Compression & WARC formatting │
+│ StatsRegistry │ stats.go │ Metrics interface │
+│ LogBackend │ logging.go │ Logging interface │
+└─────────────────────┴──────────────────┴────────────────────────────────────┘
+
+═══════════════════════════════════════════════════════════════════════════════════
diff --git a/doc/request-flow.md b/doc/request-flow.md
new file mode 100644
index 0000000..ab269eb
--- /dev/null
+++ b/doc/request-flow.md
@@ -0,0 +1,666 @@
+# Request Flow: HTTPS via IPv6 Residential Proxy → WARC File
+
+This document traces the complete journey of an HTTPS request through an IPv6 residential proxy, showing every step from the initial request to the final WARC file on disk.
+
+## Complete Flow Diagram
+
+```mermaid
+sequenceDiagram
+ autonumber
+ participant App as Application
+ participant Client as CustomHTTPClient
+ participant Transport as customTransport
+ participant Dialer as customDialer
+ participant DNS as DNS Resolver
+ participant Proxy as Proxy Selector
+ participant ProxyConn as Proxy Connection
+ participant Wrapper as wrapConnection
+ participant ReqPipe as Request Pipe
+ participant RespPipe as Response Pipe
+ participant ReqParser as readRequest Goroutine
+ participant RespParser as readResponse Goroutine
+ participant Dedupe as Deduplication
+ participant Batch as RecordBatch Channel
+ participant Writer as recordWriter
+ participant Disk as WARC File
+
+ %% Request initiation
+ App->>Client: client.Do(req)
GET https://example.com/page
+ Client->>Transport: RoundTrip(req)
+ Transport->>Transport: Add "Accept-Encoding: gzip"
+ Transport->>Dialer: CustomDialTLSContext(ctx, "tcp", "example.com:443")
+
+ %% Network type selection
+ Dialer->>Dialer: getNetworkType("tcp")
→ "tcp6" (IPv6 enabled, IPv4 disabled)
+
+ %% Proxy selection
+ Dialer->>Proxy: selectProxy(ctx, "tcp6", "example.com:443")
+ Proxy->>Proxy: Filter by ProxyType
(ctx requests Residential → use only Residential)
+ Proxy->>Proxy: Filter by Network
(tcp6 → skip IPv4-only proxies)
+ Proxy->>Proxy: Filter by AllowedDomains
(check if "example.com" matches globs)
+ Proxy->>Proxy: Round-robin select
proxyDialers[index++ % len]
+ Proxy-->>Dialer: Selected: "socks5://proxy.residential.net:1080"
Type=Residential, Network=Any
+ Dialer->>Dialer: statsRegistry.Counter("proxy_requests_total",
label="proxy.residential.net:1080").Inc()
+ Dialer->>Dialer: logBackend.Debug("proxy selected",
"proxy", "proxy.residential.net:1080")
+
+ %% DNS resolution (skipped for socks5h)
+ Note over Dialer,DNS: Proxy scheme is socks5h → remote DNS
+ Dialer->>Dialer: Skip local DNS (let proxy resolve)
+
+ %% Establish connection via proxy
+ Dialer->>ProxyConn: proxyDialer.DialContext(ctx, "tcp6", "example.com:443")
+ ProxyConn-->>Dialer: Plain TCP connection to proxy
+ Dialer->>Dialer: logBackend.Debug("connection established via proxy")
+
+ %% TLS handshake
+ Dialer->>Dialer: tlsConn = utls.UClient(conn, tlsConfig)
ServerName="example.com"
InsecureSkipVerify=false
+ Dialer->>Dialer: tlsConn.ApplyPreset(Chrome120)
+ Dialer->>ProxyConn: tlsConn.HandshakeContext(ctx, 10s timeout)
+ ProxyConn-->>Dialer: TLS handshake complete
+ Dialer->>Dialer: logBackend.Debug("TLS connection established via proxy")
+
+ %% Wrap connection for capture
+ Dialer->>Wrapper: wrapConnection(ctx, tlsConn, "https")
+ Wrapper->>ReqPipe: reqReader, reqWriter = io.Pipe()
+ Wrapper->>RespPipe: respReader, respWriter = io.Pipe()
+
+ %% Launch WARC recording goroutine
+ Wrapper->>ReqParser: go writeWARCFromConnection(...)
+ activate ReqParser
+ Wrapper->>Wrapper: Create CustomConnection:
Reader = io.TeeReader(tlsConn, respWriter)
Writer = io.MultiWriter(reqWriter, tlsConn)
+ Wrapper-->>Dialer: Return CustomConnection
+ Dialer-->>Transport: Return connection
+ Transport-->>Client: Return connection
+
+ %% Application makes request
+ Client-->>App: Return &http.Response{Body: ...}
+ App->>App: Send HTTP request
(written to CustomConnection.Writer)
+ Note over App,ReqPipe: Request bytes flow through MultiWriter
to both real connection AND reqWriter pipe
+
+ %% Parse request in background
+ ReqParser->>ReqParser: go readRequest(reqReader)
+ activate RespParser
+ ReqParser->>ReqParser: Create request Record
+ ReqParser->>ReqParser: Copy bytes from reqReader to Record.Content
(SpooledTempFile)
+ ReqParser->>ReqParser: Parse HTTP request line:
GET /page HTTP/1.1
+ ReqParser->>ReqParser: Extract Host header: example.com
+ ReqParser->>ReqParser: Construct WARC-Target-URI:
https://example.com/page
+ ReqParser->>RespParser: Send targetURI via channel
+
+ %% Application reads response
+ App->>App: io.Copy(dst, resp.Body)
+ Note over App,RespPipe: Response bytes flow through TeeReader
from real connection to both app AND respWriter pipe
+
+ %% Parse response in background
+ RespParser->>RespParser: Receive targetURI from channel
+ RespParser->>RespParser: Create response Record
+ RespParser->>RespParser: Copy bytes from respReader to Record.Content
(SpooledTempFile)
+ RespParser->>RespParser: Seek to start of Record.Content
+ RespParser->>RespParser: http.ReadResponse(bufio.Reader(Content))
+ RespParser->>RespParser: Check DiscardHook(response)
→ (false, "") → Keep record
+
+ %% Calculate payload digest
+ RespParser->>RespParser: Seek to payload start (skip HTTP headers)
+ RespParser->>RespParser: Calculate SHA1 digest of body
+ RespParser->>RespParser: payloadDigest = "sha1:ABC123..."
+ RespParser->>RespParser: Set WARC-Payload-Digest header
+
+ %% Deduplication check
+ RespParser->>Dedupe: Check if payloadDigest in dedupeHashTable
+ Dedupe-->>RespParser: Not found (first time seeing this)
+ Note over RespParser,Dedupe: If found, would create revisit record here
+
+ %% Complete record processing
+ RespParser->>RespParser: Seek back to start
+ RespParser->>RespParser: Calculate block digest (entire record)
+ RespParser-->>ReqParser: Send response record via channel
+ deactivate RespParser
+
+ %% Finalize batch
+ ReqParser->>ReqParser: Wait for both goroutines
+ ReqParser->>ReqParser: Create RecordBatch:
- Records: [request, response]
- CaptureTime: RFC3339Nano timestamp
+ ReqParser->>ReqParser: Set WARC-Record-ID (UUID) on each
+ ReqParser->>ReqParser: Set WARC-Concurrent-To (cross-reference)
+ ReqParser->>ReqParser: Set WARC-Target-URI on both
+ ReqParser->>ReqParser: Set WARC-IP-Address:
+ ReqParser->>ReqParser: Store in dedupeHashTable:
Key=payloadDigest
Value=(UUID, targetURI, date, size)
+ ReqParser->>Batch: Send RecordBatch to client.WARCWriter channel
+ deactivate ReqParser
+
+ %% WARC writing
+ Batch->>Writer: RecordBatch received by recordWriter goroutine
+ activate Writer
+ Writer->>Writer: Check current file size vs WARCSize limit
+ Note over Writer: If exceeded: rotate to new file
+ Writer->>Writer: For each record in batch:
+ Writer->>Writer: Set WARC-Date from batch.CaptureTime
+ Writer->>Writer: Set WARC-Warcinfo-ID (reference to warcinfo)
+ Writer->>Writer: Set Content-Length (from record.Content size)
+ Writer->>Writer: WriteRecord(record):
1. Write "WARC/1.1\r\n"
2. Write headers
3. Write content
4. Write "\r\n\r\n"
+ Writer->>Writer: If GZIP: Flush gzip writer
+ Writer->>Writer: Flush file buffer
+ Writer->>Disk: Data written to:
/output/WARC-20250125-00001.warc.gz
+ Writer->>Writer: statsRegistry.Counter("total_data_written").Add(bytes)
+ Writer->>Writer: logBackend.Debug("WARC record written")
+ Writer-->>Batch: Signal FeedbackChan (if provided)
+ deactivate Writer
+
+ App->>Client: Close response body
+ App->>Client: client.Close()
+ Client->>Batch: close(WARCWriter channel)
+ Client->>Writer: Wait for all WARCWriterDone signals
+ Writer->>Disk: Rename file (remove .open suffix)
+ Writer->>Disk: Close file
+```
+
+## Detailed Step Breakdown
+
+### Phase 1: Request Initiation (Steps 1-3)
+
+**Application calls `client.Do(req)`:**
+```go
+req, _ := http.NewRequest("GET", "https://example.com/page", nil)
+resp, err := client.Do(req)
+```
+
+**CustomHTTPClient → customTransport:**
+- Transport intercepts the request
+- Adds `Accept-Encoding: gzip` header (forces servers to send compressed responses)
+- Delegates to standard `http.Transport.RoundTrip()`
+
+### Phase 2: Network Type Selection (Step 4)
+
+**Dialer determines IPv4 vs IPv6:**
+
+```go
+func getNetworkType(network string) (string, error) {
+ // Given: disableIPv4=true, disableIPv6=false
+ if network == "tcp" {
+ if disableIPv4 && !disableIPv6 {
+ return "tcp6", nil // ← Returns this
+ }
+ // ... other cases
+ }
+ return network, nil
+}
+```
+
+Forces all connections to use IPv6.
+
+### Phase 3: Proxy Selection (Steps 5-11)
+
+**Configured proxies example:**
+```go
+proxies := []ProxyConfig{
+ {
+ URL: "socks5://proxy.datacenter.net:1080",
+ Network: ProxyNetworkAny,
+ Type: ProxyTypeDatacenter,
+ },
+ {
+ URL: "socks5://proxy.residential.net:1080",
+ Network: ProxyNetworkAny,
+ Type: ProxyTypeResidential, // ← This one matches
+ AllowedDomains: []string{"*.example.com"},
+ },
+ {
+ URL: "socks5://proxy.ipv4only.net:1080",
+ Network: ProxyNetworkIPv4, // Filtered out (we need IPv6)
+ Type: ProxyTypeResidential,
+ },
+}
+```
+
+**Context specifies proxy type:**
+```go
+ctx = warc.WithProxyType(ctx, warc.ProxyTypeResidential)
+req = req.WithContext(ctx)
+```
+
+**Selection algorithm:**
+
+1. **Type Filter**: Context requests `ProxyTypeResidential`
+ - ✅ Keep: `proxy.residential.net` (Type=Residential)
+ - ❌ Drop: `proxy.datacenter.net` (Type=Datacenter)
+
+2. **Network Filter**: Need `tcp6` (IPv6)
+ - ✅ Keep: `proxy.residential.net` (Network=Any)
+ - ❌ Drop: `proxy.ipv4only.net` (Network=IPv4)
+
+3. **Domain Filter**: Target is `example.com`
+ - ✅ Match: `*.example.com` pattern matches `example.com`
+
+4. **Round-Robin**: Select from eligible pool
+ - Only one remains: `proxy.residential.net:1080`
+
+**Metrics recorded:**
+```
+proxy_requests_total{proxy="proxy.residential.net:1080"} = 1
+proxy_last_used_nanoseconds{proxy="proxy.residential.net:1080"} = 1706140800000000000
+```
+
+### Phase 4: DNS Resolution (Step 12)
+
+**Proxy scheme determines DNS handling:**
+
+| Scheme | DNS Resolution | Hostname Sent to Proxy |
+|--------|----------------|------------------------|
+| socks5 | Local | No (IP sent) |
+| socks5h | Remote | Yes (hostname sent) |
+| socks4 | Local | No (IP sent) |
+| socks4a | Remote | Yes (hostname sent) |
+| http/https | Remote | Yes (in CONNECT) |
+
+Our proxy is `socks5h` → **Remote DNS**
+- Hostname `example.com:443` sent directly to proxy
+- Proxy resolves DNS on its network
+- Avoids DNS leaks from client network
+
+**If DNS were local (socks5):**
+```go
+IP, err := archiveDNS(ctx, "example.com:443")
+// Concurrent DNS lookup across multiple servers
+// Write DNS response to WARC as "resource" record
+```
+
+### Phase 5: Connection Establishment (Steps 13-18)
+
+**Connect via proxy:**
+```go
+// Proxy dialer handles SOCKS5 negotiation:
+// 1. Send: [0x05, 0x01, 0x00] (version, nmethods, no auth)
+// 2. Receive: [0x05, 0x00] (version, method accepted)
+// 3. Send: [0x05, 0x01, 0x00, 0x03, len("example.com"), "example.com", port_high, port_low]
+// Command=CONNECT, ATYP=DOMAINNAME
+// 4. Receive: [0x05, 0x00, ...] (success)
+// 5. Returns: net.Conn ready for TLS
+```
+
+**TLS Handshake:**
+```go
+tlsConfig := &tls.Config{
+ ServerName: "example.com",
+ InsecureSkipVerify: false, // Verify certificates
+}
+
+// Use utls for custom fingerprinting
+tlsConn := utls.UClient(plainConn, tlsConfig, utls.HelloChrome_120)
+tlsConn.ApplyPreset(getCustomTLSSpec())
+
+// Handshake with timeout
+ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+defer cancel()
+err := tlsConn.HandshakeContext(ctx)
+```
+
+**On error:**
+```go
+logBackend.Error("TLS handshake failed via proxy",
+ "proxy", "proxy.residential.net:1080",
+ "address", "example.com:443",
+ "error", err)
+statsRegistry.Counter("proxy_errors_total",
+ label.String("proxy", "proxy.residential.net:1080")).Inc()
+```
+
+### Phase 6: Connection Wrapping (Steps 19-24)
+
+**Create bidirectional capture pipes:**
+
+```go
+func wrapConnection(ctx, conn, scheme) *CustomConnection {
+ // Request capture
+ reqReader, reqWriter := io.Pipe()
+ // Response capture
+ respReader, respWriter := io.Pipe()
+
+ // Launch WARC recording goroutine
+ go writeWARCFromConnection(ctx, reqReader, respReader, scheme, conn)
+
+ return &CustomConnection{
+ Conn: conn,
+ Reader: io.TeeReader(conn, respWriter), // Intercept reads
+ Writer: io.MultiWriter(reqWriter, conn), // Intercept writes
+ closers: []*io.PipeWriter{reqWriter, respWriter},
+ }
+}
+```
+
+**Data flow visualization:**
+
+```
+Application Write (HTTP Request)
+ ↓
+ MultiWriter
+ ├──→ reqWriter (pipe) ──→ writeWARCFromConnection goroutine
+ └──→ conn (real network) ──→ Proxy ──→ Server
+
+Server Response
+ ↓
+ conn (real network)
+ ↓
+ TeeReader
+ ├──→ respWriter (pipe) ──→ writeWARCFromConnection goroutine
+ └──→ Application Read
+```
+
+### Phase 7: Request Parsing (Steps 25-34)
+
+**readRequest goroutine:**
+
+```go
+func readRequest(ctx, scheme, reqPipe, targetURICh, recordChan) {
+ // Create record
+ record := &Record{
+ Header: make(Header),
+ Content: NewSpooledTempFile(...),
+ }
+
+ // Copy entire HTTP request to spooled file
+ io.Copy(record.Content, reqPipe)
+
+ // Seek back to start
+ record.Content.Seek(0, io.SeekStart)
+
+ // Parse HTTP request
+ req, _ := http.ReadRequest(bufio.NewReader(record.Content))
+
+ // Extract target URI
+ host := req.Host
+ if host == "" {
+ host = req.Header.Get("Host")
+ }
+ path := req.URL.RequestURI()
+ targetURI := scheme + "://" + host + path
+ // Example: "https://example.com/page"
+
+ // Send to response parser
+ targetURICh <- targetURI
+
+ // Set WARC headers
+ record.Header.Set("WARC-Type", "request")
+ record.Header.Set("WARC-Target-URI", targetURI)
+ record.Header.Set("Content-Type", "application/http;msgtype=request")
+
+ // Send to batch assembler
+ recordChan <- record
+}
+```
+
+**HTTP request format in WARC:**
+```
+GET /page HTTP/1.1
+Host: example.com
+User-Agent: Go-http-client/1.1
+Accept-Encoding: gzip
+
+```
+
+### Phase 8: Response Parsing (Steps 35-44)
+
+**readResponse goroutine:**
+
+```go
+func readResponse(ctx, respPipe, targetURICh, recordChan) {
+ // Create record
+ record := &Record{
+ Header: make(Header),
+ Content: NewSpooledTempFile(...),
+ }
+
+ // Copy entire HTTP response
+ io.Copy(record.Content, respPipe)
+ record.Content.Seek(0, io.SeekStart)
+
+ // Parse response
+ resp, _ := http.ReadResponse(bufio.NewReader(record.Content), nil)
+
+ // Check discard hook
+ if discardHook != nil {
+ if shouldDiscard, reason := discardHook(resp); shouldDiscard {
+ logBackend.Error("response was blocked by DiscardHook",
+ "url", targetURI, "reason", reason)
+ // Still create record but mark as discarded
+ }
+ }
+
+ // Find payload start (skip HTTP headers)
+ headersEnd := findEndOfHeadersOffset(content)
+ payloadStart := headersEnd + 4 // Skip "\r\n\r\n"
+
+ // Calculate payload digest
+ record.Content.Seek(payloadStart, io.SeekStart)
+ hasher := sha1.New()
+ io.Copy(hasher, record.Content)
+ digest := base32.StdEncoding.EncodeToString(hasher.Sum(nil))
+ payloadDigest := "sha1:" + digest
+
+ record.Header.Set("WARC-Payload-Digest", payloadDigest)
+}
+```
+
+### Phase 9: Deduplication (Steps 45-46)
+
+**Check if payload already seen:**
+
+```go
+// Check local hash table
+if existing, found := dedupeHashTable.Load(payloadDigest); found {
+ revisitRecord := existing.(revisitRecord)
+
+ // Create revisit record (truncated)
+ record.Header.Set("WARC-Type", "revisit")
+ record.Header.Set("WARC-Refers-To-Target-URI", revisitRecord.targetURI)
+ record.Header.Set("WARC-Refers-To-Date", revisitRecord.date)
+ record.Header.Set("WARC-Refers-To", revisitRecord.recordID)
+ record.Header.Set("WARC-Truncated", "length")
+ record.Header.Set("WARC-Profile",
+ "http://netpreserve.org/warc/1.1/revisit/identical-payload-digest")
+
+ // Truncate content to headers only
+ record.Content = truncateToHeaders(record.Content, payloadStart)
+
+ statsRegistry.Counter("local_deduped_total").Inc()
+ statsRegistry.Counter("local_deduped_bytes_total").Add(
+ float64(revisitRecord.size))
+}
+```
+
+**External dedupe (if configured):**
+
+1. **Doppelganger API:**
+```http
+GET http://doppelganger.service/api/records/{payloadDigest}?uri={targetURI}
+Response: {"id": "...", "uri": "...", "date": "..."}
+```
+
+2. **CDX API:**
+```http
+GET http://archive.org/web/timemap/cdx?url={targetURI}&limit=-1
+Response: com,example)/page 20240101120000 ... {digest} ...
+```
+
+### Phase 10: Batch Assembly (Steps 47-54)
+
+**Finalize RecordBatch:**
+
+```go
+// Both goroutines completed
+requestRecord := <-recordChan
+responseRecord := <-recordChan
+
+// Ensure order
+batch := &RecordBatch{
+ Records: []*Record{requestRecord, responseRecord},
+ CaptureTime: time.Now().UTC().Format(time.RFC3339Nano),
+}
+
+// Generate UUIDs
+reqUUID := uuid.New().String()
+respUUID := uuid.New().String()
+
+requestRecord.Header.Set("WARC-Record-ID", "")
+responseRecord.Header.Set("WARC-Record-ID", "")
+
+// Cross-reference
+requestRecord.Header.Set("WARC-Concurrent-To", "")
+responseRecord.Header.Set("WARC-Concurrent-To", "")
+
+// Target URI (from request parser)
+targetURI := <-targetURICh
+requestRecord.Header.Set("WARC-Target-URI", targetURI)
+responseRecord.Header.Set("WARC-Target-URI", targetURI)
+
+// IP address (skip if proxied - we don't know the real server IP)
+// WARC-IP-Address NOT set when using proxy
+
+// Block digest (hash of entire record including headers)
+blockDigest := calculateBlockDigest(responseRecord.Content)
+responseRecord.Header.Set("WARC-Block-Digest", blockDigest)
+
+// Store for future dedupe
+dedupeHashTable.Store(payloadDigest, revisitRecord{
+ recordID: "",
+ targetURI: targetURI,
+ date: batch.CaptureTime,
+ size: responseRecord.Content.Size(),
+})
+```
+
+### Phase 11: WARC Writing (Steps 55-67)
+
+**recordWriter goroutine receives batch:**
+
+```go
+func recordWriter(settings, recordsChan, done) {
+ currentFile := createWARCFile() // With ".open" suffix
+ writer := NewWriter(currentFile, ...)
+
+ // Write initial warcinfo record
+ warcinfoID := writeWarcinfoRecord(writer, settings)
+
+ for batch := range recordsChan {
+ // Check file size
+ currentSize := getCurrentFileSize()
+ if currentSize >= settings.WARCSize * 1024 * 1024 {
+ rotateFile() // Rename .open → final, create new file
+ }
+
+ for _, record := range batch.Records {
+ // Set timestamp
+ record.Header.Set("WARC-Date", batch.CaptureTime)
+
+ // Reference warcinfo
+ record.Header.Set("WARC-Warcinfo-ID", warcinfoID)
+
+ // Calculate content length
+ contentLength := record.Content.Size()
+ record.Header.Set("Content-Length", strconv.FormatInt(contentLength, 10))
+
+ // Write record
+ writer.WriteRecord(record)
+ }
+
+ // Signal completion (if requested)
+ if batch.FeedbackChan != nil {
+ batch.FeedbackChan <- struct{}{}
+ }
+ }
+
+ // Shutdown
+ renameFile() // Remove .open suffix
+ writer.Close()
+ done <- true
+}
+```
+
+**WriteRecord implementation:**
+
+```go
+func (w *Writer) WriteRecord(record *Record) error {
+ // Write version line
+ fmt.Fprintf(w, "WARC/1.1\r\n")
+
+ // Write headers
+ for key, value := range record.Header {
+ fmt.Fprintf(w, "%s: %s\r\n", key, value)
+ }
+
+ // Blank line
+ fmt.Fprintf(w, "\r\n")
+
+ // Copy content
+ record.Content.Seek(0, io.SeekStart)
+ bytesWritten, _ := io.Copy(w.GZIPWriter, record.Content)
+
+ // End marker
+ fmt.Fprintf(w, "\r\n\r\n")
+
+ // Flush
+ w.GZIPWriter.Flush()
+ w.FileWriter.Flush()
+
+ // Metrics
+ w.statsRegistry.Counter("total_data_written").Add(float64(bytesWritten))
+
+ return nil
+}
+```
+
+### Final WARC File Structure
+
+```
+/output/WARC-20250125122030-00001-hostname.warc.gz
+
+[Compressed WARC content]
+├── warcinfo record
+│ ├── WARC-Type: warcinfo
+│ ├── WARC-Date: 2025-01-25T12:20:30.123456Z
+│ ├── WARC-Record-ID:
+│ ├── Content-Type: application/warc-fields
+│ └── Content: [Metadata about this WARC file]
+│
+├── request record
+│ ├── WARC-Type: request
+│ ├── WARC-Date: 2025-01-25T12:20:31.456789Z
+│ ├── WARC-Record-ID:
+│ ├── WARC-Concurrent-To:
+│ ├── WARC-Target-URI: https://example.com/page
+│ ├── WARC-Block-Digest: sha1:...
+│ ├── WARC-Payload-Digest: sha1:...
+│ ├── Content-Type: application/http;msgtype=request
+│ └── Content: GET /page HTTP/1.1\r\nHost: example.com\r\n...
+│
+└── response record
+ ├── WARC-Type: response
+ ├── WARC-Date: 2025-01-25T12:20:31.456789Z
+ ├── WARC-Record-ID:
+ ├── WARC-Concurrent-To:
+ ├── WARC-Target-URI: https://example.com/page
+ ├── WARC-Block-Digest: sha1:...
+ ├── WARC-Payload-Digest: sha1:ABC123...
+ ├── Content-Type: application/http;msgtype=response
+ └── Content: HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n...\r\n\r\n...
+```
+
+## Performance Characteristics
+
+### Memory Efficiency
+- **SpooledTempFile**: Starts in RAM, spills to disk at threshold
+- **Streaming**: Records stream through pipes, no full buffering
+- **Concurrent writing**: Multiple writer goroutines prevent blocking
+
+### Latency
+- **Async WARC writing**: Application doesn't wait for disk I/O
+- **Feedback channel**: Optional sync mode for critical writes
+- **Connection pooling**: Disabled to ensure clean capture boundaries
+
+### Throughput
+- **Parallel writers**: `WARCWriterPoolSize` goroutines (default: 4)
+- **Compression**: Hardware-accelerated when available
+- **Buffered I/O**: Multiple buffer layers reduce syscalls
+
+### Resource Usage
+- **DNS cache**: Reduces redundant queries (TTL-aware)
+- **Deduplication**: Prevents storing duplicate content
+- **File rotation**: Keeps individual files manageable size
diff --git a/gzip_interface.go b/gzip_interface.go
index 4ca2fa7..84ceb0f 100644
--- a/gzip_interface.go
+++ b/gzip_interface.go
@@ -19,4 +19,4 @@ type GzipReaderInterface interface {
io.ReadCloser
Multistream(enable bool)
Reset(r io.Reader) error
-}
\ No newline at end of file
+}
diff --git a/logging.go b/logging.go
new file mode 100644
index 0000000..376df6a
--- /dev/null
+++ b/logging.go
@@ -0,0 +1,154 @@
+package warc
+
+import (
+ "context"
+ "log/slog"
+ "sync"
+)
+
+// LogBackend provides a pluggable logging interface compatible with slog.
+// Users can implement this interface to integrate their preferred logging solution.
+// The interface matches slog.Logger method signatures for easy wrapping.
+//
+// Users control the log level cutoff by configuring their logger implementation.
+// For example, when wrapping slog.Logger, set the level in the handler:
+//
+// handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})
+// logger := slog.New(handler)
+//
+// If no LogBackend is provided, a no-op logger is used by default.
+type LogBackend interface {
+ // Debug logs a message at Debug level with optional key-value pairs
+ Debug(msg string, args ...any)
+
+ // Info logs a message at Info level with optional key-value pairs
+ Info(msg string, args ...any)
+
+ // Warn logs a message at Warn level with optional key-value pairs
+ Warn(msg string, args ...any)
+
+ // Error logs a message at Error level with optional key-value pairs
+ Error(msg string, args ...any)
+}
+
+// noopLogger is a no-op implementation of LogBackend used when no logger is provided.
+type noopLogger struct{}
+
+func (n *noopLogger) Debug(_ string, _ ...any) {}
+func (n *noopLogger) Info(_ string, _ ...any) {}
+func (n *noopLogger) Warn(_ string, _ ...any) {}
+func (n *noopLogger) Error(_ string, _ ...any) {}
+func (n *noopLogger) Log(_ context.Context, _ slog.Level, _ string, _ ...any) {}
+
+// LogEntry represents a single log entry captured by TestLogger
+type LogEntry struct {
+ Level slog.Level
+ Message string
+ Args []any
+}
+
+// testLogger is a thread-safe logger implementation for unit tests that captures
+// all log messages for verification. Messages can be retrieved sequentially.
+type testLogger struct {
+ entries []LogEntry
+ mu sync.Mutex
+}
+
+// NewTestLogger creates a new TestLogger for use in unit tests
+func NewTestLogger() *testLogger {
+ return &testLogger{
+ entries: make([]LogEntry, 0),
+ }
+}
+
+func (t *testLogger) Debug(msg string, args ...any) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ t.entries = append(t.entries, LogEntry{Level: slog.LevelDebug, Message: msg, Args: args})
+}
+
+func (t *testLogger) Info(msg string, args ...any) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ t.entries = append(t.entries, LogEntry{Level: slog.LevelInfo, Message: msg, Args: args})
+}
+
+func (t *testLogger) Warn(msg string, args ...any) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ t.entries = append(t.entries, LogEntry{Level: slog.LevelWarn, Message: msg, Args: args})
+}
+
+func (t *testLogger) Error(msg string, args ...any) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ t.entries = append(t.entries, LogEntry{Level: slog.LevelError, Message: msg, Args: args})
+}
+
+func (t *testLogger) Log(ctx context.Context, level slog.Level, msg string, args ...any) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ t.entries = append(t.entries, LogEntry{Level: level, Message: msg, Args: args})
+}
+
+// Entries returns all captured log entries
+func (t *testLogger) Entries() []LogEntry {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ // Return a copy to prevent external modification
+ result := make([]LogEntry, len(t.entries))
+ copy(result, t.entries)
+ return result
+}
+
+// Next returns the next log entry and removes it from the queue.
+// Returns nil if no entries are available.
+func (t *testLogger) Next() *LogEntry {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ if len(t.entries) == 0 {
+ return nil
+ }
+ entry := t.entries[0]
+ t.entries = t.entries[1:]
+ return &entry
+}
+
+// Count returns the number of captured log entries
+func (t *testLogger) Count() int {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ return len(t.entries)
+}
+
+// Clear removes all captured log entries
+func (t *testLogger) Clear() {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ t.entries = make([]LogEntry, 0)
+}
+
+// HasLevel returns true if any log entry with the specified level exists
+func (t *testLogger) HasLevel(level slog.Level) bool {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ for _, entry := range t.entries {
+ if entry.Level == level {
+ return true
+ }
+ }
+ return false
+}
+
+// FindByMessage returns all log entries that contain the specified message
+func (t *testLogger) FindByMessage(msg string) []LogEntry {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ result := make([]LogEntry, 0)
+ for _, entry := range t.entries {
+ if entry.Message == msg {
+ result = append(result, entry)
+ }
+ }
+ return result
+}
diff --git a/logging_test.go b/logging_test.go
new file mode 100644
index 0000000..4f5c469
--- /dev/null
+++ b/logging_test.go
@@ -0,0 +1,209 @@
+package warc
+
+import (
+ "context"
+ "log/slog"
+ "testing"
+)
+
+func TestTestLogger_BasicFunctionality(t *testing.T) {
+ logger := NewTestLogger()
+
+ // Log some messages
+ logger.Debug("debug message", "key1", "value1")
+ logger.Info("info message", "key2", "value2")
+ logger.Warn("warn message", "key3", "value3")
+ logger.Error("error message", "key4", "value4")
+
+ // Check count
+ if logger.Count() != 4 {
+ t.Errorf("Expected 4 log entries, got %d", logger.Count())
+ }
+
+ // Test Next() - sequential retrieval
+ entry := logger.Next()
+ if entry == nil {
+ t.Fatal("Expected first entry, got nil")
+ }
+ if entry.Level != slog.LevelDebug || entry.Message != "debug message" {
+ t.Errorf("Expected debug message, got %v: %s", entry.Level, entry.Message)
+ }
+
+ entry = logger.Next()
+ if entry == nil {
+ t.Fatal("Expected second entry, got nil")
+ }
+ if entry.Level != slog.LevelInfo || entry.Message != "info message" {
+ t.Errorf("Expected info message, got %v: %s", entry.Level, entry.Message)
+ }
+
+ // Count should be 2 now (2 consumed)
+ if logger.Count() != 2 {
+ t.Errorf("Expected 2 remaining entries, got %d", logger.Count())
+ }
+}
+
+func TestTestLogger_HasLevel(t *testing.T) {
+ logger := NewTestLogger()
+
+ logger.Debug("test")
+ logger.Info("test")
+
+ if !logger.HasLevel(slog.LevelDebug) {
+ t.Error("Expected to find Debug level")
+ }
+
+ if !logger.HasLevel(slog.LevelInfo) {
+ t.Error("Expected to find Info level")
+ }
+
+ if logger.HasLevel(slog.LevelError) {
+ t.Error("Did not expect to find Error level")
+ }
+}
+
+func TestTestLogger_FindByMessage(t *testing.T) {
+ logger := NewTestLogger()
+
+ logger.Info("test message 1")
+ logger.Info("test message 2")
+ logger.Info("test message 1")
+ logger.Debug("other message")
+
+ entries := logger.FindByMessage("test message 1")
+ if len(entries) != 2 {
+ t.Errorf("Expected 2 entries with 'test message 1', got %d", len(entries))
+ }
+
+ entries = logger.FindByMessage("other message")
+ if len(entries) != 1 {
+ t.Errorf("Expected 1 entry with 'other message', got %d", len(entries))
+ }
+
+ entries = logger.FindByMessage("nonexistent")
+ if len(entries) != 0 {
+ t.Errorf("Expected 0 entries with 'nonexistent', got %d", len(entries))
+ }
+}
+
+func TestTestLogger_Clear(t *testing.T) {
+ logger := NewTestLogger()
+
+ logger.Info("test 1")
+ logger.Info("test 2")
+ logger.Info("test 3")
+
+ if logger.Count() != 3 {
+ t.Errorf("Expected 3 entries before clear, got %d", logger.Count())
+ }
+
+ logger.Clear()
+
+ if logger.Count() != 0 {
+ t.Errorf("Expected 0 entries after clear, got %d", logger.Count())
+ }
+
+ // Next should return nil after clear
+ if entry := logger.Next(); entry != nil {
+ t.Error("Expected nil after clear, got entry")
+ }
+}
+
+func TestTestLogger_LogMethod(t *testing.T) {
+ logger := NewTestLogger()
+
+ ctx := context.Background()
+ logger.Log(ctx, slog.LevelWarn, "custom level message", "key", "value")
+
+ if logger.Count() != 1 {
+ t.Errorf("Expected 1 entry, got %d", logger.Count())
+ }
+
+ entry := logger.Next()
+ if entry == nil {
+ t.Fatal("Expected entry, got nil")
+ }
+ if entry.Level != slog.LevelWarn {
+ t.Errorf("Expected Warn level, got %v", entry.Level)
+ }
+ if entry.Message != "custom level message" {
+ t.Errorf("Expected 'custom level message', got %s", entry.Message)
+ }
+}
+
+func TestTestLogger_Entries(t *testing.T) {
+ logger := NewTestLogger()
+
+ logger.Debug("msg1")
+ logger.Info("msg2")
+ logger.Error("msg3")
+
+ // Get all entries
+ entries := logger.Entries()
+ if len(entries) != 3 {
+ t.Errorf("Expected 3 entries, got %d", len(entries))
+ }
+
+ // Verify it's a copy (modifying shouldn't affect logger)
+ entries[0].Message = "modified"
+ allEntries := logger.Entries()
+ if allEntries[0].Message == "modified" {
+ t.Error("Entries() should return a copy, but original was modified")
+ }
+
+ // Count should still be 3 (Entries doesn't consume)
+ if logger.Count() != 3 {
+ t.Errorf("Expected count to remain 3, got %d", logger.Count())
+ }
+}
+
+func TestTestLogger_ArgsCapture(t *testing.T) {
+ logger := NewTestLogger()
+
+ logger.Info("test message", "key1", "value1", "key2", 42, "key3", true)
+
+ entry := logger.Next()
+ if entry == nil {
+ t.Fatal("Expected entry, got nil")
+ }
+
+ if len(entry.Args) != 6 {
+ t.Errorf("Expected 6 args, got %d", len(entry.Args))
+ }
+
+ // Verify args are captured correctly
+ if entry.Args[0] != "key1" || entry.Args[1] != "value1" {
+ t.Error("Args not captured correctly")
+ }
+ if entry.Args[2] != "key2" || entry.Args[3] != 42 {
+ t.Error("Args not captured correctly")
+ }
+ if entry.Args[4] != "key3" || entry.Args[5] != true {
+ t.Error("Args not captured correctly")
+ }
+}
+
+func TestTestLogger_ThreadSafety(t *testing.T) {
+ logger := NewTestLogger()
+
+ // Concurrent writes
+ done := make(chan bool)
+ for i := 0; i < 10; i++ {
+ go func() {
+ for j := 0; j < 100; j++ {
+ logger.Info("concurrent message")
+ }
+ done <- true
+ }()
+ }
+
+ // Wait for all goroutines
+ for i := 0; i < 10; i++ {
+ <-done
+ }
+
+ // Should have 1000 entries
+ if logger.Count() != 1000 {
+ t.Errorf("Expected 1000 entries, got %d", logger.Count())
+ }
+}
diff --git a/smoke_test.go b/smoke_test.go
index 6411001..bb92844 100644
--- a/smoke_test.go
+++ b/smoke_test.go
@@ -30,11 +30,11 @@ func TestSmokeWARCFormatRegression(t *testing.T) {
// These values were extracted from a known-good WARC file and serve as
// a snapshot of correct format behavior.
expectedRecords := []struct {
- warcType string
- contentLength int64
- blockDigest string
- payloadDigest string // only for response records
- targetURI string // only for response records
+ warcType string
+ contentLength int64
+ blockDigest string
+ payloadDigest string // only for response records
+ targetURI string // only for response records
}{
{
warcType: "warcinfo",
diff --git a/stats.go b/stats.go
new file mode 100644
index 0000000..af55ff7
--- /dev/null
+++ b/stats.go
@@ -0,0 +1,355 @@
+package warc
+
+import (
+ "fmt"
+ "sort"
+ "strings"
+ "sync"
+ "sync/atomic"
+)
+
+const (
+ // totalDataWritten is the name of the metric that tracks the total data written to WARC files.
+ totalDataWritten string = "total_data_written"
+ totalDataWrittenHelp string = "Total data written to WARC files in bytes"
+
+ // localDedupedBytesTotal is the name of the metric that tracks the total bytes deduped using local dedupe.
+ localDedupedBytesTotal string = "local_deduped_bytes_total"
+ localDedupedBytesTotalHelp string = "Total bytes deduped using local dedupe"
+
+ // localDedupedTotal is the name of the metric that tracks the total records deduped using local dedupe.
+ localDedupedTotal string = "local_deduped_total"
+ localDedupedTotalHelp string = "Total records deduped using local dedupe"
+
+ // doppelgangerDedupedBytesTotal is the name of the metric that tracks the total bytes deduped using Doppelganger.
+ doppelgangerDedupedBytesTotal string = "doppelganger_deduped_bytes_total"
+ doppelgangerDedupedBytesTotalHelp string = "Total bytes deduped using Doppelganger"
+
+ // doppelgangerDedupedTotal is the name of the metric that tracks the total records deduped using Doppelganger.
+ doppelgangerDedupedTotal string = "doppelganger_deduped_total"
+ doppelgangerDedupedTotalHelp string = "Total records deduped using Doppelganger"
+
+ // cdxDedupedBytesTotal is the name of the metric that tracks the total bytes deduped using CDX.
+ cdxDedupedBytesTotal string = "cdx_deduped_bytes_total"
+ cdxDedupedBytesTotalHelp string = "Total bytes deduped using CDX"
+
+ // cdxDedupedTotal is the name of the metric that tracks the total records deduped using CDX.
+ cdxDedupedTotal string = "cdx_deduped_total"
+ cdxDedupedTotalHelp string = "Total records deduped using CDX"
+
+ // proxyRequestsTotal is the name of the metric that tracks the total number of requests gone through a proxy.
+ proxyRequestsTotal string = "proxy_requests_total"
+ proxyRequestsTotalHelp string = "Total number of requests gone through a proxy"
+
+ // proxyErrorsTotal is the name of the metric that tracks the total number of errors occurred with a proxy.
+ proxyErrorsTotal string = "proxy_errors_total"
+ proxyErrorsTotalHelp string = "Total number of errors occurred with a proxy"
+
+ // proxyLastUsedNanoseconds is the name of the metric that tracks the last time a proxy was used.
+ proxyLastUsedNanoseconds string = "proxy_last_used_nanoseconds"
+ proxyLastUsedNanosecondsHelp string = "Last time a proxy was used in seconds (unix timestamp ns)"
+)
+
+// Labels represents Prometheus-style label values as key-value pairs.
+// Labels are used with WithLabels() to specify concrete label values when recording metrics.
+//
+// Example usage with labels:
+//
+// registry := newLocalRegistry()
+//
+// // Register a counter with label names (declares which dimensions the metric tracks)
+// httpRequests := registry.RegisterCounter("http_requests_total", "Total HTTP requests", []string{"method", "status"})
+//
+// // Use WithLabels to specify label values when recording
+// httpRequests.WithLabels(Labels{"method": "GET", "status": "200"}).Inc()
+// httpRequests.WithLabels(Labels{"method": "POST", "status": "201"}).Add(5)
+// httpRequests.WithLabels(Labels{"method": "GET", "status": "404"}).Inc()
+//
+// // Each unique combination of label values creates a separate metric series
+// // The counter with method=GET,status=200 has value 1
+// // The counter with method=POST,status=201 has value 5
+// // The counter with method=GET,status=404 has value 1
+//
+// Example usage without labels:
+//
+// // For metrics without labels, pass nil or empty slice when registering
+// totalCounter := registry.RegisterCounter("requests_total", "All requests", nil)
+// // WithLabels(nil) is required but the labels parameter is ignored
+// totalCounter.WithLabels(nil).Inc()
+//
+// Labels are internally sorted by key to ensure consistent metric identification
+// regardless of the order in which they are specified. This means:
+//
+// Labels{"method": "GET", "status": "200"} == Labels{"status": "200", "method": "GET"}
+type Labels map[string]string
+
+// labelsToString converts labels to a consistent string representation for use as map keys.
+// Labels are sorted by key to ensure consistent ordering.
+func labelsToString(labels Labels) string {
+ if len(labels) == 0 {
+ return ""
+ }
+
+ // Sort keys for consistent ordering
+ keys := make([]string, 0, len(labels))
+ for k := range labels {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+
+ // Build string representation
+ parts := make([]string, 0, len(labels))
+ for _, k := range keys {
+ parts = append(parts, fmt.Sprintf("%s=%q", k, labels[k]))
+ }
+ return strings.Join(parts, ",")
+}
+
+// makeMetricKey creates a unique key for a metric with labels.
+func makeMetricKey(name string, labels Labels) string {
+ if len(labels) == 0 {
+ return name
+ }
+ return name + "{" + labelsToString(labels) + "}"
+}
+
+// RegistryOpts is an interface that provides a WithLabels method to get a metric with specific labels.
+type RegistryOpts[T any] interface {
+ // WithLabels returns a Counter for the given label values.
+ // If the counter has no labels, labels parameter is ignored.
+ WithLabels(labels Labels) T
+}
+
+// Counter represents a monotonically increasing metric.
+type Counter interface {
+ RegistryOpts[Counter]
+ // Inc increments the counter by 1.
+ Inc()
+ // Add adds the given value to the counter.
+ Add(value int64)
+ // Get returns the current value of the counter.
+ // This is used to support unit-testing of the metrics.
+ Get() int64
+}
+
+// Gauge represents a metric that can go up or down.
+type Gauge interface {
+ RegistryOpts[Gauge]
+ // Set sets the gauge to the given value.
+ Set(value int64)
+ // Inc increments the gauge by 1.
+ Inc()
+ // Dec decrements the gauge by 1.
+ Dec()
+ // Add adds the given value to the gauge.
+ Add(value int64)
+ // Sub subtracts the given value from the gauge.
+ Sub(value int64)
+ // Get returns the current value of the gauge.
+ // This is used to support unit-testing of the metrics.
+ Get() int64
+}
+
+// Histogram represents a metric for observing distributions of values.
+type Histogram interface {
+ RegistryOpts[Histogram]
+ // Observe adds a single observation to the histogram.
+ Observe(value int64)
+}
+
+// StatsRegistry provides a registry for external libraries to register and update metrics.
+// The StatsRegistry implementation is expected to be thread-safe so that gowarc can safely register and update metrics from multiple goroutines.
+type StatsRegistry interface {
+ // RegisterCounter registers a new counter metric with optional label names.
+ // labelNames specifies which labels this counter will use (e.g., []string{"method", "status"}).
+ // Returns a Counter that can be used with WithLabels() to specify label values.
+ // If labelNames is nil or empty, returns a Counter that ignores labels.
+ RegisterCounter(name, help string, labelNames []string) Counter
+
+ // RegisterGauge registers a new gauge metric with optional label names.
+ // labelNames specifies which labels this gauge will use (e.g., []string{"location", "type"}).
+ // Returns a Gauge that can be used with WithLabels() to specify label values.
+ // If labelNames is nil or empty, returns a Gauge that ignores labels.
+ RegisterGauge(name, help string, labelNames []string) Gauge
+
+ // RegisterHistogram registers a new histogram metric with the given buckets and optional label names.
+ // If buckets is nil, uses Prometheus default buckets.
+ // labelNames specifies which labels this histogram will use (e.g., []string{"endpoint", "method"}).
+ // Returns a Histogram that can be used with WithLabels() to specify label values.
+ // If labelNames is nil or empty, returns a Histogram that ignores labels.
+ RegisterHistogram(name, help string, buckets []int64, labelNames []string) Histogram
+}
+
+// Nil-safe implementations for when no StatsRegistry is provided.
+type localCounter struct {
+ v atomic.Int64
+}
+
+func (n *localCounter) WithLabels(_ Labels) Counter { return n }
+func (n *localCounter) Inc() { n.v.Add(1) }
+func (n *localCounter) Add(value int64) { n.v.Add(value) }
+func (n *localCounter) Get() int64 { return n.v.Load() }
+
+type localGauge struct {
+ v atomic.Int64
+}
+
+func (n *localGauge) WithLabels(_ Labels) Gauge { return n }
+func (n *localGauge) Set(value int64) { n.v.Store(value) }
+func (n *localGauge) Inc() { n.v.Add(1) }
+func (n *localGauge) Dec() { n.v.Add(-1) }
+func (n *localGauge) Add(value int64) { n.v.Add(value) }
+func (n *localGauge) Sub(value int64) { n.v.Add(-value) }
+func (n *localGauge) Get() int64 { return n.v.Load() }
+
+type localHistogram struct{}
+
+func (n *localHistogram) WithLabels(_ Labels) Histogram { return n }
+func (n *localHistogram) Observe(_ int64) {}
+
+// labeledCounter implements Counter with label support
+type labeledCounter struct {
+ name string
+ registry *localRegistry
+}
+
+func (l *labeledCounter) WithLabels(labels Labels) Counter {
+ return l.registry.getOrCreateCounter(l.name, labels)
+}
+
+func (l *labeledCounter) Inc() {
+ l.registry.getOrCreateCounter(l.name, nil).Inc()
+}
+
+func (l *labeledCounter) Add(value int64) {
+ l.registry.getOrCreateCounter(l.name, nil).Add(value)
+}
+
+func (l *labeledCounter) Get() int64 {
+ return l.registry.getOrCreateCounter(l.name, nil).Get()
+}
+
+// labeledGauge implements Gauge with label support
+type labeledGauge struct {
+ name string
+ registry *localRegistry
+}
+
+func (l *labeledGauge) WithLabels(labels Labels) Gauge {
+ return l.registry.getOrCreateGauge(l.name, labels)
+}
+
+func (l *labeledGauge) Set(value int64) {
+ l.registry.getOrCreateGauge(l.name, nil).Set(value)
+}
+
+func (l *labeledGauge) Inc() {
+ l.registry.getOrCreateGauge(l.name, nil).Inc()
+}
+
+func (l *labeledGauge) Dec() {
+ l.registry.getOrCreateGauge(l.name, nil).Dec()
+}
+
+func (l *labeledGauge) Add(value int64) {
+ l.registry.getOrCreateGauge(l.name, nil).Add(value)
+}
+
+func (l *labeledGauge) Sub(value int64) {
+ l.registry.getOrCreateGauge(l.name, nil).Sub(value)
+}
+
+func (l *labeledGauge) Get() int64 {
+ return l.registry.getOrCreateGauge(l.name, nil).Get()
+}
+
+// labeledHistogram implements Histogram with label support
+type labeledHistogram struct {
+ name string
+ registry *localRegistry
+}
+
+func (l *labeledHistogram) WithLabels(labels Labels) Histogram {
+ return l.registry.getOrCreateHistogram(l.name, labels)
+}
+
+func (l *labeledHistogram) Observe(value int64) {
+ l.registry.getOrCreateHistogram(l.name, nil).Observe(value)
+}
+
+type localRegistry struct {
+ sync.Mutex
+ gauges map[string]*localGauge
+ counters map[string]*localCounter
+ histograms map[string]*localHistogram
+}
+
+func newLocalRegistry() *localRegistry {
+ return &localRegistry{}
+}
+
+func (n *localRegistry) getOrCreateCounter(name string, labels Labels) Counter {
+ n.Lock()
+ defer n.Unlock()
+ if n.counters == nil {
+ n.counters = make(map[string]*localCounter)
+ }
+ key := makeMetricKey(name, labels)
+ if c, ok := n.counters[key]; ok {
+ return c
+ }
+ c := &localCounter{}
+ n.counters[key] = c
+ return c
+}
+
+func (n *localRegistry) getOrCreateGauge(name string, labels Labels) Gauge {
+ n.Lock()
+ defer n.Unlock()
+ if n.gauges == nil {
+ n.gauges = make(map[string]*localGauge)
+ }
+ key := makeMetricKey(name, labels)
+ if g, ok := n.gauges[key]; ok {
+ return g
+ }
+ g := &localGauge{}
+ n.gauges[key] = g
+ return g
+}
+
+func (n *localRegistry) getOrCreateHistogram(name string, labels Labels) Histogram {
+ n.Lock()
+ defer n.Unlock()
+ if n.histograms == nil {
+ n.histograms = make(map[string]*localHistogram)
+ }
+ key := makeMetricKey(name, labels)
+ if h, ok := n.histograms[key]; ok {
+ return h
+ }
+ h := &localHistogram{}
+ n.histograms[key] = h
+ return h
+}
+
+func (n *localRegistry) RegisterCounter(name, _ string, _ []string) Counter {
+ return &labeledCounter{
+ name: name,
+ registry: n,
+ }
+}
+
+func (n *localRegistry) RegisterGauge(name, _ string, _ []string) Gauge {
+ return &labeledGauge{
+ name: name,
+ registry: n,
+ }
+}
+
+func (n *localRegistry) RegisterHistogram(name, _ string, _ []int64, _ []string) Histogram {
+ return &labeledHistogram{
+ name: name,
+ registry: n,
+ }
+}
diff --git a/stats_test.go b/stats_test.go
new file mode 100644
index 0000000..bb9f04d
--- /dev/null
+++ b/stats_test.go
@@ -0,0 +1,607 @@
+package warc
+
+import (
+ "sync"
+ "testing"
+)
+
+// TestLocalCounter tests the localCounter implementation
+func TestLocalCounter(t *testing.T) {
+ c := &localCounter{}
+
+ // Test initial value
+ if c.Get() != 0 {
+ t.Errorf("Expected initial value 0, got %d", c.Get())
+ }
+
+ // Test Inc
+ c.Inc()
+ if c.Get() != 1 {
+ t.Errorf("Expected value 1 after Inc, got %d", c.Get())
+ }
+
+ // Test Add
+ c.Add(5)
+ if c.Get() != 6 {
+ t.Errorf("Expected value 6 after Add(5), got %d", c.Get())
+ }
+
+ // Test Add with negative value (counters should still accept it)
+ c.Add(-2)
+ if c.Get() != 4 {
+ t.Errorf("Expected value 4 after Add(-2), got %d", c.Get())
+ }
+}
+
+// TestLocalGauge tests the localGauge implementation
+func TestLocalGauge(t *testing.T) {
+ g := &localGauge{}
+
+ // Test initial value
+ if g.Get() != 0 {
+ t.Errorf("Expected initial value 0, got %d", g.Get())
+ }
+
+ // Test Set
+ g.Set(10)
+ if g.Get() != 10 {
+ t.Errorf("Expected value 10 after Set(10), got %d", g.Get())
+ }
+
+ // Test Inc
+ g.Inc()
+ if g.Get() != 11 {
+ t.Errorf("Expected value 11 after Inc, got %d", g.Get())
+ }
+
+ // Test Dec
+ g.Dec()
+ if g.Get() != 10 {
+ t.Errorf("Expected value 10 after Dec, got %d", g.Get())
+ }
+
+ // Test Add
+ g.Add(5)
+ if g.Get() != 15 {
+ t.Errorf("Expected value 15 after Add(5), got %d", g.Get())
+ }
+
+ // Test Sub
+ g.Sub(3)
+ if g.Get() != 12 {
+ t.Errorf("Expected value 12 after Sub(3), got %d", g.Get())
+ }
+
+ // Test Set to negative value
+ g.Set(-5)
+ if g.Get() != -5 {
+ t.Errorf("Expected value -5 after Set(-5), got %d", g.Get())
+ }
+}
+
+// TestLocalHistogram tests the localHistogram implementation
+func TestLocalHistogram(t *testing.T) {
+ h := &localHistogram{}
+
+ // Histogram's Observe method is a no-op, just ensure it doesn't panic
+ h.Observe(100)
+ h.Observe(0)
+ h.Observe(-50)
+}
+
+// TestLocalRegistryRegisterCounter tests the RegisterCounter method
+func TestLocalRegistryRegisterCounter(t *testing.T) {
+ registry := newLocalRegistry()
+
+ // Register a new counter
+ counter1 := registry.RegisterCounter("test_counter", "Test counter help", nil).WithLabels(nil)
+ if counter1 == nil {
+ t.Fatal("Expected counter to be created, got nil")
+ }
+
+ // Verify it's in the registry
+ if len(registry.counters) != 1 {
+ t.Errorf("Expected 1 counter in registry, got %d", len(registry.counters))
+ }
+
+ // Register the same counter again - should return existing one
+ counter2 := registry.RegisterCounter("test_counter", "Different help text", nil).WithLabels(nil)
+ if counter1 != counter2 {
+ t.Error("Expected RegisterCounter to return existing counter for same name")
+ }
+
+ // Verify still only one counter
+ if len(registry.counters) != 1 {
+ t.Errorf("Expected 1 counter in registry after re-registration, got %d", len(registry.counters))
+ }
+
+ // Register a different counter
+ counter3 := registry.RegisterCounter("another_counter", "Another counter", nil).WithLabels(nil)
+ if counter1 == counter3 {
+ t.Error("Expected different counter instances for different names")
+ }
+
+ if len(registry.counters) != 2 {
+ t.Errorf("Expected 2 counters in registry, got %d", len(registry.counters))
+ }
+}
+
+// TestLocalRegistryRegisterGauge tests the RegisterGauge method
+func TestLocalRegistryRegisterGauge(t *testing.T) {
+ registry := newLocalRegistry()
+
+ // Register a new gauge
+ gauge1 := registry.RegisterGauge("test_gauge", "Test gauge help", nil).WithLabels(nil)
+ if gauge1 == nil {
+ t.Fatal("Expected gauge to be created, got nil")
+ }
+
+ // Verify it's in the registry
+ if len(registry.gauges) != 1 {
+ t.Errorf("Expected 1 gauge in registry, got %d", len(registry.gauges))
+ }
+
+ // Register the same gauge again - should return existing one
+ gauge2 := registry.RegisterGauge("test_gauge", "Different help text", nil).WithLabels(nil)
+ if gauge1 != gauge2 {
+ t.Error("Expected RegisterGauge to return existing gauge for same name")
+ }
+
+ // Verify still only one gauge
+ if len(registry.gauges) != 1 {
+ t.Errorf("Expected 1 gauge in registry after re-registration, got %d", len(registry.gauges))
+ }
+
+ // Register a different gauge
+ gauge3 := registry.RegisterGauge("another_gauge", "Another gauge", nil).WithLabels(nil)
+ if gauge1 == gauge3 {
+ t.Error("Expected different gauge instances for different names")
+ }
+
+ if len(registry.gauges) != 2 {
+ t.Errorf("Expected 2 gauges in registry, got %d", len(registry.gauges))
+ }
+}
+
+// TestLocalRegistryRegisterHistogram tests the RegisterHistogram method
+func TestLocalRegistryRegisterHistogram(t *testing.T) {
+ registry := newLocalRegistry()
+
+ // Register a new histogram
+ labeledHist1 := registry.RegisterHistogram("test_histogram", "Test histogram help", []int64{1, 2, 3}, nil)
+ if labeledHist1 == nil {
+ t.Fatal("Expected labeled histogram to be created, got nil")
+ }
+
+ // Get the actual histogram instance with WithLabels
+ histogram1 := labeledHist1.WithLabels(nil)
+ if histogram1 == nil {
+ t.Fatal("Expected histogram to be created, got nil")
+ }
+
+ // Verify it's in the registry
+ if len(registry.histograms) != 1 {
+ t.Errorf("Expected 1 histogram in registry, got %d", len(registry.histograms))
+ }
+
+ // Register the same histogram again - calling WithLabels should return the same instance
+ histogram2 := registry.RegisterHistogram("test_histogram", "Different help text", []int64{5, 10, 15}, nil).WithLabels(nil)
+ if histogram1 != histogram2 {
+ t.Error("Expected same histogram instance for same name and labels")
+ }
+
+ // Verify still only one histogram
+ if len(registry.histograms) != 1 {
+ t.Errorf("Expected 1 histogram in registry after re-registration, got %d", len(registry.histograms))
+ }
+
+ // Register and use a different histogram
+ registry.RegisterHistogram("another_histogram", "Another histogram", nil, nil).WithLabels(nil)
+
+ if len(registry.histograms) != 2 {
+ t.Errorf("Expected 2 histograms in registry, got %d", len(registry.histograms))
+ }
+
+ // Verify both histograms are in the registry by key
+ // Note: keys are now the full metric keys including label info
+ if len(registry.histograms) < 2 {
+ t.Error("Expected both histograms to be in registry")
+ }
+}
+
+// TestLocalRegistryCounterFunctionality tests that registered counters work correctly
+func TestLocalRegistryCounterFunctionality(t *testing.T) {
+ registry := newLocalRegistry()
+ counter := registry.RegisterCounter("functional_counter", "Test", nil).WithLabels(nil)
+
+ counter.Inc()
+ if counter.Get() != 1 {
+ t.Errorf("Expected counter value 1, got %d", counter.Get())
+ }
+
+ counter.Add(10)
+ if counter.Get() != 11 {
+ t.Errorf("Expected counter value 11, got %d", counter.Get())
+ }
+}
+
+// TestLocalRegistryGaugeFunctionality tests that registered gauges work correctly
+func TestLocalRegistryGaugeFunctionality(t *testing.T) {
+ registry := newLocalRegistry()
+ gauge := registry.RegisterGauge("functional_gauge", "Test", nil).WithLabels(nil)
+
+ gauge.Set(100)
+ if gauge.Get() != 100 {
+ t.Errorf("Expected gauge value 100, got %d", gauge.Get())
+ }
+
+ gauge.Inc()
+ if gauge.Get() != 101 {
+ t.Errorf("Expected gauge value 101, got %d", gauge.Get())
+ }
+
+ gauge.Dec()
+ if gauge.Get() != 100 {
+ t.Errorf("Expected gauge value 100, got %d", gauge.Get())
+ }
+}
+
+// TestLocalRegistryConcurrentAccess tests thread-safety of the localRegistry
+func TestLocalRegistryConcurrentAccess(t *testing.T) {
+ registry := newLocalRegistry()
+ var wg sync.WaitGroup
+
+ // Number of concurrent goroutines
+ numGoroutines := 100
+
+ // Test concurrent counter registration
+ wg.Add(numGoroutines)
+ for i := 0; i < numGoroutines; i++ {
+ go func(id int) {
+ defer wg.Done()
+ // All goroutines try to register the same counter
+ counter := registry.RegisterCounter("shared_counter", "Test", nil).WithLabels(nil)
+ counter.Inc()
+ }(i)
+ }
+ wg.Wait()
+
+ // Should have exactly one counter
+ if len(registry.counters) != 1 {
+ t.Errorf("Expected 1 counter after concurrent registration, got %d", len(registry.counters))
+ }
+
+ // Counter should have been incremented by all goroutines
+ counter := registry.RegisterCounter("shared_counter", "Test", nil).WithLabels(nil)
+ if counter.Get() != int64(numGoroutines) {
+ t.Errorf("Expected counter value %d, got %d", numGoroutines, counter.Get())
+ }
+
+ // Test concurrent gauge registration
+ wg.Add(numGoroutines)
+ for i := 0; i < numGoroutines; i++ {
+ go func(id int) {
+ defer wg.Done()
+ gauge := registry.RegisterGauge("shared_gauge", "Test", nil).WithLabels(nil)
+ gauge.Inc()
+ }(i)
+ }
+ wg.Wait()
+
+ // Should have exactly one gauge
+ if len(registry.gauges) != 1 {
+ t.Errorf("Expected 1 gauge after concurrent registration, got %d", len(registry.gauges))
+ }
+
+ // Test concurrent histogram registration
+ wg.Add(numGoroutines)
+ for i := 0; i < numGoroutines; i++ {
+ go func(id int) {
+ defer wg.Done()
+ histogram := registry.RegisterHistogram("shared_histogram", "Test", nil, nil).WithLabels(nil)
+ histogram.Observe(int64(id))
+ }(i)
+ }
+ wg.Wait()
+
+ // Should have exactly one histogram
+ if len(registry.histograms) != 1 {
+ t.Errorf("Expected 1 histogram after concurrent registration, got %d", len(registry.histograms))
+ }
+}
+
+// TestLocalRegistryMultipleMetrics tests registering multiple different metrics
+func TestLocalRegistryMultipleMetrics(t *testing.T) {
+ registry := newLocalRegistry()
+
+ // Register multiple counters
+ for i := 0; i < 5; i++ {
+ name := "counter_" + string(rune('a'+i))
+ registry.RegisterCounter(name, "Test counter", nil).WithLabels(nil)
+ }
+
+ // Register multiple gauges
+ for i := 0; i < 5; i++ {
+ name := "gauge_" + string(rune('a'+i))
+ registry.RegisterGauge(name, "Test gauge", nil).WithLabels(nil)
+ }
+
+ // Register multiple histograms
+ for i := 0; i < 5; i++ {
+ name := "histogram_" + string(rune('a'+i))
+ registry.RegisterHistogram(name, "Test histogram", nil, nil).WithLabels(nil)
+ }
+
+ if len(registry.counters) != 5 {
+ t.Errorf("Expected 5 counters, got %d", len(registry.counters))
+ }
+
+ if len(registry.gauges) != 5 {
+ t.Errorf("Expected 5 gauges, got %d", len(registry.gauges))
+ }
+
+ if len(registry.histograms) != 5 {
+ t.Errorf("Expected 5 histograms, got %d", len(registry.histograms))
+ }
+}
+
+// TestLocalCounterConcurrentIncrement tests concurrent increments on a counter
+func TestLocalCounterConcurrentIncrement(t *testing.T) {
+ counter := &localCounter{}
+ var wg sync.WaitGroup
+ numGoroutines := 1000
+ incrementsPerGoroutine := 100
+
+ wg.Add(numGoroutines)
+ for i := 0; i < numGoroutines; i++ {
+ go func() {
+ defer wg.Done()
+ for j := 0; j < incrementsPerGoroutine; j++ {
+ counter.Inc()
+ }
+ }()
+ }
+ wg.Wait()
+
+ expected := int64(numGoroutines * incrementsPerGoroutine)
+ if counter.Get() != expected {
+ t.Errorf("Expected counter value %d, got %d", expected, counter.Get())
+ }
+}
+
+// TestLocalGaugeConcurrentOperations tests concurrent operations on a gauge
+func TestLocalGaugeConcurrentOperations(t *testing.T) {
+ gauge := &localGauge{}
+ var wg sync.WaitGroup
+ numGoroutines := 100
+
+ // Set initial value
+ gauge.Set(0)
+
+ // Half goroutines increment, half decrement
+ wg.Add(numGoroutines)
+ for i := 0; i < numGoroutines; i++ {
+ if i%2 == 0 {
+ go func() {
+ defer wg.Done()
+ gauge.Inc()
+ }()
+ } else {
+ go func() {
+ defer wg.Done()
+ gauge.Dec()
+ }()
+ }
+ }
+ wg.Wait()
+
+ // With equal increments and decrements, value should be 0
+ if gauge.Get() != 0 {
+ t.Errorf("Expected gauge value 0 after equal Inc/Dec operations, got %d", gauge.Get())
+ }
+}
+
+// TestLabelsToString tests the labelsToString function
+func TestLabelsToString(t *testing.T) {
+ tests := []struct {
+ name string
+ labels Labels
+ expected string
+ }{
+ {
+ name: "empty labels",
+ labels: nil,
+ expected: "",
+ },
+ {
+ name: "single label",
+ labels: Labels{"method": "GET"},
+ expected: `method="GET"`,
+ },
+ {
+ name: "multiple labels sorted",
+ labels: Labels{"method": "GET", "status": "200"},
+ expected: `method="GET",status="200"`,
+ },
+ {
+ name: "multiple labels reverse order",
+ labels: Labels{"status": "200", "method": "GET"},
+ expected: `method="GET",status="200"`, // Should be sorted
+ },
+ {
+ name: "labels with special characters",
+ labels: Labels{"path": "/api/v1/users", "method": "POST"},
+ expected: `method="POST",path="/api/v1/users"`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := labelsToString(tt.labels)
+ if result != tt.expected {
+ t.Errorf("labelsToString() = %q, expected %q", result, tt.expected)
+ }
+ })
+ }
+}
+
+// TestMakeMetricKey tests the makeMetricKey function
+func TestMakeMetricKey(t *testing.T) {
+ tests := []struct {
+ name string
+ metricName string
+ labels Labels
+ expected string
+ }{
+ {
+ name: "no labels",
+ metricName: "http_requests_total",
+ labels: nil,
+ expected: "http_requests_total",
+ },
+ {
+ name: "with labels",
+ metricName: "http_requests_total",
+ labels: Labels{"method": "GET", "status": "200"},
+ expected: `http_requests_total{method="GET",status="200"}`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := makeMetricKey(tt.metricName, tt.labels)
+ if result != tt.expected {
+ t.Errorf("makeMetricKey() = %q, expected %q", result, tt.expected)
+ }
+ })
+ }
+}
+
+// TestLocalRegistryWithLabels tests that metrics with different labels are separate
+func TestLocalRegistryWithLabels(t *testing.T) {
+ registry := newLocalRegistry()
+
+ // Register a counter with label names
+ httpRequests := registry.RegisterCounter("http_requests", "HTTP requests", []string{"method"})
+
+ // Use WithLabels to get counters for different label values
+ counter1 := httpRequests.WithLabels(Labels{"method": "GET"})
+ counter2 := httpRequests.WithLabels(Labels{"method": "POST"})
+
+ // Register a counter without labels
+ totalRequests := registry.RegisterCounter("total_requests", "Total requests", nil)
+ counter3 := totalRequests.WithLabels(nil)
+
+ // Increment each counter by different amounts
+ counter1.Add(10)
+ counter2.Add(20)
+ counter3.Add(30)
+
+ // Verify values are independent
+ if counter1.Get() != 10 {
+ t.Errorf("Expected counter1 value 10, got %d", counter1.Get())
+ }
+ if counter2.Get() != 20 {
+ t.Errorf("Expected counter2 value 20, got %d", counter2.Get())
+ }
+ if counter3.Get() != 30 {
+ t.Errorf("Expected counter3 value 30, got %d", counter3.Get())
+ }
+
+ // Verify all three are in the registry
+ if len(registry.counters) != 3 {
+ t.Errorf("Expected 3 counters in registry, got %d", len(registry.counters))
+ }
+}
+
+// TestLocalRegistryGaugeWithLabels tests gauges with labels
+func TestLocalRegistryGaugeWithLabels(t *testing.T) {
+ registry := newLocalRegistry()
+
+ // Register a gauge with label names
+ temperature := registry.RegisterGauge("temperature", "Temperature", []string{"location"})
+
+ // Use WithLabels to get gauges for different label values
+ gauge1 := temperature.WithLabels(Labels{"location": "indoor"})
+ gauge2 := temperature.WithLabels(Labels{"location": "outdoor"})
+
+ // Set different values
+ gauge1.Set(20)
+ gauge2.Set(15)
+
+ // Verify values are independent
+ if gauge1.Get() != 20 {
+ t.Errorf("Expected gauge1 value 20, got %d", gauge1.Get())
+ }
+ if gauge2.Get() != 15 {
+ t.Errorf("Expected gauge2 value 15, got %d", gauge2.Get())
+ }
+
+ // Verify both are in the registry
+ if len(registry.gauges) != 2 {
+ t.Errorf("Expected 2 gauges in registry, got %d", len(registry.gauges))
+ }
+}
+
+// TestLocalRegistryHistogramWithLabels tests histograms with labels
+func TestLocalRegistryHistogramWithLabels(t *testing.T) {
+ registry := newLocalRegistry()
+
+ // Register a histogram with label names
+ responseTime := registry.RegisterHistogram("response_time", "Response time", []int64{100, 200, 500}, []string{"endpoint"})
+
+ // Use WithLabels to get histograms for different label values
+ hist1 := responseTime.WithLabels(Labels{"endpoint": "/api"})
+ hist2 := responseTime.WithLabels(Labels{"endpoint": "/health"})
+
+ // Observe values (no-ops for local implementation, but verifies no panic)
+ hist1.Observe(150)
+ hist2.Observe(50)
+
+ // Verify both are in the registry as separate entries
+ if len(registry.histograms) != 2 {
+ t.Errorf("Expected 2 histograms in registry, got %d", len(registry.histograms))
+ }
+
+ // Verify we can retrieve them again with the same labels
+ hist1Again := responseTime.WithLabels(Labels{"endpoint": "/api"})
+ hist2Again := responseTime.WithLabels(Labels{"endpoint": "/health"})
+
+ // Should get the same instances back
+ if hist1 != hist1Again {
+ t.Error("Expected to get same histogram instance for same labels")
+ }
+ if hist2 != hist2Again {
+ t.Error("Expected to get same histogram instance for same labels")
+ }
+}
+
+// TestLocalRegistrySameLabelsDifferentOrder tests that label order doesn't matter
+func TestLocalRegistrySameLabelsDifferentOrder(t *testing.T) {
+ registry := newLocalRegistry()
+
+ // Register counter with label names
+ requests := registry.RegisterCounter("requests", "Requests", []string{"method", "status"})
+
+ // Get counter with labels in one order
+ counter1 := requests.WithLabels(Labels{"method": "GET", "status": "200"})
+ counter1.Add(5)
+
+ // Get counter with same labels in different order
+ counter2 := requests.WithLabels(Labels{"status": "200", "method": "GET"})
+
+ // Should return the same counter
+ if counter1 != counter2 {
+ t.Error("Expected same counter for same labels in different order")
+ }
+
+ // Value should be preserved
+ if counter2.Get() != 5 {
+ t.Errorf("Expected counter value 5, got %d", counter2.Get())
+ }
+
+ // Should only have one counter in registry
+ if len(registry.counters) != 1 {
+ t.Errorf("Expected 1 counter in registry, got %d", len(registry.counters))
+ }
+}
diff --git a/utils.go b/utils.go
index e053f8f..df6386c 100644
--- a/utils.go
+++ b/utils.go
@@ -6,6 +6,7 @@ import (
"encoding/binary"
"errors"
"io"
+ "net/url"
"os"
"strings"
"time"
@@ -40,7 +41,7 @@ func isHTTPRequest(line string) bool {
}
// NewWriter creates a new WARC writer.
-func NewWriter(writer io.Writer, fileName string, digestAlgorithm DigestAlgorithm, compression string, contentLengthHeader string, newFileCreation bool, dictionary []byte) (*Writer, error) {
+func NewWriter(writer io.Writer, fileName string, digestAlgorithm DigestAlgorithm, compression string, contentLengthHeader string, newFileCreation bool, dictionary []byte, stats StatsRegistry, logBackend LogBackend) (*Writer, error) {
if compression != "" {
switch strings.ToLower(compression) {
case "gzip":
@@ -52,6 +53,8 @@ func NewWriter(writer io.Writer, fileName string, digestAlgorithm DigestAlgorith
DigestAlgorithm: digestAlgorithm,
GZIPWriter: gzipWriter,
FileWriter: bufio.NewWriter(gzipWriter),
+ stats: stats,
+ logBackend: logBackend,
}, nil
case "zstd":
if newFileCreation && len(dictionary) > 0 {
@@ -94,6 +97,8 @@ func NewWriter(writer io.Writer, fileName string, digestAlgorithm DigestAlgorith
DigestAlgorithm: digestAlgorithm,
ZSTDWriter: zstdWriter,
FileWriter: bufio.NewWriter(zstdWriter),
+ stats: stats,
+ logBackend: logBackend,
}, nil
} else {
zstdWriter, err := zstd.NewWriter(writer, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
@@ -106,6 +111,8 @@ func NewWriter(writer io.Writer, fileName string, digestAlgorithm DigestAlgorith
DigestAlgorithm: digestAlgorithm,
ZSTDWriter: zstdWriter,
FileWriter: bufio.NewWriter(zstdWriter),
+ stats: stats,
+ logBackend: logBackend,
}, nil
}
default:
@@ -118,6 +125,8 @@ func NewWriter(writer io.Writer, fileName string, digestAlgorithm DigestAlgorith
Compression: "",
DigestAlgorithm: digestAlgorithm,
FileWriter: bufio.NewWriter(writer),
+ stats: stats,
+ logBackend: logBackend,
}, nil
}
@@ -227,3 +236,17 @@ func getContentLength(rwsc spooledtempfile.ReadWriteSeekCloser) int {
return int(fileInfo.Size())
}
}
+
+func proxyName(u *url.URL) string {
+ // get domain and replace dots and colons with underscores
+ domain := strings.ReplaceAll(u.Hostname(), ".", "_")
+ domain = strings.ReplaceAll(domain, ":", "_")
+ // get port and replace colons with underscores
+ port := strings.ReplaceAll(u.Port(), ":", "_")
+ // if port is empty, set it to 80
+ if port == "" {
+ port = "80"
+ }
+ // return domain and port
+ return domain + "_" + port
+}
diff --git a/utils_test.go b/utils_test.go
index 8903d6f..5b09ffb 100644
--- a/utils_test.go
+++ b/utils_test.go
@@ -1,6 +1,7 @@
package warc
import (
+ "net/url"
"testing"
)
@@ -47,3 +48,92 @@ func TestIsHTTPRequest(t *testing.T) {
}
}
}
+
+// Tests for the proxyName function
+func TestProxyName(t *testing.T) {
+ tests := []struct {
+ name string
+ urlStr string
+ expected string
+ }{
+ {
+ name: "domain with explicit port",
+ urlStr: "http://example.com:8080",
+ expected: "example_com_8080",
+ },
+ {
+ name: "domain without port (http defaults to empty, should use 80)",
+ urlStr: "http://example.com",
+ expected: "example_com_80",
+ },
+ {
+ name: "domain without port (https defaults to empty, should use 80)",
+ urlStr: "https://example.com",
+ expected: "example_com_80",
+ },
+ {
+ name: "domain with subdomain and port",
+ urlStr: "http://api.example.com:3000",
+ expected: "api_example_com_3000",
+ },
+ {
+ name: "domain with subdomain without port",
+ urlStr: "http://api.example.com",
+ expected: "api_example_com_80",
+ },
+ {
+ name: "localhost with port",
+ urlStr: "http://localhost:8080",
+ expected: "localhost_8080",
+ },
+ {
+ name: "localhost without port",
+ urlStr: "http://localhost",
+ expected: "localhost_80",
+ },
+ {
+ name: "IPv4 address with port",
+ urlStr: "http://192.168.1.1:8080",
+ expected: "192_168_1_1_8080",
+ },
+ {
+ name: "IPv4 address without port",
+ urlStr: "http://192.168.1.1",
+ expected: "192_168_1_1_80",
+ },
+ {
+ name: "IPv6 address with port",
+ urlStr: "http://[2001:db8::1]:8080",
+ expected: "2001_db8__1_8080",
+ },
+ {
+ name: "IPv6 address without port",
+ urlStr: "http://[2001:db8::1]",
+ expected: "2001_db8__1_80",
+ },
+ {
+ name: "domain with port 443",
+ urlStr: "https://example.com:443",
+ expected: "example_com_443",
+ },
+ {
+ name: "domain with multiple subdomains",
+ urlStr: "http://api.v2.example.com:9000",
+ expected: "api_v2_example_com_9000",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ u, err := url.Parse(tt.urlStr)
+ if err != nil {
+ t.Fatalf("Failed to parse URL %s: %v", tt.urlStr, err)
+ }
+
+ result := proxyName(u)
+ if result != tt.expected {
+ t.Errorf("proxyName(%s) = %s, expected %s", tt.urlStr, result, tt.expected)
+ }
+ })
+ }
+}
diff --git a/warc.go b/warc.go
index d8e111a..8987aa5 100644
--- a/warc.go
+++ b/warc.go
@@ -32,21 +32,12 @@ type RotatorSettings struct {
WARCSize float64
// WARCWriterPoolSize defines the number of parallel WARC writers
WARCWriterPoolSize int
+ // StatsRegistry is used to store stats about gowarc
+ StatsRegistry StatsRegistry
+ // LogBackend is used to log events from gowarc
+ LogBackend LogBackend
}
-var (
- // Create a couple of counters for tracking various stats
- DataTotal atomic.Int64
-
- CDXDedupeTotalBytes atomic.Int64
- DoppelgangerDedupeTotalBytes atomic.Int64
- LocalDedupeTotalBytes atomic.Int64
-
- CDXDedupeTotal atomic.Int64
- DoppelgangerDedupeTotal atomic.Int64
- LocalDedupeTotal atomic.Int64
-)
-
// NewWARCRotator creates and return a channel that can be used
// to communicate records to be written to WARC files to the
// recordWriter function running in a goroutine
@@ -113,13 +104,14 @@ func recordWriter(settings *RotatorSettings, records chan *RecordBatch, done cha
)
// Create and open the initial file
+ settings.LogBackend.Info("creating initial WARC file", "file", currentFileName)
warcFile, err := os.Create(settings.OutputDirectory + currentFileName)
if err != nil {
panic(err)
}
// Initialize WARC writer
- warcWriter, err := NewWriter(warcFile, currentFileName, settings.digestAlgorithm, settings.Compression, "", true, dictionary)
+ warcWriter, err := NewWriter(warcFile, currentFileName, settings.digestAlgorithm, settings.Compression, "", true, dictionary, settings.StatsRegistry, settings.LogBackend)
if err != nil {
panic(err)
}
@@ -137,7 +129,7 @@ func recordWriter(settings *RotatorSettings, records chan *RecordBatch, done cha
panic(err)
}
- warcWriter, err = NewWriter(warcFile, currentFileName, settings.digestAlgorithm, settings.Compression, "", false, dictionary)
+ warcWriter, err = NewWriter(warcFile, currentFileName, settings.digestAlgorithm, settings.Compression, "", false, dictionary, settings.StatsRegistry, settings.LogBackend)
if err != nil {
panic(err)
}
@@ -148,6 +140,7 @@ func recordWriter(settings *RotatorSettings, records chan *RecordBatch, done cha
if more {
if isFileSizeExceeded(warcFile, settings.WARCSize) {
// WARC file size exceeded settings.WarcSize
+ settings.LogBackend.Info("WARC file size limit exceeded, rotating to new file", "currentFile", currentFileName, "sizeLimit", settings.WARCSize)
// The WARC file is renamed to remove the .open suffix
err := os.Rename(path.Join(settings.OutputDirectory, currentFileName), strings.TrimSuffix(path.Join(settings.OutputDirectory, currentFileName), ".open"))
if err != nil {
@@ -170,13 +163,14 @@ func recordWriter(settings *RotatorSettings, records chan *RecordBatch, done cha
// Create the new file and automatically increment the serial inside of GenerateWarcFileName
currentFileName = getNextWARCFilename(settings.OutputDirectory, settings.Prefix, settings.Compression, serial)
+ settings.LogBackend.Info("creating new WARC file after rotation", "file", currentFileName)
warcFile, err = os.Create(settings.OutputDirectory + currentFileName)
if err != nil {
panic(err)
}
// Initialize new WARC writer
- warcWriter, err = NewWriter(warcFile, currentFileName, settings.digestAlgorithm, settings.Compression, "", true, dictionary)
+ warcWriter, err = NewWriter(warcFile, currentFileName, settings.digestAlgorithm, settings.Compression, "", true, dictionary, settings.StatsRegistry, settings.LogBackend)
if err != nil {
panic(err)
}
@@ -198,7 +192,7 @@ func recordWriter(settings *RotatorSettings, records chan *RecordBatch, done cha
// Write all the records of the record batch
for _, record := range recordBatch.Records {
- warcWriter, err = NewWriter(warcFile, currentFileName, settings.digestAlgorithm, settings.Compression, record.Header.Get("Content-Length"), false, dictionary)
+ warcWriter, err = NewWriter(warcFile, currentFileName, settings.digestAlgorithm, settings.Compression, record.Header.Get("Content-Length"), false, dictionary, settings.StatsRegistry, settings.LogBackend)
if err != nil {
panic(err)
}
@@ -251,6 +245,7 @@ func recordWriter(settings *RotatorSettings, records chan *RecordBatch, done cha
panic(err)
}
+ settings.LogBackend.Info("WARC writer shutting down cleanly", "finalFile", currentFileName)
done <- true
return
diff --git a/write.go b/write.go
index a173856..6088f33 100644
--- a/write.go
+++ b/write.go
@@ -8,8 +8,8 @@ import (
"strings"
"time"
- "github.com/internetarchive/gowarc/pkg/spooledtempfile"
"github.com/google/uuid"
+ "github.com/internetarchive/gowarc/pkg/spooledtempfile"
"github.com/klauspost/compress/zstd"
)
@@ -22,6 +22,9 @@ type Writer struct {
Compression string
DigestAlgorithm DigestAlgorithm
ParallelGZIP bool
+
+ stats StatsRegistry
+ logBackend LogBackend
}
// RecordBatch is a structure that contains a bunch of
@@ -104,11 +107,13 @@ func (w *Writer) WriteRecord(r *Record) (recordID string, err error) {
r.Content.Seek(0, 0)
if written, err = io.Copy(w.FileWriter, r.Content); err != nil {
+ w.logBackend.Error("failed to write WARC record content", "file", w.FileName, "error", err)
return recordID, err
}
if written > 0 {
- DataTotal.Add(written)
+ w.stats.RegisterCounter(totalDataWritten, totalDataWrittenHelp, nil).WithLabels(nil).Add(written)
+ w.logBackend.Debug("WARC record written", "file", w.FileName, "bytes", written, "recordID", recordID)
}
if _, err := io.WriteString(w.FileWriter, "\r\n\r\n"); err != nil {