Expand list of cloud IP addresses

cmd/iprange/main.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"bytes"
+
+	"go/format"
+	"slices"
+
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	ranges := [][2]string{
+		{"AWS", "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/refs/heads/main/aws/aws_ips_merged.txt"},
+		{"Azure", "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/refs/heads/main/azure/azure_ips_merged.txt"},
+		{"DigitalOcean", "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/refs/heads/main/digitalocean/digitalocean_ips_merged.txt"},
+		{"GoogleCloud", "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/refs/heads/main/googlecloud/googlecloud_ips_merged.txt"},
+		{"Hetzner", "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/refs/heads/main/hetzner/hetzner_ips_merged.txt"},
+		{"Linode", "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/refs/heads/main/linode/linode_ips_merged.txt"},
+		{"Alibaba", "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/refs/heads/main/alibaba/alibaba_ips_merged.txt"},
+		{"Oracle", "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/refs/heads/main/oracle/oracle_ips_merged.txt"},
+		{"OVH", "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/refs/heads/main/ovhcloud/ovhcloud_ips_merged.txt"},
+	}
+
+	if err := os.MkdirAll(".cache", 0o755); err != nil {
+		panic(err)
+	}
+
+	type ipRange struct {
+		bot    string
+		prefix netip.Prefix
+	}
+
+	var (
+		ranges4 = make([]ipRange, 0, 8192)
+		ranges6 = make([]ipRange, 0, 8192)
+	)
+	for _, r := range ranges {
+		cache := ".cache/" + filepath.Base(r[1])
+		data, err := os.ReadFile(cache)
+		if err != nil {
+			if !errors.Is(err, fs.ErrNotExist) {
+				panic(err)
+			}
+			resp, err := http.Get(r[1])
+			if err != nil {
+				panic(err)
+			}
+			defer resp.Body.Close()
+			data, err = io.ReadAll(resp.Body)
+			if err != nil {
+				panic(err)
+			}
+			err = os.WriteFile(cache, data, 0o644)
+			if err != nil {
+				panic(err)
+			}
+		}
+
+		for line := range strings.SplitSeq(string(data), "\n") {
+			if len(line) == 0 {
+				continue
+			}
+
+			prefix := netip.MustParsePrefix(line)
+			addr := prefix.Addr()
+			if addr.Is4() {
+				ranges4 = append(ranges4, ipRange{bot: r[0], prefix: prefix})
+			} else {
+				ranges6 = append(ranges6, ipRange{bot: r[0], prefix: prefix})
+			}
+		}
+	}
+
+	out := new(bytes.Buffer)
+	out.WriteString("// Code generated by cmd/iprange command; DO NOT EDIT.\n\npackage isbot\n\n")
+
+	out.WriteString("var ranges4 = `\n\t")
+	slices.SortFunc(ranges4, func(a, b ipRange) int { return a.prefix.Addr().Compare(b.prefix.Addr()) })
+	for i, r := range ranges4 {
+		if i > 0 && i%6 == 0 {
+			out.WriteString("\n\t")
+		}
+		fmt.Fprintf(out, "%s,%s ", r.prefix, r.bot)
+	}
+	out.WriteString("\n`\n\n")
+
+	out.WriteString("var ranges6 = `\n\t")
+	slices.SortFunc(ranges6, func(a, b ipRange) int { return a.prefix.Addr().Compare(b.prefix.Addr()) })
+	for i, r := range ranges6 {
+		if i > 0 && i%6 == 0 {
+			out.WriteString("\n\t")
+		}
+		fmt.Fprintf(out, "%s,%s ", r.prefix, r.bot)
+	}
+	out.WriteString("\n`\n\n")
+
+	out2, err := format.Source(out.Bytes())
+	if err != nil {
+		fmt.Println(err)
+		fmt.Print(out)
+		os.Exit(1)
+	}
+	err = os.WriteFile("ip_ranges.new.go", out2, 0o644)
+	if err != nil {
+		panic(err)
+	}
+	err = os.Rename("ip_ranges.new.go", "ip_ranges.go")
+	if err != nil {
+		panic(err)
+	}
+}

ip.go

@@ -1,24 +1,72 @@
-//go:generate ./mkip
+//go:generate go run ./cmd/iprange
 
 package isbot
 
 import (
 	"net/netip"
+	"strings"
 )
 
 type ipRange struct {
 	bot    Result
 	prefix netip.Prefix
 }
 
-func parseNet(ip string, b Result) ipRange {
-	prefix, err := netip.ParsePrefix(ip)
-	if err != nil {
-		panic(err)
+func botname(n string) Result {
+	switch n {
+	case "AWS":
+		return BotRangeAWS
+	case "DigitalOcean":
+		return BotRangeDigitalOcean
+	case "ServersCom":
+		return BotRangeServersCom
+	case "GoogleCloud":
+		return BotRangeGoogleCloud
+	case "Hetzner":
+		return BotRangeHetzner
+	case "Azure":
+		return BotRangeAzure
+	case "Alibaba":
+		return BotRangeAlibaba
+	case "Linode":
+		return BotRangeLinode
+	case "Oracle":
+		return BotRangeOracle
+	case "OVH":
+		return BotRangeOVH
 	}
-	return ipRange{bot: b, prefix: prefix}
+	panic(n)
 }
 
+var ipRanges4 = func() map[byte][]ipRange {
+	m := make(map[byte][]ipRange)
+	for f := range strings.FieldsSeq(ranges4) {
+		ip, name, ok := strings.Cut(f, ",")
+		if !ok {
+			panic(f)
+		}
+		prefix := netip.MustParsePrefix(ip)
+		k := prefix.Addr().As4()[0]
+		m[k] = append(m[k], ipRange{prefix: prefix, bot: botname(name)})
+	}
+	return m
+}()
+
+var ipRanges6 = func() map[[2]byte][]ipRange {
+	m := make(map[[2]byte][]ipRange)
+	for f := range strings.FieldsSeq(ranges6) {
+		ip, name, ok := strings.Cut(f, ",")
+		if !ok {
+			panic(f)
+		}
+		prefix := netip.MustParsePrefix(ip)
+		as := prefix.Addr().As16()
+		k := [2]byte{as[0], as[1]}
+		m[k] = append(m[k], ipRange{prefix: prefix, bot: botname(name)})
+	}
+	return m
+}()
+
 // IPRange checks if this IP address is from a range that should normally never
 // send browser requests, such as AWS and other cloud providers.
 func IPRange(addr string) Result {
@@ -30,7 +78,15 @@ func IPRange(addr string) Result {
 		return NoBotKnown
 	}
 
-	for _, r := range ipRanges {
+	var ranges []ipRange
+	if ip.Is4() {
+		ranges = ipRanges4[ip.As4()[0]]
+	} else {
+		as := ip.As16()
+		ranges = ipRanges6[[2]byte{as[0], as[1]}]
+	}
+
+	for _, r := range ranges {
 		if r.prefix.Contains(ip) {
 			return r.bot
 		}