VIndex for Static CT logs

go.mod

@@ -3,6 +3,7 @@ module github.com/transparency-dev/incubator
 go 1.25.0
 
 require (
+	filippo.io/sunlight v0.8.0
 	filippo.io/torchwood v0.9.1-0.20260708154658-e0a6396dd50c
 	github.com/cockroachdb/pebble v1.1.5
 	github.com/go-git/go-git/v5 v5.19.1
@@ -16,7 +17,9 @@ require (
 	go.opentelemetry.io/otel/exporters/prometheus v0.66.0
 	go.opentelemetry.io/otel/metric v1.44.0
 	go.opentelemetry.io/otel/sdk/metric v1.44.0
+	golang.org/x/crypto v0.53.0
 	golang.org/x/mod v0.38.0
+	golang.org/x/net v0.55.0
 	golang.org/x/sync v0.22.0
 	k8s.io/klog/v2 v2.140.0
 )
@@ -46,6 +49,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/certificate-transparency-go v1.3.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -69,9 +73,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
 	go.opentelemetry.io/otel/trace v1.44.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
-	golang.org/x/crypto v0.53.0 // indirect
 	golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect
-	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/sys v0.46.0 // indirect
 	golang.org/x/text v0.38.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect

go.sum

@@ -2,6 +2,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e h1:VsUbObBMxXlc23Eb9VeeJYE4jvTs87qa5RqSN2U5FJU=
 filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e/go.mod h1:32qQ5yj3R24Eu03iWFWchdC3OB653wPvoepWejkefbY=
+filippo.io/sunlight v0.8.0 h1:7ytoUj2KmU5k4ogDSLwEtCoEjjrTZsh+g++UIfTGpM4=
+filippo.io/sunlight v0.8.0/go.mod h1:gJ1qFtjHWqj9j4f5M2fnaER6ZFPUkTrRz4/pTamneDg=
 filippo.io/torchwood v0.9.1-0.20260708154658-e0a6396dd50c h1:t259W9JzlCso86LB0V92ZmRJluFg1wEsiH4OonKAB5U=
 filippo.io/torchwood v0.9.1-0.20260708154658-e0a6396dd50c/go.mod h1:kKEN0olmFcNGfsPWT3M46wig9CUz7/vHIAENPh9pLXc=
 github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
@@ -41,8 +43,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE=
 github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
 github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -72,6 +75,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J
 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A=
+github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -110,8 +115,9 @@ github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
 github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
 github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=

vindex/cmd/client/client.go

@@ -19,25 +19,29 @@ package main
 
 import (
 	"context"
+	"crypto/x509"
+	"encoding/base64"
 	"errors"
 	"flag"
 	"fmt"
 	"net/http"
 	"slices"
 
+	fnote "github.com/transparency-dev/formats/note"
 	"github.com/transparency-dev/incubator/vindex/client"
 	"golang.org/x/mod/sumdb/note"
 	"k8s.io/klog/v2"
 )
 
 var (
-	vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.")
-	inLogBaseURL  = flag.String("in_log_base_url", "", "The base URL of the input log.")
-	lookup        = flag.String("lookup", "", "The key to look up in the vindex.")
-	outLogPubKey  = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.")
-	inLogPubKey   = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint.")
-	inLogOrigin   = flag.String("in_log_origin", "", "Optional: allows the Input Log Origin string to be configured to something other than the public key name.")
-	minIdx        = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.")
+	vindexBaseURL  = flag.String("vindex_base_url", "", "The base URL of the vindex server.")
+	inLogBaseURL   = flag.String("in_log_base_url", "", "The base URL of the input log.")
+	lookup         = flag.String("lookup", "", "The key to look up in the vindex.")
+	outLogPubKey   = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint. Required.")
+	inLogPubKey    = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint. Required.")
+	inLogPubKeyDER = flag.String("in_log_pub_key_der", "", "For CT logs. The public key to use to verify the input log checkpoint. Required, along with in_log_origin.")
+	inLogOrigin    = flag.String("in_log_origin", "", "Required if in_log_pub_key_der is used. Otherwise, allows the Input Log Origin string to be configured to something other than the public key name.")
+	minIdx         = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.")
 )
 
 func main() {
@@ -104,10 +108,7 @@ func newVIndexClientFromFlags() *client.VIndexClient {
 	if *outLogPubKey == "" {
 		klog.Exitf("out_log_pub_key must be provided")
 	}
-	inV, err := note.NewVerifier(*inLogPubKey)
-	if err != nil {
-		klog.Exitf("failed to construct input log verifier: %v", err)
-	}
+	inV := inputLogVerifierFromFlags()
 	outV, err := note.NewVerifier(*outLogPubKey)
 	if err != nil {
 		klog.Exitf("failed to construct output log verifier: %v", err)
@@ -123,13 +124,7 @@ func newInputLogClientFromFlags() *client.InputLogClient {
 	if *inLogBaseURL == "" {
 		klog.Exit("in_log_base_url flag must be provided")
 	}
-	if *inLogPubKey == "" {
-		klog.Exitf("in_log_pub_key must be provided")
-	}
-	v, err := note.NewVerifier(*inLogPubKey)
-	if err != nil {
-		klog.Exitf("failed to construct input log verifier: %v", err)
-	}
+	v := inputLogVerifierFromFlags()
 	origin := *inLogOrigin
 	if len(origin) == 0 {
 		origin = v.Name()
@@ -140,3 +135,37 @@ func newInputLogClientFromFlags() *client.InputLogClient {
 	}
 	return c
 }
+
+func inputLogVerifierFromFlags() note.Verifier {
+	if (*inLogPubKey == "") == (*inLogPubKeyDER == "") {
+		klog.Exitf("Must provide exactly one --in_log_pub_key* flag")
+	}
+	if *inLogPubKeyDER != "" && *inLogOrigin == "" {
+		klog.Exitf("in_log_origin must be provided when using in_log_pub_key_der")
+	}
+	if *inLogPubKey != "" {
+		v, err := note.NewVerifier(*inLogPubKey)
+		if err != nil {
+			klog.Exitf("failed to construct input log verifier: %v", err)
+		}
+		return v
+	}
+	derBytes, err := base64.StdEncoding.DecodeString(*inLogPubKeyDER)
+	if err != nil {
+		klog.Exitf("Error decoding public key: %s", err)
+	}
+	pub, err := x509.ParsePKIXPublicKey(derBytes)
+	if err != nil {
+		klog.Exitf("Error parsing public key: %v", err)
+	}
+
+	verifierKey, err := fnote.RFC6962VerifierString(*inLogOrigin, pub)
+	if err != nil {
+		klog.Exitf("Error creating RFC6962 verifier string: %v", err)
+	}
+	v, err := fnote.NewVerifier(verifierKey)
+	if err != nil {
+		klog.Exitf("Error creating verifier: %v", err)
+	}
+	return v
+}

vindex/cmd/ct/README.md

@@ -0,0 +1,77 @@
+## Verifiable Index: CT
+
+This is a demo of pulling the contents of a tile-based CT log into a [Verifiable Index](../../README.md).
+
+[tlog-tiles]: https://c2sp.org/tlog-tiles
+[Tessera]: https://github.com/transparency-dev/tessera
+
+The CT Input Log is processed, with each entry being indexed on all common names defined in the cert. 
+This allows the owner of a domain to look up all certs for their domain, in a way that is fully verified.
+
+## Running
+
+The static CT Input Log is expected to be available for reading at a URL provided by the `--monitoring_url` flag.
+This is the base directory that should contain the checkpoint file.
+The Verifiable Index and Output Log are constructed locally, persisted to local disk (in the `--storage_dir` directory), and hosted via a web server.
+
+```shell
+OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ \
+go run ./vindex/cmd/ct \
+  --storage_dir ~/vindex-ct/ \
+  --origin="coachandhorses2026h1.staging.certificate.transparency.goog"  \
+  --public_key="MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g==" \
+  --monitoring_url="https://storage.googleapis.com/coachandhorses2026h1.staging.certificate.transparency.goog"
+```
+
+### Resource Requirements
+
+Running this demo has the following estimated system requirements (based on indexing ~180M certs from a staging log):
+*   **Architecture:** **64-bit OS** (Linux/Unix). The MPT library reserves a large virtual address space (16 TB) and cannot run on 32-bit systems.
+*   **RAM:** **32 GB+** (observed ~31 GB physical RAM usage). The current prototype stores the key-value index in a raw Go map in memory.
+*   **CPU:** **8+ cores** (actively utilizes ~6 cores during ingestion).
+*   **Disk:** **100 GB+ SSD** (uses ~45 GB for WAL and MPT files, fast I/O is required).
+
+> [!NOTE]
+> The high memory usage is a limitation of the current prototype's in-memory key-value store. The planned [v1 architecture](../../docs/v1/IMPLEMENTATION.md) will move this store to a disk-backed Pebble database, which is expected to drastically reduce RAM requirements to approximately 6-8 GB.
+
+Running the above will run a web server hosting the following URLs:
+ - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go)
+ - `/outputlog/` - the [tlog-tiles][] base URL for the output log
+
+To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above):
+
+```shell
+# To inspect the Output Log
+go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/outputlog/ --custom_log_vkey=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8
+```
+
+Use left/right cursor to browse, and `q` to quit.
+
+A domain indexed by the verifiable map can be looked up using the following command:
+
+```shell
+go run ./vindex/cmd/client \
+  --vindex_base_url http://localhost:8088/vindex/ \
+  --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 \
+  --in_log_pub_key_der=MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g== \
+  --in_log_origin=coachandhorses2026h1.staging.certificate.transparency.goog \
+  --lookup=google.com
+
+I0610 15:02:17.112527   87150 client.go:83] in_log_base_url not provided, so cannot dereference pointers
+148000245
+151898263
+152014951
+152015244
+152015262
+152015307
+...
+154306826
+154307178
+154307232
+154307321
+154368790
+154368845
+```
+
+To view the certs at the index, [woodpecker](https://github.com/mhutchinson/woodpecker) can be used.
+