diff --git a/ct b/ct new file mode 100755 index 0000000..363274b Binary files /dev/null and b/ct differ diff --git a/go.mod b/go.mod index d15ebd0..4b87ab4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/transparency-dev/incubator go 1.25.0 require ( + filippo.io/sunlight v0.8.0 filippo.io/torchwood v0.9.1-0.20260708154658-e0a6396dd50c github.com/cockroachdb/pebble v1.1.5 github.com/go-git/go-git/v5 v5.19.1 @@ -16,7 +17,9 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.66.0 go.opentelemetry.io/otel/metric v1.44.0 go.opentelemetry.io/otel/sdk/metric v1.44.0 + golang.org/x/crypto v0.53.0 golang.org/x/mod v0.38.0 + golang.org/x/net v0.55.0 golang.org/x/sync v0.22.0 k8s.io/klog/v2 v2.140.0 ) @@ -46,6 +49,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/certificate-transparency-go v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -69,9 +73,7 @@ require ( go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect - golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect google.golang.org/protobuf v1.36.11 // indirect diff --git a/go.sum b/go.sum index 96ee30e..525ef00 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e h1:VsUbObBMxXlc23Eb9VeeJYE4jvTs87qa5RqSN2U5FJU= filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e/go.mod h1:32qQ5yj3R24Eu03iWFWchdC3OB653wPvoepWejkefbY= +filippo.io/sunlight v0.8.0 h1:7ytoUj2KmU5k4ogDSLwEtCoEjjrTZsh+g++UIfTGpM4= +filippo.io/sunlight v0.8.0/go.mod h1:gJ1qFtjHWqj9j4f5M2fnaER6ZFPUkTrRz4/pTamneDg= filippo.io/torchwood v0.9.1-0.20260708154658-e0a6396dd50c h1:t259W9JzlCso86LB0V92ZmRJluFg1wEsiH4OonKAB5U= filippo.io/torchwood v0.9.1-0.20260708154658-e0a6396dd50c/go.mod h1:kKEN0olmFcNGfsPWT3M46wig9CUz7/vHIAENPh9pLXc= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= @@ -41,8 +43,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -72,6 +75,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/certificate-transparency-go v1.3.2 h1:9ahSNZF2o7SYMaKaXhAumVEzXB2QaayzII9C8rv7v+A= +github.com/google/certificate-transparency-go v1.3.2/go.mod h1:H5FpMUaGa5Ab2+KCYsxg6sELw3Flkl7pGZzWdBoYLXs= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -110,8 +115,9 @@ github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index a0b32ec..2902f70 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -19,25 +19,29 @@ package main import ( "context" + "crypto/x509" + "encoding/base64" "errors" "flag" "fmt" "net/http" "slices" + fnote "github.com/transparency-dev/formats/note" "github.com/transparency-dev/incubator/vindex/client" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" ) var ( - vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") - inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") - lookup = flag.String("lookup", "", "The key to look up in the vindex.") - outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") - inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint.") - inLogOrigin = flag.String("in_log_origin", "", "Optional: allows the Input Log Origin string to be configured to something other than the public key name.") - minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") + vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") + inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") + lookup = flag.String("lookup", "", "The key to look up in the vindex.") + outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint. Required.") + inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint. Required.") + inLogPubKeyDER = flag.String("in_log_pub_key_der", "", "For CT logs. The public key to use to verify the input log checkpoint. Required, along with in_log_origin.") + inLogOrigin = flag.String("in_log_origin", "", "Required if in_log_pub_key_der is used. Otherwise, allows the Input Log Origin string to be configured to something other than the public key name.") + minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") ) func main() { @@ -104,10 +108,7 @@ func newVIndexClientFromFlags() *client.VIndexClient { if *outLogPubKey == "" { klog.Exitf("out_log_pub_key must be provided") } - inV, err := note.NewVerifier(*inLogPubKey) - if err != nil { - klog.Exitf("failed to construct input log verifier: %v", err) - } + inV := inputLogVerifierFromFlags() outV, err := note.NewVerifier(*outLogPubKey) if err != nil { klog.Exitf("failed to construct output log verifier: %v", err) @@ -123,13 +124,7 @@ func newInputLogClientFromFlags() *client.InputLogClient { if *inLogBaseURL == "" { klog.Exit("in_log_base_url flag must be provided") } - if *inLogPubKey == "" { - klog.Exitf("in_log_pub_key must be provided") - } - v, err := note.NewVerifier(*inLogPubKey) - if err != nil { - klog.Exitf("failed to construct input log verifier: %v", err) - } + v := inputLogVerifierFromFlags() origin := *inLogOrigin if len(origin) == 0 { origin = v.Name() @@ -140,3 +135,37 @@ func newInputLogClientFromFlags() *client.InputLogClient { } return c } + +func inputLogVerifierFromFlags() note.Verifier { + if (*inLogPubKey == "") == (*inLogPubKeyDER == "") { + klog.Exitf("Must provide exactly one --in_log_pub_key* flag") + } + if *inLogPubKeyDER != "" && *inLogOrigin == "" { + klog.Exitf("in_log_origin must be provided when using in_log_pub_key_der") + } + if *inLogPubKey != "" { + v, err := note.NewVerifier(*inLogPubKey) + if err != nil { + klog.Exitf("failed to construct input log verifier: %v", err) + } + return v + } + derBytes, err := base64.StdEncoding.DecodeString(*inLogPubKeyDER) + if err != nil { + klog.Exitf("Error decoding public key: %s", err) + } + pub, err := x509.ParsePKIXPublicKey(derBytes) + if err != nil { + klog.Exitf("Error parsing public key: %v", err) + } + + verifierKey, err := fnote.RFC6962VerifierString(*inLogOrigin, pub) + if err != nil { + klog.Exitf("Error creating RFC6962 verifier string: %v", err) + } + v, err := fnote.NewVerifier(verifierKey) + if err != nil { + klog.Exitf("Error creating verifier: %v", err) + } + return v +} diff --git a/vindex/cmd/ct/README.md b/vindex/cmd/ct/README.md new file mode 100644 index 0000000..da63d19 --- /dev/null +++ b/vindex/cmd/ct/README.md @@ -0,0 +1,77 @@ +## Verifiable Index: CT + +This is a demo of pulling the contents of a tile-based CT log into a [Verifiable Index](../../README.md). + +[tlog-tiles]: https://c2sp.org/tlog-tiles +[Tessera]: https://github.com/transparency-dev/tessera + +The CT Input Log is processed, with each entry being indexed on all common names defined in the cert. +This allows the owner of a domain to look up all certs for their domain, in a way that is fully verified. + +## Running + +The static CT Input Log is expected to be available for reading at a URL provided by the `--monitoring_url` flag. +This is the base directory that should contain the checkpoint file. +The Verifiable Index and Output Log are constructed locally, persisted to local disk (in the `--storage_dir` directory), and hosted via a web server. + +```shell +OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ \ +go run ./vindex/cmd/ct \ + --storage_dir ~/vindex-ct/ \ + --origin="coachandhorses2026h1.staging.certificate.transparency.goog" \ + --public_key="MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g==" \ + --monitoring_url="https://storage.googleapis.com/coachandhorses2026h1.staging.certificate.transparency.goog" +``` + +### Resource Requirements + +Running this demo has the following estimated system requirements (based on indexing ~180M certs from a staging log): +* **Architecture:** **64-bit OS** (Linux/Unix). The MPT library reserves a large virtual address space (16 TB) and cannot run on 32-bit systems. +* **RAM:** **32 GB+** (observed ~31 GB physical RAM usage). The current prototype stores the key-value index in a raw Go map in memory. +* **CPU:** **8+ cores** (actively utilizes ~6 cores during ingestion). +* **Disk:** **100 GB+ SSD** (uses ~45 GB for WAL and MPT files, fast I/O is required). + +> [!NOTE] +> The high memory usage is a limitation of the current prototype's in-memory key-value store. The planned [v1 architecture](../../docs/v1/IMPLEMENTATION.md) will move this store to a disk-backed Pebble database, which is expected to drastically reduce RAM requirements to approximately 6-8 GB. + +Running the above will run a web server hosting the following URLs: + - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) + - `/outputlog/` - the [tlog-tiles][] base URL for the output log + +To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): + +```shell +# To inspect the Output Log +go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/outputlog/ --custom_log_vkey=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 +``` + +Use left/right cursor to browse, and `q` to quit. + +A domain indexed by the verifiable map can be looked up using the following command: + +```shell +go run ./vindex/cmd/client \ + --vindex_base_url http://localhost:8088/vindex/ \ + --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 \ + --in_log_pub_key_der=MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECHOhXfvYgTcu+Fnl7M7niFj3FgqWlQpXUSWUDw2KAaJXvhGxdJTtmyciN5rWTiDtpeNENVmsUTHFS4XQgeRE0g== \ + --in_log_origin=coachandhorses2026h1.staging.certificate.transparency.goog \ + --lookup=google.com + +I0610 15:02:17.112527 87150 client.go:83] in_log_base_url not provided, so cannot dereference pointers +148000245 +151898263 +152014951 +152015244 +152015262 +152015307 +... +154306826 +154307178 +154307232 +154307321 +154368790 +154368845 +``` + +To view the certs at the index, [woodpecker](https://github.com/mhutchinson/woodpecker) can be used. + diff --git a/vindex/cmd/ct/main.go b/vindex/cmd/ct/main.go new file mode 100644 index 0000000..9e6305e --- /dev/null +++ b/vindex/cmd/ct/main.go @@ -0,0 +1,494 @@ +// Copyright 2026 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ct is a binary that indexes a static CT log (the input log) into a verifiable +// index, and publishes the index checkpoints to a Tessera POSIX log (the output log). +// A web server is hosted that allows lookups in the index to be performed. +package main + +import ( + "context" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "errors" + "flag" + "fmt" + "iter" + "net/http" + "os" + "os/signal" + "path" + "strings" + "syscall" + "time" + + "golang.org/x/net/publicsuffix" + + "filippo.io/sunlight" + "filippo.io/torchwood" + "github.com/gorilla/mux" + "github.com/transparency-dev/formats/log" + fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/vindex" + "github.com/transparency-dev/incubator/vindex/internal/web" + "go.opentelemetry.io/otel/exporters/prometheus" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "golang.org/x/crypto/cryptobyte" + "golang.org/x/mod/sumdb/note" + "golang.org/x/mod/sumdb/tlog" + "k8s.io/klog/v2" +) + +var ( + inputLogUrl = flag.String("input_log_url", "", "Base URL of the static CT log to index. This must be the monitoring URL, not the submission URL.") + origin = flag.String("origin", "", "Origin of the log to check") + pubKey = flag.String("public_key", "", "The log's public key in base64 encoded DER format") + userAgentInfo = flag.String("user_agent_info", "", "Optional string to append to the user agent (e.g. email address for Sunlight logs)") + persistentCacheDir = flag.String("persistent_cache_dir", "", "Optional location of a directory to cache Input Log tiles") + persistIndex = flag.Bool("persist_index", true, "Set to false to use a memory-based implementation of the verifiable index.") + + outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Input Log, Output Log, and allocate space to store the verifiable map persistence.") + listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") +) + +const ( + userAgent = "TrustFabric VerifiableIndex" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx); err != nil { + klog.Exitf("Run failed: %v", err) + } +} + +func run(ctx context.Context) error { + // Set up storage for the input log, index, and output log. + if *storageDir == "" { + return errors.New("storage_dir must be set") + } + if *inputLogUrl == "" { + return errors.New("input_log_url must be set") + } + if *origin == "" { + return errors.New("origin must be set") + } + if *pubKey == "" { + return errors.New("public_key must be set") + } + + outputLogDir := path.Join(*storageDir, "outputlog") + mapRoot := path.Join(*storageDir, "vindex") + + if err := os.MkdirAll(outputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create output log directory: %v", err) + } + if err := os.MkdirAll(mapRoot, 0o755); err != nil { + return fmt.Errorf("failed to create vindex directory: %v", err) + } + + exporter, err := prometheus.New() + if err != nil { + return fmt.Errorf("failed to create prometheus exporter: %v", err) + } + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter)) + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := provider.Shutdown(shutdownCtx); err != nil { + klog.Errorf("failed to shutdown meter provider: %v", err) + } + }() + + outputLog, outputCloser, err := newOutputLogFromFlags(ctx, outputLogDir) + if err != nil { + return err + } + defer outputCloser(ctx) + + inputLog, err := newStaticCTInputLogFromFlags() + if err != nil { + return err + } + + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, mapRoot, vindex.Options{ + PersistIndex: *persistIndex, + MeterProvider: provider, + }) + if err != nil { + return fmt.Errorf("failed to create vindex: %v", err) + } + klog.Info("Created verifiable index") + + // Keeps the map synced with the latest published input log state. + go maintainMap(ctx, vi) + + // Run a web server to serve the input log, index, and output log. + go runWebServer(vi, outputLogDir) + <-ctx.Done() + return nil +} + +func cutEntry(tile []byte) (entry []byte, rh tlog.Hash, rest []byte, err error) { + // This implementation is terribly inefficient, parsing the whole entry just + // to re-serialize and throw it away. If this function shows up in profiles, + // let me know and I'll improve it. + e, rest, err := sunlight.ReadTileLeaf(tile) + if err != nil { + return nil, tlog.Hash{}, nil, err + } + rh = tlog.RecordHash(e.MerkleTreeLeaf()) + entry = tile[:len(tile)-len(rest)] + return entry, rh, rest, nil +} + +func newStaticCTInputLogFromFlags() (*staticCTInputLog, error) { + ua := userAgent + if *userAgentInfo != "" { + ua = fmt.Sprintf("%s (%s)", userAgent, *userAgentInfo) + } + fetcher, err := torchwood.NewTileFetcher(*inputLogUrl, + torchwood.WithTilePath(sunlight.TilePath), + torchwood.WithUserAgent(ua)) + if err != nil { + return nil, fmt.Errorf("failed to create tile fetcher: %w", err) + } + var tileReader torchwood.TileReader = fetcher + if *persistentCacheDir != "" { + tileReader, err = torchwood.NewPermanentCache(fetcher, *persistentCacheDir) + if err != nil { + return nil, fmt.Errorf("failed to create permanent cache: %w", err) + } + } + client, err := torchwood.NewClient(tileReader, torchwood.WithCutEntry(cutEntry)) + if err != nil { + return nil, fmt.Errorf("failed to create torchwood client: %w", err) + } + v, err := verifierFromFlags() + if err != nil { + return nil, err + } + return &staticCTInputLog{ + c: client, + f: fetcher, + v: v, + }, nil +} + +type staticCTInputLog struct { + c *torchwood.Client + f *torchwood.TileFetcher + v note.Verifier + + lastCheckpoint log.Checkpoint +} + +func (l *staticCTInputLog) Checkpoint(ctx context.Context) (checkpoint []byte, err error) { + return l.f.ReadEndpoint(ctx, "checkpoint") +} + +// Parse unmarshals and verifies a checkpoint obtained from GetCheckpoint. +func (l *staticCTInputLog) Parse(checkpoint []byte) (*log.Checkpoint, error) { + cp, _, _, err := log.ParseCheckpoint(checkpoint, l.v.Name(), l.v) + if err != nil { + return nil, err + } + l.lastCheckpoint = *cp + return cp, err +} + +// Leaves returns all the leaves in the range [start, end), outputting them via +// the returned iterator. +func (l *staticCTInputLog) Leaves(ctx context.Context, start, end uint64) iter.Seq2[[]byte, error] { + tree := tlog.Tree{ + N: int64(end), + Hash: tlog.Hash(l.lastCheckpoint.Hash), + } + return func(yield func([]byte, error) bool) { + for _, entry := range l.c.Entries(ctx, tree, int64(start)) { + e, _, err := sunlight.ReadTileLeaf(entry) + if err != nil { + if !yield(nil, err) { + return + } + } + if !yield(e.MerkleTreeLeaf(), nil) { + return + } + } + if err := l.c.Err(); err != nil { + yield(nil, l.c.Err()) + } + } +} + +func newOutputLogFromFlags(ctx context.Context, outputLogDir string) (vindex.OutputLog, func(context.Context), error) { + s, v, err := getOutputLogSignerVerifier() + if err != nil { + return nil, nil, err + } + + l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v, vindex.OutputLogOpts{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to create output log: %w", err) + } + return l, c, nil +} + +func verifierFromFlags() (note.Verifier, error) { + if *origin == "" { + return nil, errors.New("origin must be set") + } + if *pubKey == "" { + return nil, errors.New("public_key must be set") + } + derBytes, err := base64.StdEncoding.DecodeString(*pubKey) + if err != nil { + return nil, fmt.Errorf("error decoding public key: %w", err) + } + pub, err := x509.ParsePKIXPublicKey(derBytes) + if err != nil { + return nil, fmt.Errorf("error parsing public key: %w", err) + } + + verifierKey, err := fnote.RFC6962VerifierString(*origin, pub) + if err != nil { + return nil, fmt.Errorf("error creating RFC6962 verifier string: %w", err) + } + logSigV, err := fnote.NewVerifier(verifierKey) + if err != nil { + return nil, fmt.Errorf("error creating verifier: %w", err) + } + + klog.Infof("Using verifier string: %v", verifierKey) + + return logSigV, nil +} + +// maintainMap reads entries from the log and sync them to the vindex. +func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + if err := vi.Update(ctx); err != nil { + klog.Warning(err) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func runWebServer(vi *vindex.VerifiableIndex, outLogDir string) { + srv := web.NewServer(vi.Lookup) + + olfs := http.FileServer(http.Dir(outLogDir)) + r := mux.NewRouter() + r.PathPrefix("/outputlog/").Handler(http.StripPrefix("/outputlog/", olfs)) + srv.RegisterHandlers(r) + hServer := &http.Server{ + Addr: *listen, + Handler: r, + } + go func() { + if err := hServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + klog.Exitf("HTTP server failed: %v", err) + } + }() + klog.Infof("Started HTTP server listening on %s", *listen) +} + +// Read output log private key from file or environment variable and generate the +// note Signer and Verifier pair for it. +func getOutputLogSignerVerifier() (note.Signer, note.Verifier, error) { + var privKey string + var err error + if len(*outputLogPrivKeyFile) > 0 { + privKey, err = getKeyFile(*outputLogPrivKeyFile) + if err != nil { + return nil, nil, fmt.Errorf("unable to get private key: %w", err) + } + } else { + privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") + if len(privKey) == 0 { + return nil, nil, errors.New("supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable") + } + } + s, v, err := fnote.NewEd25519SignerVerifier(privKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to get signer/verifier: %w", err) + } + return s, v, nil +} + +func getKeyFile(path string) (string, error) { + k, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + return string(k), nil +} + +func mapFn(data []byte) [][sha256.Size]byte { + s := cryptobyte.String(data) + + var version, leafType uint8 + var timestamp uint64 + var certType uint16 + if !s.ReadUint8(&version) || !s.ReadUint8(&leafType) || !s.ReadUint64(×tamp) || !s.ReadUint16(&certType) { + klog.Warningf("Failed to unmarshal headers") + // This should return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + var isPreCert bool + var cert cryptobyte.String + switch certType { + case 0: + // x509 + isPreCert = false + if !s.ReadUint24LengthPrefixed(&cert) { + klog.Warning("Failed to read x509 certificate") + return nil + } + case 1: + // precert + isPreCert = true + var ikh []byte + if !s.ReadBytes(&ikh, sha256.Size) { + klog.Warning("Failed to read issuer key hash") + return nil + } + var tbsCert cryptobyte.String + if !s.ReadUint24LengthPrefixed(&tbsCert) { + klog.Warning("Failed to read precert TBSCertificate") + return nil + } + + tbsDER := []byte(tbsCert) + tbsCertCopy := tbsCert + + var tbsSeq cryptobyte.String + if !tbsCertCopy.ReadASN1(&tbsSeq, 0x30) { // SEQUENCE + klog.Warning("failed to read TBSCertificate sequence") + return nil + } + + if tbsSeq.PeekASN1Tag(0xA0) { + var version cryptobyte.String + if !tbsSeq.ReadASN1(&version, 0xA0) { + klog.Warning("failed to read version") + return nil + } + } + + var serial cryptobyte.String + if !tbsSeq.ReadASN1(&serial, 0x02) { // INTEGER + klog.Warning("failed to read serial number") + return nil + } + + var sigAlg cryptobyte.String + if !tbsSeq.ReadASN1Element(&sigAlg, 0x30) { // SEQUENCE + klog.Warning("failed to read signature algorithm") + return nil + } + + var certBuilder cryptobyte.Builder + certBuilder.AddASN1(0x30, func(b *cryptobyte.Builder) { // SEQUENCE + b.AddBytes(tbsDER) + b.AddBytes([]byte(sigAlg)) + b.AddASN1(3, func(b *cryptobyte.Builder) { + b.AddBytes([]byte{0, 0}) + }) + }) + dummyCertDER, err := certBuilder.Bytes() + if err != nil { + klog.Warningf("failed to build dummy cert: %v", err) + return nil + } + cert = cryptobyte.String(dummyCertDER) + default: + klog.Warningf("unknown cert type: %d", certType) + return nil + } + + parsedCert, err := x509.ParseCertificate(cert) + if err != nil { + klog.Warningf("failed to parse x509 cert (preCert=%t): %v", isPreCert, err) + // This could return a sentinel value (e.g. all zero hash) so unprocessable entries can be found + return nil + } + if klog.V(2).Enabled() { + klog.V(2).Info(parsedCert.DNSNames) + } + uniqueNames := make(map[string]bool) + for _, cn := range parsedCert.DNSNames { + cn = strings.ToLower(cn) + if strings.HasPrefix(cn, "*.") { + cn = cn[2:] + } else if strings.HasPrefix(cn, "*") { + cn = cn[1:] + } + if cn == "" { + continue + } + uniqueNames[cn] = true + + // Note on determinism vs PSL evolution: + // The Public Suffix List (PSL) evolves over time. If the PSL used by this binary + // is outdated and a new public suffix (e.g., a new "co.uk") is introduced, + // we will fail to recognize it as a public suffix. + // In that case, we will treat the new public suffix (e.g., "xx.yy") as the eTLD+1 + // and index it. This means all certificates under "*.xx.yy" will also be indexed + // under "xx.yy", potentially causing index bloat for that key if it becomes popular. + // This risk is accepted to maintain determinism of the indexer output for a given binary version. + etld1, err := publicsuffix.EffectiveTLDPlusOne(cn) + if err != nil { + continue + } + if cn == etld1 { + continue + } + curr := cn + for { + idx := strings.Index(curr, ".") + if idx == -1 { + break + } + curr = curr[idx+1:] + if len(curr) < len(etld1) { + break + } + uniqueNames[curr] = true + if curr == etld1 { + break + } + } + } + hashes := make([][sha256.Size]byte, 0, len(uniqueNames)) + for name := range uniqueNames { + hashes = append(hashes, sha256.Sum256([]byte(name))) + } + return hashes +} diff --git a/vindex/cmd/ct/main_test.go b/vindex/cmd/ct/main_test.go new file mode 100644 index 0000000..5ae4c66 --- /dev/null +++ b/vindex/cmd/ct/main_test.go @@ -0,0 +1,222 @@ +// Copyright 2026 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/transparency-dev/incubator/vindex/api" + "github.com/transparency-dev/incubator/vindex/internal/web" + "golang.org/x/crypto/cryptobyte" +) + +func createTestCertBytes(dnsNames []string, isPrecert bool) ([]byte, error) { + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Test Org"}, + }, + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().Add(1 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: dnsNames, + } + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, err + } + + var tbsDER []byte + if isPrecert { + parsed, err := x509.ParseCertificate(certDER) + if err != nil { + return nil, err + } + tbsDER = parsed.RawTBSCertificate + } + + // Now marshal using cryptobyte into the form expected by mapFn + // version = 0, leafType = 0, timestamp = 12345678 + var b cryptobyte.Builder + b.AddUint8(0) // version + b.AddUint8(0) // leafType + b.AddUint64(12345678) // timestamp + if isPrecert { + b.AddUint16(1) // certType (precert) + dummyIKH := make([]byte, 32) + b.AddBytes(dummyIKH) + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(tbsDER) + }) + } else { + b.AddUint16(0) // certType (x509) + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(certDER) + }) + } + return b.Bytes() +} + +func TestMapFn(t *testing.T) { + testCases := []struct { + desc string + dnsNames []string + wantKeys []string + isPrecert bool + rawBytes []byte + }{ + { + desc: "standard domains and wildcards", + dnsNames: []string{"*.google.com", "google.com", "maps.google.com"}, + wantKeys: []string{"google.com", "maps.google.com"}, + isPrecert: false, + }, + { + desc: "deeper subdomain", + dnsNames: []string{"deep.maps.google.co.uk"}, + wantKeys: []string{"deep.maps.google.co.uk", "maps.google.co.uk", "google.co.uk"}, + isPrecert: false, + }, + { + desc: "mixed case", + dnsNames: []string{"MAPS.GOOGLE.COM"}, + wantKeys: []string{"maps.google.com", "google.com"}, + isPrecert: false, + }, + { + desc: "invalid or TLD", + dnsNames: []string{"localhost", "*.co.uk"}, + wantKeys: []string{"localhost", "co.uk"}, + isPrecert: false, + }, + { + desc: "precert: standard domains and wildcards", + dnsNames: []string{"*.google.com", "google.com", "maps.google.com"}, + wantKeys: []string{"google.com", "maps.google.com"}, + isPrecert: true, + }, + { + desc: "precert: deeper subdomain", + dnsNames: []string{"deep.maps.google.co.uk"}, + wantKeys: []string{"deep.maps.google.co.uk", "maps.google.co.uk", "google.co.uk"}, + isPrecert: true, + }, + { + desc: "truncated header", + rawBytes: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, // 11 bytes, header is 12 bytes + wantKeys: []string{}, + }, + { + desc: "unknown cert type", + rawBytes: []byte{ + 0, // version + 0, // leafType + 0, 0, 0, 0, 0, 0, 0, 0, // timestamp + 0, 99, // certType = 99 + }, + wantKeys: []string{}, + }, + { + desc: "precert truncated TBS", + rawBytes: func() []byte { + var b cryptobyte.Builder + b.AddUint8(0) // version + b.AddUint8(0) // leafType + b.AddUint64(12345678) // timestamp + b.AddUint16(1) // certType (precert) + dummyIKH := make([]byte, 32) + b.AddBytes(dummyIKH) + b.AddUint24(100) // length prefix 100 + b.AddBytes([]byte{1, 2, 3, 4, 5}) // only 5 bytes of data + bytes, _ := b.Bytes() + return bytes + }(), + wantKeys: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + var data []byte + var err error + if tc.rawBytes != nil { + data = tc.rawBytes + } else { + data, err = createTestCertBytes(tc.dnsNames, tc.isPrecert) + if err != nil { + t.Fatalf("failed to create test cert: %v", err) + } + } + hashes := mapFn(data) + + gotKeys := make(map[string]bool) + for _, h := range hashes { + found := false + for _, wk := range tc.wantKeys { + if sha256.Sum256([]byte(wk)) == h { + gotKeys[wk] = true + found = true + break + } + } + if !found { + t.Errorf("got unexpected hash for a key not in wantKeys") + } + } + + for _, wk := range tc.wantKeys { + if !gotKeys[wk] { + t.Errorf("missing expected key: %s", wk) + } + } + if len(gotKeys) != len(tc.wantKeys) { + t.Errorf("got %d unique keys, want %d", len(gotKeys), len(tc.wantKeys)) + } + }) + } +} + +func TestMetricsEndpoint(t *testing.T) { + s := web.NewServer(func(ctx context.Context, h [sha256.Size]byte) (api.LookupResponse, error) { + return api.LookupResponse{}, nil + }) + r := mux.NewRouter() + s.RegisterHandlers(r) + + req := httptest.NewRequest("GET", "/metrics", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("GET /metrics: got status %d, want %d", w.Code, http.StatusOK) + } +} diff --git a/vindex/outputlog_test.go b/vindex/outputlog_test.go index d55e67f..d002767 100644 --- a/vindex/outputlog_test.go +++ b/vindex/outputlog_test.go @@ -80,7 +80,7 @@ func TestOutputLog_Lookup(t *testing.T) { t.Fatal(err) } defer func() { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() closer(ctx) }()