diff --git a/vindex/client/client.go b/vindex/client/client.go index e9b339b..8e6cf7e 100644 --- a/vindex/client/client.go +++ b/vindex/client/client.go @@ -18,6 +18,7 @@ package client import ( "context" "crypto/sha256" + "encoding/base64" "encoding/binary" "encoding/hex" "encoding/json" @@ -26,6 +27,7 @@ import ( "iter" "net/http" "net/url" + "strings" "github.com/transparency-dev/formats/log" "github.com/transparency-dev/incubator/vindex" @@ -78,7 +80,7 @@ func VerifyLookupResponse(keyHash [sha256.Size]byte, resp api.LookupResponse, in } ilcp, _, _, err := log.ParseCheckpoint(inCp, origin, inV) if err != nil { - return nil, nil, fmt.Errorf("failed to parse input log checkpoint: %v", err) + return nil, nil, fmt.Errorf("failed to parse input log checkpoint: %v (verifier: %s, keyhash: %08x; available signatures: %s)", err, inV.Name(), inV.KeyHash(), dumpSignatures(inCp)) } expectFound := len(resp.IndexValue) > 0 @@ -302,3 +304,29 @@ type logClient interface { ReadTile(ctx context.Context, l, i uint64, p uint8) ([]byte, error) ReadEntryBundle(ctx context.Context, i uint64, p uint8) ([]byte, error) } + +func dumpSignatures(noteBytes []byte) string { + lines := strings.Split(string(noteBytes), "\n") + var sigs []string + for _, line := range lines { + if strings.HasPrefix(line, "\u2014 ") { + parts := strings.SplitN(line[4:], " ", 2) + if len(parts) == 2 { + name := parts[0] + sigB64 := parts[1] + sigBytes, err := base64.StdEncoding.DecodeString(sigB64) + if err == nil && len(sigBytes) >= 4 { + kh := binary.BigEndian.Uint32(sigBytes[:4]) + sigs = append(sigs, fmt.Sprintf("%s (keyhash %08x)", name, kh)) + } else { + sigs = append(sigs, name) + } + } + } + } + if len(sigs) == 0 { + return "none" + } + return strings.Join(sigs, ", ") +} + diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index bef7b9c..6f80010 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -19,6 +19,7 @@ package main import ( "context" + "crypto/ed25519" "crypto/x509" "encoding/base64" "errors" @@ -29,19 +30,24 @@ import ( fnote "github.com/transparency-dev/formats/note" "github.com/transparency-dev/incubator/vindex/client" + "github.com/transparency-dev/incubator/vindex/internal/mtc" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" ) var ( - vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") - inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") - lookup = flag.String("lookup", "", "The key to look up in the vindex.") - outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint. Required.") - inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint. Required.") - inLogPubKeyDER = flag.String("in_log_pub_key_der", "", "For CT logs. The public key to use to verify the input log checkpoint. Required, along with in_log_origin.") - inLogOrigin = flag.String("in_log_origin", "", "Required if in_log_pub_key_der is used. Otherwise, allows the Input Log Origin string to be configured to something other than the public key name.") - minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") + vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") + inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") + lookup = flag.String("lookup", "", "The key to look up in the vindex.") + outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint. Required.") + inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint. Required.") + inLogPubKeyDER = flag.String("in_log_pub_key_der", "", "For CT logs. The public key to use to verify the input log checkpoint. Required, along with in_log_origin.") + inLogOrigin = flag.String("in_log_origin", "", "Required if in_log_pub_key_der is used. Otherwise, allows the Input Log Origin string to be configured to something other than the public key name. For MTC, this is the checkpoint origin.") + inLogKeyName = flag.String("in_log_key_name", "", "For MTC logs. The key name used in the checkpoint signature. If unset, defaults to in_log_origin.") + inLogMTC = flag.Bool("in_log_mtc", false, "Use MTC verifier for input log.") + inLogCosignerID = flag.String("in_log_cosigner_id", "", "For MTC logs. The relative OID of the cosigner.") + inLogID = flag.String("in_log_id", "", "For MTC logs. The relative OID of the log.") + minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") ) func main() { @@ -137,6 +143,41 @@ func newInputLogClientFromFlags() *client.InputLogClient { } func inputLogVerifierFromFlags() note.Verifier { + if *inLogMTC { + if *inLogPubKey == "" { + klog.Exitf("in_log_pub_key must be provided when using --in_log_mtc") + } + keyName := *inLogKeyName + if keyName == "" { + keyName = *inLogOrigin + } + if keyName == "" { + klog.Exitf("in_log_key_name (or in_log_origin) must be provided when using --in_log_mtc") + } + if *inLogCosignerID == "" { + klog.Exitf("in_log_cosigner_id must be provided when using --in_log_mtc") + } + if *inLogID == "" { + klog.Exitf("in_log_id must be provided when using --in_log_mtc") + } + pubKeyBytes, err := base64.StdEncoding.DecodeString(*inLogPubKey) + if err != nil { + klog.Exitf("failed to decode log_public_key: %v", err) + } + // At the moment this is optimized for the Cloudflare bootstrap log. + // This will need to be changed to support ML-DSA-44 when we support other MTC logs. + // Other changes are also likely to be needed, due to spec evolution since bootstrap launched. + if len(pubKeyBytes) != ed25519.PublicKeySize { + klog.Exitf("invalid log_public_key size: %d, expected %d", len(pubKeyBytes), ed25519.PublicKeySize) + } + pubKey := ed25519.PublicKey(pubKeyBytes) + v, err := mtc.NewMTCVerifier(keyName, pubKey, *inLogCosignerID, *inLogID) + if err != nil { + klog.Exitf("failed to create MTCVerifier: %v", err) + } + return v + } + if (*inLogPubKey == "") == (*inLogPubKeyDER == "") { klog.Exitf("Must provide exactly one --in_log_pub_key* flag") } diff --git a/vindex/cmd/mtcindex/README.md b/vindex/cmd/mtcindex/README.md new file mode 100644 index 0000000..2a9ac69 --- /dev/null +++ b/vindex/cmd/mtcindex/README.md @@ -0,0 +1,65 @@ +# MTC Verifiable Indexer (`mtcindex`) + +`mtcindex` is a verifiable indexer for Merkle Tree Certificates (MTC) logs. It processes leaf entries from an MTC log, parses the certificates, and indexes them by domain name in a verifiable map. This allows users to query for all certificates associated with a domain verifiably and efficiently, without downloading the entire log. + +For details on the Merkle Tree Certificates specification, see the [MTC Draft](https://datatracker.ietf.org/doc/draft-davidben-tls-merkle-tree-certs/). +For details on the Verifiable Index architecture (the "Map Sandwich"), see the [root VIndex README](../../README.md). + +## How it works + +1. **Input Log**: It monitors an MTC log (e.g., Cloudflare's bootstrap MTC log). +1. **MTC Verification**: It uses a custom checkpoint verifier that handles MTC-specific binary checkpoint signatures (`MTCSubtreeSignatureInput`) instead of standard text-based note signatures. +1. **Parsing (`mapFn`)**: It parses leaf entries of type `tbs_cert_entry` (type 1) using ASN.1, extracts the DNS names from the Subject Alternative Name (SAN) extension, and hashes them. Other entry types (like `null_entry`) are ignored. +1. **Verifiable Map**: It maintains a verifiable map (using Pebble DB for persistence) mapping domain hashes to log indices. +1. **Output Log**: It writes the map roots to an output log, which is signed by the operator. + +## Usage + +> [!WARNING] +> This demo uses hardcoded cryptographic keys. Do not use these keys in production or public deployments. Ensure you generate and use your own secure keys when deploying this service. + +### 1. Run the Indexer + +Run the indexer by pointing it to a storage directory and providing the output log private key. + +By default, it is configured to index the Cloudflare Shard 3 MTC log. We use a demo private key here: + +```bash +OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ \ +go run ./vindex/cmd/mtcindex/ \ + --storage_dir=/path/to/storage \ + --listen=:8088 +``` + +### 2. Query the Index + +Once running, the indexer hosts a web server (default `:8088`) serving the verifiable index. +A domain indexed by the verifiable map can be looked up using the client command. + +For example, to look up `minefun.io`: + +```shell +go run ./vindex/cmd/client \ + --vindex_base_url http://localhost:8088/vindex/ \ + --out_log_pub_key="example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8" \ + --in_log_pub_key="teYkXkxVoKhT1PxKODAyZFqUk8KZ4tUjzS6yAvvZ8hU=" \ + --in_log_mtc \ + --in_log_origin="bootstrap-mtca.cloudflareresearch.com/logs/shard3" \ + --in_log_cosigner_id="44363.48.9" \ + --in_log_id="44363.48.8" \ + --in_log_key_name="oid/1.3.6.1.4.1.44363.47.1.44363.48.8" \ + --lookup=minefun.io +``` +## Flags + +- `--log_url`: Base URL of the MTC log to index (default: `https://bootstrap-mtca-shard3.cloudflareresearch.com/`). +- `--key_name`: The key name used in the checkpoint signature (default: `oid/1.3.6.1.4.1.44363.47.1.44363.48.8`). +- `--log_public_key`: The log's public key, base64 encoded raw 32-byte Ed25519 key (default: `teYkXkxVoKhT1PxKODAyZFqUk8KZ4tUjzS6yAvvZ8hU=`). +- `--cosigner_id`: The relative OID of the cosigner (default: `44363.48.9`). +- `--log_id`: The relative OID of the log (default: `44363.48.8`). +- `--origin`: The expected origin string in the checkpoint text (default: `bootstrap-mtca.cloudflareresearch.com/logs/shard3`). +- `--storage_dir`: Root directory in which to store the output log data and verifiable map persistence (required). +- `--listen`: Address to set up the HTTP server on (default: `:8088`). +- `--persist_index`: Set to false to use a memory-based implementation of the verifiable index (default: `true`). +- `--oneshot`: Set to true to build the map once up to the current log size and then exit (default: `false`). +- `--output_log_private_key_path`: Location of the output log private key file. diff --git a/vindex/cmd/mtcindex/main.go b/vindex/cmd/mtcindex/main.go new file mode 100644 index 0000000..a4f201a --- /dev/null +++ b/vindex/cmd/mtcindex/main.go @@ -0,0 +1,321 @@ +// Copyright 2026 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// mtcindex brings up a verifiable index for the MTC log. +package main + +import ( + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "errors" + "flag" + "fmt" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "path" + "strings" + "sync" + "syscall" + "time" + + "github.com/gorilla/mux" + fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/vindex" + "github.com/transparency-dev/incubator/vindex/internal/mtc" + "github.com/transparency-dev/incubator/vindex/internal/web" + "go.opentelemetry.io/otel/exporters/prometheus" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "golang.org/x/net/publicsuffix" + "golang.org/x/mod/sumdb/note" + "k8s.io/klog/v2" +) + +var ( + logURL = flag.String("log_url", "https://bootstrap-mtca-shard3.cloudflareresearch.com/", "Base URL of the MTC log") + keyName = flag.String("key_name", "oid/1.3.6.1.4.1.44363.47.1.44363.48.8", "The key name used in the checkpoint signature") + logPublicKey = flag.String("log_public_key", "teYkXkxVoKhT1PxKODAyZFqUk8KZ4tUjzS6yAvvZ8hU=", "The log's public key, base64 encoded raw 32-byte Ed25519 key") + cosignerID = flag.String("cosigner_id", "44363.48.9", "The relative OID of the cosigner") + logID = flag.String("log_id", "44363.48.8", "The relative OID of the log") + origin = flag.String("origin", "bootstrap-mtca.cloudflareresearch.com/logs/shard3", "The expected origin string in the checkpoint") + storageDir = flag.String("storage_dir", "", "Root directory for storage (required)") + listen = flag.String("listen", ":8088", "Address to listen on") + persistIndex = flag.Bool("persist_index", true, "Whether to persist the index") + oneShot = flag.Bool("oneshot", false, "Run once and exit") + + outputLogPrivKeyFile = flag.String("output_log_private_key_path", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + inputLogReaders = flag.Uint("input_log_readers", 10, "Number of parallel readers for the input log") +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx); err != nil { + klog.Exitf("Run failed: %v", err) + } +} + +func run(ctx context.Context) error { + if *storageDir == "" { + return errors.New("storage_dir must be set") + } + + exporter, err := prometheus.New() + if err != nil { + return fmt.Errorf("failed to create prometheus exporter: %v", err) + } + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(exporter)) + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := provider.Shutdown(shutdownCtx); err != nil { + klog.Errorf("failed to shutdown meter provider: %v", err) + } + }() + + outputLogDir := path.Join(*storageDir, "outputlog") + mapRoot := path.Join(*storageDir, "vindex") + + if err := os.MkdirAll(outputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create output log directory: %v", err) + } + if err := os.MkdirAll(mapRoot, 0o755); err != nil { + return fmt.Errorf("failed to create vindex directory: %v", err) + } + + pubKeyBytes, err := base64.StdEncoding.DecodeString(*logPublicKey) + if err != nil { + return fmt.Errorf("failed to decode log_public_key: %v", err) + } + if len(pubKeyBytes) != ed25519.PublicKeySize { + return fmt.Errorf("invalid log_public_key size: %d, expected %d", len(pubKeyBytes), ed25519.PublicKeySize) + } + pubKey := ed25519.PublicKey(pubKeyBytes) + + mtcVerifier, err := mtc.NewMTCVerifier(*keyName, pubKey, *cosignerID, *logID) + if err != nil { + return fmt.Errorf("failed to create MTCVerifier: %v", err) + } + + parsedLogURL, err := url.Parse(*logURL) + if err != nil { + return fmt.Errorf("failed to parse log_url: %v", err) + } + + inputLog, err := vindex.NewTiledInputLog(parsedLogURL, mtcVerifier, vindex.InputLogOpts{ + HttpClient: http.DefaultClient, + Origin: *origin, + NumReaders: *inputLogReaders, + }) + if err != nil { + return fmt.Errorf("failed to create InputLog: %v", err) + } + + outputLog, outputCloser := outputLogOrDie(ctx, outputLogDir) + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + outputCloser(shutdownCtx) + }() + + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, mapRoot, vindex.Options{ + PersistIndex: *persistIndex, + MeterProvider: provider, + }) + if err != nil { + return fmt.Errorf("failed to create vindex: %v", err) + } + defer func() { + if err := vi.Close(); err != nil { + klog.Errorf("failed to close vindex: %v", err) + } + }() + + webShutdown, err := runWebServer(vi, outputLogDir) + if err != nil { + return fmt.Errorf("failed to start web server: %v", err) + } + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := webShutdown(shutdownCtx); err != nil { + klog.Errorf("failed to shutdown web server: %v", err) + } + }() + + if *oneShot { + if err := vi.Update(ctx); err != nil { + return fmt.Errorf("failed to Update index: %v", err) + } + return nil + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + maintainMap(ctx, vi) + }() + + <-ctx.Done() + klog.Info("Stopping indexer, waiting for pending updates to complete...") + wg.Wait() + return nil +} + +func outputLogOrDie(ctx context.Context, outputLogDir string) (log vindex.OutputLog, closer func(context.Context)) { + s, v := getOutputLogSignerVerifierOrDie() + l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v, vindex.OutputLogOpts{}) + if err != nil { + klog.Exitf("Failed to create Output Log: %v", err) + } + return l, c +} + +func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + if err := vi.Update(ctx); err != nil { + klog.Warningf("Failed to Update index: %v", err) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func runWebServer(vi *vindex.VerifiableIndex, outLogDir string) (func(context.Context) error, error) { + srv := web.NewServer(vi.Lookup) + + olfs := http.FileServer(http.Dir(outLogDir)) + r := mux.NewRouter() + r.PathPrefix("/outputlog/").Handler(http.StripPrefix("/outputlog/", olfs)) + srv.RegisterHandlers(r) + + listener, err := net.Listen("tcp", *listen) + if err != nil { + return nil, err + } + + hServer := &http.Server{ + Handler: r, + } + go func() { + if err := hServer.Serve(listener); err != http.ErrServerClosed { + klog.Warningf("Error from HTTP server: %v", err) + } + }() + klog.Infof("Started HTTP server listening on %s", *listen) + + return hServer.Shutdown, nil +} + +func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { + var privKey string + var err error + if len(*outputLogPrivKeyFile) > 0 { + privKey, err = getKeyFile(*outputLogPrivKeyFile) + if err != nil { + klog.Exitf("Unable to get private key: %v", err) + } + } else { + privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") + if len(privKey) == 0 { + klog.Exit("Supply private key file path using --output_log_private_key_path or set OUTPUT_LOG_PRIVATE_KEY environment variable") + } + } + s, v, err := fnote.NewEd25519SignerVerifier(privKey) + if err != nil { + klog.Exitf("Failed to get signer/verifier: %v", err) + } + return s, v +} + +func getKeyFile(path string) (string, error) { + k, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + return string(k), nil +} + +func mapFn(data []byte) [][32]byte { + if len(data) < 2 { + return nil + } + entryType := binary.BigEndian.Uint16(data[:2]) + if entryType != 1 { + return nil + } + entry, err := mtc.ParseTBSCertificateLogEntry(data[2:]) + if err != nil { + klog.Warningf("Failed to parse TBSCertificateLogEntry: %v", err) + return nil + } + dnsNames := mtc.ExtractDNSNames(entry) + uniqueNames := make(map[string]bool) + for _, cn := range dnsNames { + cn = strings.ToLower(cn) + if strings.HasPrefix(cn, "*.") { + cn = cn[2:] + } else if strings.HasPrefix(cn, "*") { + cn = cn[1:] + } + if cn == "" { + continue + } + uniqueNames[cn] = true + + etld1, err := publicsuffix.EffectiveTLDPlusOne(cn) + if err != nil { + continue + } + if cn == etld1 { + continue + } + curr := cn + for { + idx := strings.Index(curr, ".") + if idx == -1 { + break + } + curr = curr[idx+1:] + if len(curr) < len(etld1) { + break + } + uniqueNames[curr] = true + if curr == etld1 { + break + } + } + } + var hashes [][32]byte + for name := range uniqueNames { + hashes = append(hashes, sha256.Sum256([]byte(name))) + } + return hashes +} diff --git a/vindex/cmd/mtcindex/main_test.go b/vindex/cmd/mtcindex/main_test.go new file mode 100644 index 0000000..246d580 --- /dev/null +++ b/vindex/cmd/mtcindex/main_test.go @@ -0,0 +1,160 @@ +// Copyright 2026 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/sha256" + "encoding/asn1" + "encoding/hex" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/transparency-dev/incubator/vindex/internal/mtc" +) + +// helper to marshal TBSCertificateLogEntry +func marshalMTCEntry(dnsNames []string) ([]byte, error) { + rawNames := []asn1.RawValue{} + for _, name := range dnsNames { + rawNames = append(rawNames, asn1.RawValue{Tag: 2, Class: 2, Bytes: []byte(name)}) + } + sanValue, err := asn1.Marshal(rawNames) + if err != nil { + return nil, err + } + + ext := struct { + Id asn1.ObjectIdentifier + Critical bool `asn1:"optional,default:false"` + Value []byte + }{ + Id: asn1.ObjectIdentifier{2, 5, 29, 17}, + Value: sanValue, + } + extBytes, err := asn1.Marshal(ext) + if err != nil { + return nil, err + } + + entry := mtc.TBSCertificateLogEntry{ + Extensions: []asn1.RawValue{{FullBytes: extBytes}}, + } + + entryBytes, err := asn1.Marshal(entry) + if err != nil { + return nil, err + } + + return append([]byte{0, 1}, entryBytes...), nil +} + +func TestMapFn(t *testing.T) { + // Fixed DER payload from vindex/internal/mtc/mtc_test.go + derHex := "30820146a003020102301c311a3018060a2b0601040182da4b2f010c0a34343336332e34382e38301e170d3235313131313230313732365a170d3235313131383230313732365a30818e310b3009060355040613025553311330110603550408130a43616c69666f726e6961311430120603550407130b4c6f7320416e67656c6573313c303a060355040a1333496e7465726e657420436f72706f726174696f6e20666f722041737369676e6564204e616d657320616e64204e756d626572733116301406035504030c0d2a2e6578616d706c652e636f6d042088c3292097527f95650a51dac5945eca168bc4bb2664c30d022036a4c47cfccea34e304c30250603551d11041e301c820d2a2e6578616d706c652e636f6d820b6578616d706c652e636f6d300e0603551d0f0101ff04040302078030130603551d25040c300a06082b0601050507030100" + derBytes, err := hex.DecodeString(derHex) + if err != nil { + t.Fatalf("Failed to decode hex: %v", err) + } + + // Type 1: tbs_cert_entry + // Prefix with \x00\x01 (big-endian uint16 of 1 is [0, 1]) + validType1 := append([]byte{0x00, 0x01}, derBytes...) + + // Type 0: something else + validType0 := append([]byte{0x00, 0x00}, derBytes...) + + // Invalid entry data for Type 1 + invalidType1 := append([]byte{0x00, 0x01}, []byte("invalid der")...) + + // Generated test cases + subdomainCert, err := marshalMTCEntry([]string{"deep.maps.google.co.uk"}) + if err != nil { + t.Fatalf("failed to create subdomain cert: %v", err) + } + mixedCaseCert, err := marshalMTCEntry([]string{"MAPS.GOOGLE.COM"}) + if err != nil { + t.Fatalf("failed to create mixed case cert: %v", err) + } + + tests := []struct { + name string + data []byte + want [][32]byte + }{ + { + name: "Valid Type 1 (wildcard stripped)", + data: validType1, + want: [][32]byte{ + sha256.Sum256([]byte("example.com")), + }, + }, + { + name: "Type 0 (ignored)", + data: validType0, + want: nil, + }, + { + name: "Invalid Type 1 (ignored/warning)", + data: invalidType1, + want: nil, + }, + { + name: "Too short data", + data: []byte{0x00}, + want: nil, + }, + { + name: "Subdomain indexing", + data: subdomainCert, + want: [][32]byte{ + sha256.Sum256([]byte("deep.maps.google.co.uk")), + sha256.Sum256([]byte("maps.google.co.uk")), + sha256.Sum256([]byte("google.co.uk")), + }, + }, + { + name: "Mixed case normalization", + data: mixedCaseCert, + want: [][32]byte{ + sha256.Sum256([]byte("maps.google.com")), + sha256.Sum256([]byte("google.com")), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := mapFn(tt.data) + + // Sort helper for [32]byte + less := func(a, b [32]byte) bool { + for i := 0; i < 32; i++ { + if a[i] < b[i] { + return true + } + if a[i] > b[i] { + return false + } + } + return false + } + + if diff := cmp.Diff(tt.want, got, cmpopts.SortSlices(less)); diff != "" { + t.Errorf("mapFn() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/vindex/internal/mtc/mtc.go b/vindex/internal/mtc/mtc.go new file mode 100644 index 0000000..716f147 --- /dev/null +++ b/vindex/internal/mtc/mtc.go @@ -0,0 +1,229 @@ +package mtc + +import ( + "crypto/ed25519" + "crypto/sha256" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/base64" + "encoding/binary" + "fmt" + "strconv" + "strings" + "time" + + "golang.org/x/mod/sumdb/note" +) + +// Validity represents the validity period of a certificate. +type Validity struct { + NotBefore time.Time + NotAfter time.Time +} + +// TBSCertificateLogEntry represents the ASN.1 structure inside a tile entry. +type TBSCertificateLogEntry struct { + Version int `asn1:"optional,explicit,default:0,tag:0"` + Issuer pkix.RDNSequence + Validity Validity + Subject pkix.RDNSequence + SubjectPublicKeyInfoHash []byte + IssuerUniqueID asn1.BitString `asn1:"optional,implicit,tag:1"` + SubjectUniqueID asn1.BitString `asn1:"optional,implicit,tag:2"` + Extensions []asn1.RawValue `asn1:"optional,explicit,tag:3"` +} + +type extension struct { + Id asn1.ObjectIdentifier + Critical bool `asn1:"optional,default:false"` + Value []byte +} + +// ParseTBSCertificateLogEntry parses the ASN.1 DER encoded TBSCertificateLogEntry. +func ParseTBSCertificateLogEntry(der []byte) (*TBSCertificateLogEntry, error) { + var entry TBSCertificateLogEntry + rest, err := asn1.Unmarshal(der, &entry) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal TBSCertificateLogEntry: %w", err) + } + // We allow trailing data because some test cases might have it, + // but we could also check len(rest) == 0 if we wanted to be strict. + _ = rest + return &entry, nil +} + +var oidExtensionSubjectAltName = asn1.ObjectIdentifier{2, 5, 29, 17} + +// ExtractDNSNames extracts DNS names from the SAN extension of the entry. +func ExtractDNSNames(entry *TBSCertificateLogEntry) []string { + var dnsNames []string + for _, rawExt := range entry.Extensions { + var ext extension + if _, err := asn1.Unmarshal(rawExt.FullBytes, &ext); err != nil { + continue + } + if !ext.Id.Equal(oidExtensionSubjectAltName) { + continue + } + var sequence []asn1.RawValue + if _, err := asn1.Unmarshal(ext.Value, &sequence); err != nil { + continue + } + for _, raw := range sequence { + if raw.Class == 2 && raw.Tag == 2 { + dnsNames = append(dnsNames, string(raw.Bytes)) + } + } + } + return dnsNames +} + +// MTCVerifier implements note.Verifier for MTC checkpoint signatures. +type MTCVerifier struct { + name string + pubKey ed25519.PublicKey + keyHash uint32 + cosignerIDBytes []byte + logIDBytes []byte +} + +// NewMTCVerifier creates a new MTCVerifier. +func NewMTCVerifier(name string, pubKey ed25519.PublicKey, cosignerIDStr, logIDStr string) (*MTCVerifier, error) { + // Compute key hash: SHA-256(key name || 0x0A || 0xFF || "mtc-checkpoint/v1")[:4] + h := sha256.New() + h.Write([]byte(name)) + h.Write([]byte{0x0a, 0xff}) + h.Write([]byte("mtc-checkpoint/v1")) + sum := h.Sum(nil) + keyHash := binary.BigEndian.Uint32(sum[:4]) + + cosignerIDBytes, err := encodeRelativeOIDStr(cosignerIDStr) + if err != nil { + return nil, fmt.Errorf("failed to encode cosigner ID: %w", err) + } + + logIDBytes, err := encodeRelativeOIDStr(logIDStr) + if err != nil { + return nil, fmt.Errorf("failed to encode log ID: %w", err) + } + + return &MTCVerifier{ + name: name, + pubKey: pubKey, + keyHash: keyHash, + cosignerIDBytes: cosignerIDBytes, + logIDBytes: logIDBytes, + }, nil +} + +// Name returns the verifier name. +func (v *MTCVerifier) Name() string { + return v.name +} + +// KeyHash returns the verifier key hash. +func (v *MTCVerifier) KeyHash() uint32 { + return v.keyHash +} + +// Verify verifies the signature over the checkpoint message. +func (v *MTCVerifier) Verify(msg, sig []byte) bool { + // Parse checkpoint to get size and hash + // msg should be: + // [origin] + // [size] + // [hash] + lines := strings.Split(string(msg), "\n") + if len(lines) < 3 { + return false + } + sizeStr := lines[1] + hashB64 := lines[2] + + size, err := strconv.ParseUint(sizeStr, 10, 64) + if err != nil { + return false + } + + hash, err := base64.StdEncoding.DecodeString(hashB64) + if err != nil { + return false + } + + if len(hash) != 32 { + return false + } + + // Construct MTCSubtreeSignatureInput: + // "mtc-subtree/v1\n" + 0 (byte) + len(cosignerID) (1 byte) + cosignerID + len(logID) (1 byte) + logID + start (8 bytes uint64 big-endian) + end (8 bytes uint64 big-endian) + hash (32 bytes) + var input []byte + input = append(input, []byte("mtc-subtree/v1\n")...) + input = append(input, 0) + + if len(v.cosignerIDBytes) > 255 || len(v.logIDBytes) > 255 { + return false // Length must fit in 1 byte + } + + input = append(input, byte(len(v.cosignerIDBytes))) + input = append(input, v.cosignerIDBytes...) + input = append(input, byte(len(v.logIDBytes))) + input = append(input, v.logIDBytes...) + + var startBytes [8]byte + binary.BigEndian.PutUint64(startBytes[:], 0) + input = append(input, startBytes[:]...) + + var endBytes [8]byte + binary.BigEndian.PutUint64(endBytes[:], size) + input = append(input, endBytes[:]...) + + input = append(input, hash...) + + return ed25519.Verify(v.pubKey, input, sig) +} + +// Interface check +var _ note.Verifier = (*MTCVerifier)(nil) + +// Helper functions for relative OID encoding + +func encodeRelativeOIDStr(s string) ([]byte, error) { + parts := strings.Split(s, ".") + var components []uint32 + for _, p := range parts { + v, err := strconv.ParseUint(p, 10, 32) + if err != nil { + return nil, err + } + components = append(components, uint32(v)) + } + return encodeRelativeOID(components), nil +} + +func encodeRelativeOID(components []uint32) []byte { + var encoded []byte + for _, c := range components { + encoded = append(encoded, encodeBase128(c)...) + } + return encoded +} + +func encodeBase128(v uint32) []byte { + if v == 0 { + return []byte{0} + } + var chunks []byte + for v > 0 { + chunks = append(chunks, byte(v&0x7f)) + v >>= 7 + } + var encoded []byte + for i := len(chunks) - 1; i >= 0; i-- { + b := chunks[i] + if i > 0 { + b |= 0x80 + } + encoded = append(encoded, b) + } + return encoded +} diff --git a/vindex/internal/mtc/mtc_test.go b/vindex/internal/mtc/mtc_test.go new file mode 100644 index 0000000..0d1e080 --- /dev/null +++ b/vindex/internal/mtc/mtc_test.go @@ -0,0 +1,78 @@ +package mtc + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/hex" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestParseAndExtract(t *testing.T) { + // Fixed DER payload (typo fixed in validity period, and appended 00 to match outer sequence length of 330 bytes) + derHex := "30820146a003020102301c311a3018060a2b0601040182da4b2f010c0a34343336332e34382e38301e170d3235313131313230313732365a170d3235313131383230313732365a30818e310b3009060355040613025553311330110603550408130a43616c69666f726e6961311430120603550407130b4c6f7320416e67656c6573313c303a060355040a1333496e7465726e657420436f72706f726174696f6e20666f722041737369676e6564204e616d657320616e64204e756d626572733116301406035504030c0d2a2e6578616d706c652e636f6d042088c3292097527f95650a51dac5945eca168bc4bb2664c30d022036a4c47cfccea34e304c30250603551d11041e301c820d2a2e6578616d706c652e636f6d820b6578616d706c652e636f6d300e0603551d0f0101ff04040302078030130603551d25040c300a06082b0601050507030100" + der, err := hex.DecodeString(derHex) + if err != nil { + t.Fatalf("Failed to decode hex: %v", err) + } + + entry, err := ParseTBSCertificateLogEntry(der) + if err != nil { + t.Fatalf("ParseTBSCertificateLogEntry failed: %v", err) + } + + gotDNSNames := ExtractDNSNames(entry) + wantDNSNames := []string{"*.example.com", "example.com"} + + if diff := cmp.Diff(wantDNSNames, gotDNSNames); diff != "" { + t.Errorf("ExtractDNSNames mismatch (-want +got):\n%s", diff) + } +} + +func TestMTCVerifier(t *testing.T) { + name := "oid/1.3.6.1.4.1.44363.47.1.44363.48.8" + pubKeyB64 := "teYkXkxVoKhT1PxKODAyZFqUk8KZ4tUjzS6yAvvZ8hU=" + cosignerID := "44363.48.9" + logID := "44363.48.8" + + pubKeyBytes, err := base64.StdEncoding.DecodeString(pubKeyB64) + if err != nil { + t.Fatalf("Failed to decode pubKey: %v", err) + } + pubKey := ed25519.PublicKey(pubKeyBytes) + + verifier, err := NewMTCVerifier(name, pubKey, cosignerID, logID) + if err != nil { + t.Fatalf("NewMTCVerifier failed: %v", err) + } + + if verifier.Name() != name { + t.Errorf("Name() = %q, want %q", verifier.Name(), name) + } + + // Expected KeyHash: 0x8e6a3b1a + wantKeyHash := uint32(0x8e6a3b1a) + if verifier.KeyHash() != wantKeyHash { + t.Errorf("KeyHash() = 0x%x, want 0x%x", verifier.KeyHash(), wantKeyHash) + } + + checkpointText := `bootstrap-mtca.cloudflareresearch.com/logs/shard3 +197762974 +7NWRnNW49lCjHOLMyMLBYRkRTGxDhVEmQhTw2gD/Pig=` + + sigB64 := "jmo7GsZLYkXSa+C4eXII6rUTN8BECzdUogRIlzML8wqeWnFuTjOjAOGrHu79pnjuZBx1syo5FYNs5sKpj2D93QEkLws=" + sigBytes, err := base64.StdEncoding.DecodeString(sigB64) + if err != nil { + t.Fatalf("Failed to decode signature: %v", err) + } + + if len(sigBytes) < 4 { + t.Fatalf("Signature too short") + } + rawSig := sigBytes[4:] + + if !verifier.Verify([]byte(checkpointText), rawSig) { + t.Error("Verification failed") + } +}