Added MTC VIndex

vindex/client/client.go

@@ -18,6 +18,7 @@ package client
 import (
 	"context"
 	"crypto/sha256"
+	"encoding/base64"
 	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
@@ -26,6 +27,7 @@ import (
 	"iter"
 	"net/http"
 	"net/url"
+	"strings"
 
 	"github.com/transparency-dev/formats/log"
 	"github.com/transparency-dev/incubator/vindex"
@@ -78,7 +80,7 @@ func VerifyLookupResponse(keyHash [sha256.Size]byte, resp api.LookupResponse, in
 	}
 	ilcp, _, _, err := log.ParseCheckpoint(inCp, origin, inV)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to parse input log checkpoint: %v", err)
+		return nil, nil, fmt.Errorf("failed to parse input log checkpoint: %v (verifier: %s, keyhash: %08x; available signatures: %s)", err, inV.Name(), inV.KeyHash(), dumpSignatures(inCp))
 	}
 
 	expectFound := len(resp.IndexValue) > 0
@@ -302,3 +304,29 @@ type logClient interface {
 	ReadTile(ctx context.Context, l, i uint64, p uint8) ([]byte, error)
 	ReadEntryBundle(ctx context.Context, i uint64, p uint8) ([]byte, error)
 }
+
+func dumpSignatures(noteBytes []byte) string {
+	lines := strings.Split(string(noteBytes), "\n")
+	var sigs []string
+	for _, line := range lines {
+		if strings.HasPrefix(line, "\u2014 ") {
+			parts := strings.SplitN(line[4:], " ", 2)
+			if len(parts) == 2 {
+				name := parts[0]
+				sigB64 := parts[1]
+				sigBytes, err := base64.StdEncoding.DecodeString(sigB64)
+				if err == nil && len(sigBytes) >= 4 {
+					kh := binary.BigEndian.Uint32(sigBytes[:4])
+					sigs = append(sigs, fmt.Sprintf("%s (keyhash %08x)", name, kh))
+				} else {
+					sigs = append(sigs, name)
+				}
+			}
+		}
+	}
+	if len(sigs) == 0 {
+		return "none"
+	}
+	return strings.Join(sigs, ", ")
+}
+

vindex/cmd/client/client.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"context"
+	"crypto/ed25519"
 	"crypto/x509"
 	"encoding/base64"
 	"errors"
@@ -29,19 +30,24 @@ import (
 
 	fnote "github.com/transparency-dev/formats/note"
 	"github.com/transparency-dev/incubator/vindex/client"
+	"github.com/transparency-dev/incubator/vindex/internal/mtc"
 	"golang.org/x/mod/sumdb/note"
 	"k8s.io/klog/v2"
 )
 
 var (
-	vindexBaseURL  = flag.String("vindex_base_url", "", "The base URL of the vindex server.")
-	inLogBaseURL   = flag.String("in_log_base_url", "", "The base URL of the input log.")
-	lookup         = flag.String("lookup", "", "The key to look up in the vindex.")
-	outLogPubKey   = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint. Required.")
-	inLogPubKey    = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint. Required.")
-	inLogPubKeyDER = flag.String("in_log_pub_key_der", "", "For CT logs. The public key to use to verify the input log checkpoint. Required, along with in_log_origin.")
-	inLogOrigin    = flag.String("in_log_origin", "", "Required if in_log_pub_key_der is used. Otherwise, allows the Input Log Origin string to be configured to something other than the public key name.")
-	minIdx         = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.")
+	vindexBaseURL   = flag.String("vindex_base_url", "", "The base URL of the vindex server.")
+	inLogBaseURL    = flag.String("in_log_base_url", "", "The base URL of the input log.")
+	lookup          = flag.String("lookup", "", "The key to look up in the vindex.")
+	outLogPubKey    = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint. Required.")
+	inLogPubKey     = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint. Required.")
+	inLogPubKeyDER  = flag.String("in_log_pub_key_der", "", "For CT logs. The public key to use to verify the input log checkpoint. Required, along with in_log_origin.")
+	inLogOrigin     = flag.String("in_log_origin", "", "Required if in_log_pub_key_der is used. Otherwise, allows the Input Log Origin string to be configured to something other than the public key name. For MTC, this is the checkpoint origin.")
+	inLogKeyName    = flag.String("in_log_key_name", "", "For MTC logs. The key name used in the checkpoint signature. If unset, defaults to in_log_origin.")
+	inLogMTC        = flag.Bool("in_log_mtc", false, "Use MTC verifier for input log.")
+	inLogCosignerID = flag.String("in_log_cosigner_id", "", "For MTC logs. The relative OID of the cosigner.")
+	inLogID         = flag.String("in_log_id", "", "For MTC logs. The relative OID of the log.")
+	minIdx          = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.")
 )
 
 func main() {
@@ -137,6 +143,41 @@ func newInputLogClientFromFlags() *client.InputLogClient {
 }
 
 func inputLogVerifierFromFlags() note.Verifier {
+	if *inLogMTC {
+		if *inLogPubKey == "" {
+			klog.Exitf("in_log_pub_key must be provided when using --in_log_mtc")
+		}
+		keyName := *inLogKeyName
+		if keyName == "" {
+			keyName = *inLogOrigin
+		}
+		if keyName == "" {
+			klog.Exitf("in_log_key_name (or in_log_origin) must be provided when using --in_log_mtc")
+		}
+		if *inLogCosignerID == "" {
+			klog.Exitf("in_log_cosigner_id must be provided when using --in_log_mtc")
+		}
+		if *inLogID == "" {
+			klog.Exitf("in_log_id must be provided when using --in_log_mtc")
+		}
+		pubKeyBytes, err := base64.StdEncoding.DecodeString(*inLogPubKey)
+		if err != nil {
+			klog.Exitf("failed to decode log_public_key: %v", err)
+		}
+		// At the moment this is optimized for the Cloudflare bootstrap log.
+		// This will need to be changed to support ML-DSA-44 when we support other MTC logs.
+		// Other changes are also likely to be needed, due to spec evolution since bootstrap launched.
+		if len(pubKeyBytes) != ed25519.PublicKeySize {
+			klog.Exitf("invalid log_public_key size: %d, expected %d", len(pubKeyBytes), ed25519.PublicKeySize)
+		}
+		pubKey := ed25519.PublicKey(pubKeyBytes)
+		v, err := mtc.NewMTCVerifier(keyName, pubKey, *inLogCosignerID, *inLogID)
+		if err != nil {
+			klog.Exitf("failed to create MTCVerifier: %v", err)
+		}
+		return v
+	}
+
 	if (*inLogPubKey == "") == (*inLogPubKeyDER == "") {
 		klog.Exitf("Must provide exactly one --in_log_pub_key* flag")
 	}

vindex/cmd/mtcindex/README.md

@@ -0,0 +1,65 @@
+# MTC Verifiable Indexer (`mtcindex`)
+
+`mtcindex` is a verifiable indexer for Merkle Tree Certificates (MTC) logs. It processes leaf entries from an MTC log, parses the certificates, and indexes them by domain name in a verifiable map. This allows users to query for all certificates associated with a domain verifiably and efficiently, without downloading the entire log.
+
+For details on the Merkle Tree Certificates specification, see the [MTC Draft](https://datatracker.ietf.org/doc/draft-davidben-tls-merkle-tree-certs/).
+For details on the Verifiable Index architecture (the "Map Sandwich"), see the [root VIndex README](../../README.md).
+
+## How it works
+
+1. **Input Log**: It monitors an MTC log (e.g., Cloudflare's bootstrap MTC log).
+1. **MTC Verification**: It uses a custom checkpoint verifier that handles MTC-specific binary checkpoint signatures (`MTCSubtreeSignatureInput`) instead of standard text-based note signatures.
+1. **Parsing (`mapFn`)**: It parses leaf entries of type `tbs_cert_entry` (type 1) using ASN.1, extracts the DNS names from the Subject Alternative Name (SAN) extension, and hashes them. Other entry types (like `null_entry`) are ignored.
+1. **Verifiable Map**: It maintains a verifiable map (using Pebble DB for persistence) mapping domain hashes to log indices.
+1. **Output Log**: It writes the map roots to an output log, which is signed by the operator.
+
+## Usage
+
+> [!WARNING]
+> This demo uses hardcoded cryptographic keys. Do not use these keys in production or public deployments. Ensure you generate and use your own secure keys when deploying this service.
+
+### 1. Run the Indexer
+
+Run the indexer by pointing it to a storage directory and providing the output log private key.
+
+By default, it is configured to index the Cloudflare Shard 3 MTC log. We use a demo private key here:
+
+```bash
+OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ \
+go run ./vindex/cmd/mtcindex/ \
+  --storage_dir=/path/to/storage \
+  --listen=:8088
+```
+
+### 2. Query the Index
+
+Once running, the indexer hosts a web server (default `:8088`) serving the verifiable index.
+A domain indexed by the verifiable map can be looked up using the client command.
+
+For example, to look up `minefun.io`:
+
+```shell
+go run ./vindex/cmd/client \
+  --vindex_base_url http://localhost:8088/vindex/ \
+  --out_log_pub_key="example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8" \
+  --in_log_pub_key="teYkXkxVoKhT1PxKODAyZFqUk8KZ4tUjzS6yAvvZ8hU=" \
+  --in_log_mtc \
+  --in_log_origin="bootstrap-mtca.cloudflareresearch.com/logs/shard3" \
+  --in_log_cosigner_id="44363.48.9" \
+  --in_log_id="44363.48.8" \
+  --in_log_key_name="oid/1.3.6.1.4.1.44363.47.1.44363.48.8" \
+  --lookup=minefun.io
+```
+## Flags
+
+- `--log_url`: Base URL of the MTC log to index (default: `https://bootstrap-mtca-shard3.cloudflareresearch.com/`).
+- `--key_name`: The key name used in the checkpoint signature (default: `oid/1.3.6.1.4.1.44363.47.1.44363.48.8`).
+- `--log_public_key`: The log's public key, base64 encoded raw 32-byte Ed25519 key (default: `teYkXkxVoKhT1PxKODAyZFqUk8KZ4tUjzS6yAvvZ8hU=`).
+- `--cosigner_id`: The relative OID of the cosigner (default: `44363.48.9`).
+- `--log_id`: The relative OID of the log (default: `44363.48.8`).
+- `--origin`: The expected origin string in the checkpoint text (default: `bootstrap-mtca.cloudflareresearch.com/logs/shard3`).
+- `--storage_dir`: Root directory in which to store the output log data and verifiable map persistence (required).
+- `--listen`: Address to set up the HTTP server on (default: `:8088`).
+- `--persist_index`: Set to false to use a memory-based implementation of the verifiable index (default: `true`).
+- `--oneshot`: Set to true to build the map once up to the current log size and then exit (default: `false`).
+- `--output_log_private_key_path`: Location of the output log private key file.