Verifiable Index for SumDB

sumdb/README.md

@@ -6,7 +6,7 @@ This allows tooling written for the tlog-tiles API to be used with the SumDB, ev
 ### Running
 
 ```shell
-go run ./sumdb/proxy.go --listen=":8089"
+go run ./sumdb/cmd/proxy.go --listen=":8089"
 ```
 
 ### Using

sumdb/cmd/proxy.go

@@ -0,0 +1,45 @@
+// Copyright 2025 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// sumdb is a command that launches a local proxy that allows clients to query
+// using the tlog-tiles API, and retrieve results from SumDB.
+package main
+
+import (
+	"flag"
+
+	"net/http"
+
+	"github.com/transparency-dev/incubator/sumdb"
+	"k8s.io/klog/v2"
+)
+
+var (
+	listen = flag.String("listen", ":8089", "Address to set up HTTP server listening on")
+)
+
+const (
+	upstreamBase = "https://sum.golang.org"
+)
+
+func main() {
+	klog.InitFlags(nil)
+	flag.Parse()
+
+	proxy := sumdb.NewProxy(sumdb.ProxyOpts{})
+	klog.Infof("Proxying tlog-tiles API to %s on %s", upstreamBase, *listen)
+	if err := http.ListenAndServe(*listen, proxy); err != nil {
+		klog.Fatalf("ListenAndServe: %v", err)
+	}
+}

sumdb/proxy.go

@@ -12,14 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// sumdb is a command that launches a local proxy that allows clients to query
-// using the tlog-tiles API, and retrieve results from SumDB.
-package main
+// sumdb provides a utility proxy to convert to a tlog-tiles API.
+package sumdb
 
 import (
 	"bytes"
 	"encoding/binary"
-	"flag"
 	"fmt"
 
 	"net/http"
@@ -32,23 +30,24 @@ import (
 	"k8s.io/klog/v2"
 )
 
-var (
-	listen = flag.String("listen", ":8089", "Address to set up HTTP server listening on")
-)
-
 const (
 	upstreamBase = "https://sum.golang.org"
 )
 
-func main() {
-	klog.InitFlags(nil)
-	flag.Parse()
+type ProxyOpts struct {
+	// PathPrefix should be set if the proxy is hosted not at "/".
+	// Any path beyond this should be set here, so that it can be stripped.
+	PathPrefix string
+}
 
+func NewProxy(opts ProxyOpts) *httputil.ReverseProxy {
 	upstream, err := url.Parse(upstreamBase)
 	if err != nil {
 		klog.Fatalf("Failed to parse upstream URL %q: %v", upstreamBase, err)
 	}
 
+	prefix, _ := strings.CutSuffix(opts.PathPrefix, "/")
+
 	const tlogEntriesPrefix = "/tile/entries/"
 	const tlogTilePrefix = "/tile/"
 
@@ -57,14 +56,16 @@ func main() {
 	proxy := &httputil.ReverseProxy{
 		Rewrite: func(r *httputil.ProxyRequest) {
 			r.SetURL(upstream)
+			inPath := strings.TrimPrefix(r.In.URL.Path, prefix)
+
 			klog.V(2).Infof("Request for %s", r.In.URL.Path)
-			if r.In.URL.Path == "/checkpoint" {
+			if inPath == "/checkpoint" {
 				r.Out.URL.Path = "/latest"
-			} else if strings.HasPrefix(r.In.URL.Path, tlogEntriesPrefix) {
-				o := strings.TrimPrefix(r.In.URL.Path, tlogEntriesPrefix)
+			} else if strings.HasPrefix(inPath, tlogEntriesPrefix) {
+				o := strings.TrimPrefix(inPath, tlogEntriesPrefix)
 				r.Out.URL.Path = fmt.Sprintf("%s%s", sumDBTileDataPrefix, o)
-			} else if strings.HasPrefix(r.In.URL.Path, tlogTilePrefix) {
-				o := strings.TrimPrefix(r.In.URL.Path, tlogTilePrefix)
+			} else if strings.HasPrefix(inPath, tlogTilePrefix) {
+				o := strings.TrimPrefix(inPath, tlogTilePrefix)
 				r.Out.URL.Path = fmt.Sprintf("%s%s", sumDBTilePrefix, o)
 			}
 		},
@@ -91,9 +92,5 @@ func main() {
 		},
 	}
 
-	klog.Infof("Proxying tlog-tiles API to %s on %s", upstreamBase, *listen)
-	if err := http.ListenAndServe(*listen, proxy); err != nil {
-		klog.Fatalf("ListenAndServe: %v", err)
-	}
+	return proxy
 }
-

vindex/client/client.go

@@ -179,7 +179,7 @@ func (c VIndexClient) lookupUnverified(ctx context.Context, kh [sha256.Size]byte
 // NewInputLogClient returns a client that allows pointers returned from the Verifiable Index
 // to be dereferenced by looking up entries in the Input Log. All operations are verified by this
 // client, which closes the loop.
-func NewInputLogClient(inLogUrl string, inV note.Verifier, hc *http.Client) (*InputLogClient, error) {
+func NewInputLogClient(inLogUrl string, origin string, inV note.Verifier, hc *http.Client) (*InputLogClient, error) {
 	u, err := url.Parse(inLogUrl)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse URL: %v", err)
@@ -189,23 +189,25 @@ func NewInputLogClient(inLogUrl string, inV note.Verifier, hc *http.Client) (*In
 		return nil, fmt.Errorf("failed to create HTTP fetcher for %q: %v", u, err)
 	}
 	return &InputLogClient{
-		v:  inV,
-		lc: c,
+		v:      inV,
+		origin: origin,
+		lc:     c,
 	}, nil
 }
 
 // InputLogClient is a client intended to be used by users of the VIndexClient that want
 // to look up the original leaves from the Input Log.
 type InputLogClient struct {
-	v  note.Verifier
-	lc logClient
+	v      note.Verifier
+	origin string
+	lc     logClient
 }
 
 // Dereference takes pointers returned by the VIndexClient Lookup method, and fetches
 // the original leaves from the Input Log. The inclusion of any leaves returned will be
 // verified by constructing inclusion proofs to the checkpoint provided.
 func (c *InputLogClient) Dereference(ctx context.Context, cpRaw []byte, pointers []uint64) iter.Seq2[InputLogLeaf, error] {
-	cp, _, _, err := log.ParseCheckpoint(cpRaw, c.v.Name(), c.v)
+	cp, _, _, err := log.ParseCheckpoint(cpRaw, c.origin, c.v)
 	if err != nil {
 		return func(yield func(InputLogLeaf, error) bool) {
 			yield(InputLogLeaf{}, fmt.Errorf("failed to parse input log checkpoint: %v", err))

vindex/cmd/client/client.go

@@ -36,6 +36,7 @@ var (
 	lookup        = flag.String("lookup", "", "The key to look up in the vindex.")
 	outLogPubKey  = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.")
 	inLogPubKey   = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint.")
+	inLogOrigin   = flag.String("in_log_origin", "", "Optional: allows the Input Log Origin string to be configured to something other than the public key name.")
 	minIdx        = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.")
 )
 
@@ -125,7 +126,11 @@ func newInputLogClientFromFlags() *client.InputLogClient {
 	if err != nil {
 		klog.Exitf("failed to construct output log verifier: %v", err)
 	}
-	c, err := client.NewInputLogClient(*inLogBaseURL, v, http.DefaultClient)
+	origin := *inLogOrigin
+	if len(origin) == 0 {
+		origin = v.Name()
+	}
+	c, err := client.NewInputLogClient(*inLogBaseURL, origin, v, http.DefaultClient)
 	if err != nil {
 		klog.Exitf("failed to construct VIndex Client: %v", err)
 	}

vindex/cmd/logandmap/main.go

@@ -50,8 +50,8 @@ import (
 )
 
 var (
-	inputLogPrivKeyFile  = flag.String("input_log_private_key", "", "Location of private key file. If unset, uses the contents of the INPUT_LOG_PRIVATE_KEY environment variable.")
-	outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.")
+	inputLogPrivKeyFile  = flag.String("input_log_private_key_path", "", "Location of private key file. If unset, uses the contents of the INPUT_LOG_PRIVATE_KEY environment variable.")
+	outputLogPrivKeyFile = flag.String("output_log_private_key_path", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.")
 	storageDir           = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Input Log, Output Log, and allocate space to store the verifiable map persistence.")
 	listen               = flag.String("listen", ":8088", "Address to set up HTTP server listening on")
 )
@@ -291,7 +291,7 @@ func getInputLogSignerVerifierOrDie() (note.Signer, note.Verifier) {
 	} else {
 		privKey = os.Getenv("INPUT_LOG_PRIVATE_KEY")
 		if len(privKey) == 0 {
-			klog.Exit("Supply private key file path using --input_log_private_key or set INPUT_LOG_PRIVATE_KEY environment variable")
+			klog.Exit("Supply private key file path using --input_log_private_key_path or set INPUT_LOG_PRIVATE_KEY environment variable")
 		}
 	}
 	s, v, err := fnote.NewEd25519SignerVerifier(privKey)
@@ -314,7 +314,7 @@ func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) {
 	} else {
 		privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY")
 		if len(privKey) == 0 {
-			klog.Exit("Supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable")
+			klog.Exit("Supply private key file path using --output_log_private_key_path or set OUTPUT_LOG_PRIVATE_KEY environment variable")
 		}
 	}
 	s, v, err := fnote.NewEd25519SignerVerifier(privKey)

vindex/cmd/sumdb/README.md

@@ -0,0 +1,48 @@
+## Verifiable Index: SumDB
+
+This is a demo of building a [Verifiable Index](../../README.md) for Go's SumDB.
+
+The index allows package maintainers to verifiably look up all [non-pseudo](https://pkg.go.dev/golang.org/x/mod@v0.28.0/module#IsPseudoVersion) versions of their module served by the Module Proxy.
+
+[tlog-tiles]: https://c2sp.org/tlog-tiles
+[Tessera]: https://github.com/transparency-dev/tessera
+
+## Running
+
+The Verifiable Index and Output Log are managed by a single binary, which can be run using:
+
+```shell
+OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+SumDBIndex+a5ed0e81+AYT6tfHpqGaSoH0gYpM7fhj1tEkM3wwYR/IhtiYh1pnj \
+go run ./vindex/cmd/sumdb \
+  --storage_dir ~/vindex-sumdb/
+```
+
+Running the above will run a web server hosting the following URLs:
+ - `/inputlog/` - the [tlog-tiles][] base URL for a proxy of the SumDB API
+ - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go)
+ - `/outputlog/` - the [tlog-tiles][] base URL for the output log
+
+To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above):
+
+```shell
+# To inspect the Output Log
+go run github.com/mhutchinson/woodpecker@main \
+  --custom_log_type=tiles \
+  --custom_log_url=http://localhost:8088/outputlog/ \
+  --custom_log_vkey=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd
+```
+
+Use left/right cursor to browse, and `q` to quit.
+
+This log is processed into a verifiable map which can be looked up using the following command:
+
+```shell
+go run ./vindex/cmd/client \
+  --vindex_base_url http://localhost:8088/vindex/ \
+  --in_log_base_url http://localhost:8088/inputlog/ \
+  --out_log_pub_key=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd \
+  --in_log_pub_key=sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 \
+  --in_log_origin="go.sum database tree" \
+  --lookup=github.com/transparency-dev/tessera
+```
+