Lookup returns mostly provable response

vindex/api/api.go

@@ -44,7 +44,6 @@ type LookupResponse struct {
 	// These values represent the lookup operation in the index at the root hash
 	// committed to by OutputLogLeaf. The values contain all indices for the given
 	// key, and the proof binds these values at this key at the index root hash.
-	IndexKey   [sha256.Size]byte   `json:"index_key"`
 	IndexValue []uint64            `json:"index_value"`
 	IndexProof [][sha256.Size]byte `json:"index_proof"`
 }

vindex/cmd/client/client.go

@@ -18,8 +18,10 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha256"
+	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -29,13 +31,18 @@ import (
 	"net/http"
 	"net/url"
 
+	"github.com/transparency-dev/formats/log"
 	"github.com/transparency-dev/incubator/vindex/api"
+	"github.com/transparency-dev/merkle/proof"
+	"github.com/transparency-dev/merkle/rfc6962"
+	"golang.org/x/mod/sumdb/note"
 	"k8s.io/klog/v2"
 )
 
 var (
-	baseURL = flag.String("base_url", "", "The base URL of the vindex server.")
-	lookup  = flag.String("lookup", "", "The key to look up in the vindex.")
+	baseURL      = flag.String("base_url", "", "The base URL of the vindex server.")
+	lookup       = flag.String("lookup", "", "The key to look up in the vindex.")
+	outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.")
 )
 
 func main() {
@@ -50,35 +57,15 @@ func run(ctx context.Context) error {
 	c := newVIndexClientFromFlags()
 
 	if *lookup == "" {
-		return errors.New("key flag must be provided")
+		return errors.New("lookup flag must be provided")
 	}
 
-	resp, err := c.Lookup(ctx, *lookup)
+	idxes, err := c.Lookup(ctx, *lookup)
 	if err != nil {
-		return fmt.Errorf("lookup for %q failed: %v", *lookup, err)
+		return fmt.Errorf("failed to look up key: %v", err)
 	}
 
-	// For now, pretty print the JSON response
-	jsonResp, err := json.MarshalIndent(resp, "", "  ")
-	if err != nil {
-		return fmt.Errorf("failed to marshal response: %v", err)
-	}
-	fmt.Println(string(jsonResp))
-
-	// This needs to verify the proofs in the response.
-	// We can't verify inclusion with the info we currently have.
-	// We at least need the index of the leaf returned. We could add
-	// that to the response object, but this is kinda rfc6962.
-	// What would it look like if the output log is tiled? What should
-	// the lookup response contain? Can it contain less, and thus put
-	// more on the client as per tlog-tiles? If so, what stops clients
-	// requesting old views of the index based on non-recent leaves from
-	// the output log?
-	// What if we flip this all around, and the OutputLog part of the response
-	// only returns an index into the output log, and the client has to look up
-	// that leaf, checkpoint, and generate inclusion proof?
-
-	// proof.VerifyInclusion()
+	fmt.Printf("Indices: %v\n", idxes)
 
 	return nil
 }
@@ -92,16 +79,75 @@ func newVIndexClientFromFlags() VIndexClient {
 		klog.Exitf("failed to parse URL: %v", err)
 	}
 	lookupURL := u.JoinPath(api.PathLookup)
+
+	if *outLogPubKey == "" {
+		klog.Exitf("out_log_pub_key must be provided")
+	}
+	outV, err := note.NewVerifier(*outLogPubKey)
+	if err != nil {
+		klog.Exitf("failed to construct output log verifier: %v", err)
+	}
 	return VIndexClient{
 		lookupURL: lookupURL,
+		outV:      outV,
 	}
 }
 
 type VIndexClient struct {
 	lookupURL *url.URL
+	outV      note.Verifier
+}
+
+func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, error) {
+	resp, err := c.lookupUnverified(ctx, key)
+	if err != nil {
+		return nil, fmt.Errorf("lookup for %q failed: %v", *lookup, err)
+	}
+
+	// Currently the response contains the RFC6962 style response type; leaf, proof, etc.
+	// What if we flip this all around, and the OutputLog part of the response
+	// only returns an index into the output log, and the client has to look up
+	// that leaf, checkpoint, and generate inclusion proof?
+
+	cp, _, _, err := log.ParseCheckpoint(resp.OutputLogCP, c.outV.Name(), c.outV)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse output log checkpoint: %v", err)
+	}
+	outLeafHash := rfc6962.DefaultHasher.HashLeaf(resp.OutputLogLeaf)
+	olp := make([][]byte, len(resp.OutputLogProof))
+	for i := range olp {
+		olp[i] = resp.OutputLogProof[i][:]
+	}
+	oli := cp.Size - 1 // TODO(mhutchinson): include this in the response?
+	if err := proof.VerifyInclusion(rfc6962.DefaultHasher, oli, cp.Size, outLeafHash[:], olp, cp.Hash); err != nil {
+		return nil, fmt.Errorf("failed to verify inclusion in output log: %v", err)
+	}
+	var mapRoot []byte
+	if idx := bytes.Index(resp.OutputLogLeaf, []byte{'\n', '\n'}); idx < 0 {
+		return nil, fmt.Errorf("failed to parse output log leaf: %q", resp.OutputLogLeaf)
+	} else {
+		mapRoot = resp.OutputLogLeaf[:idx]
+		mapRoot, err = hex.AppendDecode(nil, mapRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode map root: %v", err)
+		}
+	}
+
+	idxLeafHash := sha256.New()
+	for _, idx := range resp.IndexValue {
+		if err := binary.Write(idxLeafHash, binary.BigEndian, idx); err != nil {
+			return nil, fmt.Errorf("failed to calculate leaf hash for indices: %v", err)
+		}
+	}
+	vindexLeafHash := idxLeafHash.Sum(nil)
+	vindexKeyHash := sha256.Sum256([]byte(key))
+	// TODO(mhutchinson): verify inclusion in the vindex!
+	klog.Warningf("TODO: confirm inclusion of leaf hash %x at key location %x with root hash %x", vindexLeafHash, vindexKeyHash, mapRoot)
+
+	return resp.IndexValue, nil
 }
 
-func (c VIndexClient) Lookup(ctx context.Context, key string) (api.LookupResponse, error) {
+func (c VIndexClient) lookupUnverified(ctx context.Context, key string) (api.LookupResponse, error) {
 	var lookupResp api.LookupResponse
 
 	// For now, keys are stored under the hash of the key

vindex/cmd/logandmap/README.md

@@ -0,0 +1,62 @@
+## Verifiable Index: Log & Map
+
+This is a demo of using a [Tessera][] Verifiable Log as the Input Log, with all entries indexed by a [Verifiable Index](../../README.md).
+
+[tlog-tiles]: https://c2sp.org/tlog-tiles
+[Tessera]: https://github.com/transparency-dev/tessera
+
+The entries in the Input Log loosely represent Binary/Artifact Registry entries, committing to a triple of `{module name, module version, artifact hash}`:
+
+```
+{
+  "module": "bar",
+  "version": "2025-08-07T10:41:56.527888424Z",
+  "hash": "vsOru/9zZqrLjamAgzvQCaSvpMmF9jy+r75HpMvncZc="
+}
+```
+
+This pattern is very common: committing that a module at a given version has a particular hash.
+This hash could represent the git commit fingerprint the release was tagged from, the hash of a compiled binary, etc.
+See [Transparency.dev: Add tamper checking to a package manager](https://transparency.dev/application/add-tamper-checking-to-a-package-manager/) for more background on this pattern.
+
+This Input Log is processed, with each entry being indexed solely on the `module`,
+i.e. the key that is put into the map has the following value:
+
+```go
+		sha256.Sum256([]byte(entry.Module))
+```
+
+This allows the owner of a given module to look up the modules they are responsible for, and verifiably
+find the index of all entries in the Input Log for their modules.
+
+## Running
+
+The Input Log, Verifiable Index, and Output Log are all managed by a single binary:
+
+```shell
+INPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/inputlog+bd6268fb+ATPZW5UsUYHJo24lwgK1ykm9VafhyUtUxX5evV4ZIokY OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ go run ./vindex/cmd/logandmap --storage_dir ~/logandmap/
+```
+
+Running the above will run a web server hosting the following URLs:
+ - `/inputlog/` - the [tlog-tiles][] base URL for the input log
+ - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go)
+ - `/outputlog/` - TODO(mhutchinson): this is where the output log will be hosted
+
+The input log has entries for packages in the set {`foo`, `bar`, `baz`, `splat`}.
+To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above):
+
+```shell
+# To inspect the Input Log
+go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/inputlog/ --custom_log_vkey=example.com/inputlog+bd6268fb+AWdGkrHKBm+pOubTrcBTV8JMDLFlF1Y8WUH1nrtLNXDr
+
+# To inspect the Output Log
+go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/outputlog/ --custom_log_vkey=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8
+```
+
+Use left/right cursor to browse, and `q` to quit.
+
+This log is processed into a verifiable map which can be looked up using the following command:
+
+```shell
+go run ./vindex/cmd/client --base_url http://localhost:8088/vindex/ --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 --lookup=foo
+```

vindex/cmd/logandmap/main.go

@@ -255,13 +255,7 @@ func submitEntries(ctx context.Context, appender *tessera.Appender) {
 }
 
 func runWebServer(vi *vindex.VerifiableIndex, inLogDir, outLogDir string) {
-	web := NewServer(func(h [sha256.Size]byte) ([]uint64, error) {
-		idxes, size := vi.Lookup(h)
-		if size == 0 {
-			return nil, errors.New("index not populated")
-		}
-		return idxes, nil
-	})
+	web := NewServer(vi.Lookup)
 
 	ilfs := http.FileServer(http.Dir(inLogDir))
 	olfs := http.FileServer(http.Dir(outLogDir))

vindex/cmd/logandmap/web.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"context"
 	"crypto/sha256"
 	_ "embed"
 	"encoding/hex"
@@ -27,14 +28,14 @@ import (
 	"k8s.io/klog/v2"
 )
 
-func NewServer(lookup func([sha256.Size]byte) ([]uint64, error)) Server {
+func NewServer(lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error)) Server {
 	return Server{
 		lookup: lookup,
 	}
 }
 
 type Server struct {
-	lookup func([sha256.Size]byte) ([]uint64, error)
+	lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error)
 }
 
 // handleLookup handles GET requests for looking up map entries.
@@ -58,15 +59,13 @@ func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) {
 
 	klog.V(2).Infof("Received hash from request: '%s'", h)
 
-	idxes, err := s.lookup([sha256.Size]byte(h))
+	resp, err := s.lookup(r.Context(), [sha256.Size]byte(h))
 	if err != nil {
 		http.Error(w, fmt.Sprintf("lookup failed: %v", err), http.StatusInternalServerError)
 		return
 	}
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
-	var resp api.LookupResponse
-	resp.IndexValue = idxes
 	if err := json.NewEncoder(w).Encode(resp); err != nil {
 		klog.Warningf("failed to encode response: %v", err)
 	}