From 67ff450e3a1ab1d4f159ba2348a5572144721e49 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Wed, 20 Aug 2025 15:59:34 +0000 Subject: [PATCH 1/2] [VIndex client] Deference input log pointers This allows the pointers into the input log to be automatically dereferenced by the client tool. For now, this just prints out the string value of whatever is there. This isn't appropriate for every input log, but it's good enough to demo the service end-to-end, which is its main purpose in life. --- vindex/cmd/client/client.go | 129 +++++++++++++++++++++++++++++++-- vindex/cmd/logandmap/README.md | 2 +- 2 files changed, 123 insertions(+), 8 deletions(-) diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index 2f0ffaa..10c28e4 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -30,19 +30,25 @@ import ( "io" "net/http" "net/url" + "slices" "github.com/transparency-dev/formats/log" "github.com/transparency-dev/incubator/vindex/api" "github.com/transparency-dev/merkle/proof" "github.com/transparency-dev/merkle/rfc6962" + "github.com/transparency-dev/tessera/api/layout" + "github.com/transparency-dev/tessera/client" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" ) var ( - baseURL = flag.String("base_url", "", "The base URL of the vindex server.") - lookup = flag.String("lookup", "", "The key to look up in the vindex.") - outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") + vindexBaseURL = flag.String("vindex_base_url", "", "The base URL of the vindex server.") + inLogBaseURL = flag.String("in_log_base_url", "", "The base URL of the input log.") + lookup = flag.String("lookup", "", "The key to look up in the vindex.") + outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") + inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint.") + minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") ) func main() { @@ -65,16 +71,43 @@ func run(ctx context.Context) error { return fmt.Errorf("failed to look up key: %v", err) } - fmt.Printf("Indices: %v\n", idxes) + if i := slices.IndexFunc(idxes, func(idx uint64) bool { + return idx >= *minIdx + }); i > 0 { + klog.Infof("Dropping %d pointers to index less than min_idx %d", i, *minIdx) + idxes = idxes[i:] + } else if i < 0 { + klog.Infof("Dropping %d pointers to index less than min_idx %d", len(idxes), *minIdx) + idxes = []uint64{} + } + if len(idxes) == 0 { + klog.Infof("No values found for key %q", *lookup) + return nil + } + + lr, err := NewLeafReaderFromFlags(ctx) + if err != nil { + return err + } + + klog.Infof("Dereferencing %d pointers", len(idxes)) + for _, idx := range idxes { + leaf, err := lr.getLeaf(ctx, idx) + if err != nil { + klog.Errorf("failed to get leaf at index %d: %v", idx, err) + continue + } + fmt.Printf("%d)\n%s\n\n", idx, leaf) + } return nil } func newVIndexClientFromFlags() VIndexClient { - if *baseURL == "" { - klog.Exit("base_url flag must be provided") + if *vindexBaseURL == "" { + klog.Exit("vindex_base_url flag must be provided") } - u, err := url.Parse(*baseURL) + u, err := url.Parse(*vindexBaseURL) if err != nil { klog.Exitf("failed to parse URL: %v", err) } @@ -183,3 +216,85 @@ func (c VIndexClient) lookupUnverified(ctx context.Context, key string) (api.Loo } return lookupResp, nil } + +func NewLeafReaderFromFlags(ctx context.Context) (*LeafReader, error) { + inV, err := note.NewVerifier(*inLogPubKey) + if err != nil { + klog.Exitf("failed to construct input log verifier: %v", err) + } + + if *inLogBaseURL == "" { + klog.Exit("in_log_base_url flag must be provided") + } + u, err := url.Parse(*inLogBaseURL) + if err != nil { + return nil, fmt.Errorf("failed to parse URL: %v", err) + } + c, err := client.NewHTTPFetcher(u, http.DefaultClient) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP fetcher for %q: %v", u, err) + } + + cpRaw, err := c.ReadCheckpoint(ctx) + if err != nil { + return nil, fmt.Errorf("failed to read checkpoint from input log: %v", err) + } + cp, _, _, err := log.ParseCheckpoint(cpRaw, inV.Name(), inV) + if err != nil { + klog.Warning(string(cpRaw)) + return nil, fmt.Errorf("failed to parse input log checkpoint: %v", err) + } + lr := &LeafReader{ + f: c.ReadEntryBundle, + cp: cp, + } + return lr, nil +} + +// LeafReader reads leaves from the tree. +// This class is not thread safe. +type LeafReader struct { + f client.EntryBundleFetcherFunc + cp *log.Checkpoint + c leafBundleCache +} + +// getLeaf fetches the raw contents committed to at a given leaf index. +func (r *LeafReader) getLeaf(ctx context.Context, i uint64) ([]byte, error) { + if i >= r.cp.Size { + return nil, fmt.Errorf("requested leaf %d >= log size %d", i, r.cp.Size) + } + if cached, _ := r.c.get(i); cached != nil { + klog.V(2).Infof("Using cached result for index %d", i) + return cached, nil + } + + // TODO(mhutchinson): Check the inclusion proof of a fetched bundle + bundle, err := client.GetEntryBundle(ctx, r.f, i/layout.EntryBundleWidth, r.cp.Size) + if err != nil { + return nil, fmt.Errorf("failed to get entry bundle: %v", err) + } + ti := i % layout.EntryBundleWidth + r.c = leafBundleCache{ + start: i - ti, + leaves: bundle.Entries, + } + return r.c.leaves[ti], nil +} + +// leafBundleCache stores the results of the last fetched tile. Assuming that the client +// accesses leaves in order, then this avoids fetching the same bundle multiple times if +// multiple leaves are in the same bundle. +type leafBundleCache struct { + start uint64 + leaves [][]byte +} + +func (tc leafBundleCache) get(i uint64) ([]byte, error) { + end := tc.start + uint64(len(tc.leaves)) + if i >= tc.start && i < end { + leaf := tc.leaves[i-tc.start] + return leaf, nil + } + return nil, errors.New("not found") +} diff --git a/vindex/cmd/logandmap/README.md b/vindex/cmd/logandmap/README.md index 4c92d99..bc2dcac 100644 --- a/vindex/cmd/logandmap/README.md +++ b/vindex/cmd/logandmap/README.md @@ -58,5 +58,5 @@ Use left/right cursor to browse, and `q` to quit. This log is processed into a verifiable map which can be looked up using the following command: ```shell -go run ./vindex/cmd/client --base_url http://localhost:8088/vindex/ --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 --lookup=foo +go run ./vindex/cmd/client --vindex_base_url http://localhost:8088/vindex/ --in_log_base_url http://localhost:8088/inputlog/ --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 --in_log_pub_key=example.com/inputlog+bd6268fb+AWdGkrHKBm+pOubTrcBTV8JMDLFlF1Y8WUH1nrtLNXDr --lookup=foo ``` From b2dd3e847b32b57ba089b9c942b3c8bbb11326c5 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Thu, 21 Aug 2025 10:16:26 +0000 Subject: [PATCH 2/2] Address comments --- vindex/cmd/client/client.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index 10c28e4..3becc97 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -131,6 +131,15 @@ type VIndexClient struct { outV note.Verifier } +// Lookup returns all indices, in ascending order, where the given key appears in the Input Log. +// This will be verified before being returned from this method, so a caller can be assured that +// any results (including the empty slice, i.e. non-presence) were found in the verifiable index, +// and committed to by the output log. +// +// Note that it is up to the caller to ensure that any leaves looked up in the Input Log are +// verified by an inclusion proof. +// TODO(mhutchinson): maybe this should return the Input Log Checkpoint that was committed to in +// the Output Log leaf? func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, error) { resp, err := c.lookupUnverified(ctx, key) if err != nil { @@ -264,7 +273,7 @@ func (r *LeafReader) getLeaf(ctx context.Context, i uint64) ([]byte, error) { if i >= r.cp.Size { return nil, fmt.Errorf("requested leaf %d >= log size %d", i, r.cp.Size) } - if cached, _ := r.c.get(i); cached != nil { + if cached := r.c.get(i); cached != nil { klog.V(2).Infof("Using cached result for index %d", i) return cached, nil } @@ -290,11 +299,11 @@ type leafBundleCache struct { leaves [][]byte } -func (tc leafBundleCache) get(i uint64) ([]byte, error) { +func (tc leafBundleCache) get(i uint64) []byte { end := tc.start + uint64(len(tc.leaves)) if i >= tc.start && i < end { leaf := tc.leaves[i-tc.start] - return leaf, nil + return leaf } - return nil, errors.New("not found") + return nil }