From 309c2c273b249af99810f30225a2674b4f80073e Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Tue, 26 Aug 2025 10:22:44 +0000 Subject: [PATCH] Better client, easier reading This contains refactoring, and improved comments, etc. Most noticeably, the client has been moved to its own library. Functional changes: - Output Leaf now has one newline instead of two to separate hash and Input Log Checkpoint - Full checking of entries in the Input Log --- vindex/client/client.go | 286 +++++++++++++++++++++++++++++++++ vindex/cmd/client/client.go | 245 +++------------------------- vindex/cmd/logandmap/README.md | 2 +- vindex/cmd/logandmap/main.go | 7 +- vindex/map.go | 47 ++---- vindex/outputlog.go | 24 +++ vindex/outputlog_test.go | 20 +++ 7 files changed, 371 insertions(+), 260 deletions(-) create mode 100644 vindex/client/client.go diff --git a/vindex/client/client.go b/vindex/client/client.go new file mode 100644 index 0000000..80a3b78 --- /dev/null +++ b/vindex/client/client.go @@ -0,0 +1,286 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// client contains a library for interacting with a Verifiable Index. +package client + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "iter" + "net/http" + "net/url" + + "filippo.io/torchwood/prefix" + "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/incubator/vindex" + "github.com/transparency-dev/incubator/vindex/api" + "github.com/transparency-dev/merkle/proof" + "github.com/transparency-dev/merkle/rfc6962" + "github.com/transparency-dev/tessera/api/layout" + "github.com/transparency-dev/tessera/client" + "golang.org/x/mod/sumdb/note" + "k8s.io/klog/v2" +) + +// NewVIndexClient returns a client that can perform verified lookups into the index at the +// given base URL, using the supplied verifier to check checkpoint signatures on the output +// log. +func NewVIndexClient(vindexUrl string, outV note.Verifier) (*VIndexClient, error) { + viu, err := url.Parse(vindexUrl) + if err != nil { + return nil, fmt.Errorf("failed to parse URL: %v", err) + } + lookupURL := viu.JoinPath(api.PathLookup) + + return &VIndexClient{ + lookupURL: lookupURL, + outV: outV, + }, nil +} + +// VIndexClient allows verified lookups into a verifiable index. +type VIndexClient struct { + lookupURL *url.URL + outV note.Verifier +} + +// Lookup returns all indices, in ascending order, where the given key appears in the Input Log. +// This will be verified before being returned from this method, so a caller can be assured that +// any results (including the empty slice, i.e. non-presence) were found in the verifiable index, +// and committed to by the output log. +// On success, this also returns the Checkpoint for the Input Log that was relied upon by the +// verifiable index. This may be used by the caller when constructing inclusion proofs when +// dereferencing any pointers returned. +// +// Note that it is up to the caller to ensure that any leaves looked up in the Input Log are +// verified by an inclusion proof. The checkpoint returned by this method can be used. +// The easiest way to do this is to use the InputLogClient. +func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, []byte, error) { + kh := sha256.Sum256([]byte(key)) + resp, err := c.lookupUnverified(ctx, kh) + if err != nil { + return nil, nil, fmt.Errorf("lookup failed: %v", err) + } + + // Currently the response contains the RFC6962 style response type; leaf, proof, etc. + // What if we flip this all around, and the OutputLog part of the response + // only returns an index into the output log, and the client has to look up + // that leaf, checkpoint, and generate inclusion proof? + + cp, _, _, err := log.ParseCheckpoint(resp.OutputLogCP, c.outV.Name(), c.outV) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse output log checkpoint: %v", err) + } + outLeafHash := rfc6962.DefaultHasher.HashLeaf(resp.OutputLogLeaf) + olp := make([][]byte, len(resp.OutputLogProof)) + for i := range olp { + olp[i] = resp.OutputLogProof[i][:] + } + oli := cp.Size - 1 // TODO(mhutchinson): include this in the response? + if err := proof.VerifyInclusion(rfc6962.DefaultHasher, oli, cp.Size, outLeafHash[:], olp, cp.Hash); err != nil { + return nil, nil, fmt.Errorf("failed to verify inclusion in output log: %v", err) + } + + mapRoot, inCp, err := vindex.UnmarshalLeaf(resp.OutputLogLeaf) + if err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal output log leaf: %v", err) + } + + idxLeafHash := sha256.New() + for _, idx := range resp.IndexValue { + if err := binary.Write(idxLeafHash, binary.BigEndian, idx); err != nil { + return nil, nil, fmt.Errorf("failed to calculate leaf hash for indices: %v", err) + } + } + vindexLeafHash := idxLeafHash.Sum(nil) + + pns := make([]prefix.ProofNode, len(resp.IndexProof)) + for i, p := range resp.IndexProof { + label, err := prefix.NewLabel(p.LabelBitLen, p.LabelPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to create label: %v", err) + } + pns[i] = prefix.ProofNode{ + Label: label, + Hash: p.Hash, + } + } + + if len(resp.IndexValue) > 0 { + if err := prefix.VerifyMembershipProof(sha256.Sum256, kh, [32]byte(vindexLeafHash), pns, mapRoot); err != nil { + return nil, nil, fmt.Errorf("failed to verify membership: %v", err) + } + } else { + if err := prefix.VerifyNonMembershipProof(sha256.Sum256, kh, pns, mapRoot); err != nil { + return nil, nil, fmt.Errorf("failed to verify non-membership: %v", err) + } + } + + return resp.IndexValue, inCp, nil +} + +func (c VIndexClient) lookupUnverified(ctx context.Context, kh [sha256.Size]byte) (api.LookupResponse, error) { + var lookupResp api.LookupResponse + + // For now, keys are stored under the hash of the key + u := c.lookupURL.JoinPath(hex.EncodeToString(kh[:])) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return lookupResp, fmt.Errorf("failed to create request: %v", err) + } + + klog.V(1).Infof("Making request to %q", u.String()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return lookupResp, fmt.Errorf("failed to get URL %q: %v", u, err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return lookupResp, fmt.Errorf("got non-200 status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return lookupResp, fmt.Errorf("failed to read response body: %v", err) + } + + if err := json.Unmarshal(body, &lookupResp); err != nil { + return lookupResp, fmt.Errorf("failed to unmarshal response: %v", err) + } + return lookupResp, nil +} + +// NewInputLogClient returns a client that allows pointers returned from the Verifiable Index +// to be dereferenced by looking up entries in the Input Log. All operations are verified by this +// client, which closes the loop. +func NewInputLogClient(inLogUrl string, inV note.Verifier, hc *http.Client) (*InputLogClient, error) { + u, err := url.Parse(inLogUrl) + if err != nil { + return nil, fmt.Errorf("failed to parse URL: %v", err) + } + c, err := client.NewHTTPFetcher(u, hc) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP fetcher for %q: %v", u, err) + } + return &InputLogClient{ + v: inV, + lc: c, + }, nil +} + +// InputLogClient is a client intended to be used by users of the VIndexClient that want +// to look up the original leaves from the Input Log. +type InputLogClient struct { + v note.Verifier + lc logClient +} + +// Dereference takes pointers returned by the VIndexClient Lookup method, and fetches +// the original leaves from the Input Log. The inclusion of any leaves returned will be +// verified by constructing inclusion proofs to the checkpoint provided. +func (c *InputLogClient) Dereference(ctx context.Context, cpRaw []byte, pointers []uint64) iter.Seq2[InputLogLeaf, error] { + cp, _, _, err := log.ParseCheckpoint(cpRaw, c.v.Name(), c.v) + if err != nil { + return func(yield func(InputLogLeaf, error) bool) { + yield(InputLogLeaf{}, fmt.Errorf("failed to parse input log checkpoint: %v", err)) + } + } + pb, err := client.NewProofBuilder(ctx, cp.Size, c.lc.ReadTile) + if err != nil { + return func(yield func(InputLogLeaf, error) bool) { + yield(InputLogLeaf{}, fmt.Errorf("failed to parse input log checkpoint: %v", err)) + } + } + return func(yield func(InputLogLeaf, error) bool) { + var cache leafBundleCache + for _, i := range pointers { + if i >= cp.Size { + yield(InputLogLeaf{}, fmt.Errorf("requested leaf %d >= log size %d", i, cp.Size)) + return + } + ip, err := pb.InclusionProof(ctx, i) + if err != nil { + yield(InputLogLeaf{}, fmt.Errorf("failed to get inclusion proof: %v", err)) + return + } + + var entry []byte + if entry = cache.get(i); entry == nil { + bundle, err := client.GetEntryBundle(ctx, c.lc.ReadEntryBundle, i/layout.EntryBundleWidth, cp.Size) + if err != nil { + yield(InputLogLeaf{}, fmt.Errorf("failed to get entry bundle: %v", err)) + return + } + + // Store the bundle in a cache in case the next index is in the same bundle. + ti := i % layout.EntryBundleWidth + cache = leafBundleCache{ + start: i - ti, + leaves: bundle.Entries, + } + entry = cache.leaves[ti] + } + + lh := rfc6962.DefaultHasher.HashLeaf(entry) + if err := proof.VerifyInclusion(rfc6962.DefaultHasher, i, cp.Size, lh, ip, cp.Hash); err != nil { + yield(InputLogLeaf{}, fmt.Errorf("failed to verify inclusion proof: %v", err)) + return + } + + if !yield(InputLogLeaf{i, entry}, nil) { + return + } + } + } +} + +// InputLogLeaf is an entry in the Input Log. +type InputLogLeaf struct { + Index uint64 + Data []byte +} + +// leafBundleCache stores the results of the last fetched tile. Assuming that the client +// accesses leaves in order, then this avoids fetching the same bundle multiple times if +// multiple leaves are in the same bundle. +type leafBundleCache struct { + start uint64 + leaves [][]byte +} + +func (tc leafBundleCache) get(i uint64) []byte { + end := tc.start + uint64(len(tc.leaves)) + if i >= tc.start && i < end { + leaf := tc.leaves[i-tc.start] + return leaf + } + return nil +} + +// logClient describes what we need from a log client. +type logClient interface { + ReadTile(ctx context.Context, l, i uint64, p uint8) ([]byte, error) + ReadEntryBundle(ctx context.Context, i uint64, p uint8) ([]byte, error) +} diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index d7deff8..41e85fc 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -18,27 +18,14 @@ package main import ( - "bytes" "context" - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "encoding/json" "errors" "flag" "fmt" - "io" "net/http" - "net/url" "slices" - "filippo.io/torchwood/prefix" - "github.com/transparency-dev/formats/log" - "github.com/transparency-dev/incubator/vindex/api" - "github.com/transparency-dev/merkle/proof" - "github.com/transparency-dev/merkle/rfc6962" - "github.com/transparency-dev/tessera/api/layout" - "github.com/transparency-dev/tessera/client" + "github.com/transparency-dev/incubator/vindex/client" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" ) @@ -61,13 +48,13 @@ func main() { } func run(ctx context.Context) error { - c := newVIndexClientFromFlags() + vic := newVIndexClientFromFlags() if *lookup == "" { return errors.New("lookup flag must be provided") } - idxes, err := c.Lookup(ctx, *lookup) + idxes, inCp, err := vic.Lookup(ctx, *lookup) if err != nil { return fmt.Errorf("failed to look up key: %v", err) } @@ -86,34 +73,24 @@ func run(ctx context.Context) error { return nil } - lr, err := NewLeafReaderFromFlags(ctx) - if err != nil { - return err - } + lr := newInputLogClientFromFlags() - klog.Infof("Dereferencing %d pointers", len(idxes)) - for _, idx := range idxes { - leaf, err := lr.getLeaf(ctx, idx) + klog.V(1).Infof("Dereferencing %d pointers", len(idxes)) + for leaf, err := range lr.Dereference(ctx, inCp, idxes) { if err != nil { - klog.Errorf("failed to get leaf at index %d: %v", idx, err) + klog.Errorf("failed to get leaf at index %d: %v", leaf.Index, err) continue } - fmt.Printf("%d)\n%s\n\n", idx, leaf) + fmt.Printf("%d)\n%s\n\n", leaf.Index, leaf.Data) } return nil } -func newVIndexClientFromFlags() VIndexClient { +func newVIndexClientFromFlags() *client.VIndexClient { if *vindexBaseURL == "" { klog.Exit("vindex_base_url flag must be provided") } - u, err := url.Parse(*vindexBaseURL) - if err != nil { - klog.Exitf("failed to parse URL: %v", err) - } - lookupURL := u.JoinPath(api.PathLookup) - if *outLogPubKey == "" { klog.Exitf("out_log_pub_key must be provided") } @@ -121,209 +98,27 @@ func newVIndexClientFromFlags() VIndexClient { if err != nil { klog.Exitf("failed to construct output log verifier: %v", err) } - return VIndexClient{ - lookupURL: lookupURL, - outV: outV, - } -} - -type VIndexClient struct { - lookupURL *url.URL - outV note.Verifier -} - -// Lookup returns all indices, in ascending order, where the given key appears in the Input Log. -// This will be verified before being returned from this method, so a caller can be assured that -// any results (including the empty slice, i.e. non-presence) were found in the verifiable index, -// and committed to by the output log. -// -// Note that it is up to the caller to ensure that any leaves looked up in the Input Log are -// verified by an inclusion proof. -// TODO(mhutchinson): maybe this should return the Input Log Checkpoint that was committed to in -// the Output Log leaf? -func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, error) { - kh := sha256.Sum256([]byte(key)) - resp, err := c.lookupUnverified(ctx, kh) - if err != nil { - return nil, fmt.Errorf("lookup for %q failed: %v", *lookup, err) - } - - // Currently the response contains the RFC6962 style response type; leaf, proof, etc. - // What if we flip this all around, and the OutputLog part of the response - // only returns an index into the output log, and the client has to look up - // that leaf, checkpoint, and generate inclusion proof? - - cp, _, _, err := log.ParseCheckpoint(resp.OutputLogCP, c.outV.Name(), c.outV) + c, err := client.NewVIndexClient(*vindexBaseURL, outV) if err != nil { - return nil, fmt.Errorf("failed to parse output log checkpoint: %v", err) - } - outLeafHash := rfc6962.DefaultHasher.HashLeaf(resp.OutputLogLeaf) - olp := make([][]byte, len(resp.OutputLogProof)) - for i := range olp { - olp[i] = resp.OutputLogProof[i][:] - } - oli := cp.Size - 1 // TODO(mhutchinson): include this in the response? - if err := proof.VerifyInclusion(rfc6962.DefaultHasher, oli, cp.Size, outLeafHash[:], olp, cp.Hash); err != nil { - return nil, fmt.Errorf("failed to verify inclusion in output log: %v", err) - } - var mapRoot []byte - if idx := bytes.Index(resp.OutputLogLeaf, []byte{'\n', '\n'}); idx < 0 { - return nil, fmt.Errorf("failed to parse output log leaf: %q", resp.OutputLogLeaf) - } else { - mapRoot = resp.OutputLogLeaf[:idx] - mapRoot, err = hex.AppendDecode(nil, mapRoot) - if err != nil { - return nil, fmt.Errorf("failed to decode map root: %v", err) - } - } - - idxLeafHash := sha256.New() - for _, idx := range resp.IndexValue { - if err := binary.Write(idxLeafHash, binary.BigEndian, idx); err != nil { - return nil, fmt.Errorf("failed to calculate leaf hash for indices: %v", err) - } - } - vindexLeafHash := idxLeafHash.Sum(nil) - - pns := make([]prefix.ProofNode, len(resp.IndexProof)) - for i, p := range resp.IndexProof { - label, err := prefix.NewLabel(p.LabelBitLen, p.LabelPath) - if err != nil { - return nil, fmt.Errorf("failed to create label: %v", err) - } - pns[i] = prefix.ProofNode{ - Label: label, - Hash: p.Hash, - } + klog.Exitf("failed to construct VIndex Client: %v", err) } - - if len(resp.IndexValue) > 0 { - if err := prefix.VerifyMembershipProof(sha256.Sum256, kh, [32]byte(vindexLeafHash), pns, [32]byte(mapRoot)); err != nil { - return nil, fmt.Errorf("failed to verify membership: %v", err) - } - } else { - if err := prefix.VerifyNonMembershipProof(sha256.Sum256, kh, pns, [32]byte(mapRoot)); err != nil { - return nil, fmt.Errorf("failed to verify non-membership: %v", err) - } - } - - return resp.IndexValue, nil + return c } -func (c VIndexClient) lookupUnverified(ctx context.Context, kh [sha256.Size]byte) (api.LookupResponse, error) { - var lookupResp api.LookupResponse - - // For now, keys are stored under the hash of the key - u := c.lookupURL.JoinPath(hex.EncodeToString(kh[:])) - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return lookupResp, fmt.Errorf("failed to create request: %v", err) - } - - klog.Infof("Making request to %q", u.String()) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return lookupResp, fmt.Errorf("failed to get URL %q: %v", u, err) - } - defer func() { - _ = resp.Body.Close() - }() - - if resp.StatusCode != http.StatusOK { - return lookupResp, fmt.Errorf("got non-200 status code: %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return lookupResp, fmt.Errorf("failed to read response body: %v", err) - } - - if err := json.Unmarshal(body, &lookupResp); err != nil { - return lookupResp, fmt.Errorf("failed to unmarshal response: %v", err) - } - return lookupResp, nil -} - -func NewLeafReaderFromFlags(ctx context.Context) (*LeafReader, error) { - inV, err := note.NewVerifier(*inLogPubKey) - if err != nil { - klog.Exitf("failed to construct input log verifier: %v", err) - } - +func newInputLogClientFromFlags() *client.InputLogClient { if *inLogBaseURL == "" { klog.Exit("in_log_base_url flag must be provided") } - u, err := url.Parse(*inLogBaseURL) - if err != nil { - return nil, fmt.Errorf("failed to parse URL: %v", err) - } - c, err := client.NewHTTPFetcher(u, http.DefaultClient) - if err != nil { - return nil, fmt.Errorf("failed to create HTTP fetcher for %q: %v", u, err) + if *inLogPubKey == "" { + klog.Exitf("in_log_pub_key must be provided") } - - cpRaw, err := c.ReadCheckpoint(ctx) + v, err := note.NewVerifier(*inLogPubKey) if err != nil { - return nil, fmt.Errorf("failed to read checkpoint from input log: %v", err) - } - cp, _, _, err := log.ParseCheckpoint(cpRaw, inV.Name(), inV) - if err != nil { - klog.Warning(string(cpRaw)) - return nil, fmt.Errorf("failed to parse input log checkpoint: %v", err) - } - lr := &LeafReader{ - f: c.ReadEntryBundle, - cp: cp, - } - return lr, nil -} - -// LeafReader reads leaves from the tree. -// This class is not thread safe. -type LeafReader struct { - f client.EntryBundleFetcherFunc - cp *log.Checkpoint - c leafBundleCache -} - -// getLeaf fetches the raw contents committed to at a given leaf index. -func (r *LeafReader) getLeaf(ctx context.Context, i uint64) ([]byte, error) { - if i >= r.cp.Size { - return nil, fmt.Errorf("requested leaf %d >= log size %d", i, r.cp.Size) - } - if cached := r.c.get(i); cached != nil { - klog.V(2).Infof("Using cached result for index %d", i) - return cached, nil + klog.Exitf("failed to construct output log verifier: %v", err) } - - // TODO(mhutchinson): Check the inclusion proof of a fetched bundle - bundle, err := client.GetEntryBundle(ctx, r.f, i/layout.EntryBundleWidth, r.cp.Size) + c, err := client.NewInputLogClient(*inLogBaseURL, v, http.DefaultClient) if err != nil { - return nil, fmt.Errorf("failed to get entry bundle: %v", err) - } - ti := i % layout.EntryBundleWidth - r.c = leafBundleCache{ - start: i - ti, - leaves: bundle.Entries, + klog.Exitf("failed to construct VIndex Client: %v", err) } - return r.c.leaves[ti], nil -} - -// leafBundleCache stores the results of the last fetched tile. Assuming that the client -// accesses leaves in order, then this avoids fetching the same bundle multiple times if -// multiple leaves are in the same bundle. -type leafBundleCache struct { - start uint64 - leaves [][]byte -} - -func (tc leafBundleCache) get(i uint64) []byte { - end := tc.start + uint64(len(tc.leaves)) - if i >= tc.start && i < end { - leaf := tc.leaves[i-tc.start] - return leaf - } - return nil + return c } diff --git a/vindex/cmd/logandmap/README.md b/vindex/cmd/logandmap/README.md index bc2dcac..2d16a9a 100644 --- a/vindex/cmd/logandmap/README.md +++ b/vindex/cmd/logandmap/README.md @@ -40,7 +40,7 @@ INPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/inputlog+bd6268fb+ATPZW5UsUYHJo24l Running the above will run a web server hosting the following URLs: - `/inputlog/` - the [tlog-tiles][] base URL for the input log - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) - - `/outputlog/` - TODO(mhutchinson): this is where the output log will be hosted + - `/outputlog/` - the [tlog-tiles][] base URL for the output log The input log has entries for packages in the set {`foo`, `bar`, `baz`, `splat`}. To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): diff --git a/vindex/cmd/logandmap/main.go b/vindex/cmd/logandmap/main.go index dd6962a..41340da 100644 --- a/vindex/cmd/logandmap/main.go +++ b/vindex/cmd/logandmap/main.go @@ -75,6 +75,7 @@ type LogEntry struct { } func run(ctx context.Context) error { + // Set up storage for the input log, index, and output log. if *storageDir == "" { return errors.New("storage_dir must be set") } @@ -92,6 +93,8 @@ func run(ctx context.Context) error { return fmt.Errorf("failed to create vindex directory: %v", err) } + // Create the input log, output log, and verifiable index. + // The input log is continuously getting new leaves written to it. inputLog, inputCloser := inputLogOrDie(ctx, inputLogDir) defer inputCloser() @@ -103,10 +106,10 @@ func run(ctx context.Context) error { return fmt.Errorf("failed to create vindex: %v", err) } - // Keeps the map synced with the latest published log state. + // Keeps the map synced with the latest published input log state. go maintainMap(ctx, vi) - // Run a web server to handle queries over the verifiable index. + // Run a web server to serve the input log, index, and output log. go runWebServer(vi, inputLogDir, outputLogDir) <-ctx.Done() return nil diff --git a/vindex/map.go b/vindex/map.go index 446301a..dca524e 100644 --- a/vindex/map.go +++ b/vindex/map.go @@ -24,7 +24,6 @@ import ( "crypto/sha256" "encoding/base64" "encoding/binary" - "encoding/hex" "errors" "fmt" "io" @@ -193,11 +192,6 @@ func (m *inputLogMapper) close() error { // syncFromInputLog reads the latest checkpoint from the input log, and ensures that the WAL // contains a corresponding entry for every index committed to by that checkpoint. -// -// TODO(mhutchinson): this doesn't perform any validation on the input log to check the -// leaves correspond to the checkpoint root hash. This was reasonable while it was based on the -// cloneDB, which performed this validation. Implementing this will require the index to store some -// state alongside the WAL which contains a compact range of its current progress. func (m *inputLogMapper) syncFromInputLog(ctx context.Context) error { rawCp, err := m.inputLog.Checkpoint(ctx) if err != nil { @@ -331,7 +325,6 @@ func (b *VerifiableIndex) Close() error { } // Lookup returns the values stored for the given key. -// TODO(mhutchinson): This needs to return verifiable stuff func (b *VerifiableIndex) Lookup(ctx context.Context, key [sha256.Size]byte) (api.LookupResponse, error) { // Scope the lock to be as minimal as possible lookupLocked := func(key [sha256.Size]byte) []uint64 { @@ -361,25 +354,13 @@ func (b *VerifiableIndex) Lookup(ctx context.Context, key [sha256.Size]byte) (ap } // Parse the output log entry to get the input log tree size that the vindex was built from. - var size uint64 - if startPos := bytes.Index(data, []byte{'\n', '\n'}); startPos < 0 { - return result, fmt.Errorf("output leaf does not have expected format: %s", data) - } else { - working := data[2+startPos:] - if startPos = bytes.Index(working, []byte{'\n'}); startPos < 0 { - return result, fmt.Errorf("output leaf does not have expected format: %s", data) - } - working = working[1+startPos:] - endPos := bytes.Index(working, []byte{'\n'}) - if endPos < 0 { - return result, fmt.Errorf("output leaf does not have expected format: %s", data) - } - working = working[:endPos] - sizeStr := string(working) - size, err = strconv.ParseUint(sizeStr, 10, 64) - if err != nil { - return result, fmt.Errorf("output leaf does not have expected format (found @ %d: %q):\n%s", startPos, sizeStr, data) - } + _, inCp, err := UnmarshalLeaf(data) + if err != nil { + return result, fmt.Errorf("failed to unmarshal output leaf: %v", err) + } + _, size, _, err := checkpointUnsafe(inCp) + if err != nil { + return result, fmt.Errorf("failed to unmarshal input log checkpoint from output leaf: %v", err) } result.OutputLogLeaf = data @@ -448,10 +429,7 @@ func (b *VerifiableIndex) publish(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to load vindex root: %v", err) } - leaf := append(hex.AppendEncode(nil, rootNode.Hash[:]), '\n', '\n') - leaf = append(leaf, inCp...) - - outIdx, rawCp, err := b.outputLog.Append(ctx, leaf) + outIdx, rawCp, err := b.outputLog.Append(ctx, MarshalLeaf(rootNode.Hash, inCp)) if err != nil { return fmt.Errorf("failed to append to output log: %v", err) } @@ -474,10 +452,15 @@ func (b *VerifiableIndex) publish(ctx context.Context) error { // buildMap reads from the WAL until the file has been consumed and the map has been // built up the provided size. +// TODO(mhutchinson): tighten the semantics here. What is the provided size? +// It does double duty: rebuilding the log to a previous size (output log): this doesn't +// need to update the OL, but normal usage should in the mutex as described below. func (b *VerifiableIndex) buildMap(ctx context.Context) error { startWal := time.Now() - updatedKeys := make(map[[sha256.Size]byte]bool) // Allows us to efficiently update vindex after first init + updatedKeys := make(map[[sha256.Size]byte]struct{}) // Allows us to efficiently update vindex after first init + // Load the last input log checkpoint we synced to, verified, and flushed the mapped + // entries into the WAL. cpRaw, closer, err := b.db.Get([]byte(dbLatestCheckpointKey)) if err != nil { if err == pebble.ErrNotFound { @@ -523,7 +506,7 @@ func (b *VerifiableIndex) buildMap(ctx context.Context) error { idxes := b.data[h] idxes = append(idxes, idx) b.data[h] = idxes - updatedKeys[h] = true + updatedKeys[h] = struct{}{} } }() } diff --git a/vindex/outputlog.go b/vindex/outputlog.go index 47b0471..e2bb766 100644 --- a/vindex/outputlog.go +++ b/vindex/outputlog.go @@ -17,6 +17,7 @@ package vindex import ( "context" "crypto/sha256" + "encoding/hex" "errors" "fmt" "time" @@ -111,3 +112,26 @@ func (l posixOutputLog) Lookup(ctx context.Context, idx, size uint64) ([]byte, [ } return data, proofRes, nil } + +// MarshalLeaf creates the leaf to be committed to the Output Log given the root hash +// of the verifiable index, and the checkpoint from the Input Log. +func MarshalLeaf(vindexRootHash [sha256.Size]byte, inLogCp []byte) []byte { + m := append(hex.AppendEncode(nil, vindexRootHash[:]), '\n') + m = append(m, inLogCp...) + return m +} + +// UnmarshalLeaf returns the root hash of the Verifiable Index and the checkpoint from the +// Input Log by unmarshalling a leaf from the Output Log, previously marshalled with +// MarshalLeaf. +func UnmarshalLeaf(leaf []byte) ([sha256.Size]byte, []byte, error) { + split := hex.EncodedLen(sha256.Size) + if split > len(leaf) { + return [sha256.Size]byte{}, nil, fmt.Errorf("failed to parse output log leaf: %q", leaf) + } + mapRoot, err := hex.AppendDecode(nil, leaf[:split]) + if err != nil { + return [sha256.Size]byte{}, nil, fmt.Errorf("failed to decode map root: %v", err) + } + return [32]byte(mapRoot), leaf[split+1:], nil +} diff --git a/vindex/outputlog_test.go b/vindex/outputlog_test.go index 71057de..1c3d802 100644 --- a/vindex/outputlog_test.go +++ b/vindex/outputlog_test.go @@ -17,6 +17,8 @@ package vindex import ( + "bytes" + "crypto/sha256" "os" "testing" @@ -106,3 +108,21 @@ func TestOutputLog_Lookup(t *testing.T) { }) } } + +func TestOutpuLogLeafRoundtrip(t *testing.T) { + inH := sha256.Sum256([]byte("test123")) + inCp := []byte("example.com/test\n123\ndeadbeef") + + leaf := MarshalLeaf(inH, inCp) + + outH, outCp, err := UnmarshalLeaf(leaf) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(outCp, inCp) { + t.Errorf("expected %x but got %x", inCp, outCp) + } + if !bytes.Equal(inH[:], outH[:]) { + t.Errorf("expected %x but got %x", inH, outH) + } +}