From feaf722dea6c50dbf6000839067118f07c3bfe2b Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Thu, 11 Sep 2025 11:35:36 +0000 Subject: [PATCH] Clarify and test concurrency Test concurrency of VIndex. Also verify response from index using client code. --- vindex/client/client.go | 90 ++++++++------- vindex/cmd/logandmap/main.go | 4 +- vindex/map.go | 15 +-- vindex/map_test.go | 213 ++++++++++++++++++++++++++++++++--- vindex/outputlog_test.go | 9 +- vindex/wal_test.go | 6 + 6 files changed, 264 insertions(+), 73 deletions(-) diff --git a/vindex/client/client.go b/vindex/client/client.go index 80a3b78..eb632a3 100644 --- a/vindex/client/client.go +++ b/vindex/client/client.go @@ -39,52 +39,13 @@ import ( "k8s.io/klog/v2" ) -// NewVIndexClient returns a client that can perform verified lookups into the index at the -// given base URL, using the supplied verifier to check checkpoint signatures on the output -// log. -func NewVIndexClient(vindexUrl string, outV note.Verifier) (*VIndexClient, error) { - viu, err := url.Parse(vindexUrl) - if err != nil { - return nil, fmt.Errorf("failed to parse URL: %v", err) - } - lookupURL := viu.JoinPath(api.PathLookup) - - return &VIndexClient{ - lookupURL: lookupURL, - outV: outV, - }, nil -} - -// VIndexClient allows verified lookups into a verifiable index. -type VIndexClient struct { - lookupURL *url.URL - outV note.Verifier -} - -// Lookup returns all indices, in ascending order, where the given key appears in the Input Log. -// This will be verified before being returned from this method, so a caller can be assured that -// any results (including the empty slice, i.e. non-presence) were found in the verifiable index, -// and committed to by the output log. -// On success, this also returns the Checkpoint for the Input Log that was relied upon by the -// verifiable index. This may be used by the caller when constructing inclusion proofs when -// dereferencing any pointers returned. -// -// Note that it is up to the caller to ensure that any leaves looked up in the Input Log are -// verified by an inclusion proof. The checkpoint returned by this method can be used. -// The easiest way to do this is to use the InputLogClient. -func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, []byte, error) { - kh := sha256.Sum256([]byte(key)) - resp, err := c.lookupUnverified(ctx, kh) - if err != nil { - return nil, nil, fmt.Errorf("lookup failed: %v", err) - } - +func VerifyLookupResponse(keyHash [sha256.Size]byte, resp api.LookupResponse, outV note.Verifier) ([]uint64, []byte, error) { // Currently the response contains the RFC6962 style response type; leaf, proof, etc. // What if we flip this all around, and the OutputLog part of the response // only returns an index into the output log, and the client has to look up // that leaf, checkpoint, and generate inclusion proof? - cp, _, _, err := log.ParseCheckpoint(resp.OutputLogCP, c.outV.Name(), c.outV) + cp, _, _, err := log.ParseCheckpoint(resp.OutputLogCP, outV.Name(), outV) if err != nil { return nil, nil, fmt.Errorf("failed to parse output log checkpoint: %v", err) } @@ -124,11 +85,11 @@ func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, []byte, } if len(resp.IndexValue) > 0 { - if err := prefix.VerifyMembershipProof(sha256.Sum256, kh, [32]byte(vindexLeafHash), pns, mapRoot); err != nil { + if err := prefix.VerifyMembershipProof(sha256.Sum256, keyHash, [32]byte(vindexLeafHash), pns, mapRoot); err != nil { return nil, nil, fmt.Errorf("failed to verify membership: %v", err) } } else { - if err := prefix.VerifyNonMembershipProof(sha256.Sum256, kh, pns, mapRoot); err != nil { + if err := prefix.VerifyNonMembershipProof(sha256.Sum256, keyHash, pns, mapRoot); err != nil { return nil, nil, fmt.Errorf("failed to verify non-membership: %v", err) } } @@ -136,6 +97,49 @@ func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, []byte, return resp.IndexValue, inCp, nil } +// NewVIndexClient returns a client that can perform verified lookups into the index at the +// given base URL, using the supplied verifier to check checkpoint signatures on the output +// log. +func NewVIndexClient(vindexUrl string, outV note.Verifier) (*VIndexClient, error) { + viu, err := url.Parse(vindexUrl) + if err != nil { + return nil, fmt.Errorf("failed to parse URL: %v", err) + } + lookupURL := viu.JoinPath(api.PathLookup) + + return &VIndexClient{ + lookupURL: lookupURL, + outV: outV, + }, nil +} + +// VIndexClient allows verified lookups into a verifiable index. +type VIndexClient struct { + lookupURL *url.URL + outV note.Verifier +} + +// Lookup returns all indices, in ascending order, where the given key appears in the Input Log. +// This will be verified before being returned from this method, so a caller can be assured that +// any results (including the empty slice, i.e. non-presence) were found in the verifiable index, +// and committed to by the output log. +// On success, this also returns the Checkpoint for the Input Log that was relied upon by the +// verifiable index. This may be used by the caller when constructing inclusion proofs when +// dereferencing any pointers returned. +// +// Note that it is up to the caller to ensure that any leaves looked up in the Input Log are +// verified by an inclusion proof. The checkpoint returned by this method can be used. +// The easiest way to do this is to use the InputLogClient. +func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, []byte, error) { + kh := sha256.Sum256([]byte(key)) + resp, err := c.lookupUnverified(ctx, kh) + if err != nil { + return nil, nil, fmt.Errorf("lookup failed: %v", err) + } + + return VerifyLookupResponse(kh, resp, c.outV) +} + func (c VIndexClient) lookupUnverified(ctx context.Context, kh [sha256.Size]byte) (api.LookupResponse, error) { var lookupResp api.LookupResponse diff --git a/vindex/cmd/logandmap/main.go b/vindex/cmd/logandmap/main.go index ae5da91..ec6ebbf 100644 --- a/vindex/cmd/logandmap/main.go +++ b/vindex/cmd/logandmap/main.go @@ -143,7 +143,9 @@ func inputLogOrDie(ctx context.Context, inputLogDir string) (log logReaderSource go submitEntries(ctx, inputAppender) return inputLog, func() { - _ = inputShutdown(ctx) + if err := inputShutdown(ctx); err != nil { + klog.Warningf("Error shutting down Input Log appender: %v", err) + } } } diff --git a/vindex/map.go b/vindex/map.go index 539eccc..18a5d64 100644 --- a/vindex/map.go +++ b/vindex/map.go @@ -365,10 +365,12 @@ func (b *VerifiableIndex) Close() error { // Lookup returns the values stored for the given key. func (b *VerifiableIndex) Lookup(ctx context.Context, key [sha256.Size]byte) (api.LookupResponse, error) { // Scope the lock to be as minimal as possible - lookupLocked := func(key [sha256.Size]byte) []uint64 { + // This looks up the indices from the in-memory map, and the proof from the vindex. + lookupLocked := func(key [sha256.Size]byte) (bool, []prefix.ProofNode, []uint64, error) { b.indexMu.RLock() defer b.indexMu.RUnlock() - return b.data[key] + found, viProof, err := b.vindex.Lookup(ctx, key) + return found, viProof, b.data[key], err } result := api.LookupResponse{} @@ -404,7 +406,10 @@ func (b *VerifiableIndex) Lookup(ctx context.Context, key [sha256.Size]byte) (ap result.OutputLogLeaf = data result.OutputLogProof = proof - allIndices := lookupLocked(key) + found, viProof, allIndices, err := lookupLocked(key) + if err != nil { + return result, fmt.Errorf("failed to get inclusion proof from vindex: %v", err) + } cutoff := slices.IndexFunc(allIndices, func(idx uint64) bool { return idx >= size @@ -415,10 +420,6 @@ func (b *VerifiableIndex) Lookup(ctx context.Context, key [sha256.Size]byte) (ap } result.IndexValue = allIndices - found, viProof, err := b.vindex.Lookup(ctx, key) - if err != nil { - return result, fmt.Errorf("failed to get inclusion proof from vindex: %v", err) - } if expectFound := len(allIndices) > 0; expectFound != found { return result, fmt.Errorf("found = %t, but expected %t (number of indices: %d)", found, expectFound, len(allIndices)) } diff --git a/vindex/map_test.go b/vindex/map_test.go index 39cf6dc..3f603e4 100644 --- a/vindex/map_test.go +++ b/vindex/map_test.go @@ -14,24 +14,30 @@ // vindex contains a prototype of an in-memory verifiable index. // This version uses the clone tool DB as the log source. -package vindex +package vindex_test import ( "bytes" "context" "crypto/sha256" - "encoding/hex" + "fmt" "iter" "os" "path" + "sync" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/transparency-dev/formats/log" fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/vindex" + "github.com/transparency-dev/incubator/vindex/api" + "github.com/transparency-dev/incubator/vindex/client" "github.com/transparency-dev/merkle/rfc6962" "github.com/transparency-dev/merkle/testonly" "golang.org/x/mod/sumdb/note" + "golang.org/x/sync/errgroup" ) const ( @@ -77,12 +83,12 @@ func TestVerifiableIndex(t *testing.T) { } old := path.Join(f.Name(), "outputlog") - outputLog, closer, err := NewOutputLog(ctx, old, s, v) + outputLog, closer, err := vindex.NewOutputLog(ctx, old, s, v) if err != nil { t.Fatal(err) } defer closer() - vi, err := NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, f.Name(), Options{}) + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, f.Name(), vindex.Options{}) if err != nil { t.Fatal(err) } @@ -91,41 +97,213 @@ func TestVerifiableIndex(t *testing.T) { t.Fatal(err) } - resp, err := vi.Lookup(t.Context(), sha256.Sum256([]byte("foo"))) + kh := sha256.Sum256([]byte("foo")) + resp, err := vi.Lookup(t.Context(), kh) if err != nil { t.Fatal(err) } - if got, want := resp.IndexValue, []uint64{0, 3}; !cmp.Equal(got, want) { + indices, _, err := client.VerifyLookupResponse(kh, resp, v) + if err != nil { + t.Fatalf("failed to verify vindex response: %v", err) + } + if got, want := indices, []uint64{0, 3}; !cmp.Equal(got, want) { t.Errorf("expected %v but got %v", want, got) } - resp, err = vi.Lookup(t.Context(), sha256.Sum256([]byte("bar"))) + kh = sha256.Sum256([]byte("bar")) + resp, err = vi.Lookup(t.Context(), kh) if err != nil { t.Fatal(err) } - if got, want := resp.IndexValue, []uint64{1, 2}; !cmp.Equal(got, want) { + indices, _, err = client.VerifyLookupResponse(kh, resp, v) + if err != nil { + t.Fatalf("failed to verify vindex response: %v", err) + } + if got, want := indices, []uint64{1, 2}; !cmp.Equal(got, want) { t.Errorf("expected %v but got %v", want, got) } - resp, err = vi.Lookup(t.Context(), sha256.Sum256([]byte("banana"))) + kh = sha256.Sum256([]byte("banana")) + resp, err = vi.Lookup(t.Context(), kh) if err != nil { t.Fatal(err) } - if resp.IndexValue != nil { + indices, _, err = client.VerifyLookupResponse(kh, resp, v) + if err != nil { + t.Fatalf("failed to verify vindex response: %v", err) + } + if indices != nil { t.Errorf("expected no results but got %+v", resp.IndexValue) } } +func TestVerifiableIndex_concurrency(t *testing.T) { + testCases := []struct { + desc string + persist bool + }{ + { + desc: "in memory", + persist: false, + }, + { + desc: "on disk", + persist: true, + }, + } + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + s, v, err := fnote.NewEd25519SignerVerifier(skey) + if err != nil { + t.Fatal(err) + } + inputLog := &inMemoryTreeSource{ + t: testonly.New(rfc6962.DefaultHasher), + leaves: make([][]byte, 0), + s: s, + v: v, + } + for _, str := range []string{"foo: 2", "bar: 5", "bar: 10", "foo: 8"} { + inputLog.Append(str) + } + + mapFn := func(leaf []byte) [][sha256.Size]byte { + key, _, found := bytes.Cut(leaf, []byte(":")) + if !found { + panic("colon not found") + } + return [][sha256.Size]byte{sha256.Sum256(key)} + } + f, err := os.CreateTemp("", "vindexTestDir") + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + if err := os.Remove(f.Name()); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(f.Name(), 0o755); err != nil { + t.Fatal(err) + } + + old := path.Join(f.Name(), "outputlog") + outputLog, closer, err := vindex.NewOutputLog(ctx, old, s, v) + if err != nil { + t.Fatal(err) + } + defer closer() + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, f.Name(), vindex.Options{PersistIndex: tC.persist}) + if err != nil { + t.Fatal(err) + } + if err := vi.Update(ctx); err != nil { + t.Fatal(err) + } + + var eg errgroup.Group + + // Constantly add entries to the input log. + eg.Go(func() error { + i := 101 + + for { + inputLog.Append(fmt.Sprintf("key%d: %d", i, i)) + i++ + select { + case <-ctx.Done(): + return nil + case <-time.After(2 * time.Millisecond): + } + } + }) + + // Periodically update the map from the input log. + eg.Go(func() error { + for { + select { + case <-ctx.Done(): + return nil + case <-time.After(100 * time.Millisecond): + if err := vi.Update(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + } + } + }) + + // Regularly perform lookups in the index. + eg.Go(func() error { + var kh [sha256.Size]byte + var resp api.LookupResponse + var indices []uint64 + + kh = sha256.Sum256([]byte("bar")) + resp, err = vi.Lookup(t.Context(), kh) + if err != nil { + return err + } + indices, _, err = client.VerifyLookupResponse(kh, resp, v) + if err != nil { + return fmt.Errorf("failed to verify vindex response: %v", err) + } + if got, want := indices, []uint64{1, 2}; !cmp.Equal(got, want) { + return fmt.Errorf("expected %v but got %v", want, got) + } + + kh = sha256.Sum256([]byte("banana")) + resp, err = vi.Lookup(t.Context(), kh) + if err != nil { + return err + } + indices, _, err = client.VerifyLookupResponse(kh, resp, v) + if err != nil { + return fmt.Errorf("failed to verify vindex response: %v", err) + } + if indices != nil { + return fmt.Errorf("expected no results but got %+v", resp.IndexValue) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-time.After(8 * time.Millisecond): + } + } + }) + + <-time.After(4 * time.Second) + cancel() + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + }) + } +} + type inMemoryTreeSource struct { t *testonly.Tree leaves [][]byte s note.Signer v note.Verifier + + mu sync.RWMutex } func (s *inMemoryTreeSource) Checkpoint(ctx context.Context) (checkpoint []byte, err error) { - rootHash := s.t.Hash() - size := uint64(len(s.leaves)) + var rootHash []byte + var size uint64 + rootHash, size = func() ([]byte, uint64) { + s.mu.RLock() + defer s.mu.RUnlock() + return s.t.Hash(), uint64(len(s.leaves)) + }() cp := log.Checkpoint{ Origin: s.s.Name(), @@ -143,7 +321,9 @@ func (s *inMemoryTreeSource) Parse(cpRaw []byte) (*log.Checkpoint, error) { func (s *inMemoryTreeSource) Leaves(ctx context.Context, start, end uint64) iter.Seq2[[]byte, error] { return func(yield func([]byte, error) bool) { - for _, entry := range s.leaves { + s.mu.RLock() + defer s.mu.RUnlock() + for _, entry := range s.leaves[start:end] { if !yield(entry, nil) { return } @@ -153,11 +333,8 @@ func (s *inMemoryTreeSource) Leaves(ctx context.Context, start, end uint64) iter func (s *inMemoryTreeSource) Append(leafStr string) { leaf := []byte(leafStr) + s.mu.Lock() + defer s.mu.Unlock() s.leaves = append(s.leaves, leaf) s.t.Append(rfc6962.DefaultHasher.HashLeaf(leaf)) } - -func mustHashEncode(data string) string { - h := sha256.Sum256([]byte(data)) - return hex.EncodeToString(h[:]) -} diff --git a/vindex/outputlog_test.go b/vindex/outputlog_test.go index 1c3d802..b82a120 100644 --- a/vindex/outputlog_test.go +++ b/vindex/outputlog_test.go @@ -14,7 +14,7 @@ // vindex contains a prototype of an in-memory verifiable index. // This version uses the clone tool DB as the log source. -package vindex +package vindex_test import ( "bytes" @@ -23,6 +23,7 @@ import ( "testing" fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/vindex" "github.com/transparency-dev/merkle/proof" "github.com/transparency-dev/merkle/rfc6962" ) @@ -72,7 +73,7 @@ func TestOutputLog_Lookup(t *testing.T) { _ = os.RemoveAll(dir) }() - log, closer, err := NewOutputLog(t.Context(), dir, s, v) + log, closer, err := vindex.NewOutputLog(t.Context(), dir, s, v) if err != nil { t.Fatal(err) } @@ -113,9 +114,9 @@ func TestOutpuLogLeafRoundtrip(t *testing.T) { inH := sha256.Sum256([]byte("test123")) inCp := []byte("example.com/test\n123\ndeadbeef") - leaf := MarshalLeaf(inH, inCp) + leaf := vindex.MarshalLeaf(inH, inCp) - outH, outCp, err := UnmarshalLeaf(leaf) + outH, outCp, err := vindex.UnmarshalLeaf(leaf) if err != nil { t.Fatal(err) } diff --git a/vindex/wal_test.go b/vindex/wal_test.go index da87d35..06dc83f 100644 --- a/vindex/wal_test.go +++ b/vindex/wal_test.go @@ -19,6 +19,7 @@ package vindex import ( "bytes" "crypto/sha256" + "encoding/hex" "fmt" "io" "os" @@ -314,3 +315,8 @@ func TestUnmarshal(t *testing.T) { }) } } + +func mustHashEncode(data string) string { + h := sha256.Sum256([]byte(data)) + return hex.EncodeToString(h[:]) +}