diff --git a/vindex/api/api.go b/vindex/api/api.go index c905bdb..12b087d 100644 --- a/vindex/api/api.go +++ b/vindex/api/api.go @@ -44,7 +44,6 @@ type LookupResponse struct { // These values represent the lookup operation in the index at the root hash // committed to by OutputLogLeaf. The values contain all indices for the given // key, and the proof binds these values at this key at the index root hash. - IndexKey [sha256.Size]byte `json:"index_key"` IndexValue []uint64 `json:"index_value"` IndexProof [][sha256.Size]byte `json:"index_proof"` } diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index b01a036..2f0ffaa 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -18,8 +18,10 @@ package main import ( + "bytes" "context" "crypto/sha256" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -29,13 +31,18 @@ import ( "net/http" "net/url" + "github.com/transparency-dev/formats/log" "github.com/transparency-dev/incubator/vindex/api" + "github.com/transparency-dev/merkle/proof" + "github.com/transparency-dev/merkle/rfc6962" + "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" ) var ( - baseURL = flag.String("base_url", "", "The base URL of the vindex server.") - lookup = flag.String("lookup", "", "The key to look up in the vindex.") + baseURL = flag.String("base_url", "", "The base URL of the vindex server.") + lookup = flag.String("lookup", "", "The key to look up in the vindex.") + outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") ) func main() { @@ -50,35 +57,15 @@ func run(ctx context.Context) error { c := newVIndexClientFromFlags() if *lookup == "" { - return errors.New("key flag must be provided") + return errors.New("lookup flag must be provided") } - resp, err := c.Lookup(ctx, *lookup) + idxes, err := c.Lookup(ctx, *lookup) if err != nil { - return fmt.Errorf("lookup for %q failed: %v", *lookup, err) + return fmt.Errorf("failed to look up key: %v", err) } - // For now, pretty print the JSON response - jsonResp, err := json.MarshalIndent(resp, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal response: %v", err) - } - fmt.Println(string(jsonResp)) - - // This needs to verify the proofs in the response. - // We can't verify inclusion with the info we currently have. - // We at least need the index of the leaf returned. We could add - // that to the response object, but this is kinda rfc6962. - // What would it look like if the output log is tiled? What should - // the lookup response contain? Can it contain less, and thus put - // more on the client as per tlog-tiles? If so, what stops clients - // requesting old views of the index based on non-recent leaves from - // the output log? - // What if we flip this all around, and the OutputLog part of the response - // only returns an index into the output log, and the client has to look up - // that leaf, checkpoint, and generate inclusion proof? - - // proof.VerifyInclusion() + fmt.Printf("Indices: %v\n", idxes) return nil } @@ -92,16 +79,75 @@ func newVIndexClientFromFlags() VIndexClient { klog.Exitf("failed to parse URL: %v", err) } lookupURL := u.JoinPath(api.PathLookup) + + if *outLogPubKey == "" { + klog.Exitf("out_log_pub_key must be provided") + } + outV, err := note.NewVerifier(*outLogPubKey) + if err != nil { + klog.Exitf("failed to construct output log verifier: %v", err) + } return VIndexClient{ lookupURL: lookupURL, + outV: outV, } } type VIndexClient struct { lookupURL *url.URL + outV note.Verifier +} + +func (c VIndexClient) Lookup(ctx context.Context, key string) ([]uint64, error) { + resp, err := c.lookupUnverified(ctx, key) + if err != nil { + return nil, fmt.Errorf("lookup for %q failed: %v", *lookup, err) + } + + // Currently the response contains the RFC6962 style response type; leaf, proof, etc. + // What if we flip this all around, and the OutputLog part of the response + // only returns an index into the output log, and the client has to look up + // that leaf, checkpoint, and generate inclusion proof? + + cp, _, _, err := log.ParseCheckpoint(resp.OutputLogCP, c.outV.Name(), c.outV) + if err != nil { + return nil, fmt.Errorf("failed to parse output log checkpoint: %v", err) + } + outLeafHash := rfc6962.DefaultHasher.HashLeaf(resp.OutputLogLeaf) + olp := make([][]byte, len(resp.OutputLogProof)) + for i := range olp { + olp[i] = resp.OutputLogProof[i][:] + } + oli := cp.Size - 1 // TODO(mhutchinson): include this in the response? + if err := proof.VerifyInclusion(rfc6962.DefaultHasher, oli, cp.Size, outLeafHash[:], olp, cp.Hash); err != nil { + return nil, fmt.Errorf("failed to verify inclusion in output log: %v", err) + } + var mapRoot []byte + if idx := bytes.Index(resp.OutputLogLeaf, []byte{'\n', '\n'}); idx < 0 { + return nil, fmt.Errorf("failed to parse output log leaf: %q", resp.OutputLogLeaf) + } else { + mapRoot = resp.OutputLogLeaf[:idx] + mapRoot, err = hex.AppendDecode(nil, mapRoot) + if err != nil { + return nil, fmt.Errorf("failed to decode map root: %v", err) + } + } + + idxLeafHash := sha256.New() + for _, idx := range resp.IndexValue { + if err := binary.Write(idxLeafHash, binary.BigEndian, idx); err != nil { + return nil, fmt.Errorf("failed to calculate leaf hash for indices: %v", err) + } + } + vindexLeafHash := idxLeafHash.Sum(nil) + vindexKeyHash := sha256.Sum256([]byte(key)) + // TODO(mhutchinson): verify inclusion in the vindex! + klog.Warningf("TODO: confirm inclusion of leaf hash %x at key location %x with root hash %x", vindexLeafHash, vindexKeyHash, mapRoot) + + return resp.IndexValue, nil } -func (c VIndexClient) Lookup(ctx context.Context, key string) (api.LookupResponse, error) { +func (c VIndexClient) lookupUnverified(ctx context.Context, key string) (api.LookupResponse, error) { var lookupResp api.LookupResponse // For now, keys are stored under the hash of the key diff --git a/vindex/cmd/logandmap/README.md b/vindex/cmd/logandmap/README.md new file mode 100644 index 0000000..4c92d99 --- /dev/null +++ b/vindex/cmd/logandmap/README.md @@ -0,0 +1,62 @@ +## Verifiable Index: Log & Map + +This is a demo of using a [Tessera][] Verifiable Log as the Input Log, with all entries indexed by a [Verifiable Index](../../README.md). + +[tlog-tiles]: https://c2sp.org/tlog-tiles +[Tessera]: https://github.com/transparency-dev/tessera + +The entries in the Input Log loosely represent Binary/Artifact Registry entries, committing to a triple of `{module name, module version, artifact hash}`: + +``` +{ + "module": "bar", + "version": "2025-08-07T10:41:56.527888424Z", + "hash": "vsOru/9zZqrLjamAgzvQCaSvpMmF9jy+r75HpMvncZc=" +} +``` + +This pattern is very common: committing that a module at a given version has a particular hash. +This hash could represent the git commit fingerprint the release was tagged from, the hash of a compiled binary, etc. +See [Transparency.dev: Add tamper checking to a package manager](https://transparency.dev/application/add-tamper-checking-to-a-package-manager/) for more background on this pattern. + +This Input Log is processed, with each entry being indexed solely on the `module`, +i.e. the key that is put into the map has the following value: + +```go + sha256.Sum256([]byte(entry.Module)) +``` + +This allows the owner of a given module to look up the modules they are responsible for, and verifiably +find the index of all entries in the Input Log for their modules. + +## Running + +The Input Log, Verifiable Index, and Output Log are all managed by a single binary: + +```shell +INPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/inputlog+bd6268fb+ATPZW5UsUYHJo24lwgK1ykm9VafhyUtUxX5evV4ZIokY OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+example.com/outputlog+07392c46+ATPJ4crkyUbPeaRffN/4NUof3KV0pQznVIPGOQm3SDEJ go run ./vindex/cmd/logandmap --storage_dir ~/logandmap/ +``` + +Running the above will run a web server hosting the following URLs: + - `/inputlog/` - the [tlog-tiles][] base URL for the input log + - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) + - `/outputlog/` - TODO(mhutchinson): this is where the output log will be hosted + +The input log has entries for packages in the set {`foo`, `bar`, `baz`, `splat`}. +To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): + +```shell +# To inspect the Input Log +go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/inputlog/ --custom_log_vkey=example.com/inputlog+bd6268fb+AWdGkrHKBm+pOubTrcBTV8JMDLFlF1Y8WUH1nrtLNXDr + +# To inspect the Output Log +go run github.com/mhutchinson/woodpecker@main --custom_log_type=tiles --custom_log_url=http://localhost:8088/outputlog/ --custom_log_vkey=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 +``` + +Use left/right cursor to browse, and `q` to quit. + +This log is processed into a verifiable map which can be looked up using the following command: + +```shell +go run ./vindex/cmd/client --base_url http://localhost:8088/vindex/ --out_log_pub_key=example.com/outputlog+07392c46+AWyS8y8ZsRmQnTr6Fr2knaa8+t6CPYFh5Ho3wJEr14B8 --lookup=foo +``` diff --git a/vindex/cmd/logandmap/main.go b/vindex/cmd/logandmap/main.go index 2e71340..dd6962a 100644 --- a/vindex/cmd/logandmap/main.go +++ b/vindex/cmd/logandmap/main.go @@ -255,13 +255,7 @@ func submitEntries(ctx context.Context, appender *tessera.Appender) { } func runWebServer(vi *vindex.VerifiableIndex, inLogDir, outLogDir string) { - web := NewServer(func(h [sha256.Size]byte) ([]uint64, error) { - idxes, size := vi.Lookup(h) - if size == 0 { - return nil, errors.New("index not populated") - } - return idxes, nil - }) + web := NewServer(vi.Lookup) ilfs := http.FileServer(http.Dir(inLogDir)) olfs := http.FileServer(http.Dir(outLogDir)) diff --git a/vindex/cmd/logandmap/web.go b/vindex/cmd/logandmap/web.go index c0efd92..68c8c25 100644 --- a/vindex/cmd/logandmap/web.go +++ b/vindex/cmd/logandmap/web.go @@ -15,6 +15,7 @@ package main import ( + "context" "crypto/sha256" _ "embed" "encoding/hex" @@ -27,14 +28,14 @@ import ( "k8s.io/klog/v2" ) -func NewServer(lookup func([sha256.Size]byte) ([]uint64, error)) Server { +func NewServer(lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error)) Server { return Server{ lookup: lookup, } } type Server struct { - lookup func([sha256.Size]byte) ([]uint64, error) + lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error) } // handleLookup handles GET requests for looking up map entries. @@ -58,15 +59,13 @@ func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { klog.V(2).Infof("Received hash from request: '%s'", h) - idxes, err := s.lookup([sha256.Size]byte(h)) + resp, err := s.lookup(r.Context(), [sha256.Size]byte(h)) if err != nil { http.Error(w, fmt.Sprintf("lookup failed: %v", err), http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - var resp api.LookupResponse - resp.IndexValue = idxes if err := json.NewEncoder(w).Encode(resp); err != nil { klog.Warningf("failed to encode response: %v", err) } diff --git a/vindex/map.go b/vindex/map.go index 73a1e37..6c9df22 100644 --- a/vindex/map.go +++ b/vindex/map.go @@ -25,11 +25,13 @@ import ( "encoding/base64" "encoding/binary" "encoding/hex" + "errors" "fmt" "io" "iter" "os" "path" + "slices" "strconv" "sync" "time" @@ -37,6 +39,7 @@ import ( "filippo.io/torchwood/mpt" "github.com/cockroachdb/pebble" "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/incubator/vindex/api" "github.com/transparency-dev/merkle/compact" "github.com/transparency-dev/merkle/rfc6962" "k8s.io/klog/v2" @@ -78,14 +81,10 @@ type OutputLog interface { Parse(checkpoint []byte) (*log.Checkpoint, error) // Append adds a new leaf and returns the checkpoint that commits to it. Append(ctx context.Context, data []byte) (idx uint64, checkpoint []byte, err error) + // Lookup fetches the data, with a proof, at the given index and tree size. + Lookup(ctx context.Context, idx, size uint64) ([]byte, [][sha256.Size]byte, error) } -// OpenCheckpointFn is a function that parses a checkpoint, validating it, and returns a parsed -// checkpoint. This is expected to be a thin wrapper around log.ParseCheckpoint with the -// validators set up according to the index operator's policy on the number of witnesses -// required. -type OpenCheckpointFn func(cpRaw []byte) (*log.Checkpoint, error) - // NewVerifiableIndex returns an IndexBuilder that pulls entries from the given inputLog, determines // indices for each one using the mapFn, and then writes the entries out to a Write Ahead Log at the given // path. @@ -333,7 +332,7 @@ func (b *VerifiableIndex) Close() error { // Lookup returns the values stored for the given key. // TODO(mhutchinson): This needs to return verifiable stuff -func (b *VerifiableIndex) Lookup(key [sha256.Size]byte) (indices []uint64, size uint64) { +func (b *VerifiableIndex) Lookup(ctx context.Context, key [sha256.Size]byte) (api.LookupResponse, error) { // Scope the lock to be as minimal as possible lookupLocked := func(key [sha256.Size]byte) []uint64 { b.indexMu.RLock() @@ -341,19 +340,66 @@ func (b *VerifiableIndex) Lookup(key [sha256.Size]byte) (indices []uint64, size return b.data[key] } - // TODO(mhutchinson): this should come from the latest map root in the (witnessed) output log. - // This map root, the witnessed output log checkpoint, and all proofs should also be served here. - size = b.servingSize + result := api.LookupResponse{} - allIndices := lookupLocked(key) - for i, idx := range allIndices { - if idx >= size { - // If we have indices past the current size we are serving, drop them. - // Doing this allows us to update b.data with new indices while still serving from it. - return allIndices[:i], size + olcp, err := b.outputLog.Checkpoint(ctx) + if err != nil { + return result, err + } + result.OutputLogCP = olcp + cp, err := b.outputLog.Parse(olcp) + if err != nil { + return result, err + } + if cp.Size == 0 { + return result, errors.New("map is empty") + } + + data, proof, err := b.outputLog.Lookup(ctx, cp.Size-1, cp.Size) + if err != nil { + return result, fmt.Errorf("failed to lookup last leaf in output log: %v", err) + } + + // Parse the output log entry to get the input log tree size that the vindex was built from. + var size uint64 + if startPos := bytes.Index(data, []byte{'\n', '\n'}); startPos < 0 { + return result, fmt.Errorf("output leaf does not have expected format: %s", data) + } else { + working := data[2+startPos:] + if startPos = bytes.Index(working, []byte{'\n'}); startPos < 0 { + return result, fmt.Errorf("output leaf does not have expected format: %s", data) + } + working = working[1+startPos:] + endPos := bytes.Index(working, []byte{'\n'}) + if endPos < 0 { + return result, fmt.Errorf("output leaf does not have expected format: %s", data) + } + working = working[:endPos] + sizeStr := string(working) + size, err = strconv.ParseUint(sizeStr, 10, 64) + if err != nil { + return result, fmt.Errorf("output leaf does not have expected format (found @ %d: %q):\n%s", startPos, sizeStr, data) } } - return allIndices, size + + result.OutputLogLeaf = data + result.OutputLogProof = proof + + allIndices := lookupLocked(key) + + cutoff := slices.IndexFunc(allIndices, func(idx uint64) bool { + return idx >= size + }) + + if cutoff >= 0 { + result.IndexValue = allIndices[:cutoff] + } + result.IndexValue = allIndices + + // TODO(filosottile): Generate proof for the vindex + result.IndexProof = nil + + return result, nil } // Update checks the input log for a new Checkpoint, and ensures that the Verifiable Index diff --git a/vindex/map_test.go b/vindex/map_test.go index e453ffb..b661faa 100644 --- a/vindex/map_test.go +++ b/vindex/map_test.go @@ -40,7 +40,7 @@ const ( ) func TestVerifiableIndex(t *testing.T) { - ctx := context.Background() + ctx := t.Context() s, v, err := fnote.NewEd25519SignerVerifier(skey) if err != nil { t.Fatal(err) @@ -91,28 +91,28 @@ func TestVerifiableIndex(t *testing.T) { t.Fatal(err) } - idxes, size := vi.Lookup(sha256.Sum256([]byte("foo"))) - if size != 4 { - t.Errorf("expected size 4 but got %d", size) + resp, err := vi.Lookup(t.Context(), sha256.Sum256([]byte("foo"))) + if err != nil { + t.Fatal(err) } - if want := []uint64{0, 3}; !cmp.Equal(idxes, want) { - t.Errorf("expected %v but got %v", want, idxes) + if got, want := resp.IndexValue, []uint64{0, 3}; !cmp.Equal(got, want) { + t.Errorf("expected %v but got %v", want, got) } - idxes, size = vi.Lookup(sha256.Sum256([]byte("bar"))) - if size != 4 { - t.Errorf("expected size 4 but got %d", size) + resp, err = vi.Lookup(t.Context(), sha256.Sum256([]byte("bar"))) + if err != nil { + t.Fatal(err) } - if want := []uint64{1, 2}; !cmp.Equal(idxes, want) { - t.Errorf("expected %v but got %v", want, idxes) + if got, want := resp.IndexValue, []uint64{1, 2}; !cmp.Equal(got, want) { + t.Errorf("expected %v but got %v", want, got) } - idxes, size = vi.Lookup(sha256.Sum256([]byte("banana"))) - if size != 4 { - t.Errorf("expected size 4 but got %d", size) + resp, err = vi.Lookup(t.Context(), sha256.Sum256([]byte("banana"))) + if err != nil { + t.Fatal(err) } - if idxes != nil { - t.Errorf("expected no results but got %+v", idxes) + if resp.IndexValue != nil { + t.Errorf("expected no results but got %+v", resp.IndexValue) } } diff --git a/vindex/outputlog.go b/vindex/outputlog.go index 594bbcd..47b0471 100644 --- a/vindex/outputlog.go +++ b/vindex/outputlog.go @@ -16,11 +16,15 @@ package vindex import ( "context" + "crypto/sha256" + "errors" "fmt" "time" "github.com/transparency-dev/formats/log" "github.com/transparency-dev/tessera" + "github.com/transparency-dev/tessera/api" + "github.com/transparency-dev/tessera/client" "github.com/transparency-dev/tessera/storage/posix" "golang.org/x/mod/sumdb/note" ) @@ -34,7 +38,7 @@ func NewOutputLog(ctx context.Context, outputLogDir string, s note.Signer, v not appender, shutdown, reader, err := tessera.NewAppender(ctx, driver, tessera.NewAppendOptions(). WithCheckpointSigner(s). - WithCheckpointInterval(5*time.Second). + WithCheckpointInterval(1*time.Second). WithBatching(1, time.Second)) if err != nil { return nil, nil, fmt.Errorf("failed to get appender: %v", err) @@ -73,3 +77,37 @@ func (l posixOutputLog) Append(ctx context.Context, data []byte) (idx uint64, ch index, cp, err := l.w.Await(ctx, l.a.Add(ctx, tessera.NewEntry(data))) return index.Index, cp, err } + +func (l posixOutputLog) Lookup(ctx context.Context, idx, size uint64) ([]byte, [][sha256.Size]byte, error) { + pb, err := client.NewProofBuilder(ctx, size, l.r.ReadTile) + if err != nil { + return nil, nil, fmt.Errorf("failed to create proof builder: %v", err) + } + proof, err := pb.InclusionProof(ctx, idx) + if err != nil { + return nil, nil, fmt.Errorf("failed to create proof: %v", err) + } + sizeFn := func(_ context.Context) (uint64, error) { + return size, nil + } + + var data []byte + var done bool + for b := range client.EntryBundles(ctx, 1, sizeFn, l.r.ReadEntryBundle, idx, 1) { + if done { + panic(errors.New("got 2 entries, expected 1")) + } + var eb api.EntryBundle + if err := eb.UnmarshalText(b.Data); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal bundle: %v", err) + } + data = eb.Entries[b.RangeInfo.First] + done = true + } + + proofRes := make([][sha256.Size]byte, len(proof)) + for i, p := range proof { + proofRes[i] = [sha256.Size]byte(p) + } + return data, proofRes, nil +} diff --git a/vindex/outputlog_test.go b/vindex/outputlog_test.go new file mode 100644 index 0000000..71057de --- /dev/null +++ b/vindex/outputlog_test.go @@ -0,0 +1,108 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// vindex contains a prototype of an in-memory verifiable index. +// This version uses the clone tool DB as the log source. +package vindex + +import ( + "os" + "testing" + + fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/merkle/proof" + "github.com/transparency-dev/merkle/rfc6962" +) + +func TestOutputLog_Lookup(t *testing.T) { + s, v, err := fnote.NewEd25519SignerVerifier(skey) + if err != nil { + t.Fatal(err) + } + testCases := []struct { + desc string + leaves []string + lookupIdx uint64 + wantErr bool + }{ + { + desc: "single entry log", + leaves: []string{"foo"}, + lookupIdx: 0, + wantErr: false, + }, + { + desc: "two entry log", + leaves: []string{"foo", "bar"}, + lookupIdx: 1, + wantErr: false, + }, + { + desc: "multi entry log: last", + leaves: []string{"foo", "bar", "baz"}, + lookupIdx: 2, + wantErr: false, + }, { + desc: "multi entry log: penultimate", + leaves: []string{"foo", "bar", "baz"}, + lookupIdx: 1, + wantErr: false, + }, + } + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + dir, err := os.MkdirTemp("", "testOutputLog") + if err != nil { + t.Fatal(err) + } + defer func() { + _ = os.RemoveAll(dir) + }() + + log, closer, err := NewOutputLog(t.Context(), dir, s, v) + if err != nil { + t.Fatal(err) + } + defer closer() + + for _, l := range tC.leaves { + if _, _, err := log.Append(t.Context(), []byte(l)); err != nil { + t.Fatal(err) + } + } + rawCp, err := log.Checkpoint(t.Context()) + if err != nil { + t.Fatal(err) + } + cp, err := log.Parse(rawCp) + if err != nil { + t.Fatal(err) + } + + data, incProof, err := log.Lookup(t.Context(), tC.lookupIdx, cp.Size) + if err != nil { + t.Fatal(err) + } + + hash := rfc6962.DefaultHasher.HashLeaf(data) + incProof2 := make([][]byte, len(incProof)) + for i := range incProof { + incProof2[i] = incProof[i][:] + } + if err := proof.VerifyInclusion(rfc6962.DefaultHasher, tC.lookupIdx, cp.Size, hash, incProof2, cp.Hash); err != nil { + t.Fatal(err) + } + }) + } +}