From 6e9cd19f2913c6ca5aaa18e3226af568ebf4661d Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Tue, 23 Sep 2025 09:17:58 +0000 Subject: [PATCH 1/2] Verifiable Index for SumDB This one reads straight from the log (with a slight indirection via the local proxy). This obsoletes the version that reads from the CloneDB in trillian-examples. The server brings up a tlog-tiles proxy for SumDB on the same port it serves the index and output log on. This allows the index client to read from this to dereference the leaf indices returned by the index. --- sumdb/README.md | 2 +- sumdb/cmd/proxy.go | 45 +++++++ sumdb/proxy.go | 39 +++--- vindex/client/client.go | 14 ++- vindex/cmd/client/client.go | 7 +- vindex/cmd/sumdb/README.md | 48 ++++++++ vindex/cmd/sumdb/main.go | 237 ++++++++++++++++++++++++++++++++++++ vindex/cmd/sumdb/web.go | 76 ++++++++++++ vindex/inputlog.go | 97 +++++++++++++++ 9 files changed, 536 insertions(+), 29 deletions(-) create mode 100644 sumdb/cmd/proxy.go create mode 100644 vindex/cmd/sumdb/README.md create mode 100644 vindex/cmd/sumdb/main.go create mode 100644 vindex/cmd/sumdb/web.go create mode 100644 vindex/inputlog.go diff --git a/sumdb/README.md b/sumdb/README.md index 18f3d16..5a77501 100644 --- a/sumdb/README.md +++ b/sumdb/README.md @@ -6,7 +6,7 @@ This allows tooling written for the tlog-tiles API to be used with the SumDB, ev ### Running ```shell -go run ./sumdb/proxy.go --listen=":8089" +go run ./sumdb/cmd/proxy.go --listen=":8089" ``` ### Using diff --git a/sumdb/cmd/proxy.go b/sumdb/cmd/proxy.go new file mode 100644 index 0000000..e58c71a --- /dev/null +++ b/sumdb/cmd/proxy.go @@ -0,0 +1,45 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// sumdb is a command that launches a local proxy that allows clients to query +// using the tlog-tiles API, and retrieve results from SumDB. +package main + +import ( + "flag" + + "net/http" + + "github.com/transparency-dev/incubator/sumdb" + "k8s.io/klog/v2" +) + +var ( + listen = flag.String("listen", ":8089", "Address to set up HTTP server listening on") +) + +const ( + upstreamBase = "https://sum.golang.org" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + proxy := sumdb.NewProxy(sumdb.ProxyOpts{}) + klog.Infof("Proxying tlog-tiles API to %s on %s", upstreamBase, *listen) + if err := http.ListenAndServe(*listen, proxy); err != nil { + klog.Fatalf("ListenAndServe: %v", err) + } +} diff --git a/sumdb/proxy.go b/sumdb/proxy.go index caa3ec4..0fd6253 100644 --- a/sumdb/proxy.go +++ b/sumdb/proxy.go @@ -12,14 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -// sumdb is a command that launches a local proxy that allows clients to query -// using the tlog-tiles API, and retrieve results from SumDB. -package main +// sumdb provides a utility proxy to convert to a tlog-tiles API. +package sumdb import ( "bytes" "encoding/binary" - "flag" "fmt" "net/http" @@ -32,23 +30,24 @@ import ( "k8s.io/klog/v2" ) -var ( - listen = flag.String("listen", ":8089", "Address to set up HTTP server listening on") -) - const ( upstreamBase = "https://sum.golang.org" ) -func main() { - klog.InitFlags(nil) - flag.Parse() +type ProxyOpts struct { + // PathPrefix should be set if the proxy is hosted not at "/". + // Any path beyond this should be set here, so that it can be stripped. + PathPrefix string +} +func NewProxy(opts ProxyOpts) *httputil.ReverseProxy { upstream, err := url.Parse(upstreamBase) if err != nil { klog.Fatalf("Failed to parse upstream URL %q: %v", upstreamBase, err) } + prefix, _ := strings.CutSuffix(opts.PathPrefix, "/") + const tlogEntriesPrefix = "/tile/entries/" const tlogTilePrefix = "/tile/" @@ -57,14 +56,16 @@ func main() { proxy := &httputil.ReverseProxy{ Rewrite: func(r *httputil.ProxyRequest) { r.SetURL(upstream) + inPath := strings.TrimPrefix(r.In.URL.Path, prefix) + klog.V(2).Infof("Request for %s", r.In.URL.Path) - if r.In.URL.Path == "/checkpoint" { + if inPath == "/checkpoint" { r.Out.URL.Path = "/latest" - } else if strings.HasPrefix(r.In.URL.Path, tlogEntriesPrefix) { - o := strings.TrimPrefix(r.In.URL.Path, tlogEntriesPrefix) + } else if strings.HasPrefix(inPath, tlogEntriesPrefix) { + o := strings.TrimPrefix(inPath, tlogEntriesPrefix) r.Out.URL.Path = fmt.Sprintf("%s%s", sumDBTileDataPrefix, o) - } else if strings.HasPrefix(r.In.URL.Path, tlogTilePrefix) { - o := strings.TrimPrefix(r.In.URL.Path, tlogTilePrefix) + } else if strings.HasPrefix(inPath, tlogTilePrefix) { + o := strings.TrimPrefix(inPath, tlogTilePrefix) r.Out.URL.Path = fmt.Sprintf("%s%s", sumDBTilePrefix, o) } }, @@ -91,9 +92,5 @@ func main() { }, } - klog.Infof("Proxying tlog-tiles API to %s on %s", upstreamBase, *listen) - if err := http.ListenAndServe(*listen, proxy); err != nil { - klog.Fatalf("ListenAndServe: %v", err) - } + return proxy } - diff --git a/vindex/client/client.go b/vindex/client/client.go index eb632a3..4e9821d 100644 --- a/vindex/client/client.go +++ b/vindex/client/client.go @@ -179,7 +179,7 @@ func (c VIndexClient) lookupUnverified(ctx context.Context, kh [sha256.Size]byte // NewInputLogClient returns a client that allows pointers returned from the Verifiable Index // to be dereferenced by looking up entries in the Input Log. All operations are verified by this // client, which closes the loop. -func NewInputLogClient(inLogUrl string, inV note.Verifier, hc *http.Client) (*InputLogClient, error) { +func NewInputLogClient(inLogUrl string, origin string, inV note.Verifier, hc *http.Client) (*InputLogClient, error) { u, err := url.Parse(inLogUrl) if err != nil { return nil, fmt.Errorf("failed to parse URL: %v", err) @@ -189,23 +189,25 @@ func NewInputLogClient(inLogUrl string, inV note.Verifier, hc *http.Client) (*In return nil, fmt.Errorf("failed to create HTTP fetcher for %q: %v", u, err) } return &InputLogClient{ - v: inV, - lc: c, + v: inV, + origin: origin, + lc: c, }, nil } // InputLogClient is a client intended to be used by users of the VIndexClient that want // to look up the original leaves from the Input Log. type InputLogClient struct { - v note.Verifier - lc logClient + v note.Verifier + origin string + lc logClient } // Dereference takes pointers returned by the VIndexClient Lookup method, and fetches // the original leaves from the Input Log. The inclusion of any leaves returned will be // verified by constructing inclusion proofs to the checkpoint provided. func (c *InputLogClient) Dereference(ctx context.Context, cpRaw []byte, pointers []uint64) iter.Seq2[InputLogLeaf, error] { - cp, _, _, err := log.ParseCheckpoint(cpRaw, c.v.Name(), c.v) + cp, _, _, err := log.ParseCheckpoint(cpRaw, c.origin, c.v) if err != nil { return func(yield func(InputLogLeaf, error) bool) { yield(InputLogLeaf{}, fmt.Errorf("failed to parse input log checkpoint: %v", err)) diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index 73f7070..152a833 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -36,6 +36,7 @@ var ( lookup = flag.String("lookup", "", "The key to look up in the vindex.") outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint.") + inLogOrigin = flag.String("in_log_origin", "", "Optional: allows the Input Log Origin string to be configured to something other than the public key name.") minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") ) @@ -125,7 +126,11 @@ func newInputLogClientFromFlags() *client.InputLogClient { if err != nil { klog.Exitf("failed to construct output log verifier: %v", err) } - c, err := client.NewInputLogClient(*inLogBaseURL, v, http.DefaultClient) + origin := *inLogOrigin + if len(origin) == 0 { + origin = v.Name() + } + c, err := client.NewInputLogClient(*inLogBaseURL, origin, v, http.DefaultClient) if err != nil { klog.Exitf("failed to construct VIndex Client: %v", err) } diff --git a/vindex/cmd/sumdb/README.md b/vindex/cmd/sumdb/README.md new file mode 100644 index 0000000..48dc11c --- /dev/null +++ b/vindex/cmd/sumdb/README.md @@ -0,0 +1,48 @@ +## Verifiable Index: SumDB + +This is a demo of building a [Verifiable Index](../../README.md) for Go's SumDB. + +The index allows package maintainers to verifiable look up all [non-psuedo](https://pkg.go.dev/golang.org/x/mod@v0.28.0/module#IsPseudoVersion) versions of their module served by the Module Proxy. + +[tlog-tiles]: https://c2sp.org/tlog-tiles +[Tessera]: https://github.com/transparency-dev/tessera + +## Running + +The Verifiable Index and Output Log are managed by a single binary, which can be run using: + +```shell +OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+SumDBIndex+a5ed0e81+AYT6tfHpqGaSoH0gYpM7fhj1tEkM3wwYR/IhtiYh1pnj \ +go run ./vindex/cmd/sumdb \ + --storage_dir ~/vindex-sumdb/ +``` + +Running the above will run a web server hosting the following URLs: + - `/inputlog/` - the [tlog-tiles][] base URL for a proxy of the SumDB API + - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) + - `/outputlog/` - the [tlog-tiles][] base URL for the output log + +To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): + +```shell +# To inspect the Output Log +go run github.com/mhutchinson/woodpecker@main \ + --custom_log_type=tiles \ + --custom_log_url=http://localhost:8088/outputlog/ \ + --custom_log_vkey=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd +``` + +Use left/right cursor to browse, and `q` to quit. + +This log is processed into a verifiable map which can be looked up using the following command: + +```shell +go run ./vindex/cmd/client \ + --vindex_base_url http://localhost:8088/vindex/ \ + --in_log_base_url http://localhost:8088/inputlog/ \ + --out_log_pub_key=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd \ + --in_log_pub_key=sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 \ + --in_log_origin="go.sum database tree" \ + --lookup=github.com/transparency-dev/tessera +``` + diff --git a/vindex/cmd/sumdb/main.go b/vindex/cmd/sumdb/main.go new file mode 100644 index 0000000..b37ceeb --- /dev/null +++ b/vindex/cmd/sumdb/main.go @@ -0,0 +1,237 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// sumdb brings up a verifiable index for the Go SumDB. +// This requires a proxy to be running to bridge to a tlog-tiles API. +// See the README for usage details. +package main + +import ( + "context" + "crypto/sha256" + "errors" + "flag" + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/signal" + "path" + "regexp" + "strings" + "syscall" + "time" + + "github.com/gorilla/mux" + fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/sumdb" + "github.com/transparency-dev/incubator/vindex" + "golang.org/x/mod/module" + "golang.org/x/mod/sumdb/note" + "k8s.io/klog/v2" +) + +var ( + outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Output Log, and allocate space to store the verifiable map persistence.") + listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") +) + +var ( + // Example leaf: + // golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= + // golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= + // + line0RE = regexp.MustCompile(`(.*) (.*) h1:(.*)`) + line1RE = regexp.MustCompile(`(.*) (.*)/go.mod h1:(.*)`) +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx); err != nil { + klog.Exitf("Run failed: %v", err) + } +} + +func run(ctx context.Context) error { + // Set up storage for the input log, index, and output log. + if *storageDir == "" { + return errors.New("storage_dir must be set") + } + inputLogDir := path.Join(*storageDir, "inputlog") + outputLogDir := path.Join(*storageDir, "outputlog") + mapRoot := path.Join(*storageDir, "vindex") + + if err := os.MkdirAll(inputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create input log directory: %v", err) + } + if err := os.MkdirAll(outputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create output log directory: %v", err) + } + if err := os.MkdirAll(mapRoot, 0o755); err != nil { + return fmt.Errorf("failed to create vindex directory: %v", err) + } + + sumV, err := note.NewVerifier("sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8") + if err != nil { + return err + } + sumUrl, err := url.Parse(fmt.Sprintf("http://%s/inputlog/", *listen)) + if err != nil { + return err + } + inputLog, err := vindex.NewTiledInputLog(sumUrl, sumV, vindex.InputLogOpts{ + HttpClient: http.DefaultClient, + Origin: "go.sum database tree", + }) + if err != nil { + return err + } + sumProxy := sumdb.NewProxy(sumdb.ProxyOpts{ + PathPrefix: "/inputlog/", + }) + + outputLog, outputCloser := outputLogOrDie(ctx, outputLogDir) + defer outputCloser() + + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFnOrDie(), outputLog, mapRoot, vindex.Options{}) + if err != nil { + return fmt.Errorf("failed to create vindex: %v", err) + } + + // Run a web server to serve the input log, index, and output log. + go runWebServer(sumProxy, vi, outputLogDir) + + // Keeps the map synced with the latest published input log state. + go maintainMap(ctx, vi) + + <-ctx.Done() + return nil +} + +// outputLogOrDie returns an output log using a POSIX log in the given directory. +func outputLogOrDie(ctx context.Context, outputLogDir string) (log vindex.OutputLog, closer func()) { + s, v := getOutputLogSignerVerifierOrDie() + + l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v) + if err != nil { + klog.Exit(err) + } + return l, c +} + +// maintainMap reads entries from the log and sync them to the vindex. +func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + if err := vi.Update(ctx); err != nil { + klog.Warning(err) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func runWebServer(inLog *httputil.ReverseProxy, vi *vindex.VerifiableIndex, outLogDir string) { + web := NewServer(vi.Lookup) + + olfs := http.FileServer(http.Dir(outLogDir)) + r := mux.NewRouter() + r.PathPrefix("/inputlog/").Handler(inLog) + r.PathPrefix("/outputlog/").Handler(http.StripPrefix("/outputlog/", olfs)) + web.registerHandlers(r) + hServer := &http.Server{ + Addr: *listen, + Handler: r, + } + go func() { + if err := hServer.ListenAndServe(); err != http.ErrServerClosed { + klog.Warning(err) + } + }() + klog.Infof("Started HTTP server listening on %s", *listen) +} + +// Read output log private key from file or environment variable and generate the +// note Signer and Verifier pair for it. +func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { + var privKey string + var err error + if len(*outputLogPrivKeyFile) > 0 { + privKey, err = getKeyFile(*outputLogPrivKeyFile) + if err != nil { + klog.Exitf("Unable to get private key: %v", err) + } + } else { + privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") + if len(privKey) == 0 { + klog.Exit("Supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable") + } + } + s, v, err := fnote.NewEd25519SignerVerifier(privKey) + if err != nil { + klog.Exitf("Failed to get signer/verifier: %v", err) + } + return s, v +} + +func getKeyFile(path string) (string, error) { + k, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + return string(k), nil +} + +func mapFnOrDie() vindex.MapFn { + mapFn := func(data []byte) [][32]byte { + lines := strings.Split(string(data), "\n") + if len(lines) < 2 { + panic(fmt.Errorf("expected 2 lines but got %d", len(lines))) + } + + line0Parts := line0RE.FindStringSubmatch(lines[0]) + line0Module, line0Version := line0Parts[1], line0Parts[2] + + line1Parts := line1RE.FindStringSubmatch(lines[1]) + line1Module, line1Version := line1Parts[1], line1Parts[2] + + if line0Module != line1Module { + klog.Errorf("mismatched module names: (%s, %s)", line0Module, line1Module) + } + if line0Version != line1Version { + klog.Errorf("mismatched version names: (%s, %s)", line0Version, line0Version) + } + if module.IsPseudoVersion(line0Version) { + // Drop any emphemeral builds + return nil + } + + klog.V(2).Infof("MapFn found: Module: %s:\t%s", line0Module, line0Version) + + return [][32]byte{sha256.Sum256([]byte(line0Module))} + } + return mapFn +} diff --git a/vindex/cmd/sumdb/web.go b/vindex/cmd/sumdb/web.go new file mode 100644 index 0000000..68c8c25 --- /dev/null +++ b/vindex/cmd/sumdb/web.go @@ -0,0 +1,76 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + + "github.com/gorilla/mux" + "github.com/transparency-dev/incubator/vindex/api" + "k8s.io/klog/v2" +) + +func NewServer(lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error)) Server { + return Server{ + lookup: lookup, + } +} + +type Server struct { + lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error) +} + +// handleLookup handles GET requests for looking up map entries. +func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + hashStr, ok := vars["hash"] + if !ok { + http.Error(w, "hash parameter not found", http.StatusBadRequest) + return + } + + h, err := hex.DecodeString(hashStr) + if err != nil { + http.Error(w, fmt.Sprintf("invalid hex hash: %v", err), http.StatusBadRequest) + return + } + if l := len(h); l != sha256.Size { + http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", l), http.StatusBadRequest) + return + } + + klog.V(2).Infof("Received hash from request: '%s'", h) + + resp, err := s.lookup(r.Context(), [sha256.Size]byte(h)) + if err != nil { + http.Error(w, fmt.Sprintf("lookup failed: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + klog.Warningf("failed to encode response: %v", err) + } +} + +func (s Server) registerHandlers(r *mux.Router) { + r.HandleFunc("/vindex/lookup/{hash}", s.handleLookup).Methods("GET") +} diff --git a/vindex/inputlog.go b/vindex/inputlog.go new file mode 100644 index 0000000..f6b8b91 --- /dev/null +++ b/vindex/inputlog.go @@ -0,0 +1,97 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vindex + +import ( + "context" + "iter" + "net/http" + "net/url" + + "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/tessera/api" + "github.com/transparency-dev/tessera/client" + "golang.org/x/mod/sumdb/note" +) + +type InputLogOpts struct { + HttpClient *http.Client + Origin string +} + +func NewTiledInputLog(base *url.URL, v note.Verifier, o InputLogOpts) (InputLog, error) { + c := o.HttpClient + if c == nil { + c = http.DefaultClient + } + f, err := client.NewHTTPFetcher(base, c) + if err != nil { + return nil, err + } + return logReaderSource{ + f: f, + v: v, + opts: o, + }, nil +} + +// logReaderSource adapts a tessera.LogReader to a vindex.InputLog. +type logReaderSource struct { + f *client.HTTPFetcher + v note.Verifier + opts InputLogOpts +} + +func (s logReaderSource) Checkpoint(ctx context.Context) (checkpoint []byte, err error) { + return s.f.ReadCheckpoint(ctx) +} + +func (s logReaderSource) Parse(cpRaw []byte) (*log.Checkpoint, error) { + origin := s.opts.Origin + if len(origin) == 0 { + origin = s.v.Name() + } + cp, _, _, err := log.ParseCheckpoint(cpRaw, origin, s.v) + return cp, err +} + +func (s logReaderSource) Leaves(ctx context.Context, start, end uint64) iter.Seq2[[]byte, error] { + tsf := func(ctx context.Context) (uint64, error) { + return end, nil + } + bi := client.EntryBundles(ctx, 2, tsf, s.f.ReadEntryBundle, start, end-start) + unbundleFn := func(bundle []byte) ([][]byte, error) { + eb := &api.EntryBundle{} + if err := eb.UnmarshalText(bundle); err != nil { + return nil, err + } + return eb.Entries, nil + } + + return func(yield func([]byte, error) bool) { + // Unwrap the client.Entry type to return an iterator of []byte only. + for entry, err := range client.Entries(bi, unbundleFn) { + if err != nil { + if !yield(nil, err) { + return + } + continue + } + if !yield(entry.Entry, nil) { + return + } + } + } +} From b78058d1d3976bd98f79aa6c8f105d345dd60a01 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Wed, 24 Sep 2025 10:53:16 +0000 Subject: [PATCH 2/2] Review comments --- vindex/cmd/logandmap/main.go | 8 ++--- vindex/cmd/sumdb/README.md | 2 +- vindex/cmd/sumdb/main.go | 57 +++++++++++++++++------------------- vindex/inputlog.go | 18 ++++++------ 4 files changed, 41 insertions(+), 44 deletions(-) diff --git a/vindex/cmd/logandmap/main.go b/vindex/cmd/logandmap/main.go index ec6ebbf..4276292 100644 --- a/vindex/cmd/logandmap/main.go +++ b/vindex/cmd/logandmap/main.go @@ -50,8 +50,8 @@ import ( ) var ( - inputLogPrivKeyFile = flag.String("input_log_private_key", "", "Location of private key file. If unset, uses the contents of the INPUT_LOG_PRIVATE_KEY environment variable.") - outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + inputLogPrivKeyFile = flag.String("input_log_private_key_path", "", "Location of private key file. If unset, uses the contents of the INPUT_LOG_PRIVATE_KEY environment variable.") + outputLogPrivKeyFile = flag.String("output_log_private_key_path", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Input Log, Output Log, and allocate space to store the verifiable map persistence.") listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") ) @@ -291,7 +291,7 @@ func getInputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { } else { privKey = os.Getenv("INPUT_LOG_PRIVATE_KEY") if len(privKey) == 0 { - klog.Exit("Supply private key file path using --input_log_private_key or set INPUT_LOG_PRIVATE_KEY environment variable") + klog.Exit("Supply private key file path using --input_log_private_key_path or set INPUT_LOG_PRIVATE_KEY environment variable") } } s, v, err := fnote.NewEd25519SignerVerifier(privKey) @@ -314,7 +314,7 @@ func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { } else { privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") if len(privKey) == 0 { - klog.Exit("Supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable") + klog.Exit("Supply private key file path using --output_log_private_key_path or set OUTPUT_LOG_PRIVATE_KEY environment variable") } } s, v, err := fnote.NewEd25519SignerVerifier(privKey) diff --git a/vindex/cmd/sumdb/README.md b/vindex/cmd/sumdb/README.md index 48dc11c..f4ca193 100644 --- a/vindex/cmd/sumdb/README.md +++ b/vindex/cmd/sumdb/README.md @@ -2,7 +2,7 @@ This is a demo of building a [Verifiable Index](../../README.md) for Go's SumDB. -The index allows package maintainers to verifiable look up all [non-psuedo](https://pkg.go.dev/golang.org/x/mod@v0.28.0/module#IsPseudoVersion) versions of their module served by the Module Proxy. +The index allows package maintainers to verifiably look up all [non-pseudo](https://pkg.go.dev/golang.org/x/mod@v0.28.0/module#IsPseudoVersion) versions of their module served by the Module Proxy. [tlog-tiles]: https://c2sp.org/tlog-tiles [Tessera]: https://github.com/transparency-dev/tessera diff --git a/vindex/cmd/sumdb/main.go b/vindex/cmd/sumdb/main.go index b37ceeb..6e4ef87 100644 --- a/vindex/cmd/sumdb/main.go +++ b/vindex/cmd/sumdb/main.go @@ -44,7 +44,7 @@ import ( ) var ( - outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + outputLogPrivKeyFile = flag.String("output_log_private_key_path", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Output Log, and allocate space to store the verifiable map persistence.") listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") ) @@ -111,7 +111,7 @@ func run(ctx context.Context) error { outputLog, outputCloser := outputLogOrDie(ctx, outputLogDir) defer outputCloser() - vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFnOrDie(), outputLog, mapRoot, vindex.Options{}) + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, mapRoot, vindex.Options{}) if err != nil { return fmt.Errorf("failed to create vindex: %v", err) } @@ -132,7 +132,7 @@ func outputLogOrDie(ctx context.Context, outputLogDir string) (log vindex.Output l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v) if err != nil { - klog.Exit(err) + klog.Exitf("Failed to create Output Log: %v", err) } return l, c } @@ -144,7 +144,7 @@ func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { for { if err := vi.Update(ctx); err != nil { - klog.Warning(err) + klog.Warningf("Failed to Update index: %v", err) } select { case <-ctx.Done(): @@ -168,7 +168,7 @@ func runWebServer(inLog *httputil.ReverseProxy, vi *vindex.VerifiableIndex, outL } go func() { if err := hServer.ListenAndServe(); err != http.ErrServerClosed { - klog.Warning(err) + klog.Warningf("Error from HTTP server: %v", err) } }() klog.Infof("Started HTTP server listening on %s", *listen) @@ -187,7 +187,7 @@ func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { } else { privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") if len(privKey) == 0 { - klog.Exit("Supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable") + klog.Exit("Supply private key file path using --output_log_private_key_path or set OUTPUT_LOG_PRIVATE_KEY environment variable") } } s, v, err := fnote.NewEd25519SignerVerifier(privKey) @@ -205,33 +205,30 @@ func getKeyFile(path string) (string, error) { return string(k), nil } -func mapFnOrDie() vindex.MapFn { - mapFn := func(data []byte) [][32]byte { - lines := strings.Split(string(data), "\n") - if len(lines) < 2 { - panic(fmt.Errorf("expected 2 lines but got %d", len(lines))) - } +func mapFn(data []byte) [][32]byte { + lines := strings.Split(string(data), "\n") + if len(lines) < 2 { + panic(fmt.Errorf("expected 2 lines but got %d", len(lines))) + } - line0Parts := line0RE.FindStringSubmatch(lines[0]) - line0Module, line0Version := line0Parts[1], line0Parts[2] + line0Parts := line0RE.FindStringSubmatch(lines[0]) + line0Module, line0Version := line0Parts[1], line0Parts[2] - line1Parts := line1RE.FindStringSubmatch(lines[1]) - line1Module, line1Version := line1Parts[1], line1Parts[2] + line1Parts := line1RE.FindStringSubmatch(lines[1]) + line1Module, line1Version := line1Parts[1], line1Parts[2] - if line0Module != line1Module { - klog.Errorf("mismatched module names: (%s, %s)", line0Module, line1Module) - } - if line0Version != line1Version { - klog.Errorf("mismatched version names: (%s, %s)", line0Version, line0Version) - } - if module.IsPseudoVersion(line0Version) { - // Drop any emphemeral builds - return nil - } + if line0Module != line1Module { + klog.Errorf("mismatched module names: (%s, %s)", line0Module, line1Module) + } + if line0Version != line1Version { + klog.Errorf("mismatched version names: (%s, %s)", line0Version, line0Version) + } + if module.IsPseudoVersion(line0Version) { + // Drop any emphemeral builds + return nil + } - klog.V(2).Infof("MapFn found: Module: %s:\t%s", line0Module, line0Version) + klog.V(2).Infof("MapFn found: Module: %s:\t%s", line0Module, line0Version) - return [][32]byte{sha256.Sum256([]byte(line0Module))} - } - return mapFn + return [][32]byte{sha256.Sum256([]byte(line0Module))} } diff --git a/vindex/inputlog.go b/vindex/inputlog.go index f6b8b91..d7cb655 100644 --- a/vindex/inputlog.go +++ b/vindex/inputlog.go @@ -32,11 +32,15 @@ type InputLogOpts struct { } func NewTiledInputLog(base *url.URL, v note.Verifier, o InputLogOpts) (InputLog, error) { - c := o.HttpClient - if c == nil { - c = http.DefaultClient + // Set any missing optional values to their defaults + if o.HttpClient == nil { + o.HttpClient = http.DefaultClient } - f, err := client.NewHTTPFetcher(base, c) + if len(o.Origin) == 0 { + o.Origin = v.Name() + } + + f, err := client.NewHTTPFetcher(base, o.HttpClient) if err != nil { return nil, err } @@ -59,11 +63,7 @@ func (s logReaderSource) Checkpoint(ctx context.Context) (checkpoint []byte, err } func (s logReaderSource) Parse(cpRaw []byte) (*log.Checkpoint, error) { - origin := s.opts.Origin - if len(origin) == 0 { - origin = s.v.Name() - } - cp, _, _, err := log.ParseCheckpoint(cpRaw, origin, s.v) + cp, _, _, err := log.ParseCheckpoint(cpRaw, s.opts.Origin, s.v) return cp, err }