diff --git a/sumdb/README.md b/sumdb/README.md index 18f3d16..5a77501 100644 --- a/sumdb/README.md +++ b/sumdb/README.md @@ -6,7 +6,7 @@ This allows tooling written for the tlog-tiles API to be used with the SumDB, ev ### Running ```shell -go run ./sumdb/proxy.go --listen=":8089" +go run ./sumdb/cmd/proxy.go --listen=":8089" ``` ### Using diff --git a/sumdb/cmd/proxy.go b/sumdb/cmd/proxy.go new file mode 100644 index 0000000..e58c71a --- /dev/null +++ b/sumdb/cmd/proxy.go @@ -0,0 +1,45 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// sumdb is a command that launches a local proxy that allows clients to query +// using the tlog-tiles API, and retrieve results from SumDB. +package main + +import ( + "flag" + + "net/http" + + "github.com/transparency-dev/incubator/sumdb" + "k8s.io/klog/v2" +) + +var ( + listen = flag.String("listen", ":8089", "Address to set up HTTP server listening on") +) + +const ( + upstreamBase = "https://sum.golang.org" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + proxy := sumdb.NewProxy(sumdb.ProxyOpts{}) + klog.Infof("Proxying tlog-tiles API to %s on %s", upstreamBase, *listen) + if err := http.ListenAndServe(*listen, proxy); err != nil { + klog.Fatalf("ListenAndServe: %v", err) + } +} diff --git a/sumdb/proxy.go b/sumdb/proxy.go index caa3ec4..0fd6253 100644 --- a/sumdb/proxy.go +++ b/sumdb/proxy.go @@ -12,14 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -// sumdb is a command that launches a local proxy that allows clients to query -// using the tlog-tiles API, and retrieve results from SumDB. -package main +// sumdb provides a utility proxy to convert to a tlog-tiles API. +package sumdb import ( "bytes" "encoding/binary" - "flag" "fmt" "net/http" @@ -32,23 +30,24 @@ import ( "k8s.io/klog/v2" ) -var ( - listen = flag.String("listen", ":8089", "Address to set up HTTP server listening on") -) - const ( upstreamBase = "https://sum.golang.org" ) -func main() { - klog.InitFlags(nil) - flag.Parse() +type ProxyOpts struct { + // PathPrefix should be set if the proxy is hosted not at "/". + // Any path beyond this should be set here, so that it can be stripped. + PathPrefix string +} +func NewProxy(opts ProxyOpts) *httputil.ReverseProxy { upstream, err := url.Parse(upstreamBase) if err != nil { klog.Fatalf("Failed to parse upstream URL %q: %v", upstreamBase, err) } + prefix, _ := strings.CutSuffix(opts.PathPrefix, "/") + const tlogEntriesPrefix = "/tile/entries/" const tlogTilePrefix = "/tile/" @@ -57,14 +56,16 @@ func main() { proxy := &httputil.ReverseProxy{ Rewrite: func(r *httputil.ProxyRequest) { r.SetURL(upstream) + inPath := strings.TrimPrefix(r.In.URL.Path, prefix) + klog.V(2).Infof("Request for %s", r.In.URL.Path) - if r.In.URL.Path == "/checkpoint" { + if inPath == "/checkpoint" { r.Out.URL.Path = "/latest" - } else if strings.HasPrefix(r.In.URL.Path, tlogEntriesPrefix) { - o := strings.TrimPrefix(r.In.URL.Path, tlogEntriesPrefix) + } else if strings.HasPrefix(inPath, tlogEntriesPrefix) { + o := strings.TrimPrefix(inPath, tlogEntriesPrefix) r.Out.URL.Path = fmt.Sprintf("%s%s", sumDBTileDataPrefix, o) - } else if strings.HasPrefix(r.In.URL.Path, tlogTilePrefix) { - o := strings.TrimPrefix(r.In.URL.Path, tlogTilePrefix) + } else if strings.HasPrefix(inPath, tlogTilePrefix) { + o := strings.TrimPrefix(inPath, tlogTilePrefix) r.Out.URL.Path = fmt.Sprintf("%s%s", sumDBTilePrefix, o) } }, @@ -91,9 +92,5 @@ func main() { }, } - klog.Infof("Proxying tlog-tiles API to %s on %s", upstreamBase, *listen) - if err := http.ListenAndServe(*listen, proxy); err != nil { - klog.Fatalf("ListenAndServe: %v", err) - } + return proxy } - diff --git a/vindex/client/client.go b/vindex/client/client.go index eb632a3..4e9821d 100644 --- a/vindex/client/client.go +++ b/vindex/client/client.go @@ -179,7 +179,7 @@ func (c VIndexClient) lookupUnverified(ctx context.Context, kh [sha256.Size]byte // NewInputLogClient returns a client that allows pointers returned from the Verifiable Index // to be dereferenced by looking up entries in the Input Log. All operations are verified by this // client, which closes the loop. -func NewInputLogClient(inLogUrl string, inV note.Verifier, hc *http.Client) (*InputLogClient, error) { +func NewInputLogClient(inLogUrl string, origin string, inV note.Verifier, hc *http.Client) (*InputLogClient, error) { u, err := url.Parse(inLogUrl) if err != nil { return nil, fmt.Errorf("failed to parse URL: %v", err) @@ -189,23 +189,25 @@ func NewInputLogClient(inLogUrl string, inV note.Verifier, hc *http.Client) (*In return nil, fmt.Errorf("failed to create HTTP fetcher for %q: %v", u, err) } return &InputLogClient{ - v: inV, - lc: c, + v: inV, + origin: origin, + lc: c, }, nil } // InputLogClient is a client intended to be used by users of the VIndexClient that want // to look up the original leaves from the Input Log. type InputLogClient struct { - v note.Verifier - lc logClient + v note.Verifier + origin string + lc logClient } // Dereference takes pointers returned by the VIndexClient Lookup method, and fetches // the original leaves from the Input Log. The inclusion of any leaves returned will be // verified by constructing inclusion proofs to the checkpoint provided. func (c *InputLogClient) Dereference(ctx context.Context, cpRaw []byte, pointers []uint64) iter.Seq2[InputLogLeaf, error] { - cp, _, _, err := log.ParseCheckpoint(cpRaw, c.v.Name(), c.v) + cp, _, _, err := log.ParseCheckpoint(cpRaw, c.origin, c.v) if err != nil { return func(yield func(InputLogLeaf, error) bool) { yield(InputLogLeaf{}, fmt.Errorf("failed to parse input log checkpoint: %v", err)) diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index 73f7070..152a833 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -36,6 +36,7 @@ var ( lookup = flag.String("lookup", "", "The key to look up in the vindex.") outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") inLogPubKey = flag.String("in_log_pub_key", "", "The public key to use to verify the input log checkpoint.") + inLogOrigin = flag.String("in_log_origin", "", "Optional: allows the Input Log Origin string to be configured to something other than the public key name.") minIdx = flag.Uint64("min_idx", 0, "The minimum index to look up in the input log.") ) @@ -125,7 +126,11 @@ func newInputLogClientFromFlags() *client.InputLogClient { if err != nil { klog.Exitf("failed to construct output log verifier: %v", err) } - c, err := client.NewInputLogClient(*inLogBaseURL, v, http.DefaultClient) + origin := *inLogOrigin + if len(origin) == 0 { + origin = v.Name() + } + c, err := client.NewInputLogClient(*inLogBaseURL, origin, v, http.DefaultClient) if err != nil { klog.Exitf("failed to construct VIndex Client: %v", err) } diff --git a/vindex/cmd/logandmap/main.go b/vindex/cmd/logandmap/main.go index ec6ebbf..4276292 100644 --- a/vindex/cmd/logandmap/main.go +++ b/vindex/cmd/logandmap/main.go @@ -50,8 +50,8 @@ import ( ) var ( - inputLogPrivKeyFile = flag.String("input_log_private_key", "", "Location of private key file. If unset, uses the contents of the INPUT_LOG_PRIVATE_KEY environment variable.") - outputLogPrivKeyFile = flag.String("output_log_private_key", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + inputLogPrivKeyFile = flag.String("input_log_private_key_path", "", "Location of private key file. If unset, uses the contents of the INPUT_LOG_PRIVATE_KEY environment variable.") + outputLogPrivKeyFile = flag.String("output_log_private_key_path", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Input Log, Output Log, and allocate space to store the verifiable map persistence.") listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") ) @@ -291,7 +291,7 @@ func getInputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { } else { privKey = os.Getenv("INPUT_LOG_PRIVATE_KEY") if len(privKey) == 0 { - klog.Exit("Supply private key file path using --input_log_private_key or set INPUT_LOG_PRIVATE_KEY environment variable") + klog.Exit("Supply private key file path using --input_log_private_key_path or set INPUT_LOG_PRIVATE_KEY environment variable") } } s, v, err := fnote.NewEd25519SignerVerifier(privKey) @@ -314,7 +314,7 @@ func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { } else { privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") if len(privKey) == 0 { - klog.Exit("Supply private key file path using --output_log_private_key or set OUTPUT_LOG_PRIVATE_KEY environment variable") + klog.Exit("Supply private key file path using --output_log_private_key_path or set OUTPUT_LOG_PRIVATE_KEY environment variable") } } s, v, err := fnote.NewEd25519SignerVerifier(privKey) diff --git a/vindex/cmd/sumdb/README.md b/vindex/cmd/sumdb/README.md new file mode 100644 index 0000000..f4ca193 --- /dev/null +++ b/vindex/cmd/sumdb/README.md @@ -0,0 +1,48 @@ +## Verifiable Index: SumDB + +This is a demo of building a [Verifiable Index](../../README.md) for Go's SumDB. + +The index allows package maintainers to verifiably look up all [non-pseudo](https://pkg.go.dev/golang.org/x/mod@v0.28.0/module#IsPseudoVersion) versions of their module served by the Module Proxy. + +[tlog-tiles]: https://c2sp.org/tlog-tiles +[Tessera]: https://github.com/transparency-dev/tessera + +## Running + +The Verifiable Index and Output Log are managed by a single binary, which can be run using: + +```shell +OUTPUT_LOG_PRIVATE_KEY=PRIVATE+KEY+SumDBIndex+a5ed0e81+AYT6tfHpqGaSoH0gYpM7fhj1tEkM3wwYR/IhtiYh1pnj \ +go run ./vindex/cmd/sumdb \ + --storage_dir ~/vindex-sumdb/ +``` + +Running the above will run a web server hosting the following URLs: + - `/inputlog/` - the [tlog-tiles][] base URL for a proxy of the SumDB API + - `/vindex/lookup` - the provisional [vindex lookup API](./api/api.go) + - `/outputlog/` - the [tlog-tiles][] base URL for the output log + +To inspect the log, you can use the woodpecker tool (using the corresponding public key to the private key used above): + +```shell +# To inspect the Output Log +go run github.com/mhutchinson/woodpecker@main \ + --custom_log_type=tiles \ + --custom_log_url=http://localhost:8088/outputlog/ \ + --custom_log_vkey=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd +``` + +Use left/right cursor to browse, and `q` to quit. + +This log is processed into a verifiable map which can be looked up using the following command: + +```shell +go run ./vindex/cmd/client \ + --vindex_base_url http://localhost:8088/vindex/ \ + --in_log_base_url http://localhost:8088/inputlog/ \ + --out_log_pub_key=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd \ + --in_log_pub_key=sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 \ + --in_log_origin="go.sum database tree" \ + --lookup=github.com/transparency-dev/tessera +``` + diff --git a/vindex/cmd/sumdb/main.go b/vindex/cmd/sumdb/main.go new file mode 100644 index 0000000..6e4ef87 --- /dev/null +++ b/vindex/cmd/sumdb/main.go @@ -0,0 +1,234 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// sumdb brings up a verifiable index for the Go SumDB. +// This requires a proxy to be running to bridge to a tlog-tiles API. +// See the README for usage details. +package main + +import ( + "context" + "crypto/sha256" + "errors" + "flag" + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/signal" + "path" + "regexp" + "strings" + "syscall" + "time" + + "github.com/gorilla/mux" + fnote "github.com/transparency-dev/formats/note" + "github.com/transparency-dev/incubator/sumdb" + "github.com/transparency-dev/incubator/vindex" + "golang.org/x/mod/module" + "golang.org/x/mod/sumdb/note" + "k8s.io/klog/v2" +) + +var ( + outputLogPrivKeyFile = flag.String("output_log_private_key_path", "", "Location of private key file. If unset, uses the contents of the OUTPUT_LOG_PRIVATE_KEY environment variable.") + storageDir = flag.String("storage_dir", "", "Root directory in which to store the data for the demo. This will create subdirectories for the Output Log, and allocate space to store the verifiable map persistence.") + listen = flag.String("listen", ":8088", "Address to set up HTTP server listening on") +) + +var ( + // Example leaf: + // golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= + // golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= + // + line0RE = regexp.MustCompile(`(.*) (.*) h1:(.*)`) + line1RE = regexp.MustCompile(`(.*) (.*)/go.mod h1:(.*)`) +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx); err != nil { + klog.Exitf("Run failed: %v", err) + } +} + +func run(ctx context.Context) error { + // Set up storage for the input log, index, and output log. + if *storageDir == "" { + return errors.New("storage_dir must be set") + } + inputLogDir := path.Join(*storageDir, "inputlog") + outputLogDir := path.Join(*storageDir, "outputlog") + mapRoot := path.Join(*storageDir, "vindex") + + if err := os.MkdirAll(inputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create input log directory: %v", err) + } + if err := os.MkdirAll(outputLogDir, 0o755); err != nil { + return fmt.Errorf("failed to create output log directory: %v", err) + } + if err := os.MkdirAll(mapRoot, 0o755); err != nil { + return fmt.Errorf("failed to create vindex directory: %v", err) + } + + sumV, err := note.NewVerifier("sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8") + if err != nil { + return err + } + sumUrl, err := url.Parse(fmt.Sprintf("http://%s/inputlog/", *listen)) + if err != nil { + return err + } + inputLog, err := vindex.NewTiledInputLog(sumUrl, sumV, vindex.InputLogOpts{ + HttpClient: http.DefaultClient, + Origin: "go.sum database tree", + }) + if err != nil { + return err + } + sumProxy := sumdb.NewProxy(sumdb.ProxyOpts{ + PathPrefix: "/inputlog/", + }) + + outputLog, outputCloser := outputLogOrDie(ctx, outputLogDir) + defer outputCloser() + + vi, err := vindex.NewVerifiableIndex(ctx, inputLog, mapFn, outputLog, mapRoot, vindex.Options{}) + if err != nil { + return fmt.Errorf("failed to create vindex: %v", err) + } + + // Run a web server to serve the input log, index, and output log. + go runWebServer(sumProxy, vi, outputLogDir) + + // Keeps the map synced with the latest published input log state. + go maintainMap(ctx, vi) + + <-ctx.Done() + return nil +} + +// outputLogOrDie returns an output log using a POSIX log in the given directory. +func outputLogOrDie(ctx context.Context, outputLogDir string) (log vindex.OutputLog, closer func()) { + s, v := getOutputLogSignerVerifierOrDie() + + l, c, err := vindex.NewOutputLog(ctx, outputLogDir, s, v) + if err != nil { + klog.Exitf("Failed to create Output Log: %v", err) + } + return l, c +} + +// maintainMap reads entries from the log and sync them to the vindex. +func maintainMap(ctx context.Context, vi *vindex.VerifiableIndex) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + if err := vi.Update(ctx); err != nil { + klog.Warningf("Failed to Update index: %v", err) + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func runWebServer(inLog *httputil.ReverseProxy, vi *vindex.VerifiableIndex, outLogDir string) { + web := NewServer(vi.Lookup) + + olfs := http.FileServer(http.Dir(outLogDir)) + r := mux.NewRouter() + r.PathPrefix("/inputlog/").Handler(inLog) + r.PathPrefix("/outputlog/").Handler(http.StripPrefix("/outputlog/", olfs)) + web.registerHandlers(r) + hServer := &http.Server{ + Addr: *listen, + Handler: r, + } + go func() { + if err := hServer.ListenAndServe(); err != http.ErrServerClosed { + klog.Warningf("Error from HTTP server: %v", err) + } + }() + klog.Infof("Started HTTP server listening on %s", *listen) +} + +// Read output log private key from file or environment variable and generate the +// note Signer and Verifier pair for it. +func getOutputLogSignerVerifierOrDie() (note.Signer, note.Verifier) { + var privKey string + var err error + if len(*outputLogPrivKeyFile) > 0 { + privKey, err = getKeyFile(*outputLogPrivKeyFile) + if err != nil { + klog.Exitf("Unable to get private key: %v", err) + } + } else { + privKey = os.Getenv("OUTPUT_LOG_PRIVATE_KEY") + if len(privKey) == 0 { + klog.Exit("Supply private key file path using --output_log_private_key_path or set OUTPUT_LOG_PRIVATE_KEY environment variable") + } + } + s, v, err := fnote.NewEd25519SignerVerifier(privKey) + if err != nil { + klog.Exitf("Failed to get signer/verifier: %v", err) + } + return s, v +} + +func getKeyFile(path string) (string, error) { + k, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + return string(k), nil +} + +func mapFn(data []byte) [][32]byte { + lines := strings.Split(string(data), "\n") + if len(lines) < 2 { + panic(fmt.Errorf("expected 2 lines but got %d", len(lines))) + } + + line0Parts := line0RE.FindStringSubmatch(lines[0]) + line0Module, line0Version := line0Parts[1], line0Parts[2] + + line1Parts := line1RE.FindStringSubmatch(lines[1]) + line1Module, line1Version := line1Parts[1], line1Parts[2] + + if line0Module != line1Module { + klog.Errorf("mismatched module names: (%s, %s)", line0Module, line1Module) + } + if line0Version != line1Version { + klog.Errorf("mismatched version names: (%s, %s)", line0Version, line0Version) + } + if module.IsPseudoVersion(line0Version) { + // Drop any emphemeral builds + return nil + } + + klog.V(2).Infof("MapFn found: Module: %s:\t%s", line0Module, line0Version) + + return [][32]byte{sha256.Sum256([]byte(line0Module))} +} diff --git a/vindex/cmd/sumdb/web.go b/vindex/cmd/sumdb/web.go new file mode 100644 index 0000000..68c8c25 --- /dev/null +++ b/vindex/cmd/sumdb/web.go @@ -0,0 +1,76 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + + "github.com/gorilla/mux" + "github.com/transparency-dev/incubator/vindex/api" + "k8s.io/klog/v2" +) + +func NewServer(lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error)) Server { + return Server{ + lookup: lookup, + } +} + +type Server struct { + lookup func(context.Context, [sha256.Size]byte) (api.LookupResponse, error) +} + +// handleLookup handles GET requests for looking up map entries. +func (s Server) handleLookup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + hashStr, ok := vars["hash"] + if !ok { + http.Error(w, "hash parameter not found", http.StatusBadRequest) + return + } + + h, err := hex.DecodeString(hashStr) + if err != nil { + http.Error(w, fmt.Sprintf("invalid hex hash: %v", err), http.StatusBadRequest) + return + } + if l := len(h); l != sha256.Size { + http.Error(w, fmt.Sprintf("hash wrong length (decoded %d bytes)", l), http.StatusBadRequest) + return + } + + klog.V(2).Infof("Received hash from request: '%s'", h) + + resp, err := s.lookup(r.Context(), [sha256.Size]byte(h)) + if err != nil { + http.Error(w, fmt.Sprintf("lookup failed: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + klog.Warningf("failed to encode response: %v", err) + } +} + +func (s Server) registerHandlers(r *mux.Router) { + r.HandleFunc("/vindex/lookup/{hash}", s.handleLookup).Methods("GET") +} diff --git a/vindex/inputlog.go b/vindex/inputlog.go new file mode 100644 index 0000000..d7cb655 --- /dev/null +++ b/vindex/inputlog.go @@ -0,0 +1,97 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vindex + +import ( + "context" + "iter" + "net/http" + "net/url" + + "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/tessera/api" + "github.com/transparency-dev/tessera/client" + "golang.org/x/mod/sumdb/note" +) + +type InputLogOpts struct { + HttpClient *http.Client + Origin string +} + +func NewTiledInputLog(base *url.URL, v note.Verifier, o InputLogOpts) (InputLog, error) { + // Set any missing optional values to their defaults + if o.HttpClient == nil { + o.HttpClient = http.DefaultClient + } + if len(o.Origin) == 0 { + o.Origin = v.Name() + } + + f, err := client.NewHTTPFetcher(base, o.HttpClient) + if err != nil { + return nil, err + } + return logReaderSource{ + f: f, + v: v, + opts: o, + }, nil +} + +// logReaderSource adapts a tessera.LogReader to a vindex.InputLog. +type logReaderSource struct { + f *client.HTTPFetcher + v note.Verifier + opts InputLogOpts +} + +func (s logReaderSource) Checkpoint(ctx context.Context) (checkpoint []byte, err error) { + return s.f.ReadCheckpoint(ctx) +} + +func (s logReaderSource) Parse(cpRaw []byte) (*log.Checkpoint, error) { + cp, _, _, err := log.ParseCheckpoint(cpRaw, s.opts.Origin, s.v) + return cp, err +} + +func (s logReaderSource) Leaves(ctx context.Context, start, end uint64) iter.Seq2[[]byte, error] { + tsf := func(ctx context.Context) (uint64, error) { + return end, nil + } + bi := client.EntryBundles(ctx, 2, tsf, s.f.ReadEntryBundle, start, end-start) + unbundleFn := func(bundle []byte) ([][]byte, error) { + eb := &api.EntryBundle{} + if err := eb.UnmarshalText(bundle); err != nil { + return nil, err + } + return eb.Entries, nil + } + + return func(yield func([]byte, error) bool) { + // Unwrap the client.Entry type to return an iterator of []byte only. + for entry, err := range client.Entries(bi, unbundleFn) { + if err != nil { + if !yield(nil, err) { + return + } + continue + } + if !yield(entry.Entry, nil) { + return + } + } + } +}