From 1422a9441a89ff709fc616a74e78e54b8495164e Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Thu, 25 Sep 2025 10:59:53 +0000 Subject: [PATCH 1/2] [VIndex] SumDB verification tool Added a custom client for SumDB targeted at module maintainers. This queries a local checkout of a Go Module, and checks that every entry in SumDB has a corresponding git tag. It also prints out informational output in the case that there are local tags not present in SumDB. This is not a problem for SumDB, but it should be interesting to module authors if a version was never served. Note: the vanilla client still works and is documented for completeness. --- vindex/cmd/client/client.go | 4 +- vindex/cmd/sumdb/README.md | 37 ++++- vindex/cmd/sumdbverify/client.go | 260 +++++++++++++++++++++++++++++++ 3 files changed, 296 insertions(+), 5 deletions(-) create mode 100644 vindex/cmd/sumdbverify/client.go diff --git a/vindex/cmd/client/client.go b/vindex/cmd/client/client.go index 152a833..2a6b669 100644 --- a/vindex/cmd/client/client.go +++ b/vindex/cmd/client/client.go @@ -124,7 +124,7 @@ func newInputLogClientFromFlags() *client.InputLogClient { } v, err := note.NewVerifier(*inLogPubKey) if err != nil { - klog.Exitf("failed to construct output log verifier: %v", err) + klog.Exitf("failed to construct input log verifier: %v", err) } origin := *inLogOrigin if len(origin) == 0 { @@ -132,7 +132,7 @@ func newInputLogClientFromFlags() *client.InputLogClient { } c, err := client.NewInputLogClient(*inLogBaseURL, origin, v, http.DefaultClient) if err != nil { - klog.Exitf("failed to construct VIndex Client: %v", err) + klog.Exitf("failed to construct Input Log client: %v", err) } return c } diff --git a/vindex/cmd/sumdb/README.md b/vindex/cmd/sumdb/README.md index f96ad25..d6a0513 100644 --- a/vindex/cmd/sumdb/README.md +++ b/vindex/cmd/sumdb/README.md @@ -56,6 +56,38 @@ The command above starts a web server that hosts the following URLs: ### Querying +======= + +#### Auditing Local Modules + +A tool is provided that ensures that every version in SumDB has a corresponding git tag in a local checkout of the module. +The command below shows the output for this command querying a local checkout of `github.com/transparency-dev/tessera`: + +```shell +go run ./vindex/cmd/sumdbverify \ + --base_url http://localhost:8088/ \ + --out_log_pub_key=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd \ + --mod_root ~/git/tessera + +v0.1.0 found at index 37258761: ✅ found in git tags +v0.1.1 found at index 37258762: ✅ found in git tags +v0.1.2 found at index 37258746: ✅ found in git tags +v0.2.0 found at index 38108519: ✅ found in git tags +v1.0.0-rc1 found at index 41510961: ✅ found in git tags +v1.0.0-rc2 found at index 42710781: ✅ found in git tags +v1.0.0-rc3 found at index 43267373: ✅ found in git tags +v1.0.0 found at index 43930254: ✅ found in git tags +``` + +This tool only checks for differences in the known version strings between the Checksum DB and the git checkout. + +> [!IMPORTANT] +> This tool does not yet check that the hashes in the SumDB correspond to the state of the git repository. +> A useful extension would be to support checking out each git tag to confirm hashes. +> A [similar monitor](https://github.com/usbarmory/armory-drive-log/tree/master/cmd/monitor) exists for the armory-drive log. + +#### General Query + The SumDB log is processed into a verifiable map which can be looked up using the client in `./vindex/cmd/client`. Below is an example of querying the index to list all releases of `github.com/transparency-dev/tessera`. The output lists all of the indices where the module is logged, and the entry from the log at this index. @@ -67,8 +99,7 @@ go run ./vindex/cmd/client \ --out_log_pub_key=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd \ --in_log_pub_key=sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 \ --in_log_origin="go.sum database tree" \ - --lookup=github.com/transparency-dev/tessera \ - --min_idx=40000000 + --lookup=github.com/transparency-dev/tessera 37258746) github.com/transparency-dev/tessera v0.1.2 h1:s8h0HQ5knhvCmQ2TdATw7FMTvdZY+RMcAcgsaPrKm1k= @@ -115,7 +146,7 @@ Note that this matches the list of entries on the unverifiable proxy endpoint: h When running this command on a regular basis, you can avoid seeing entries you have already processed by providing the `--min_idx` flag. ```shell -❯ go run ./vindex/cmd/client \ +go run ./vindex/cmd/client \ --vindex_base_url http://localhost:8088/vindex/ \ --in_log_base_url http://localhost:8088/inputlog/ \ --out_log_pub_key=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd \ diff --git a/vindex/cmd/sumdbverify/client.go b/vindex/cmd/sumdbverify/client.go new file mode 100644 index 0000000..05f3a1d --- /dev/null +++ b/vindex/cmd/sumdbverify/client.go @@ -0,0 +1,260 @@ +// Copyright 2025 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// sumdbverify defines a binary that uses the Go Checksum Verifiable Index +// to look up versions for a given Go module, and compares these entries with +// the local git tags. +package main + +import ( + "context" + "encoding/base64" + "errors" + "flag" + "fmt" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/transparency-dev/incubator/vindex/client" + "golang.org/x/mod/modfile" + "golang.org/x/mod/semver" + "golang.org/x/mod/sumdb/note" + "golang.org/x/sync/errgroup" + "k8s.io/klog/v2" +) + +var ( + baseURL = flag.String("base_url", "", "The base URL of the server hosting the logs and vindex.") + outLogPubKey = flag.String("out_log_pub_key", "", "The public key to use to verify the output log checkpoint.") + modRoot = flag.String("mod_root", "", "The path to a go module checked out locally via git.") +) + +var ( + // Example leaf: + // golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= + // golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= + // + line0RE = regexp.MustCompile(`(.*) (.*) h1:(.*)`) + line1RE = regexp.MustCompile(`(.*) (.*)/go.mod h1:(.*)`) +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + if err := run(context.Background()); err != nil { + klog.Exitf("run failed: %v", err) + } +} + +func run(ctx context.Context) error { + if *baseURL == "" { + return errors.New("base_url flag must be provided") + } + if *outLogPubKey == "" { + return errors.New("out_log_pub_key flag must be provided") + } + if *modRoot == "" { + return errors.New("mod_root flag must be provided") + } + if s, err := os.Stat(*modRoot); err != nil || !s.IsDir() { + return errors.New("mod_root flag must be a directory") + } + modPathBytes, err := os.ReadFile(filepath.Join(*modRoot, "go.mod")) + if err != nil { + return fmt.Errorf("failed to read go.mod file: %v", err) + } + modPath, err := modfile.Parse("go.mod", modPathBytes, nil) + if err != nil { + return fmt.Errorf("failed to parse go.mod file: %v", err) + } + modName := modPath.Module.Mod.Path + + eg, egctx := errgroup.WithContext(ctx) + var versions map[string]modData + var tags map[string]struct{} + + eg.Go(func() error { + // This function gets all version info from the verifiable index. + vic := newVIndexClientFromFlags() + + versions, err = queryIndex(egctx, vic, modName) + if err != nil { + return fmt.Errorf("error querying index: %v", err) + } + return nil + }) + eg.Go(func() error { + // This function gets all tags from the local git checkout. + rawTags, err := queryTags(egctx, *modRoot) + if err != nil { + return fmt.Errorf("error enumerating git tags: %v", err) + } + tags = make(map[string]struct{}, len(rawTags)) + for _, t := range rawTags { + tags[t] = struct{}{} + } + return nil + }) + if err := eg.Wait(); err != nil { + return err + } + + // Create a sorted slice of versions + sv := make([]string, 0, len(versions)) + for v := range versions { + sv = append(sv, v) + } + semver.Sort(sv) + + fmt.Println(modName) + for _, v := range sv { + d := versions[v] + presence := "✅ found in git tags" + if _, found := tags[v]; !found { + presence = "❌ missing from git tags" + } + + fmt.Printf("%s found at index %d: %s\n", v, d.index, presence) + delete(tags, v) + } + + if len(tags) > 0 { + fmt.Println("----------") + fmt.Println("> INFO: The tagged versions below were never downloaded via the Module Proxy") + for t := range tags { + fmt.Printf("%s found locally but missing from SumDB\n", t) + } + } + return nil +} + +type modData struct { + index uint64 + zipHash []byte + modHash []byte +} + +func queryTags(ctx context.Context, modRoot string) ([]string, error) { + cmd := exec.CommandContext(ctx, "git", "tag") + cmd.Dir = modRoot + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("failed to run git tag: %w", err) + } + trimmed := strings.TrimSpace(string(out)) + if trimmed == "" { + return []string{}, nil + } + tags := strings.Split(trimmed, "\n") + return tags, nil +} + +func queryIndex(ctx context.Context, vic *client.VIndexClient, modName string) (map[string]modData, error) { + idxes, inCp, err := vic.Lookup(ctx, modName) + if err != nil { + return nil, fmt.Errorf("failed to look up key: %v", err) + } + + versions := make(map[string]modData, len(idxes)) + + lr := newInputLogClientFromFlags() + klog.V(1).Infof("Dereferencing %d pointers", len(idxes)) + for leaf, err := range lr.Dereference(ctx, inCp, idxes) { + if err != nil { + return nil, fmt.Errorf("failed to get leaf at index %d: %v", leaf.Index, err) + } + version, data, err := parseLeaf(leaf.Index, leaf.Data) + if err != nil { + return nil, fmt.Errorf("failed to parse leaf at index %d: %v", leaf.Index, err) + } + if prev, found := versions[version]; found { + return nil, fmt.Errorf("conflicting versions for version %q found!\n%v\n%v", version, prev, data) + } + versions[version] = data + } + return versions, nil +} + +// parseLeaf extracts the version string and the hashes from the raw leaf data. +func parseLeaf(idx uint64, data []byte) (string, modData, error) { + lines := strings.Split(string(data), "\n") + if len(lines) < 2 { + panic(fmt.Errorf("expected 2 lines but got %d", len(lines))) + } + + line0Parts := line0RE.FindStringSubmatch(lines[0]) + line0Module, line0Version, zipHashB64 := line0Parts[1], line0Parts[2], line0Parts[3] + + line1Parts := line1RE.FindStringSubmatch(lines[1]) + line1Module, line1Version, modHashB64 := line1Parts[1], line1Parts[2], line1Parts[3] + + if line0Module != line1Module { + return "", modData{}, fmt.Errorf("mismatched module names: (%s, %s)", line0Module, line1Module) + } + if line0Version != line1Version { + return "", modData{}, fmt.Errorf("mismatched version names: (%s, %s)", line0Version, line1Version) + } + + zipHash, err := base64.StdEncoding.DecodeString(zipHashB64) + if err != nil { + return "", modData{}, fmt.Errorf("failed to decode hash %q: %v", zipHashB64, err) + } + modHash, err := base64.StdEncoding.DecodeString(modHashB64) + if err != nil { + return "", modData{}, fmt.Errorf("failed to decode hash %q: %v", modHashB64, err) + } + return line0Version, modData{ + index: idx, + zipHash: zipHash, + modHash: modHash, + }, nil +} + +func newVIndexClientFromFlags() *client.VIndexClient { + outV, err := note.NewVerifier(*outLogPubKey) + if err != nil { + klog.Exitf("failed to construct VIndex verifier: %v", err) + } + u, err := url.JoinPath(*baseURL, "/vindex/") + if err != nil { + klog.Exitf("failed to construct VIndex URL: %v", err) + } + c, err := client.NewVIndexClient(u, outV) + if err != nil { + klog.Exitf("failed to construct VIndex Client: %v", err) + } + return c +} + +func newInputLogClientFromFlags() *client.InputLogClient { + v, err := note.NewVerifier("sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8") + if err != nil { + klog.Exitf("failed to construct Input Log verifier: %v", err) + } + u, err := url.JoinPath(*baseURL, "/inputlog/") + if err != nil { + klog.Exitf("failed to construct Input Log URL: %v", err) + } + origin := "go.sum database tree" + c, err := client.NewInputLogClient(u, origin, v, http.DefaultClient) + if err != nil { + klog.Exitf("failed to construct Input Log client: %v", err) + } + return c +} From c8ac6a93e475ab1b30d68f4f140da3a075faa93d Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Thu, 25 Sep 2025 14:54:44 +0000 Subject: [PATCH 2/2] review comments --- vindex/cmd/sumdbverify/client.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/vindex/cmd/sumdbverify/client.go b/vindex/cmd/sumdbverify/client.go index 05f3a1d..7d48baf 100644 --- a/vindex/cmd/sumdbverify/client.go +++ b/vindex/cmd/sumdbverify/client.go @@ -157,12 +157,7 @@ func queryTags(ctx context.Context, modRoot string) ([]string, error) { if err != nil { return nil, fmt.Errorf("failed to run git tag: %w", err) } - trimmed := strings.TrimSpace(string(out)) - if trimmed == "" { - return []string{}, nil - } - tags := strings.Split(trimmed, "\n") - return tags, nil + return strings.Split(strings.TrimSpace(string(out)), "\n"), nil } func queryIndex(ctx context.Context, vic *client.VIndexClient, modName string) (map[string]modData, error) { @@ -195,7 +190,7 @@ func queryIndex(ctx context.Context, vic *client.VIndexClient, modName string) ( func parseLeaf(idx uint64, data []byte) (string, modData, error) { lines := strings.Split(string(data), "\n") if len(lines) < 2 { - panic(fmt.Errorf("expected 2 lines but got %d", len(lines))) + return "", modData{}, fmt.Errorf("expected 2 lines but got %d", len(lines)) } line0Parts := line0RE.FindStringSubmatch(lines[0])