From 4f28d4a6e9cd86678c39a30b3bf80020337fe056 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Tue, 30 Sep 2025 16:18:01 +0000 Subject: [PATCH 1/2] Improve the sumdbverify client tool Now compares the go.mod file contents match between the git repository for each version, and that served in SumDB. Note that this still doesn't calculate and compare the zip file hash. Misc smaller changes: - Much better output: proper tabular output and granular info - Pre-refactoring ready to accept non-VIndex SumDB info - Switch away from os.Exec to use proper git implementation - README for the sumdbverify command directly --- go.mod | 22 ++- go.sum | 87 ++++++++++- vindex/cmd/sumdb/README.md | 26 +--- vindex/cmd/sumdbverify/README.md | 37 +++++ vindex/cmd/sumdbverify/client.go | 245 +++++++++++++++++++++++++------ 5 files changed, 337 insertions(+), 80 deletions(-) create mode 100644 vindex/cmd/sumdbverify/README.md diff --git a/go.mod b/go.mod index 36d57d2..e42d88a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24.1 require ( filippo.io/torchwood v0.5.1-0.20250821141945-7cf4555d7644 github.com/cockroachdb/pebble v1.1.5 + github.com/go-git/go-git/v5 v5.16.2 github.com/google/go-cmp v0.7.0 github.com/gorilla/mux v1.8.1 github.com/transparency-dev/formats v0.0.0-20250723101439-be3b1008ec3a @@ -16,46 +17,63 @@ require ( ) require ( + dario.cat/mergo v1.0.0 // indirect github.com/DataDog/zstd v1.4.5 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/circl v1.6.1 // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.6.2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.15.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel v1.38.0 // indirect go.opentelemetry.io/otel/metric v1.38.0 // indirect go.opentelemetry.io/otel/trace v1.38.0 // indirect golang.org/x/crypto v0.42.0 // indirect golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect + golang.org/x/net v0.44.0 // indirect golang.org/x/sys v0.36.0 // indirect golang.org/x/text v0.29.0 // indirect google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect modernc.org/libc v1.65.7 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index bada0ea..a7156f6 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,26 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/torchwood v0.5.1-0.20250821141945-7cf4555d7644 h1:xPQ8RTWOsXdseR4XG7RRuERLvJOBcbjMohcZ66Nj2AY= filippo.io/torchwood v0.5.1-0.20250821141945-7cf4555d7644/go.mod h1:Z+iz3Syg0RCaVkL9nBjG2STp/9HpuFl1+SbaNSZ/Ez8= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= @@ -23,14 +36,31 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= +github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.16.2 h1:fT6ZIOjE5iEnkzKyxTHK1W4HGAsPhqEqiSAssSO77hM= +github.com/go-git/go-git/v5 v5.16.2/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -38,14 +68,14 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -54,14 +84,21 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -70,8 +107,12 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -88,8 +129,16 @@ github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/transparency-dev/formats v0.0.0-20250723101439-be3b1008ec3a h1:l1RrmDw9xrVN/lbW/rzPJhjQ+dmsqOyItES9Ku/njFA= @@ -98,6 +147,8 @@ github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= github.com/transparency-dev/tessera v1.0.0 h1:4OT1V9xJLa5NnYlFWWlCdZkCm18/o12rdd+bCTje7XE= github.com/transparency-dev/tessera v1.0.0/go.mod h1:TLvfjlkbmsmKVEJUtzO2eb9Q2IBnK3EJ0dI4G0oxEOU= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= @@ -111,6 +162,7 @@ go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42s golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= @@ -123,6 +175,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -131,12 +186,22 @@ golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -149,10 +214,16 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/vindex/cmd/sumdb/README.md b/vindex/cmd/sumdb/README.md index 7500208..3a95f2f 100644 --- a/vindex/cmd/sumdb/README.md +++ b/vindex/cmd/sumdb/README.md @@ -58,31 +58,7 @@ The command above starts a web server that hosts the following URLs: #### Auditing Local Modules -A tool is provided that ensures that every version in SumDB has a corresponding git tag in a local checkout of the module. -The command below shows the output for this command querying a local checkout of `github.com/transparency-dev/tessera`: - -```shell -go run ./vindex/cmd/sumdbverify \ - --base_url http://localhost:8088/ \ - --out_log_pub_key=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd \ - --mod_root ~/git/tessera - -v0.1.0 found at index 37258761: ✅ found in git tags -v0.1.1 found at index 37258762: ✅ found in git tags -v0.1.2 found at index 37258746: ✅ found in git tags -v0.2.0 found at index 38108519: ✅ found in git tags -v1.0.0-rc1 found at index 41510961: ✅ found in git tags -v1.0.0-rc2 found at index 42710781: ✅ found in git tags -v1.0.0-rc3 found at index 43267373: ✅ found in git tags -v1.0.0 found at index 43930254: ✅ found in git tags -``` - -This tool only checks for differences in the known version strings between the Checksum DB and the git checkout. - -> [!IMPORTANT] -> This tool does not yet check that the hashes in the SumDB correspond to the state of the git repository. -> A useful extension would be to support checking out each git tag to confirm hashes. -> A [similar monitor](https://github.com/usbarmory/armory-drive-log/tree/master/cmd/monitor) exists for the armory-drive log. +The best tool to use is the SumDB-specific verification tool: [sumdbverify](../sumdbverify/). #### General Query diff --git a/vindex/cmd/sumdbverify/README.md b/vindex/cmd/sumdbverify/README.md new file mode 100644 index 0000000..ae2ea57 --- /dev/null +++ b/vindex/cmd/sumdbverify/README.md @@ -0,0 +1,37 @@ +## SumDB Verify + +> [!IMPORTANT] +> This tool requires a [SumDB VIndex](../sumdb/) to be running. +> Functionality may be added to support reading SumDB contents from non-verifiable endpoints. + +This tool checks that the contents for a module in SumDB match the state as represented in a local git repository. +The command below shows the output for this command querying a local checkout of `github.com/transparency-dev/tessera`: + +```shell +go run ./vindex/cmd/sumdbverify \ + --base_url http://localhost:8088/ \ + --out_log_pub_key=SumDBIndex+a5ed0e81+AXEnbaKj+9gCH3f69vcQokgkcFocCl+GlaMXrAg8mRzd \ + --mod_root ~/git/tessera + +github.com/transparency-dev/tessera +VERSION INDEX FOUND go.mod +v0.1.0 37258761 ✅ ✅ +v0.1.1 37258762 ✅ ✅ +v0.1.2 37258746 ✅ ✅ +v0.2.0 38108519 ✅ ✅ +v1.0.0-rc1 41510961 ✅ ✅ +v1.0.0-rc2 42710781 ✅ ✅ +v1.0.0-rc3 43267373 ✅ ✅ +v1.0.0 43930254 ✅ ✅ +``` + +The output shows all versions present in SumDB, and for each: + - INDEX is the leaf index of this `module@version` in SumDB + - FOUND shows that a tag with the same version string was found in the git version + - go.mod shows that the hashes for the `go.mod` file match. In addition to the green tick, there are two other states: + - ⚠️: no `go.mod` file was found in the git repo at the tagged version; this _could_ be a release from before modules were adopted + - ❌: a `go.mod` file was found in the git repo, but the hash doesn't match that in SumDB. Either the tag was changed, or SumDB is hosting bad content. + +> [!IMPORTANT] +> This tool does NOT yet check that the zip file hash matches. + diff --git a/vindex/cmd/sumdbverify/client.go b/vindex/cmd/sumdbverify/client.go index 7d48baf..906bfff 100644 --- a/vindex/cmd/sumdbverify/client.go +++ b/vindex/cmd/sumdbverify/client.go @@ -18,19 +18,26 @@ package main import ( + "bytes" "context" + "crypto/sha256" "encoding/base64" "errors" "flag" "fmt" + "io" "net/http" "net/url" "os" - "os/exec" "path/filepath" "regexp" + "strconv" "strings" + "text/tabwriter" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/transparency-dev/incubator/vindex/client" "golang.org/x/mod/modfile" "golang.org/x/mod/semver" @@ -72,47 +79,114 @@ func run(ctx context.Context) error { if *modRoot == "" { return errors.New("mod_root flag must be provided") } - if s, err := os.Stat(*modRoot); err != nil || !s.IsDir() { - return errors.New("mod_root flag must be a directory") + + // TODO(mhutchinson): Support a non-VIndex version of this that reads the non-verifiable proxy endpoints: + // 1) https://proxy.golang.org/github.com/transparency-dev/tessera/@v/list + // 2) https://sum.golang.org/lookup/github.com/transparency-dev/tessera@v1.0.0 + // This will provide a way to use this tool before the VIndex is widely available + sumFetcher := func(ctx context.Context, modName string, versions map[string]modData) error { + vic := newVIndexClientFromFlags() + + vs, err := queryIndex(ctx, vic, modName) + if err != nil { + return fmt.Errorf("error querying index: %v", err) + } + for k, v := range vs { + versions[k] = v + } + return nil + } + + report, reportErr := getReport(ctx, *modRoot, sumFetcher) + if reportErr != nil && len(report.versions) == 0 { + return fmt.Errorf("failed to compile report: %v", reportErr) + } + + fmt.Println(report.modName) + tw := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0) + if _, err := fmt.Fprintln(tw, "VERSION\tINDEX\tFOUND\tgo.mod\t"); err != nil { + return fmt.Errorf("failed to output report: %v", err) } - modPathBytes, err := os.ReadFile(filepath.Join(*modRoot, "go.mod")) + for _, v := range report.versions { + var sumIndex string + if v.sumFound { + sumIndex = strconv.FormatUint(v.sumIndex, 10) + } else { + sumIndex = "--" + } + + gitHash := "✅" + if v.gitCommitHash == nil { + gitHash = "❌" + } + + goMod := "✅" + if v.gitCommitHash == nil || !v.gitGoMod { + goMod = "⚠️" + } else if !bytes.Equal(v.gitModHash, v.sumModHash) { + goMod = "❌" + } + if _, err := fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t\n", v.version, sumIndex, gitHash, goMod); err != nil { + return fmt.Errorf("failed to output report: %v", err) + } + } + if err := tw.Flush(); err != nil { + return fmt.Errorf("failed to flush report to stdout: %v", err) + } + + return reportErr +} + +func getReport(ctx context.Context, modRoot string, sumFetcher func(context.Context, string, map[string]modData) error) (diffReport, error) { + if s, err := os.Stat(modRoot); err != nil || !s.IsDir() { + return diffReport{}, errors.New("mod_root flag must be a directory") + } + modPathBytes, err := os.ReadFile(filepath.Join(modRoot, "go.mod")) if err != nil { - return fmt.Errorf("failed to read go.mod file: %v", err) + return diffReport{}, fmt.Errorf("failed to read go.mod file: %v", err) } modPath, err := modfile.Parse("go.mod", modPathBytes, nil) if err != nil { - return fmt.Errorf("failed to parse go.mod file: %v", err) + return diffReport{}, fmt.Errorf("failed to parse go.mod file: %v", err) } modName := modPath.Module.Mod.Path + report := &diffReport{ + modName: modName, + versions: make([]versionReport, 0), + } + + repo, err := git.PlainOpen(modRoot) + if err != nil { + return *report, fmt.Errorf("directory at mod_root %q cannot be opened as git repo: %v", modRoot, err) + } eg, egctx := errgroup.WithContext(ctx) - var versions map[string]modData + versions := make(map[string]modData) var tags map[string]struct{} eg.Go(func() error { - // This function gets all version info from the verifiable index. - vic := newVIndexClientFromFlags() - - versions, err = queryIndex(egctx, vic, modName) - if err != nil { - return fmt.Errorf("error querying index: %v", err) - } - return nil + return sumFetcher(egctx, modName, versions) }) eg.Go(func() error { - // This function gets all tags from the local git checkout. - rawTags, err := queryTags(egctx, *modRoot) + refIter, err := repo.References() if err != nil { - return fmt.Errorf("error enumerating git tags: %v", err) + return fmt.Errorf("failed to list references: %v", err) } - tags = make(map[string]struct{}, len(rawTags)) - for _, t := range rawTags { - tags[t] = struct{}{} + + tags = make(map[string]struct{}) + if err := refIter.ForEach(func(ref *plumbing.Reference) error { + if ref.Name().IsTag() { + tagName := ref.Name().Short() + tags[tagName] = struct{}{} + } + return nil + }); err != nil { + return fmt.Errorf("failed to list tags: %v", err) } return nil }) if err := eg.Wait(); err != nil { - return err + return *report, err } // Create a sorted slice of versions @@ -122,26 +196,117 @@ func run(ctx context.Context) error { } semver.Sort(sv) - fmt.Println(modName) + vErrors := make([]error, 0) for _, v := range sv { - d := versions[v] - presence := "✅ found in git tags" - if _, found := tags[v]; !found { - presence = "❌ missing from git tags" + sumHashes := versions[v] + + vr, err := reportVersion(ctx, repo, v) + if err != nil { + vErrors = append(vErrors, fmt.Errorf("failed to get report for version %q: %v", v, err)) } + vr.sumFound = true + vr.sumModHash = sumHashes.modHash + vr.sumIndex = sumHashes.index + report.versions = append(report.versions, vr) + + if vr.gitCommitHash != nil { + delete(tags, v) + } + } + + // TODO(mhutchinson): Include information on tags in git that aren't in SumDB + // if len(tags) > 0 { + // for t := range tags { + // report.versions = append(report.versions, versionReport{ + // version: t, + // sumFound: false, + // // also include git info here + // }) + // } + // } + return *report, errors.Join(vErrors...) +} + +func reportVersion(ctx context.Context, repo *git.Repository, v string) (versionReport, error) { + report := &versionReport{ + version: v, + } - fmt.Printf("%s found at index %d: %s\n", v, d.index, presence) - delete(tags, v) + // Find the commit this tag points to. + ref := plumbing.NewTagReferenceName(v) + hash, err := repo.ResolveRevision(plumbing.Revision(ref)) + if err != nil { + return *report, fmt.Errorf("failed to resolve tag '%s' to a commit: %v", ref, err) } - if len(tags) > 0 { - fmt.Println("----------") - fmt.Println("> INFO: The tagged versions below were never downloaded via the Module Proxy") - for t := range tags { - fmt.Printf("%s found locally but missing from SumDB\n", t) + // Update report to include the sha1 commit hash the tag points to. + report.gitCommitHash = hash[:] + + // Find the go.mod file in this commit. + commit, err := repo.CommitObject(*hash) + if err != nil { + return *report, fmt.Errorf("failed to get commit object: %v", err) + } + tree, err := commit.Tree() + if err != nil { + return *report, fmt.Errorf("failed to get commit tree: %v", err) + } + modFile, err := tree.File("go.mod") + if err != nil { + if err == object.ErrFileNotFound { + // This isn't necessarily an error: old versions didn't have go.mod files + return *report, nil } + return *report, fmt.Errorf("go.mod not found: %v", err) } - return nil + + // Update report to note that we found a go.mod file. + report.gitGoMod = true + + // Compute the SumDB hash of the file (this requires a double hash). + blob, err := repo.BlobObject(modFile.Hash) + if err != nil { + return *report, fmt.Errorf("failed to get blob object: %v", err) + } + gmh := sha256.New() + br, err := blob.Reader() + if err != nil { + return *report, fmt.Errorf("failed to get blob reader: %v", err) + } + defer func() { + if err := br.Close(); err != nil { + klog.Warningf("failed to close blob reader: %v", err) + } + }() + if _, err := io.Copy(gmh, br); err != nil { + return *report, fmt.Errorf("failed to calculate hash: %v", err) + } + preimage := fmt.Sprintf("%x go.mod\n", gmh.Sum(nil)) + got := sha256.Sum256([]byte(preimage)) + report.gitModHash = got[:] + + return *report, nil +} + +type diffReport struct { + modName string + versions []versionReport +} + +type versionReport struct { + version string + + // These fields are written in-order, as further info available. + // If not commit hash present, then the version tag was not found in git. + gitCommitHash []byte + gitGoMod bool + gitModHash []byte + + // If sumFound is not set, then no entry was found in SumDB so fields below + // should not be referenced. + sumFound bool + sumIndex uint64 + sumModHash []byte } type modData struct { @@ -150,16 +315,6 @@ type modData struct { modHash []byte } -func queryTags(ctx context.Context, modRoot string) ([]string, error) { - cmd := exec.CommandContext(ctx, "git", "tag") - cmd.Dir = modRoot - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("failed to run git tag: %w", err) - } - return strings.Split(strings.TrimSpace(string(out)), "\n"), nil -} - func queryIndex(ctx context.Context, vic *client.VIndexClient, modName string) (map[string]modData, error) { idxes, inCp, err := vic.Lookup(ctx, modName) if err != nil { From 365f97a82bcfa86fa01986994f088206c9e02f38 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Wed, 1 Oct 2025 14:51:56 +0000 Subject: [PATCH 2/2] address comments --- vindex/cmd/sumdbverify/client.go | 105 ++++++++++++------------------- 1 file changed, 39 insertions(+), 66 deletions(-) diff --git a/vindex/cmd/sumdbverify/client.go b/vindex/cmd/sumdbverify/client.go index 906bfff..fe7cd2c 100644 --- a/vindex/cmd/sumdbverify/client.go +++ b/vindex/cmd/sumdbverify/client.go @@ -18,10 +18,7 @@ package main import ( - "bytes" "context" - "crypto/sha256" - "encoding/base64" "errors" "flag" "fmt" @@ -41,6 +38,7 @@ import ( "github.com/transparency-dev/incubator/vindex/client" "golang.org/x/mod/modfile" "golang.org/x/mod/semver" + "golang.org/x/mod/sumdb/dirhash" "golang.org/x/mod/sumdb/note" "golang.org/x/sync/errgroup" "k8s.io/klog/v2" @@ -57,8 +55,8 @@ var ( // golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= // golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= // - line0RE = regexp.MustCompile(`(.*) (.*) h1:(.*)`) - line1RE = regexp.MustCompile(`(.*) (.*)/go.mod h1:(.*)`) + line0RE = regexp.MustCompile(`(.*) (.*) (h1:.*)`) + line1RE = regexp.MustCompile(`(.*) (.*)/go.mod (h1:.*)`) ) func main() { @@ -84,17 +82,10 @@ func run(ctx context.Context) error { // 1) https://proxy.golang.org/github.com/transparency-dev/tessera/@v/list // 2) https://sum.golang.org/lookup/github.com/transparency-dev/tessera@v1.0.0 // This will provide a way to use this tool before the VIndex is widely available - sumFetcher := func(ctx context.Context, modName string, versions map[string]modData) error { + sumFetcher := func(ctx context.Context, modName string) (map[string]modData, error) { vic := newVIndexClientFromFlags() - vs, err := queryIndex(ctx, vic, modName) - if err != nil { - return fmt.Errorf("error querying index: %v", err) - } - for k, v := range vs { - versions[k] = v - } - return nil + return queryIndex(ctx, vic, modName) } report, reportErr := getReport(ctx, *modRoot, sumFetcher) @@ -121,9 +112,9 @@ func run(ctx context.Context) error { } goMod := "✅" - if v.gitCommitHash == nil || !v.gitGoMod { + if v.gitCommitHash == nil || len(v.gitModHash) == 0 { goMod = "⚠️" - } else if !bytes.Equal(v.gitModHash, v.sumModHash) { + } else if v.gitModHash != v.sumModHash { goMod = "❌" } if _, err := fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t\n", v.version, sumIndex, gitHash, goMod); err != nil { @@ -137,7 +128,7 @@ func run(ctx context.Context) error { return reportErr } -func getReport(ctx context.Context, modRoot string, sumFetcher func(context.Context, string, map[string]modData) error) (diffReport, error) { +func getReport(ctx context.Context, modRoot string, sumFetcher func(context.Context, string) (map[string]modData, error)) (diffReport, error) { if s, err := os.Stat(modRoot); err != nil || !s.IsDir() { return diffReport{}, errors.New("mod_root flag must be a directory") } @@ -151,21 +142,23 @@ func getReport(ctx context.Context, modRoot string, sumFetcher func(context.Cont } modName := modPath.Module.Mod.Path - report := &diffReport{ + report := diffReport{ modName: modName, versions: make([]versionReport, 0), } repo, err := git.PlainOpen(modRoot) if err != nil { - return *report, fmt.Errorf("directory at mod_root %q cannot be opened as git repo: %v", modRoot, err) + return report, fmt.Errorf("directory at mod_root %q cannot be opened as git repo: %v", modRoot, err) } eg, egctx := errgroup.WithContext(ctx) - versions := make(map[string]modData) + var versions map[string]modData var tags map[string]struct{} eg.Go(func() error { - return sumFetcher(egctx, modName, versions) + var err error + versions, err = sumFetcher(egctx, modName) + return err }) eg.Go(func() error { refIter, err := repo.References() @@ -186,7 +179,7 @@ func getReport(ctx context.Context, modRoot string, sumFetcher func(context.Cont return nil }) if err := eg.Wait(); err != nil { - return *report, err + return report, err } // Create a sorted slice of versions @@ -224,7 +217,7 @@ func getReport(ctx context.Context, modRoot string, sumFetcher func(context.Cont // }) // } // } - return *report, errors.Join(vErrors...) + return report, errors.Join(vErrors...) } func reportVersion(ctx context.Context, repo *git.Repository, v string) (versionReport, error) { @@ -251,39 +244,28 @@ func reportVersion(ctx context.Context, repo *git.Repository, v string) (version if err != nil { return *report, fmt.Errorf("failed to get commit tree: %v", err) } - modFile, err := tree.File("go.mod") + hs, err := dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) { + modFile, err := tree.File("go.mod") + if err != nil { + return nil, err + } + + blob, err := repo.BlobObject(modFile.Hash) + if err != nil { + return nil, fmt.Errorf("failed to get blob object: %v", err) + } + return blob.Reader() + }) if err != nil { - if err == object.ErrFileNotFound { + if errors.Is(err, object.ErrFileNotFound) { // This isn't necessarily an error: old versions didn't have go.mod files return *report, nil } - return *report, fmt.Errorf("go.mod not found: %v", err) + return *report, fmt.Errorf("failed to calculate file hash: %v", err) } // Update report to note that we found a go.mod file. - report.gitGoMod = true - - // Compute the SumDB hash of the file (this requires a double hash). - blob, err := repo.BlobObject(modFile.Hash) - if err != nil { - return *report, fmt.Errorf("failed to get blob object: %v", err) - } - gmh := sha256.New() - br, err := blob.Reader() - if err != nil { - return *report, fmt.Errorf("failed to get blob reader: %v", err) - } - defer func() { - if err := br.Close(); err != nil { - klog.Warningf("failed to close blob reader: %v", err) - } - }() - if _, err := io.Copy(gmh, br); err != nil { - return *report, fmt.Errorf("failed to calculate hash: %v", err) - } - preimage := fmt.Sprintf("%x go.mod\n", gmh.Sum(nil)) - got := sha256.Sum256([]byte(preimage)) - report.gitModHash = got[:] + report.gitModHash = hs return *report, nil } @@ -296,23 +278,22 @@ type diffReport struct { type versionReport struct { version string - // These fields are written in-order, as further info available. - // If not commit hash present, then the version tag was not found in git. + // These fields are written in-order, as further info becomes available. + // If no commit hash is present, then the version tag was not found in git. gitCommitHash []byte - gitGoMod bool - gitModHash []byte + gitModHash string // If sumFound is not set, then no entry was found in SumDB so fields below // should not be referenced. sumFound bool sumIndex uint64 - sumModHash []byte + sumModHash string } type modData struct { index uint64 - zipHash []byte - modHash []byte + zipHash string + modHash string } func queryIndex(ctx context.Context, vic *client.VIndexClient, modName string) (map[string]modData, error) { @@ -361,18 +342,10 @@ func parseLeaf(idx uint64, data []byte) (string, modData, error) { return "", modData{}, fmt.Errorf("mismatched version names: (%s, %s)", line0Version, line1Version) } - zipHash, err := base64.StdEncoding.DecodeString(zipHashB64) - if err != nil { - return "", modData{}, fmt.Errorf("failed to decode hash %q: %v", zipHashB64, err) - } - modHash, err := base64.StdEncoding.DecodeString(modHashB64) - if err != nil { - return "", modData{}, fmt.Errorf("failed to decode hash %q: %v", modHashB64, err) - } return line0Version, modData{ index: idx, - zipHash: zipHash, - modHash: modHash, + zipHash: zipHashB64, + modHash: modHashB64, }, nil }