From 45c73beac488b05525f88240c6ebfec7c0758697 Mon Sep 17 00:00:00 2001 From: Alexey Sharov Date: Mon, 17 Aug 2026 13:31:48 +0700 Subject: [PATCH] crypto: limit sha1 or sha512 blocks processed at once in assembly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem GC stop-the-world is slow if a large input is passed to sha1/sha512 Root cause: Write passes unbounded input to block(), and assembly is not preemptible. CL 671098 bounded md5 and sha256 for #64417; sha1 and sha512 have the same problem and were not covered. Stop-the-world (improved) /sched/pauses/stopping/gc:seconds, the time the runtime spends waiting for every P to halt, worst pause while hashing a 64 MiB buffer. linux/amd64, EPYC 4344P, GOMAXPROCS=2 before after pauses >1ms crypto/sha1 25.166ms 197us 2 -> 0 crypto/sha512 67.109ms 1.049ms 2 -> 0 BenchmarkSTW, added here, reports the same effect as the wall time of one runtime.GC() that overlaps the hash: ``` GOMAXPROCS=2 go test -run='^$' -bench=BenchmarkSTW -count=6 -benchtime=2s crypto/sha1 crypto/sha512 benchstat before.txt after.txt │ gcwait-sec/op │ gcwait-sec/op vs base │ sha1 24988.7µ ± 0% 319.3µ ± 13% -98.72% (p=0.002 n=6) sha512 61965.8µ ± 0% 475.8µ ± 60% -99.23% (p=0.002 n=6) ``` Throughput (no degradations) ``` for i in $(seq 8); do GOMAXPROCS=2 go test -run='^$' -bench=BenchmarkHash -count=3 -benchtime=200ms crypto/sha1 crypto/sha512 done benchstat before.txt after.txt sha1 sec/op sec/op vs base Hash8Bytes/New-2 57.70n ± 6% 57.78n ± 5% ~ (p=0.649 n=24) Hash8Bytes/Sum-2 52.50n ± 0% 52.46n ± 0% ~ (p=0.770 n=24) Hash320Bytes/New-2 174.3n ± 0% 174.3n ± 0% ~ (p=0.888 n=24) Hash320Bytes/Sum-2 171.6n ± 0% 171.7n ± 0% ~ (p=0.368 n=24) Hash1K/New-2 433.8n ± 0% 433.9n ± 0% ~ (p=0.922 n=24) Hash1K/Sum-2 432.1n ± 0% 432.0n ± 0% ~ (p=0.988 n=24) Hash8K/New-2 3.083µ ± 0% 3.083µ ± 0% ~ (p=0.896 n=24) Hash8K/Sum-2 3.083µ ± 0% 3.082µ ± 0% ~ (p=0.938 n=24) Hash256K/New-2 96.96µ ± 0% 96.94µ ± 0% ~ (p=0.657 n=24) Hash256K/Sum-2 96.90µ ± 0% 96.96µ ± 0% ~ (p=0.092 n=24) Hash1M/New-2 387.9µ ± 0% 388.0µ ± 0% ~ (p=0.214 n=24) Hash1M/Sum-2 387.8µ ± 0% 387.9µ ± 0% ~ (p=0.428 n=24) geomean 2.796µ 2.796µ +0.01% sha512 sec/op sec/op vs base Hash8Bytes/New-2 149.4n ± 0% 149.3n ± 0% ~ (p=0.849 n=24) Hash8Bytes/Sum384-2 157.6n ± 0% 158.0n ± 0% +0.25% (p=0.003 n=24) Hash8Bytes/Sum512-2 157.5n ± 0% 157.8n ± 0% ~ (p=0.106 n=24) Hash1K/New-2 1.082µ ± 0% 1.084µ ± 0% ~ (p=0.053 n=24) Hash1K/Sum384-2 1.089µ ± 0% 1.089µ ± 0% ~ (p=0.329 n=24) Hash1K/Sum512-2 1.093µ ± 0% 1.091µ ± 0% ~ (p=0.538 n=24) Hash8K/New-2 7.655µ ± 0% 7.650µ ± 0% -0.07% (p=0.045 n=24) Hash8K/Sum384-2 7.668µ ± 0% 7.652µ ± 0% -0.21% (p=0.000 n=24) Hash8K/Sum512-2 7.660µ ± 0% 7.649µ ± 0% -0.14% (p=0.000 n=24) Hash256K/New-2 240.5µ ± 0% 239.9µ ± 0% ~ (p=0.173 n=24) Hash256K/Sum384-2 240.7µ ± 0% 240.0µ ± 0% -0.32% (p=0.024 n=24) Hash256K/Sum512-2 240.2µ ± 0% 240.0µ ± 0% ~ (p=0.136 n=24) Hash1M/New-2 960.8µ ± 0% 962.4µ ± 0% ~ (p=0.519 n=24) Hash1M/Sum384-2 961.2µ ± 0% 961.0µ ± 0% ~ (p=0.814 n=24) Hash1M/Sum512-2 960.8µ ± 0% 960.5µ ± 0% ~ (p=0.878 n=24) geomean 12.44µ 12.44µ -0.04% ``` References: - Go stdlib sha256, md5: https://github.com/golang/go/issues/64417 --- src/crypto/internal/fips140/sha512/sha512.go | 10 +++++ src/crypto/sha1/sha1.go | 10 +++++ src/crypto/sha1/sha1_test.go | 45 +++++++++++++++++-- src/crypto/sha512/sha512_test.go | 47 ++++++++++++++++++-- 4 files changed, 105 insertions(+), 7 deletions(-) diff --git a/src/crypto/internal/fips140/sha512/sha512.go b/src/crypto/internal/fips140/sha512/sha512.go index 3e7a5e11f15198..79bd61a4e39ab6 100644 --- a/src/crypto/internal/fips140/sha512/sha512.go +++ b/src/crypto/internal/fips140/sha512/sha512.go @@ -31,6 +31,11 @@ const ( blockSize = 128 ) +// The maximum number of bytes that can be passed to block(). The limit exists +// because implementations that rely on assembly routines are not preemptible. +const maxAsmIters = 512 +const maxAsmSize = blockSize * maxAsmIters // 64KiB + const ( chunk = 128 init0 = 0x6a09e667f3bcc908 @@ -248,6 +253,11 @@ func (d *Digest) Write(p []byte) (nn int, err error) { } if len(p) >= chunk { n := len(p) &^ (chunk - 1) + for n > maxAsmSize { + block(d, p[:maxAsmSize]) + p = p[maxAsmSize:] + n -= maxAsmSize + } block(d, p[:n]) p = p[n:] } diff --git a/src/crypto/sha1/sha1.go b/src/crypto/sha1/sha1.go index 46e47df1d32cf2..4e117201485bf3 100644 --- a/src/crypto/sha1/sha1.go +++ b/src/crypto/sha1/sha1.go @@ -27,6 +27,11 @@ const Size = 20 // The blocksize of SHA-1 in bytes. const BlockSize = 64 +// The maximum number of bytes that can be passed to block(). The limit exists +// because implementations that rely on assembly routines are not preemptible. +const maxAsmIters = 1024 +const maxAsmSize = BlockSize * maxAsmIters // 64KiB + const ( chunk = 64 init0 = 0x67452301 @@ -143,6 +148,11 @@ func (d *digest) Write(p []byte) (nn int, err error) { } if len(p) >= chunk { n := len(p) &^ (chunk - 1) + for n > maxAsmSize { + block(d, p[:maxAsmSize]) + p = p[maxAsmSize:] + n -= maxAsmSize + } block(d, p[:n]) p = p[n:] } diff --git a/src/crypto/sha1/sha1_test.go b/src/crypto/sha1/sha1_test.go index 8b4618df556271..7ab7fba078e9de 100644 --- a/src/crypto/sha1/sha1_test.go +++ b/src/crypto/sha1/sha1_test.go @@ -14,7 +14,9 @@ import ( "fmt" "hash" "io" + "runtime" "testing" + "time" ) type sha1Test struct { @@ -267,16 +269,16 @@ func TestOutOfBoundsRead(t *testing.T) { } var bench = New() -var buf = make([]byte, 8192) func benchmarkSize(b *testing.B, size int) { + buf := make([]byte, size) sum := make([]byte, bench.Size()) b.Run("New", func(b *testing.B) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { bench.Reset() - bench.Write(buf[:size]) + bench.Write(buf) bench.Sum(sum[:0]) } }) @@ -284,7 +286,7 @@ func benchmarkSize(b *testing.B, size int) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { - Sum(buf[:size]) + Sum(buf) } }) } @@ -304,3 +306,40 @@ func BenchmarkHash1K(b *testing.B) { func BenchmarkHash8K(b *testing.B) { benchmarkSize(b, 8192) } + +func BenchmarkHash256K(b *testing.B) { + benchmarkSize(b, 256*1024) +} + +func BenchmarkHash1M(b *testing.B) { + benchmarkSize(b, 1024*1024) +} + +var sinkSTW []byte + +// BenchmarkSTW reports how long a garbage collection had to wait while a hash +// ran alongside it, as gcwait-ns/op. Assembly is not preemptible, so a call +// that covers the whole input blocks every goroutine in the process for as +// long as it runs; bounding the call gives the collector a way in between +// chunks. Run with GOMAXPROCS>=2 so the two actually overlap. +func BenchmarkSTW(b *testing.B) { + buf := make([]byte, 64<<20) + var total time.Duration + var iters int + b.SetBytes(int64(len(buf))) + for b.Loop() { + done := make(chan struct{}) + go func() { + defer close(done) + h := New() + h.Write(buf) + sinkSTW = h.Sum(nil) + }() + start := time.Now() + runtime.GC() // one per iteration, so the mean is well defined + total += time.Since(start) + iters++ + <-done + } + b.ReportMetric(float64(total.Nanoseconds())/float64(iters), "gcwait-ns/op") +} diff --git a/src/crypto/sha512/sha512_test.go b/src/crypto/sha512/sha512_test.go index 080bf694f03652..a222a5f3ce4d43 100644 --- a/src/crypto/sha512/sha512_test.go +++ b/src/crypto/sha512/sha512_test.go @@ -14,7 +14,9 @@ import ( "fmt" "hash" "io" + "runtime" "testing" + "time" ) type sha512Test struct { @@ -998,16 +1000,16 @@ func maybeCloner(h hash.Hash) any { } var bench = New() -var buf = make([]byte, 8192) func benchmarkSize(b *testing.B, size int) { + buf := make([]byte, size) sum := make([]byte, bench.Size()) b.Run("New", func(b *testing.B) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { bench.Reset() - bench.Write(buf[:size]) + bench.Write(buf) bench.Sum(sum[:0]) } }) @@ -1015,14 +1017,14 @@ func benchmarkSize(b *testing.B, size int) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { - Sum384(buf[:size]) + Sum384(buf) } }) b.Run("Sum512", func(b *testing.B) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { - Sum512(buf[:size]) + Sum512(buf) } }) } @@ -1038,3 +1040,40 @@ func BenchmarkHash1K(b *testing.B) { func BenchmarkHash8K(b *testing.B) { benchmarkSize(b, 8192) } + +func BenchmarkHash256K(b *testing.B) { + benchmarkSize(b, 256*1024) +} + +func BenchmarkHash1M(b *testing.B) { + benchmarkSize(b, 1024*1024) +} + +var sinkSTW []byte + +// BenchmarkSTW reports how long a garbage collection had to wait while a hash +// ran alongside it, as gcwait-ns/op. Assembly is not preemptible, so a call +// that covers the whole input blocks every goroutine in the process for as +// long as it runs; bounding the call gives the collector a way in between +// chunks. Run with GOMAXPROCS>=2 so the two actually overlap. +func BenchmarkSTW(b *testing.B) { + buf := make([]byte, 64<<20) + var total time.Duration + var iters int + b.SetBytes(int64(len(buf))) + for b.Loop() { + done := make(chan struct{}) + go func() { + defer close(done) + h := New() + h.Write(buf) + sinkSTW = h.Sum(nil) + }() + start := time.Now() + runtime.GC() // one per iteration, so the mean is well defined + total += time.Since(start) + iters++ + <-done + } + b.ReportMetric(float64(total.Nanoseconds())/float64(iters), "gcwait-ns/op") +}