From d4ceb67465a0fe19b10c77f72fd91a7354663291 Mon Sep 17 00:00:00 2001 From: Alexey Sharov Date: Mon, 17 Aug 2026 13:31:48 +0700 Subject: [PATCH] crypto: limit sha1 or sha512 blocks processed at once in assembly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem GC stop-the-world is slow if a large input is passed to sha1/sha512: worst stopping pause 67 ms for sha512 and 25 ms for sha1 on a 64 MiB buffer. Every goroutine in the process is frozen for that window. Root cause: Write passes unbounded input to block(), and assembly is not preemptible. CL 671098 bounded md5 and sha256 for #64417; sha1 and sha512 have the same problem and were not covered. Write now feeds block() at most maxAsmSize (64 KiB) per call, as md5 and sha256 already do. Nothing else changes: on the arches with assembly, block() is an ordinary Go function dispatching to blockAVX2/blockSHANI, so its prologue already carries the stack check the runtime poisons to request preemption. Calling it 1024 times instead of once is what lets the collector in. Stop-the-world (improved) Worst /sched/pauses/stopping/gc:seconds sample, which is the time the runtime spends waiting for every P to halt, while hashing a 64 MiB buffer. linux/amd64, EPYC 4344P, GOMAXPROCS=2. Values are histogram bucket upper bounds, so read them as "at most": before after pauses over 1ms crypto/sha1 25.166ms 197us 2 -> 0 crypto/sha512 67.109ms 1.049ms 2 -> 0 The pause is linear in what a single call is handed, about 0.4 ms of stop-the-world per MiB for sha1 and 1.0 ms per MiB for sha512, so the bound caps it rather than removing it. BenchmarkSTW, added here, reports the same effect as the wall time of one runtime.GC() that overlaps the hash: GOMAXPROCS=2 go test -run='^$' -bench=BenchmarkSTW -count=6 -benchtime=2s crypto/sha1 crypto/sha512 benchstat before.txt after.txt gcwait-sec/op gcwait-sec/op vs base sha1 24988.7µ ± 0% 319.3µ ± 13% -98.72% (p=0.002 n=6) sha512 61965.8µ ± 0% 475.8µ ± 60% -99.23% (p=0.002 n=6) The before column is flat at 0% because it is not measuring the collector at all, it is measuring the hash. The variance that appears afterwards is the collector's own work. The benchmark counts an iteration only if the hash had started before the collection finished, so a sample that overlapped nothing cannot skew the mean. Throughput (no degradations) The existing benchmarks stop at 8 KiB, below the bound, so Hash256K and Hash1M are added. That required benchmarkSize to allocate its own buffer rather than slice a fixed 8192-byte package var, matching crypto/sha256. Sampled across 8 separate process instances per side. Hash8Bytes is bimodal per-process, landing at either ~57.6ns or ~62ns depending on heap alignment in both the old and the new binary, so a single process per side attributes that coin flip to the change. for i in $(seq 8); do GOMAXPROCS=2 go test -run='^$' -bench=BenchmarkHash -count=3 -benchtime=200ms crypto/sha1 crypto/sha512 done benchstat before.txt after.txt sha1 sec/op sec/op vs base Hash8Bytes/New-2 57.70n ± 6% 57.78n ± 5% ~ (p=0.649 n=24) Hash8Bytes/Sum-2 52.50n ± 0% 52.46n ± 0% ~ (p=0.770 n=24) Hash320Bytes/New-2 174.3n ± 0% 174.3n ± 0% ~ (p=0.888 n=24) Hash320Bytes/Sum-2 171.6n ± 0% 171.7n ± 0% ~ (p=0.368 n=24) Hash1K/New-2 433.8n ± 0% 433.9n ± 0% ~ (p=0.922 n=24) Hash1K/Sum-2 432.1n ± 0% 432.0n ± 0% ~ (p=0.988 n=24) Hash8K/New-2 3.083µ ± 0% 3.083µ ± 0% ~ (p=0.896 n=24) Hash8K/Sum-2 3.083µ ± 0% 3.082µ ± 0% ~ (p=0.938 n=24) Hash256K/New-2 96.96µ ± 0% 96.94µ ± 0% ~ (p=0.657 n=24) Hash256K/Sum-2 96.90µ ± 0% 96.96µ ± 0% ~ (p=0.092 n=24) Hash1M/New-2 387.9µ ± 0% 388.0µ ± 0% ~ (p=0.214 n=24) Hash1M/Sum-2 387.8µ ± 0% 387.9µ ± 0% ~ (p=0.428 n=24) geomean 2.796µ 2.796µ +0.01% sha512 sec/op sec/op vs base Hash8Bytes/New-2 149.4n ± 0% 149.3n ± 0% ~ (p=0.849 n=24) Hash8Bytes/Sum384-2 157.6n ± 0% 158.0n ± 0% +0.25% (p=0.003 n=24) Hash8Bytes/Sum512-2 157.5n ± 0% 157.8n ± 0% ~ (p=0.106 n=24) Hash1K/New-2 1.082µ ± 0% 1.084µ ± 0% ~ (p=0.053 n=24) Hash1K/Sum384-2 1.089µ ± 0% 1.089µ ± 0% ~ (p=0.329 n=24) Hash1K/Sum512-2 1.093µ ± 0% 1.091µ ± 0% ~ (p=0.538 n=24) Hash8K/New-2 7.655µ ± 0% 7.650µ ± 0% -0.07% (p=0.045 n=24) Hash8K/Sum384-2 7.668µ ± 0% 7.652µ ± 0% -0.21% (p=0.000 n=24) Hash8K/Sum512-2 7.660µ ± 0% 7.649µ ± 0% -0.14% (p=0.000 n=24) Hash256K/New-2 240.5µ ± 0% 239.9µ ± 0% ~ (p=0.173 n=24) Hash256K/Sum384-2 240.7µ ± 0% 240.0µ ± 0% -0.32% (p=0.024 n=24) Hash256K/Sum512-2 240.2µ ± 0% 240.0µ ± 0% ~ (p=0.136 n=24) Hash1M/New-2 960.8µ ± 0% 962.4µ ± 0% ~ (p=0.519 n=24) Hash1M/Sum384-2 961.2µ ± 0% 961.0µ ± 0% ~ (p=0.814 n=24) Hash1M/Sum512-2 960.8µ ± 0% 960.5µ ± 0% ~ (p=0.878 n=24) geomean 12.44µ 12.44µ -0.04% Nothing moves outside noise, in either direction, at any size. Updates #64417 --- src/crypto/internal/fips140/sha512/sha512.go | 10 ++++ src/crypto/sha1/sha1.go | 10 ++++ src/crypto/sha1/sha1_test.go | 58 ++++++++++++++++++- src/crypto/sha512/sha512_test.go | 60 ++++++++++++++++++-- 4 files changed, 131 insertions(+), 7 deletions(-) diff --git a/src/crypto/internal/fips140/sha512/sha512.go b/src/crypto/internal/fips140/sha512/sha512.go index 3e7a5e11f15198..79bd61a4e39ab6 100644 --- a/src/crypto/internal/fips140/sha512/sha512.go +++ b/src/crypto/internal/fips140/sha512/sha512.go @@ -31,6 +31,11 @@ const ( blockSize = 128 ) +// The maximum number of bytes that can be passed to block(). The limit exists +// because implementations that rely on assembly routines are not preemptible. +const maxAsmIters = 512 +const maxAsmSize = blockSize * maxAsmIters // 64KiB + const ( chunk = 128 init0 = 0x6a09e667f3bcc908 @@ -248,6 +253,11 @@ func (d *Digest) Write(p []byte) (nn int, err error) { } if len(p) >= chunk { n := len(p) &^ (chunk - 1) + for n > maxAsmSize { + block(d, p[:maxAsmSize]) + p = p[maxAsmSize:] + n -= maxAsmSize + } block(d, p[:n]) p = p[n:] } diff --git a/src/crypto/sha1/sha1.go b/src/crypto/sha1/sha1.go index 46e47df1d32cf2..4e117201485bf3 100644 --- a/src/crypto/sha1/sha1.go +++ b/src/crypto/sha1/sha1.go @@ -27,6 +27,11 @@ const Size = 20 // The blocksize of SHA-1 in bytes. const BlockSize = 64 +// The maximum number of bytes that can be passed to block(). The limit exists +// because implementations that rely on assembly routines are not preemptible. +const maxAsmIters = 1024 +const maxAsmSize = BlockSize * maxAsmIters // 64KiB + const ( chunk = 64 init0 = 0x67452301 @@ -143,6 +148,11 @@ func (d *digest) Write(p []byte) (nn int, err error) { } if len(p) >= chunk { n := len(p) &^ (chunk - 1) + for n > maxAsmSize { + block(d, p[:maxAsmSize]) + p = p[maxAsmSize:] + n -= maxAsmSize + } block(d, p[:n]) p = p[n:] } diff --git a/src/crypto/sha1/sha1_test.go b/src/crypto/sha1/sha1_test.go index 8b4618df556271..745dce5c9ff158 100644 --- a/src/crypto/sha1/sha1_test.go +++ b/src/crypto/sha1/sha1_test.go @@ -14,7 +14,10 @@ import ( "fmt" "hash" "io" + "runtime" + "sync/atomic" "testing" + "time" ) type sha1Test struct { @@ -267,16 +270,16 @@ func TestOutOfBoundsRead(t *testing.T) { } var bench = New() -var buf = make([]byte, 8192) func benchmarkSize(b *testing.B, size int) { + buf := make([]byte, size) sum := make([]byte, bench.Size()) b.Run("New", func(b *testing.B) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { bench.Reset() - bench.Write(buf[:size]) + bench.Write(buf) bench.Sum(sum[:0]) } }) @@ -284,7 +287,7 @@ func benchmarkSize(b *testing.B, size int) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { - Sum(buf[:size]) + Sum(buf) } }) } @@ -304,3 +307,52 @@ func BenchmarkHash1K(b *testing.B) { func BenchmarkHash8K(b *testing.B) { benchmarkSize(b, 8192) } + +func BenchmarkHash256K(b *testing.B) { + benchmarkSize(b, 256*1024) +} + +func BenchmarkHash1M(b *testing.B) { + benchmarkSize(b, 1024*1024) +} + +var sinkSTW []byte + +// BenchmarkSTW reports how long a garbage collection had to wait while a hash +// ran alongside it, as gcwait-ns/op. Assembly is not preemptible, so a call +// that covers the whole input blocks every goroutine in the process for as +// long as it runs; bounding the call gives the collector a way in between +// chunks. Run with GOMAXPROCS>=2 so the two actually overlap. +func BenchmarkSTW(b *testing.B) { + buf := make([]byte, 64<<20) + var total time.Duration + var iters int + b.SetBytes(int64(len(buf))) + for b.Loop() { + done := make(chan struct{}) + var began atomic.Int64 + go func() { + defer close(done) + began.Store(time.Now().UnixNano()) + h := New() + h.Write(buf) + sinkSTW = h.Sum(nil) + }() + start := time.Now() + runtime.GC() // one per iteration, so the mean is well defined + end := time.Now() + <-done + + // Count the iteration only if the hash had started before the + // collection finished. Otherwise there was nothing to overlap and the + // sample says nothing about preemptibility. + if t := began.Load(); t != 0 && t < end.UnixNano() { + total += end.Sub(start) + iters++ + } + } + if iters == 0 { + b.Skip("no iteration overlapped a collection") + } + b.ReportMetric(float64(total.Nanoseconds())/float64(iters), "gcwait-ns/op") +} diff --git a/src/crypto/sha512/sha512_test.go b/src/crypto/sha512/sha512_test.go index 080bf694f03652..214b8686db9ffe 100644 --- a/src/crypto/sha512/sha512_test.go +++ b/src/crypto/sha512/sha512_test.go @@ -14,7 +14,10 @@ import ( "fmt" "hash" "io" + "runtime" + "sync/atomic" "testing" + "time" ) type sha512Test struct { @@ -998,16 +1001,16 @@ func maybeCloner(h hash.Hash) any { } var bench = New() -var buf = make([]byte, 8192) func benchmarkSize(b *testing.B, size int) { + buf := make([]byte, size) sum := make([]byte, bench.Size()) b.Run("New", func(b *testing.B) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { bench.Reset() - bench.Write(buf[:size]) + bench.Write(buf) bench.Sum(sum[:0]) } }) @@ -1015,14 +1018,14 @@ func benchmarkSize(b *testing.B, size int) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { - Sum384(buf[:size]) + Sum384(buf) } }) b.Run("Sum512", func(b *testing.B) { b.ReportAllocs() b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { - Sum512(buf[:size]) + Sum512(buf) } }) } @@ -1038,3 +1041,52 @@ func BenchmarkHash1K(b *testing.B) { func BenchmarkHash8K(b *testing.B) { benchmarkSize(b, 8192) } + +func BenchmarkHash256K(b *testing.B) { + benchmarkSize(b, 256*1024) +} + +func BenchmarkHash1M(b *testing.B) { + benchmarkSize(b, 1024*1024) +} + +var sinkSTW []byte + +// BenchmarkSTW reports how long a garbage collection had to wait while a hash +// ran alongside it, as gcwait-ns/op. Assembly is not preemptible, so a call +// that covers the whole input blocks every goroutine in the process for as +// long as it runs; bounding the call gives the collector a way in between +// chunks. Run with GOMAXPROCS>=2 so the two actually overlap. +func BenchmarkSTW(b *testing.B) { + buf := make([]byte, 64<<20) + var total time.Duration + var iters int + b.SetBytes(int64(len(buf))) + for b.Loop() { + done := make(chan struct{}) + var began atomic.Int64 + go func() { + defer close(done) + began.Store(time.Now().UnixNano()) + h := New() + h.Write(buf) + sinkSTW = h.Sum(nil) + }() + start := time.Now() + runtime.GC() // one per iteration, so the mean is well defined + end := time.Now() + <-done + + // Count the iteration only if the hash had started before the + // collection finished. Otherwise there was nothing to overlap and the + // sample says nothing about preemptibility. + if t := began.Load(); t != 0 && t < end.UnixNano() { + total += end.Sub(start) + iters++ + } + } + if iters == 0 { + b.Skip("no iteration overlapped a collection") + } + b.ReportMetric(float64(total.Nanoseconds())/float64(iters), "gcwait-ns/op") +}