diff --git a/src/internal/runtime/maps/export_test.go b/src/internal/runtime/maps/export_test.go index adce44ba935085..43d870b7f7faa4 100644 --- a/src/internal/runtime/maps/export_test.go +++ b/src/internal/runtime/maps/export_test.go @@ -22,6 +22,11 @@ const MaxAvgGroupLoad = maxAvgGroupLoad // we can't properly test hint alloc overflows with this. const maxAllocTest = 1 << 30 +// Do not convert to var-of-funcval, these are used in benchmarks and must be +// inlined. +func MemHashAES(p unsafe.Pointer, h, s uintptr) uintptr { return memHashAES(p, h, s) } +func MemHashFallback(p unsafe.Pointer, h, s uintptr) uintptr { return memHashFallback(p, h, s) } + func newTestMapType[K comparable, V any]() *abi.MapType { var m map[K]V mTyp := abi.TypeOf(m) diff --git a/src/internal/runtime/maps/memhash_aes.go b/src/internal/runtime/maps/memhash_aes.go index 8893cdd35b55ef..ad292535a9aa7a 100644 --- a/src/internal/runtime/maps/memhash_aes.go +++ b/src/internal/runtime/maps/memhash_aes.go @@ -13,21 +13,21 @@ import ( const memHashAESImplemented = true func MemHash(p unsafe.Pointer, h, s uintptr) uintptr { - if UseAeshash { + if s >= MinAeshashSize { return memHashAES(p, h, s) } return memHashFallback(p, h, s) } func MemHash32(k uint32, h uintptr) uintptr { - if UseAeshash { + if useAeshash32 { return memHash32AES(k, h) } return memHash32Fallback(k, h) } func MemHash64(k uint64, h uintptr) uintptr { - if UseAeshash { + if useAeshash64 { return memHash64AES(k, h) } return memHash64Fallback(k, h) diff --git a/src/internal/runtime/maps/memhash_bench_test.go b/src/internal/runtime/maps/memhash_bench_test.go new file mode 100644 index 00000000000000..97b413fe1e4d7b --- /dev/null +++ b/src/internal/runtime/maps/memhash_bench_test.go @@ -0,0 +1,91 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build amd64 || arm64 + +package maps_test + +import ( + "fmt" + "testing" + "unsafe" + + "internal/runtime/maps" +) + +var sink uintptr + +// BenchmarkHashBakeoff measures the AES and scalar memory hashers at +// various sizes to try to empirically determine when one becomes better than +// the other, for some target uarch. Results are very uarch-dependent! +// +// The datapoints should be compared something like benchstat which uses the +// appropriate statistical tests to knock out outliers. +// +// Latency (i.e., serial pipeline performance) matters for probing, because +// there is a data dependency between the hash and the probe sequence. We can +// measure this by making each iteration of the benchmark depend on the previous +// one. This tends to favor scalar-only hashing more. +// +// Throughput (i.e., how long matters when many independent things are being +// hashed, resulting in better IPC. We measure this by using a seed of 0 for +// each iteration. This tends to favor AES more. +// +// Conservatively, we treat throughput as more important. However, more study +// is needed to determine if prioritizing latency (and thus picking a higher +// cutoff, such as MinLen = 112 on Zen4) results in better macrobenchmarks. +func BenchmarkHashBakeoff(b *testing.B) { + if !maps.AeshashEnabled() { + b.Skip("AES hashing not available on this machine") + } + + buf := make([]byte, 1024+8) + for i := range buf { + buf[i] = byte(i * 63) + } + p := unsafe.Pointer(unsafe.SliceData(buf)) + + var sizes = []uintptr{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 100, 104, 108, 112, 116, + 120, 124, 128, 192, 256, 512, 1024, + } + + for _, s := range sizes { + b.Run(fmt.Sprintf("scalar/latency/%d", s), func(b *testing.B) { + var h uintptr + for b.Loop() { + h = maps.MemHashFallback(p, h, s) + } + sink = h + }) + } + for _, s := range sizes { + b.Run(fmt.Sprintf("scalar/throughput/%d", s), func(b *testing.B) { + var h uintptr + for b.Loop() { + h ^= maps.MemHashFallback(p, 0, s) + } + sink = h + }) + } + for _, s := range sizes { + b.Run(fmt.Sprintf("aes/latency/%d", s), func(b *testing.B) { + var h uintptr + for b.Loop() { + h = maps.MemHashAES(p, h, s) + } + sink = h + }) + } + for _, s := range sizes { + b.Run(fmt.Sprintf("aes/throughput/%d", s), func(b *testing.B) { + var h uintptr + for b.Loop() { + h ^= maps.MemHashAES(p, 0, s) + } + sink = h + }) + } +} diff --git a/src/internal/runtime/maps/runtime_alg.go b/src/internal/runtime/maps/runtime_alg.go index 1c80e7b9c1b537..cfe4a56001438a 100644 --- a/src/internal/runtime/maps/runtime_alg.go +++ b/src/internal/runtime/maps/runtime_alg.go @@ -11,10 +11,27 @@ import ( "unsafe" ) -// runtime variable to check if the processor we're running on -// actually supports the instructions used by the AES-based -// hash implementation. -var UseAeshash bool +// MinAeshashSize is the smallest key size hashed with the AES-based +// implementation. Selecting hash is a size comparison against this value. +// Setting this to MaxUintptr disables AES altogether. +// +// When support is detected, the threshold is lowered to select between the +// scalar-based fallback hash and the vector-based AES hash. This value is +// selected on a per-platform basis based on what value produces the best +// benchmark results. +// +// Scalar hashes are faster on small values because it avoids taking a trip +// into the vector unit, which hurts latency (and for very small values, +// throughput). +var MinAeshashSize uintptr = ^uintptr(0) + +// AeshashEnabled reports whether this machine hashes any sizes with AES. +// +// Test-only; compare against MinAeshashSize in non-test code to fuse this +// comparison with the MinAeshashSize check. +func AeshashEnabled() bool { + return MinAeshashSize != ^uintptr(0) +} const hashRandomBytes = goarch.PtrSize / 4 * 64 @@ -24,6 +41,13 @@ var aeskeysched [hashRandomBytes]byte // used in hash{32,64}.go to seed the hash function var hashkey [4]uintptr +// Pre-computed comparisons against MinAeshashSize, which reduces a +// load-and-compare-and-branch to a load-and-branch. +var ( + useAeshash32 bool // = MinAeshashSize <= 4 + useAeshash64 bool // = MinAeshashSize <= 8 +) + func AlgInit() { // Always intialize hashkey. // @@ -46,23 +70,32 @@ func AlgInit() { } initAlgAES() - if memHashUsesVAES { + if memHashUsesVAES && !cpu.X86.HasAVX { // We are using intrinsics hash implementation. // Override the UseAeshash in this case, since it uses VAES (AVX) instructions. // While assembly implementation used AES-NI instructions, // simd intrinsics only provide access to AVX ones. - UseAeshash = cpu.X86.HasAVX + MinAeshashSize = ^uintptr(0) } - return - } - if goarch.GOARCH == "arm64" && cpu.ARM64.HasAES { + } else if goarch.GOARCH == "arm64" && cpu.ARM64.HasAES { initAlgAES() - return } + + useAeshash32 = 4 >= MinAeshashSize + useAeshash64 = 8 >= MinAeshashSize } func initAlgAES() { - UseAeshash = true + // TODO(mcy): investigate cutoffs on a per-uarch basis. + // See memhash_bench_test.go. + switch goarch.ArchFamily { + case goarch.AMD64: + // Measured on AMD Ryzen Threadripper PRO 7995WX (Zen4). + MinAeshashSize = 9 + default: + MinAeshashSize = 0 + } + // Initialize with random data so hash collisions will be hard to engineer. key := (*[hashRandomBytes / 8]uint64)(unsafe.Pointer(&aeskeysched)) for i := range key { diff --git a/src/internal/runtime/maps/runtime_fast32.go b/src/internal/runtime/maps/runtime_fast32.go index 66666823250319..b11d71d9901c11 100644 --- a/src/internal/runtime/maps/runtime_fast32.go +++ b/src/internal/runtime/maps/runtime_fast32.go @@ -69,9 +69,9 @@ func runtime_mapaccess2_fast32(typ *abi.MapType, m *Map, key uint32) (unsafe.Poi // But when we are using intrinsic implementation we want it to be inlined, // since it improves performance. // - // Note: memHashAESImplemented is compile time constant. We use it to remove runtime UseAeshash check + // Note: memHashAESImplemented is compile time constant. We use it to remove the useAeshash32 check // for architectures where we don't have AES hashing implementations. - if memHashAESImplemented && UseAeshash { + if memHashAESImplemented && useAeshash32 { hash = memHash32AES(key, m.seed) } else { hash = memHash32Fallback(key, m.seed) @@ -199,7 +199,7 @@ func runtime_mapassign_fast32(typ *abi.MapType, m *Map, key uint32) unsafe.Point var hash uintptr // See the related comment in runtime_mapaccess2_fast32 - if memHashAESImplemented && UseAeshash { + if memHashAESImplemented && useAeshash32 { hash = memHash32AES(key, m.seed) } else { hash = memHash32Fallback(key, m.seed) @@ -348,7 +348,7 @@ func runtime_mapassign_fast32ptr(typ *abi.MapType, m *Map, key unsafe.Pointer) u var hash uintptr // See the related comment in runtime_mapaccess2_fast32 - if memHashAESImplemented && UseAeshash { + if memHashAESImplemented && useAeshash32 { hash = memHash32AES(uint32((uintptr)(key)), m.seed) } else { hash = memHash32Fallback(uint32((uintptr)(key)), m.seed) diff --git a/src/internal/runtime/maps/runtime_fast64.go b/src/internal/runtime/maps/runtime_fast64.go index c76dc28130c210..081615e6d8b28d 100644 --- a/src/internal/runtime/maps/runtime_fast64.go +++ b/src/internal/runtime/maps/runtime_fast64.go @@ -65,7 +65,7 @@ func runtime_mapaccess2_fast64(typ *abi.MapType, m *Map, key uint64) (unsafe.Poi var hash uintptr // See the related comment in runtime_mapaccess2_fast32 - if memHashAESImplemented && UseAeshash { + if memHashAESImplemented && useAeshash64 { hash = memHash64AES(key, m.seed) } else { hash = memHash64Fallback(key, m.seed) @@ -194,7 +194,7 @@ func runtime_mapassign_fast64(typ *abi.MapType, m *Map, key uint64) unsafe.Point var hash uintptr // See the related comment in runtime_mapaccess2_fast32 - if memHashAESImplemented && UseAeshash { + if memHashAESImplemented && useAeshash64 { hash = memHash64AES(key, m.seed) } else { hash = memHash64Fallback(key, m.seed) @@ -412,7 +412,7 @@ func runtime_mapassign_fast64ptr(typ *abi.MapType, m *Map, key unsafe.Pointer) u var hash uintptr // See the related comment in runtime_mapaccess2_fast32 - if memHashAESImplemented && UseAeshash { + if memHashAESImplemented && useAeshash64 { hash = memHash64AES(uint64((uintptr)(key)), m.seed) } else { hash = memHash64Fallback(uint64((uintptr)(key)), m.seed) diff --git a/src/internal/runtime/maps/runtime_faststr.go b/src/internal/runtime/maps/runtime_faststr.go index 85cdccdf532170..8cedccc7a994f6 100644 --- a/src/internal/runtime/maps/runtime_faststr.go +++ b/src/internal/runtime/maps/runtime_faststr.go @@ -66,7 +66,7 @@ dohash: // This path will cost 1 hash and 1+ε comparisons. var hash uintptr // See the related comment in runtime_mapaccess2_fast32 - if memHashAESImplemented && UseAeshash { + if memHashAESImplemented && uintptr(len(key)) >= MinAeshashSize { hash = memHashAES(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) } else { hash = memHashFallback(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) @@ -143,7 +143,7 @@ func runtime_mapaccess2_faststr(typ *abi.MapType, m *Map, key string) (unsafe.Po var hash uintptr // See the related comment in runtime_mapaccess2_fast32 - if memHashAESImplemented && UseAeshash { + if memHashAESImplemented && uintptr(len(key)) >= MinAeshashSize { hash = memHashAES(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) } else { hash = memHashFallback(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) @@ -273,7 +273,7 @@ func runtime_mapassign_faststr(typ *abi.MapType, m *Map, key string) unsafe.Poin var hash uintptr // See the related comment in runtime_mapaccess2_fast32 - if memHashAESImplemented && UseAeshash { + if memHashAESImplemented && uintptr(len(key)) >= MinAeshashSize { hash = memHashAES(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) } else { hash = memHashFallback(unsafe.Pointer(unsafe.StringData(key)), m.seed, uintptr(len(key))) diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index ae5c5168502f0c..39308f06cd54da 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -212,7 +212,9 @@ var ( IfaceHash = ifaceHash ) -var UseAeshash = &maps.UseAeshash +var MinAeshashSize = &maps.MinAeshashSize + +var AeshashEnabled = maps.AeshashEnabled func MemclrBytes(b []byte) { s := (*slice)(unsafe.Pointer(&b)) diff --git a/src/runtime/hash_test.go b/src/runtime/hash_test.go index a530aa7de89673..0c241d8df7039a 100644 --- a/src/runtime/hash_test.go +++ b/src/runtime/hash_test.go @@ -32,7 +32,7 @@ func TestMemHash32AlignAccess(t *testing.T) { } func TestMemHash32Equality(t *testing.T) { - if *UseAeshash { + if *MinAeshashSize <= 4 { t.Skip("skipping since AES hash implementation is used") } var b [4]byte @@ -60,7 +60,7 @@ func TestMemHash64AlignAccess(t *testing.T) { } func TestMemHash64Equality(t *testing.T) { - if *UseAeshash { + if *MinAeshashSize <= 8 { t.Skip("skipping since AES hash implementation is used") } var b [8]byte @@ -658,7 +658,7 @@ func TestSmhasherSeed(t *testing.T) { } func TestIssue66841(t *testing.T) { - if *UseAeshash && os.Getenv("TEST_ISSUE_66841") == "" { + if AeshashEnabled() && os.Getenv("TEST_ISSUE_66841") == "" { // We want to test the backup hash, so if we're running on a machine // that uses aeshash, exec ourselves while turning aes off. cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t), "-test.run=^TestIssue66841$")) diff --git a/src/runtime/map_benchmark_test.go b/src/runtime/map_benchmark_test.go index 9e93b219f17795..4164d743c556c7 100644 --- a/src/runtime/map_benchmark_test.go +++ b/src/runtime/map_benchmark_test.go @@ -1226,3 +1226,68 @@ func BenchmarkMapAccessEmpty(b *testing.B) { b.Run("Key=mediumType", mapAccessEmptyBenchmark[mediumType]) b.Run("Key=bigType", mapAccessEmptyBenchmark[bigType]) } + +type groupByStats struct { + min, max, sum float64 + count int64 +} + +const ( + groupByRows = 1 << 18 + groupByGroups = 512 +) + +var groupBySink int + +// mapGroupByBenchmark aggregates a large row set into per-group +// statistics, one map lookup plus an update through the stored pointer +// per row. +// +// Unlike the access benchmarks above, whose lookups are independent, +// each row here chains the lookup into loads and stores of the group's +// stats, so the benchmark is sensitive to hash latency rather than +// hash throughput. +func mapGroupByBenchmark[K comparable](genKey func(int) K) func(*testing.B) { + return func(b *testing.B) { + r := rand.New(rand.NewSource(1234)) + rows := make([]K, groupByRows) + temps := make([]float64, groupByRows) + for i := range rows { + rows[i] = genKey(r.Intn(groupByGroups)) + temps[i] = float64(r.Intn(999)-499) / 10 + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + m := make(map[K]*groupByStats, groupByGroups) + for j, k := range rows { + s := m[k] + if s == nil { + s = &groupByStats{min: 1000, max: -1000} + m[k] = s + } + t := temps[j] + if t < s.min { + s.min = t + } + if t > s.max { + s.max = t + } + s.sum += t + s.count++ + } + groupBySink = len(m) + } + } +} + +func BenchmarkMapGroupBy(b *testing.B) { + b.Run("Key=int64", mapGroupByBenchmark(func(g int) int64 { + return int64(g)*7919 + 13 + })) + b.Run("Key=string", mapGroupByBenchmark(func(g int) string { + // Realistic short identifier keys, 9 to 13 bytes. + return fmt.Sprintf("station_%x", g*7919) + })) +} diff --git a/src/runtime/map_test.go b/src/runtime/map_test.go index a6d6ea47a0637a..ba244fbcc0dfe4 100644 --- a/src/runtime/map_test.go +++ b/src/runtime/map_test.go @@ -1080,7 +1080,7 @@ func TestMemHashGlobalSeed(t *testing.T) { // aeshash and memHashFallback use separate per-process seeds, so test // both. t.Run("aes", func(t *testing.T) { - if !*runtime.UseAeshash { + if !runtime.AeshashEnabled() { t.Skip("No AES") } @@ -1098,7 +1098,7 @@ func TestMemHashGlobalSeed(t *testing.T) { t.Run("noaes", func(t *testing.T) { env := "" - if *runtime.UseAeshash { + if runtime.AeshashEnabled() { env = "GODEBUG=cpu.aes=off" }