Optimizations

database/dbtest/benchmark.go

@@ -16,15 +16,15 @@ import (
 var (
 	// Benchmarks is a list of all database benchmarks
 	Benchmarks = map[string]func(b *testing.B, db database.Database, keys, values [][]byte){
-		"Get":            BenchmarkGet,
-		"Put":            BenchmarkPut,
-		"Delete":         BenchmarkDelete,
-		"BatchPut":       BenchmarkBatchPut,
-		"BatchDelete":    BenchmarkBatchDelete,
+		// "Get":            BenchmarkGet,
+		// "Put":            BenchmarkPut,
+		// "Delete":         BenchmarkDelete,
+		// // "BatchPut":       BenchmarkBatchPut,
+		// // "BatchDelete":    BenchmarkBatchDelete,
 		"BatchWrite":     BenchmarkBatchWrite,
-		"ParallelGet":    BenchmarkParallelGet,
-		"ParallelPut":    BenchmarkParallelPut,
-		"ParallelDelete": BenchmarkParallelDelete,
+		// "ParallelGet":    BenchmarkParallelGet,
+		// "ParallelPut":    BenchmarkParallelPut,
+		// "ParallelDelete": BenchmarkParallelDelete,
 	}
 	// BenchmarkSizes to use with each benchmark
 	BenchmarkSizes = [][]int{

go.mod

@@ -95,11 +95,13 @@ require (
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
+	github.com/dgraph-io/ristretto v0.2.0 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/frankban/quicktest v1.14.4 // indirect

go.sum

@@ -162,6 +162,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
 github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE=
+github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
@@ -178,6 +180,8 @@ github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14
 github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
 github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=

x/merkledb/db_test.go

@@ -159,6 +159,37 @@ func Benchmark_MerkleDB_DBInterface(b *testing.B) {
 	}
 }
 
+
+func Benchmark_MerkleDB_DBInterface_Timed(b *testing.B) {
+	totalTime := time.Duration(0)
+    for _, size := range dbtest.BenchmarkSizes {
+        keys, values := dbtest.SetupBenchmark(b, size[0], size[1], size[2])
+        for _, bf := range validBranchFactors {
+            for name, bench := range dbtest.Benchmarks {
+                // Run the benchmark and track its time
+                b.Run(fmt.Sprintf("merkledb_%d_%d_pairs_%d_keys_%d_values_%s", bf, size[0], size[1], size[2], name), func(b *testing.B) {
+                    db, err := getBasicDBWithBranchFactor(bf)
+                    require.NoError(b, err)
+                    b.Cleanup(func() {
+                        db.Close()
+                    })
+
+                    // Record the start time for each individual benchmark
+                    start := time.Now()
+                    bench(b, db, keys, values)
+                    // Calculate the time taken for the specific benchmark
+                    duration := time.Since(start)
+					totalTime += duration
+                    // Log the time taken for the specific benchmark
+                })
+            }
+        }
+    }
+	// Log the total time taken for all benchmarks
+	fmt.Printf("Total time taken for all benchmarks: %v\n", totalTime)
+	b.Log("Total time taken for all benchmarks: ", totalTime)
+}
+
 // PASSES
 func Test_MerkleDB_DB_Load_Root_From_DB(t *testing.T) {
 	require := require.New(t)

x/merkledb/disk_manager.go

@@ -85,7 +85,7 @@ func newDiskManager(metaData []byte, dir string, fileName string) (*diskMgr, err
 	}
 
 	// start freelist
-	maxSize := 4096 * 4 * 4
+	maxSize := 4096 * 4 * 4 * 4
 	f := newFreeList(maxSize)
 	f.load(dir)
 
@@ -119,8 +119,6 @@ func (dm *diskMgr) putBack(addr diskAddress) error {
 }
 
 func (dm *diskMgr) writeRoot(rootNode dbNode) (diskAddress, error) {
-
-
 	// first check the size of rootNode without the disk address
 	bytes := encodeDBNode_disk(&rootNode)
 	freeSpace, ok := dm.free.get(int64(len(bytes)) + 16)
@@ -190,6 +188,22 @@ func (dm *diskMgr) writeRoot(rootNode dbNode) (diskAddress, error) {
 
 }
 
+func (dm *diskMgr) fetch(byteLength int64) (diskAddress, error) {
+	freeSpace, ok := dm.free.get(int64(byteLength))
+	if !ok {
+		endOffset, err := dm.endOfFile()
+		if err != nil {
+			log.Fatalf("failed to get end of file: %v", err)
+			return diskAddress{}, err
+		}
+		return diskAddress{offset: endOffset, size: int64(byteLength)}, nil
+	} else {
+		return diskAddress{offset: freeSpace.offset, size: int64(byteLength)}, nil
+	}
+
+
+}
+
 // returning diskaddress that it wrote to
 // if we write to freelist: diskaddress would be the size of freespace
 // if we dont write to freelist: append bytes to end, return endoffset and size

x/merkledb/freelist.go

@@ -50,7 +50,12 @@ func (f *freeList) get(size int64) (diskAddress, bool) {
 		return diskAddress{}, false
 	}*/
 	bucket := f.bucketIndex(size)
+
 
+	// first check whether or not the bucket would be out of bounds
+	if bucket >= len(f.buckets) {
+		return diskAddress{}, false
+	}
 	if len(f.buckets[bucket]) > 0 {
 		space := f.buckets[bucket][len(f.buckets[bucket])-1]
 		f.buckets[bucket] = f.buckets[bucket][:len(f.buckets[bucket])-1]

x/merkledb/helpers_test.go

@@ -11,9 +11,16 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/ava-labs/avalanchego/database/memdb"
+
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/utils/hashing"
 	"github.com/ava-labs/avalanchego/utils/maybe"
+
+
+	"github.com/ava-labs/avalanchego/database/leveldb"
+	"github.com/ava-labs/avalanchego/utils/logging"
+	"github.com/prometheus/client_golang/prometheus"
+
 )
 const disk = true
 func getBasicDB(tb testing.TB) (*merkleDB, error) {
@@ -29,21 +36,24 @@ func getBasicDB(tb testing.TB) (*merkleDB, error) {
 }
 
 func getBasicDBWithBranchFactor(bf BranchFactor) (*merkleDB, error) {
-	config := newDefaultConfig()
-	config.BranchFactor = bf
-	t := &testing.T{}
-	dir := t.TempDir()
-	if disk{
-		return getBasicDBWithBranchFactor_disk(bf,dir)
-	}
-	return newDatabase(
-		context.Background(),
-		memdb.New(),
-		config,
-		&mockMetrics{},
-	)
-}
+    config := newDefaultConfig()
+    config.BranchFactor = bf
+    t := &testing.T{}
+
+    folder := t.TempDir()
+    db, err := leveldb.New(folder, nil, logging.NoLog{}, prometheus.NewRegistry())
+    require.NoError(t, err)
 
+    if disk {
+        return getBasicDBWithBranchFactor_disk(bf, folder)
+    }
+    return newDatabase(
+        context.Background(),
+        db,
+        config,
+        &mockMetrics{},
+    )
+}
 // Writes []byte{i} -> []byte{i} for i in [0, 4]
 func writeBasicBatch(t *testing.T, db *merkleDB) {
 	require := require.New(t)

x/merkledb/key.go

@@ -37,7 +37,7 @@ var (
 		BranchFactor2,
 		BranchFactor4,
 		BranchFactor16,
-		// BranchFactor256,
+		BranchFactor256,
 	}
 )