Skip to content

Fix memory lease issues with Pebble #147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mhutchinson merged 2 commits into from
Jul 22, 2026
Merged

Fix memory lease issues with Pebble #147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
26c6fcd
Fix memory lease issues with Pebble
mhutchinson Jul 22, 2026
59dc895
Reviewer comments
mhutchinson Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file removed ct
View file
Open in desktop
Binary file not shown.
22 changes: 11 additions & 11 deletions vindex/map.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"sync"
"time"

"filippo.io/torchwood/mpt"
"github.com/cockroachdb/pebble"
"github.com/transparency-dev/formats/log"
"github.com/transparency-dev/incubator/vindex/api"
Expand All @@ -44,7 +45,6 @@ import (
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/metric"
"k8s.io/klog/v2"
"filippo.io/torchwood/mpt"
)

const (
Expand Down Expand Up @@ -110,12 +110,7 @@ func NewVerifiableIndex(ctx context.Context, inputLog InputLog, mapFn MapFn, out
return nil, fmt.Errorf("pebble.Open(): %v", err)
}

// Helpful wrapper to convert Closer to something that can be safely deferred (according to the linter)
logClose := func(c io.Closer) {
if err := c.Close(); err != nil {
klog.Error(err)
}
}


// Load the compact range we have calculated so far, and the size persisted. We MUST start
// from this index in order to have properly verified the state of the input log.
Expand All @@ -142,7 +137,7 @@ func NewVerifiableIndex(ctx context.Context, inputLog InputLog, mapFn MapFn, out
size = binary.BigEndian.Uint64(sizeBs)
crHashes := make([][]byte, len(crBs)/sha256.Size)
for i := range crHashes {
crHashes[i] = crBs[i*sha256.Size : (i+1)*sha256.Size]
crHashes[i] = slices.Clone(crBs[i*sha256.Size : (i+1)*sha256.Size])
}
klog.V(1).Infof("Loaded compact range state from PebbleDB: size=%d, hashes=%d", size, len(crHashes))
cr, err = crf.NewRange(0, size, crHashes)
Expand Down Expand Up @@ -654,9 +649,7 @@ func (b *VerifiableIndex) buildMap(ctx context.Context, updateIndex bool) error
}
return fmt.Errorf("failed to read latest checkpoint: %v", err)
}
if err := closer.Close(); err != nil {
return fmt.Errorf("failed to close: %v", err)
}
defer logClose(closer)
_, size, _, err := checkpointUnsafe(cpRaw)
if err != nil {
return fmt.Errorf("failed to parse checkpoint: %v", err)
Expand Down Expand Up @@ -785,6 +778,13 @@ func checkpointUnsafe(rawCp []byte) (string, uint64, []byte, error) {
return origin, size, hash, nil
}

// Helpful wrapper to convert Closer to something that can be safely deferred (according to the linter)
func logClose(c io.Closer) {
if err := c.Close(); err != nil {
klog.Error(err)
}
}

type reporter struct {
// treeSize is fixed for the lifetime of the reporter.
treeSize uint64
Expand Down
Loading