perf: opt-in WAND/MAXSCORE competitive scoring, parallel segment search, zapx v18 (ScoreMode API)

explain_disjunction_regression_test.go

@@ -0,0 +1,134 @@
+package bleve
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/search"
+)
+
+func floatClose(a, b float64) bool {
+	return math.Abs(a-b) <= 1e-6*(1+math.Abs(b))
+}
+
+// walkExpl fails on any nil node and, for nodes whose message declares an
+// aggregation ("sum of:" / "product of:"), verifies the node's Value equals the
+// sum/product of its children. Nodes with a non-aggregating message (e.g.
+// "saturation(...)") are only checked for nil children, not arithmetic.
+// Returns the node count.
+func walkExpl(t *testing.T, e *search.Explanation, path string) int {
+	t.Helper()
+	if e == nil {
+		t.Fatalf("NIL explanation node at %s", path)
+	}
+	n := 1
+	for i, c := range e.Children {
+		cp := fmt.Sprintf("%s > child[%d]", path, i)
+		if c == nil {
+			t.Fatalf("NIL child at %s (parent message=%q)", cp, e.Message)
+		}
+		n += walkExpl(t, c, cp)
+	}
+	if len(e.Children) == 0 {
+		return n
+	}
+	isSum := strings.Contains(e.Message, "sum of:")
+	isProduct := strings.Contains(e.Message, "product of:")
+	if !isSum && !isProduct {
+		return n // non-aggregating node (e.g. saturation): child is explanatory only
+	}
+	agg := 1.0
+	if isSum {
+		agg = 0.0
+	}
+	for _, c := range e.Children {
+		if isSum {
+			agg += c.Value
+		} else {
+			agg *= c.Value
+		}
+	}
+	kind := "product"
+	if isSum {
+		kind = "sum"
+	}
+	if !floatClose(agg, e.Value) {
+		t.Errorf("arithmetic mismatch at %s (message=%q): node.Value=%.8f but %s-of-children=%.8f",
+			path, e.Message, e.Value, kind, agg)
+	}
+	return n
+}
+
+// TestExplainDisjunctionTreeWellFormed is the regression test for the
+// DisjunctionQueryScorer explain bug (first "sum of:" child came back nil).
+// Runs under both scoring models; BM25 is the model FTS uses and the one the
+// §25 impact-table / NormByte optimizations target.
+func TestExplainDisjunctionTreeWellFormed(t *testing.T) {
+	for _, model := range []string{"", "bm25"} {
+		name := model
+		if name == "" {
+			name = "tfidf-default"
+		}
+		t.Run(name, func(t *testing.T) {
+			tmp, err := os.MkdirTemp("", "bleve-explain-*")
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer os.RemoveAll(tmp)
+
+			m := NewIndexMapping()
+			m.ScoringModel = model // "" also defaults to tf-idf; explicit for clarity
+
+			idx, err := New(tmp, m) // default index type = scorch (writes v18)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer idx.Close()
+
+			docs := map[string]string{
+				"a": "the quick brown fox jumps over the lazy dog",
+				"b": "quick brown quick brown clever",
+				"c": "brown bears and brown crates",
+				"d": "nothing relevant here at all",
+			}
+			batch := idx.NewBatch()
+			for id, text := range docs {
+				if err := batch.Index(id, map[string]any{"text": text}); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if err := idx.Batch(batch); err != nil {
+				t.Fatal(err)
+			}
+
+			// Multi-term match (OR) => disjunction of TermSearchers => DisjunctionQueryScorer.Score.
+			q := NewMatchQuery("quick brown fox")
+			q.SetField("text")
+			req := NewSearchRequest(q)
+			req.Explain = true
+			req.Size = 10
+
+			res, err := idx.Search(req)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(res.Hits) == 0 {
+				t.Fatal("expected hits")
+			}
+
+			for _, hit := range res.Hits {
+				if hit.Expl == nil {
+					t.Fatalf("doc %s: nil top-level explanation", hit.ID)
+				}
+				if !floatClose(hit.Score, hit.Expl.Value) {
+					t.Errorf("doc %s: hit.Score=%.8f != Expl.Value=%.8f", hit.ID, hit.Score, hit.Expl.Value)
+				}
+				nodes := walkExpl(t, hit.Expl, "doc "+hit.ID)
+				t.Logf("doc %s: score=%.6f, explanation nodes=%d\n%s", hit.ID, hit.Score, nodes, hit.Expl)
+			}
+		})
+	}
+}

go.mod

@@ -3,9 +3,9 @@ module github.com/blevesearch/bleve/v2
 go 1.25.0
 
 require (
-	github.com/RoaringBitmap/roaring/v2 v2.14.5
+	github.com/RoaringBitmap/roaring/v2 v2.18.2
 	github.com/bits-and-blooms/bitset v1.24.2
-	github.com/blevesearch/bleve_index_api v1.4.0
+	github.com/blevesearch/bleve_index_api v1.4.1-0.20260707065413-4b241f361ec2
 	github.com/blevesearch/geo v0.2.5
 	github.com/blevesearch/go-faiss v1.1.5
 	github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
@@ -25,7 +25,8 @@ require (
 	github.com/blevesearch/zapx/v14 v14.4.3
 	github.com/blevesearch/zapx/v15 v15.4.3
 	github.com/blevesearch/zapx/v16 v16.3.4
-	github.com/blevesearch/zapx/v17 v17.1.9
+	github.com/blevesearch/zapx/v17 v17.1.8
+	github.com/blevesearch/zapx/v18 v18.0.0-20260707065421-f1ceb9717cca
 	github.com/couchbase/moss v0.2.0
 	github.com/spf13/cobra v1.10.2
 	go.etcd.io/bbolt v1.4.0

go.sum

@@ -1,9 +1,9 @@
-github.com/RoaringBitmap/roaring/v2 v2.14.5 h1:ckd0o545JqDPeVJDgeFoaM21eBixUnlWfYgjE5VnyWw=
-github.com/RoaringBitmap/roaring/v2 v2.14.5/go.mod h1:eq4wdNXxtJIS/oikeCzdX1rBzek7ANzbth041hrU8Q4=
+github.com/RoaringBitmap/roaring/v2 v2.18.2 h1:oPq3Cgx//iDuJQVp6xSInAKW34J9CEwE5GmLI2z+Eic=
+github.com/RoaringBitmap/roaring/v2 v2.18.2/go.mod h1:eq4wdNXxtJIS/oikeCzdX1rBzek7ANzbth041hrU8Q4=
 github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0=
 github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
-github.com/blevesearch/bleve_index_api v1.4.0 h1:xoCC4dvTizjcsZu7yO9Ua+/259K09BrirjTwLkx2MpY=
-github.com/blevesearch/bleve_index_api v1.4.0/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
+github.com/blevesearch/bleve_index_api v1.4.1-0.20260707065413-4b241f361ec2 h1:Nc8lOn/fmchbYX2bi9hdAxhngW3XC0wf31xEjSKzY1U=
+github.com/blevesearch/bleve_index_api v1.4.1-0.20260707065413-4b241f361ec2/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
 github.com/blevesearch/geo v0.2.5 h1:yJg9FX1oRwLnjXSXF+ECHfXFTF4diF02Ca/qUGVjJhE=
 github.com/blevesearch/geo v0.2.5/go.mod h1:Jhq7WE2K6mJTx1xS44M2pUO6Io+wjCSHh1+co3YOgH4=
 github.com/blevesearch/go-faiss v1.1.5 h1:/IU5lkOahH9Ghfk9n3F6N0XD7PYVXZJWmNDc9TtXuco=
@@ -45,8 +45,10 @@ github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2l
 github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
 github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0=
 github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg=
-github.com/blevesearch/zapx/v17 v17.1.9 h1:K5MsArRyuwfylDTN1+cU7plGVIFz4gPoH4HD5M7I8ik=
-github.com/blevesearch/zapx/v17 v17.1.9/go.mod h1:34TIaJmdo5hMh2IBLoE4Day65j7DJ++8s5trz1yrsGY=
+github.com/blevesearch/zapx/v17 v17.1.8 h1:RhlEYVsjJuXbPaIHdKi3qbKpc8HLIToiTYf7A8CVkoo=
+github.com/blevesearch/zapx/v17 v17.1.8/go.mod h1:34TIaJmdo5hMh2IBLoE4Day65j7DJ++8s5trz1yrsGY=
+github.com/blevesearch/zapx/v18 v18.0.0-20260707065421-f1ceb9717cca h1:k6+l1lMGDpW+YHWEVVPLI8zozhIVzZ+jFlp4kYgmnRo=
+github.com/blevesearch/zapx/v18 v18.0.0-20260707065421-f1ceb9717cca/go.mod h1:OQ/HrAtXAmUG40C02RCRb+5PjJIhcZFivmkw+acaGvw=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=

index/scorch/empty.go

@@ -14,7 +14,10 @@
 
 package scorch
 
-import segment "github.com/blevesearch/scorch_segment_api/v2"
+import (
+	"github.com/RoaringBitmap/roaring/v2"
+	segment "github.com/blevesearch/scorch_segment_api/v2"
+)
 
 type emptyPostingsIterator struct{}
 
@@ -38,4 +41,12 @@ func (e *emptyPostingsIterator) ResetBytesRead(uint64) {}
 
 func (e *emptyPostingsIterator) BytesWritten() uint64 { return 0 }
 
+// Implement OptimizablePostingsIterator so that anEmptyPostingsIterator can
+// participate in nested conjunction/disjunction optimizations without aborting
+// them.  ActualBitmap returning nil and DocNum1Hit returning false cause the
+// iterator to contribute nothing to any AND or OR, which is correct.
+func (e *emptyPostingsIterator) ActualBitmap() *roaring.Bitmap { return nil }
+func (e *emptyPostingsIterator) DocNum1Hit() (uint64, bool)    { return 0, false }
+func (e *emptyPostingsIterator) ReplaceActual(*roaring.Bitmap) {}
+
 var anEmptyPostingsIterator = &emptyPostingsIterator{}

index/scorch/optimize.go

@@ -308,24 +308,6 @@ func (o *OptimizeTFRDisjunctionUnadorned) Finish() (rv index.Optimized, err erro
 		return nil, nil
 	}
 
-	for i := range o.snapshot.segment {
-		var cMax uint64
-
-		for _, tfr := range o.tfrs {
-			itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator)
-			if !ok {
-				return nil, nil
-			}
-
-			if itr.ActualBitmap() != nil {
-				c := itr.ActualBitmap().GetCardinality()
-				if cMax < c {
-					cMax = c
-				}
-			}
-		}
-	}
-
 	// We use an artificial term and field because the optimized
 	// termFieldReader can represent multiple terms and fields.
 	oTFR := o.snapshot.unadornedTermFieldReader(
@@ -355,16 +337,26 @@ func (o *OptimizeTFRDisjunctionUnadorned) Finish() (rv index.Optimized, err erro
 			}
 		}
 
+		// Fast path: no hits in this segment — reuse the zero-alloc empty sentinel.
+		if len(actualBMs) == 0 && len(docNums) == 0 {
+			oTFR.iterators[i] = anEmptyPostingsIterator
+			continue
+		}
+
+		// Fast path: exactly one 1-hit doc with no bitmaps.
+		if len(actualBMs) == 0 && len(docNums) == 1 {
+			oTFR.iterators[i] = newUnadornedPostingsIteratorFrom1Hit(uint64(docNums[0]))
+			continue
+		}
+
 		var bm *roaring.Bitmap
 		if len(actualBMs) > 2 {
 			bm = roaring.HeapOr(actualBMs...)
 		} else if len(actualBMs) == 2 {
 			bm = roaring.Or(actualBMs[0], actualBMs[1])
 		} else if len(actualBMs) == 1 {
 			bm = actualBMs[0].Clone()
-		}
-
-		if bm == nil {
+		} else {
 			bm = roaring.New()
 		}