MB-72489: fuzzy/regexp candidate-collection optimizations

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/blevesearch/zapx/v14 v14.4.3
 	github.com/blevesearch/zapx/v15 v15.4.3
 	github.com/blevesearch/zapx/v16 v16.3.4
-	github.com/blevesearch/zapx/v17 v17.2.0
+	github.com/blevesearch/zapx/v17 v17.2.1
 	github.com/couchbase/moss v0.2.0
 	github.com/spf13/cobra v1.10.2
 	go.etcd.io/bbolt v1.4.0

go.sum

@@ -45,8 +45,8 @@ github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2l
 github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
 github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0=
 github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg=
-github.com/blevesearch/zapx/v17 v17.2.0 h1:+4Zn/lDFqnoD2XRvwcw3K1ivwUjIQKIxST3Jr1LCcDE=
-github.com/blevesearch/zapx/v17 v17.2.0/go.mod h1:qwCdOYIG1kG0V4iYxwzNMwhxZxw7IXnSRrMobR1C/S0=
+github.com/blevesearch/zapx/v17 v17.2.1 h1:yIfupsQn2YlP5uKoWK8iGZ3uvxeQ2TQSNPEMnRVqPoA=
+github.com/blevesearch/zapx/v17 v17.2.1/go.mod h1:qwCdOYIG1kG0V4iYxwzNMwhxZxw7IXnSRrMobR1C/S0=
 github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
 github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
 github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=

index/scorch/fuzzy_omitcount_test.go

@@ -0,0 +1,98 @@
+//	Copyright (c) 2026 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package scorch
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/document"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+// TestFieldDictFuzzyAutomatonOmitsCount verifies that the fuzzy automaton field
+// dictionary is wired to the count-omitting iterator: the segment dictionary
+// implements the optional interface, and the collected candidate entries have
+// their (discarded) Count left at zero rather than incurring a postings read.
+func TestFieldDictFuzzyAutomatonOmitsCount(t *testing.T) {
+	cfg := CreateConfig("TestFieldDictFuzzyAutomatonOmitsCount")
+	if err := InitTest(cfg); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = DestroyTest(cfg) }()
+	analysisQueue := index.NewAnalysisQueue(1)
+	idx, err := NewScorch(Name, cfg, analysisQueue)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = idx.Open(); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = idx.Close() }()
+
+	// Each doc's "desc" field is a single token (nil analyzer => whole value is
+	// one term). Repeating terms across docs yields multi-doc postings lists.
+	b := index.NewBatch()
+	for d := 0; d < 300; d++ {
+		doc := document.NewDocument(fmt.Sprintf("%d", d))
+		doc.AddField(document.NewTextField("desc", nil, []byte(fmt.Sprintf("term%04d", d%100))))
+		b.Update(doc)
+	}
+	if err = idx.Batch(b); err != nil {
+		t.Fatal(err)
+	}
+
+	r, err := idx.Reader()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = r.Close() }()
+	is := r.(*IndexSnapshot)
+
+	// the segment dictionary must implement the count-omit optional interface,
+	// otherwise the wiring silently falls back to the count-reading path.
+	seg := is.segment[0].segment
+	dict, err := seg.Dictionary("desc")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := dict.(termDictionaryOmitCount); !ok {
+		// The omit-count wiring only activates once the zapx dependency ships
+		// AutomatonIteratorOmitCount; until then the code falls back to the
+		// count-reading path, so there is nothing to assert.
+		t.Skipf("segment dict %T does not implement termDictionaryOmitCount "+
+			"(zapx dependency predates the omit-count iterator)", dict)
+	}
+
+	fd, _, err := is.FieldDictFuzzyAutomaton("desc", "term0050", 2, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	n := 0
+	tfd, err := fd.Next()
+	for err == nil && tfd != nil {
+		n++
+		if tfd.Count != 0 {
+			t.Fatalf("expected omitted (zero) count for %q, got %d", tfd.Term, tfd.Count)
+		}
+		tfd, err = fd.Next()
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n == 0 {
+		t.Fatal("expected at least one fuzzy candidate term")
+	}
+	t.Logf("collected %d candidate terms, all with omitted count", n)
+}

index/scorch/snapshot_index.go

@@ -329,8 +329,9 @@ func (is *IndexSnapshot) fieldDictRegexp(field string,
 		return nil, nil, err
 	}
 
-	fd, err := is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {
-		return is.AutomatonIterator(a, prefixBeg, prefixEnd)
+	fd, err := is.newIndexSnapshotFieldDict(field, func(dict segment.TermDictionary) segment.DictionaryIterator {
+		// regexp/wildcard candidate collection discards DictEntry.Count.
+		return automatonIteratorOmitCount(dict, a, prefixBeg, prefixEnd)
 	}, false)
 	if err != nil {
 		return nil, nil, err
@@ -379,8 +380,9 @@ func (is *IndexSnapshot) fieldDictFuzzy(field string,
 		prefixBeg = []byte(prefix)
 		prefixEnd = calculateExclusiveEndFromPrefix(prefixBeg)
 	}
-	fd, err := is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {
-		return is.AutomatonIterator(a, prefixBeg, prefixEnd)
+	fd, err := is.newIndexSnapshotFieldDict(field, func(dict segment.TermDictionary) segment.DictionaryIterator {
+		// fuzzy candidate collection discards DictEntry.Count.
+		return automatonIteratorOmitCount(dict, a, prefixBeg, prefixEnd)
 	}, false)
 	if err != nil {
 		return nil, nil, err

index/scorch/snapshot_index_dict.go

@@ -21,6 +21,26 @@ import (
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 )
 
+// termDictionaryOmitCount is an optional interface a segment's TermDictionary
+// may implement to return an automaton iterator that skips the per-term
+// postings read used to populate DictEntry.Count. It is used by candidate-term
+// collection (fuzzy/regexp) where the count is discarded, and falls back to the
+// regular AutomatonIterator when the segment does not implement it.
+type termDictionaryOmitCount interface {
+	AutomatonIteratorOmitCount(a segment.Automaton,
+		startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator
+}
+
+// automatonIteratorOmitCount returns a count-omitting automaton iterator when
+// the dictionary supports it, otherwise the standard one.
+func automatonIteratorOmitCount(dict segment.TermDictionary, a segment.Automaton,
+	startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator {
+	if oc, ok := dict.(termDictionaryOmitCount); ok {
+		return oc.AutomatonIteratorOmitCount(a, startKeyInclusive, endKeyExclusive)
+	}
+	return dict.AutomatonIterator(a, startKeyInclusive, endKeyExclusive)
+}
+
 type segmentDictCursor struct {
 	dict segment.TermDictionary
 	itr  segment.DictionaryIterator

search/searcher/search_fuzzy.go

@@ -66,11 +66,11 @@ func NewFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term s
 	}
 
 	// Note: we don't byte slice the term for a prefix because of runes.
-	prefixTerm := ""
-	for i, r := range term {
-		if i < prefix {
-			prefixTerm += string(r)
-		} else {
+	// we instead slice the term at the first rune boundary at/after prefix index
+	prefixTerm := term
+	for i := range term {
+		if i >= prefix {
+			prefixTerm = term[:i]
 			break
 		}
 	}
@@ -89,21 +89,21 @@ func NewFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term s
 		dictBytesRead = fuzzyCandidates.bytesRead
 	}
 
+	var fuzzyTermMatches map[string][]string
 	if ctx != nil {
 		reportIOStats(ctx, dictBytesRead)
 		search.RecordSearchCost(ctx, search.AddM, dictBytesRead)
-		fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey)
-		if fuzzyTermMatches != nil {
-			fuzzyTermMatches.(map[string][]string)[term] = candidates
+		if ftm := ctx.Value(search.FuzzyMatchPhraseKey); ftm != nil {
+			fuzzyTermMatches = ftm.(map[string][]string)
 		}
 	}
+	if fuzzyTermMatches != nil {
+		fuzzyTermMatches[term] = candidates
+	}
 	// check if the candidates are empty or have one term which is the term itself
 	if len(candidates) == 0 || (len(candidates) == 1 && candidates[0] == term) {
-		if ctx != nil {
-			fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey)
-			if fuzzyTermMatches != nil {
-				fuzzyTermMatches.(map[string][]string)[term] = []string{term}
-			}
+		if fuzzyTermMatches != nil {
+			fuzzyTermMatches[term] = []string{term}
 		}
 		return NewTermSearcher(ctx, indexReader, term, field, boost, options)
 	}
@@ -156,15 +156,29 @@ func findFuzzyCandidateTerms(ctx context.Context, indexReader index.IndexReader,
 	// the levenshtein automaton based iterator to collect the
 	// candidate terms
 	if ir, ok := indexReader.(index.IndexReaderFuzzy); ok {
-		termSet := make(map[string]struct{})
+		var synonymTerms map[string][]string
+		if ctx != nil {
+			if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {
+				synonymTerms = fts[field]
+			}
+		}
+		// termSet is used to de-duplicate synonym terms against the candidates
+		// gathered across all the segments' dictionaries.
+		var termSet map[string]struct{}
+		if len(synonymTerms) > 0 {
+			termSet = make(map[string]struct{}, len(synonymTerms))
+		}
 		addCandidateTerm := func(term string, editDistance uint8) error {
-			if _, exists := termSet[term]; !exists {
-				termSet[term] = struct{}{}
-				rv.candidates = append(rv.candidates, term)
-				rv.editDistances = append(rv.editDistances, editDistance)
-				if tooManyClauses(len(rv.candidates)) {
-					return tooManyClausesErr(field, len(rv.candidates))
+			if termSet != nil {
+				if _, exists := termSet[term]; exists {
+					return nil
 				}
+				termSet[term] = struct{}{}
+			}
+			rv.candidates = append(rv.candidates, term)
+			rv.editDistances = append(rv.editDistances, editDistance)
+			if tooManyClauses(len(rv.candidates)) {
+				return tooManyClausesErr(field, len(rv.candidates))
 			}
 			return nil
 		}
@@ -188,24 +202,18 @@ func findFuzzyCandidateTerms(ctx context.Context, indexReader index.IndexReader,
 		if err != nil {
 			return nil, err
 		}
-		if ctx != nil {
-			if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {
-				if ts, exists := fts[field]; exists {
-					for term := range ts {
-						if _, exists := termSet[term]; exists {
-							continue
-						}
-						if !strings.HasPrefix(term, prefixTerm) {
-							continue
-						}
-						match, editDistance := a.MatchAndDistance(term)
-						if match {
-							err = addCandidateTerm(term, editDistance)
-							if err != nil {
-								return nil, err
-							}
-						}
-					}
+		for term := range synonymTerms {
+			if _, exists := termSet[term]; exists {
+				continue
+			}
+			if !strings.HasPrefix(term, prefixTerm) {
+				continue
+			}
+			match, editDistance := a.MatchAndDistance(term)
+			if match {
+				err = addCandidateTerm(term, editDistance)
+				if err != nil {
+					return nil, err
 				}
 			}
 		}