diff --git a/go.mod b/go.mod index 71d398ae0..ac081ecc4 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/blevesearch/zapx/v14 v14.4.3 github.com/blevesearch/zapx/v15 v15.4.3 github.com/blevesearch/zapx/v16 v16.3.4 - github.com/blevesearch/zapx/v17 v17.2.0 + github.com/blevesearch/zapx/v17 v17.2.1 github.com/couchbase/moss v0.2.0 github.com/spf13/cobra v1.10.2 go.etcd.io/bbolt v1.4.0 diff --git a/go.sum b/go.sum index b397b2283..e8f693b78 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2l github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0= github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg= -github.com/blevesearch/zapx/v17 v17.2.0 h1:+4Zn/lDFqnoD2XRvwcw3K1ivwUjIQKIxST3Jr1LCcDE= -github.com/blevesearch/zapx/v17 v17.2.0/go.mod h1:qwCdOYIG1kG0V4iYxwzNMwhxZxw7IXnSRrMobR1C/S0= +github.com/blevesearch/zapx/v17 v17.2.1 h1:yIfupsQn2YlP5uKoWK8iGZ3uvxeQ2TQSNPEMnRVqPoA= +github.com/blevesearch/zapx/v17 v17.2.1/go.mod h1:qwCdOYIG1kG0V4iYxwzNMwhxZxw7IXnSRrMobR1C/S0= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= diff --git a/index/scorch/fuzzy_omitcount_test.go b/index/scorch/fuzzy_omitcount_test.go new file mode 100644 index 000000000..8a2dfdf8b --- /dev/null +++ b/index/scorch/fuzzy_omitcount_test.go @@ -0,0 +1,98 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package scorch + +import ( + "fmt" + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +// TestFieldDictFuzzyAutomatonOmitsCount verifies that the fuzzy automaton field +// dictionary is wired to the count-omitting iterator: the segment dictionary +// implements the optional interface, and the collected candidate entries have +// their (discarded) Count left at zero rather than incurring a postings read. +func TestFieldDictFuzzyAutomatonOmitsCount(t *testing.T) { + cfg := CreateConfig("TestFieldDictFuzzyAutomatonOmitsCount") + if err := InitTest(cfg); err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + if err = idx.Open(); err != nil { + t.Fatal(err) + } + defer func() { _ = idx.Close() }() + + // Each doc's "desc" field is a single token (nil analyzer => whole value is + // one term). Repeating terms across docs yields multi-doc postings lists. + b := index.NewBatch() + for d := 0; d < 300; d++ { + doc := document.NewDocument(fmt.Sprintf("%d", d)) + doc.AddField(document.NewTextField("desc", nil, []byte(fmt.Sprintf("term%04d", d%100)))) + b.Update(doc) + } + if err = idx.Batch(b); err != nil { + t.Fatal(err) + } + + r, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + is := r.(*IndexSnapshot) + + // the segment dictionary must implement the count-omit optional interface, + // otherwise the wiring silently falls back to the count-reading path. + seg := is.segment[0].segment + dict, err := seg.Dictionary("desc") + if err != nil { + t.Fatal(err) + } + if _, ok := dict.(termDictionaryOmitCount); !ok { + // The omit-count wiring only activates once the zapx dependency ships + // AutomatonIteratorOmitCount; until then the code falls back to the + // count-reading path, so there is nothing to assert. + t.Skipf("segment dict %T does not implement termDictionaryOmitCount "+ + "(zapx dependency predates the omit-count iterator)", dict) + } + + fd, _, err := is.FieldDictFuzzyAutomaton("desc", "term0050", 2, "") + if err != nil { + t.Fatal(err) + } + n := 0 + tfd, err := fd.Next() + for err == nil && tfd != nil { + n++ + if tfd.Count != 0 { + t.Fatalf("expected omitted (zero) count for %q, got %d", tfd.Term, tfd.Count) + } + tfd, err = fd.Next() + } + if err != nil { + t.Fatal(err) + } + if n == 0 { + t.Fatal("expected at least one fuzzy candidate term") + } + t.Logf("collected %d candidate terms, all with omitted count", n) +} diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go index 3836bd29c..d4242c460 100644 --- a/index/scorch/snapshot_index.go +++ b/index/scorch/snapshot_index.go @@ -329,8 +329,9 @@ func (is *IndexSnapshot) fieldDictRegexp(field string, return nil, nil, err } - fd, err := is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { - return is.AutomatonIterator(a, prefixBeg, prefixEnd) + fd, err := is.newIndexSnapshotFieldDict(field, func(dict segment.TermDictionary) segment.DictionaryIterator { + // regexp/wildcard candidate collection discards DictEntry.Count. + return automatonIteratorOmitCount(dict, a, prefixBeg, prefixEnd) }, false) if err != nil { return nil, nil, err @@ -379,8 +380,9 @@ func (is *IndexSnapshot) fieldDictFuzzy(field string, prefixBeg = []byte(prefix) prefixEnd = calculateExclusiveEndFromPrefix(prefixBeg) } - fd, err := is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { - return is.AutomatonIterator(a, prefixBeg, prefixEnd) + fd, err := is.newIndexSnapshotFieldDict(field, func(dict segment.TermDictionary) segment.DictionaryIterator { + // fuzzy candidate collection discards DictEntry.Count. + return automatonIteratorOmitCount(dict, a, prefixBeg, prefixEnd) }, false) if err != nil { return nil, nil, err diff --git a/index/scorch/snapshot_index_dict.go b/index/scorch/snapshot_index_dict.go index 2ae789c6b..b3204e822 100644 --- a/index/scorch/snapshot_index_dict.go +++ b/index/scorch/snapshot_index_dict.go @@ -21,6 +21,26 @@ import ( segment "github.com/blevesearch/scorch_segment_api/v2" ) +// termDictionaryOmitCount is an optional interface a segment's TermDictionary +// may implement to return an automaton iterator that skips the per-term +// postings read used to populate DictEntry.Count. It is used by candidate-term +// collection (fuzzy/regexp) where the count is discarded, and falls back to the +// regular AutomatonIterator when the segment does not implement it. +type termDictionaryOmitCount interface { + AutomatonIteratorOmitCount(a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator +} + +// automatonIteratorOmitCount returns a count-omitting automaton iterator when +// the dictionary supports it, otherwise the standard one. +func automatonIteratorOmitCount(dict segment.TermDictionary, a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { + if oc, ok := dict.(termDictionaryOmitCount); ok { + return oc.AutomatonIteratorOmitCount(a, startKeyInclusive, endKeyExclusive) + } + return dict.AutomatonIterator(a, startKeyInclusive, endKeyExclusive) +} + type segmentDictCursor struct { dict segment.TermDictionary itr segment.DictionaryIterator diff --git a/search/searcher/search_fuzzy.go b/search/searcher/search_fuzzy.go index 187486efc..f53502ffe 100644 --- a/search/searcher/search_fuzzy.go +++ b/search/searcher/search_fuzzy.go @@ -66,11 +66,11 @@ func NewFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term s } // Note: we don't byte slice the term for a prefix because of runes. - prefixTerm := "" - for i, r := range term { - if i < prefix { - prefixTerm += string(r) - } else { + // we instead slice the term at the first rune boundary at/after prefix index + prefixTerm := term + for i := range term { + if i >= prefix { + prefixTerm = term[:i] break } } @@ -89,21 +89,21 @@ func NewFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term s dictBytesRead = fuzzyCandidates.bytesRead } + var fuzzyTermMatches map[string][]string if ctx != nil { reportIOStats(ctx, dictBytesRead) search.RecordSearchCost(ctx, search.AddM, dictBytesRead) - fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey) - if fuzzyTermMatches != nil { - fuzzyTermMatches.(map[string][]string)[term] = candidates + if ftm := ctx.Value(search.FuzzyMatchPhraseKey); ftm != nil { + fuzzyTermMatches = ftm.(map[string][]string) } } + if fuzzyTermMatches != nil { + fuzzyTermMatches[term] = candidates + } // check if the candidates are empty or have one term which is the term itself if len(candidates) == 0 || (len(candidates) == 1 && candidates[0] == term) { - if ctx != nil { - fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey) - if fuzzyTermMatches != nil { - fuzzyTermMatches.(map[string][]string)[term] = []string{term} - } + if fuzzyTermMatches != nil { + fuzzyTermMatches[term] = []string{term} } return NewTermSearcher(ctx, indexReader, term, field, boost, options) } @@ -156,15 +156,29 @@ func findFuzzyCandidateTerms(ctx context.Context, indexReader index.IndexReader, // the levenshtein automaton based iterator to collect the // candidate terms if ir, ok := indexReader.(index.IndexReaderFuzzy); ok { - termSet := make(map[string]struct{}) + var synonymTerms map[string][]string + if ctx != nil { + if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok { + synonymTerms = fts[field] + } + } + // termSet is used to de-duplicate synonym terms against the candidates + // gathered across all the segments' dictionaries. + var termSet map[string]struct{} + if len(synonymTerms) > 0 { + termSet = make(map[string]struct{}, len(synonymTerms)) + } addCandidateTerm := func(term string, editDistance uint8) error { - if _, exists := termSet[term]; !exists { - termSet[term] = struct{}{} - rv.candidates = append(rv.candidates, term) - rv.editDistances = append(rv.editDistances, editDistance) - if tooManyClauses(len(rv.candidates)) { - return tooManyClausesErr(field, len(rv.candidates)) + if termSet != nil { + if _, exists := termSet[term]; exists { + return nil } + termSet[term] = struct{}{} + } + rv.candidates = append(rv.candidates, term) + rv.editDistances = append(rv.editDistances, editDistance) + if tooManyClauses(len(rv.candidates)) { + return tooManyClausesErr(field, len(rv.candidates)) } return nil } @@ -188,24 +202,18 @@ func findFuzzyCandidateTerms(ctx context.Context, indexReader index.IndexReader, if err != nil { return nil, err } - if ctx != nil { - if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok { - if ts, exists := fts[field]; exists { - for term := range ts { - if _, exists := termSet[term]; exists { - continue - } - if !strings.HasPrefix(term, prefixTerm) { - continue - } - match, editDistance := a.MatchAndDistance(term) - if match { - err = addCandidateTerm(term, editDistance) - if err != nil { - return nil, err - } - } - } + for term := range synonymTerms { + if _, exists := termSet[term]; exists { + continue + } + if !strings.HasPrefix(term, prefixTerm) { + continue + } + match, editDistance := a.MatchAndDistance(term) + if match { + err = addCandidateTerm(term, editDistance) + if err != nil { + return nil, err } } } diff --git a/search/searcher/search_fuzzy_bench_test.go b/search/searcher/search_fuzzy_bench_test.go new file mode 100644 index 000000000..63b90f675 --- /dev/null +++ b/search/searcher/search_fuzzy_bench_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +// buildFuzzyBenchIndex builds a scorch index whose "desc" field has numTerms +// distinct terms spread across numDocs documents (perDoc terms per document), +// so candidate terms have real multi-doc postings lists. +func buildFuzzyBenchIndex(b *testing.B, numTerms, numDocs, perDoc int) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + idx, err := scorch.NewScorch(scorch.Name, + map[string]interface{}{"path": b.TempDir()}, analysisQueue) + if err != nil { + b.Fatal(err) + } + if err = idx.Open(); err != nil { + b.Fatal(err) + } + + terms := make([]string, numTerms) + for i := range terms { + terms[i] = fmt.Sprintf("term%04d", i) + } + + batch := index.NewBatch() + for d := 0; d < numDocs; d++ { + doc := document.NewDocument(fmt.Sprintf("%d", d)) + buf := make([]byte, 0, perDoc*9) + for j := 0; j < perDoc; j++ { + if j > 0 { + buf = append(buf, ' ') + } + buf = append(buf, terms[(d*perDoc+j)%numTerms]...) + } + doc.AddField(document.NewTextFieldCustom("desc", nil, buf, + twoDocIndexDescIndexingOptions, testAnalyzer)) + batch.Update(doc) + } + if err = idx.Batch(batch); err != nil { + b.Fatal(err) + } + return idx +} + +// BenchmarkFuzzyCandidateCollection isolates candidate-term collection +// (findFuzzyCandidateTerms) over a real scorch index for a fuzziness-2 query. +// This is the step the fuzzy optimizations touch; the full NewFuzzySearcher is +// dominated by building one term searcher per candidate downstream, which these +// changes do not affect and which would dilute the measurement. +func BenchmarkFuzzyCandidateCollection(b *testing.B) { + idx := buildFuzzyBenchIndex(b, 1000, 3000, 40) + defer func() { _ = idx.Close() }() + + r, err := idx.Reader() + if err != nil { + b.Fatal(err) + } + defer func() { _ = r.Close() }() + + ctx := context.Background() + + // sanity: report how many candidates the query matches, once. + fc, err := findFuzzyCandidateTerms(ctx, r, "term0500", 2, "desc", "") + if err != nil { + b.Fatal(err) + } + b.Logf("candidates=%d", len(fc.candidates)) + + b.ResetTimer() + b.ReportAllocs() + for n := 0; n < b.N; n++ { + _, err := findFuzzyCandidateTerms(ctx, r, "term0500", 2, "desc", "") + if err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkFuzzySearcherEndToEnd measures the full NewFuzzySearcher path for +// reference (candidate collection + building the boosted disjunction). +func BenchmarkFuzzySearcherEndToEnd(b *testing.B) { + idx := buildFuzzyBenchIndex(b, 1000, 3000, 40) + defer func() { _ = idx.Close() }() + + r, err := idx.Reader() + if err != nil { + b.Fatal(err) + } + defer func() { _ = r.Close() }() + + opts := search.SearcherOptions{Score: "none"} + ctx := context.Background() + + b.ResetTimer() + b.ReportAllocs() + for n := 0; n < b.N; n++ { + s, err := NewFuzzySearcher(ctx, r, "term0500", 0, 2, "desc", 1.0, opts) + if err != nil { + b.Fatal(err) + } + if err = s.Close(); err != nil { + b.Fatal(err) + } + } +}