diff --git a/pkg/bindinfo/tests/BUILD.bazel b/pkg/bindinfo/tests/BUILD.bazel index 732e64505debd..a1a162f48d39b 100644 --- a/pkg/bindinfo/tests/BUILD.bazel +++ b/pkg/bindinfo/tests/BUILD.bazel @@ -11,7 +11,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 25, + shard_count = 26, deps = [ "//pkg/bindinfo", "//pkg/domain", diff --git a/pkg/planner/cardinality/row_count_index.go b/pkg/planner/cardinality/row_count_index.go index c0e9533aeec55..132a29b153507 100644 --- a/pkg/planner/cardinality/row_count_index.go +++ b/pkg/planner/cardinality/row_count_index.go @@ -16,6 +16,7 @@ package cardinality import ( "bytes" + "math" "slices" "strings" "time" @@ -568,6 +569,119 @@ func expBackoffEstimation(sctx planctx.PlanContext, idx *statistics.Index, coll return multResult, minSel, maxSel, true, nil } +// AdjustRowCountForAppendedHandleColumns damps a row count estimated from the declared +// index columns with the selectivity of the handle columns that fillIndexPath appended +// to the range columns of a non-unique index path. Index statistics only cover the +// declared columns, so prefixCount was computed from ranges pruned back to +// declaredColCnt dimensions and gives the appended handle predicates no credit; without +// an adjustment, a path whose benefit is the handle seek looks as expensive as one +// without it. Following expBackoffEstimation, the prefix estimate keeps the full-weight +// slot and each handle column contributes sel^(1/2), sel^(1/4), ... starting from the +// most selective one, which credits the handle predicates without assuming full +// independence between the index columns and the primary key. The full ranges must not +// reach the index statistics directly: bounds encoded from the appended dimensions sort +// past the truncated statistics keys and would collapse the estimate. +// +// idxColsWithHandle must be the declared index columns followed by the complete handle: +// fillIndexPath only appends the handle when the table's primary key is the single +// integer handle column, so the dimensions past declaredColCnt always identify a row +// exactly. A partial handle suffix would break the full-point cap below. +func AdjustRowCountForAppendedHandleColumns( + sctx planctx.PlanContext, + coll *statistics.HistColl, + ranges []*ranger.Range, + idxColsWithHandle []*expression.Column, + declaredColCnt int, + prefixCount statistics.RowEstimate, +) statistics.RowEstimate { + realtimeCount := float64(coll.RealtimeCount) + if realtimeCount <= 0 || len(ranges) == 0 || len(idxColsWithHandle) <= declaredColCnt { + return prefixCount + } + sels := make([]float64, 0, len(idxColsWithHandle)-declaredColCnt) + for dim := declaredColCnt; dim < len(idxColsWithHandle); dim++ { + col := idxColsWithHandle[dim] + if col == nil || statistics.ColumnStatsIsInvalid(coll.GetCol(col.UniqueID), sctx, coll, col.UniqueID) { + continue + } + colRanges := make(ranger.Ranges, 0, len(ranges)) + allBound := true + for _, ran := range ranges { + if len(ran.LowVal) <= dim || len(ran.HighVal) <= dim { + // Some range does not constrain this dimension, so the column is not + // bound across the whole path and must not contribute selectivity. + allBound = false + break + } + colRanges = append(colRanges, &ranger.Range{ + LowVal: []types.Datum{ran.LowVal[dim]}, + HighVal: []types.Datum{ran.HighVal[dim]}, + Collators: []collate.Collator{ran.Collators[dim]}, + // The exclusion flags of a multi-column range apply to its last dimension. + LowExclude: ran.LowExclude && dim == len(ran.LowVal)-1, + HighExclude: ran.HighExclude && dim == len(ran.HighVal)-1, + }) + } + if !allBound { + continue + } + // Ranges that differ only in earlier dimensions repeat the same handle bound; + // merge them so the column row count is not summed once per range. + merged, err := ranger.UnionRanges(sctx.GetRangerCtx(), colRanges, false) + if err != nil { + continue + } + countEst, err := GetRowCountByColumnRanges(sctx, coll, col.UniqueID, merged, false) + if err != nil { + continue + } + if sel := countEst.Est / realtimeCount; sel > 0 && sel < 1 { + sels = append(sels, sel) + } + } + adjusted := prefixCount + if len(sels) > 0 { + slices.Sort(sels) + factor, indepFactor := 1.0, 1.0 + for i, sel := range sels { + indepFactor *= sel + // The prefix estimate occupies the full-weight slot, so the i-th handle + // selectivity gets weight 1/2^(i+1). + if i+1 < MaxExponentialBackoffCols { + for range i + 1 { + sel = math.Sqrt(sel) + } + factor *= sel + } + } + adjusted.Est *= factor + // Damping should not push the estimate below 1 row unless the prefix estimate is already < 1. + adjusted.Est = max(adjusted.Est, min(prefixCount.Est, 1)) + // Full independence gives the most optimistic count; the unadjusted prefix + // estimate remains the upper bound in MaxEst. + adjusted.MinEst = min(adjusted.MinEst*indepFactor, adjusted.Est) + } + // A point range over the declared columns plus the full handle identifies at most + // one row, because the physical key of a non-unique index ends with the complete + // handle and is therefore unique. This relies on the contract above: the appended + // dimensions cover the complete handle, not a prefix of a multi-column primary key. + fullPoints := true + for _, ran := range ranges { + if len(ran.LowVal) != len(idxColsWithHandle) || len(ran.HighVal) != len(idxColsWithHandle) || + !ran.IsPoint(sctx.GetRangerCtx()) { + fullPoints = false + break + } + } + if fullPoints { + pointCap := float64(len(ranges)) + adjusted.Est = min(adjusted.Est, pointCap) + adjusted.MinEst = min(adjusted.MinEst, adjusted.Est) + adjusted.MaxEst = min(adjusted.MaxEst, pointCap) + } + return adjusted +} + // outOfRangeOnIndex checks if the datum is out of the range. func outOfRangeOnIndex(idx *statistics.Index, val types.Datum) bool { if !idx.Histogram.OutOfRange(val) { diff --git a/pkg/planner/cardinality/selectivity_test.go b/pkg/planner/cardinality/selectivity_test.go index 34686eec0203d..4887816113e87 100644 --- a/pkg/planner/cardinality/selectivity_test.go +++ b/pkg/planner/cardinality/selectivity_test.go @@ -1743,22 +1743,53 @@ func TestIndexRangeEstimationWithTruncatedHandleRange(t *testing.T) { return "", "" } - // Each distinct value of a has 10 rows; the handle predicates below must estimate as if - // the handle dimension were absent, i.e. the 10 rows of a = 5. - - // Handle range with an exclusive low bound. + // Each distinct value of a has 10 rows. The estimate starts from the 10 rows of the + // a = 5 prefix (index statistics only cover the declared column), is damped by the + // handle column's selectivity with exponential backoff, and is then aligned upward to + // stats.RowCount (the Selectivity() result over all predicates) for consistency — + // without the SelectionFactor penalty that adjustCountAfterAccess would otherwise add. + + // Handle range with an exclusive low bound. Before the exclusion-flag fix the pruned + // range collapsed to the empty (5, 5] and the consistency penalty produced 12.50. + // Non-point handle predicates receive no net credit yet: Selectivity() floors the + // backoff result at 1/NDV of the declared index columns, so the aligned estimate + // stays at the prefix row count. estRows, opInfo := indexScanRow("select * from t use index(ia) where a = 5 and id > 10") require.Contains(t, opInfo, "range:(5 10,5 +inf]", "execution range must keep the handle dimension") require.Equal(t, "10.00", estRows) - // Handle range with an exclusive high bound. + // Handle range with an exclusive high bound: same shape as above. estRows, opInfo = indexScanRow("select * from t use index(ia) where a = 5 and id < 10") require.Contains(t, opInfo, "range:[5 -inf,5 10)", "execution range must keep the handle dimension") require.Equal(t, "10.00", estRows) - // Multiple handle points under the same prefix must not double count. + // Point-bound handle predicates get real credit. The damped estimate + // 10 * sqrt(sel(id in (11, 22))) = 1.41 aligns to the Selectivity() result of 2.00, + // instead of the uncredited 10.00 (or 12.50 with the consistency penalty). estRows, opInfo = indexScanRow("select * from t use index(ia) where a = 5 and id in (11, 22)") require.Contains(t, opInfo, "range:[5 11,5 11], [5 22,5 22]", "execution range must keep the handle dimension") + require.Equal(t, "2.00", estRows) + + // A point over the index column plus the full handle matches at most one row, because + // the physical key of a non-unique index ends with the complete handle. + estRows, opInfo = indexScanRow("select * from t use index(ia) where a = 5 and id = 7") + require.Contains(t, opInfo, "range:[5 7,5 7]", "execution range must keep the handle dimension") + require.Equal(t, "1.00", estRows) + + // An unsigned int handle is stored in the index key suffix in signed-encoded order, + // which wraps at the int64 boundary: values in [MaxInt64+1, MaxUint64] sort before + // [0, MaxInt64]. fillIndexPath therefore never appends an unsigned handle to the + // index columns, so unsigned handle predicates must stay out of the index ranges and + // receive no appended-handle credit: the estimate remains the prefix row count. + tk.MustExec("create table tu(id bigint unsigned primary key clustered, a int, key ia(a))") + tk.MustExec("insert into tu values " + strings.Join(vals, ",")) + tk.MustExec("analyze table tu all columns") + estRows, opInfo = indexScanRow("select * from tu use index(ia) where a = 5 and id in (11, 22)") + require.Contains(t, opInfo, "range:[5,5]", "unsigned handle must not extend the execution range") + require.NotContains(t, opInfo, "5 11", "unsigned handle must not extend the execution range") + require.Equal(t, "10.00", estRows) + estRows, opInfo = indexScanRow("select * from tu use index(ia) where a = 5 and id > 10") + require.Contains(t, opInfo, "range:[5,5]", "unsigned handle must not extend the execution range") require.Equal(t, "10.00", estRows) } diff --git a/pkg/planner/core/stats.go b/pkg/planner/core/stats.go index cd96b75cc9c1a..02ccbfa8f9d2e 100644 --- a/pkg/planner/core/stats.go +++ b/pkg/planner/core/stats.go @@ -202,6 +202,20 @@ func fillIndexPath(ds *logicalop.DataSource, path *util.AccessPath, conds []expr return err } +// pathRangesIncludeAppendedHandle reports whether the ranges of a non-unique index path +// extend past the declared index columns into the appended handle columns. +func pathRangesIncludeAppendedHandle(path *util.AccessPath) bool { + if path.Index == nil || len(path.IdxCols) <= len(path.Index.Columns) { + return false + } + for _, ran := range path.Ranges { + if len(ran.LowVal) > len(path.Index.Columns) || len(ran.HighVal) > len(path.Index.Columns) { + return true + } + } + return false +} + // tryAppendCommonHandleColsToIndexPath is the common-handle counterpart of the int-handle // append in fillIndexPath. The key of a non-unique secondary index on a clustered table // physically ends with the full common handle, so appending the primary key columns to the @@ -258,6 +272,21 @@ func adjustCountAfterAccess(ds *logicalop.DataSource, path *util.AccessPath) { // We prefer the `stats.RowCount` to provide consistency in estimation across all paths. // Add an arbitrary tolerance factor to account for comparison with floating point if (path.CountAfterAccess + cost.ToleranceFactor) < ds.StatsInfo().RowCount { + // When the ranges include the appended handle columns, the handle predicates were + // credited with deliberately damped exponential backoff, so falling below the + // independence-leaning stats.RowCount is expected rather than a sign of + // inconsistent assumptions. Align to stats.RowCount without the SelectionFactor + // penalty so the credited path is not made more expensive than an uncredited one. + if pathRangesIncludeAppendedHandle(path) { + if path.MinCountAfterAccess > 0 { + path.MinCountAfterAccess = min(path.MinCountAfterAccess, path.CountAfterAccess) + } else { + path.MinCountAfterAccess = path.CountAfterAccess + } + path.CountAfterAccess = ds.StatsInfo().RowCount + path.MaxCountAfterAccess = max(path.CountAfterAccess, path.MaxCountAfterAccess) + return + } // Store the MinCountAfterAccess "before" adjusting the "CountAfterAccess". This can be used to differentiate // the "Min" estimate for each index/inthandle path when CountAfterAccess has been equalized. if path.MinCountAfterAccess > 0 { @@ -496,8 +525,16 @@ func detachCondAndBuildRangeForPath( } } count, err := cardinality.GetRowCountByIndexRanges(sctx, histColl, path.Index.ID, estimateRanges, indexCols) + if err != nil { + return err + } + if needPruneEstimateRange { + // The pruned estimate gives the appended handle predicates no credit; damp it + // with the handle columns' selectivities. + count = cardinality.AdjustRowCountForAppendedHandleColumns(sctx, histColl, path.Ranges, path.IdxCols, len(indexCols), count) + } path.CountAfterAccess, path.MinCountAfterAccess, path.MaxCountAfterAccess = count.Est, count.MinEst, count.MaxEst - return err + return nil } // pruneEstimateRange truncates ranges built over the index columns plus the appended handle