planner: credit appended handle predicates in index row count estimation

pkg/bindinfo/tests/BUILD.bazel

@@ -11,7 +11,7 @@ go_test(
     ],
     flaky = True,
     race = "on",
-    shard_count = 25,
+    shard_count = 26,
     deps = [
         "//pkg/bindinfo",
         "//pkg/domain",

pkg/planner/cardinality/row_count_index.go

@@ -16,6 +16,7 @@ package cardinality
 
 import (
 	"bytes"
+	"math"
 	"slices"
 	"strings"
 	"time"
@@ -568,6 +569,119 @@ func expBackoffEstimation(sctx planctx.PlanContext, idx *statistics.Index, coll
 	return multResult, minSel, maxSel, true, nil
 }
 
+// AdjustRowCountForAppendedHandleColumns damps a row count estimated from the declared
+// index columns with the selectivity of the handle columns that fillIndexPath appended
+// to the range columns of a non-unique index path. Index statistics only cover the
+// declared columns, so prefixCount was computed from ranges pruned back to
+// declaredColCnt dimensions and gives the appended handle predicates no credit; without
+// an adjustment, a path whose benefit is the handle seek looks as expensive as one
+// without it. Following expBackoffEstimation, the prefix estimate keeps the full-weight
+// slot and each handle column contributes sel^(1/2), sel^(1/4), ... starting from the
+// most selective one, which credits the handle predicates without assuming full
+// independence between the index columns and the primary key. The full ranges must not
+// reach the index statistics directly: bounds encoded from the appended dimensions sort
+// past the truncated statistics keys and would collapse the estimate.
+//
+// idxColsWithHandle must be the declared index columns followed by the complete handle:
+// fillIndexPath only appends the handle when the table's primary key is the single
+// integer handle column, so the dimensions past declaredColCnt always identify a row
+// exactly. A partial handle suffix would break the full-point cap below.
+func AdjustRowCountForAppendedHandleColumns(
+	sctx planctx.PlanContext,
+	coll *statistics.HistColl,
+	ranges []*ranger.Range,
+	idxColsWithHandle []*expression.Column,
+	declaredColCnt int,
+	prefixCount statistics.RowEstimate,
+) statistics.RowEstimate {
+	realtimeCount := float64(coll.RealtimeCount)
+	if realtimeCount <= 0 || len(ranges) == 0 || len(idxColsWithHandle) <= declaredColCnt {
+		return prefixCount
+	}
+	sels := make([]float64, 0, len(idxColsWithHandle)-declaredColCnt)
+	for dim := declaredColCnt; dim < len(idxColsWithHandle); dim++ {
+		col := idxColsWithHandle[dim]
+		if col == nil || statistics.ColumnStatsIsInvalid(coll.GetCol(col.UniqueID), sctx, coll, col.UniqueID) {
+			continue
+		}
+		colRanges := make(ranger.Ranges, 0, len(ranges))
+		allBound := true
+		for _, ran := range ranges {
+			if len(ran.LowVal) <= dim || len(ran.HighVal) <= dim {
+				// Some range does not constrain this dimension, so the column is not
+				// bound across the whole path and must not contribute selectivity.
+				allBound = false
+				break
+			}
+			colRanges = append(colRanges, &ranger.Range{
+				LowVal:    []types.Datum{ran.LowVal[dim]},
+				HighVal:   []types.Datum{ran.HighVal[dim]},
+				Collators: []collate.Collator{ran.Collators[dim]},
+				// The exclusion flags of a multi-column range apply to its last dimension.
+				LowExclude:  ran.LowExclude && dim == len(ran.LowVal)-1,
+				HighExclude: ran.HighExclude && dim == len(ran.HighVal)-1,
+			})
+		}
+		if !allBound {
+			continue
+		}
+		// Ranges that differ only in earlier dimensions repeat the same handle bound;
+		// merge them so the column row count is not summed once per range.
+		merged, err := ranger.UnionRanges(sctx.GetRangerCtx(), colRanges, false)
+		if err != nil {
+			continue
+		}
+		countEst, err := GetRowCountByColumnRanges(sctx, coll, col.UniqueID, merged, false)
+		if err != nil {
+			continue
+		}
+		if sel := countEst.Est / realtimeCount; sel > 0 && sel < 1 {
+			sels = append(sels, sel)
+		}
+	}
+	adjusted := prefixCount
+	if len(sels) > 0 {
+		slices.Sort(sels)
+		factor, indepFactor := 1.0, 1.0
+		for i, sel := range sels {
+			indepFactor *= sel
+			// The prefix estimate occupies the full-weight slot, so the i-th handle
+			// selectivity gets weight 1/2^(i+1).
+			if i+1 < MaxExponentialBackoffCols {
+				for range i + 1 {
+					sel = math.Sqrt(sel)
+				}
+				factor *= sel
+			}
+		}
+		adjusted.Est *= factor
+		// Damping should not push the estimate below 1 row unless the prefix estimate is already < 1.
+		adjusted.Est = max(adjusted.Est, min(prefixCount.Est, 1))
+		// Full independence gives the most optimistic count; the unadjusted prefix
+		// estimate remains the upper bound in MaxEst.
+		adjusted.MinEst = min(adjusted.MinEst*indepFactor, adjusted.Est)
+	}
+	// A point range over the declared columns plus the full handle identifies at most
+	// one row, because the physical key of a non-unique index ends with the complete
+	// handle and is therefore unique. This relies on the contract above: the appended
+	// dimensions cover the complete handle, not a prefix of a multi-column primary key.
+	fullPoints := true
+	for _, ran := range ranges {
+		if len(ran.LowVal) != len(idxColsWithHandle) || len(ran.HighVal) != len(idxColsWithHandle) ||
+			!ran.IsPoint(sctx.GetRangerCtx()) {
+			fullPoints = false
+			break
+		}
+	}
+	if fullPoints {
+		pointCap := float64(len(ranges))
+		adjusted.Est = min(adjusted.Est, pointCap)
+		adjusted.MinEst = min(adjusted.MinEst, adjusted.Est)
+		adjusted.MaxEst = min(adjusted.MaxEst, pointCap)
+	}
+	return adjusted
+}
+
 // outOfRangeOnIndex checks if the datum is out of the range.
 func outOfRangeOnIndex(idx *statistics.Index, val types.Datum) bool {
 	if !idx.Histogram.OutOfRange(val) {

pkg/planner/cardinality/selectivity_test.go

@@ -1743,22 +1743,53 @@ func TestIndexRangeEstimationWithTruncatedHandleRange(t *testing.T) {
 		return "", ""
 	}
 
-	// Each distinct value of a has 10 rows; the handle predicates below must estimate as if
-	// the handle dimension were absent, i.e. the 10 rows of a = 5.
-
-	// Handle range with an exclusive low bound.
+	// Each distinct value of a has 10 rows. The estimate starts from the 10 rows of the
+	// a = 5 prefix (index statistics only cover the declared column), is damped by the
+	// handle column's selectivity with exponential backoff, and is then aligned upward to
+	// stats.RowCount (the Selectivity() result over all predicates) for consistency —
+	// without the SelectionFactor penalty that adjustCountAfterAccess would otherwise add.
+
+	// Handle range with an exclusive low bound. Before the exclusion-flag fix the pruned
+	// range collapsed to the empty (5, 5] and the consistency penalty produced 12.50.
+	// Non-point handle predicates receive no net credit yet: Selectivity() floors the
+	// backoff result at 1/NDV of the declared index columns, so the aligned estimate
+	// stays at the prefix row count.
 	estRows, opInfo := indexScanRow("select * from t use index(ia) where a = 5 and id > 10")
 	require.Contains(t, opInfo, "range:(5 10,5 +inf]", "execution range must keep the handle dimension")
 	require.Equal(t, "10.00", estRows)
 
-	// Handle range with an exclusive high bound.
+	// Handle range with an exclusive high bound: same shape as above.
 	estRows, opInfo = indexScanRow("select * from t use index(ia) where a = 5 and id < 10")
 	require.Contains(t, opInfo, "range:[5 -inf,5 10)", "execution range must keep the handle dimension")
 	require.Equal(t, "10.00", estRows)
 
-	// Multiple handle points under the same prefix must not double count.
+	// Point-bound handle predicates get real credit. The damped estimate
+	// 10 * sqrt(sel(id in (11, 22))) = 1.41 aligns to the Selectivity() result of 2.00,
+	// instead of the uncredited 10.00 (or 12.50 with the consistency penalty).
 	estRows, opInfo = indexScanRow("select * from t use index(ia) where a = 5 and id in (11, 22)")
 	require.Contains(t, opInfo, "range:[5 11,5 11], [5 22,5 22]", "execution range must keep the handle dimension")
+	require.Equal(t, "2.00", estRows)
+
+	// A point over the index column plus the full handle matches at most one row, because
+	// the physical key of a non-unique index ends with the complete handle.
+	estRows, opInfo = indexScanRow("select * from t use index(ia) where a = 5 and id = 7")
+	require.Contains(t, opInfo, "range:[5 7,5 7]", "execution range must keep the handle dimension")
+	require.Equal(t, "1.00", estRows)
+
+	// An unsigned int handle is stored in the index key suffix in signed-encoded order,
+	// which wraps at the int64 boundary: values in [MaxInt64+1, MaxUint64] sort before
+	// [0, MaxInt64]. fillIndexPath therefore never appends an unsigned handle to the
+	// index columns, so unsigned handle predicates must stay out of the index ranges and
+	// receive no appended-handle credit: the estimate remains the prefix row count.
+	tk.MustExec("create table tu(id bigint unsigned primary key clustered, a int, key ia(a))")
+	tk.MustExec("insert into tu values " + strings.Join(vals, ","))
+	tk.MustExec("analyze table tu all columns")
+	estRows, opInfo = indexScanRow("select * from tu use index(ia) where a = 5 and id in (11, 22)")
+	require.Contains(t, opInfo, "range:[5,5]", "unsigned handle must not extend the execution range")
+	require.NotContains(t, opInfo, "5 11", "unsigned handle must not extend the execution range")
+	require.Equal(t, "10.00", estRows)
+	estRows, opInfo = indexScanRow("select * from tu use index(ia) where a = 5 and id > 10")
+	require.Contains(t, opInfo, "range:[5,5]", "unsigned handle must not extend the execution range")
 	require.Equal(t, "10.00", estRows)
 }
 

pkg/planner/core/stats.go

@@ -202,6 +202,20 @@ func fillIndexPath(ds *logicalop.DataSource, path *util.AccessPath, conds []expr
 	return err
 }
 
+// pathRangesIncludeAppendedHandle reports whether the ranges of a non-unique index path
+// extend past the declared index columns into the appended handle columns.
+func pathRangesIncludeAppendedHandle(path *util.AccessPath) bool {
+	if path.Index == nil || len(path.IdxCols) <= len(path.Index.Columns) {
+		return false
+	}
+	for _, ran := range path.Ranges {
+		if len(ran.LowVal) > len(path.Index.Columns) || len(ran.HighVal) > len(path.Index.Columns) {
+			return true
+		}
+	}
+	return false
+}
+
 // tryAppendCommonHandleColsToIndexPath is the common-handle counterpart of the int-handle
 // append in fillIndexPath. The key of a non-unique secondary index on a clustered table
 // physically ends with the full common handle, so appending the primary key columns to the
@@ -258,6 +272,21 @@ func adjustCountAfterAccess(ds *logicalop.DataSource, path *util.AccessPath) {
 	// We prefer the `stats.RowCount` to provide consistency in estimation across all paths.
 	// Add an arbitrary tolerance factor to account for comparison with floating point
 	if (path.CountAfterAccess + cost.ToleranceFactor) < ds.StatsInfo().RowCount {
+		// When the ranges include the appended handle columns, the handle predicates were
+		// credited with deliberately damped exponential backoff, so falling below the
+		// independence-leaning stats.RowCount is expected rather than a sign of
+		// inconsistent assumptions. Align to stats.RowCount without the SelectionFactor
+		// penalty so the credited path is not made more expensive than an uncredited one.
+		if pathRangesIncludeAppendedHandle(path) {
+			if path.MinCountAfterAccess > 0 {
+				path.MinCountAfterAccess = min(path.MinCountAfterAccess, path.CountAfterAccess)
+			} else {
+				path.MinCountAfterAccess = path.CountAfterAccess
+			}
+			path.CountAfterAccess = ds.StatsInfo().RowCount
+			path.MaxCountAfterAccess = max(path.CountAfterAccess, path.MaxCountAfterAccess)
+			return
+		}
 		// Store the MinCountAfterAccess "before" adjusting the "CountAfterAccess". This can be used to differentiate
 		// the "Min" estimate for each index/inthandle path when CountAfterAccess has been equalized.
 		if path.MinCountAfterAccess > 0 {
@@ -496,8 +525,16 @@ func detachCondAndBuildRangeForPath(
 		}
 	}
 	count, err := cardinality.GetRowCountByIndexRanges(sctx, histColl, path.Index.ID, estimateRanges, indexCols)
+	if err != nil {
+		return err
+	}
+	if needPruneEstimateRange {
+		// The pruned estimate gives the appended handle predicates no credit; damp it
+		// with the handle columns' selectivities.
+		count = cardinality.AdjustRowCountForAppendedHandleColumns(sctx, histColl, path.Ranges, path.IdxCols, len(indexCols), count)
+	}
 	path.CountAfterAccess, path.MinCountAfterAccess, path.MaxCountAfterAccess = count.Est, count.MinEst, count.MaxEst
-	return err
+	return nil
 }
 
 // pruneEstimateRange truncates ranges built over the index columns plus the appended handle