From 4df342164438b4e52ac14bcd51f8bf56f8845239 Mon Sep 17 00:00:00 2001 From: Jerome Jiang Date: Thu, 27 Aug 2026 13:18:27 -0400 Subject: [PATCH] av2-rtc: Tune variance based partition thresholds Tune variance-based partition (VBP) thresholds and sub-block split evaluation for real-time (RTC) encoding mode: 1. Update threshold_base formula to properly model AV2's exponential quantizer step size across QP, replacing the linear AC quantizer scale. 2. Refine resolution-based threshold scaling across <= 288p, 360p, 720p, and 1080p+, and smooth the high-QP scaling slope. 3. Enhance sub-block variance spread evaluation to avoid unnecessary 8x8 splits on static/homogeneous blocks while preserving splits for fine details and screen content. 4. Guard 64x64 block splitting based on maximum sub-block variance. BD-rate and speedup summary: rtc_derf (20 clips): avg_psnr: -2.17% (median: -2.29%) ovr_psnr: -2.13% (median: -2.29%) vmaf: -3.60% (median: -5.14%) encoding speedup: +1.25% (median: +0.47%) rtc (28 clips): avg_psnr: -3.04% (median: -2.80%) ovr_psnr: -3.03% (median: -2.79%) vmaf: -4.97% (median: -5.01%) encoding speedup: +2.15% (median: +2.55%) STATS_CHANGED for RTC mode --- av2/encoder/var_based_part.c | 180 +++++++++++++++++++++++++++++------ 1 file changed, 151 insertions(+), 29 deletions(-) diff --git a/av2/encoder/var_based_part.c b/av2/encoder/var_based_part.c index 2cf7ad052f..49da2fc4a0 100644 --- a/av2/encoder/var_based_part.c +++ b/av2/encoder/var_based_part.c @@ -32,6 +32,8 @@ // Macros for common video resolutions: width x height // For example, 720p represents video resolution of 1280x720 pixels. +#define RESOLUTION_180P (320 * 180) +#define RESOLUTION_240P (320 * 240) #define RESOLUTION_288P (352 * 288) #define RESOLUTION_360P (640 * 360) #define RESOLUTION_480P (640 * 480) @@ -338,49 +340,145 @@ static inline void fill_variance_4x4avg(const uint16_t *src_buf, int src_stride, } } +static inline PART_EVAL_STATUS get_part_eval_based_on_sub_blk_var( + VP16x16 *var_16x16_info, int64_t threshold_base, int current_qindex, + bool is_lowres, bool is_screen) { + int max_8x8_var = 0, min_8x8_var = INT_MAX; + for (int split_idx = 0; split_idx < 4; split_idx++) { + get_variance(&var_16x16_info->split[split_idx].part_variances.none); + int this_8x8_var = + var_16x16_info->split[split_idx].part_variances.none.variance; + max_8x8_var = AVMMAX(this_8x8_var, max_8x8_var); + min_8x8_var = AVMMIN(this_8x8_var, min_8x8_var); + } + const int diff = max_8x8_var - min_8x8_var; + if (is_screen) { + if (current_qindex > 125) { + return (diff > threshold_base) ? PART_EVAL_ONLY_SPLIT + : PART_EVAL_ALL; + } + return (diff > (threshold_base >> 1)) ? PART_EVAL_ONLY_SPLIT + : PART_EVAL_ALL; + } + if (is_lowres) { + if (current_qindex > 130) { + return (diff > (threshold_base << 1)) ? PART_EVAL_ONLY_SPLIT + : PART_EVAL_ONLY_NONE; + } + return (diff > (threshold_base >> 1)) ? PART_EVAL_ONLY_SPLIT + : PART_EVAL_ALL; + } + if (current_qindex <= 110) { + return (diff > threshold_base) ? PART_EVAL_ONLY_SPLIT : PART_EVAL_ALL; + } + return (diff > (threshold_base << 1)) ? PART_EVAL_ONLY_SPLIT + : PART_EVAL_ONLY_NONE; +} + static inline void tune_thresh_based_on_resolution(AV2_COMP *cpi, int64_t thresholds[], int64_t threshold_base, int current_qindex, int num_pixels) { + const bool is_screen = (cpi->oxcf.tune_cfg.content == AVM_CONTENT_SCREEN); if (num_pixels >= RESOLUTION_720P) thresholds[4] = thresholds[4] << 1; - if (num_pixels <= RESOLUTION_288P) { - const int qindex_low_thr = 200; - const int qindex_high_thr = 220; + if (num_pixels <= RESOLUTION_180P) { + const int qindex_low_thr = 110; + const int qindex_high_thr = 135; + if (current_qindex >= qindex_high_thr) { + thresholds[2] = threshold_base >> 3; + thresholds[3] = (7 * threshold_base) >> 3; + thresholds[4] = threshold_base << 4; + } else if (current_qindex < qindex_low_thr) { + thresholds[2] = threshold_base >> 3; + thresholds[3] = (3 * threshold_base) >> 3; + thresholds[4] = threshold_base << 2; + } else { + int64_t qi_diff_low = current_qindex - qindex_low_thr; + int64_t qi_diff_high = qindex_high_thr - current_qindex; + int64_t threshold_diff = qindex_high_thr - qindex_low_thr; + thresholds[2] = threshold_base >> 3; + thresholds[3] = (qi_diff_low * ((7 * threshold_base) >> 3) + + qi_diff_high * ((3 * threshold_base) >> 3)) / + threshold_diff; + thresholds[4] = (qi_diff_low * (threshold_base << 4) + + qi_diff_high * (threshold_base << 2)) / + threshold_diff; + } + } else if (num_pixels <= RESOLUTION_240P) { + const int qindex_low_thr = 110; + const int qindex_high_thr = 135; + if (current_qindex >= qindex_high_thr) { + thresholds[2] = threshold_base >> 3; + thresholds[3] = (7 * threshold_base) >> 3; + thresholds[4] = threshold_base << 4; + } else if (current_qindex < qindex_low_thr) { + thresholds[2] = threshold_base >> 3; + thresholds[3] = (7 * threshold_base) >> 4; + thresholds[4] = threshold_base << 2; + } else { + int64_t qi_diff_low = current_qindex - qindex_low_thr; + int64_t qi_diff_high = qindex_high_thr - current_qindex; + int64_t threshold_diff = qindex_high_thr - qindex_low_thr; + thresholds[2] = threshold_base >> 3; + thresholds[3] = (qi_diff_low * ((7 * threshold_base) >> 3) + + qi_diff_high * ((7 * threshold_base) >> 4)) / + threshold_diff; + thresholds[4] = (qi_diff_low * (threshold_base << 4) + + qi_diff_high * (threshold_base << 2)) / + threshold_diff; + } + } else if (num_pixels <= RESOLUTION_288P) { + const int qindex_low_thr = 110; + const int qindex_high_thr = 135; if (current_qindex >= qindex_high_thr) { - threshold_base = (5 * threshold_base) >> 1; thresholds[2] = threshold_base >> 3; - thresholds[3] = threshold_base << 2; - thresholds[4] = threshold_base << 5; + thresholds[3] = (9 * threshold_base) >> 2; + thresholds[4] = threshold_base << 4; } else if (current_qindex < qindex_low_thr) { thresholds[2] = threshold_base >> 3; - thresholds[3] = threshold_base >> 1; + thresholds[3] = threshold_base; thresholds[4] = threshold_base << 3; } else { int64_t qi_diff_low = current_qindex - qindex_low_thr; int64_t qi_diff_high = qindex_high_thr - current_qindex; int64_t threshold_diff = qindex_high_thr - qindex_low_thr; - int64_t threshold_base_high = (5 * threshold_base) >> 1; - - threshold_diff = threshold_diff > 0 ? threshold_diff : 1; - threshold_base = - (qi_diff_low * threshold_base_high + qi_diff_high * threshold_base) / - threshold_diff; thresholds[2] = threshold_base >> 3; - thresholds[3] = ((qi_diff_low * threshold_base) + - qi_diff_high * (threshold_base >> 1)) / + thresholds[3] = (qi_diff_low * ((9 * threshold_base) >> 2) + + qi_diff_high * threshold_base) / threshold_diff; - thresholds[4] = ((qi_diff_low * (threshold_base << 5)) + + thresholds[4] = (qi_diff_low * (threshold_base << 4) + qi_diff_high * (threshold_base << 3)) / threshold_diff; } + } else if (num_pixels <= RESOLUTION_360P) { + thresholds[2] = threshold_base; + thresholds[3] = (3 * threshold_base) >> 1; + thresholds[4] = threshold_base << 5; } else if (num_pixels < RESOLUTION_720P) { thresholds[3] = (3 * threshold_base) >> 1; + if (current_qindex > 115) { + const int q_diff = current_qindex - 115; + thresholds[2] = thresholds[2] + ((q_diff * thresholds[2]) >> 5); + thresholds[3] = thresholds[3] + ((q_diff * thresholds[3]) >> 5); + } + if (current_qindex > 135) { + thresholds[4] = INT64_MAX; + } } else if (num_pixels < RESOLUTION_1080P) { thresholds[3] = threshold_base << 1; + if (current_qindex > 115) { + const int q_diff = current_qindex - 115; + thresholds[1] = thresholds[1] + ((q_diff * thresholds[1]) >> 5); + thresholds[2] = thresholds[2] + ((q_diff * thresholds[2]) >> 5); + thresholds[3] = thresholds[3] + ((q_diff * thresholds[3]) >> 5); + } + if (current_qindex > 135) { + thresholds[4] = INT64_MAX; + } } else { // num_pixels >= RESOLUTION_1080P - if (cpi->oxcf.tune_cfg.content == AVM_CONTENT_SCREEN) { + if (is_screen) { if (num_pixels < RESOLUTION_1440P) { thresholds[3] = (5 * threshold_base) >> 1; } else { @@ -391,6 +489,9 @@ static inline void tune_thresh_based_on_resolution(AV2_COMP *cpi, thresholds[3] = threshold_base << 2; } } + if (current_qindex > 135) { + thresholds[4] = INT64_MAX; + } } } @@ -435,8 +536,18 @@ static inline void set_vbp_thresholds(AV2_COMP *cpi, int64_t thresholds[], AV2_COMMON *const cm = &cpi->common; const int is_key_frame = frame_is_intra_only(cm); const int threshold_multiplier = is_key_frame ? 120 : 1; - const int ac_q = av2_ac_quant_QTX(qindex, 0, 0, cm->seq_params.bit_depth); - int64_t threshold_base = (int64_t)(threshold_multiplier * ac_q) >> 3; + // In AV2, qindex is linear in dB (exponential step size 2^(q/24)), whereas + // AV1 had a linear quantizer table. We compute threshold_base directly as a + // function of qindex to account for the exponential range in AV2 vs linear in AV1. + int64_t threshold_base; + if (qindex <= 65) { + threshold_base = 12; + } else { + const int64_t q_offset = qindex - 65; + threshold_base = 12 + ((q_offset * (qindex + 15)) / 40); + } + threshold_base *= threshold_multiplier; + const int current_qindex = cm->quant_params.base_qindex; const int threshold_left_shift = 7; const int num_pixels = cm->width * cm->height; @@ -494,7 +605,8 @@ static inline void set_vbp_thresholds(AV2_COMP *cpi, int64_t thresholds[], * \param[in] sb_size Superblock size (BLOCK_64X64, BLOCK_128X128, or * BLOCK_256X256) */ -static void fill_variance_tree_leaves(MACROBLOCK *x, VP128x128 *vt, +static void fill_variance_tree_leaves(AV2_COMP *cpi, MACROBLOCK *x, + VP128x128 *vt, PART_EVAL_STATUS *force_split, int avg_16x16[][4], int maxvar_16x16[][4], int minvar_16x16[][4], @@ -518,6 +630,9 @@ static void fill_variance_tree_leaves(MACROBLOCK *x, VP128x128 *vt, const int offset_64x64 = offsets.offset_64x64; const int offset_32x32 = offsets.offset_32x32; const int offset_16x16 = offsets.offset_16x16; + const bool is_lowres = + (cpi->common.width * cpi->common.height <= RESOLUTION_288P); + const bool is_screen = (cpi->oxcf.tune_cfg.content == AVM_CONTENT_SCREEN); for (int blk64_idx = 0; blk64_idx < num_64x64_blocks; blk64_idx++) { int x64_idx, y64_idx; VP64x64 *vt64; @@ -571,10 +686,12 @@ static void fill_variance_tree_leaves(MACROBLOCK *x, VP128x128 *vt, maxvar_16x16[blk64_idx][lvl1_idx] = AVMMAX(maxvar_16x16[blk64_idx][lvl1_idx], val_none_var); if (val_none_var > thresholds[4]) { - // 16X16 variance is above threshold for split, so force split to - // 8x8 for this 16x16 block (this also forces splits for upper - // levels). - force_split[split_index] = PART_EVAL_ONLY_SPLIT; + // 16X16 variance is above threshold for split, so evaluate + // split based on sub-block variance variation (this also forces + // splits for upper levels). + const int current_qindex = cpi->common.quant_params.base_qindex; + force_split[split_index] = get_part_eval_based_on_sub_blk_var( + vst, thresholds[2], current_qindex, is_lowres, is_screen); force_split[offset_32x32 + blk64_scale_idx + lvl1_idx] = PART_EVAL_ONLY_SPLIT; force_split[blk64_idx + offset_64x64] = PART_EVAL_ONLY_SPLIT; @@ -771,7 +888,7 @@ void av2_choose_var_based_partitioning(AV2_COMP *cpi, // Fill in the entire tree of 8x8 (for inter frames) or 4x4 (for key frames) // variances for splits. - fill_variance_tree_leaves(x, vt, force_split, avg_16x16, maxvar_16x16, + fill_variance_tree_leaves(cpi, x, vt, force_split, avg_16x16, maxvar_16x16, minvar_16x16, thresholds, src_buf, src_stride, dst_buf, dst_stride, is_key_frame, cm->seq_params.sb_size); @@ -780,6 +897,8 @@ void av2_choose_var_based_partitioning(AV2_COMP *cpi, const int offset_64x64 = offsets.offset_64x64; const int offset_32x32 = offsets.offset_32x32; const int offset_16x16 = offsets.offset_16x16; + const bool is_lowres = (cm->width * cm->height <= RESOLUTION_288P); + const bool is_screen = (cpi->oxcf.tune_cfg.content == AVM_CONTENT_SCREEN); avg_64x64 = 0; for (int blk64_idx = 0; blk64_idx < num_64x64_blocks; ++blk64_idx) { @@ -802,7 +921,9 @@ void av2_choose_var_based_partitioning(AV2_COMP *cpi, get_variance(&vtemp->part_variances.none); if (vtemp->part_variances.none.variance > thresholds[4]) { const int split_index = offset_16x16 + lvl1_scale_idx + lvl2_idx; - force_split[split_index] = PART_EVAL_ONLY_SPLIT; + const int current_qindex = cm->quant_params.base_qindex; + force_split[split_index] = get_part_eval_based_on_sub_blk_var( + vtemp, thresholds[2], current_qindex, is_lowres, is_screen); force_split[offset_32x32 + blk64_scale_idx + lvl1_idx] = PART_EVAL_ONLY_SPLIT; force_split[blk64_idx + offset_64x64] = PART_EVAL_ONLY_SPLIT; @@ -851,13 +972,14 @@ void av2_choose_var_based_partitioning(AV2_COMP *cpi, min_var_64x64 = AVMMIN(var_64x64, min_var_64x64); // If the difference of the max-min variances of sub-blocks or max // variance of a sub-block is above some threshold of then force this - // block to split. Only checking this for noise level >= medium, if - // encoder is in SVC or if we already forced large blocks. + // block to split. const int max_min_var_32x32_diff = max_var_32x32[blk64_idx] - min_var_32x32[blk64_idx]; + const int check_max_var = max_var_32x32[blk64_idx] > (thresholds[2] >> 1); const int64_t set_threshold = 3 * (thresholds[2] >> 3); - if (!is_key_frame && max_min_var_32x32_diff > set_threshold) { + if (!is_key_frame && max_min_var_32x32_diff > set_threshold && + check_max_var) { force_split[offset_64x64 + blk64_idx] = PART_EVAL_ONLY_SPLIT; force_split[0] = PART_EVAL_ONLY_SPLIT; }