From 55713f4b7b745650c74f181d34b576cf99b185e3 Mon Sep 17 00:00:00 2001 From: Mudassir Galaganath Date: Sun, 23 Aug 2026 22:55:49 +0530 Subject: [PATCH 1/5] Unify rectangular sub-block search in av2_rd_pick_inter_mode_sb This change unifies the rectangular sub-block search to improve the readabilty and avoid code duplication. Unified code is extracted to new function has_searched_rect_subblock() which scans HORZ/VERT sub-blocks for a previously searched partition. No stats changed Change-Id: I331cf4a82fadc549122a91f2acb7f1e9eef1b2cd --- av2/encoder/rdopt.c | 92 ++++++++++++++++----------------------------- 1 file changed, 33 insertions(+), 59 deletions(-) diff --git a/av2/encoder/rdopt.c b/av2/encoder/rdopt.c index c0d508c81b..16688e04e9 100644 --- a/av2/encoder/rdopt.c +++ b/av2/encoder/rdopt.c @@ -9138,6 +9138,36 @@ static_assert(sizeof(ref_frame_centric_eval_order) / 210, "ref_frame_centric_eval_order size changed; audit callers."); +// Returns true if any rectangular (HORZ or VERT) sub-block of `bsize` at +// (mi_row, mi_col) has a previously searched partition recorded. +static bool has_searched_rect_subblock(MACROBLOCK *x, int mi_row, int mi_col, + BLOCK_SIZE bsize, BLOCK_SIZE sb_size, + int8_t region_type) { + for (RECT_PART_TYPE rect_type = HORZ; rect_type < NUM_RECT_PARTS; + ++rect_type) { + const PARTITION_TYPE part = + (rect_type == HORZ) ? PARTITION_HORZ : PARTITION_VERT; + const BLOCK_SIZE subsize = get_partition_subsize(bsize, part); + if (subsize == BLOCK_INVALID) continue; + + const int limit = + (rect_type == HORZ) ? mi_size_high[bsize] / 2 : mi_size_wide[bsize] / 2; + // Extended ratio blocks (1:4, 4:1) step through every mi offset (step = + // 1). Standard 1:2 / 2:1 and square blocks step directly to the two + // sub-block origins (step = limit). + const int step = (bsize > BLOCK_LARGEST) ? 1 : AVMMAX(1, limit); + + for (int offset = 0; offset <= limit; offset += step) { + const int r = (rect_type == HORZ) ? offset : 0; + const int c = (rect_type == HORZ) ? 0 : offset; + const PARTITION_TYPE prev_part = av2_get_prev_partition( + x, mi_row + r, mi_col + c, subsize, sb_size, region_type); + if (prev_part != PARTITION_INVALID) return true; + } + } + return false; +} + // TODO(chiyotsai@google.com): See the todo for av2_rd_pick_intra_mode_sb. void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, struct TileDataEnc *tile_data, @@ -9259,69 +9289,13 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, // Ref frames that are selected by square partition blocks. uint64_t picked_ref_frames_mask = 0; if (cpi->sf.inter_sf.prune_ref_frames && !x->inter_mode_cache[0]) { - bool prune_ref_frames = false; assert(should_reuse_mode(x, REUSE_PARTITION_MODE_FLAG)); // Prune reference frames if we are either a 1:4 block, or if we are a 1:2 // block, and we have searched any of the rectangular subblock. - if (!is_partition_point(bsize)) { - prune_ref_frames = true; - } else if (bsize > BLOCK_LARGEST) { - // Check horz sub-blocks at different row offsets. - BLOCK_SIZE subsize = get_partition_subsize(bsize, PARTITION_HORZ); - if (subsize != BLOCK_INVALID) { - for (int r = 0; r <= mi_size_high[bsize] / 2; ++r) { - const PARTITION_TYPE prev_part = - av2_get_prev_partition(x, xd->mi_row + r, xd->mi_col, subsize, - cm->sb_size, (int8_t)mbmi->region_type); - if (prev_part != PARTITION_INVALID) { - prune_ref_frames = true; - break; - } - } - } - // Check vert sub-blocks at different col offsets. - subsize = get_partition_subsize(bsize, PARTITION_VERT); - if (subsize != BLOCK_INVALID) { - for (int c = 0; c <= mi_size_wide[bsize] / 2; ++c) { - const PARTITION_TYPE prev_part = - av2_get_prev_partition(x, xd->mi_row, xd->mi_col + c, subsize, - cm->sb_size, (int8_t)mbmi->region_type); - if (prev_part != PARTITION_INVALID) { - prune_ref_frames = true; - break; - } - } - } - } else { - for (RECT_PART_TYPE rect_type = HORZ; rect_type < NUM_RECT_PARTS; - rect_type++) { - const int mi_pos_rect[NUM_RECT_PARTS][SUB_PARTITIONS_RECT][2] = { - { { xd->mi_row, xd->mi_col }, - { xd->mi_row + mi_size_high[bsize] / 2, xd->mi_col } }, - { { xd->mi_row, xd->mi_col }, - { xd->mi_row, xd->mi_col + mi_size_wide[bsize] / 2 } } - }; - const PARTITION_TYPE part = - (rect_type == HORZ) ? PARTITION_HORZ : PARTITION_VERT; - const BLOCK_SIZE subsize = get_partition_subsize(bsize, part); - if (subsize == BLOCK_INVALID) { - continue; - } - for (int sub_idx = 0; sub_idx < 2; sub_idx++) { - const PARTITION_TYPE prev_part = av2_get_prev_partition( - x, mi_pos_rect[rect_type][sub_idx][0], - mi_pos_rect[rect_type][sub_idx][1], subsize, cm->sb_size, - (int8_t)mbmi->region_type); - if (prev_part != PARTITION_INVALID) { - prune_ref_frames = true; - break; - } - } - } - } - - if (prune_ref_frames) { + if (!is_partition_point(bsize) || + has_searched_rect_subblock(x, xd->mi_row, xd->mi_col, bsize, + cm->sb_size, (int8_t)mbmi->region_type)) { picked_ref_frames_mask = fetch_picked_ref_frames_mask(x, bsize, cm->mib_size); } From c691d1f009f14c5c761bce3c32f1c6337f5efb33 Mon Sep 17 00:00:00 2001 From: Mudassir Galaganath Date: Mon, 24 Aug 2026 00:59:38 +0530 Subject: [PATCH 2/5] Abstract TPL cost and IntraBC pred code into helper functions - Abstracted the TPL stage inter/intra cost accumulation to calculate_cost_from_tpl_data(). - Abstracted the IntraBC prediction code to try_intrabc_after_inter_search(). No stats changed Change-Id: Icc70b559709078236cc96b4d3af34c5d10cff6ac --- av2/encoder/rdopt.c | 200 ++++++++++++++++++++++++-------------------- 1 file changed, 111 insertions(+), 89 deletions(-) diff --git a/av2/encoder/rdopt.c b/av2/encoder/rdopt.c index 16688e04e9..944f69c7ec 100644 --- a/av2/encoder/rdopt.c +++ b/av2/encoder/rdopt.c @@ -9168,6 +9168,113 @@ static bool has_searched_rect_subblock(MACROBLOCK *x, int mi_row, int mi_col, return false; } +// Prepare inter_cost and intra_cost from TPL stats, which are used as ML +// features in intra mode pruning. +static AVM_INLINE void calculate_cost_from_tpl_data( + const AV2_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, int mi_row, + int mi_col, int64_t *inter_cost, int64_t *intra_cost) { + const AV2_COMMON *const cm = &cpi->common; + // Only consider full SB. + const BLOCK_SIZE sb_size = cm->sb_size; + const int tpl_bsize_1d = cpi->tpl_data.tpl_bsize_1d; + const int len = (block_size_wide[sb_size] / tpl_bsize_1d) * + (block_size_high[sb_size] / tpl_bsize_1d); + SuperBlockEnc *sb_enc = &x->sb_enc; + if (sb_enc->tpl_data_count == len) { + const BLOCK_SIZE tpl_bsize = convert_length_to_bsize(tpl_bsize_1d); + const int tpl_stride = sb_enc->tpl_stride; + const int tplw = mi_size_wide[tpl_bsize]; + const int tplh = mi_size_high[tpl_bsize]; + const int nw = mi_size_wide[bsize] / tplw; + const int nh = mi_size_high[bsize] / tplh; + if (nw >= 1 && nh >= 1) { + const int of_h = mi_row % mi_size_high[sb_size]; + const int of_w = mi_col % mi_size_wide[sb_size]; + const int start = of_h / tplh * tpl_stride + of_w / tplw; + + for (int k = 0; k < nh; ++k) { + for (int l = 0; l < nw; ++l) { + *inter_cost += sb_enc->tpl_inter_cost[start + k * tpl_stride + l]; + *intra_cost += sb_enc->tpl_intra_cost[start + k * tpl_stride + l]; + } + } + *inter_cost /= nw * nh; + *intra_cost /= nw * nh; + } + } +} + +// Tries intrabc prediction as a final candidate after the inter/intra mode +// search. +static void try_intrabc_after_inter_search( + AV2_COMP *cpi, MACROBLOCK *x, PICK_MODE_CONTEXT *ctx, BLOCK_SIZE bsize, + unsigned int intra_ref_frame_cost, int is_intra_mode_allowed, + InterModeSearchState *search_state, RD_STATS *rd_cost) { + if (search_state->best_skip2 != 0) return; + + const AV2_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = xd->mi[0]; + + const int try_intrabc = + cpi->oxcf.kf_cfg.enable_intrabc && cpi->oxcf.kf_cfg.enable_intrabc_ext && + !cpi->sf.inter_sf.skip_eval_intrabc_in_inter_frame && + av2_allow_intrabc(cm, xd, bsize) && (xd->tree_type != CHROMA_PART); + if (!(try_intrabc && is_intra_mode_allowed)) return; + + RD_STATS this_rd_cost; + this_rd_cost.rdcost = INT64_MAX; + mbmi->ref_frame[0] = INTRA_FRAME; + mbmi->ref_frame[1] = NONE_FRAME; + mbmi->use_intrabc[xd->tree_type == CHROMA_PART] = 0; + mbmi->mv[0].as_int = 0; + mbmi->skip_mode = 0; + mbmi->mode = DC_PRED; + mbmi->motion_mode = SIMPLE_TRANSLATION; + mbmi->warp_ref_idx = 0; + mbmi->max_num_warp_candidates = 0; + mbmi->warpmv_with_mvd_flag = 0; + mbmi->morph_pred = 0; + mbmi->six_param_warp_model_flag = 0; + mbmi->warp_precision_idx = 0; + mbmi->warp_inter_intra = 0; + + int skip_interintra_cost = intra_ref_frame_cost; + if (is_skip_mode_allowed(cm, xd)) { + // Compare the use of skip_mode with the best intra/inter mode obtained. + const int skip_mode_ctx = av2_get_skip_mode_context(xd); + skip_interintra_cost += x->mode_costs.skip_mode_cost[skip_mode_ctx][0]; + } + + rd_pick_intrabc_mode_sb(cpi, x, ctx, &this_rd_cost, bsize, INT64_MAX, + skip_interintra_cost); + + if (this_rd_cost.rdcost >= search_state->best_rd) return; + + rd_cost->rate = this_rd_cost.rate; + rd_cost->dist = this_rd_cost.dist; + rd_cost->rdcost = this_rd_cost.rdcost; + + search_state->best_rd = rd_cost->rdcost; + search_state->best_mbmode = *mbmi; + search_state->best_skip2 = mbmi->skip_txfm[xd->tree_type == CHROMA_PART]; + search_state->best_mode_skippable = + mbmi->skip_txfm[xd->tree_type == CHROMA_PART]; + + const int num_planes = av2_num_planes(cm); + TxfmSearchInfo *const txfm_info = &x->txfm_search_info; + for (int i = 0; i < num_planes; ++i) { + const int num_blk_plane = + (i == AVM_PLANE_Y) ? ctx->num_4x4_blk : ctx->num_4x4_blk_chroma; + memcpy(ctx->blk_skip[i], txfm_info->blk_skip[i], + sizeof(*txfm_info->blk_skip[i]) * num_blk_plane); + } + av2_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); + av2_copy_array(ctx->cctx_type_map, xd->cctx_type_map, + ctx->num_4x4_blk_chroma); + ctx->rd_stats.skip_txfm = mbmi->skip_txfm[xd->tree_type == CHROMA_PART]; +} + // TODO(chiyotsai@google.com): See the todo for av2_rd_pick_intra_mode_sb. void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, struct TileDataEnc *tile_data, @@ -9384,34 +9491,8 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, const int do_pruning = (AVMMIN(cm->width, cm->height) > 480 && cpi->speed <= 2) ? 0 : 1; if (do_pruning && sf->intra_sf.skip_intra_in_interframe) { - // Only consider full SB. - const BLOCK_SIZE sb_size = cm->sb_size; - const int tpl_bsize_1d = cpi->tpl_data.tpl_bsize_1d; - const int len = (block_size_wide[sb_size] / tpl_bsize_1d) * - (block_size_high[sb_size] / tpl_bsize_1d); - SuperBlockEnc *sb_enc = &x->sb_enc; - if (sb_enc->tpl_data_count == len) { - const BLOCK_SIZE tpl_bsize = convert_length_to_bsize(tpl_bsize_1d); - const int tpl_stride = sb_enc->tpl_stride; - const int tplw = mi_size_wide[tpl_bsize]; - const int tplh = mi_size_high[tpl_bsize]; - const int nw = mi_size_wide[bsize] / tplw; - const int nh = mi_size_high[bsize] / tplh; - if (nw >= 1 && nh >= 1) { - const int of_h = mi_row % mi_size_high[sb_size]; - const int of_w = mi_col % mi_size_wide[sb_size]; - const int start = of_h / tplh * tpl_stride + of_w / tplw; - - for (int k = 0; k < nh; k++) { - for (int l = 0; l < nw; l++) { - inter_cost += sb_enc->tpl_inter_cost[start + k * tpl_stride + l]; - intra_cost += sb_enc->tpl_intra_cost[start + k * tpl_stride + l]; - } - } - inter_cost /= nw * nh; - intra_cost /= nw * nh; - } - } + calculate_cost_from_tpl_data(cpi, x, bsize, mi_row, mi_col, &inter_cost, + &intra_cost); } // Initialize best mode stats for winner mode processing @@ -9802,67 +9883,8 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, rd_pick_skip_mode(&search_state, cpi, x, bsize, yv12_mb, ctx, rd_cost); } - if (search_state.best_skip2 == 0) { - const int try_intrabc = cpi->oxcf.kf_cfg.enable_intrabc && - cpi->oxcf.kf_cfg.enable_intrabc_ext && - !sf->inter_sf.skip_eval_intrabc_in_inter_frame && - av2_allow_intrabc(cm, xd, bsize - - ) && - (xd->tree_type != CHROMA_PART); - if (try_intrabc && is_intra_mode_allowed) { - RD_STATS this_rd_cost; - this_rd_cost.rdcost = INT64_MAX; - mbmi->ref_frame[0] = INTRA_FRAME; - mbmi->ref_frame[1] = NONE_FRAME; - mbmi->use_intrabc[xd->tree_type == CHROMA_PART] = 0; - mbmi->mv[0].as_int = 0; - mbmi->skip_mode = 0; - mbmi->mode = 0; - mbmi->motion_mode = SIMPLE_TRANSLATION; - mbmi->warp_ref_idx = 0; - mbmi->max_num_warp_candidates = 0; - mbmi->warpmv_with_mvd_flag = 0; - mbmi->morph_pred = 0; - mbmi->six_param_warp_model_flag = 0; - mbmi->warp_precision_idx = 0; - - mbmi->warp_inter_intra = 0; - - int skip_interintra_cost = intra_ref_frame_cost; - if (is_skip_mode_allowed(cm, xd)) { - // Compare the use of skip_mode with the best intra/inter mode obtained. - const int skip_mode_ctx = av2_get_skip_mode_context(xd); - skip_interintra_cost += x->mode_costs.skip_mode_cost[skip_mode_ctx][0]; - } - - rd_pick_intrabc_mode_sb(cpi, x, ctx, &this_rd_cost, bsize, INT64_MAX, - skip_interintra_cost); - - if (this_rd_cost.rdcost < search_state.best_rd) { - rd_cost->rate = this_rd_cost.rate; - rd_cost->dist = this_rd_cost.dist; - rd_cost->rdcost = this_rd_cost.rdcost; - - search_state.best_rd = rd_cost->rdcost; - search_state.best_mbmode = *mbmi; - search_state.best_skip2 = mbmi->skip_txfm[xd->tree_type == CHROMA_PART]; - search_state.best_mode_skippable = - mbmi->skip_txfm[xd->tree_type == CHROMA_PART]; - - for (i = 0; i < num_planes; ++i) { - const int num_blk_plane = - (i == AVM_PLANE_Y) ? ctx->num_4x4_blk : ctx->num_4x4_blk_chroma; - memcpy(ctx->blk_skip[i], txfm_info->blk_skip[i], - sizeof(*txfm_info->blk_skip[i]) * num_blk_plane); - } - av2_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk); - av2_copy_array(ctx->cctx_type_map, xd->cctx_type_map, - ctx->num_4x4_blk_chroma); - ctx->rd_stats.skip_txfm = mbmi->skip_txfm[xd->tree_type == CHROMA_PART]; - } - } - } + try_intrabc_after_inter_search(cpi, x, ctx, bsize, intra_ref_frame_cost, + is_intra_mode_allowed, &search_state, rd_cost); // Make sure that the ref_mv_idx is only nonzero when we're // using a mode which can support ref_mv_idx From 3bf1bbe06198553672907b0e3761be13fbc9c8ea Mon Sep 17 00:00:00 2001 From: Mudassir Galaganath Date: Mon, 24 Aug 2026 01:05:17 +0530 Subject: [PATCH 3/5] Move loop-invariant conditions out of mode evaluation loop - Hoist the frame/sequence level reads out of the mode loop into locals. - Fold the loop-invariant BLOCK_4X4 check into the mode loop bound. - Flatten the BRU gate from three nesting levels to two. No stats changed Change-Id: I7d9e8fa4794e1959185e9aa39aedd7aad0cfc319 --- av2/encoder/rdopt.c | 52 +++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/av2/encoder/rdopt.c b/av2/encoder/rdopt.c index 944f69c7ec..01bc7b0693 100644 --- a/av2/encoder/rdopt.c +++ b/av2/encoder/rdopt.c @@ -9554,14 +9554,20 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, sf->inter_sf.prune_compound_using_single_ref; const int motion_mode_winner = cpi->sf.winner_mode_sf.motion_mode_for_winner_cand; - const int reference_mode_select = - cm->current_frame.reference_mode == REFERENCE_MODE_SELECT; + const int reference_mode = cm->current_frame.reference_mode; + const int reference_mode_select = reference_mode == REFERENCE_MODE_SELECT; const int num_total_refs = cm->ref_frames_info.num_total_refs; + const int num_same_ref_compound = cm->ref_frames_info.num_same_ref_compound; + const int has_both_sides_refs = cm->has_both_sides_refs; + const int ref_frame_flags = cm->ref_frame_flags; + const int enable_joint_mvd = cm->seq_params.enable_joint_mvd; + const int enable_adaptive_mvd = cm->seq_params.enable_adaptive_mvd; + const int bru_enabled = cm->bru.enabled; + const int bru_update_ref_idx = cm->bru.update_ref_idx; const int tip_allowed = is_tip_allowed(cm, xd); const int comp_ref_allowed = - cm->current_frame.reference_mode != SINGLE_REFERENCE && - is_comp_ref_allowed(bsize); + reference_mode != SINGLE_REFERENCE && is_comp_ref_allowed(bsize); const int warpmv_allowed = (cm->features.enabled_motion_modes & (1 << WARP_DELTA)) != 0 && cm->features.allow_warpmv_mode && is_warpmv_allowed_bsize(bsize); @@ -9577,10 +9583,11 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, cm->features.opfl_refine_type == REFINE_SWITCHABLE && !frame_is_sframe(cm) && !thin_4xn_nx4; - for (int mode_refs_pair_idx = 0; - mode_refs_pair_idx < ref_frame_centric_eval_order_num; + const int num_mode_ref_pairs = + (bsize == BLOCK_4X4) ? 0 : ref_frame_centric_eval_order_num; + + for (int mode_refs_pair_idx = 0; mode_refs_pair_idx < num_mode_ref_pairs; ++mode_refs_pair_idx) { - if (bsize == BLOCK_4X4) break; const PREDICTION_MODE this_mode = ref_frame_centric_eval_order[mode_refs_pair_idx].mode; const MV_REFERENCE_FRAME ref_frame = @@ -9602,7 +9609,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, if ((int)second_ref_frame >= num_total_refs) continue; // Same-ref compound only when permitted at runtime. if (ref_frame == second_ref_frame && - (int)ref_frame >= cm->ref_frames_info.num_same_ref_compound) + (int)ref_frame >= num_same_ref_compound) continue; } @@ -9632,35 +9639,31 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, // top same_ref_compound_rank_cap ranks are kept. if (apply_dry_pass_shortcuts && is_comp_mode && ref_frame == second_ref_frame && - (cm->has_both_sides_refs || + (has_both_sides_refs || ref_frame > dry_pass_cfg.same_ref_compound_rank_cap)) continue; if (this_mode == WARPMV && !warpmv_allowed) continue; if (this_mode == WARP_NEWMV && (!warpmv_allowed || !warp_newmv_allowed)) continue; if (this_mode >= NEAR_NEARMV_OPTFLOW && !opfl_modes_allowed) continue; - if (is_joint_mvd_coding_mode(this_mode) && - cm->seq_params.enable_joint_mvd == 0) - continue; + if (is_joint_mvd_coding_mode(this_mode) && enable_joint_mvd == 0) continue; const int num_amvd_modes = - 1 + (cm->seq_params.enable_adaptive_mvd && allow_amvd_mode(this_mode)); + 1 + (enable_adaptive_mvd && allow_amvd_mode(this_mode)); // Asymmetric NEAR/NEW compound modes default to AMVD on, invert the flag const int amvd_inverted = (this_mode == NEW_NEARMV || this_mode == NEAR_NEWMV || this_mode == NEAR_NEWMV_OPTFLOW || this_mode == NEW_NEARMV_OPTFLOW); - if (cm->bru.enabled) { + if (bru_enabled) { assert(xd->sbi->sb_active_mode == BRU_ACTIVE_SB); - if (xd->sbi->sb_active_mode == BRU_ACTIVE_SB) { - if (cm->bru.update_ref_idx == ref_frame) continue; - if (second_ref_frame != NONE_FRAME && - cm->bru.update_ref_idx == second_ref_frame) - continue; - } + if (xd->sbi->sb_active_mode == BRU_ACTIVE_SB && + (ref_frame == bru_update_ref_idx || + (is_comp_mode && second_ref_frame == bru_update_ref_idx))) + continue; } - if (comp_pred && !(cm->ref_frame_flags & (1 << second_ref_frame))) continue; + if (comp_pred && !(ref_frame_flags & (1 << second_ref_frame))) continue; if (comp_pred && prune_comp_ref_by_priority(&sf->inter_sf, this_mode, ref_frame, second_ref_frame)) { @@ -9793,8 +9796,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, continue; } - assert(IMPLIES(comp_pred, - cm->current_frame.reference_mode != SINGLE_REFERENCE)); + assert(IMPLIES(comp_pred, reference_mode != SINGLE_REFERENCE)); update_search_state(&search_state, rd_cost, ctx, &rd_stats, &rd_stats_y, &rd_stats_uv, this_mode, x, do_tx_search, cm); if (do_tx_search) search_state.best_skip_rd[0] = skip_rd[0]; @@ -9807,8 +9809,8 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, } /* keep record of best compound/single-only prediction */ - record_best_compound(cm->current_frame.reference_mode, &rd_stats, - comp_pred, x->rdmult, &search_state, compmode_cost); + record_best_compound(reference_mode, &rd_stats, comp_pred, x->rdmult, + &search_state, compmode_cost); } // end of use_amvd mode loop } // end of LUT entry loop From 9019e95b7dd467179076f6ce0be6a280a20774ec Mon Sep 17 00:00:00 2001 From: Mudassir Galaganath Date: Tue, 25 Aug 2026 01:22:44 +0530 Subject: [PATCH 4/5] Cosmetic cleanups in av2_rd_pick_inter_mode_sb - Drop is_comp_mode, a second name for comp_pred used interchangeably with it in the same loop body. - Drop the no-op guard around the search_state.best_rd assignment. - Compute num_amvd_modes and amvd_inverted just above the use_amvd loop that consumes them. - Scope the loop counter i to the loops that use it. - Initialize the motion mode prune pool with a single flat loop. - Use the local aliases consistently (e.g: sf-> over cpi->sf etc). - Add the missing const to the pointer locals that are never reassigned. - Move the main loop and hdres threshold comments next to the code they describe. - Use prefix increment in the two loops that used postfix. No stats changed Change-Id: Ic88588aa0ce5986bdabb471cf4322141975a00c5 --- av2/encoder/rdopt.c | 175 +++++++++++++++++++++----------------------- 1 file changed, 82 insertions(+), 93 deletions(-) diff --git a/av2/encoder/rdopt.c b/av2/encoder/rdopt.c index 01bc7b0693..cd122b6736 100644 --- a/av2/encoder/rdopt.c +++ b/av2/encoder/rdopt.c @@ -9138,6 +9138,13 @@ static_assert(sizeof(ref_frame_centric_eval_order) / 210, "ref_frame_centric_eval_order size changed; audit callers."); +// Asymmetric NEAR/NEW compound modes default to AMVD enabled, so the +// use_amvd loop counter runs inverted for them. +static AVM_INLINE int is_amvd_inverted_mode(PREDICTION_MODE mode) { + return mode == NEW_NEARMV || mode == NEAR_NEWMV || + mode == NEAR_NEWMV_OPTFLOW || mode == NEW_NEARMV_OPTFLOW; +} + // Returns true if any rectangular (HORZ or VERT) sub-block of `bsize` at // (mi_row, mi_col) has a previously searched partition recorded. static bool has_searched_rect_subblock(MACROBLOCK *x, int mi_row, int mi_col, @@ -9322,12 +9329,12 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, const FeatureFlags *const features = &cm->features; const int num_planes = av2_num_planes(cm); const SPEED_FEATURES *const sf = &cpi->sf; + const INTER_MODE_SPEED_FEATURES *const inter_sf = &sf->inter_sf; MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; - TxfmSearchInfo *txfm_info = &x->txfm_search_info; - int i; - const ModeCosts *mode_costs = &x->mode_costs; - const int *comp_inter_cost = + TxfmSearchInfo *const txfm_info = &x->txfm_search_info; + const ModeCosts *const mode_costs = &x->mode_costs; + const int *const comp_inter_cost = mode_costs->comp_inter_cost[av2_get_reference_mode_context(cm, xd)]; mbmi->use_intrabc[xd->tree_type == CHROMA_PART] = 0; mbmi->local_rest_type = 1; @@ -9347,9 +9354,8 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, INT_MAX, INT_MAX, search_state.simple_rd, - cpi->sf.inter_sf.prune_newmv_modes_using_prior_rd - ? search_state.single_inter_rd - : NULL, + inter_sf->prune_newmv_modes_using_prior_rd ? search_state.single_inter_rd + : NULL, 0, interintra_modes, { { 0, { { 0 } }, { 0 }, 0, 0, 0 } }, @@ -9374,19 +9380,18 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, }; // Indicates the appropriate number of simple translation winner modes for - // exhaustive motion mode evaluation + // exhaustive motion mode evaluation. const int max_winner_motion_mode_cand = - num_winner_motion_modes[cpi->sf.winner_mode_sf - .motion_mode_for_winner_cand]; + num_winner_motion_modes[sf->winner_mode_sf.motion_mode_for_winner_cand]; assert(max_winner_motion_mode_cand <= MAX_WINNER_MOTION_MODES); motion_mode_candidate motion_mode_cand; motion_mode_best_st_candidate best_motion_mode_cands; // Initializing the number of motion mode candidates to zero. best_motion_mode_cands.num_motion_mode_cand = 0; - for (i = 0; i < MAX_WINNER_MOTION_MODES; ++i) + for (int i = 0; i < MAX_WINNER_MOTION_MODES; ++i) best_motion_mode_cands.motion_mode_cand[i].rd_cost = INT64_MAX; - for (i = 0; i < SINGLE_REF_FRAMES; ++i) x->pred_sse[i] = INT_MAX; + for (int i = 0; i < SINGLE_REF_FRAMES; ++i) x->pred_sse[i] = INT_MAX; av2_invalid_rd_stats(rd_cost); @@ -9395,7 +9400,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, // Ref frames that are selected by square partition blocks. uint64_t picked_ref_frames_mask = 0; - if (cpi->sf.inter_sf.prune_ref_frames && !x->inter_mode_cache[0]) { + if (inter_sf->prune_ref_frames && !x->inter_mode_cache[0]) { assert(should_reuse_mode(x, REUSE_PARTITION_MODE_FLAG)); // Prune reference frames if we are either a 1:4 block, or if we are a 1:2 @@ -9427,23 +9432,22 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, mbmi->skip_mode = 0; mbmi->refinemv_flag = 0; mbmi->mode = NEARMV; - // init params, set frame modes, speed features + // Initialize params, set frame modes, speed features. set_params_rd_pick_inter_mode(cpi, x, bsize, &mode_skip_mask, ref_costs_single, ref_costs_comp, yv12_mb); int64_t best_est_rd = INT64_MAX; - const InterModeRdModel *md = &tile_data->inter_mode_rd_models[bsize]; + const InterModeRdModel *const md = &tile_data->inter_mode_rd_models[bsize]; // If do_tx_search is 0, only estimated RD should be computed. // If do_tx_search is 1, all modes have TX search performed. // Dry pass: transform search stays on, but the settings above keep it cheap. const int do_tx_search = apply_dry_pass_shortcuts ? 1 - : !((cpi->sf.inter_sf.inter_mode_rd_model_estimation == 1 && - md->ready) || - (cpi->sf.inter_sf.inter_mode_rd_model_estimation == 2 && + : !((inter_sf->inter_mode_rd_model_estimation == 1 && md->ready) || + (inter_sf->inter_mode_rd_model_estimation == 2 && num_pels_log2_lookup[bsize] > 8)); - InterModesInfo *inter_modes_info = x->inter_modes_info; + InterModesInfo *const inter_modes_info = x->inter_modes_info; inter_modes_info->num = 0; // Temporary buffers used by handle_inter_mode(). @@ -9452,7 +9456,6 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, // The best RD found for the reference frame, among single reference modes. // Read by in_single_ref_cutoff() to check whether either ref of a compound // trial is within ~10% of the best single-ref-mode RD. - int64_t ref_frame_rd[SINGLE_REF_FRAMES] = { INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX, INT64_MAX }; @@ -9463,13 +9466,12 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, // Prepared stats used later to check if we could skip intra mode eval. int64_t inter_cost = -1; int64_t intra_cost = -1; - // Need to tweak the threshold for hdres speed 0 & 1. const int mi_row = xd->mi_row; const int mi_col = xd->mi_col; - // Obtain the relevant tpl stats for pruning inter modes + // Obtain the relevant tpl stats for pruning inter modes. PruneInfoFromTpl inter_cost_info_from_tpl; - if (cpi->sf.inter_sf.prune_inter_modes_based_on_tpl) { + if (inter_sf->prune_inter_modes_based_on_tpl) { // x->tpl_keep_ref_frame[id] = 1 => no pruning in // prune_ref_by_selective_ref_frame() // x->tpl_keep_ref_frame[id] = 0 => ref frame can be pruned in @@ -9488,6 +9490,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, get_block_level_tpl_stats(cpi, bsize, mi_row, mi_col, valid_refs, &inter_cost_info_from_tpl); } + // Need to tweak the threshold for hdres speed<=2. const int do_pruning = (AVMMIN(cm->width, cm->height) > 480 && cpi->speed <= 2) ? 0 : 1; if (do_pruning && sf->intra_sf.skip_intra_in_interframe) { @@ -9495,25 +9498,25 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, &intra_cost); } - // Initialize best mode stats for winner mode processing + // Initialize best mode stats for winner mode processing. av2_zero(x->winner_mode_stats); x->winner_mode_count = 0; - const MV_REFERENCE_FRAME init_refs[2] = { -1, -1 }; - store_winner_mode_stats(&cpi->common, x, mbmi, NULL, NULL, NULL, init_refs, + const MV_REFERENCE_FRAME init_refs[2] = { NONE_FRAME, NONE_FRAME }; + store_winner_mode_stats(cm, x, mbmi, NULL, NULL, NULL, init_refs, MODE_INVALID, NULL, bsize, best_rd_so_far, - cpi->sf.winner_mode_sf.multi_winner_mode_type, 0); + sf->winner_mode_sf.multi_winner_mode_type, 0); int mode_thresh_mul_fact = (1 << MODE_THRESH_QBITS); - if (sf->inter_sf.prune_inter_modes_if_skippable) { + if (inter_sf->prune_inter_modes_if_skippable) { // Higher multiplication factor values for lower quantizers. mode_thresh_mul_fact = mode_threshold_mul_factor[x->qindex]; } init_top_tx_part_rd_for_inter_modes(x, sf->tx_sf.prune_inter_tx_part_rd_eval); - init_top_comp_est_rd(x, sf->inter_sf.prune_comp_mode_eval_using_est_rd); + init_top_comp_est_rd(x, inter_sf->prune_comp_mode_eval_using_est_rd); - // Initialize arguments for mode loop speed features + // Initialize arguments for mode loop speed features. InterModeSFArgs sf_args = { &args.skip_motion_mode, &mode_skip_mask, &search_state, @@ -9526,34 +9529,25 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, // share_across_modes is 1, every prediction mode shares pool_shared // (smaller working set, more pruning). When 0, each prediction mode // keeps its own row in pool_per_mode. - const int share_across_modes = cpi->sf.inter_sf.share_motion_mode_prune_pool; + const int share_across_modes = inter_sf->share_motion_mode_prune_pool; int64_t pool_shared[TOP_MOTION_MODE_MODEL_COUNT]; int64_t pool_per_mode[MB_MODE_COUNT][TOP_MOTION_MODE_MODEL_COUNT]; if (enable_tx_prune) { - if (share_across_modes) { - for (int k = 0; k < TOP_MOTION_MODE_MODEL_COUNT; k++) { - pool_shared[k] = INT64_MAX; - } - } else { - for (int m = 0; m < MB_MODE_COUNT; m++) { - for (int k = 0; k < TOP_MOTION_MODE_MODEL_COUNT; k++) { - pool_per_mode[m][k] = INT64_MAX; - } - } - } + int64_t *const pool = + share_across_modes ? pool_shared : &pool_per_mode[0][0]; + const int pool_size = share_across_modes + ? TOP_MOTION_MODE_MODEL_COUNT + : MB_MODE_COUNT * TOP_MOTION_MODE_MODEL_COUNT; + for (int k = 0; k < pool_size; ++k) pool[k] = INT64_MAX; } - // This is the main loop of this function. It loops over all inter modes - // and calls handle_inter_mode() to compute the RD for each. - // Intra modes are evaluated separately after this loop. const int prune_comp_by_single = - sf->inter_sf.prune_comp_search_by_single_result > 0; + inter_sf->prune_comp_search_by_single_result > 0; const int prune_comp_best_single = - sf->inter_sf.prune_comp_using_best_single_mode_ref > 0; + inter_sf->prune_comp_using_best_single_mode_ref > 0; const int prune_comp_using_single_ref = - sf->inter_sf.prune_compound_using_single_ref; - const int motion_mode_winner = - cpi->sf.winner_mode_sf.motion_mode_for_winner_cand; + inter_sf->prune_compound_using_single_ref; + const int motion_mode_winner = sf->winner_mode_sf.motion_mode_for_winner_cand; const int reference_mode = cm->current_frame.reference_mode; const int reference_mode_select = reference_mode == REFERENCE_MODE_SELECT; @@ -9569,23 +9563,25 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, const int comp_ref_allowed = reference_mode != SINGLE_REFERENCE && is_comp_ref_allowed(bsize); const int warpmv_allowed = - (cm->features.enabled_motion_modes & (1 << WARP_DELTA)) != 0 && - cm->features.allow_warpmv_mode && is_warpmv_allowed_bsize(bsize); + (features->enabled_motion_modes & (1 << WARP_DELTA)) != 0 && + features->allow_warpmv_mode && is_warpmv_allowed_bsize(bsize); const int warp_newmv_allowed = is_motion_variation_allowed_bsize(bsize, mi_row, mi_col) && !xd->cur_frame_force_integer_mv; - const int thin_4xn_nx4 = is_thin_4xn_nx4_block(bsize); // OPTFLOW modes are only useful with REFINE_SWITCHABLE; REFINE_NONE disables // opfl entirely, REFINE_ALL applies opfl to all compound (explicit modes // redundant), and sframes/seq-level disable also kill it. const int opfl_modes_allowed = cm->seq_params.enable_opfl_refine != AVM_OPFL_REFINE_NONE && - cm->features.opfl_refine_type == REFINE_SWITCHABLE && - !frame_is_sframe(cm) && !thin_4xn_nx4; + features->opfl_refine_type == REFINE_SWITCHABLE && !frame_is_sframe(cm) && + !is_thin_4xn_nx4_block(bsize); const int num_mode_ref_pairs = (bsize == BLOCK_4X4) ? 0 : ref_frame_centric_eval_order_num; + // This is the main loop of this function. It loops over all inter modes and + // reference frames combinations and calls handle_inter_mode() to compute the + // RD for each. Intra modes are evaluated separately after this loop. for (int mode_refs_pair_idx = 0; mode_refs_pair_idx < num_mode_ref_pairs; ++mode_refs_pair_idx) { const PREDICTION_MODE this_mode = @@ -9594,9 +9590,8 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, ref_frame_centric_eval_order[mode_refs_pair_idx].rf0; const MV_REFERENCE_FRAME second_ref_frame = ref_frame_centric_eval_order[mode_refs_pair_idx].rf1; - const int is_comp_mode = (second_ref_frame != NONE_FRAME); - const int is_single_pred = !is_comp_mode; - const int comp_pred = is_comp_mode; + const int comp_pred = (second_ref_frame != NONE_FRAME); + const int is_single_pred = !comp_pred; // Gate LUT entries by runtime ref-frame availability. if (is_tip_ref_frame(ref_frame)) { @@ -9604,7 +9599,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, } else if ((int)ref_frame >= num_total_refs) { continue; } - if (is_comp_mode) { + if (comp_pred) { if (!comp_ref_allowed) continue; if ((int)second_ref_frame >= num_total_refs) continue; // Same-ref compound only when permitted at runtime. @@ -9637,7 +9632,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, // They pay for two MVs but rarely win the partition choice. When the frame // has references on both sides they are dropped completely; otherwise the // top same_ref_compound_rank_cap ranks are kept. - if (apply_dry_pass_shortcuts && is_comp_mode && + if (apply_dry_pass_shortcuts && comp_pred && ref_frame == second_ref_frame && (has_both_sides_refs || ref_frame > dry_pass_cfg.same_ref_compound_rank_cap)) @@ -9648,25 +9643,18 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, if (this_mode >= NEAR_NEARMV_OPTFLOW && !opfl_modes_allowed) continue; if (is_joint_mvd_coding_mode(this_mode) && enable_joint_mvd == 0) continue; - const int num_amvd_modes = - 1 + (enable_adaptive_mvd && allow_amvd_mode(this_mode)); - // Asymmetric NEAR/NEW compound modes default to AMVD on, invert the flag - const int amvd_inverted = - (this_mode == NEW_NEARMV || this_mode == NEAR_NEWMV || - this_mode == NEAR_NEWMV_OPTFLOW || this_mode == NEW_NEARMV_OPTFLOW); - if (bru_enabled) { assert(xd->sbi->sb_active_mode == BRU_ACTIVE_SB); if (xd->sbi->sb_active_mode == BRU_ACTIVE_SB && (ref_frame == bru_update_ref_idx || - (is_comp_mode && second_ref_frame == bru_update_ref_idx))) + (comp_pred && second_ref_frame == bru_update_ref_idx))) continue; } if (comp_pred && !(ref_frame_flags & (1 << second_ref_frame))) continue; - if (comp_pred && prune_comp_ref_by_priority(&sf->inter_sf, this_mode, - ref_frame, second_ref_frame)) { + if (comp_pred && prune_comp_ref_by_priority(inter_sf, this_mode, ref_frame, + second_ref_frame)) { continue; } @@ -9694,7 +9682,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, continue; // Select prediction reference frames. - for (i = 0; i < num_planes; i++) { + for (int i = 0; i < num_planes; ++i) { xd->plane[i].pre[0] = yv12_mb[COMPACT_INDEX0_NRS(ref_frame)][i]; if (comp_pred) xd->plane[i].pre[1] = yv12_mb[COMPACT_INDEX0_NRS(second_ref_frame)][i]; @@ -9713,6 +9701,10 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, args.single_comp_cost = real_compmode_cost; args.ref_frame_cost = ref_frame_cost; + const int num_amvd_modes = + 1 + (enable_adaptive_mvd && allow_amvd_mode(this_mode)); + const int amvd_inverted = is_amvd_inverted_mode(this_mode); + for (int use_amvd_mode = 0; use_amvd_mode < num_amvd_modes; ++use_amvd_mode) { mbmi->use_amvd = (num_amvd_modes > 1 && amvd_inverted) ? !use_amvd_mode @@ -9723,11 +9715,10 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, continue; } - if (sf->inter_sf.skip_amvd_new_near_near_new_modes && amvd_inverted && + if (inter_sf->skip_amvd_new_near_near_new_modes && amvd_inverted && mbmi->use_amvd && cm->current_frame.pyramid_level > 3) continue; - if (sf->inter_sf.prune_amvd_newmv && - cm->current_frame.pyramid_level >= 4 && + if (inter_sf->prune_amvd_newmv && cm->current_frame.pyramid_level >= 4 && (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV || mbmi->mode == NEW_NEWMV_OPTFLOW)) { if (mbmi->use_amvd && search_state.best_mbmode.mode != mbmi->mode) { @@ -9766,7 +9757,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, search_state.best_mbmode.mode, pool, &inter_cost_info_from_tpl, is_best_mode_warp); - // collect_single_states uses simple_rd/modelled_rd populated by + // Collect_single_states uses simple_rd/modelled_rd populated by // handle_inter_mode even when this_rd == INT64_MAX, so call before // the early exit. if (is_single_pred && prune_comp_by_single) @@ -9789,7 +9780,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, rd_stats_uv.rate = 0; } - // Did this mode help, i.e., is it the new best mode + // Did this mode help, i.e., is it the new best mode. if (this_rd < search_state.best_rd) { if (is_tip_ref_frame(ref_frame) && this_rd + TIP_RD_CORRECTION > search_state.best_rd) { @@ -9808,11 +9799,11 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, &motion_mode_cand, args.skip_motion_mode); } - /* keep record of best compound/single-only prediction */ + // Keep record of best compound/single-only prediction. record_best_compound(reference_mode, &rd_stats, comp_pred, x->rdmult, &search_state, compmode_cost); - } // end of use_amvd mode loop - } // end of LUT entry loop + } // End of use_amvd mode loop. + } // End of LUT entry loop. if (motion_mode_winner) { // For the single ref winner candidates, evaluate other motion modes (non @@ -9828,7 +9819,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, #endif if (do_tx_search != 1) { // A full tx search has not yet been done, do tx search for - // top mode candidates + // top mode candidates. tx_search_best_inter_candidates(cpi, tile_data, x, best_rd_so_far, bsize, yv12_mb, mi_row, mi_col, &search_state, rd_cost, ctx, &args); @@ -9857,8 +9848,8 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, end_timing(cpi, handle_intra_mode_time); #endif - int winner_mode_count = - cpi->sf.winner_mode_sf.multi_winner_mode_type ? x->winner_mode_count : 1; + const int winner_mode_count = + sf->winner_mode_sf.multi_winner_mode_type ? x->winner_mode_count : 1; if (!(winner_mode_count == 1 && search_state.best_mode_skippable)) { // In effect only when fast tx search speed features are enabled. @@ -9868,10 +9859,8 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, &search_state.best_skip2, winner_mode_count); } - if (search_state.best_rd != rd_cost->rdcost) { - search_state.best_rd = rd_cost->rdcost; - } - // Initialize default mode evaluation params + search_state.best_rd = rd_cost->rdcost; + // Initialize default mode evaluation params. set_mode_eval_params(cpi, x, DEFAULT_EVAL); av2_search_palette_mode( @@ -9889,7 +9878,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, is_intra_mode_allowed, &search_state, rd_cost); // Make sure that the ref_mv_idx is only nonzero when we're - // using a mode which can support ref_mv_idx + // using a mode which can support ref_mv_idx. if ((search_state.best_mbmode.ref_mv_idx[0] != 0 || search_state.best_mbmode.ref_mv_idx[1] != 0) && !(have_newmv_in_each_reference(search_state.best_mbmode.mode) || @@ -9913,12 +9902,12 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, // (interp_filter == search_state.best_mbmode.interp_fltr) || // !is_inter_block(&search_state.best_mbmode, xd->tree_type)); - if (!cpi->rc.is_src_frame_alt_ref && cpi->sf.inter_sf.adaptive_rd_thresh) { + if (!cpi->rc.is_src_frame_alt_ref && inter_sf->adaptive_rd_thresh) { av2_update_rd_thresh_fact(cm, x->thresh_freq_fact, - sf->inter_sf.adaptive_rd_thresh, bsize, + inter_sf->adaptive_rd_thresh, bsize, search_state.best_mbmode.mode); } - // macroblock modes + // Macroblock modes. *mbmi = search_state.best_mbmode; assert(av2_check_newmv_joint_nonzero(cm, x)); @@ -9930,12 +9919,12 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, // GLOBALMV by the all-zero mode handling of ref-mv. if (mbmi->mode == GLOBALMV || mbmi->mode == GLOBAL_GLOBALMV) { // Correct the interp filters for GLOBALMV - if (is_nontrans_global_motion(xd, xd->mi[0])) { + if (is_nontrans_global_motion(xd, mbmi)) { assert(mbmi->interp_fltr == av2_unswitchable_filter(interp_filter)); } } - for (i = 0; i < REFERENCE_MODES; ++i) { + for (int i = 0; i < REFERENCE_MODES; ++i) { if (search_state.intra_search_state.best_pred_rd[i] == INT64_MAX) { search_state.best_pred_diff[i] = INT_MIN; } else { @@ -9952,7 +9941,7 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, assert(search_state.best_mbmode.mode != MODE_INVALID); if (mbmi->motion_mode == WARP_DELTA) { - // Rebuild the warp candidate list with the best coding results + // Rebuild the warp candidate list with the best coding results. av2_find_warp_delta_base_candidates( xd, mbmi, x->mbmi_ext->warp_param_stack[av2_ref_frame_type(mbmi->ref_frame)], From 27e69f9cb9b9bb51a4d98fb614c3ecf78168f597 Mon Sep 17 00:00:00 2001 From: Mudassir Galaganath Date: Thu, 27 Aug 2026 23:58:35 +0530 Subject: [PATCH 5/5] Fix gcc array bounds warning on ref_costs_comp access Gcc reports "array subscript 8 is above array bounds of unsigned int[8][8]" for ref_costs_comp[ref_frame_index]. The index only reaches TIP_FRAME_INDEX(8) for TIP, which is single reference only, so the compound branch is never evaluated with that index. ref_costs_single is sized to include the TIP slot, ref_costs_comp is not. Added the bounds check to the condition so that the invariant is visible to the compiler. No stats changed Change-Id: I518f7414c5958f886e6750ee7402a48ecc1a895a --- av2/encoder/rdopt.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/av2/encoder/rdopt.c b/av2/encoder/rdopt.c index cd122b6736..84bf8b43a1 100644 --- a/av2/encoder/rdopt.c +++ b/av2/encoder/rdopt.c @@ -9690,9 +9690,14 @@ void av2_rd_pick_inter_mode_sb(struct AV2_COMP *cpi, const int ref_frame_index = COMPACT_INDEX0_NRS(ref_frame); const int sec_ref_frame_index = COMPACT_INDEX1_NRS(second_ref_frame); + // ref_frame_index < MAX_COMPOUND_REF_INDEX is added to silence a warning + // about array out of bounds access. It is never false when comp_pred is + // set, because ref_frame_index only reaches TIP_FRAME_INDEX for TIP and + // TIP is single reference only. const int ref_frame_cost = - comp_pred ? ref_costs_comp[ref_frame_index][sec_ref_frame_index] - : ref_costs_single[ref_frame_index]; + (comp_pred && ref_frame_index < MAX_COMPOUND_REF_INDEX) + ? ref_costs_comp[ref_frame_index][sec_ref_frame_index] + : ref_costs_single[ref_frame_index]; const int compmode_cost = (comp_ref_allowed && !is_tip_ref_frame(ref_frame)) ? comp_inter_cost[comp_pred] : 0;