From 7405cad9405485152e8da82eff4b866359ddf613 Mon Sep 17 00:00:00 2001 From: Cheng Chen Date: Wed, 26 Aug 2026 13:19:36 -0700 Subject: [PATCH] Optimize TCQ with block-level AVX2 dispatch Previously, TCQ evaluated coefficients along anti-diagonal scan lines in a portable C loop (trellis_loop_diagonal_st8), invoking individual fine-grained RTCD SIMD functions per coefficient. This fine-grained dispatch structure incurred significant overhead: - Repeated RTCD function calls and indirect branches on every coeff. - Register spills to memory structures across subroutine boundaries. - Optimization barriers preventing compiler vector scheduling across the full diagonal loop. This patch introduces a coarse-grained, block-level RTCD entry point: av2_trellis_loop_diagonal_st8() with AVX2 specialization (av2_trellis_loop_diagonal_st8_avx2). Updates are now executed contiguously within AVX2 registers without exiting to scalar C code per coefficient. Unit tests are added to compare the the new and original implementation. We can see that the TCQ function becomes 3-10% faster. ===================================================================== TX Size Pure C (us) Base AVX2 (us) Patch AVX2 (us) vs Base AVX2 --------------------------------------------------------------- 4x4 110313 45646 44360 1.03x (+2.8%) 8x8 369747 102636 96537 1.06x (+5.9%) 16x16 343618 81354 74201 1.10x (+8.8%) 32x32 268794 60375 54646 1.10x (+9.5%) 4x8 195707 64939 61781 1.05x (+4.9%) 8x4 201683 65227 61872 1.05x (+5.1%) 8x16 175484 44150 40669 1.09x (+7.9%) 16x8 180465 44455 40943 1.09x (+7.9%) 16x32 134924 30975 28011 1.11x (+9.6%) 32x16 134311 30946 28192 1.10x (+8.9%) 4x16 364581 103343 96494 1.07x (+6.6%) 16x4 390933 104326 97702 1.07x (+6.3%) 8x32 343005 81586 74379 1.10x (+8.8%) 32x8 352547 81916 74820 1.09x (+8.7%) ===================================================================== Change-Id: Ib14c7e995e37e4c0af0750af50c78db3d2adb3b5 --- av2/common/av2_rtcd_defs.pl | 2 + av2/encoder/trellis_quant.c | 72 ++-- av2/encoder/x86/trellis_quant_avx2.c | 262 ++++++++++++-- test/trellis_test.cc | 512 +++++++++++++++++++++++++++ 4 files changed, 779 insertions(+), 69 deletions(-) diff --git a/av2/common/av2_rtcd_defs.pl b/av2/common/av2_rtcd_defs.pl index b99a5428e8..5c478823f1 100644 --- a/av2/common/av2_rtcd_defs.pl +++ b/av2/common/av2_rtcd_defs.pl @@ -246,6 +246,8 @@ () specialize qw/av2_get_coeff_ctx avx2/; add_proto qw/void av2_update_nbr_diagonal/, "struct tcq_ctx_t *tcq_ctx, int row, int col, int bwl"; specialize qw/av2_update_nbr_diagonal avx2/; + add_proto qw/void av2_trellis_loop_diagonal_st8/, "const struct tcq_param_t *p, int scan_hi, int scan_lo, struct tcq_ctx_t *tcq_ctx, struct tcq_node_t *trellis"; + specialize qw/av2_trellis_loop_diagonal_st8 avx2/; # fdct functions diff --git a/av2/encoder/trellis_quant.c b/av2/encoder/trellis_quant.c index f822d176a6..f1acae7b69 100644 --- a/av2/encoder/trellis_quant.c +++ b/av2/encoder/trellis_quant.c @@ -935,10 +935,12 @@ static AVM_INLINE int get_diag_ctx(int lf, int blk_pos, int scan_pos, int bwl) { return diag_ctx; } -// TCQ 8-state for a 2D luma block. -static void trellis_loop_diagonal_st8(const tcq_param_t *p, int scan_hi, - int scan_lo, tcq_ctx_t *tcq_ctx, - tcq_node_t *trellis) { +// TCQ 8-state for a 2D luma block. Dispatch this whole loop once per block so +// SIMD implementations do not pay indirect-call overhead for every kernel at +// every coefficient. +void av2_trellis_loop_diagonal_st8_c(const tcq_param_t *p, int scan_hi, + int scan_lo, tcq_ctx_t *tcq_ctx, + tcq_node_t *trellis) { int plane = p->plane; int log_scale = p->log_scale; int try_eob = p->sharpness == 0; @@ -985,35 +987,35 @@ static void trellis_loop_diagonal_st8(const tcq_param_t *p, int scan_hi, // Get coeff contexts tcq_coeff_ctx_t coeff_ctx; - av2_get_coeff_ctx(tcq_ctx, col, &coeff_ctx); + av2_get_coeff_ctx_c(tcq_ctx, col, &coeff_ctx); coeff_ctx.coef_eob = get_lower_levels_ctx_eob(bwl, height, scan_pos); int eob_rate = block_eob_rate[scan_pos]; tcq_rate_t rd; if (pqData.orig_qIdx < 2) { - av2_pre_quant_q1(tcoeff[blk_pos], &pqData, quant, tempdqv, log_scale, - scan_pos); - av2_get_rate_dist_def_luma_q1(p, &pqData, &coeff_ctx, blk_pos, diag_ctx, - eob_rate, &rd); - av2_decide_states_q1(prev_decision, &rd, &pqData, lf, try_eob, rdmult, - decision); + av2_pre_quant_q1_c(tcoeff[blk_pos], &pqData, quant, tempdqv, log_scale, + scan_pos); + av2_get_rate_dist_def_luma_q1_c(p, &pqData, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, &rd); + av2_decide_states_q1_c(prev_decision, &rd, &pqData, lf, try_eob, rdmult, + decision); } else { - av2_pre_quant(tcoeff[blk_pos], &pqData, quant, tempdqv, log_scale, - scan_pos); - av2_get_rate_dist_def_luma(p, &pqData, &coeff_ctx, blk_pos, diag_ctx, - eob_rate, &rd); + av2_pre_quant_c(tcoeff[blk_pos], &pqData, quant, tempdqv, log_scale, + scan_pos); + av2_get_rate_dist_def_luma_c(p, &pqData, &coeff_ctx, blk_pos, diag_ctx, + eob_rate, &rd); - av2_decide_states(prev_decision, &rd, &pqData, lf, try_eob, rdmult, - decision); + av2_decide_states_c(prev_decision, &rd, &pqData, lf, try_eob, rdmult, + decision); } - av2_update_states(decision, col, tcq_ctx); + av2_update_states_c(decision, col, tcq_ctx); blk_pos += blk_pos_inc; col--; row++; } - av2_update_nbr_diagonal(tcq_ctx, row - 1, col + 1, bwl); + av2_update_nbr_diagonal_c(tcq_ctx, row - 1, col + 1, bwl); scan_hi = scan_lo - 1; } // Handle LF region. @@ -1041,36 +1043,36 @@ static void trellis_loop_diagonal_st8(const tcq_param_t *p, int scan_hi, // Get coeff contexts tcq_coeff_ctx_t coeff_ctx; - av2_get_coeff_ctx(tcq_ctx, col, &coeff_ctx); + av2_get_coeff_ctx_c(tcq_ctx, col, &coeff_ctx); coeff_ctx.coef_eob = get_lower_levels_ctx_eob(bwl, height, scan_pos); int eob_rate = block_eob_rate[scan_pos]; tcq_rate_t rd; if (pqData.orig_qIdx < 2) { - av2_pre_quant_q1(tcoeff[blk_pos], &pqData, quant, tempdqv, log_scale, - scan_pos); - av2_get_rate_dist_lf_luma_q1(p, &pqData, &coeff_ctx, blk_pos, diag_ctx, - eob_rate, dc_coeff_sign, &rd); - av2_decide_states_q1(prev_decision, &rd, &pqData, lf, try_eob, rdmult, - decision); + av2_pre_quant_q1_c(tcoeff[blk_pos], &pqData, quant, tempdqv, log_scale, + scan_pos); + av2_get_rate_dist_lf_luma_q1_c(p, &pqData, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, dc_coeff_sign, &rd); + av2_decide_states_q1_c(prev_decision, &rd, &pqData, lf, try_eob, rdmult, + decision); } else { // Calculate rate and distortion. - av2_pre_quant(tcoeff[blk_pos], &pqData, quant, tempdqv, log_scale, - scan_pos); - av2_get_rate_dist_lf_luma(p, &pqData, &coeff_ctx, blk_pos, diag_ctx, - eob_rate, dc_coeff_sign, &rd); - av2_decide_states(prev_decision, &rd, &pqData, lf, try_eob, rdmult, - decision); + av2_pre_quant_c(tcoeff[blk_pos], &pqData, quant, tempdqv, log_scale, + scan_pos); + av2_get_rate_dist_lf_luma_c(p, &pqData, &coeff_ctx, blk_pos, diag_ctx, + eob_rate, dc_coeff_sign, &rd); + av2_decide_states_c(prev_decision, &rd, &pqData, lf, try_eob, rdmult, + decision); } - av2_update_states(decision, col, tcq_ctx); + av2_update_states_c(decision, col, tcq_ctx); blk_pos += blk_pos_inc; col--; row++; } if (scan_hi != 0) { - av2_update_nbr_diagonal(tcq_ctx, row - 1, col + 1, bwl); + av2_update_nbr_diagonal_c(tcq_ctx, row - 1, col + 1, bwl); } scan_hi = scan_lo - 1; } @@ -1353,7 +1355,7 @@ int av2_trellis_quant(const struct AV2_COMP *cpi, MACROBLOCK *x, int plane, // Speed-up version for 2D Luma by exploiting parallelism // Process coeffs diagonal-by-diagonal. if (scan_hi >= 0) { - trellis_loop_diagonal_st8(¶m, scan_hi, 0, &tcq_ctx, trellis); + av2_trellis_loop_diagonal_st8(¶m, scan_hi, 0, &tcq_ctx, trellis); } // find best path diff --git a/av2/encoder/x86/trellis_quant_avx2.c b/av2/encoder/x86/trellis_quant_avx2.c index c249c192b3..9f00dc7776 100644 --- a/av2/encoder/x86/trellis_quant_avx2.c +++ b/av2/encoder/x86/trellis_quant_avx2.c @@ -75,11 +75,10 @@ static const uint8_t kConst[4][16] = { { 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 }, }; -void av2_decide_states_avx2(const struct tcq_node_t *prev, - const struct tcq_rate_t *rd, - const struct prequant_t *pq, int limits, - int try_eob, int64_t rdmult, - struct tcq_node_t *decision) { +static AVM_FORCE_INLINE void decide_states_avx2_impl( + const struct tcq_node_t *prev, const struct tcq_rate_t *rd, + const struct prequant_t *pq, int limits, int try_eob, int64_t rdmult, + struct tcq_node_t *decision) { (void)limits; assert((rdmult >> 32) == 0); static_assert(sizeof(tcq_node_t) == 16, ""); @@ -198,11 +197,10 @@ void av2_decide_states_avx2(const struct tcq_node_t *prev, } } -void av2_decide_states_q1_avx2(const struct tcq_node_t *prev, - const struct tcq_rate_t *rd, - const struct prequant_t *pq, int limits, - int try_eob, int64_t rdmult, - struct tcq_node_t *decision) { +static AVM_FORCE_INLINE void decide_states_q1_avx2_impl( + const struct tcq_node_t *prev, const struct tcq_rate_t *rd, + const struct prequant_t *pq, int limits, int try_eob, int64_t rdmult, + struct tcq_node_t *decision) { (void)limits; assert((rdmult >> 32) == 0); @@ -368,8 +366,8 @@ void av2_pre_quant_q1_avx2(tran_low_t tqc, struct prequant_t *pqData, _mm_storeu_si128((__m128i *)&pqData->deltaDist[1], deltaDist); } -void av2_update_states_avx2(const tcq_node_t *decision, int col, - struct tcq_ctx_t *tcq_ctx) { +static AVM_FORCE_INLINE void update_states_avx2_impl( + const tcq_node_t *decision, int col, struct tcq_ctx_t *tcq_ctx) { // Extract prevId, absLevel from decision[] __m256i dec01 = _mm256_lddqu_si256((__m256i *)&decision[0]); __m256i dec23 = _mm256_lddqu_si256((__m256i *)&decision[2]); @@ -454,11 +452,10 @@ static int get_mid_cost_lf_dc(tran_low_t abs_qc, int sign, int coeff_ctx, return cost; } -void av2_get_rate_dist_def_luma_avx2(const struct tcq_param_t *p, - const struct prequant_t *pq, - const struct tcq_coeff_ctx_t *coeff_ctx, - int blk_pos, int diag_ctx, int eob_rate, - struct tcq_rate_t *rd) { +static AVM_FORCE_INLINE void get_rate_dist_def_luma_avx2_impl( + const struct tcq_param_t *p, const struct prequant_t *pq, + const struct tcq_coeff_ctx_t *coeff_ctx, int blk_pos, int diag_ctx, + int eob_rate, struct tcq_rate_t *rd) { const LV_MAP_COEFF_COST *txb_costs = p->txb_costs; (void)blk_pos; const int32_t(*cost_zero)[SIG_COEF_CONTEXTS] = txb_costs->base_cost_zero; @@ -606,11 +603,10 @@ void av2_get_rate_dist_def_luma_avx2(const struct tcq_param_t *p, } } -void av2_get_rate_dist_def_luma_q1_avx2(const struct tcq_param_t *p, - const struct prequant_t *pq, - const struct tcq_coeff_ctx_t *coeff_ctx, - int blk_pos, int diag_ctx, int eob_rate, - struct tcq_rate_t *rd) { +static AVM_FORCE_INLINE void get_rate_dist_def_luma_q1_avx2_impl( + const struct tcq_param_t *p, const struct prequant_t *pq, + const struct tcq_coeff_ctx_t *coeff_ctx, int blk_pos, int diag_ctx, + int eob_rate, struct tcq_rate_t *rd) { const LV_MAP_COEFF_COST *txb_costs = p->txb_costs; (void)blk_pos; (void)pq; @@ -773,11 +769,10 @@ void av2_update_nbr_diagonal_avx2(struct tcq_ctx_t *tcq_ctx, int row, int col, } } -void av2_get_rate_dist_lf_luma_avx2(const struct tcq_param_t *p, - const struct prequant_t *pq, - const struct tcq_coeff_ctx_t *coeff_ctx, - int blk_pos, int diag_ctx, int eob_rate, - int coeff_sign, struct tcq_rate_t *rd) { +static AVM_FORCE_INLINE void get_rate_dist_lf_luma_avx2_impl( + const struct tcq_param_t *p, const struct prequant_t *pq, + const struct tcq_coeff_ctx_t *coeff_ctx, int blk_pos, int diag_ctx, + int eob_rate, int coeff_sign, struct tcq_rate_t *rd) { const LV_MAP_COEFF_COST *txb_costs = p->txb_costs; static const int8_t kShuf[2][32] = { { 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, @@ -947,11 +942,10 @@ void av2_get_rate_dist_lf_luma_avx2(const struct tcq_param_t *p, } } -void av2_get_rate_dist_lf_luma_q1_avx2(const struct tcq_param_t *p, - const struct prequant_t *pq, - const struct tcq_coeff_ctx_t *coeff_ctx, - int blk_pos, int diag_ctx, int eob_rate, - int coeff_sign, struct tcq_rate_t *rd) { +static AVM_FORCE_INLINE void get_rate_dist_lf_luma_q1_avx2_impl( + const struct tcq_param_t *p, const struct prequant_t *pq, + const struct tcq_coeff_ctx_t *coeff_ctx, int blk_pos, int diag_ctx, + int eob_rate, int coeff_sign, struct tcq_rate_t *rd) { (void)pq; const LV_MAP_COEFF_COST *txb_costs = p->txb_costs; static const int8_t kShuf[2][32] = { @@ -1049,6 +1043,66 @@ void av2_get_rate_dist_lf_luma_q1_avx2(const struct tcq_param_t *p, _mm_storeu_si64(&rd->rate_eob[0], rate_eob); } +// Keep the RTCD entry points for unit tests and non-fused callers. The block +// loop below uses the inline implementations so its per-coefficient kernels +// can be optimized together. +void av2_decide_states_avx2(const struct tcq_node_t *prev, + const struct tcq_rate_t *rd, + const struct prequant_t *pq, int limits, + int try_eob, int64_t rdmult, + struct tcq_node_t *decision) { + decide_states_avx2_impl(prev, rd, pq, limits, try_eob, rdmult, decision); +} + +void av2_decide_states_q1_avx2(const struct tcq_node_t *prev, + const struct tcq_rate_t *rd, + const struct prequant_t *pq, int limits, + int try_eob, int64_t rdmult, + struct tcq_node_t *decision) { + decide_states_q1_avx2_impl(prev, rd, pq, limits, try_eob, rdmult, decision); +} + +void av2_update_states_avx2(const tcq_node_t *decision, int col, + struct tcq_ctx_t *tcq_ctx) { + update_states_avx2_impl(decision, col, tcq_ctx); +} + +void av2_get_rate_dist_def_luma_avx2(const struct tcq_param_t *p, + const struct prequant_t *pq, + const struct tcq_coeff_ctx_t *coeff_ctx, + int blk_pos, int diag_ctx, int eob_rate, + struct tcq_rate_t *rd) { + get_rate_dist_def_luma_avx2_impl(p, pq, coeff_ctx, blk_pos, diag_ctx, + eob_rate, rd); +} + +void av2_get_rate_dist_def_luma_q1_avx2(const struct tcq_param_t *p, + const struct prequant_t *pq, + const struct tcq_coeff_ctx_t *coeff_ctx, + int blk_pos, int diag_ctx, int eob_rate, + struct tcq_rate_t *rd) { + get_rate_dist_def_luma_q1_avx2_impl(p, pq, coeff_ctx, blk_pos, diag_ctx, + eob_rate, rd); +} + +void av2_get_rate_dist_lf_luma_avx2(const struct tcq_param_t *p, + const struct prequant_t *pq, + const struct tcq_coeff_ctx_t *coeff_ctx, + int blk_pos, int diag_ctx, int eob_rate, + int coeff_sign, struct tcq_rate_t *rd) { + get_rate_dist_lf_luma_avx2_impl(p, pq, coeff_ctx, blk_pos, diag_ctx, eob_rate, + coeff_sign, rd); +} + +void av2_get_rate_dist_lf_luma_q1_avx2(const struct tcq_param_t *p, + const struct prequant_t *pq, + const struct tcq_coeff_ctx_t *coeff_ctx, + int blk_pos, int diag_ctx, int eob_rate, + int coeff_sign, struct tcq_rate_t *rd) { + get_rate_dist_lf_luma_q1_avx2_impl(p, pq, coeff_ctx, blk_pos, diag_ctx, + eob_rate, coeff_sign, rd); +} + // Pre-calculate eob bits (rate) for each EOB candidate position from 1 // to the initial eob location. Store rate in array block_eob_rate[], // starting with index. @@ -1173,9 +1227,9 @@ int av2_find_best_path_avx2(const struct tcq_node_t *trellis, for (; prev_id >= 0; scan_pos++) { const int32_t *decision = (int32_t *)&trellis[(scan_pos << TCQ_N_STATES_LOG) + prev_id]; - __m128i info = _mm_loadu_si64(&decision[3]); + __m128i info = _mm_cvtsi32_si128(decision[3]); int blk_pos = scan[scan_pos]; - __m128i sign = _mm_loadu_si64(&tcoeff[blk_pos]); + __m128i sign = _mm_cvtsi32_si128(tcoeff[blk_pos]); sign = _mm_srai_epi32(sign, 31); __m128i abs_lev = _mm_slli_epi32(info, 8); __m128i abs_lev2 = _mm_srli_epi32(abs_lev, 7); @@ -1231,3 +1285,143 @@ int av2_find_best_path_avx2(const struct tcq_node_t *trellis, *min_cost = min_path_cost; return eob; } + +static AVM_INLINE int get_diag_ctx_avx2(int lf, int blk_pos, int scan_pos, + int bwl) { + int diag_ctx; + if (lf) { + diag_ctx = get_nz_map_ctx_from_stats_lf(0, blk_pos, bwl, TX_CLASS_2D); + if (scan_pos > 0) diag_ctx += 7 << 8; + } else { + diag_ctx = get_nz_map_ctx_from_stats(0, blk_pos, bwl, TX_CLASS_2D, 0); + } + return diag_ctx; +} + +// Keep runtime dispatch outside the coefficient loops. Each call below is the +// same AVX2 kernel selected by RTCD before this change, but it is now a direct +// call that the compiler can schedule with the surrounding loop. +void av2_trellis_loop_diagonal_st8_avx2(const tcq_param_t *p, int scan_hi, + int scan_lo, tcq_ctx_t *tcq_ctx, + tcq_node_t *trellis) { + const int log_scale = p->log_scale; + const int try_eob = p->sharpness == 0; + const int64_t rdmult = p->rdmult; + const int16_t *scan = p->scan; + const tran_low_t *tcoeff = p->tcoeff; + const int32_t *quant = p->quant; + const int32_t *dequant = p->dequant; + const qm_val_t *iqmatrix = p->iqmatrix; + const uint16_t *block_eob_rate = p->block_eob_rate; + const int bwl = p->bwl; + const int height = p->txb_height; + assert(p->plane == 0); + assert(p->tx_class == TX_CLASS_2D); + + const int dc_coeff_sign = tcoeff[0] < 0; + const int blk_pos_inc = (1 << bwl) - 1; + const int shift = 16 - log_scale + QUANT_FP_BITS; + int blk_pos, row, col; + + while (scan_hi >= 10) { + blk_pos = scan[scan_hi]; + row = blk_pos >> bwl; + col = blk_pos - (row << bwl); + const int inc = AVMMIN(height - 1 - row, col); + scan_lo = scan_hi - inc; + const int lf = 0; + const int diag_ctx = get_diag_ctx_avx2(lf, blk_pos, scan_lo, bwl); + assert(scan_lo >= 0); + + for (int scan_pos = scan_hi; scan_pos >= scan_lo; --scan_pos) { + tcq_node_t *decision = &trellis[scan_pos << TCQ_N_STATES_LOG]; + const tcq_node_t *prev_decision = &decision[TCQ_N_STATES]; + prequant_t pq_data; + const int temp_dqv = get_dqv(dequant, scan[scan_pos], iqmatrix); + pq_data.orig_qIdx = + (tran_low_t)(((int64_t)abs(tcoeff[blk_pos]) * quant[scan_pos != 0]) >> + shift); + + tcq_coeff_ctx_t coeff_ctx; + av2_get_coeff_ctx_avx2(tcq_ctx, col, &coeff_ctx); + coeff_ctx.coef_eob = get_lower_levels_ctx_eob(bwl, height, scan_pos); + const int eob_rate = block_eob_rate[scan_pos]; + tcq_rate_t rd; + + if (pq_data.orig_qIdx < 2) { + av2_pre_quant_q1_avx2(tcoeff[blk_pos], &pq_data, quant, temp_dqv, + log_scale, scan_pos); + get_rate_dist_def_luma_q1_avx2_impl(p, &pq_data, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, &rd); + decide_states_q1_avx2_impl(prev_decision, &rd, &pq_data, lf, try_eob, + rdmult, decision); + } else { + av2_pre_quant_avx2(tcoeff[blk_pos], &pq_data, quant, temp_dqv, + log_scale, scan_pos); + get_rate_dist_def_luma_avx2_impl(p, &pq_data, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, &rd); + decide_states_avx2_impl(prev_decision, &rd, &pq_data, lf, try_eob, + rdmult, decision); + } + update_states_avx2_impl(decision, col, tcq_ctx); + + blk_pos += blk_pos_inc; + --col; + ++row; + } + av2_update_nbr_diagonal_avx2(tcq_ctx, row - 1, col + 1, bwl); + scan_hi = scan_lo - 1; + } + + while (scan_hi >= 0) { + blk_pos = scan[scan_hi]; + row = blk_pos >> bwl; + col = blk_pos - (row << bwl); + const int inc = AVMMIN(height - 1 - row, col); + scan_lo = scan_hi - inc; + const int lf = 1; + const int diag_ctx = get_diag_ctx_avx2(lf, blk_pos, scan_lo, bwl); + assert(scan_lo >= 0); + + for (int scan_pos = scan_hi; scan_pos >= scan_lo; --scan_pos) { + tcq_node_t *decision = &trellis[scan_pos << TCQ_N_STATES_LOG]; + const tcq_node_t *prev_decision = &decision[TCQ_N_STATES]; + prequant_t pq_data; + const int temp_dqv = get_dqv(dequant, scan[scan_pos], iqmatrix); + pq_data.orig_qIdx = + (tran_low_t)(((int64_t)abs(tcoeff[blk_pos]) * quant[scan_pos != 0]) >> + shift); + + tcq_coeff_ctx_t coeff_ctx; + av2_get_coeff_ctx_avx2(tcq_ctx, col, &coeff_ctx); + coeff_ctx.coef_eob = get_lower_levels_ctx_eob(bwl, height, scan_pos); + const int eob_rate = block_eob_rate[scan_pos]; + tcq_rate_t rd; + + if (pq_data.orig_qIdx < 2) { + av2_pre_quant_q1_avx2(tcoeff[blk_pos], &pq_data, quant, temp_dqv, + log_scale, scan_pos); + get_rate_dist_lf_luma_q1_avx2_impl(p, &pq_data, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, dc_coeff_sign, + &rd); + decide_states_q1_avx2_impl(prev_decision, &rd, &pq_data, lf, try_eob, + rdmult, decision); + } else { + av2_pre_quant_avx2(tcoeff[blk_pos], &pq_data, quant, temp_dqv, + log_scale, scan_pos); + get_rate_dist_lf_luma_avx2_impl(p, &pq_data, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, dc_coeff_sign, &rd); + decide_states_avx2_impl(prev_decision, &rd, &pq_data, lf, try_eob, + rdmult, decision); + } + update_states_avx2_impl(decision, col, tcq_ctx); + + blk_pos += blk_pos_inc; + --col; + ++row; + } + if (scan_hi != 0) + av2_update_nbr_diagonal_avx2(tcq_ctx, row - 1, col + 1, bwl); + scan_hi = scan_lo - 1; + } +} diff --git a/test/trellis_test.cc b/test/trellis_test.cc index fac2ab4fad..be70f9d08c 100644 --- a/test/trellis_test.cc +++ b/test/trellis_test.cc @@ -17,6 +17,7 @@ #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "test/register_state_check.h" #include "test/function_equivalence_test.h" +#include "test/util.h" #include "config/avm_config.h" #include "config/avm_dsp_rtcd.h" @@ -24,7 +25,10 @@ #include "avm/avm_integer.h" #include "av2/common/enums.h" +#include "av2/common/idct.h" +#include "av2/common/scan.h" #include "av2/encoder/trellis_quant.h" +#include "av2/common/txb_common.h" using libavm_test::FunctionEquivalenceTest; @@ -878,6 +882,507 @@ class TcqDecideStatesQ1Test TEST_P(TcqDecideStatesQ1Test, RandomValues) { RunTest(); } +typedef void (*TcqLoopDiagonalSt8Func)(const struct tcq_param_t *p, int scan_hi, + int scan_lo, struct tcq_ctx_t *tcq_ctx, + struct tcq_node_t *trellis); +typedef libavm_test::FuncParam + TcqLoopDiagonalSt8TestFuncs; + +class TcqLoopDiagonalSt8Test + : public FunctionEquivalenceTest { + protected: + static const int kIterations = 10000; + + void InitParam(TX_SIZE tx_size, tcq_param_t *param, + LV_MAP_COEFF_COST *txb_costs, tran_low_t *tcoeff, + int32_t *tmp_sign, int32_t *quant, int32_t *dequant, + uint16_t *block_eob_rate) { + const int bwl = get_txb_bwl(tx_size); + const int height = get_txb_high(tx_size); + const int width = 1 << bwl; + const int num_coeffs = width * height; + const int log_scale = av2_get_tx_scale(tx_size) + 1; + const int shift = 16 - log_scale + QUANT_FP_BITS; + + const SCAN_ORDER *scan_order = get_scan(tx_size, DCT_DCT); + param->plane = 0; + param->bwl = bwl; + param->txb_height = height; + param->tx_size = tx_size; + param->tx_class = TX_CLASS_2D; + param->sharpness = rng_.Rand8() & 1; + param->rdmult = rng_(1 << 16) + 100; + param->log_scale = log_scale; + param->dc_sign_ctx = rng_.Rand8() & 3; + param->scan = scan_order->scan; + param->tmp_sign = tmp_sign; + param->qcoeff = NULL; + param->tcoeff = tcoeff; + param->quant = quant; + param->dequant = dequant; + param->iqmatrix = NULL; + param->block_eob_rate = block_eob_rate; + param->txb_costs = txb_costs; + + quant[0] = 1 << shift; + quant[1] = 1 << shift; + dequant[0] = (1 << QUANT_TABLE_BITS) << (log_scale - 1); + dequant[1] = (1 << QUANT_TABLE_BITS) << (log_scale - 1); + + generate_random_cost_tables(&rng_, txb_costs); + + for (int i = 0; i < num_coeffs; i++) { + tcoeff[i] = (tran_low_t)((rng_.Rand16() % 2048) - 1024); + tmp_sign[i] = rng_.Rand8() & 1; + } + for (int i = 0; i < MAX_TRELLIS; i++) { + block_eob_rate[i] = rng_(512 * 4); + } + } + + void InitContextAndTrellis(const tcq_param_t *param, int first_scan_pos, + tcq_ctx_t *tcq_ctx, tcq_node_t *trellis) { + int blk_pos = param->scan[first_scan_pos]; + TX_SIZE tx_size = param->tx_size; + const int bwl = get_txb_bwl(tx_size); + const int height = get_txb_high(tx_size); + const int row = blk_pos >> bwl; + const int col = blk_pos - (row << bwl); + + int diag = AVMMIN(row + col, MAX_DIAG) + 2; + int ctx_array_size = diag << TCQ_N_STATES_LOG; + + static const int8_t init_st[4][TCQ_MAX_STATES] = { + { 0, 1, 2, 3, 4, 5, 6, 7 }, + { 0, 1, 2, 3, 4, 5, 6, 7 }, + { 0, 1, 2, 3, 4, 5, 6, 7 }, + { 0, 1, 2, 3, 4, 5, 6, 7 }, + }; + + memset(&tcq_ctx->mag_base, 0, ctx_array_size); + memset(&tcq_ctx->mag_mid, 0, ctx_array_size); + memset(&tcq_ctx->ctx, 0, ctx_array_size); + memset(&tcq_ctx->lev_new, 0, ctx_array_size); + + for (int i = 0; i < diag; i += 4) { + memcpy(tcq_ctx->prev_st[i], init_st, sizeof(init_st)); + } + + memset(trellis, 0, sizeof(tcq_node_t) * MAX_TRELLIS * TCQ_MAX_STATES); + + tcq_node_t *decision = &trellis[first_scan_pos << TCQ_N_STATES_LOG]; + static const tcq_node_t def = { INT64_MAX >> 10, 0, -1, -2 }; + for (int i = 0; i < TCQ_N_STATES; i++) { + decision[i] = def; + } + decision[0].rdCost = rng_(1 << 20); + decision[0].rate = rng_(1 << 16); + decision[0].absLevel = rng_.Rand8() % 16; + decision[0].prevId = -1; + decision[4].rdCost = rng_(1 << 20); + decision[4].rate = rng_(1 << 16); + decision[4].absLevel = rng_.Rand8() % 16; + decision[4].prevId = -1; + + for (int i = 0; i < TCQ_MAX_STATES; i++) { + tcq_ctx->orig_st[i] = i; + tcq_ctx->prev_st[col][i] = -1; + tcq_ctx->lev_new[col][i] = 0; + } + tcq_ctx->lev_new[col][0] = + AVMMIN(AVMMAX(0, decision[0].absLevel), MAX_VAL_BR_CTX); + tcq_ctx->lev_new[col][4] = + AVMMIN(AVMMAX(0, decision[4].absLevel), MAX_VAL_BR_CTX); + + if ((col == 0 && row != 0) || row == height - 1) { + av2_update_nbr_diagonal_c(tcq_ctx, row, col, bwl); + } + } + + void RunEquivalenceTest() { + static const TX_SIZE kTxSizes[] = { TX_4X4, TX_8X8, TX_16X16, TX_32X32, + TX_4X8, TX_8X4, TX_8X16, TX_16X8, + TX_16X32, TX_32X16, TX_4X16, TX_16X4, + TX_8X32, TX_32X8 }; + const int kNumTxSizes = sizeof(kTxSizes) / sizeof(kTxSizes[0]); + + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + TX_SIZE tx_size = kTxSizes[iter % kNumTxSizes]; + tcq_param_t param; + LV_MAP_COEFF_COST txb_costs; + tran_low_t tcoeff[MAX_TRELLIS]; + int32_t tmp_sign[MAX_TRELLIS]; + int32_t quant[2]; + int32_t dequant[2]; + uint16_t block_eob_rate[MAX_TRELLIS]; + + InitParam(tx_size, ¶m, &txb_costs, tcoeff, tmp_sign, quant, dequant, + block_eob_rate); + + const int width = 1 << param.bwl; + const int height = param.txb_height; + const int num_coeffs = width * height; + const int max_diag = width + height - 2; + + // Select a diagonal (from 1 to max_diag) to start from. + int start_diag = max_diag; + if (iter % 2 != 0 && max_diag > 1) { + start_diag = 1 + (rng_.Rand16() % max_diag); + } + + // Find bottom-left scan pos of start_diag. + int col_bot = AVMMAX(0, start_diag - (height - 1)); + int row_bot = start_diag - col_bot; + int blk_pos_bot = (row_bot << param.bwl) + col_bot; + int first_scan_pos = -1; + for (int s = 0; s < num_coeffs; ++s) { + if (param.scan[s] == blk_pos_bot) { + first_scan_pos = s; + break; + } + } + ASSERT_GE(first_scan_pos, 0); + + tcq_ctx_t tcq_ctx_ref, tcq_ctx_tst; + tcq_node_t trellis_ref[MAX_TRELLIS * TCQ_MAX_STATES]; + tcq_node_t trellis_tst[MAX_TRELLIS * TCQ_MAX_STATES]; + + InitContextAndTrellis(¶m, first_scan_pos, &tcq_ctx_ref, trellis_ref); + tcq_ctx_tst = tcq_ctx_ref; + memcpy(trellis_tst, trellis_ref, sizeof(trellis_ref)); + + int scan_hi = first_scan_pos - 1; + params_.ref_func(¶m, scan_hi, 0, &tcq_ctx_ref, trellis_ref); + ASM_REGISTER_STATE_CHECK( + params_.tst_func(¶m, scan_hi, 0, &tcq_ctx_tst, trellis_tst)); + + const int64_t kUnreachable = (INT64_MAX >> 12); + for (int scan_pos = 0; scan_pos <= scan_hi; ++scan_pos) { + for (int st = 0; st < TCQ_N_STATES; ++st) { + int idx = (scan_pos << TCQ_N_STATES_LOG) + st; + if ((trellis_ref[idx].rdCost >= 0 && + trellis_ref[idx].rdCost < kUnreachable) || + (trellis_tst[idx].rdCost >= 0 && + trellis_tst[idx].rdCost < kUnreachable)) { + ASSERT_EQ(trellis_ref[idx].rdCost, trellis_tst[idx].rdCost) + << "rdCost mismatch at scan_pos=" << scan_pos << " st=" << st + << " tx_size=" << tx_size; + ASSERT_EQ(trellis_ref[idx].rate, trellis_tst[idx].rate) + << "rate mismatch at scan_pos=" << scan_pos << " st=" << st + << " tx_size=" << tx_size; + ASSERT_EQ(trellis_ref[idx].absLevel, trellis_tst[idx].absLevel) + << "absLevel mismatch at scan_pos=" << scan_pos << " st=" << st + << " tx_size=" << tx_size; + ASSERT_EQ(trellis_ref[idx].prevId, trellis_tst[idx].prevId) + << "prevId mismatch at scan_pos=" << scan_pos << " st=" << st + << " tx_size=" << tx_size; + } + } + } + + for (int st = 0; st < TCQ_MAX_STATES; ++st) { + if ((trellis_ref[st].rdCost >= 0 && + trellis_ref[st].rdCost < kUnreachable) || + (trellis_tst[st].rdCost >= 0 && + trellis_tst[st].rdCost < kUnreachable)) { + ASSERT_EQ((int)tcq_ctx_ref.orig_st[st], (int)tcq_ctx_tst.orig_st[st]) + << "orig_st mismatch at iter=" << iter << " st=" << st + << " tx_size=" << tx_size; + ASSERT_EQ((int)tcq_ctx_ref.lev_new[0][st], + (int)tcq_ctx_tst.lev_new[0][st]) + << "lev_new[0] mismatch at iter=" << iter << " st=" << st + << " tx_size=" << tx_size; + ASSERT_EQ((int)tcq_ctx_ref.prev_st[0][st], + (int)tcq_ctx_tst.prev_st[0][st]) + << "prev_st[0] mismatch at iter=" << iter << " st=" << st + << " tx_size=" << tx_size; + } + } + + tran_low_t qcoeff_ref[MAX_TRELLIS] = { 0 }, + qcoeff_tst[MAX_TRELLIS] = { 0 }; + tran_low_t dqcoeff_ref[MAX_TRELLIS] = { 0 }, + dqcoeff_tst[MAX_TRELLIS] = { 0 }; + int rate_ref = 0, rate_tst = 0; + int64_t cost_ref = INT64_MAX, cost_tst = INT64_MAX; + int eob_ref = av2_find_best_path( + trellis_ref, param.scan, param.dequant, param.iqmatrix, param.tcoeff, + first_scan_pos, param.log_scale, qcoeff_ref, dqcoeff_ref, &rate_ref, + &cost_ref); + int eob_tst = av2_find_best_path( + trellis_tst, param.scan, param.dequant, param.iqmatrix, param.tcoeff, + first_scan_pos, param.log_scale, qcoeff_tst, dqcoeff_tst, &rate_tst, + &cost_tst); + ASSERT_EQ(eob_ref, eob_tst); + ASSERT_EQ(rate_ref, rate_tst); + ASSERT_EQ(cost_ref, cost_tst); + for (int i = 0; i < num_coeffs; ++i) { + ASSERT_EQ(qcoeff_ref[i], qcoeff_tst[i]); + ASSERT_EQ(dqcoeff_ref[i], dqcoeff_tst[i]); + } + } + } + +#if HAVE_AVX2 + static AVM_FORCE_INLINE int get_diag_ctx_bench(int lf, int blk_pos, + int scan_pos, int bwl) { + int diag_ctx; + if (lf) { + diag_ctx = get_nz_map_ctx_from_stats_lf(0, blk_pos, bwl, TX_CLASS_2D); + if (scan_pos > 0) diag_ctx += 7 << 8; + } else { + diag_ctx = get_nz_map_ctx_from_stats(0, blk_pos, bwl, TX_CLASS_2D, 0); + } + return diag_ctx; + } + + static AVM_FORCE_INLINE int get_dqv_bench(const int32_t *dequant, + int coeff_idx, + const qm_val_t *iqmatrix) { + int dqv = dequant[coeff_idx != 0]; + if (iqmatrix != NULL) { + dqv = + (dqv * iqmatrix[coeff_idx] + (1 << (AVM_QM_BITS - 1))) >> AVM_QM_BITS; + } + return dqv; + } + + static void trellis_loop_diagonal_st8_prepatch_baseline_avx2( + const tcq_param_t *p, int scan_hi, int scan_lo, tcq_ctx_t *tcq_ctx, + tcq_node_t *trellis) { + int log_scale = p->log_scale; + int try_eob = p->sharpness == 0; + int64_t rdmult = p->rdmult; + const int16_t *scan = p->scan; + const tran_low_t *tcoeff = p->tcoeff; + const int32_t *quant = p->quant; + const int32_t *dequant = p->dequant; + const qm_val_t *iqmatrix = p->iqmatrix; + const uint16_t *block_eob_rate = p->block_eob_rate; + int bwl = p->bwl; + int height = p->txb_height; + int dc_coeff_sign = tcoeff[0] < 0; + int blk_pos_inc = (1 << bwl) - 1; + int blk_pos, row, col; + int shift = 16 - log_scale + QUANT_FP_BITS; + + while (scan_hi >= 10) { + blk_pos = scan[scan_hi]; + row = blk_pos >> bwl; + col = blk_pos - (row << bwl); + int inc = AVMMIN(height - 1 - row, col); + scan_lo = scan_hi - inc; + int lf = 0; + int diag_ctx = get_diag_ctx_bench(lf, blk_pos, scan_lo, bwl); + + for (int scan_pos = scan_hi; scan_pos >= scan_lo; scan_pos--) { + tcq_node_t *decision = &trellis[scan_pos << TCQ_N_STATES_LOG]; + tcq_node_t *prev_decision = &decision[TCQ_N_STATES]; + prequant_t pqData; + int tempdqv = get_dqv_bench(dequant, scan[scan_pos], iqmatrix); + tran_low_t orig_qIdx = (tran_low_t)(((int64_t)abs(tcoeff[blk_pos]) * + quant[scan_pos != 0]) >> + shift); + pqData.orig_qIdx = orig_qIdx; + + tcq_coeff_ctx_t coeff_ctx; + av2_get_coeff_ctx_avx2(tcq_ctx, col, &coeff_ctx); + coeff_ctx.coef_eob = get_lower_levels_ctx_eob(bwl, height, scan_pos); + int eob_rate = block_eob_rate[scan_pos]; + tcq_rate_t rd; + + if (pqData.orig_qIdx < 2) { + av2_pre_quant_q1_avx2(tcoeff[blk_pos], &pqData, quant, tempdqv, + log_scale, scan_pos); + av2_get_rate_dist_def_luma_q1_avx2(p, &pqData, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, &rd); + av2_decide_states_q1_avx2(prev_decision, &rd, &pqData, lf, try_eob, + rdmult, decision); + } else { + av2_pre_quant_avx2(tcoeff[blk_pos], &pqData, quant, tempdqv, + log_scale, scan_pos); + av2_get_rate_dist_def_luma_avx2(p, &pqData, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, &rd); + av2_decide_states_avx2(prev_decision, &rd, &pqData, lf, try_eob, + rdmult, decision); + } + av2_update_states_avx2(decision, col, tcq_ctx); + blk_pos += blk_pos_inc; + col--; + row++; + } + av2_update_nbr_diagonal_avx2(tcq_ctx, row - 1, col + 1, bwl); + scan_hi = scan_lo - 1; + } + while (scan_hi >= 0) { + blk_pos = scan[scan_hi]; + row = blk_pos >> bwl; + col = blk_pos - (row << bwl); + int inc = AVMMIN(height - 1 - row, col); + scan_lo = scan_hi - inc; + int lf = 1; + int diag_ctx = get_diag_ctx_bench(lf, blk_pos, scan_lo, bwl); + + for (int scan_pos = scan_hi; scan_pos >= scan_lo; scan_pos--) { + tcq_node_t *decision = &trellis[scan_pos << TCQ_N_STATES_LOG]; + tcq_node_t *prev_decision = &decision[TCQ_N_STATES]; + prequant_t pqData; + int tempdqv = get_dqv_bench(dequant, scan[scan_pos], iqmatrix); + tran_low_t orig_qIdx = (tran_low_t)(((int64_t)abs(tcoeff[blk_pos]) * + quant[scan_pos != 0]) >> + shift); + pqData.orig_qIdx = orig_qIdx; + + tcq_coeff_ctx_t coeff_ctx; + av2_get_coeff_ctx_avx2(tcq_ctx, col, &coeff_ctx); + coeff_ctx.coef_eob = get_lower_levels_ctx_eob(bwl, height, scan_pos); + int eob_rate = block_eob_rate[scan_pos]; + tcq_rate_t rd; + + if (pqData.orig_qIdx < 2) { + av2_pre_quant_q1_avx2(tcoeff[blk_pos], &pqData, quant, tempdqv, + log_scale, scan_pos); + av2_get_rate_dist_lf_luma_q1_avx2(p, &pqData, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, dc_coeff_sign, + &rd); + av2_decide_states_q1_avx2(prev_decision, &rd, &pqData, lf, try_eob, + rdmult, decision); + } else { + av2_pre_quant_avx2(tcoeff[blk_pos], &pqData, quant, tempdqv, + log_scale, scan_pos); + av2_get_rate_dist_lf_luma_avx2(p, &pqData, &coeff_ctx, blk_pos, + diag_ctx, eob_rate, dc_coeff_sign, + &rd); + av2_decide_states_avx2(prev_decision, &rd, &pqData, lf, try_eob, + rdmult, decision); + } + av2_update_states_avx2(decision, col, tcq_ctx); + blk_pos += blk_pos_inc; + col--; + row++; + } + if (scan_hi != 0) { + av2_update_nbr_diagonal_avx2(tcq_ctx, row - 1, col + 1, bwl); + } + scan_hi = scan_lo - 1; + } + } +#endif // HAVE_AVX2 + + void RunSpeedTest() { + static const TX_SIZE kTxSizes[] = { TX_4X4, TX_8X8, TX_16X16, TX_32X32, + TX_4X8, TX_8X4, TX_8X16, TX_16X8, + TX_16X32, TX_32X16, TX_4X16, TX_16X4, + TX_8X32, TX_32X8 }; + const int kNumTxSizes = sizeof(kTxSizes) / sizeof(kTxSizes[0]); + + printf( + "\n============================ TCQ Diagonal Loop: Direct Speed " + "Benchmark ============================\n"); + printf("%-8s %-6s %-12s %-15s %-15s %-16s %-16s\n", "TX Size", "Coeffs", + "Pure C (us)", "Base AVX2 (us)", "Patch AVX2 (us)", "vs Base AVX2", + "vs Pure C"); + printf( + "----------------------------------------------------------------------" + "------------------------------\n"); + + for (int txi = 0; txi < kNumTxSizes; ++txi) { + TX_SIZE tx_size = kTxSizes[txi]; + const int width = 1 << get_txb_bwl(tx_size); + const int height = get_txb_high(tx_size); + const int num_coeffs = width * height; + const int kNumTests = (num_coeffs <= 64) ? 20000 + : (num_coeffs <= 256) ? 5000 + : 1000; + + tcq_param_t param; + LV_MAP_COEFF_COST txb_costs; + tran_low_t tcoeff[MAX_TRELLIS]; + int32_t tmp_sign[MAX_TRELLIS]; + int32_t quant[2]; + int32_t dequant[2]; + uint16_t block_eob_rate[MAX_TRELLIS]; + + InitParam(tx_size, ¶m, &txb_costs, tcoeff, tmp_sign, quant, dequant, + block_eob_rate); + + tcq_ctx_t tcq_ctx_c, tcq_ctx_opt; + tcq_node_t trellis_c[MAX_TRELLIS * TCQ_MAX_STATES]; + tcq_node_t trellis_opt[MAX_TRELLIS * TCQ_MAX_STATES]; +#if HAVE_AVX2 + tcq_ctx_t tcq_ctx_base; + tcq_node_t trellis_base[MAX_TRELLIS * TCQ_MAX_STATES]; +#endif + + int first_scan_pos = num_coeffs - 1; + int scan_hi = first_scan_pos - 1; + + avm_usec_timer timer_c; + avm_usec_timer_start(&timer_c); + for (int i = 0; i < kNumTests; ++i) { + InitContextAndTrellis(¶m, first_scan_pos, &tcq_ctx_c, trellis_c); + params_.ref_func(¶m, scan_hi, 0, &tcq_ctx_c, trellis_c); + } + avm_usec_timer_mark(&timer_c); + const int64_t elapsed_time_c = avm_usec_timer_elapsed(&timer_c); + +#if HAVE_AVX2 + avm_usec_timer timer_base; + avm_usec_timer_start(&timer_base); + for (int i = 0; i < kNumTests; ++i) { + InitContextAndTrellis(¶m, first_scan_pos, &tcq_ctx_base, + trellis_base); + trellis_loop_diagonal_st8_prepatch_baseline_avx2( + ¶m, scan_hi, 0, &tcq_ctx_base, trellis_base); + } + avm_usec_timer_mark(&timer_base); + const int64_t elapsed_time_base = avm_usec_timer_elapsed(&timer_base); +#else + const int64_t elapsed_time_base = elapsed_time_c; +#endif + + avm_usec_timer timer_opt; + avm_usec_timer_start(&timer_opt); + for (int i = 0; i < kNumTests; ++i) { + InitContextAndTrellis(¶m, first_scan_pos, &tcq_ctx_opt, + trellis_opt); + params_.tst_func(¶m, scan_hi, 0, &tcq_ctx_opt, trellis_opt); + } + avm_usec_timer_mark(&timer_opt); + const int64_t elapsed_time_opt = avm_usec_timer_elapsed(&timer_opt); + + const double gain_vs_base = + (double)elapsed_time_base / (double)elapsed_time_opt; + const double speedup_vs_base = + (1.0 - (double)elapsed_time_opt / (double)elapsed_time_base) * 100.0; + + const double gain_vs_c = + (double)elapsed_time_c / (double)elapsed_time_opt; + const double speedup_vs_c = + (1.0 - (double)elapsed_time_opt / (double)elapsed_time_c) * 100.0; + + char size_str[32]; + snprintf(size_str, sizeof(size_str), "%dx%d", width, height); + char vs_base_str[32]; + snprintf(vs_base_str, sizeof(vs_base_str), "%.2fx (+%.1f%%)", + gain_vs_base, speedup_vs_base); + char vs_c_str[32]; + snprintf(vs_c_str, sizeof(vs_c_str), "%.2fx (+%.1f%%)", gain_vs_c, + speedup_vs_c); + + printf("%-8s %-6d %-12ld %-15ld %-15ld %-16s %-16s\n", size_str, + num_coeffs, (long)elapsed_time_c, (long)elapsed_time_base, + (long)elapsed_time_opt, vs_base_str, vs_c_str); + } + printf( + "======================================================================" + "==============================\n\n"); + } +}; + +TEST_P(TcqLoopDiagonalSt8Test, RandomValues) { RunEquivalenceTest(); } +TEST_P(TcqLoopDiagonalSt8Test, Speed) { RunSpeedTest(); } + #if HAVE_AVX2 INSTANTIATE_TEST_SUITE_P(AVX2, TcqDecideStatesTest, ::testing::Values(TcqDecideStatesTestFuncs( @@ -920,6 +1425,11 @@ INSTANTIATE_TEST_SUITE_P(AVX2, TcqUpdateNbrDiagonalTest, ::testing::Values(TcqUpdateNbrDiagonalTestFuncs( av2_update_nbr_diagonal_c, av2_update_nbr_diagonal_avx2))); + +INSTANTIATE_TEST_SUITE_P(AVX2, TcqLoopDiagonalSt8Test, + ::testing::Values(TcqLoopDiagonalSt8TestFuncs( + av2_trellis_loop_diagonal_st8_c, + av2_trellis_loop_diagonal_st8_avx2))); #endif // HAVE_AVX2 GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TcqDecideStatesTest); @@ -940,4 +1450,6 @@ GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TcqRateLfLumaQ1Test); GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TcqUpdateNbrDiagonalTest); +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TcqLoopDiagonalSt8Test); + } // namespace