/******************************************************************************
|
*
|
* Copyright (C) 2018 The Android Open Source Project
|
*
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at:
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
*
|
*****************************************************************************
|
* Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore
|
*/
|
/**
|
*******************************************************************************
|
* @file
|
* ihevce_had_compute_neon.c
|
*
|
* @brief
|
* Contains intrinsic definitions of functions for computing had
|
*
|
* @author
|
* Ittiam
|
*
|
* @par List of Functions:
|
*
|
* @remarks
|
* None
|
*
|
********************************************************************************
|
*/
|
|
/*****************************************************************************/
|
/* File Includes */
|
/*****************************************************************************/
|
/* System include files */
|
#include <string.h>
|
#include <assert.h>
|
#include <arm_neon.h>
|
|
/* User include files */
|
#include "ihevc_typedefs.h"
|
#include "itt_video_api.h"
|
#include "ihevc_cmn_utils_neon.h"
|
#include "ihevce_had_satd.h"
|
#include "ihevce_cmn_utils_instr_set_router.h"
|
|
/*****************************************************************************/
|
/* Globals */
|
/*****************************************************************************/
|
const int16_t gu2_dc_mask[8] = { 0x0000, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff };
|
|
/*****************************************************************************/
|
/* Function Macros */
|
/*****************************************************************************/
|
#define RESIDUE(k, is_chroma) \
|
if(!is_chroma) \
|
{ \
|
const uint8x8_t s##k = vld1_u8(pu1_src); \
|
const uint8x8_t p##k = vld1_u8(pu1_pred); \
|
*r##k = vreinterpretq_s16_u16(vsubl_u8(s##k, p##k)); \
|
pu1_src += src_strd; \
|
pu1_pred += pred_strd; \
|
} \
|
else \
|
{ \
|
const uint8x8_t s##k = vld2_u8(pu1_src).val[0]; \
|
const uint8x8_t p##k = vld2_u8(pu1_pred).val[0]; \
|
*r##k = vreinterpretq_s16_u16(vsubl_u8(s##k, p##k)); \
|
pu1_src += src_strd; \
|
pu1_pred += pred_strd; \
|
}
|
|
/*****************************************************************************/
|
/* Function Definitions */
|
/*****************************************************************************/
|
|
static INLINE void
|
hadamard4x4_2_one_pass(int16x8_t *r0, int16x8_t *r1, int16x8_t *r2, int16x8_t *r3)
|
{
|
const int16x8_t a0 = vaddq_s16(*r0, *r2);
|
const int16x8_t a1 = vaddq_s16(*r1, *r3);
|
const int16x8_t a2 = vsubq_s16(*r0, *r2);
|
const int16x8_t a3 = vsubq_s16(*r1, *r3);
|
|
*r0 = vaddq_s16(a0, a1);
|
*r1 = vsubq_s16(a0, a1);
|
*r2 = vaddq_s16(a2, a3);
|
*r3 = vsubq_s16(a2, a3);
|
}
|
|
static INLINE void hadamard4x4_2(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
int16x8_t *r0,
|
int16x8_t *r1,
|
int16x8_t *r2,
|
int16x8_t *r3)
|
{
|
// compute error between src and pred
|
RESIDUE(0, 0);
|
RESIDUE(1, 0);
|
RESIDUE(2, 0);
|
RESIDUE(3, 0);
|
|
// vertical hadamard tx
|
hadamard4x4_2_one_pass(r0, r1, r2, r3);
|
|
// transpose
|
transpose_s16_4x4q(r0, r1, r2, r3);
|
|
// horizontal hadamard tx
|
hadamard4x4_2_one_pass(r0, r1, r2, r3);
|
}
|
|
static INLINE void hadamard4x4_4(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
int16x8_t *r0,
|
int16x8_t *r1,
|
int16x8_t *r2,
|
int16x8_t *r3,
|
int16x8_t *r4,
|
int16x8_t *r5,
|
int16x8_t *r6,
|
int16x8_t *r7)
|
{
|
// hadamard 4x4_2n
|
hadamard4x4_2(pu1_src, src_strd, pu1_pred, pred_strd, r0, r1, r2, r3);
|
|
// hadamard 4x4_2n
|
pu1_src += (4 * src_strd);
|
pu1_pred += (4 * pred_strd);
|
hadamard4x4_2(pu1_src, src_strd, pu1_pred, pred_strd, r4, r5, r6, r7);
|
}
|
|
static INLINE WORD32 hadamard_sad4x4_4(int16x8_t *a, WORD32 *pi4_hsad, WORD32 hsad_stride)
|
{
|
int16x8_t p[8];
|
int32x4_t b01, b23;
|
int64x2_t c01, c23;
|
int32x2_t d01, d23;
|
|
// satd
|
p[0] = vabsq_s16(a[0]);
|
p[1] = vabsq_s16(a[1]);
|
p[0] = vaddq_s16(p[0], p[1]);
|
p[2] = vabsq_s16(a[2]);
|
p[3] = vabsq_s16(a[3]);
|
p[2] = vaddq_s16(p[2], p[3]);
|
|
p[4] = vabsq_s16(a[4]);
|
p[5] = vabsq_s16(a[5]);
|
p[4] = vaddq_s16(p[4], p[5]);
|
p[6] = vabsq_s16(a[6]);
|
p[7] = vabsq_s16(a[7]);
|
p[6] = vaddq_s16(p[6], p[7]);
|
|
p[0] = vaddq_s16(p[0], p[2]);
|
b01 = vpaddlq_s16(p[0]);
|
c01 = vpaddlq_s32(b01);
|
d01 = vrshrn_n_s64(c01, 2);
|
vst1_s32(pi4_hsad, d01);
|
pi4_hsad += hsad_stride;
|
|
p[4] = vaddq_s16(p[4], p[6]);
|
b23 = vpaddlq_s16(p[4]);
|
c23 = vpaddlq_s32(b23);
|
d23 = vrshrn_n_s64(c23, 2);
|
vst1_s32(pi4_hsad, d23);
|
|
d01 = vadd_s32(d01, d23);
|
|
return (WORD32)(vget_lane_s64(vpaddl_s32(d01), 0));
|
}
|
|
static INLINE WORD32 hadamard_sad8x8_using4x4(int16x8_t *a, WORD32 *early_cbf, WORD32 i4_frm_qstep)
|
{
|
int16x8_t p[8];
|
const int16x8_t threshold = vdupq_n_s16((int16_t)(i4_frm_qstep >> 8));
|
int32x4_t b;
|
int64x2_t c;
|
int64_t satd;
|
WORD32 i;
|
|
for(i = 0; i < 4; i++)
|
{
|
int16x8_t p0 = vaddq_s16(a[i], a[i + 4]);
|
int16x8_t p1 = vsubq_s16(a[i], a[i + 4]);
|
|
int16x4_t q0 = vadd_s16(vget_low_s16(p0), vget_high_s16(p0));
|
int16x4_t q1 = vsub_s16(vget_low_s16(p0), vget_high_s16(p0));
|
int16x4_t q2 = vadd_s16(vget_low_s16(p1), vget_high_s16(p1));
|
int16x4_t q3 = vsub_s16(vget_low_s16(p1), vget_high_s16(p1));
|
|
a[i] = vcombine_s16(q0, q2);
|
a[i + 4] = vcombine_s16(q1, q3);
|
}
|
|
#define EARLY_EXIT(k) \
|
{ \
|
p[k] = vabsq_s16(a[k]); \
|
if(*early_cbf == 0) \
|
{ \
|
uint16x8_t cmp; \
|
cmp = vcgtq_s16(p[k], threshold); \
|
if(vget_lane_s64(vreinterpret_s64_u16(vget_low_u16(cmp)), 0) || \
|
vget_lane_s64(vreinterpret_s64_u16(vget_high_u16(cmp)), 0)) \
|
{ \
|
*early_cbf = 1; \
|
} \
|
} \
|
}
|
// satd
|
EARLY_EXIT(0);
|
EARLY_EXIT(1);
|
p[0] = vaddq_s16(p[0], p[1]);
|
EARLY_EXIT(2);
|
EARLY_EXIT(3);
|
p[2] = vaddq_s16(p[2], p[3]);
|
|
EARLY_EXIT(4);
|
EARLY_EXIT(5);
|
p[4] = vaddq_s16(p[4], p[5]);
|
EARLY_EXIT(6);
|
EARLY_EXIT(7);
|
#undef EARLY_EXIT
|
p[6] = vaddq_s16(p[6], p[7]);
|
|
p[0] = vaddq_s16(p[0], p[2]);
|
p[4] = vaddq_s16(p[4], p[6]);
|
p[0] = vaddq_s16(p[0], p[4]);
|
b = vpaddlq_s16(p[0]);
|
c = vpaddlq_s32(b);
|
satd = vget_lane_s64(vadd_s64(vget_low_s64(c), vget_high_s64(c)), 0);
|
|
return ((satd + 4) >> 3);
|
}
|
|
static INLINE void hadamard8x8_one_pass(
|
int16x8_t *r0,
|
int16x8_t *r1,
|
int16x8_t *r2,
|
int16x8_t *r3,
|
int16x8_t *r4,
|
int16x8_t *r5,
|
int16x8_t *r6,
|
int16x8_t *r7)
|
{
|
const int16x8_t a0 = vaddq_s16(*r0, *r4);
|
const int16x8_t a4 = vsubq_s16(*r0, *r4);
|
const int16x8_t a1 = vaddq_s16(*r1, *r5);
|
const int16x8_t a5 = vsubq_s16(*r1, *r5);
|
const int16x8_t a2 = vaddq_s16(*r2, *r6);
|
const int16x8_t a6 = vsubq_s16(*r2, *r6);
|
const int16x8_t a3 = vaddq_s16(*r3, *r7);
|
const int16x8_t a7 = vsubq_s16(*r3, *r7);
|
|
const int16x8_t b0 = vaddq_s16(a0, a2);
|
const int16x8_t b2 = vsubq_s16(a0, a2);
|
const int16x8_t b1 = vaddq_s16(a1, a3);
|
const int16x8_t b3 = vsubq_s16(a1, a3);
|
const int16x8_t b4 = vaddq_s16(a4, a6);
|
const int16x8_t b6 = vsubq_s16(a4, a6);
|
const int16x8_t b5 = vaddq_s16(a5, a7);
|
const int16x8_t b7 = vsubq_s16(a5, a7);
|
|
*r0 = vaddq_s16(b0, b1);
|
*r1 = vsubq_s16(b0, b1);
|
*r2 = vaddq_s16(b2, b3);
|
*r3 = vsubq_s16(b2, b3);
|
*r4 = vaddq_s16(b4, b5);
|
*r5 = vsubq_s16(b4, b5);
|
*r6 = vaddq_s16(b6, b7);
|
*r7 = vsubq_s16(b6, b7);
|
}
|
|
static INLINE void hadamard8x8(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
int16x8_t *r0,
|
int16x8_t *r1,
|
int16x8_t *r2,
|
int16x8_t *r3,
|
int16x8_t *r4,
|
int16x8_t *r5,
|
int16x8_t *r6,
|
int16x8_t *r7,
|
WORD32 is_chroma)
|
{
|
// compute error between src and pred
|
RESIDUE(0, is_chroma);
|
RESIDUE(1, is_chroma);
|
RESIDUE(2, is_chroma);
|
RESIDUE(3, is_chroma);
|
RESIDUE(4, is_chroma);
|
RESIDUE(5, is_chroma);
|
RESIDUE(6, is_chroma);
|
RESIDUE(7, is_chroma);
|
|
// vertical hadamard tx
|
hadamard8x8_one_pass(r0, r1, r2, r3, r4, r5, r6, r7);
|
|
// transpose
|
transpose_s16_8x8(r0, r1, r2, r3, r4, r5, r6, r7);
|
|
// horizontal hadamard tx
|
hadamard8x8_one_pass(r0, r1, r2, r3, r4, r5, r6, r7);
|
}
|
|
static INLINE UWORD32 ihevce_HAD_8x8_8bit_plane_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD32 is_chroma,
|
WORD32 ac_only)
|
{
|
int16x8_t a0, a1, a2, a3, a4, a5, a6, a7;
|
int32x4_t b;
|
int64x2_t c;
|
int64_t satd;
|
|
// hadamard 8x8
|
hadamard8x8(
|
pu1_src, src_strd, pu1_pred, pred_strd, &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, is_chroma);
|
|
if(ac_only)
|
{
|
const int16x8_t mask = vld1q_s16(gu2_dc_mask);
|
a0 = vandq_s16(a0, mask);
|
}
|
|
// satd
|
a0 = vabsq_s16(a0);
|
a1 = vabsq_s16(a1);
|
a0 = vaddq_s16(a0, a1);
|
a2 = vabsq_s16(a2);
|
a3 = vabsq_s16(a3);
|
a2 = vaddq_s16(a2, a3);
|
|
a4 = vabsq_s16(a4);
|
a5 = vabsq_s16(a5);
|
a4 = vaddq_s16(a4, a5);
|
a6 = vabsq_s16(a6);
|
a7 = vabsq_s16(a7);
|
a6 = vaddq_s16(a6, a7);
|
|
a0 = vaddq_s16(a0, a2);
|
a4 = vaddq_s16(a4, a6);
|
a0 = vaddq_s16(a0, a4);
|
b = vpaddlq_s16(a0);
|
c = vpaddlq_s32(b);
|
satd = vget_lane_s64(vadd_s64(vget_low_s64(c), vget_high_s64(c)), 0);
|
|
return ((satd + 4) >> 3);
|
}
|
|
static INLINE UWORD32 ihevce_HAD_4x4_8bit_plane_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD32 is_chroma,
|
WORD32 ac_only)
|
{
|
uint8x16_t src_u8, pred_u8;
|
int16x8_t res_01, res_23;
|
int16x4_t h[4];
|
int16x4_t v[4];
|
int16x4x2_t trans_4[2];
|
int16x8_t combined_rows[4];
|
int32x4x2_t trans_8;
|
int32x4_t sad_32_4[3];
|
int32x2_t sad_32_2;
|
int64x1_t sad_64_1;
|
int32_t sad;
|
|
if(!is_chroma)
|
{
|
src_u8 = load_unaligned_u8q(pu1_src, src_strd);
|
pred_u8 = load_unaligned_u8q(pu1_pred, pred_strd);
|
}
|
else
|
{
|
src_u8 = load_unaligned_u8qi(pu1_src, src_strd);
|
pred_u8 = load_unaligned_u8qi(pu1_pred, pred_strd);
|
}
|
res_01 = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(src_u8), vget_low_u8(pred_u8)));
|
res_23 = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(src_u8), vget_high_u8(pred_u8)));
|
|
h[0] = vadd_s16(vget_low_s16(res_01), vget_high_s16(res_23));
|
h[1] = vadd_s16(vget_high_s16(res_01), vget_low_s16(res_23));
|
h[2] = vsub_s16(vget_high_s16(res_01), vget_low_s16(res_23));
|
h[3] = vsub_s16(vget_low_s16(res_01), vget_high_s16(res_23));
|
|
v[0] = vadd_s16(h[0], h[1]);
|
v[1] = vadd_s16(h[3], h[2]);
|
v[2] = vsub_s16(h[0], h[1]);
|
v[3] = vsub_s16(h[3], h[2]);
|
|
trans_4[0] = vtrn_s16(v[0], v[2]);
|
trans_4[1] = vtrn_s16(v[1], v[3]);
|
|
combined_rows[0] = vcombine_s16(trans_4[0].val[0], trans_4[1].val[0]);
|
combined_rows[1] = vcombine_s16(trans_4[0].val[1], trans_4[1].val[1]);
|
|
combined_rows[2] = vaddq_s16(combined_rows[0], combined_rows[1]);
|
combined_rows[3] = vsubq_s16(combined_rows[0], combined_rows[1]);
|
|
trans_8 =
|
vtrnq_s32(vreinterpretq_s32_s16(combined_rows[2]), vreinterpretq_s32_s16(combined_rows[3]));
|
|
combined_rows[0] =
|
vaddq_s16(vreinterpretq_s16_s32(trans_8.val[0]), vreinterpretq_s16_s32(trans_8.val[1]));
|
combined_rows[0] = vabsq_s16(combined_rows[0]);
|
combined_rows[1] =
|
vsubq_s16(vreinterpretq_s16_s32(trans_8.val[0]), vreinterpretq_s16_s32(trans_8.val[1]));
|
combined_rows[1] = vabsq_s16(combined_rows[1]);
|
|
if(ac_only)
|
{
|
const int16x8_t mask = vld1q_s16(gu2_dc_mask);
|
combined_rows[0] = vandq_s16(combined_rows[0], mask);
|
}
|
|
sad_32_4[0] = vpaddlq_s16(combined_rows[0]);
|
sad_32_4[1] = vpaddlq_s16(combined_rows[1]);
|
sad_32_4[2] = vaddq_s32(sad_32_4[0], sad_32_4[1]);
|
sad_32_2 = vadd_s32(vget_high_s32(sad_32_4[2]), vget_low_s32(sad_32_4[2]));
|
sad_64_1 = vpaddl_s32(sad_32_2);
|
sad = vget_lane_s64(sad_64_1, 0);
|
|
return ((sad + 2) >> 2);
|
}
|
|
UWORD32 ihevce_HAD_4x4_8bit_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd)
|
{
|
(void)pi2_dst;
|
(void)dst_strd;
|
return ihevce_HAD_4x4_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 0, 0);
|
}
|
|
UWORD32 ihevce_chroma_compute_AC_HAD_4x4_8bit_neon(
|
UWORD8 *pu1_origin,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred_buf,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd)
|
{
|
(void)pi2_dst;
|
(void)dst_strd;
|
return ihevce_HAD_4x4_8bit_plane_neon(pu1_origin, src_strd, pu1_pred_buf, pred_strd, 1, 1);
|
}
|
|
UWORD32 ihevce_HAD_8x8_8bit_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd)
|
{
|
(void)pi2_dst;
|
(void)dst_strd;
|
return ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 0, 0);
|
}
|
|
UWORD32 ihevce_compute_ac_had_8x8_8bit_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd)
|
{
|
(void)pi2_dst;
|
(void)dst_strd;
|
return ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 0, 1);
|
}
|
|
UWORD32 ihevce_HAD_16x16_8bit_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd)
|
{
|
int16x8_t b0[8];
|
int16x8_t b1[8];
|
int16x8_t b2[8];
|
int16x8_t b3[8];
|
uint32x4_t sum = vdupq_n_u32(0);
|
uint64x2_t c;
|
uint64_t satd;
|
WORD32 i;
|
|
(void)pi2_dst;
|
(void)dst_strd;
|
|
// hadamard 8x8 - b0
|
hadamard8x8(
|
pu1_src,
|
src_strd,
|
pu1_pred,
|
pred_strd,
|
&b0[0],
|
&b0[1],
|
&b0[2],
|
&b0[3],
|
&b0[4],
|
&b0[5],
|
&b0[6],
|
&b0[7],
|
0);
|
// hadamard 8x8 - b1
|
hadamard8x8(
|
pu1_src + 8,
|
src_strd,
|
pu1_pred + 8,
|
pred_strd,
|
&b1[0],
|
&b1[1],
|
&b1[2],
|
&b1[3],
|
&b1[4],
|
&b1[5],
|
&b1[6],
|
&b1[7],
|
0);
|
// hadamard 8x8 - b2
|
hadamard8x8(
|
pu1_src + (8 * src_strd),
|
src_strd,
|
pu1_pred + (8 * pred_strd),
|
pred_strd,
|
&b2[0],
|
&b2[1],
|
&b2[2],
|
&b2[3],
|
&b2[4],
|
&b2[5],
|
&b2[6],
|
&b2[7],
|
0);
|
// hadamard 8x8 - b3
|
hadamard8x8(
|
pu1_src + (8 * src_strd) + 8,
|
src_strd,
|
pu1_pred + (8 * pred_strd) + 8,
|
pred_strd,
|
&b3[0],
|
&b3[1],
|
&b3[2],
|
&b3[3],
|
&b3[4],
|
&b3[5],
|
&b3[6],
|
&b3[7],
|
0);
|
|
for(i = 0; i < 8; i++)
|
{
|
int16x8_t p0 = vhaddq_s16(b0[i], b1[i]);
|
int16x8_t p1 = vhsubq_s16(b0[i], b1[i]);
|
int16x8_t p2 = vhaddq_s16(b2[i], b3[i]);
|
int16x8_t p3 = vhsubq_s16(b2[i], b3[i]);
|
|
int16x8_t q0 = vaddq_s16(p0, p2);
|
int16x8_t q1 = vsubq_s16(p0, p2);
|
int16x8_t q2 = vaddq_s16(p1, p3);
|
int16x8_t q3 = vsubq_s16(p1, p3);
|
|
uint16x8_t r0 =
|
vaddq_u16(vreinterpretq_u16_s16(vabsq_s16(q0)), vreinterpretq_u16_s16(vabsq_s16(q1)));
|
uint16x8_t r1 =
|
vaddq_u16(vreinterpretq_u16_s16(vabsq_s16(q2)), vreinterpretq_u16_s16(vabsq_s16(q3)));
|
|
uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0));
|
uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1));
|
|
sum = vaddq_u32(sum, s0);
|
sum = vaddq_u32(sum, s1);
|
}
|
|
c = vpaddlq_u32(sum);
|
satd = vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0);
|
|
return ((satd + 4) >> 3);
|
}
|
|
UWORD32 ihevce_chroma_HAD_4x4_8bit_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd)
|
{
|
(void)pi2_dst;
|
(void)dst_strd;
|
return ihevce_HAD_4x4_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 1, 0);
|
}
|
|
UWORD32 ihevce_chroma_HAD_8x8_8bit_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd)
|
{
|
(void)pi2_dst;
|
(void)dst_strd;
|
return ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 1, 0);
|
}
|
|
UWORD32 ihevce_chroma_HAD_16x16_8bit_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd)
|
{
|
UWORD32 au4_satd[4];
|
|
(void)pi2_dst;
|
(void)dst_strd;
|
au4_satd[0] = ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 1, 0);
|
au4_satd[1] =
|
ihevce_HAD_8x8_8bit_plane_neon(pu1_src + 16, src_strd, pu1_pred + 16, pred_strd, 1, 0);
|
au4_satd[2] = ihevce_HAD_8x8_8bit_plane_neon(
|
pu1_src + 8 * src_strd, src_strd, pu1_pred + 8 * pred_strd, pred_strd, 1, 0);
|
au4_satd[3] = ihevce_HAD_8x8_8bit_plane_neon(
|
pu1_src + 8 * src_strd + 16, src_strd, pu1_pred + 8 * pred_strd + 16, pred_strd, 1, 0);
|
|
return au4_satd[0] + au4_satd[1] + au4_satd[2] + au4_satd[3];
|
}
|
|
UWORD32 ihevce_HAD_32x32_8bit_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd)
|
{
|
int16x8_t a[4][4][8];
|
uint32x4_t sum = vdupq_n_u32(0);
|
WORD32 b8, b16;
|
uint64x2_t c;
|
uint64_t satd;
|
WORD32 i, j;
|
|
(void)pi2_dst;
|
(void)dst_strd;
|
// hadamard 32x32
|
for(b16 = 0; b16 < 4; b16++)
|
{
|
UWORD8 *pu1_src_b16 = pu1_src + (b16 >> 1) * (src_strd * 16) + ((b16 & 1) * 16);
|
UWORD8 *pu1_pred_b16 = pu1_pred + (b16 >> 1) * (pred_strd * 16) + ((b16 & 1) * 16);
|
// hadamard 16x16
|
for(b8 = 0; b8 < 4; b8++)
|
{
|
UWORD8 *pu1_src_b8 = pu1_src_b16 + (b8 >> 1) * (src_strd * 8) + ((b8 & 1) * 8);
|
UWORD8 *pu1_pred_b8 = pu1_pred_b16 + (b8 >> 1) * (pred_strd * 8) + ((b8 & 1) * 8);
|
// hadamard 8x8
|
hadamard8x8(
|
pu1_src_b8,
|
src_strd,
|
pu1_pred_b8,
|
pred_strd,
|
&a[b16][b8][0],
|
&a[b16][b8][1],
|
&a[b16][b8][2],
|
&a[b16][b8][3],
|
&a[b16][b8][4],
|
&a[b16][b8][5],
|
&a[b16][b8][6],
|
&a[b16][b8][7],
|
0);
|
}
|
for(i = 0; i < 8; i++)
|
{
|
int16x8_t p0 = vhaddq_s16(a[b16][0][i], a[b16][1][i]);
|
int16x8_t p1 = vhsubq_s16(a[b16][0][i], a[b16][1][i]);
|
int16x8_t p2 = vhaddq_s16(a[b16][2][i], a[b16][3][i]);
|
int16x8_t p3 = vhsubq_s16(a[b16][2][i], a[b16][3][i]);
|
|
a[b16][0][i] = vaddq_s16(p0, p2);
|
a[b16][1][i] = vsubq_s16(p0, p2);
|
a[b16][2][i] = vaddq_s16(p1, p3);
|
a[b16][3][i] = vsubq_s16(p1, p3);
|
|
a[b16][0][i] = vshrq_n_s16(a[b16][0][i], 2);
|
a[b16][1][i] = vshrq_n_s16(a[b16][1][i], 2);
|
a[b16][2][i] = vshrq_n_s16(a[b16][2][i], 2);
|
a[b16][3][i] = vshrq_n_s16(a[b16][3][i], 2);
|
}
|
}
|
for(j = 0; j < 4; j++)
|
{
|
for(i = 0; i < 8; i++)
|
{
|
int16x8_t p0 = vaddq_s16(a[0][j][i], a[1][j][i]);
|
int16x8_t p1 = vsubq_s16(a[0][j][i], a[1][j][i]);
|
int16x8_t p2 = vaddq_s16(a[2][j][i], a[3][j][i]);
|
int16x8_t p3 = vsubq_s16(a[2][j][i], a[3][j][i]);
|
|
int16x8_t q0 = vaddq_s16(p0, p2);
|
int16x8_t q1 = vsubq_s16(p0, p2);
|
int16x8_t q2 = vaddq_s16(p1, p3);
|
int16x8_t q3 = vsubq_s16(p1, p3);
|
|
uint16x8_t r0 = vaddq_u16(
|
vreinterpretq_u16_s16(vabsq_s16(q0)), vreinterpretq_u16_s16(vabsq_s16(q1)));
|
uint16x8_t r1 = vaddq_u16(
|
vreinterpretq_u16_s16(vabsq_s16(q2)), vreinterpretq_u16_s16(vabsq_s16(q3)));
|
|
uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0));
|
uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1));
|
|
sum = vaddq_u32(sum, s0);
|
sum = vaddq_u32(sum, s1);
|
}
|
}
|
c = vpaddlq_u32(sum);
|
satd = vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0);
|
|
return ((satd + 2) >> 2);
|
}
|
|
WORD32 ihevce_had4_4x4_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst4x4,
|
WORD32 dst_strd,
|
WORD32 *pi4_hsad,
|
WORD32 hsad_stride,
|
WORD32 i4_frm_qstep)
|
{
|
int16x8_t a[8];
|
|
(void)pi2_dst4x4;
|
(void)dst_strd;
|
(void)i4_frm_qstep;
|
|
/* -------- Compute four 4x4 HAD Transforms of 8x8 in one call--------- */
|
hadamard4x4_4(
|
pu1_src,
|
src_strd,
|
pu1_pred,
|
pred_strd,
|
&a[0],
|
&a[1],
|
&a[2],
|
&a[3],
|
&a[4],
|
&a[5],
|
&a[6],
|
&a[7]);
|
|
return hadamard_sad4x4_4(a, pi4_hsad, hsad_stride);
|
}
|
|
WORD32 ihevce_had_8x8_using_4_4x4_r_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd,
|
WORD32 **ppi4_hsad,
|
WORD32 **ppi4_tu_split,
|
WORD32 **ppi4_tu_early_cbf,
|
WORD32 pos_x_y_4x4,
|
WORD32 num_4x4_in_row,
|
WORD32 lambda,
|
WORD32 lambda_q_shift,
|
WORD32 i4_frm_qstep,
|
WORD32 i4_cur_depth,
|
WORD32 i4_max_depth,
|
WORD32 i4_max_tr_size,
|
WORD32 *pi4_tu_split_cost,
|
void *pv_func_sel)
|
{
|
WORD32 pos_x = pos_x_y_4x4 & 0xFFFF;
|
WORD32 pos_y = (pos_x_y_4x4 >> 16) & 0xFFFF;
|
|
WORD32 *pi4_4x4_hsad;
|
WORD32 *pi4_8x8_hsad;
|
WORD32 *pi4_8x8_tu_split;
|
WORD32 *pi4_8x8_tu_early_cbf;
|
|
WORD32 cost_child, cost_parent;
|
WORD32 best_cost;
|
WORD32 early_cbf = 0;
|
const UWORD8 u1_cur_tr_size = 8;
|
|
WORD32 i;
|
|
int16x8_t a[8];
|
|
(void)pv_func_sel;
|
|
assert(pos_x >= 0);
|
assert(pos_y >= 0);
|
|
/* Initialize pointers to store 4x4 and 8x8 HAD SATDs */
|
pi4_4x4_hsad = ppi4_hsad[HAD_4x4] + pos_x + pos_y * num_4x4_in_row;
|
pi4_8x8_hsad = ppi4_hsad[HAD_8x8] + (pos_x >> 1) + (pos_y >> 1) * (num_4x4_in_row >> 1);
|
pi4_8x8_tu_split = ppi4_tu_split[HAD_8x8] + (pos_x >> 1) + (pos_y >> 1) * (num_4x4_in_row >> 1);
|
pi4_8x8_tu_early_cbf =
|
ppi4_tu_early_cbf[HAD_8x8] + (pos_x >> 1) + (pos_y >> 1) * (num_4x4_in_row >> 1);
|
|
/* -------- Compute four 4x4 HAD Transforms of 8x8 in one call--------- */
|
hadamard4x4_4(
|
pu1_src,
|
src_strd,
|
pu1_pred,
|
pred_strd,
|
&a[0],
|
&a[1],
|
&a[2],
|
&a[3],
|
&a[4],
|
&a[5],
|
&a[6],
|
&a[7]);
|
|
/* -------- cost child -------- */
|
cost_child = hadamard_sad4x4_4(a, pi4_4x4_hsad, num_4x4_in_row);
|
/* 4 CBF Flags, extra 1 becoz of the 0.5 bits per bin is assumed */
|
cost_child += ((4) * lambda) >> (lambda_q_shift + 1);
|
|
/* -------- cost parent -------- */
|
cost_parent = hadamard_sad8x8_using4x4(a, &early_cbf, i4_frm_qstep);
|
for(i = 0; i < 8; i++, pi2_dst += dst_strd)
|
vst1q_s16(pi2_dst, a[i]);
|
|
if(i4_cur_depth < i4_max_depth)
|
{
|
if((cost_child < cost_parent) || (i4_max_tr_size < u1_cur_tr_size))
|
{
|
*pi4_tu_split_cost += (4 * lambda) >> (lambda_q_shift + 1);
|
best_cost = cost_child;
|
best_cost <<= 1;
|
best_cost++;
|
pi4_8x8_tu_split[0] = 1;
|
pi4_8x8_hsad[0] = cost_child;
|
}
|
else
|
{
|
best_cost = cost_parent;
|
best_cost <<= 1;
|
pi4_8x8_tu_split[0] = 0;
|
pi4_8x8_hsad[0] = cost_parent;
|
}
|
}
|
else
|
{
|
best_cost = cost_parent;
|
best_cost <<= 1;
|
pi4_8x8_tu_split[0] = 0;
|
pi4_8x8_hsad[0] = cost_parent;
|
}
|
|
pi4_8x8_tu_early_cbf[0] = early_cbf;
|
|
/* best cost has tu_split_flag at LSB(Least significant bit) */
|
return ((best_cost << 1) + early_cbf);
|
}
|
|
static WORD32 ihevce_compute_16x16HAD_using_8x8_neon(
|
WORD16 *pi2_8x8_had,
|
WORD32 had8_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd,
|
WORD32 i4_frm_qstep,
|
WORD32 *pi4_cbf)
|
{
|
int16x8_t b0[8];
|
int16x8_t b1[8];
|
int16x8_t b2[8];
|
int16x8_t b3[8];
|
const int16x8_t threshold = vdupq_n_s16((int16_t)(i4_frm_qstep >> 8));
|
uint32x4_t sum = vdupq_n_u32(0);
|
uint64x2_t c;
|
uint64_t satd;
|
WORD32 i;
|
|
for(i = 0; i < 8; i++, pi2_8x8_had += had8_strd)
|
{
|
b0[i] = vld1q_s16(pi2_8x8_had);
|
b1[i] = vld1q_s16(pi2_8x8_had + 8);
|
}
|
for(i = 0; i < 8; i++, pi2_8x8_had += had8_strd)
|
{
|
b2[i] = vld1q_s16(pi2_8x8_had);
|
b3[i] = vld1q_s16(pi2_8x8_had + 8);
|
}
|
|
#define EARLY_EXIT(k) \
|
{ \
|
p##k = vabsq_s16(q##k); \
|
if(*pi4_cbf == 0) \
|
{ \
|
uint16x8_t cmp; \
|
cmp = vcgtq_s16(p##k, threshold); \
|
if(vget_lane_s64(vreinterpret_s64_u16(vget_low_u16(cmp)), 0) || \
|
vget_lane_s64(vreinterpret_s64_u16(vget_high_u16(cmp)), 0)) \
|
{ \
|
*pi4_cbf = 1; \
|
} \
|
} \
|
}
|
for(i = 0; i < 8; i++, pi2_dst += dst_strd)
|
{
|
int16x8_t p0 = vhaddq_s16(b0[i], b1[i]);
|
int16x8_t p1 = vhsubq_s16(b0[i], b1[i]);
|
int16x8_t p2 = vhaddq_s16(b2[i], b3[i]);
|
int16x8_t p3 = vhsubq_s16(b2[i], b3[i]);
|
|
int16x8_t q0 = vaddq_s16(p0, p2);
|
int16x8_t q1 = vsubq_s16(p0, p2);
|
int16x8_t q2 = vaddq_s16(p1, p3);
|
int16x8_t q3 = vsubq_s16(p1, p3);
|
|
vst1q_s16(pi2_dst, q0);
|
EARLY_EXIT(0);
|
vst1q_s16(pi2_dst + 8, q1);
|
EARLY_EXIT(1);
|
vst1q_s16(pi2_dst + 8 * dst_strd, q2);
|
EARLY_EXIT(2);
|
vst1q_s16(pi2_dst + 8 * dst_strd + 8, q3);
|
EARLY_EXIT(3);
|
uint16x8_t r0 = vaddq_u16(vreinterpretq_u16_s16(p0), vreinterpretq_u16_s16(p1));
|
uint16x8_t r1 = vaddq_u16(vreinterpretq_u16_s16(p2), vreinterpretq_u16_s16(p3));
|
|
uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0));
|
uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1));
|
|
sum = vaddq_u32(sum, s0);
|
sum = vaddq_u32(sum, s1);
|
}
|
|
c = vpaddlq_u32(sum);
|
satd = vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0);
|
|
return ((satd + 4) >> 3);
|
}
|
|
WORD32 ihevce_had_16x16_r_neon(
|
UWORD8 *pu1_src,
|
WORD32 src_strd,
|
UWORD8 *pu1_pred,
|
WORD32 pred_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd,
|
WORD32 **ppi4_hsad,
|
WORD32 **ppi4_tu_split,
|
WORD32 **ppi4_tu_early_cbf,
|
WORD32 pos_x_y_4x4,
|
WORD32 num_4x4_in_row,
|
WORD32 lambda,
|
WORD32 lambda_q_shift,
|
WORD32 i4_frm_qstep,
|
WORD32 i4_cur_depth,
|
WORD32 i4_max_depth,
|
WORD32 i4_max_tr_size,
|
WORD32 *pi4_tu_split_cost,
|
void *pv_func_sel)
|
{
|
WORD16 ai2_8x8_had[256];
|
|
WORD32 *pi4_16x16_hsad;
|
WORD32 *pi4_16x16_tu_split;
|
WORD32 *pi4_16x16_tu_early_cbf;
|
|
WORD32 best_cost, best_cost_tu_split;
|
WORD32 tu_split_flag = 0;
|
WORD32 i4_early_cbf_flag = 0, early_cbf = 0;
|
WORD32 cost_parent, cost_child = 0;
|
|
const UWORD8 u1_cur_tr_size = 16;
|
|
WORD32 i;
|
|
WORD16 *pi2_y0;
|
UWORD8 *src, *pred;
|
WORD32 pos_x_y_4x4_0;
|
|
WORD32 pos_x = pos_x_y_4x4 & 0xFFFF;
|
WORD32 pos_y = (pos_x_y_4x4 >> 16) & 0xFFFF;
|
|
assert(pos_x >= 0);
|
assert(pos_y >= 0);
|
|
/* Initialize pointers to store 16x16 SATDs */
|
pi4_16x16_hsad = ppi4_hsad[HAD_16x16] + (pos_x >> 2) + (pos_y >> 2) * (num_4x4_in_row >> 2);
|
|
pi4_16x16_tu_split =
|
ppi4_tu_split[HAD_16x16] + (pos_x >> 2) + (pos_y >> 2) * (num_4x4_in_row >> 2);
|
|
pi4_16x16_tu_early_cbf =
|
ppi4_tu_early_cbf[HAD_16x16] + (pos_x >> 2) + (pos_y >> 2) * (num_4x4_in_row >> 2);
|
|
/* -------- Compute four 8x8 HAD Transforms of 16x16 call--------- */
|
for(i = 0; i < 4; i++)
|
{
|
src = pu1_src + (i & 0x01) * 8 + (i >> 1) * src_strd * 8;
|
pred = pu1_pred + (i & 0x01) * 8 + (i >> 1) * pred_strd * 8;
|
pi2_y0 = ai2_8x8_had + (i & 0x01) * 8 + (i >> 1) * 16 * 8;
|
pos_x_y_4x4_0 = pos_x_y_4x4 + (i & 0x01) * 2 + (i >> 1) * (2 << 16);
|
|
best_cost_tu_split = ihevce_had_8x8_using_4_4x4_r_neon(
|
src,
|
src_strd,
|
pred,
|
pred_strd,
|
pi2_y0,
|
16,
|
ppi4_hsad,
|
ppi4_tu_split,
|
ppi4_tu_early_cbf,
|
pos_x_y_4x4_0,
|
num_4x4_in_row,
|
lambda,
|
lambda_q_shift,
|
i4_frm_qstep,
|
i4_cur_depth + 1,
|
i4_max_depth,
|
i4_max_tr_size,
|
pi4_tu_split_cost,
|
pv_func_sel);
|
|
/* Cost is shifted by two bits for Tu_split_flag and early cbf flag */
|
best_cost = (best_cost_tu_split >> 2);
|
|
/* Last but one bit stores the information regarding the TU_Split */
|
tu_split_flag += (best_cost_tu_split & 0x3) >> 1;
|
|
/* Last bit stores the information regarding the early_cbf */
|
i4_early_cbf_flag += (best_cost_tu_split & 0x1);
|
|
cost_child += best_cost;
|
|
tu_split_flag <<= 1;
|
i4_early_cbf_flag <<= 1;
|
}
|
|
/* -------- Compute 16x16 HAD Transform using 8x8 results ------------- */
|
pi2_y0 = ai2_8x8_had;
|
|
/* Threshold currently passed as "0" */
|
cost_parent = ihevce_compute_16x16HAD_using_8x8_neon(
|
pi2_y0, 16, pi2_dst, dst_strd, i4_frm_qstep, &early_cbf);
|
|
/* 4 TU_Split flags , 4 CBF Flags, extra 1 becoz of the 0.5 bits per bin is assumed */
|
cost_child += ((4 + 4) * lambda) >> (lambda_q_shift + 1);
|
|
i4_early_cbf_flag += early_cbf;
|
|
/* Right now the depth is hard-coded to 4: The depth can be modified from the config file
|
which decides the extent to which TU_REC needs to be done */
|
if(i4_cur_depth < i4_max_depth)
|
{
|
if((cost_child < cost_parent) || (i4_max_tr_size < u1_cur_tr_size))
|
{
|
*pi4_tu_split_cost += ((4 + 4) * lambda) >> (lambda_q_shift + 1);
|
tu_split_flag += 1;
|
best_cost = cost_child;
|
}
|
else
|
{
|
tu_split_flag += 0;
|
best_cost = cost_parent;
|
}
|
}
|
else
|
{
|
tu_split_flag += 0;
|
best_cost = cost_parent;
|
}
|
|
pi4_16x16_hsad[0] = best_cost;
|
pi4_16x16_tu_split[0] = tu_split_flag;
|
pi4_16x16_tu_early_cbf[0] = i4_early_cbf_flag;
|
|
/*returning two values(best cost & tu_split_flag) as a single value*/
|
return ((best_cost << 10) + (tu_split_flag << 5) + i4_early_cbf_flag);
|
}
|
|
UWORD32 ihevce_compute_32x32HAD_using_16x16_neon(
|
WORD16 *pi2_16x16_had,
|
WORD32 had16_strd,
|
WORD16 *pi2_dst,
|
WORD32 dst_strd,
|
WORD32 i4_frm_qstep,
|
WORD32 *pi4_cbf)
|
{
|
int16x8_t a[4][4][8];
|
uint32x4_t sum = vdupq_n_u32(0);
|
const int16x8_t threshold = vdupq_n_s16((int16_t)(i4_frm_qstep >> 8));
|
WORD32 b8, b16;
|
uint64x2_t c;
|
WORD32 i, j;
|
|
(void)pi2_dst;
|
(void)dst_strd;
|
|
for(b16 = 0; b16 < 4; b16++)
|
{
|
WORD16 *pi2_b16 = pi2_16x16_had + (b16 >> 1) * (had16_strd * 16) + ((b16 & 1) * 16);
|
|
for(b8 = 0; b8 < 4; b8++)
|
{
|
WORD16 *pi2_b8 = pi2_b16 + (b8 >> 1) * (had16_strd * 8) + ((b8 & 1) * 8);
|
|
for(i = 0; i < 8; i++, pi2_b8 += had16_strd)
|
{
|
a[b16][b8][i] = vld1q_s16(pi2_b8);
|
a[b16][b8][i] = vshrq_n_s16(a[b16][b8][i], 2);
|
}
|
}
|
}
|
|
for(j = 0; j < 4; j++)
|
{
|
for(i = 0; i < 8; i++)
|
{
|
int16x8_t p0 = vaddq_s16(a[0][j][i], a[1][j][i]);
|
int16x8_t p1 = vsubq_s16(a[0][j][i], a[1][j][i]);
|
int16x8_t p2 = vaddq_s16(a[2][j][i], a[3][j][i]);
|
int16x8_t p3 = vsubq_s16(a[2][j][i], a[3][j][i]);
|
|
int16x8_t q0 = vaddq_s16(p0, p2);
|
int16x8_t q1 = vsubq_s16(p0, p2);
|
int16x8_t q2 = vaddq_s16(p1, p3);
|
int16x8_t q3 = vsubq_s16(p1, p3);
|
|
EARLY_EXIT(0);
|
EARLY_EXIT(1);
|
EARLY_EXIT(2);
|
EARLY_EXIT(3);
|
|
uint16x8_t r0 = vaddq_u16(vreinterpretq_u16_s16(p0), vreinterpretq_u16_s16(p1));
|
uint16x8_t r1 = vaddq_u16(vreinterpretq_u16_s16(p2), vreinterpretq_u16_s16(p3));
|
|
uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0));
|
uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1));
|
|
sum = vaddq_u32(sum, s0);
|
sum = vaddq_u32(sum, s1);
|
}
|
}
|
c = vpaddlq_u32(sum);
|
|
return vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0);
|
}
|
