From f5b470781ae6b69e672baf1fbe022c5017db681b Mon Sep 17 00:00:00 2001 From: Shiyou Yin Date: Thu, 3 Sep 2020 14:51:18 +0800 Subject: [PATCH] Add optimizations for loongarch Change-Id: I005f2d8c0f874f2732628de6daea33ee4441fda8 Signed-off-by: Shiyou Yin --- Makefile | 2 +- configure | 58 +- ffbuild/arch.mak | 4 +- ffbuild/common.mak | 8 + fftools/ffmpeg.c | 2 +- libavcodec/cabac_functions.h | 3 + libavcodec/h264chroma.c | 2 + libavcodec/h264chroma.h | 1 + libavcodec/h264dsp.c | 1 + libavcodec/h264dsp.h | 2 + libavcodec/h264qpel.c | 2 + libavcodec/h264qpel.h | 1 + libavcodec/hevcdsp.c | 2 + libavcodec/hevcdsp.h | 1 + libavcodec/hpeldsp.c | 2 + libavcodec/hpeldsp.h | 1 + libavcodec/idctdsp.c | 2 + libavcodec/idctdsp.h | 2 + libavcodec/loongarch/Makefile | 30 + libavcodec/loongarch/cabac.h | 198 + libavcodec/loongarch/h264_deblock_lasx.c | 144 + .../loongarch/h264chroma_init_loongarch.c | 35 + libavcodec/loongarch/h264chroma_lasx.c | 837 ++++ libavcodec/loongarch/h264chroma_lasx.h | 34 + libavcodec/loongarch/h264dsp_init_loongarch.c | 81 + libavcodec/loongarch/h264dsp_lasx.c | 1753 ++++++++ libavcodec/loongarch/h264dsp_lasx.h | 99 + libavcodec/loongarch/h264idct_lasx.c | 461 +++ .../loongarch/h264qpel_init_loongarch.c | 96 + libavcodec/loongarch/h264qpel_lasx.c | 2053 ++++++++++ libavcodec/loongarch/h264qpel_lasx.h | 158 + libavcodec/loongarch/hevc_idct_lsx.c | 874 ++++ libavcodec/loongarch/hevc_lpf_sao_lsx.c | 2422 +++++++++++ libavcodec/loongarch/hevc_macros_lsx.h | 71 + libavcodec/loongarch/hevc_mc_bi_lsx.c | 3075 ++++++++++++++ libavcodec/loongarch/hevc_mc_uni_lsx.c | 1476 +++++++ libavcodec/loongarch/hevc_mc_uniw_lsx.c | 338 ++ libavcodec/loongarch/hevcdsp_init_loongarch.c | 189 + libavcodec/loongarch/hevcdsp_lsx.c | 3338 +++++++++++++++ libavcodec/loongarch/hevcdsp_lsx.h | 233 ++ libavcodec/loongarch/hpeldsp_init_loongarch.c | 71 + libavcodec/loongarch/hpeldsp_lasx.c | 1041 +++++ libavcodec/loongarch/hpeldsp_lasx.h | 57 + libavcodec/loongarch/idctdsp_init_loongarch.c | 45 + libavcodec/loongarch/idctdsp_lasx.c | 116 + libavcodec/loongarch/idctdsp_loongarch.h | 42 + libavcodec/loongarch/simple_idct_lasx.c | 320 ++ libavcodec/loongarch/vc1dsp_init_loongarch.c | 67 + libavcodec/loongarch/vc1dsp_lasx.c | 997 +++++ libavcodec/loongarch/vc1dsp_loongarch.h | 79 + libavcodec/loongarch/videodsp_init.c | 46 + libavcodec/loongarch/vp8_lpf_lsx.c | 437 ++ libavcodec/loongarch/vp8dsp_init_loongarch.c | 41 + libavcodec/loongarch/vp8dsp_loongarch.h | 39 + libavcodec/loongarch/vp9_idct_lsx.c | 1513 +++++++ libavcodec/loongarch/vp9_intra_lsx.c | 612 +++ libavcodec/loongarch/vp9_lpf_lsx.c | 3140 +++++++++++++++ libavcodec/loongarch/vp9_mc_lsx.c | 2318 +++++++++++ libavcodec/loongarch/vp9dsp_init_loongarch.c | 178 + libavcodec/loongarch/vp9dsp_loongarch.h | 182 + libavcodec/mips/Makefile | 3 +- libavcodec/mips/blockdsp_mmi.c | 8 +- libavcodec/mips/cabac.h | 129 +- libavcodec/mips/constants.c | 89 +- libavcodec/mips/constants.h | 88 +- libavcodec/mips/h264_deblock_msa.c | 153 + libavcodec/mips/h264chroma_init_mips.c | 19 +- libavcodec/mips/h264chroma_mmi.c | 177 +- libavcodec/mips/h264dsp_init_mips.c | 6 +- libavcodec/mips/h264dsp_mips.h | 4 + libavcodec/mips/h264dsp_mmi.c | 308 +- libavcodec/mips/h264dsp_msa.c | 465 +-- libavcodec/mips/h264pred_mmi.c | 41 +- libavcodec/mips/h264qpel_mmi.c | 60 +- libavcodec/mips/hevcdsp_mmi.c | 91 +- libavcodec/mips/hpeldsp_mmi.c | 26 +- libavcodec/mips/idctdsp_mmi.c | 4 +- libavcodec/mips/mpegvideo_mmi.c | 114 +- libavcodec/mips/pixblockdsp_mmi.c | 8 +- libavcodec/mips/simple_idct_mmi.c | 14 +- libavcodec/mips/vc1dsp_mmi.c | 210 +- libavcodec/mips/vp3dsp_idct_mmi.c | 132 +- libavcodec/mips/vp8dsp_mmi.c | 343 +- libavcodec/mips/vp9_mc_mmi.c | 26 +- libavcodec/mips/wmv2dsp_mmi.c | 2 +- libavcodec/vc1dsp.c | 3 +- libavcodec/vc1dsp.h | 1 + libavcodec/videodsp.c | 2 + libavcodec/videodsp.h | 1 + libavcodec/vp8dsp.c | 2 + libavcodec/vp8dsp.h | 1 + libavcodec/vp9dsp.c | 1 + libavcodec/vp9dsp.h | 1 + libavutil/cpu.c | 10 + libavutil/cpu.h | 4 + libavutil/cpu_internal.h | 2 + libavutil/loongarch/Makefile | 1 + libavutil/loongarch/cpu.c | 69 + libavutil/loongarch/cpu.h | 31 + libavutil/loongarch/generic_macros_lasx.h | 3561 +++++++++++++++++ libavutil/loongarch/generic_macros_lsx.h | 670 ++++ libavutil/loongarch/generic_macros_lsx.h.bak | 425 ++ libavutil/mips/asmdefs.h | 8 + libavutil/mips/generic_macros_msa.h | 37 - libavutil/tests/cpu.c | 3 + libswscale/loongarch/Makefile | 6 + libswscale/loongarch/input_lasx.c | 191 + libswscale/loongarch/output_lasx.c | 2436 +++++++++++ libswscale/loongarch/rgb2rgb_lasx.c | 54 + libswscale/loongarch/swscale_init_loongarch.c | 95 + libswscale/loongarch/swscale_lasx.c | 944 +++++ libswscale/loongarch/swscale_loongarch.h | 78 + libswscale/loongarch/yuv2rgb_lasx.c | 209 + libswscale/mips/Makefile | 4 + libswscale/mips/rgb2rgb_init_mips.c | 33 + libswscale/mips/rgb2rgb_mips.h | 32 + libswscale/mips/rgb2rgb_msa.c | 51 + libswscale/mips/swscale_init_mips.c | 211 + libswscale/mips/swscale_mips.h | 404 ++ libswscale/mips/swscale_msa.c | 1805 +++++++++ libswscale/rgb2rgb.c | 4 + libswscale/rgb2rgb.h | 2 + libswscale/swscale.c | 4 + libswscale/swscale_internal.h | 3 + libswscale/utils.c | 24 +- libswscale/yuv2rgb.c | 2 + tests/checkasm/checkasm.c | 3 + 127 files changed, 42187 insertions(+), 1193 deletions(-) create mode 100644 libavcodec/loongarch/Makefile create mode 100644 libavcodec/loongarch/cabac.h create mode 100644 libavcodec/loongarch/h264_deblock_lasx.c create mode 100644 libavcodec/loongarch/h264chroma_init_loongarch.c create mode 100644 libavcodec/loongarch/h264chroma_lasx.c create mode 100644 libavcodec/loongarch/h264chroma_lasx.h create mode 100644 libavcodec/loongarch/h264dsp_init_loongarch.c create mode 100644 libavcodec/loongarch/h264dsp_lasx.c create mode 100644 libavcodec/loongarch/h264dsp_lasx.h create mode 100644 libavcodec/loongarch/h264idct_lasx.c create mode 100644 libavcodec/loongarch/h264qpel_init_loongarch.c create mode 100644 libavcodec/loongarch/h264qpel_lasx.c create mode 100644 libavcodec/loongarch/h264qpel_lasx.h create mode 100644 libavcodec/loongarch/hevc_idct_lsx.c create mode 100644 libavcodec/loongarch/hevc_lpf_sao_lsx.c create mode 100644 libavcodec/loongarch/hevc_macros_lsx.h create mode 100644 libavcodec/loongarch/hevc_mc_bi_lsx.c create mode 100644 libavcodec/loongarch/hevc_mc_uni_lsx.c create mode 100644 libavcodec/loongarch/hevc_mc_uniw_lsx.c create mode 100644 libavcodec/loongarch/hevcdsp_init_loongarch.c create mode 100644 libavcodec/loongarch/hevcdsp_lsx.c create mode 100644 libavcodec/loongarch/hevcdsp_lsx.h create mode 100644 libavcodec/loongarch/hpeldsp_init_loongarch.c create mode 100644 libavcodec/loongarch/hpeldsp_lasx.c create mode 100644 libavcodec/loongarch/hpeldsp_lasx.h create mode 100644 libavcodec/loongarch/idctdsp_init_loongarch.c create mode 100644 libavcodec/loongarch/idctdsp_lasx.c create mode 100644 libavcodec/loongarch/idctdsp_loongarch.h create mode 100644 libavcodec/loongarch/simple_idct_lasx.c create mode 100644 libavcodec/loongarch/vc1dsp_init_loongarch.c create mode 100644 libavcodec/loongarch/vc1dsp_lasx.c create mode 100644 libavcodec/loongarch/vc1dsp_loongarch.h create mode 100644 libavcodec/loongarch/videodsp_init.c create mode 100644 libavcodec/loongarch/vp8_lpf_lsx.c create mode 100644 libavcodec/loongarch/vp8dsp_init_loongarch.c create mode 100644 libavcodec/loongarch/vp8dsp_loongarch.h create mode 100644 libavcodec/loongarch/vp9_idct_lsx.c create mode 100644 libavcodec/loongarch/vp9_intra_lsx.c create mode 100644 libavcodec/loongarch/vp9_lpf_lsx.c create mode 100644 libavcodec/loongarch/vp9_mc_lsx.c create mode 100644 libavcodec/loongarch/vp9dsp_init_loongarch.c create mode 100644 libavcodec/loongarch/vp9dsp_loongarch.h create mode 100644 libavcodec/mips/h264_deblock_msa.c create mode 100644 libavutil/loongarch/Makefile create mode 100644 libavutil/loongarch/cpu.c create mode 100644 libavutil/loongarch/cpu.h create mode 100644 libavutil/loongarch/generic_macros_lasx.h create mode 100644 libavutil/loongarch/generic_macros_lsx.h create mode 100644 libavutil/loongarch/generic_macros_lsx.h.bak create mode 100644 libswscale/loongarch/Makefile create mode 100644 libswscale/loongarch/input_lasx.c create mode 100644 libswscale/loongarch/output_lasx.c create mode 100644 libswscale/loongarch/rgb2rgb_lasx.c create mode 100644 libswscale/loongarch/swscale_init_loongarch.c create mode 100644 libswscale/loongarch/swscale_lasx.c create mode 100644 libswscale/loongarch/swscale_loongarch.h create mode 100644 libswscale/loongarch/yuv2rgb_lasx.c create mode 100644 libswscale/mips/Makefile create mode 100644 libswscale/mips/rgb2rgb_init_mips.c create mode 100644 libswscale/mips/rgb2rgb_mips.h create mode 100644 libswscale/mips/rgb2rgb_msa.c create mode 100644 libswscale/mips/swscale_init_mips.c create mode 100644 libswscale/mips/swscale_mips.h create mode 100644 libswscale/mips/swscale_msa.c diff --git a/Makefile b/Makefile index 4bf1dfedcf..ec260157f9 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ SUBDIR_VARS := CLEANFILES FFLIBS HOSTPROGS TESTPROGS TOOLS \ ARMV5TE-OBJS ARMV6-OBJS ARMV8-OBJS VFP-OBJS NEON-OBJS \ ALTIVEC-OBJS VSX-OBJS MMX-OBJS X86ASM-OBJS \ MIPSFPU-OBJS MIPSDSPR2-OBJS MIPSDSP-OBJS MSA-OBJS \ - MMI-OBJS OBJS SLIBOBJS HOSTOBJS TESTOBJS + MMI-OBJS LSX-OBJS LASX-OBJS OBJS SLIBOBJS HOSTOBJS TESTOBJS define RESET $(1) := diff --git a/configure b/configure index db57f4d544..1ac28abde9 100755 --- a/configure +++ b/configure @@ -439,10 +439,11 @@ Optimization options (experts only): --disable-mipsdsp disable MIPS DSP ASE R1 optimizations --disable-mipsdspr2 disable MIPS DSP ASE R2 optimizations --disable-msa disable MSA optimizations - --disable-msa2 disable MSA2 optimizations --disable-mipsfpu disable floating point MIPS optimizations --disable-mmi disable Loongson SIMD optimizations --disable-fast-unaligned consider unaligned accesses slow + --disable-lsx disable LSX optimizations + --disable-lasx disable LASX optimizations Developer options (useful when working on FFmpeg itself): --disable-debug disable debugging symbols @@ -1931,6 +1932,9 @@ ARCH_LIST=" x86 x86_32 x86_64 + loongarch + loongarch32 + loongarch64 " ARCH_EXT_LIST_ARM=" @@ -1954,13 +1958,14 @@ ARCH_EXT_LIST_MIPS=" mipsdsp mipsdspr2 msa - msa2 " ARCH_EXT_LIST_LOONGSON=" loongson2 loongson3 mmi + lsx + lasx " ARCH_EXT_LIST_X86_SIMD=" @@ -2470,6 +2475,10 @@ power8_deps="vsx" loongson2_deps="mips" loongson3_deps="mips" +mmi_deps_any="loongson2 loongson3" +lsx_deps="loongarch" +lasx_deps="lsx" + mips32r2_deps="mips" mips32r5_deps="mips" mips32r6_deps="mips" @@ -2478,9 +2487,7 @@ mips64r6_deps="mips" mipsfpu_deps="mips" mipsdsp_deps="mips" mipsdspr2_deps="mips" -mmi_deps_any="loongson2 loongson3" msa_deps="mipsfpu" -msa2_deps="msa" cpunop_deps="i686" x86_64_select="i686" @@ -2517,8 +2524,8 @@ for ext in $(filter_out mmx $ARCH_EXT_LIST_X86_SIMD); do done aligned_stack_if_any="aarch64 ppc x86" -fast_64bit_if_any="aarch64 alpha ia64 mips64 parisc64 ppc64 sparc64 x86_64" -fast_clz_if_any="aarch64 alpha avr32 mips ppc x86" +fast_64bit_if_any="aarch64 alpha ia64 mips64 parisc64 ppc64 sparc64 x86_64 loongarch64" +fast_clz_if_any="aarch64 alpha avr32 mips ppc x86 loongarch" fast_unaligned_if_any="aarch64 ppc x86" simd_align_16_if_any="altivec neon sse" simd_align_32_if_any="avx" @@ -4670,6 +4677,9 @@ case "$arch" in arm*|iPad*|iPhone*) arch="arm" ;; + loongarch*) + arch="loongarch" + ;; mips*|IP*) case "$arch" in *el) @@ -5016,6 +5026,20 @@ elif enabled x86; then ;; esac +elif enabled loongarch; then + # TODO: pending to test. + enable local_aligned + enable simd_align_16 + enable fast_64bit + enable fast_clz + enable fast_cmov + enable fast_unaligned + disable aligned_stack + case $cpu in + gs464v) + cpuflags="-march=$cpu" + ;; + esac fi if [ "$cpu" != generic ]; then @@ -5098,6 +5122,11 @@ case "$arch" in objformat=elf64 fi ;; + loongarch) + check_64bit loongarch32 loongarch64 + enabled loongarch64 && disable loongarch32 + enabled shared && enable_weak pic + ;; esac # OS specific @@ -5669,9 +5698,8 @@ elif enabled mips; then enabled mipsdsp && check_inline_asm_flags mipsdsp '"addu.qb $t0, $t1, $t2"' '-mdsp' enabled mipsdspr2 && check_inline_asm_flags mipsdspr2 '"absq_s.qb $t0, $t1"' '-mdspr2' - # MSA and MSA2 can be detected at runtime so we supply extra flags here + # MSA can be detected at runtime so we supply extra flags here enabled mipsfpu && enabled msa && check_inline_asm msa '"addvi.b $w0, $w1, 1"' '-mmsa' && append MSAFLAGS '-mmsa' - enabled msa && enabled msa2 && check_inline_asm msa2 '"nxbits.any.b $w0, $w0"' '-mmsa2' && append MSAFLAGS '-mmsa2' # loongson2 have no switch cflag so we can only probe toolchain ability enabled loongson2 && check_inline_asm loongson2 '"dmult.g $8, $9, $10"' && disable loongson3 @@ -5682,6 +5710,10 @@ elif enabled mips; then # MMI can be detected at runtime too enabled mmi && check_inline_asm mmi '"punpcklhw $f0, $f0, $f0"' '-mloongson-mmi' && append MMIFLAGS '-mloongson-mmi' + # TODO: The following code is intended to fix h264 decoding module compilation error, + # but it is not resonable. Maybe we should fix it in the feature. + enabled mmi && h264_decoder_select="$h264_decoder_select hpeldsp" + if enabled bigendian && enabled msa; then disable msa fi @@ -5799,6 +5831,9 @@ EOF ;; esac +elif enabled loongarch; then + enabled lsx && check_inline_asm lsx '"vadd.b $vr0, $vr1, $vr2"' '-mlsx' && append LSXFLAGS '-mlsx' + enabled lasx && check_inline_asm lasx '"xvadd.b $xr0, $xr1, $xr2"' '-mlasx' && append LASXFLAGS '-mlasx' fi check_cc intrinsics_neon arm_neon.h "int16x8_t test = vdupq_n_s16(0)" @@ -7027,7 +7062,6 @@ if enabled mips; then echo "MIPS DSP R1 enabled ${mipsdsp-no}" echo "MIPS DSP R2 enabled ${mipsdspr2-no}" echo "MIPS MSA enabled ${msa-no}" - echo "MIPS MSA2 enabled ${msa2-no}" echo "LOONGSON MMI enabled ${mmi-no}" fi if enabled ppc; then @@ -7037,6 +7071,10 @@ if enabled ppc; then echo "PPC 4xx optimizations ${ppc4xx-no}" echo "dcbzl available ${dcbzl-no}" fi +if enabled loongarch; then + echo "LSX enabled ${lsx-no}" + echo "LASX enabled ${lasx-no}" +fi echo "debug symbols ${debug-no}" echo "strip symbols ${stripping-no}" echo "optimize for size ${small-no}" @@ -7189,6 +7227,8 @@ ASMSTRIPFLAGS=$ASMSTRIPFLAGS X86ASMFLAGS=$X86ASMFLAGS MSAFLAGS=$MSAFLAGS MMIFLAGS=$MMIFLAGS +LSXFLAGS=$LSXFLAGS +LASXFLAGS=$LASXFLAGS BUILDSUF=$build_suffix PROGSSUF=$progs_suffix FULLNAME=$FULLNAME diff --git a/ffbuild/arch.mak b/ffbuild/arch.mak index e09006efca..997e31e85e 100644 --- a/ffbuild/arch.mak +++ b/ffbuild/arch.mak @@ -8,7 +8,9 @@ OBJS-$(HAVE_MIPSFPU) += $(MIPSFPU-OBJS) $(MIPSFPU-OBJS-yes) OBJS-$(HAVE_MIPSDSP) += $(MIPSDSP-OBJS) $(MIPSDSP-OBJS-yes) OBJS-$(HAVE_MIPSDSPR2) += $(MIPSDSPR2-OBJS) $(MIPSDSPR2-OBJS-yes) OBJS-$(HAVE_MSA) += $(MSA-OBJS) $(MSA-OBJS-yes) -OBJS-$(HAVE_MMI) += $(MMI-OBJS) $(MMI-OBJS-yes) +OBJS-$(HAVE_MMI) += $(MMI-OBJS) $(MMI-OBJS-yes) +OBJS-$(HAVE_LSX) += $(LSX-OBJS) $(LSX-OBJS-yes) +OBJS-$(HAVE_LASX) += $(LASX-OBJS) $(LASX-OBJS-yes) OBJS-$(HAVE_ALTIVEC) += $(ALTIVEC-OBJS) $(ALTIVEC-OBJS-yes) OBJS-$(HAVE_VSX) += $(VSX-OBJS) $(VSX-OBJS-yes) diff --git a/ffbuild/common.mak b/ffbuild/common.mak index a00d40d735..d321506f96 100644 --- a/ffbuild/common.mak +++ b/ffbuild/common.mak @@ -57,6 +57,8 @@ COMPILE_HOSTC = $(call COMPILE,HOSTCC) COMPILE_NVCC = $(call COMPILE,NVCC) COMPILE_MMI = $(call COMPILE,CC,MMIFLAGS) COMPILE_MSA = $(call COMPILE,CC,MSAFLAGS) +COMPILE_LSX = $(call COMPILE,CC,LSXFLAGS) +COMPILE_LASX = $(call COMPILE,CC,LASXFLAGS) %_mmi.o: %_mmi.c $(COMPILE_MMI) @@ -64,6 +66,12 @@ COMPILE_MSA = $(call COMPILE,CC,MSAFLAGS) %_msa.o: %_msa.c $(COMPILE_MSA) +%_lsx.o: %_lsx.c + $(COMPILE_LSX) + +%_lasx.o: %_lasx.c + $(COMPILE_LASX) + %.o: %.c $(COMPILE_C) diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index 24307c3cae..9c20351ff8 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -1709,7 +1709,7 @@ static void print_report(int is_last_report, int64_t timer_start, int64_t cur_ti frame_number = ost->frame_number; fps = t > 1 ? frame_number / t : 0; av_bprintf(&buf, "frame=%5d fps=%3.*f q=%3.1f ", - frame_number, fps < 9.95, fps, q); + frame_number, fps < 100, fps, q); av_bprintf(&buf_script, "frame=%d\n", frame_number); av_bprintf(&buf_script, "fps=%.2f\n", fps); av_bprintf(&buf_script, "stream_%d_%d_q=%.1f\n", diff --git a/libavcodec/cabac_functions.h b/libavcodec/cabac_functions.h index bb2b4210b7..fad9d7c10f 100644 --- a/libavcodec/cabac_functions.h +++ b/libavcodec/cabac_functions.h @@ -48,6 +48,9 @@ #if ARCH_MIPS # include "mips/cabac.h" #endif +#if ARCH_LOONGARCH64 +# include "loongarch/cabac.h" +#endif static const uint8_t * const ff_h264_norm_shift = ff_h264_cabac_tables + H264_NORM_SHIFT_OFFSET; static const uint8_t * const ff_h264_lps_range = ff_h264_cabac_tables + H264_LPS_RANGE_OFFSET; diff --git a/libavcodec/h264chroma.c b/libavcodec/h264chroma.c index c2f1f30f5a..279a6ada7f 100644 --- a/libavcodec/h264chroma.c +++ b/libavcodec/h264chroma.c @@ -56,4 +56,6 @@ av_cold void ff_h264chroma_init(H264ChromaContext *c, int bit_depth) ff_h264chroma_init_x86(c, bit_depth); if (ARCH_MIPS) ff_h264chroma_init_mips(c, bit_depth); + if (ARCH_LOONGARCH) + ff_h264chroma_init_loongarch(c, bit_depth); } diff --git a/libavcodec/h264chroma.h b/libavcodec/h264chroma.h index 5c89fd12df..3259b4935f 100644 --- a/libavcodec/h264chroma.h +++ b/libavcodec/h264chroma.h @@ -36,5 +36,6 @@ void ff_h264chroma_init_arm(H264ChromaContext *c, int bit_depth); void ff_h264chroma_init_ppc(H264ChromaContext *c, int bit_depth); void ff_h264chroma_init_x86(H264ChromaContext *c, int bit_depth); void ff_h264chroma_init_mips(H264ChromaContext *c, int bit_depth); +void ff_h264chroma_init_loongarch(H264ChromaContext *c, int bit_depth); #endif /* AVCODEC_H264CHROMA_H */ diff --git a/libavcodec/h264dsp.c b/libavcodec/h264dsp.c index d26f552369..436ce68f0b 100644 --- a/libavcodec/h264dsp.c +++ b/libavcodec/h264dsp.c @@ -158,4 +158,5 @@ av_cold void ff_h264dsp_init(H264DSPContext *c, const int bit_depth, if (ARCH_PPC) ff_h264dsp_init_ppc(c, bit_depth, chroma_format_idc); if (ARCH_X86) ff_h264dsp_init_x86(c, bit_depth, chroma_format_idc); if (ARCH_MIPS) ff_h264dsp_init_mips(c, bit_depth, chroma_format_idc); + if (ARCH_LOONGARCH) ff_h264dsp_init_loongarch(c, bit_depth, chroma_format_idc); } diff --git a/libavcodec/h264dsp.h b/libavcodec/h264dsp.h index bcd76abcc1..4a394299d8 100644 --- a/libavcodec/h264dsp.h +++ b/libavcodec/h264dsp.h @@ -129,5 +129,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, const int chroma_format_idc); void ff_h264dsp_init_mips(H264DSPContext *c, const int bit_depth, const int chroma_format_idc); +void ff_h264dsp_init_loongarch(H264DSPContext *c, const int bit_depth, + const int chroma_format_idc); #endif /* AVCODEC_H264DSP_H */ diff --git a/libavcodec/h264qpel.c b/libavcodec/h264qpel.c index 50e82e23b0..9f8cfbb474 100644 --- a/libavcodec/h264qpel.c +++ b/libavcodec/h264qpel.c @@ -106,4 +106,6 @@ av_cold void ff_h264qpel_init(H264QpelContext *c, int bit_depth) ff_h264qpel_init_x86(c, bit_depth); if (ARCH_MIPS) ff_h264qpel_init_mips(c, bit_depth); + if (ARCH_LOONGARCH) + ff_h264qpel_init_loongarch(c, bit_depth); } diff --git a/libavcodec/h264qpel.h b/libavcodec/h264qpel.h index 7c57ad001c..0259e8de23 100644 --- a/libavcodec/h264qpel.h +++ b/libavcodec/h264qpel.h @@ -36,5 +36,6 @@ void ff_h264qpel_init_arm(H264QpelContext *c, int bit_depth); void ff_h264qpel_init_ppc(H264QpelContext *c, int bit_depth); void ff_h264qpel_init_x86(H264QpelContext *c, int bit_depth); void ff_h264qpel_init_mips(H264QpelContext *c, int bit_depth); +void ff_h264qpel_init_loongarch(H264QpelContext *c, int bit_depth); #endif /* AVCODEC_H264QPEL_H */ diff --git a/libavcodec/hevcdsp.c b/libavcodec/hevcdsp.c index 957e40d5ff..b26f7f5f38 100644 --- a/libavcodec/hevcdsp.c +++ b/libavcodec/hevcdsp.c @@ -265,4 +265,6 @@ int i = 0; ff_hevc_dsp_init_x86(hevcdsp, bit_depth); if (ARCH_MIPS) ff_hevc_dsp_init_mips(hevcdsp, bit_depth); + if (ARCH_LOONGARCH) + ff_hevc_dsp_init_loongarch(hevcdsp, bit_depth); } diff --git a/libavcodec/hevcdsp.h b/libavcodec/hevcdsp.h index 0ae67cba85..27432ebc8c 100644 --- a/libavcodec/hevcdsp.h +++ b/libavcodec/hevcdsp.h @@ -131,5 +131,6 @@ void ff_hevc_dsp_init_arm(HEVCDSPContext *c, const int bit_depth); void ff_hevc_dsp_init_ppc(HEVCDSPContext *c, const int bit_depth); void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth); void ff_hevc_dsp_init_mips(HEVCDSPContext *c, const int bit_depth); +void ff_hevc_dsp_init_loongarch(HEVCDSPContext *c, const int bit_depth); #endif /* AVCODEC_HEVCDSP_H */ diff --git a/libavcodec/hpeldsp.c b/libavcodec/hpeldsp.c index 8e2fd8fcf5..681f839ff8 100644 --- a/libavcodec/hpeldsp.c +++ b/libavcodec/hpeldsp.c @@ -367,4 +367,6 @@ av_cold void ff_hpeldsp_init(HpelDSPContext *c, int flags) ff_hpeldsp_init_x86(c, flags); if (ARCH_MIPS) ff_hpeldsp_init_mips(c, flags); + if (ARCH_LOONGARCH) + ff_hpeldsp_init_loongarch(c, flags); } diff --git a/libavcodec/hpeldsp.h b/libavcodec/hpeldsp.h index 768139bfc9..45e81b10a5 100644 --- a/libavcodec/hpeldsp.h +++ b/libavcodec/hpeldsp.h @@ -102,5 +102,6 @@ void ff_hpeldsp_init_arm(HpelDSPContext *c, int flags); void ff_hpeldsp_init_ppc(HpelDSPContext *c, int flags); void ff_hpeldsp_init_x86(HpelDSPContext *c, int flags); void ff_hpeldsp_init_mips(HpelDSPContext *c, int flags); +void ff_hpeldsp_init_loongarch(HpelDSPContext *c, int flags); #endif /* AVCODEC_HPELDSP_H */ diff --git a/libavcodec/idctdsp.c b/libavcodec/idctdsp.c index 846ed0b0f8..71bd03c606 100644 --- a/libavcodec/idctdsp.c +++ b/libavcodec/idctdsp.c @@ -315,6 +315,8 @@ av_cold void ff_idctdsp_init(IDCTDSPContext *c, AVCodecContext *avctx) ff_idctdsp_init_x86(c, avctx, high_bit_depth); if (ARCH_MIPS) ff_idctdsp_init_mips(c, avctx, high_bit_depth); + if (ARCH_LOONGARCH) + ff_idctdsp_init_loongarch(c, avctx, high_bit_depth); ff_init_scantable_permutation(c->idct_permutation, c->perm_type); diff --git a/libavcodec/idctdsp.h b/libavcodec/idctdsp.h index ca21a31a02..014488aec3 100644 --- a/libavcodec/idctdsp.h +++ b/libavcodec/idctdsp.h @@ -118,5 +118,7 @@ void ff_idctdsp_init_x86(IDCTDSPContext *c, AVCodecContext *avctx, unsigned high_bit_depth); void ff_idctdsp_init_mips(IDCTDSPContext *c, AVCodecContext *avctx, unsigned high_bit_depth); +void ff_idctdsp_init_loongarch(IDCTDSPContext *c, AVCodecContext *avctx, + unsigned high_bit_depth); #endif /* AVCODEC_IDCTDSP_H */ diff --git a/libavcodec/loongarch/Makefile b/libavcodec/loongarch/Makefile new file mode 100644 index 0000000000..2a8d7e430f --- /dev/null +++ b/libavcodec/loongarch/Makefile @@ -0,0 +1,30 @@ +OBJS-$(CONFIG_H264QPEL) += loongarch/h264qpel_init_loongarch.o +OBJS-$(CONFIG_H264CHROMA) += loongarch/h264chroma_init_loongarch.o +OBJS-$(CONFIG_H264DSP) += loongarch/h264dsp_init_loongarch.o +OBJS-$(CONFIG_IDCTDSP) += loongarch/idctdsp_init_loongarch.o +OBJS-$(CONFIG_VC1DSP) += loongarch/vc1dsp_init_loongarch.o +OBJS-$(CONFIG_HPELDSP) += loongarch/hpeldsp_init_loongarch.o +OBJS-$(CONFIG_HEVC_DECODER) += loongarch/hevcdsp_init_loongarch.o +OBJS-$(CONFIG_VP8_DECODER) += loongarch/vp8dsp_init_loongarch.o +OBJS-$(CONFIG_VP9_DECODER) += loongarch/vp9dsp_init_loongarch.o +OBJS-$(CONFIG_VIDEODSP) += loongarch/videodsp_init.o +LASX-OBJS-$(CONFIG_H264DSP) += loongarch/h264dsp_lasx.o \ + loongarch/h264idct_lasx.o \ + loongarch/h264_deblock_lasx.o +LASX-OBJS-$(CONFIG_H264CHROMA) += loongarch/h264chroma_lasx.o +LASX-OBJS-$(CONFIG_H264CHROMA) += loongarch/h264qpel_lasx.o +LASX-OBJS-$(CONFIG_IDCTDSP) += loongarch/simple_idct_lasx.o \ + loongarch/idctdsp_lasx.o +LASX-OBJS-$(CONFIG_VC1_DECODER) += loongarch/vc1dsp_lasx.o +LASX-OBJS-$(CONFIG_HPELDSP) += loongarch/hpeldsp_lasx.o +LSX-OBJS-$(CONFIG_HEVC_DECODER) += loongarch/hevcdsp_lsx.o +LSX-OBJS-$(CONFIG_HEVC_DECODER) += loongarch/hevc_idct_lsx.o +LSX-OBJS-$(CONFIG_HEVC_DECODER) += loongarch/hevc_mc_bi_lsx.o +LSX-OBJS-$(CONFIG_HEVC_DECODER) += loongarch/hevc_lpf_sao_lsx.o +LSX-OBJS-$(CONFIG_HEVC_DECODER) += loongarch/hevc_mc_uni_lsx.o +LSX-OBJS-$(CONFIG_HEVC_DECODER) += loongarch/hevc_mc_uniw_lsx.o +LSX-OBJS-$(CONFIG_VP9_DECODER) += loongarch/vp9_mc_lsx.o \ + loongarch/vp9_intra_lsx.o \ + loongarch/vp9_lpf_lsx.o \ + loongarch/vp9_idct_lsx.o +LSX-OBJS-$(CONFIG_VP8_DECODER) += loongarch/vp8_lpf_lsx.o diff --git a/libavcodec/loongarch/cabac.h b/libavcodec/loongarch/cabac.h new file mode 100644 index 0000000000..c0e17e3462 --- /dev/null +++ b/libavcodec/loongarch/cabac.h @@ -0,0 +1,198 @@ +/* + * Loongson optimized cabac + * + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * Gu Xiwei(guxiwei-hf@loongson.cn) + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_CABAC_H +#define AVCODEC_LOONGARCH_CABAC_H + +#include "libavcodec/cabac.h" +#include "config.h" + +#define get_cabac_inline get_cabac_inline_loongarch +static av_always_inline int get_cabac_inline_loongarch(CABACContext *c, + uint8_t * const state){ + int64_t tmp0, tmp1, tmp2, bit; + + __asm__ volatile ( + "ld.bu %[bit], %[state], 0x0 \n\t" + "andi %[tmp0], %[c_range], 0xC0 \n\t" + "slli.d %[tmp0], %[tmp0], 0x01 \n\t" + "add.d %[tmp0], %[tmp0], %[tables] \n\t" + "add.d %[tmp0], %[tmp0], %[bit] \n\t" + /* tmp1: RangeLPS */ + "ld.bu %[tmp1], %[tmp0], %[lps_off] \n\t" + + "sub.d %[c_range], %[c_range], %[tmp1] \n\t" + "slli.d %[tmp0], %[c_range], 0x11 \n\t" + "bge %[tmp0], %[c_low], 1f \n\t" + "move %[c_range], %[tmp1] \n\t" + "nor %[bit], %[bit], %[bit] \n\t" + "sub.d %[c_low], %[c_low], %[tmp0] \n\t" + + "1: \n\t" + /* tmp1: *state */ + "add.d %[tmp0], %[tables], %[bit] \n\t" + "ld.bu %[tmp1], %[tmp0], %[mlps_off] \n\t" + /* tmp2: lps_mask */ + "add.d %[tmp0], %[tables], %[c_range] \n\t" + "ld.bu %[tmp2], %[tmp0], %[norm_off] \n\t" + + "andi %[bit], %[bit], 0x01 \n\t" + "st.b %[tmp1], %[state], 0x0 \n\t" + "sll.d %[c_range], %[c_range], %[tmp2] \n\t" + "sll.d %[c_low], %[c_low], %[tmp2] \n\t" + + "and %[tmp1], %[c_low], %[cabac_mask] \n\t" + "bnez %[tmp1], 1f \n\t" + "ld.hu %[tmp1], %[c_bytestream], 0x0 \n\t" + "addi.d %[tmp0], %[c_low], -0X01 \n\t" + "xor %[tmp0], %[c_low], %[tmp0] \n\t" + "srai.d %[tmp0], %[tmp0], 0x0f \n\t" + "add.d %[tmp0], %[tmp0], %[tables] \n\t" + /* tmp2: ff_h264_norm_shift[x >> (CABAC_BITS - 1)] */ + "ld.bu %[tmp2], %[tmp0], %[norm_off] \n\t" + + "revb.2h %[tmp0], %[tmp1] \n\t" + "slli.d %[tmp0], %[tmp0], 0x01 \n\t" + "sub.d %[tmp0], %[tmp0], %[cabac_mask] \n\t" + + "li.d %[tmp1], 0x07 \n\t" + "sub.d %[tmp1], %[tmp1], %[tmp2] \n\t" + "sll.d %[tmp0], %[tmp0], %[tmp1] \n\t" + "add.d %[c_low], %[c_low], %[tmp0] \n\t" + +#if UNCHECKED_BITSTREAM_READER + "addi.d %[c_bytestream], %[c_bytestream], 0x02 \n\t" +#else + "slt %[tmp0], %[c_bytestream], %[c_bytestream_end] \n\t" + "add.d %[c_bytestream], %[c_bytestream], %[tmp0] \n\t" + "add.d %[c_bytestream], %[c_bytestream], %[tmp0] \n\t" +#endif + "1: \n\t" + : [bit]"=&r"(bit), [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [tmp2]"=&r"(tmp2), + [c_range]"+&r"(c->range), [c_low]"+&r"(c->low), + [c_bytestream]"+&r"(c->bytestream) + : [state]"r"(state), [tables]"r"(ff_h264_cabac_tables), +#if !UNCHECKED_BITSTREAM_READER + [c_bytestream_end]"r"(c->bytestream_end), +#endif + [lps_off]"i"(H264_LPS_RANGE_OFFSET), + [mlps_off]"i"(H264_MLPS_STATE_OFFSET + 128), + [norm_off]"i"(H264_NORM_SHIFT_OFFSET), + [cabac_mask]"r"(CABAC_MASK) + : "memory" + ); + + return bit; +} + +#define get_cabac_bypass get_cabac_bypass_loongarch +static av_always_inline int get_cabac_bypass_loongarch(CABACContext *c) +{ + int64_t tmp0, tmp1, tmp2; + int res = 0; + __asm__ volatile( + "slli.d %[c_low], %[c_low], 0x01 \n\t" + "and %[tmp0], %[c_low], %[cabac_mask] \n\t" + "bnez %[tmp0], 1f \n\t" + "ld.hu %[tmp1], %[c_bytestream], 0x0 \n\t" +#if UNCHECKED_BITSTREAM_READER + "addi.d %[c_bytestream], %[c_bytestream], 0x02 \n\t" +#else + "slt %[tmp0], %[c_bytestream], %[c_bytestream_end] \n\t" + "add.d %[c_bytestream], %[c_bytestream], %[tmp0] \n\t" + "add.d %[c_bytestream], %[c_bytestream], %[tmp0] \n\t" +#endif + "revb.2h %[tmp1], %[tmp1] \n\t" + "slli.d %[tmp1], %[tmp1], 0x01 \n\t" + "sub.d %[tmp1], %[tmp1], %[cabac_mask] \n\t" + "add.d %[c_low], %[c_low], %[tmp1] \n\t" + "1: \n\t" + "slli.d %[tmp1], %[c_range], 0x11 \n\t" + "slt %[tmp0], %[c_low], %[tmp1] \n\t" + "sub.d %[tmp1], %[c_low], %[tmp1] \n\t" + "masknez %[tmp2], %[one], %[tmp0] \n\t" + "maskeqz %[res], %[res], %[tmp0] \n\t" + "or %[res], %[res], %[tmp2] \n\t" + "masknez %[tmp2], %[tmp1], %[tmp0] \n\t" + "maskeqz %[c_low], %[c_low], %[tmp0] \n\t" + "or %[c_low], %[c_low], %[tmp2] \n\t" + : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [tmp2]"=&r"(tmp2), + [c_range]"+&r"(c->range), [c_low]"+&r"(c->low), + [c_bytestream]"+&r"(c->bytestream), [res]"+&r"(res) + : [cabac_mask]"r"(CABAC_MASK), +#if !UNCHECKED_BITSTREAM_READER + [c_bytestream_end]"r"(c->bytestream_end), +#endif + [one]"r"(0x01) + : "memory" + ); + return res; +} + +#define get_cabac_bypass_sign get_cabac_bypass_sign_loongarch +static av_always_inline int get_cabac_bypass_sign_loongarch(CABACContext *c, int val) +{ + int64_t tmp0, tmp1; + int res = val; + __asm__ volatile( + "slli.d %[c_low], %[c_low], 0x01 \n\t" + "and %[tmp0], %[c_low], %[cabac_mask] \n\t" + "bnez %[tmp0], 1f \n\t" + "ld.hu %[tmp1], %[c_bytestream], 0x0 \n\t" +#if UNCHECKED_BITSTREAM_READER + "addi.d %[c_bytestream], %[c_bytestream], 0x02 \n\t" +#else + "slt %[tmp0], %[c_bytestream], %[c_bytestream_end] \n\t" + "add.d %[c_bytestream], %[c_bytestream], %[tmp0] \n\t" + "add.d %[c_bytestream], %[c_bytestream], %[tmp0] \n\t" +#endif + "revb.2h %[tmp1], %[tmp1] \n\t" + "slli.d %[tmp1], %[tmp1], 0x01 \n\t" + "sub.d %[tmp1], %[tmp1], %[cabac_mask] \n\t" + "add.d %[c_low], %[c_low], %[tmp1] \n\t" + "1: \n\t" + "slli.d %[tmp1], %[c_range], 0x11 \n\t" + "slt %[tmp0], %[c_low], %[tmp1] \n\t" + "sub.d %[tmp1], %[c_low], %[tmp1] \n\t" + "masknez %[tmp1], %[tmp1], %[tmp0] \n\t" + "maskeqz %[c_low], %[c_low], %[tmp0] \n\t" + "or %[c_low], %[c_low], %[tmp1] \n\t" + "sub.d %[tmp1], %[zero], %[res] \n\t" + "maskeqz %[tmp1], %[tmp1], %[tmp0] \n\t" + "masknez %[res], %[res], %[tmp0] \n\t" + "or %[res], %[res], %[tmp1] \n\t" + : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [res]"+&r"(res), + [c_range]"+&r"(c->range), [c_low]"+&r"(c->low), + [c_bytestream]"+&r"(c->bytestream) + : [cabac_mask]"r"(CABAC_MASK), +#if !UNCHECKED_BITSTREAM_READER + [c_bytestream_end]"r"(c->bytestream_end), +#endif + [zero]"r"(0x0) + : "memory" + ); + + return res; +} +#endif /* AVCODEC_LOONGARCH_CABAC_H */ diff --git a/libavcodec/loongarch/h264_deblock_lasx.c b/libavcodec/loongarch/h264_deblock_lasx.c new file mode 100644 index 0000000000..73706739a0 --- /dev/null +++ b/libavcodec/loongarch/h264_deblock_lasx.c @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Xiwei Gu + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavcodec/bit_depth_template.c" +#include "h264dsp_lasx.h" +#include "libavutil/loongarch/generic_macros_lasx.h" + +#define H264_LOOP_FILTER_STRENGTH_ITERATION_LASX(edges, step, mask_mv, dir, \ + d_idx, mask_dir) \ +do { \ + int b_idx = 0; \ + int step_x4 = step << 2; \ + int d_idx_12 = d_idx + 12; \ + int d_idx_52 = d_idx + 52; \ + int d_idx_x4 = d_idx << 2; \ + int d_idx_x4_48 = d_idx_x4 + 48; \ + int dir_x32 = dir * 32; \ + uint8_t *ref_t = (uint8_t*)ref; \ + uint8_t *mv_t = (uint8_t*)mv; \ + uint8_t *nnz_t = (uint8_t*)nnz; \ + uint8_t *bS_t = (uint8_t*)bS; \ + mask_mv <<= 3; \ + for (; b_idx < edges; b_idx += step) { \ + out &= mask_dir; \ + if (!(mask_mv & b_idx)) { \ + if (bidir) { \ + ref2 = LASX_LD(ref_t + d_idx_12); \ + ref3 = LASX_LD(ref_t + d_idx_52); \ + ref0 = LASX_LD(ref_t + 12); \ + ref1 = LASX_LD(ref_t + 52); \ + ref2 = __lasx_xvilvl_w(ref3, ref2); \ + ref0 = __lasx_xvilvl_w(ref0, ref0); \ + ref1 = __lasx_xvilvl_w(ref1, ref1); \ + ref3 = __lasx_xvshuf4i_w(ref2, 0xB1); \ + ref0 = __lasx_xvsub_b(ref0, ref2); \ + ref1 = __lasx_xvsub_b(ref1, ref3); \ + ref0 = __lasx_xvor_v(ref0, ref1); \ +\ + tmp2 = LASX_LD(mv_t + d_idx_x4_48); \ + tmp3 = LASX_LD(mv_t + 48); \ + tmp4 = LASX_LD(mv_t + 208); \ + tmp5 = LASX_LD(mv_t + 208 + d_idx_x4); \ + LASX_PCKEV_Q_2(tmp2, tmp2, tmp5, tmp5, tmp2, tmp5); \ + LASX_PCKEV_Q(tmp4, tmp3, tmp3); \ + tmp2 = __lasx_xvsub_h(tmp2, tmp3); \ + tmp5 = __lasx_xvsub_h(tmp5, tmp3); \ + LASX_SAT_H_2(tmp2, tmp5, tmp2, tmp5, 7); \ + LASX_PCKEV_B(tmp5, tmp2, tmp0); \ + tmp0 = __lasx_xvadd_b(tmp0, cnst_1); \ + tmp0 = __lasx_xvssub_bu(tmp0, cnst_0); \ + LASX_SAT_H(tmp0, tmp0, 7); \ + LASX_PCKEV_B(tmp0, tmp0, tmp0); \ + LASX_PCKOD_D_128SV(tmp0, tmp0, tmp1); \ + out = __lasx_xvor_v(ref0, tmp0); \ + tmp1 = __lasx_xvshuf4i_w(tmp1, 0xB1); \ + out = __lasx_xvor_v(out, tmp1); \ + tmp0 = __lasx_xvshuf4i_w(out, 0xB1); \ + out = __lasx_xvmin_bu(out, tmp0); \ + } else { \ + ref0 = LASX_LD(ref_t + d_idx_12); \ + ref3 = LASX_LD(ref_t + 12); \ + tmp2 = LASX_LD(mv_t + d_idx_x4_48); \ + tmp3 = LASX_LD(mv_t + 48); \ + tmp4 = __lasx_xvsub_h(tmp3, tmp2); \ + LASX_SAT_H(tmp4, tmp1, 7); \ + tmp1 = __lasx_xvpickev_b(tmp1, tmp1); \ + tmp1 = __lasx_xvadd_b(tmp1, cnst_1); \ + out = __lasx_xvssub_bu(tmp1, cnst_0); \ + LASX_SAT_H(out, out, 7); \ + out = __lasx_xvpickev_b(out, out); \ + ref0 = __lasx_xvsub_b(ref3, ref0); \ + out = __lasx_xvor_v(out, ref0); \ + } \ + } \ + tmp0 = LASX_LD(nnz_t + 12); \ + tmp1 = LASX_LD(nnz_t + d_idx_12); \ + tmp0 = __lasx_xvor_v(tmp0, tmp1); \ + tmp0 = __lasx_xvmin_bu(tmp0, cnst_2); \ + out = __lasx_xvmin_bu(out, cnst_2); \ + tmp0 = __lasx_xvslli_h(tmp0, 1); \ + tmp0 = __lasx_xvmax_bu(out, tmp0); \ + LASX_UNPCK_L_HU_BU(tmp0, tmp0); \ + LASX_ST_D(tmp0, 0, bS_t + dir_x32); \ + ref_t += step; \ + mv_t += step_x4; \ + nnz_t += step; \ + bS_t += step; \ + } \ +} while(0) + +void ff_h264_loop_filter_strength_lasx(int16_t bS[2][4][4], uint8_t nnz[40], + int8_t ref[2][40], int16_t mv[2][40][2], + int bidir, int edges, int step, + int mask_mv0, int mask_mv1, int field) +{ + __m256i out; + __m256i ref0, ref1, ref2, ref3; + __m256i tmp0, tmp1; + __m256i tmp2, tmp3, tmp4, tmp5; + __m256i cnst_0, cnst_1, cnst_2; + __m256i zero = __lasx_xvldi(0); + __m256i one = __lasx_xvnor_v(zero, zero); + int64_t cnst3 = 0x0206020602060206, cnst4 = 0x0103010301030103; + if (field) { + cnst_0 = __lasx_xvreplgr2vr_d(cnst3); + cnst_1 = __lasx_xvreplgr2vr_d(cnst4); + cnst_2 = __lasx_xvldi(0x01); + } else { + cnst_0 = __lasx_xvldi(0x06); + cnst_1 = __lasx_xvldi(0x03); + cnst_2 = __lasx_xvldi(0x01); + } + step <<= 3; + edges <<= 3; + + H264_LOOP_FILTER_STRENGTH_ITERATION_LASX(edges, step, mask_mv1, 1, -8, zero); + H264_LOOP_FILTER_STRENGTH_ITERATION_LASX(32, 8, mask_mv0, 0, -1, one); + + LASX_LD_2((int8_t*)bS, 16, tmp0, tmp1); + LASX_ILVH_D_2_128SV(tmp0, tmp0, tmp1, tmp1, tmp2, tmp3); + LASX_TRANSPOSE4x4_H_128SV(tmp0, tmp2, tmp1, tmp3, tmp2, tmp3, tmp4, tmp5); + __lasx_xvstelm_d(tmp2, (int8_t*)bS, 0, 0); + __lasx_xvstelm_d(tmp3, (int8_t*)bS + 8, 0, 0); + __lasx_xvstelm_d(tmp4, (int8_t*)bS + 16, 0, 0); + __lasx_xvstelm_d(tmp5, (int8_t*)bS + 24, 0, 0); +} diff --git a/libavcodec/loongarch/h264chroma_init_loongarch.c b/libavcodec/loongarch/h264chroma_init_loongarch.c new file mode 100644 index 0000000000..5789b5f573 --- /dev/null +++ b/libavcodec/loongarch/h264chroma_init_loongarch.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "h264chroma_lasx.h" +#include "libavutil/loongarch/cpu.h" +#include "libavcodec/h264chroma.h" + +av_cold void ff_h264chroma_init_loongarch(H264ChromaContext *c, int bit_depth) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lasx(cpu_flags)) { + if (bit_depth <= 8) { + c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_lasx; + c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_lasx; + } + } +} diff --git a/libavcodec/loongarch/h264chroma_lasx.c b/libavcodec/loongarch/h264chroma_lasx.c new file mode 100644 index 0000000000..4fbe99ba73 --- /dev/null +++ b/libavcodec/loongarch/h264chroma_lasx.c @@ -0,0 +1,837 @@ +/* + * Loongson LASX optimized h264chroma + * + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "h264chroma_lasx.h" +#include "libavutil/attributes.h" +#include "libavutil/avassert.h" +#include "libavutil/loongarch/generic_macros_lasx.h" + +static const uint8_t chroma_mask_arr[32] = { + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, +}; + +static av_always_inline void avc_chroma_hv_8x4_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride, uint32_t coef_hor0, + uint32_t coef_hor1, uint32_t coef_ver0, + uint32_t coef_ver1) +{ + __m256i src0, src1, src2, src3, src4, out; + __m256i res_hz0, res_hz1, res_hz2, res_vt0, res_vt1; + __m256i mask; + __m256i coeff_hz_vec0 = __lasx_xvreplgr2vr_b(coef_hor0); + __m256i coeff_hz_vec1 = __lasx_xvreplgr2vr_b(coef_hor1); + __m256i coeff_hz_vec = __lasx_xvilvl_b(coeff_hz_vec0, coeff_hz_vec1); + __m256i coeff_vt_vec0 = __lasx_xvreplgr2vr_h(coef_ver0); + __m256i coeff_vt_vec1 = __lasx_xvreplgr2vr_h(coef_ver1); + + mask = LASX_LD(chroma_mask_arr); + src0 = LASX_LD(src); + LASX_LD_4(src + stride, stride, src1, src2, src3, src4); + LASX_PCKEV_Q_2(src2, src1, src4, src3, src1, src3); + LASX_SHUF_B_128SV(src0, src0, mask, src0); + LASX_SHUF_B_2_128SV(src1, src1, src3, src3, mask, mask, src1, src3); + LASX_DP2_H_BU(src0, coeff_hz_vec, res_hz0); + LASX_DP2_H_BU(src1, coeff_hz_vec, res_hz1); + LASX_DP2_H_BU(src3, coeff_hz_vec, res_hz2); + res_vt0 = __lasx_xvmul_h(res_hz1, coeff_vt_vec0); + res_vt1 = __lasx_xvmul_h(res_hz2, coeff_vt_vec0); + LASX_PCKEV_Q(res_hz1, res_hz0, res_hz0); + res_hz1 = __lasx_xvpermi_q(res_hz1, res_hz2, 0x3); + res_hz0 = __lasx_xvmul_h(res_hz0, coeff_vt_vec1); + res_hz1 = __lasx_xvmul_h(res_hz1, coeff_vt_vec1); + res_vt0 = __lasx_xvadd_h(res_vt0, res_hz0); + res_vt1 = __lasx_xvadd_h(res_vt1, res_hz1); + LASX_SRARI_H_2(res_vt0, res_vt1, res_vt0, res_vt1, 6); + res_vt0 = __lasx_xvsat_hu(res_vt0, 7); + res_vt1 = __lasx_xvsat_hu(res_vt1, 7); + LASX_PCKEV_B_128SV(res_vt1, res_vt0, out); + LASX_ST_D_4(out, 0, 2, 1, 3, dst, stride); +} + +static av_always_inline void avc_chroma_hv_8x8_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride, uint32_t coef_hor0, + uint32_t coef_hor1, uint32_t coef_ver0, + uint32_t coef_ver1) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m256i out0, out1; + __m256i res_hz0, res_hz1, res_hz2, res_hz3, res_hz4; + __m256i res_vt0, res_vt1, res_vt2, res_vt3; + __m256i mask; + __m256i coeff_hz_vec0 = __lasx_xvreplgr2vr_b(coef_hor0); + __m256i coeff_hz_vec1 = __lasx_xvreplgr2vr_b(coef_hor1); + __m256i coeff_hz_vec = __lasx_xvilvl_b(coeff_hz_vec0, coeff_hz_vec1); + __m256i coeff_vt_vec0 = __lasx_xvreplgr2vr_h(coef_ver0); + __m256i coeff_vt_vec1 = __lasx_xvreplgr2vr_h(coef_ver1); + + mask = LASX_LD(chroma_mask_arr); + src0 = LASX_LD(src); + LASX_LD_8(src + stride, stride, src1, src2, src3, src4, + src5, src6, src7, src8); + LASX_PCKEV_Q_4(src2, src1, src4, src3, src6, src5, src8, src7, + src1, src3, src5, src7); + LASX_SHUF_B_128SV(src0, src0, mask, src0); + LASX_SHUF_B_4_128SV(src1, src1, src3, src3, src5, src5, src7, src7, + mask, mask, mask, mask, src1, src3, src5, src7); + LASX_DP2_H_BU(src0, coeff_hz_vec, res_hz0); + LASX_DP2_H_BU(src1, coeff_hz_vec, res_hz1); + LASX_DP2_H_BU(src3, coeff_hz_vec, res_hz2); + LASX_DP2_H_BU(src5, coeff_hz_vec, res_hz3); + LASX_DP2_H_BU(src7, coeff_hz_vec, res_hz4); + res_vt0 = __lasx_xvmul_h(res_hz1, coeff_vt_vec0); + res_vt1 = __lasx_xvmul_h(res_hz2, coeff_vt_vec0); + res_vt2 = __lasx_xvmul_h(res_hz3, coeff_vt_vec0); + res_vt3 = __lasx_xvmul_h(res_hz4, coeff_vt_vec0); + LASX_PCKEV_Q(res_hz1, res_hz0, res_hz0); + res_hz1 = __lasx_xvpermi_q(res_hz1, res_hz2, 0x3); + res_hz2 = __lasx_xvpermi_q(res_hz2, res_hz3, 0x3); + res_hz3 = __lasx_xvpermi_q(res_hz3, res_hz4, 0x3); + res_hz0 = __lasx_xvmul_h(res_hz0, coeff_vt_vec1); + res_hz1 = __lasx_xvmul_h(res_hz1, coeff_vt_vec1); + res_hz2 = __lasx_xvmul_h(res_hz2, coeff_vt_vec1); + res_hz3 = __lasx_xvmul_h(res_hz3, coeff_vt_vec1); + res_vt0 = __lasx_xvadd_h(res_vt0, res_hz0); + res_vt1 = __lasx_xvadd_h(res_vt1, res_hz1); + res_vt2 = __lasx_xvadd_h(res_vt2, res_hz2); + res_vt3 = __lasx_xvadd_h(res_vt3, res_hz3); + LASX_SRARI_H_4(res_vt0, res_vt1, res_vt2, res_vt3, + res_vt0, res_vt1, res_vt2, res_vt3, 6); + res_vt0 = __lasx_xvsat_hu(res_vt0, 7); + res_vt1 = __lasx_xvsat_hu(res_vt1, 7); + res_vt2 = __lasx_xvsat_hu(res_vt2, 7); + res_vt3 = __lasx_xvsat_hu(res_vt3, 7); + LASX_PCKEV_B_2_128SV(res_vt1, res_vt0, res_vt3, res_vt2, out0, out1); + LASX_ST_D_4(out0, 0, 2, 1, 3, dst, stride); + LASX_ST_D_4(out1, 0, 2, 1, 3, dst + stride * 4, stride); +} + +static av_always_inline void avc_chroma_hz_8x4_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride, uint32_t coeff0, uint32_t coeff1) +{ + __m256i src0, src1, src2, src3, out; + __m256i res0, res1; + __m256i mask; + __m256i coeff_vec0 = __lasx_xvreplgr2vr_b(coeff0); + __m256i coeff_vec1 = __lasx_xvreplgr2vr_b(coeff1); + __m256i coeff_vec = __lasx_xvilvl_b(coeff_vec0, coeff_vec1); + + mask = LASX_LD(chroma_mask_arr); + LASX_LD_4(src, stride, src0, src1, src2, src3); + LASX_PCKEV_Q_2(src1, src0, src3, src2, src0, src2); + LASX_SHUF_B_2_128SV(src0, src0, src2, src2, mask, mask, src0, src2); + LASX_DP2_H_BU(src0, coeff_vec, res0); + LASX_DP2_H_BU(src2, coeff_vec, res1); + res0 = __lasx_xvslli_h(res0, 3); + res1 = __lasx_xvslli_h(res1, 3); + LASX_SRARI_H_2(res0, res1, res0, res1, 6); + res0 = __lasx_xvsat_hu(res0, 7); + res1 = __lasx_xvsat_hu(res1, 7); + LASX_PCKEV_B_128SV(res1, res0, out); + LASX_ST_D_4(out, 0, 2, 1, 3, dst, stride); +} + +static av_always_inline void avc_chroma_hz_8x8_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride, uint32_t coeff0, uint32_t coeff1) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i out0, out1; + __m256i res0, res1, res2, res3; + __m256i mask; + __m256i coeff_vec0 = __lasx_xvreplgr2vr_b(coeff0); + __m256i coeff_vec1 = __lasx_xvreplgr2vr_b(coeff1); + __m256i coeff_vec = __lasx_xvilvl_b(coeff_vec0, coeff_vec1); + + mask = LASX_LD(chroma_mask_arr); + LASX_LD_8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7); + LASX_PCKEV_Q_4(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src2, src4, src6); + LASX_SHUF_B_4_128SV(src0, src0, src2, src2, src4, src4, src6, src6, + mask, mask, mask, mask, src0, src2, src4, src6); + LASX_DP2_H_BU(src0, coeff_vec, res0); + LASX_DP2_H_BU(src2, coeff_vec, res1); + LASX_DP2_H_BU(src4, coeff_vec, res2); + LASX_DP2_H_BU(src6, coeff_vec, res3); + res0 = __lasx_xvslli_h(res0, 3); + res1 = __lasx_xvslli_h(res1, 3); + res2 = __lasx_xvslli_h(res2, 3); + res3 = __lasx_xvslli_h(res3, 3); + LASX_SRARI_H_4(res0, res1, res2, res3, + res0, res1, res2, res3, 6); + res0 = __lasx_xvsat_hu(res0, 7); + res1 = __lasx_xvsat_hu(res1, 7); + res2 = __lasx_xvsat_hu(res2, 7); + res3 = __lasx_xvsat_hu(res3, 7); + LASX_PCKEV_B_2_128SV(res1, res0, res3, res2, out0, out1); + LASX_ST_D_4(out0, 0, 2, 1, 3, dst, stride); + LASX_ST_D_4(out1, 0, 2, 1, 3, dst + stride * 4, stride) +} + +static av_always_inline void avc_chroma_hz_nonmult_lasx(uint8_t *src, + uint8_t *dst, ptrdiff_t stride, uint32_t coeff0, + uint32_t coeff1, int32_t height) +{ + uint32_t row; + __m256i src0, src1, src2, src3, out; + __m256i res0, res1; + __m256i mask; + __m256i coeff_vec0 = __lasx_xvreplgr2vr_b(coeff0); + __m256i coeff_vec1 = __lasx_xvreplgr2vr_b(coeff1); + __m256i coeff_vec = __lasx_xvilvl_b(coeff_vec0, coeff_vec1); + + mask = LASX_LD(chroma_mask_arr); + + for (row = height >> 2; row--;) { + LASX_LD_4(src, stride, src0, src1, src2, src3); + src += (stride * 4); + LASX_PCKEV_Q_2(src1, src0, src3, src2, src0, src2); + LASX_SHUF_B_2_128SV(src0, src0, src2, src2, mask, mask, src0, src2); + LASX_DP2_H_BU(src0, coeff_vec, res0); + LASX_DP2_H_BU(src2, coeff_vec, res1); + res0 = __lasx_xvslli_h(res0, 3); + res1 = __lasx_xvslli_h(res1, 3); + LASX_SRARI_H_2(res0, res1, res0, res1, 6); + res0 = __lasx_xvsat_hu(res0, 7); + res1 = __lasx_xvsat_hu(res1, 7); + LASX_PCKEV_B_128SV(res1, res0, out); + LASX_ST_D_4(out, 0, 2, 1, 3, dst, stride); + dst += (stride * 4); + } + + if ((height & 3)) { + for (row = (height & 3); row--;) { + src0 = LASX_LD(src); + src += stride; + LASX_SHUF_B_128SV(src0, src0, mask, src0); + LASX_DP2_H_BU(src0, coeff_vec, res0); + res0 = __lasx_xvslli_h(res0, 3); + LASX_SRARI_H(res0, res0, 6); + res0 = __lasx_xvsat_hu(res0, 7); + LASX_PCKEV_B_128SV(res0, res0, out); + LASX_ST_D(out, 0, dst); + dst += stride; + } + } +} + +static av_always_inline void avc_chroma_vt_8x4_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride, uint32_t coeff0, uint32_t coeff1) +{ + __m256i src0, src1, src2, src3, src4, out; + __m256i res0, res1; + __m256i coeff_vec0 = __lasx_xvreplgr2vr_b(coeff0); + __m256i coeff_vec1 = __lasx_xvreplgr2vr_b(coeff1); + __m256i coeff_vec = __lasx_xvilvl_b(coeff_vec0, coeff_vec1); + + src0 = LASX_LD(src); + LASX_LD_4(src + stride, stride, src1, src2, src3, src4); + LASX_PCKEV_Q_4(src1, src0, src2, src1, src3, src2, src4, src3, + src0, src1, src2, src3); + LASX_ILVL_B_2_128SV(src1, src0, src3, src2, src0, src2); + LASX_DP2_H_BU(src0, coeff_vec, res0); + LASX_DP2_H_BU(src2, coeff_vec, res1); + res0 = __lasx_xvslli_h(res0, 3); + res1 = __lasx_xvslli_h(res1, 3); + LASX_SRARI_H_2(res0, res1, res0, res1, 6); + res0 = __lasx_xvsat_hu(res0, 7); + res1 = __lasx_xvsat_hu(res1, 7); + LASX_PCKEV_B_128SV(res1, res0, out); + LASX_ST_D_4(out, 0, 2, 1, 3, dst, stride); +} + +static av_always_inline void avc_chroma_vt_8x8_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride, uint32_t coeff0, uint32_t coeff1) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m256i out0, out1; + __m256i res0, res1, res2, res3; + __m256i coeff_vec0 = __lasx_xvreplgr2vr_b(coeff0); + __m256i coeff_vec1 = __lasx_xvreplgr2vr_b(coeff1); + __m256i coeff_vec = __lasx_xvilvl_b(coeff_vec0, coeff_vec1); + + src0 = LASX_LD(src); + LASX_LD_8(src+stride, stride, src1, src2, src3, src4, + src5, src6, src7, src8); + LASX_PCKEV_Q_4(src1, src0, src2, src1, src3, src2, src4, src3, + src0, src1, src2, src3); + LASX_PCKEV_Q_4(src5, src4, src6, src5, src7, src6, src8, src7, + src4, src5, src6, src7); + LASX_ILVL_B_4_128SV(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src2, src4, src6); + LASX_DP2_H_BU(src0, coeff_vec, res0); + LASX_DP2_H_BU(src2, coeff_vec, res1); + LASX_DP2_H_BU(src4, coeff_vec, res2); + LASX_DP2_H_BU(src6, coeff_vec, res3); + res0 = __lasx_xvslli_h(res0, 3); + res1 = __lasx_xvslli_h(res1, 3); + res2 = __lasx_xvslli_h(res2, 3); + res3 = __lasx_xvslli_h(res3, 3); + LASX_SRARI_H_4(res0, res1, res2, res3, res0, res1, res2, res3, 6); + res0 = __lasx_xvsat_hu(res0, 7); + res1 = __lasx_xvsat_hu(res1, 7); + res2 = __lasx_xvsat_hu(res2, 7); + res3 = __lasx_xvsat_hu(res3, 7); + LASX_PCKEV_B_2_128SV(res1, res0, res3, res2, out0, out1); + LASX_ST_D_4(out0, 0, 2, 1, 3, dst, stride); + LASX_ST_D_4(out1, 0, 2, 1, 3, dst + stride * 4, stride); +} + +static av_always_inline void copy_width8x8_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride) +{ + uint64_t tmp[8]; + __asm__ volatile ( + "ld.d %[tmp0], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp1], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp2], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp3], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp4], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp5], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp6], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp7], %[src], 0x0 \n\t" + + "st.d %[tmp0], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp1], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp2], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp3], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp4], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp5], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp6], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp7], %[dst], 0x0 \n\t" + : [tmp0]"=&r"(tmp[0]), [tmp1]"=&r"(tmp[1]), + [tmp2]"=&r"(tmp[2]), [tmp3]"=&r"(tmp[3]), + [tmp4]"=&r"(tmp[4]), [tmp5]"=&r"(tmp[5]), + [tmp6]"=&r"(tmp[6]), [tmp7]"=&r"(tmp[7]), + [dst]"+&r"(dst), [src]"+&r"(src) + : [stride]"r"(stride) + : "memory" + ); +} + +static av_always_inline void copy_width8x4_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride) +{ + uint64_t tmp[4]; + __asm__ volatile ( + "ld.d %[tmp0], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp1], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp2], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp3], %[src], 0x0 \n\t" + + "st.d %[tmp0], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp1], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp2], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp3], %[dst], 0x0 \n\t" + : [tmp0]"=&r"(tmp[0]), [tmp1]"=&r"(tmp[1]), + [tmp2]"=&r"(tmp[2]), [tmp3]"=&r"(tmp[3]), + [dst]"+&r"(dst), [src]"+&r"(src) + : [stride]"r"(stride) + : "memory" + ); +} + +static void avc_chroma_hv_8w_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + uint32_t coef_hor0, uint32_t coef_hor1, + uint32_t coef_ver0, uint32_t coef_ver1, + int32_t height) +{ + if (4 == height) { + avc_chroma_hv_8x4_lasx(src, dst, stride, coef_hor0, coef_hor1, coef_ver0, + coef_ver1); + } else if (8 == height) { + avc_chroma_hv_8x8_lasx(src, dst, stride, coef_hor0, coef_hor1, coef_ver0, + coef_ver1); + } +} + +static void avc_chroma_hz_8w_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + uint32_t coeff0, uint32_t coeff1, + int32_t height) +{ + if (4 == height) { + avc_chroma_hz_8x4_lasx(src, dst, stride, coeff0, coeff1); + } else if (8 == height) { + avc_chroma_hz_8x8_lasx(src, dst, stride, coeff0, coeff1); + } else { + avc_chroma_hz_nonmult_lasx(src, dst, stride, coeff0, coeff1, height); + } +} + +static void avc_chroma_vt_8w_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + uint32_t coeff0, uint32_t coeff1, + int32_t height) +{ + if (4 == height) { + avc_chroma_vt_8x4_lasx(src, dst, stride, coeff0, coeff1); + } else if (8 == height) { + avc_chroma_vt_8x8_lasx(src, dst, stride, coeff0, coeff1); + } +} + +static void copy_width8_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + int32_t height) +{ + if (8 == height) { + copy_width8x8_lasx(src, dst, stride); + } else if (4 == height) { + copy_width8x4_lasx(src, dst, stride); + } +} + +void ff_put_h264_chroma_mc8_lasx(uint8_t *dst, uint8_t *src, ptrdiff_t stride, + int height, int x, int y) +{ + av_assert2(x < 8 && y < 8 && x >= 0 && y >= 0); + + if (!(x || y)) { + copy_width8_lasx(src, dst, stride, height); + } else if (x && y) { + avc_chroma_hv_8w_lasx(src, dst, stride, x, (8 - x), y, (8 - y), height); + } else if (x) { + avc_chroma_hz_8w_lasx(src, dst, stride, x, (8 - x), height); + } else { + avc_chroma_vt_8w_lasx(src, dst, stride, y, (8 - y), height); + } +} + +static av_always_inline void avc_chroma_hv_and_aver_dst_8x4_lasx(uint8_t *src, + uint8_t *dst, ptrdiff_t stride, uint32_t coef_hor0, + uint32_t coef_hor1, uint32_t coef_ver0, + uint32_t coef_ver1) +{ + __m256i tp0, tp1, tp2, tp3; + __m256i src0, src1, src2, src3, src4, out; + __m256i res_hz0, res_hz1, res_hz2, res_vt0, res_vt1; + __m256i mask; + __m256i coeff_hz_vec0 = __lasx_xvreplgr2vr_b(coef_hor0); + __m256i coeff_hz_vec1 = __lasx_xvreplgr2vr_b(coef_hor1); + __m256i coeff_hz_vec = __lasx_xvilvl_b(coeff_hz_vec0, coeff_hz_vec1); + __m256i coeff_vt_vec0 = __lasx_xvreplgr2vr_h(coef_ver0); + __m256i coeff_vt_vec1 = __lasx_xvreplgr2vr_h(coef_ver1); + + mask = LASX_LD(chroma_mask_arr); + src0 = LASX_LD(src); + LASX_LD_4(src + stride, stride, src1, src2, src3, src4); + LASX_PCKEV_Q_2(src2, src1, src4, src3, src1, src3); + LASX_SHUF_B_128SV(src0, src0, mask, src0); + LASX_SHUF_B_2_128SV(src1, src1, src3, src3, mask, mask, src1, src3); + LASX_DP2_H_BU(src0, coeff_hz_vec, res_hz0); + LASX_DP2_H_BU(src1, coeff_hz_vec, res_hz1); + LASX_DP2_H_BU(src3, coeff_hz_vec, res_hz2); + res_vt0 = __lasx_xvmul_h(res_hz1, coeff_vt_vec0); + res_vt1 = __lasx_xvmul_h(res_hz2, coeff_vt_vec0); + LASX_PCKEV_Q(res_hz1, res_hz0, res_hz0); + res_hz1 = __lasx_xvpermi_q(res_hz1, res_hz2, 0x3); + res_hz0 = __lasx_xvmul_h(res_hz0, coeff_vt_vec1); + res_hz1 = __lasx_xvmul_h(res_hz1, coeff_vt_vec1); + res_vt0 = __lasx_xvadd_h(res_vt0, res_hz0); + res_vt1 = __lasx_xvadd_h(res_vt1, res_hz1); + LASX_SRARI_H_2(res_vt0, res_vt1, res_vt0, res_vt1, 6); + res_vt0 = __lasx_xvsat_hu(res_vt0, 7); + res_vt1 = __lasx_xvsat_hu(res_vt1, 7); + LASX_PCKEV_B_128SV(res_vt1, res_vt0, out); + LASX_LD_4(dst, stride, tp0, tp1, tp2, tp3); + LASX_ILVL_D_2_128SV(tp2, tp0, tp3, tp1, tp0, tp2); + LASX_PCKEV_Q(tp2, tp0, tp0); + out = __lasx_xvavgr_bu(out, tp0); + LASX_ST_D_4(out, 0, 2, 1, 3, dst, stride); +} + +static av_always_inline void avc_chroma_hv_and_aver_dst_8x8_lasx(uint8_t *src, + uint8_t *dst, ptrdiff_t stride, uint32_t coef_hor0, + uint32_t coef_hor1, uint32_t coef_ver0, + uint32_t coef_ver1) +{ + __m256i tp0, tp1, tp2, tp3, dst0, dst1; + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m256i out0, out1; + __m256i res_hz0, res_hz1, res_hz2, res_hz3, res_hz4; + __m256i res_vt0, res_vt1, res_vt2, res_vt3; + __m256i mask; + __m256i coeff_hz_vec0 = __lasx_xvreplgr2vr_b(coef_hor0); + __m256i coeff_hz_vec1 = __lasx_xvreplgr2vr_b(coef_hor1); + __m256i coeff_vt_vec0 = __lasx_xvreplgr2vr_h(coef_ver0); + __m256i coeff_vt_vec1 = __lasx_xvreplgr2vr_h(coef_ver1); + __m256i coeff_hz_vec = __lasx_xvilvl_b(coeff_hz_vec0, coeff_hz_vec1); + + mask = LASX_LD(chroma_mask_arr); + src0 = LASX_LD(src); + LASX_LD_8(src + stride, stride, src1, src2, src3, src4, + src5, src6, src7, src8); + LASX_PCKEV_Q_4(src2, src1, src4, src3, src6, src5, src8, src7, + src1, src3, src5, src7); + LASX_SHUF_B_128SV(src0, src0, mask, src0); + LASX_SHUF_B_4_128SV(src1, src1, src3, src3, src5, src5, src7, src7, + mask, mask, mask, mask, src1, src3, src5, src7); + LASX_DP2_H_BU(src0, coeff_hz_vec, res_hz0); + LASX_DP2_H_BU(src1, coeff_hz_vec, res_hz1); + LASX_DP2_H_BU(src3, coeff_hz_vec, res_hz2); + LASX_DP2_H_BU(src5, coeff_hz_vec, res_hz3); + LASX_DP2_H_BU(src7, coeff_hz_vec, res_hz4); + res_vt0 = __lasx_xvmul_h(res_hz1, coeff_vt_vec0); + res_vt1 = __lasx_xvmul_h(res_hz2, coeff_vt_vec0); + res_vt2 = __lasx_xvmul_h(res_hz3, coeff_vt_vec0); + res_vt3 = __lasx_xvmul_h(res_hz4, coeff_vt_vec0); + LASX_PCKEV_Q(res_hz1, res_hz0, res_hz0); + res_hz1 = __lasx_xvpermi_q(res_hz1, res_hz2, 0x3); + res_hz2 = __lasx_xvpermi_q(res_hz2, res_hz3, 0x3); + res_hz3 = __lasx_xvpermi_q(res_hz3, res_hz4, 0x3); + res_hz0 = __lasx_xvmul_h(res_hz0, coeff_vt_vec1); + res_hz1 = __lasx_xvmul_h(res_hz1, coeff_vt_vec1); + res_hz2 = __lasx_xvmul_h(res_hz2, coeff_vt_vec1); + res_hz3 = __lasx_xvmul_h(res_hz3, coeff_vt_vec1); + res_vt0 = __lasx_xvadd_h(res_vt0, res_hz0); + res_vt1 = __lasx_xvadd_h(res_vt1, res_hz1); + res_vt2 = __lasx_xvadd_h(res_vt2, res_hz2); + res_vt3 = __lasx_xvadd_h(res_vt3, res_hz3); + LASX_SRARI_H_4(res_vt0, res_vt1, res_vt2, res_vt3, + res_vt0, res_vt1, res_vt2, res_vt3, 6); + res_vt0 = __lasx_xvsat_hu(res_vt0, 7); + res_vt1 = __lasx_xvsat_hu(res_vt1, 7); + res_vt2 = __lasx_xvsat_hu(res_vt2, 7); + res_vt3 = __lasx_xvsat_hu(res_vt3, 7); + LASX_PCKEV_B_2_128SV(res_vt1, res_vt0, res_vt3, res_vt2, out0, out1); + LASX_LD_4(dst, stride, tp0, tp1, tp2, tp3); + LASX_ILVL_D_2_128SV(tp2, tp0, tp3, tp1, tp0, tp2); + LASX_PCKEV_Q(tp2, tp0, dst0); + LASX_LD_4(dst + 4 * stride, stride, tp0, tp1, tp2, tp3); + LASX_ILVL_D_2_128SV(tp2, tp0, tp3, tp1, tp0, tp2); + LASX_PCKEV_Q(tp2, tp0, dst1); + out0 = __lasx_xvavgr_bu(out0, dst0); + out1 = __lasx_xvavgr_bu(out1, dst1); + LASX_ST_D_4(out0, 0, 2, 1, 3, dst, stride); + LASX_ST_D_4(out1, 0, 2, 1, 3, dst + stride * 4, stride); +} + +static av_always_inline void avc_chroma_hz_and_aver_dst_8x4_lasx(uint8_t *src, + uint8_t *dst, ptrdiff_t stride, uint32_t coeff0, + uint32_t coeff1) +{ + __m256i tp0, tp1, tp2, tp3; + __m256i src0, src1, src2, src3, out; + __m256i res0, res1; + __m256i mask; + __m256i coeff_vec0 = __lasx_xvreplgr2vr_b(coeff0); + __m256i coeff_vec1 = __lasx_xvreplgr2vr_b(coeff1); + __m256i coeff_vec = __lasx_xvilvl_b(coeff_vec0, coeff_vec1); + + mask = LASX_LD(chroma_mask_arr); + LASX_LD_4(src, stride, src0, src1, src2, src3); + LASX_PCKEV_Q_2(src1, src0, src3, src2, src0, src2); + LASX_SHUF_B_2_128SV(src0, src0, src2, src2, mask, mask, src0, src2); + LASX_DP2_H_BU(src0, coeff_vec, res0); + LASX_DP2_H_BU(src2, coeff_vec, res1); + res0 = __lasx_xvslli_h(res0, 3); + res1 = __lasx_xvslli_h(res1, 3); + LASX_SRARI_H_2(res0, res1, res0, res1, 6); + res0 = __lasx_xvsat_hu(res0, 7); + res1 = __lasx_xvsat_hu(res1, 7); + LASX_PCKEV_B_128SV(res1, res0, out); + LASX_LD_4(dst, stride, tp0, tp1, tp2, tp3); + LASX_ILVL_D_2_128SV(tp2, tp0, tp3, tp1, tp0, tp2); + LASX_PCKEV_Q(tp2, tp0, tp0); + out = __lasx_xvavgr_bu(out, tp0); + LASX_ST_D_4(out, 0, 2, 1, 3, dst, stride); +} + +static av_always_inline void avc_chroma_hz_and_aver_dst_8x8_lasx(uint8_t *src, + uint8_t *dst, ptrdiff_t stride, uint32_t coeff0, + uint32_t coeff1) +{ + __m256i tp0, tp1, tp2, tp3, dst0, dst1; + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i out0, out1; + __m256i res0, res1, res2, res3; + __m256i mask; + __m256i coeff_vec0 = __lasx_xvreplgr2vr_b(coeff0); + __m256i coeff_vec1 = __lasx_xvreplgr2vr_b(coeff1); + __m256i coeff_vec = __lasx_xvilvl_b(coeff_vec0, coeff_vec1); + + mask = LASX_LD(chroma_mask_arr); + LASX_LD_8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7); + LASX_PCKEV_Q_4(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src2, src4, src6); + LASX_SHUF_B_4_128SV(src0, src0, src2, src2, src4, src4, src6, src6, + mask, mask, mask, mask, src0, src2, src4, src6); + LASX_DP2_H_BU(src0, coeff_vec, res0); + LASX_DP2_H_BU(src2, coeff_vec, res1); + LASX_DP2_H_BU(src4, coeff_vec, res2); + LASX_DP2_H_BU(src6, coeff_vec, res3); + res0 = __lasx_xvslli_h(res0, 3); + res1 = __lasx_xvslli_h(res1, 3); + res2 = __lasx_xvslli_h(res2, 3); + res3 = __lasx_xvslli_h(res3, 3); + LASX_SRARI_H_4(res0, res1, res2, res3, + res0, res1, res2, res3, 6); + res0 = __lasx_xvsat_hu(res0, 7); + res1 = __lasx_xvsat_hu(res1, 7); + res2 = __lasx_xvsat_hu(res2, 7); + res3 = __lasx_xvsat_hu(res3, 7); + LASX_PCKEV_B_2_128SV(res1, res0, res3, res2, out0, out1); + LASX_LD_4(dst, stride, tp0, tp1, tp2, tp3); + LASX_ILVL_D_2_128SV(tp2, tp0, tp3, tp1, tp0, tp2); + LASX_PCKEV_Q(tp2, tp0, dst0); + LASX_LD_4(dst + 4 * stride, stride, tp0, tp1, tp2, tp3); + LASX_ILVL_D_2_128SV(tp2, tp0, tp3, tp1, tp0, tp2); + LASX_PCKEV_Q(tp2, tp0, dst1); + out0 = __lasx_xvavgr_bu(out0, dst0); + out1 = __lasx_xvavgr_bu(out1, dst1); + LASX_ST_D_4(out0, 0, 2, 1, 3, dst, stride); + LASX_ST_D_4(out1, 0, 2, 1, 3, dst + stride * 4, stride) +} + +static av_always_inline void avc_chroma_vt_and_aver_dst_8x4_lasx(uint8_t *src, + uint8_t *dst, ptrdiff_t stride, uint32_t coeff0, + uint32_t coeff1) +{ + __m256i tp0, tp1, tp2, tp3; + __m256i src0, src1, src2, src3, src4, out; + __m256i res0, res1; + __m256i coeff_vec0 = __lasx_xvreplgr2vr_b(coeff0); + __m256i coeff_vec1 = __lasx_xvreplgr2vr_b(coeff1); + __m256i coeff_vec = __lasx_xvilvl_b(coeff_vec0, coeff_vec1); + + src0 = LASX_LD(src); + LASX_LD_4(src + stride, stride, src1, src2, src3, src4); + LASX_PCKEV_Q_4(src1, src0, src2, src1, src3, src2, src4, src3, + src0, src1, src2, src3); + LASX_ILVL_B_2_128SV(src1, src0, src3, src2, src0, src2); + LASX_DP2_H_BU(src0, coeff_vec, res0); + LASX_DP2_H_BU(src2, coeff_vec, res1); + res0 = __lasx_xvslli_h(res0, 3); + res1 = __lasx_xvslli_h(res1, 3); + LASX_SRARI_H_2(res0, res1, res0, res1, 6); + res0 = __lasx_xvsat_hu(res0, 7); + res1 = __lasx_xvsat_hu(res1, 7); + LASX_PCKEV_B_128SV(res1, res0, out); + LASX_LD_4(dst, stride, tp0, tp1, tp2, tp3); + LASX_ILVL_D_2_128SV(tp2, tp0, tp3, tp1, tp0, tp2); + LASX_PCKEV_Q(tp2, tp0, tp0); + out = __lasx_xvavgr_bu(out, tp0); + LASX_ST_D_4(out, 0, 2, 1, 3, dst, stride); +} + +static av_always_inline void avc_chroma_vt_and_aver_dst_8x8_lasx(uint8_t *src, + uint8_t *dst, ptrdiff_t stride, uint32_t coeff0, + uint32_t coeff1) +{ + __m256i tp0, tp1, tp2, tp3, dst0, dst1; + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m256i out0, out1; + __m256i res0, res1, res2, res3; + __m256i coeff_vec0 = __lasx_xvreplgr2vr_b(coeff0); + __m256i coeff_vec1 = __lasx_xvreplgr2vr_b(coeff1); + __m256i coeff_vec = __lasx_xvilvl_b(coeff_vec0, coeff_vec1); + + src0 = LASX_LD(src); + LASX_LD_8(src + stride, stride, src1, src2, src3, src4, + src5, src6, src7, src8); + LASX_PCKEV_Q_4(src1, src0, src2, src1, src3, src2, src4, src3, + src0, src1, src2, src3); + LASX_PCKEV_Q_4(src5, src4, src6, src5, src7, src6, src8, src7, + src4, src5, src6, src7); + LASX_ILVL_B_4_128SV(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src2, src4, src6); + LASX_DP2_H_BU(src0, coeff_vec, res0); + LASX_DP2_H_BU(src2, coeff_vec, res1); + LASX_DP2_H_BU(src4, coeff_vec, res2); + LASX_DP2_H_BU(src6, coeff_vec, res3); + res0 = __lasx_xvslli_h(res0, 3); + res1 = __lasx_xvslli_h(res1, 3); + res2 = __lasx_xvslli_h(res2, 3); + res3 = __lasx_xvslli_h(res3, 3); + LASX_SRARI_H_4(res0, res1, res2, res3, res0, res1, res2, res3, 6); + res0 = __lasx_xvsat_hu(res0, 7); + res1 = __lasx_xvsat_hu(res1, 7); + res2 = __lasx_xvsat_hu(res2, 7); + res3 = __lasx_xvsat_hu(res3, 7); + LASX_PCKEV_B_2_128SV(res1, res0, res3, res2, out0, out1); + LASX_LD_4(dst, stride, tp0, tp1, tp2, tp3); + LASX_ILVL_D_2_128SV(tp2, tp0, tp3, tp1, tp0, tp2); + LASX_PCKEV_Q(tp2, tp0, dst0); + LASX_LD_4(dst + 4 * stride, stride, tp0, tp1, tp2, tp3); + LASX_ILVL_D_2_128SV(tp2, tp0, tp3, tp1, tp0, tp2); + LASX_PCKEV_Q(tp2, tp0, dst1); + out0 = __lasx_xvavgr_bu(out0, dst0); + out1 = __lasx_xvavgr_bu(out1, dst1); + LASX_ST_D_4(out0, 0, 2, 1, 3, dst, stride); + LASX_ST_D_4(out1, 0, 2, 1, 3, dst + stride * 4, stride); +} + +static av_always_inline void avg_width8x8_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride) +{ + __m256i src0, src1, src2, src3; + __m256i dst0, dst1, dst2, dst3; + ptrdiff_t stride_x2 = stride << 1; + ptrdiff_t stride_x3 = stride_x2 + stride; + ptrdiff_t stride_x4 = stride << 2; + + src0 = __lasx_xvldrepl_d(src, 0); + src1 = __lasx_xvldrepl_d(src + stride, 0); + src2 = __lasx_xvldrepl_d(src + stride_x2, 0); + src3 = __lasx_xvldrepl_d(src + stride_x3, 0); + dst0 = __lasx_xvldrepl_d(dst, 0); + dst1 = __lasx_xvldrepl_d(dst + stride, 0); + dst2 = __lasx_xvldrepl_d(dst + stride_x2, 0); + dst3 = __lasx_xvldrepl_d(dst + stride_x3, 0); + src0 = __lasx_xvpackev_d(src1,src0); + src2 = __lasx_xvpackev_d(src3,src2); + src0 = __lasx_xvpermi_q(src0, src2, 0x02); + dst0 = __lasx_xvpackev_d(dst1,dst0); + dst2 = __lasx_xvpackev_d(dst3,dst2); + dst0 = __lasx_xvpermi_q(dst0, dst2, 0x02); + dst0 = __lasx_xvavgr_bu(src0, dst0); + LASX_ST_D_4(dst0, 0, 1, 2, 3, dst, stride); + + src += stride_x4; + dst += stride_x4; + src0 = __lasx_xvldrepl_d(src, 0); + src1 = __lasx_xvldrepl_d(src + stride, 0); + src2 = __lasx_xvldrepl_d(src + stride_x2, 0); + src3 = __lasx_xvldrepl_d(src + stride_x3, 0); + dst0 = __lasx_xvldrepl_d(dst, 0); + dst1 = __lasx_xvldrepl_d(dst + stride, 0); + dst2 = __lasx_xvldrepl_d(dst + stride_x2, 0); + dst3 = __lasx_xvldrepl_d(dst + stride_x3, 0); + src0 = __lasx_xvpackev_d(src1,src0); + src2 = __lasx_xvpackev_d(src3,src2); + src0 = __lasx_xvpermi_q(src0, src2, 0x02); + dst0 = __lasx_xvpackev_d(dst1,dst0); + dst2 = __lasx_xvpackev_d(dst3,dst2); + dst0 = __lasx_xvpermi_q(dst0, dst2, 0x02); + dst0 = __lasx_xvavgr_bu(src0, dst0); + LASX_ST_D_4(dst0, 0, 1, 2, 3, dst, stride); +} + +static av_always_inline void avg_width8x4_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride) +{ + __m256i src0, src1, src2, src3; + __m256i dst0, dst1, dst2, dst3; + ptrdiff_t stride_x2 = stride << 1; + ptrdiff_t stride_x3 = stride_x2 + stride; + + src0 = __lasx_xvldrepl_d(src, 0); + src1 = __lasx_xvldrepl_d(src + stride, 0); + src2 = __lasx_xvldrepl_d(src + stride_x2, 0); + src3 = __lasx_xvldrepl_d(src + stride_x3, 0); + dst0 = __lasx_xvldrepl_d(dst, 0); + dst1 = __lasx_xvldrepl_d(dst + stride, 0); + dst2 = __lasx_xvldrepl_d(dst + stride_x2, 0); + dst3 = __lasx_xvldrepl_d(dst + stride_x3, 0); + src0 = __lasx_xvpackev_d(src1,src0); + src2 = __lasx_xvpackev_d(src3,src2); + src0 = __lasx_xvpermi_q(src0, src2, 0x02); + dst0 = __lasx_xvpackev_d(dst1,dst0); + dst2 = __lasx_xvpackev_d(dst3,dst2); + dst0 = __lasx_xvpermi_q(dst0, dst2, 0x02); + dst0 = __lasx_xvavgr_bu(src0, dst0); + LASX_ST_D_4(dst0, 0, 1, 2, 3, dst, stride); +} + +static void avc_chroma_hv_and_aver_dst_8w_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride, + uint32_t coef_hor0, + uint32_t coef_hor1, + uint32_t coef_ver0, + uint32_t coef_ver1, + int32_t height) +{ + if (4 == height) { + avc_chroma_hv_and_aver_dst_8x4_lasx(src, dst, stride, coef_hor0, + coef_hor1, coef_ver0, coef_ver1); + } else if (8 == height) { + avc_chroma_hv_and_aver_dst_8x8_lasx(src, dst, stride, coef_hor0, + coef_hor1, coef_ver0, coef_ver1); + } +} + +static void avc_chroma_hz_and_aver_dst_8w_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride, uint32_t coeff0, + uint32_t coeff1, int32_t height) +{ + if (4 == height) { + avc_chroma_hz_and_aver_dst_8x4_lasx(src, dst, stride, coeff0, coeff1); + } else if (8 == height) { + avc_chroma_hz_and_aver_dst_8x8_lasx(src, dst, stride, coeff0, coeff1); + } +} + +static void avc_chroma_vt_and_aver_dst_8w_lasx(uint8_t *src, uint8_t *dst, + ptrdiff_t stride, uint32_t coeff0, + uint32_t coeff1, int32_t height) +{ + if (4 == height) { + avc_chroma_vt_and_aver_dst_8x4_lasx(src, dst, stride, coeff0, coeff1); + } else if (8 == height) { + avc_chroma_vt_and_aver_dst_8x8_lasx(src, dst, stride, coeff0, coeff1); + } +} + +static void avg_width8_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + int32_t height) +{ + if (8 == height) { + avg_width8x8_lasx(src, dst, stride); + } else if (4 == height) { + avg_width8x4_lasx(src, dst, stride); + } +} + +void ff_avg_h264_chroma_mc8_lasx(uint8_t *dst, uint8_t *src, ptrdiff_t stride, + int height, int x, int y) +{ + av_assert2(x < 8 && y < 8 && x >= 0 && y >= 0); + + if (!(x || y)) { + avg_width8_lasx(src, dst, stride, height); + } else if (x && y) { + avc_chroma_hv_and_aver_dst_8w_lasx(src, dst, stride, x, (8 - x), y, + (8 - y), height); + } else if (x) { + avc_chroma_hz_and_aver_dst_8w_lasx(src, dst, stride, x, (8 - x), height); + } else { + avc_chroma_vt_and_aver_dst_8w_lasx(src, dst, stride, y, (8 - y), height); + } +} diff --git a/libavcodec/loongarch/h264chroma_lasx.h b/libavcodec/loongarch/h264chroma_lasx.h new file mode 100644 index 0000000000..3b70e1dccb --- /dev/null +++ b/libavcodec/loongarch/h264chroma_lasx.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_H264CHROMA_LASX_H +#define AVCODEC_LOONGARCH_H264CHROMA_LASX_H + +#include +#include +#include "libavcodec/h264.h" + +void ff_put_h264_chroma_mc8_lasx(uint8_t *dst, uint8_t *src, ptrdiff_t stride, + int h, int x, int y); +void ff_avg_h264_chroma_mc8_lasx(uint8_t *dst, uint8_t *src, ptrdiff_t stride, + int h, int x, int y); + +#endif /* AVCODEC_LOONGARCH_H264CHROMA_LASX_H */ diff --git a/libavcodec/loongarch/h264dsp_init_loongarch.c b/libavcodec/loongarch/h264dsp_init_loongarch.c new file mode 100644 index 0000000000..f1b927c72e --- /dev/null +++ b/libavcodec/loongarch/h264dsp_init_loongarch.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * Xiwei Gu + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/cpu.h" +#include "h264dsp_lasx.h" + +av_cold void ff_h264dsp_init_loongarch(H264DSPContext *c, const int bit_depth, + const int chroma_format_idc) +{ + int cpu_flags = av_get_cpu_flags(); + + if (have_lasx(cpu_flags)) { + if (chroma_format_idc <= 1) + c->h264_loop_filter_strength = ff_h264_loop_filter_strength_lasx; + if (bit_depth == 8) { + c->h264_v_loop_filter_luma = ff_h264_v_lpf_luma_8_lasx; + c->h264_h_loop_filter_luma = ff_h264_h_lpf_luma_8_lasx; + c->h264_v_loop_filter_luma_intra = ff_h264_v_lpf_luma_intra_8_lasx; + c->h264_h_loop_filter_luma_intra = ff_h264_h_lpf_luma_intra_8_lasx; +// c->h264_h_loop_filter_luma_mbaff = ff_h264_h_loop_filter_luma_mbaff_lasx; +// c->h264_h_loop_filter_luma_mbaff_intra = ff_h264_h_loop_filter_luma_mbaff_intra_lasx; + c->h264_v_loop_filter_chroma = ff_h264_v_lpf_chroma_8_lasx; + + if (chroma_format_idc <= 1) + c->h264_h_loop_filter_chroma = ff_h264_h_lpf_chroma_8_lasx; +// else +// c->h264_h_loop_filter_chroma = ff_h264_h_loop_filter_chroma422_lasx; +// +// if (chroma_format_idc > 1) +// c->h264_h_loop_filter_chroma_mbaff = ff_h264_h_loop_filter_chroma422_mbaff_lasx; +// + c->h264_v_loop_filter_chroma_intra = ff_h264_v_lpf_chroma_intra_8_lasx; + + if (chroma_format_idc <= 1) + c->h264_h_loop_filter_chroma_intra = ff_h264_h_lpf_chroma_intra_8_lasx; + + /* Weighted MC */ + c->weight_h264_pixels_tab[0] = ff_weight_h264_pixels16_8_lasx; + c->weight_h264_pixels_tab[1] = ff_weight_h264_pixels8_8_lasx; + c->weight_h264_pixels_tab[2] = ff_weight_h264_pixels4_8_lasx; + + c->biweight_h264_pixels_tab[0] = ff_biweight_h264_pixels16_8_lasx; + c->biweight_h264_pixels_tab[1] = ff_biweight_h264_pixels8_8_lasx; + c->biweight_h264_pixels_tab[2] = ff_biweight_h264_pixels4_8_lasx; + + c->h264_idct_add = ff_h264_idct_add_lasx; + c->h264_idct8_add = ff_h264_idct8_addblk_lasx; + c->h264_idct_dc_add = ff_h264_idct4x4_addblk_dc_lasx; + c->h264_idct8_dc_add = ff_h264_idct8_dc_addblk_lasx; + c->h264_idct_add16 = ff_h264_idct_add16_lasx; + c->h264_idct8_add4 = ff_h264_idct8_add4_lasx; + + if (chroma_format_idc <= 1) + c->h264_idct_add8 = ff_h264_idct_add8_lasx; + else + c->h264_idct_add8 = ff_h264_idct_add8_422_lasx; + + c->h264_idct_add16intra = ff_h264_idct_add16_intra_lasx; + c->h264_luma_dc_dequant_idct = ff_h264_deq_idct_luma_dc_lasx; + } + } +} diff --git a/libavcodec/loongarch/h264dsp_lasx.c b/libavcodec/loongarch/h264dsp_lasx.c new file mode 100644 index 0000000000..21efa69672 --- /dev/null +++ b/libavcodec/loongarch/h264dsp_lasx.c @@ -0,0 +1,1753 @@ +/* + * Loongson LASX optimized h264dsp + * + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * Xiwei Gu + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lasx.h" +#include "h264dsp_lasx.h" + +#define AVC_LPF_P1_OR_Q1(p0_or_q0_org_in, q0_or_p0_org_in, \ + p1_or_q1_org_in, p2_or_q2_org_in, \ + neg_tc_in, tc_in, p1_or_q1_out) \ +{ \ + __m256i clip3, temp; \ + \ + clip3 = __lasx_xvavgr_hu(p0_or_q0_org_in, \ + q0_or_p0_org_in); \ + temp = __lasx_xvslli_h(p1_or_q1_org_in, 1); \ + clip3 = __lasx_xvsub_h(clip3, temp); \ + clip3 = __lasx_xvavg_h(p2_or_q2_org_in, clip3); \ + LASX_CLIP_H(clip3, neg_tc_in, tc_in); \ + p1_or_q1_out = __lasx_xvadd_h(p1_or_q1_org_in, clip3); \ +} + +#define AVC_LPF_P0Q0(q0_or_p0_org_in, p0_or_q0_org_in, \ + p1_or_q1_org_in, q1_or_p1_org_in, \ + neg_threshold_in, threshold_in, \ + p0_or_q0_out, q0_or_p0_out) \ +{ \ + __m256i q0_sub_p0, p1_sub_q1, delta; \ + \ + q0_sub_p0 = __lasx_xvsub_h(q0_or_p0_org_in, \ + p0_or_q0_org_in); \ + p1_sub_q1 = __lasx_xvsub_h(p1_or_q1_org_in, \ + q1_or_p1_org_in); \ + q0_sub_p0 = __lasx_xvslli_h(q0_sub_p0, 2); \ + p1_sub_q1 = __lasx_xvaddi_hu(p1_sub_q1, 4); \ + delta = __lasx_xvadd_h(q0_sub_p0, p1_sub_q1); \ + delta = __lasx_xvsrai_h(delta, 3); \ + \ + LASX_CLIP_H(delta, neg_threshold_in, threshold_in); \ + \ + p0_or_q0_out = __lasx_xvadd_h(p0_or_q0_org_in, delta); \ + q0_or_p0_out = __lasx_xvsub_h(q0_or_p0_org_in, delta); \ + \ + LASX_CLIP_H_0_255(p0_or_q0_out, p0_or_q0_out); \ + LASX_CLIP_H_0_255(q0_or_p0_out, q0_or_p0_out); \ +} + +void ff_h264_h_lpf_luma_8_lasx(uint8_t *data, int img_width, + int alpha_in, int beta_in, int8_t *tc) +{ + int img_width_2x = img_width << 1; + int img_width_8x = img_width << 3; + __m256i tmp_vec0, bs_vec; + __m256i tc_vec = {0x0101010100000000, 0x0303030302020202, 0x0101010100000000, 0x0303030302020202}; + + tmp_vec0 = __lasx_xvldrepl_w((uint32_t*)tc, 0); + tc_vec = __lasx_xvshuf_b(tmp_vec0, tmp_vec0, tc_vec); + bs_vec = __lasx_xvslti_b(tc_vec, 0); + bs_vec = __lasx_xvxori_b(bs_vec, 255); + bs_vec = __lasx_xvandi_b(bs_vec, 1); + + if (__lasx_xbnz_v(bs_vec)) { + uint8_t *src = data - 4; + __m256i p3_org, p2_org, p1_org, p0_org, q0_org, q1_org, q2_org, q3_org; + __m256i p0_asub_q0, p1_asub_p0, q1_asub_q0, alpha, beta; + __m256i is_less_than, is_less_than_beta, is_less_than_alpha; + __m256i is_bs_greater_than0; + __m256i zero = __lasx_xvldi(0); + + is_bs_greater_than0 = __lasx_xvslt_bu(zero, bs_vec); + + { + uint8_t *src_tmp = src + img_width_8x; + __m256i row0, row1, row2, row3, row4, row5, row6, row7; + __m256i row8, row9, row10, row11, row12, row13, row14, row15; + + LASX_LD_8(src, img_width, row0, row1, row2, row3, + row4, row5, row6, row7); + LASX_LD_8(src_tmp, img_width, row8, row9, row10, row11, + row12, row13, row14, row15); + + LASX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, + row8, row9, row10, row11, + row12, row13, row14, row15, + p3_org, p2_org, p1_org, p0_org, + q0_org, q1_org, q2_org, q3_org); + } + + p0_asub_q0 = __lasx_xvabsd_bu(p0_org, q0_org); + p1_asub_p0 = __lasx_xvabsd_bu(p1_org, p0_org); + q1_asub_q0 = __lasx_xvabsd_bu(q1_org, q0_org); + + alpha = __lasx_xvreplgr2vr_b(alpha_in); + beta = __lasx_xvreplgr2vr_b(beta_in); + + is_less_than_alpha = __lasx_xvslt_bu(p0_asub_q0, alpha); + is_less_than_beta = __lasx_xvslt_bu(p1_asub_p0, beta); + is_less_than = is_less_than_alpha & is_less_than_beta; + is_less_than_beta = __lasx_xvslt_bu(q1_asub_q0, beta); + is_less_than = is_less_than_beta & is_less_than; + is_less_than = is_less_than & is_bs_greater_than0; + + if (__lasx_xbnz_v(is_less_than)) { + __m256i neg_tc_h, tc_h, p1_org_h, p0_org_h, q0_org_h, q1_org_h; + __m256i p2_asub_p0, q2_asub_q0; + + neg_tc_h = __lasx_xvneg_b(tc_vec); + neg_tc_h = __lasx_vext2xv_h_b(neg_tc_h); + tc_h = __lasx_vext2xv_hu_bu(tc_vec); + p1_org_h = __lasx_vext2xv_hu_bu(p1_org); + p0_org_h = __lasx_vext2xv_hu_bu(p0_org); + q0_org_h = __lasx_vext2xv_hu_bu(q0_org); + + p2_asub_p0 = __lasx_xvabsd_bu(p2_org, p0_org); + is_less_than_beta = __lasx_xvslt_bu(p2_asub_p0, beta); + is_less_than_beta = is_less_than_beta & is_less_than; + + if (__lasx_xbnz_v(is_less_than_beta)) { + __m256i p2_org_h, p1_h; + + p2_org_h = __lasx_vext2xv_hu_bu(p2_org); + AVC_LPF_P1_OR_Q1(p0_org_h, q0_org_h, p1_org_h, p2_org_h, + neg_tc_h, tc_h, p1_h); + LASX_PCKEV_B(p1_h, p1_h, p1_h); + + p1_org = __lasx_xvbitsel_v(p1_org, p1_h, is_less_than_beta); + is_less_than_beta = __lasx_xvandi_b(is_less_than_beta, 1); + tc_vec = __lasx_xvadd_b(tc_vec, is_less_than_beta); + } + + q2_asub_q0 = __lasx_xvabsd_bu(q2_org, q0_org); + is_less_than_beta = __lasx_xvslt_bu(q2_asub_q0, beta); + is_less_than_beta = is_less_than_beta & is_less_than; + + q1_org_h = __lasx_vext2xv_hu_bu(q1_org); + + if (__lasx_xbnz_v(is_less_than_beta)) { + __m256i q2_org_h, q1_h; + + q2_org_h = __lasx_vext2xv_hu_bu(q2_org); + AVC_LPF_P1_OR_Q1(p0_org_h, q0_org_h, q1_org_h, q2_org_h, + neg_tc_h, tc_h, q1_h); + LASX_PCKEV_B(q1_h, q1_h, q1_h); + q1_org = __lasx_xvbitsel_v(q1_org, q1_h, is_less_than_beta); + + is_less_than_beta = __lasx_xvandi_b(is_less_than_beta, 1); + tc_vec = __lasx_xvadd_b(tc_vec, is_less_than_beta); + } + + { + __m256i neg_thresh_h, p0_h, q0_h; + + neg_thresh_h = __lasx_xvneg_b(tc_vec); + neg_thresh_h = __lasx_vext2xv_h_b(neg_thresh_h); + tc_h = __lasx_vext2xv_hu_bu(tc_vec); + + AVC_LPF_P0Q0(q0_org_h, p0_org_h, p1_org_h, q1_org_h, + neg_thresh_h, tc_h, p0_h, q0_h); + LASX_PCKEV_B_2(p0_h, p0_h, q0_h, q0_h, p0_h, q0_h); + p0_org = __lasx_xvbitsel_v(p0_org, p0_h, is_less_than); + q0_org = __lasx_xvbitsel_v(q0_org, q0_h, is_less_than); + } + + { + __m256i row0, row1, row2, row3, row4, row5, row6, row7; + + LASX_ILVL_B_4(p1_org, p3_org, p0_org, p2_org, q2_org, q0_org, q3_org, + q1_org, row0, row1, row2, row3); + LASX_ILVLH_B_2(row1, row0, row3, row2, row5, row4, row7, row6); + LASX_ILVLH_W_2(row6, row4, row7, row5, row1, row0, row3, row2); + LASX_PCKEV_Q_4(row0, row0, row1, row1, row2, row2, row3, row3, + row4, row5, row6, row7); + LASX_ST_D_2(row4, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row0, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row5, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row1, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row6, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row2, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row7, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row3, 2, 3, src, img_width); + } + } + } +} + +void ff_h264_v_lpf_luma_8_lasx(uint8_t *data, int img_width, + int alpha_in, int beta_in, int8_t *tc) +{ + int img_width_2x = img_width << 1; + int img_width_3x = img_width + img_width_2x; + __m256i tmp_vec0, bs_vec; + __m256i tc_vec = {0x0101010100000000, 0x0303030302020202, 0x0101010100000000, 0x0303030302020202}; + + tmp_vec0 = __lasx_xvldrepl_w((uint32_t*)tc, 0); + tc_vec = __lasx_xvshuf_b(tmp_vec0, tmp_vec0, tc_vec); + bs_vec = __lasx_xvslti_b(tc_vec, 0); + bs_vec = __lasx_xvxori_b(bs_vec, 255); + bs_vec = __lasx_xvandi_b(bs_vec, 1); + + if (__lasx_xbnz_v(bs_vec)) { + __m256i p2_org, p1_org, p0_org, q0_org, q1_org, q2_org; + __m256i p0_asub_q0, p1_asub_p0, q1_asub_q0, alpha, beta; + __m256i is_less_than, is_less_than_beta, is_less_than_alpha; + __m256i p1_org_h, p0_org_h, q0_org_h, q1_org_h; + __m256i is_bs_greater_than0; + __m256i zero = __lasx_xvldi(0); + + alpha = __lasx_xvreplgr2vr_b(alpha_in); + beta = __lasx_xvreplgr2vr_b(beta_in); + + p2_org = LASX_LD(data - img_width_3x); + p1_org = LASX_LD(data - img_width_2x); + p0_org = LASX_LD(data - img_width); + LASX_LD_2(data, img_width, q0_org, q1_org); + + is_bs_greater_than0 = __lasx_xvslt_bu(zero, bs_vec); + p0_asub_q0 = __lasx_xvabsd_bu(p0_org, q0_org); + p1_asub_p0 = __lasx_xvabsd_bu(p1_org, p0_org); + q1_asub_q0 = __lasx_xvabsd_bu(q1_org, q0_org); + + is_less_than_alpha = __lasx_xvslt_bu(p0_asub_q0, alpha); + is_less_than_beta = __lasx_xvslt_bu(p1_asub_p0, beta); + is_less_than = is_less_than_alpha & is_less_than_beta; + is_less_than_beta = __lasx_xvslt_bu(q1_asub_q0, beta); + is_less_than = is_less_than_beta & is_less_than; + is_less_than = is_less_than & is_bs_greater_than0; + + if (__lasx_xbnz_v(is_less_than)) { + __m256i neg_tc_h, tc_h, p2_asub_p0, q2_asub_q0; + + q2_org = LASX_LD(data + img_width_2x); + + neg_tc_h = __lasx_xvneg_b(tc_vec); + neg_tc_h = __lasx_vext2xv_h_b(neg_tc_h); + tc_h = __lasx_vext2xv_hu_bu(tc_vec); + p1_org_h = __lasx_vext2xv_hu_bu(p1_org); + p0_org_h = __lasx_vext2xv_hu_bu(p0_org); + q0_org_h = __lasx_vext2xv_hu_bu(q0_org); + + p2_asub_p0 = __lasx_xvabsd_bu(p2_org, p0_org); + is_less_than_beta = __lasx_xvslt_bu(p2_asub_p0, beta); + is_less_than_beta = is_less_than_beta & is_less_than; + + if (__lasx_xbnz_v(is_less_than_beta)) { + __m256i p1_h, p2_org_h; + + p2_org_h = __lasx_vext2xv_hu_bu(p2_org); + AVC_LPF_P1_OR_Q1(p0_org_h, q0_org_h, p1_org_h, p2_org_h, + neg_tc_h, tc_h, p1_h); + LASX_PCKEV_B(p1_h, p1_h, p1_h); + p1_h = __lasx_xvbitsel_v(p1_org, p1_h, is_less_than_beta); + p1_org = __lasx_xvpermi_q(p1_org, p1_h, 0x30); + LASX_ST(p1_org, data - img_width_2x); + + is_less_than_beta = __lasx_xvandi_b(is_less_than_beta, 1); + tc_vec = __lasx_xvadd_b(tc_vec, is_less_than_beta); + } + + q2_asub_q0 = __lasx_xvabsd_bu(q2_org, q0_org); + is_less_than_beta = __lasx_xvslt_bu(q2_asub_q0, beta); + is_less_than_beta = is_less_than_beta & is_less_than; + + q1_org_h = __lasx_vext2xv_hu_bu(q1_org); + + if (__lasx_xbnz_v(is_less_than_beta)) { + __m256i q1_h, q2_org_h; + + q2_org_h = __lasx_vext2xv_hu_bu(q2_org); + AVC_LPF_P1_OR_Q1(p0_org_h, q0_org_h, q1_org_h, q2_org_h, + neg_tc_h, tc_h, q1_h); + LASX_PCKEV_B(q1_h, q1_h, q1_h); + q1_h = __lasx_xvbitsel_v(q1_org, q1_h, is_less_than_beta); + q1_org = __lasx_xvpermi_q(q1_org, q1_h, 0x30); + LASX_ST(q1_org, data + img_width); + + is_less_than_beta = __lasx_xvandi_b(is_less_than_beta, 1); + tc_vec = __lasx_xvadd_b(tc_vec, is_less_than_beta); + + } + + { + __m256i neg_thresh_h, p0_h, q0_h; + + neg_thresh_h = __lasx_xvneg_b(tc_vec); + neg_thresh_h = __lasx_vext2xv_h_b(neg_thresh_h); + tc_h = __lasx_vext2xv_hu_bu(tc_vec); + + AVC_LPF_P0Q0(q0_org_h, p0_org_h, p1_org_h, q1_org_h, + neg_thresh_h, tc_h, p0_h, q0_h); + LASX_PCKEV_B_2(p0_h, p0_h, q0_h, q0_h, p0_h, q0_h); + p0_h = __lasx_xvbitsel_v(p0_org, p0_h, is_less_than); + q0_h = __lasx_xvbitsel_v(q0_org, q0_h, is_less_than); + p0_org = __lasx_xvpermi_q(p0_org, p0_h, 0x30); + q0_org = __lasx_xvpermi_q(q0_org, q0_h, 0x30); + LASX_ST(p0_org, data - img_width); + LASX_ST(q0_org, data); + } + } + } +} + +void ff_h264_h_lpf_chroma_8_lasx(uint8_t *data, int img_width, + int alpha_in, int beta_in, int8_t *tc) +{ + __m256i tmp_vec0, bs_vec; + __m256i tc_vec = {0x0303020201010000, 0x0303020201010000, 0x0, 0x0}; + __m256i zero = __lasx_xvldi(0); + + tmp_vec0 = __lasx_xvldrepl_w((uint32_t*)tc, 0); + tc_vec = __lasx_xvshuf_b(tmp_vec0, tmp_vec0, tc_vec); + bs_vec = __lasx_xvslti_b(tc_vec, 0); + bs_vec = __lasx_xvxori_b(bs_vec, 255); + bs_vec = __lasx_xvandi_b(bs_vec, 1); + bs_vec = __lasx_xvpermi_q(zero, bs_vec, 0x30); + + if (__lasx_xbnz_v(bs_vec)) { + uint8_t *src = data - 2; + __m256i p1_org, p0_org, q0_org, q1_org; + __m256i p0_asub_q0, p1_asub_p0, q1_asub_q0, alpha, beta; + __m256i is_less_than, is_less_than_beta, is_less_than_alpha; + __m256i is_bs_greater_than0; + + is_bs_greater_than0 = __lasx_xvslt_bu(zero, bs_vec); + + { + __m256i row0, row1, row2, row3, row4, row5, row6, row7; + + LASX_LD_8(src, img_width, row0, row1, row2, row3, + row4, row5, row6, row7); + /* LASX_TRANSPOSE8x4_B */ + LASX_ILVL_B_4(row2, row0, row3, row1, row6, row4, row7, row5, + p1_org, p0_org, q0_org, q1_org); + LASX_ILVL_B_2(p0_org, p1_org, q1_org, q0_org, row0, row1); + LASX_ILVLH_W_128SV(row1, row0, row3, row2); + p1_org = __lasx_xvpermi_d(row2, 0x00); + p0_org = __lasx_xvpermi_d(row2, 0x55); + q0_org = __lasx_xvpermi_d(row3, 0x00); + q1_org = __lasx_xvpermi_d(row3, 0x55); + } + + p0_asub_q0 = __lasx_xvabsd_bu(p0_org, q0_org); + p1_asub_p0 = __lasx_xvabsd_bu(p1_org, p0_org); + q1_asub_q0 = __lasx_xvabsd_bu(q1_org, q0_org); + + alpha = __lasx_xvreplgr2vr_b(alpha_in); + beta = __lasx_xvreplgr2vr_b(beta_in); + + is_less_than_alpha = __lasx_xvslt_bu(p0_asub_q0, alpha); + is_less_than_beta = __lasx_xvslt_bu(p1_asub_p0, beta); + is_less_than = is_less_than_alpha & is_less_than_beta; + is_less_than_beta = __lasx_xvslt_bu(q1_asub_q0, beta); + is_less_than = is_less_than_beta & is_less_than; + is_less_than = is_less_than & is_bs_greater_than0; + + if (__lasx_xbnz_v(is_less_than)) { + __m256i p1_org_h, p0_org_h, q0_org_h, q1_org_h; + + p1_org_h = __lasx_vext2xv_hu_bu(p1_org); + p0_org_h = __lasx_vext2xv_hu_bu(p0_org); + q0_org_h = __lasx_vext2xv_hu_bu(q0_org); + q1_org_h = __lasx_vext2xv_hu_bu(q1_org); + + { + __m256i tc_h, neg_thresh_h, p0_h, q0_h; + + neg_thresh_h = __lasx_xvneg_b(tc_vec); + neg_thresh_h = __lasx_vext2xv_h_b(neg_thresh_h); + tc_h = __lasx_vext2xv_hu_bu(tc_vec); + + AVC_LPF_P0Q0(q0_org_h, p0_org_h, p1_org_h, q1_org_h, + neg_thresh_h, tc_h, p0_h, q0_h); + LASX_PCKEV_B_2(p0_h, p0_h, q0_h, q0_h, p0_h, q0_h); + p0_org = __lasx_xvbitsel_v(p0_org, p0_h, is_less_than); + q0_org = __lasx_xvbitsel_v(q0_org, q0_h, is_less_than); + } + + p0_org = __lasx_xvilvl_b(q0_org, p0_org); + src = data - 1; + __lasx_xvstelm_h(p0_org, src, 0, 0); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 1); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 2); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 3); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 4); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 5); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 6); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 7); + } + } +} + +void ff_h264_v_lpf_chroma_8_lasx(uint8_t *data, int img_width, + int alpha_in, int beta_in, int8_t *tc) +{ + int img_width_2x = img_width << 1; + __m256i tmp_vec0, bs_vec; + __m256i tc_vec = {0x0303020201010000, 0x0303020201010000, 0x0, 0x0}; + __m256i zero = __lasx_xvldi(0); + + tmp_vec0 = __lasx_xvldrepl_w((uint32_t*)tc, 0); + tc_vec = __lasx_xvshuf_b(tmp_vec0, tmp_vec0, tc_vec); + bs_vec = __lasx_xvslti_b(tc_vec, 0); + bs_vec = __lasx_xvxori_b(bs_vec, 255); + bs_vec = __lasx_xvandi_b(bs_vec, 1); + bs_vec = __lasx_xvpermi_q(zero, bs_vec, 0x30); + + if (__lasx_xbnz_v(bs_vec)) { + __m256i p1_org, p0_org, q0_org, q1_org; + __m256i p0_asub_q0, p1_asub_p0, q1_asub_q0, alpha, beta; + __m256i is_less_than, is_less_than_beta, is_less_than_alpha; + __m256i is_bs_greater_than0; + + alpha = __lasx_xvreplgr2vr_b(alpha_in); + beta = __lasx_xvreplgr2vr_b(beta_in); + + p1_org = LASX_LD(data - img_width_2x); + p0_org = LASX_LD(data - img_width); + LASX_LD_2(data, img_width, q0_org, q1_org); + + is_bs_greater_than0 = __lasx_xvslt_bu(zero, bs_vec); + p0_asub_q0 = __lasx_xvabsd_bu(p0_org, q0_org); + p1_asub_p0 = __lasx_xvabsd_bu(p1_org, p0_org); + q1_asub_q0 = __lasx_xvabsd_bu(q1_org, q0_org); + + is_less_than_alpha = __lasx_xvslt_bu(p0_asub_q0, alpha); + is_less_than_beta = __lasx_xvslt_bu(p1_asub_p0, beta); + is_less_than = is_less_than_alpha & is_less_than_beta; + is_less_than_beta = __lasx_xvslt_bu(q1_asub_q0, beta); + is_less_than = is_less_than_beta & is_less_than; + is_less_than = is_less_than & is_bs_greater_than0; + + if (__lasx_xbnz_v(is_less_than)) { + __m256i p1_org_h, p0_org_h, q0_org_h, q1_org_h; + + p1_org_h = __lasx_vext2xv_hu_bu(p1_org); + p0_org_h = __lasx_vext2xv_hu_bu(p0_org); + q0_org_h = __lasx_vext2xv_hu_bu(q0_org); + q1_org_h = __lasx_vext2xv_hu_bu(q1_org); + + { + __m256i neg_thresh_h, tc_h, p0_h, q0_h; + + neg_thresh_h = __lasx_xvneg_b(tc_vec); + neg_thresh_h = __lasx_vext2xv_h_b(neg_thresh_h); + tc_h = __lasx_vext2xv_hu_bu(tc_vec); + + AVC_LPF_P0Q0(q0_org_h, p0_org_h, p1_org_h, q1_org_h, + neg_thresh_h, tc_h, p0_h, q0_h); + LASX_PCKEV_B_2(p0_h, p0_h, q0_h, q0_h, p0_h, q0_h); + p0_h = __lasx_xvbitsel_v(p0_org, p0_h, is_less_than); + q0_h = __lasx_xvbitsel_v(q0_org, q0_h, is_less_than); + LASX_ST_D(p0_h, 0, data - img_width); + LASX_ST_D(q0_h, 0, data); + } + } + } +} + +#define AVC_LPF_P0P1P2_OR_Q0Q1Q2(p3_or_q3_org_in, p0_or_q0_org_in, \ + q3_or_p3_org_in, p1_or_q1_org_in, \ + p2_or_q2_org_in, q1_or_p1_org_in, \ + p0_or_q0_out, p1_or_q1_out, p2_or_q2_out) \ +{ \ + __m256i threshold; \ + __m256i const2, const3 = __lasx_xvldi(0); \ + \ + const2 = __lasx_xvaddi_hu(const3, 2); \ + const3 = __lasx_xvaddi_hu(const3, 3); \ + threshold = __lasx_xvadd_h(p0_or_q0_org_in, q3_or_p3_org_in); \ + threshold = __lasx_xvadd_h(p1_or_q1_org_in, threshold); \ + \ + p0_or_q0_out = __lasx_xvslli_h(threshold, 1); \ + p0_or_q0_out = __lasx_xvadd_h(p0_or_q0_out, p2_or_q2_org_in); \ + p0_or_q0_out = __lasx_xvadd_h(p0_or_q0_out, q1_or_p1_org_in); \ + p0_or_q0_out = __lasx_xvsrar_h(p0_or_q0_out, const3); \ + \ + p1_or_q1_out = __lasx_xvadd_h(p2_or_q2_org_in, threshold); \ + p1_or_q1_out = __lasx_xvsrar_h(p1_or_q1_out, const2); \ + \ + p2_or_q2_out = __lasx_xvmul_h(p2_or_q2_org_in, const3); \ + p2_or_q2_out = __lasx_xvadd_h(p2_or_q2_out, p3_or_q3_org_in); \ + p2_or_q2_out = __lasx_xvadd_h(p2_or_q2_out, p3_or_q3_org_in); \ + p2_or_q2_out = __lasx_xvadd_h(p2_or_q2_out, threshold); \ + p2_or_q2_out = __lasx_xvsrar_h(p2_or_q2_out, const3); \ +} + +/* data[-u32_img_width] = (uint8_t)((2 * p1 + p0 + q1 + 2) >> 2); */ +#define AVC_LPF_P0_OR_Q0(p0_or_q0_org_in, q1_or_p1_org_in, \ + p1_or_q1_org_in, p0_or_q0_out) \ +{ \ + __m256i const2 = __lasx_xvldi(0); \ + const2 = __lasx_xvaddi_hu(const2, 2); \ + p0_or_q0_out = __lasx_xvadd_h(p0_or_q0_org_in, q1_or_p1_org_in); \ + p0_or_q0_out = __lasx_xvadd_h(p0_or_q0_out, p1_or_q1_org_in); \ + p0_or_q0_out = __lasx_xvadd_h(p0_or_q0_out, p1_or_q1_org_in); \ + p0_or_q0_out = __lasx_xvsrar_h(p0_or_q0_out, const2); \ +} + +void ff_h264_h_lpf_luma_intra_8_lasx(uint8_t *data, int img_width, + int alpha_in, int beta_in) +{ + int img_width_2x = img_width << 1; + uint8_t *src = data - 4; + __m256i p0_asub_q0, p1_asub_p0, q1_asub_q0, alpha, beta; + __m256i is_less_than, is_less_than_beta, is_less_than_alpha; + __m256i p3_org, p2_org, p1_org, p0_org, q0_org, q1_org, q2_org, q3_org; + __m256i zero = __lasx_xvldi(0); + + { + __m256i row0, row1, row2, row3, row4, row5, row6, row7; + __m256i row8, row9, row10, row11, row12, row13, row14, row15; + + LASX_LD_8(src, img_width, row0, row1, row2, row3, + row4, row5, row6, row7); + src += img_width << 3; + LASX_LD_8(src, img_width, row8, row9, row10, row11, + row12, row13, row14, row15); + + LASX_TRANSPOSE16x8_B(row0, row1, row2, row3, + row4, row5, row6, row7, + row8, row9, row10, row11, + row12, row13, row14, row15, + p3_org, p2_org, p1_org, p0_org, + q0_org, q1_org, q2_org, q3_org); + } + + alpha = __lasx_xvreplgr2vr_b(alpha_in); + beta = __lasx_xvreplgr2vr_b(beta_in); + p0_asub_q0 = __lasx_xvabsd_bu(p0_org, q0_org); + p1_asub_p0 = __lasx_xvabsd_bu(p1_org, p0_org); + q1_asub_q0 = __lasx_xvabsd_bu(q1_org, q0_org); + + is_less_than_alpha = __lasx_xvslt_bu(p0_asub_q0, alpha); + is_less_than_beta = __lasx_xvslt_bu(p1_asub_p0, beta); + is_less_than = is_less_than_beta & is_less_than_alpha; + is_less_than_beta = __lasx_xvslt_bu(q1_asub_q0, beta); + is_less_than = is_less_than_beta & is_less_than; + is_less_than = __lasx_xvpermi_q(zero, is_less_than, 0x30); + + if (__lasx_xbnz_v(is_less_than)) { + __m256i p2_asub_p0, q2_asub_q0, p0_h, q0_h, negate_is_less_than_beta; + __m256i p1_org_h, p0_org_h, q0_org_h, q1_org_h; + __m256i less_alpha_shift2_add2 = __lasx_xvsrli_b(alpha, 2); + + less_alpha_shift2_add2 = __lasx_xvaddi_bu(less_alpha_shift2_add2, 2); + less_alpha_shift2_add2 = __lasx_xvslt_bu(p0_asub_q0, less_alpha_shift2_add2); + + p1_org_h = __lasx_vext2xv_hu_bu(p1_org); + p0_org_h = __lasx_vext2xv_hu_bu(p0_org); + q0_org_h = __lasx_vext2xv_hu_bu(q0_org); + q1_org_h = __lasx_vext2xv_hu_bu(q1_org); + + p2_asub_p0 = __lasx_xvabsd_bu(p2_org, p0_org); + is_less_than_beta = __lasx_xvslt_bu(p2_asub_p0, beta); + is_less_than_beta = is_less_than_beta & less_alpha_shift2_add2; + negate_is_less_than_beta = __lasx_xvxori_b(is_less_than_beta, 0xff); + is_less_than_beta = is_less_than_beta & is_less_than; + negate_is_less_than_beta = negate_is_less_than_beta & is_less_than; + + /* combine and store */ + if (__lasx_xbnz_v(is_less_than_beta)) { + __m256i p2_org_h, p3_org_h, p1_h, p2_h; + + p2_org_h = __lasx_vext2xv_hu_bu(p2_org); + p3_org_h = __lasx_vext2xv_hu_bu(p3_org); + + AVC_LPF_P0P1P2_OR_Q0Q1Q2(p3_org_h, p0_org_h, q0_org_h, p1_org_h, + p2_org_h, q1_org_h, p0_h, p1_h, p2_h); + + LASX_PCKEV_B(p0_h, p0_h, p0_h); + LASX_PCKEV_B_2(p1_h, p1_h, p2_h, p2_h, p1_h, p2_h); + p0_org = __lasx_xvbitsel_v(p0_org, p0_h, is_less_than_beta); + p1_org = __lasx_xvbitsel_v(p1_org, p1_h, is_less_than_beta); + p2_org = __lasx_xvbitsel_v(p2_org, p2_h, is_less_than_beta); + } + + AVC_LPF_P0_OR_Q0(p0_org_h, q1_org_h, p1_org_h, p0_h); + /* combine */ + LASX_PCKEV_B(p0_h, p0_h, p0_h); + p0_org = __lasx_xvbitsel_v(p0_org, p0_h, negate_is_less_than_beta); + + /* if (tmpFlag && (unsigned)ABS(q2-q0) < thresholds->beta_in) */ + q2_asub_q0 = __lasx_xvabsd_bu(q2_org, q0_org); + is_less_than_beta = __lasx_xvslt_bu(q2_asub_q0, beta); + is_less_than_beta = is_less_than_beta & less_alpha_shift2_add2; + negate_is_less_than_beta = __lasx_xvxori_b(is_less_than_beta, 0xff); + is_less_than_beta = is_less_than_beta & is_less_than; + negate_is_less_than_beta = negate_is_less_than_beta & is_less_than; + + /* combine and store */ + if (__lasx_xbnz_v(is_less_than_beta)) { + __m256i q2_org_h, q3_org_h, q1_h, q2_h; + + q2_org_h = __lasx_vext2xv_hu_bu(q2_org); + q3_org_h = __lasx_vext2xv_hu_bu(q3_org); + + AVC_LPF_P0P1P2_OR_Q0Q1Q2(q3_org_h, q0_org_h, p0_org_h, q1_org_h, + q2_org_h, p1_org_h, q0_h, q1_h, q2_h); + + LASX_PCKEV_B(q0_h, q0_h, q0_h); + LASX_PCKEV_B_2(q1_h, q1_h, q2_h, q2_h, q1_h, q2_h); + q0_org = __lasx_xvbitsel_v(q0_org, q0_h, is_less_than_beta); + q1_org = __lasx_xvbitsel_v(q1_org, q1_h, is_less_than_beta); + q2_org = __lasx_xvbitsel_v(q2_org, q2_h, is_less_than_beta); + + } + + AVC_LPF_P0_OR_Q0(q0_org_h, p1_org_h, q1_org_h, q0_h); + + /* combine */ + LASX_PCKEV_B(q0_h, q0_h, q0_h); + q0_org = __lasx_xvbitsel_v(q0_org, q0_h, negate_is_less_than_beta); + + /* transpose and store */ + { + __m256i row0, row1, row2, row3, row4, row5, row6, row7; + + LASX_ILVL_B_4(p1_org, p3_org, p0_org, p2_org, q2_org, q0_org, q3_org, + q1_org, row0, row1, row2, row3); + LASX_ILVLH_B_2(row1, row0, row3, row2, row5, row4, row7, row6); + LASX_ILVLH_W_2(row6, row4, row7, row5, row1, row0, row3, row2); + LASX_PCKEV_Q_4(row0, row0, row1, row1, row2, row2, row3, row3, + row4, row5, row6, row7); + src = data - 4; + LASX_ST_D_2(row4, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row0, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row5, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row1, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row6, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row2, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row7, 2, 3, src, img_width); + src += img_width_2x; + LASX_ST_D_2(row3, 2, 3, src, img_width); + } + } +} + +void ff_h264_v_lpf_luma_intra_8_lasx(uint8_t *data, int img_width, + int alpha_in, int beta_in) +{ + int img_width_2x = img_width << 1; + uint8_t *src = data - img_width_2x; + __m256i p0_asub_q0, p1_asub_p0, q1_asub_q0, alpha, beta; + __m256i is_less_than, is_less_than_beta, is_less_than_alpha; + __m256i p1_org, p0_org, q0_org, q1_org; + __m256i zero = __lasx_xvldi(0); + + LASX_LD_4(src, img_width, p1_org, p0_org, q0_org, q1_org) + alpha = __lasx_xvreplgr2vr_b(alpha_in); + beta = __lasx_xvreplgr2vr_b(beta_in); + p0_asub_q0 = __lasx_xvabsd_bu(p0_org, q0_org); + p1_asub_p0 = __lasx_xvabsd_bu(p1_org, p0_org); + q1_asub_q0 = __lasx_xvabsd_bu(q1_org, q0_org); + + is_less_than_alpha = __lasx_xvslt_bu(p0_asub_q0, alpha); + is_less_than_beta = __lasx_xvslt_bu(p1_asub_p0, beta); + is_less_than = is_less_than_beta & is_less_than_alpha; + is_less_than_beta = __lasx_xvslt_bu(q1_asub_q0, beta); + is_less_than = is_less_than_beta & is_less_than; + is_less_than = __lasx_xvpermi_q(zero, is_less_than, 0x30); + + if (__lasx_xbnz_v(is_less_than)) { + __m256i p2_asub_p0, q2_asub_q0, p0_h, q0_h, negate_is_less_than_beta; + __m256i p1_org_h, p0_org_h, q0_org_h, q1_org_h; + __m256i p2_org = LASX_LD(src - img_width); + __m256i q2_org = LASX_LD(data + img_width_2x); + __m256i less_alpha_shift2_add2 = __lasx_xvsrli_b(alpha, 2); + less_alpha_shift2_add2 = __lasx_xvaddi_bu(less_alpha_shift2_add2, 2); + less_alpha_shift2_add2 = __lasx_xvslt_bu(p0_asub_q0, less_alpha_shift2_add2); + + p1_org_h = __lasx_vext2xv_hu_bu(p1_org); + p0_org_h = __lasx_vext2xv_hu_bu(p0_org); + q0_org_h = __lasx_vext2xv_hu_bu(q0_org); + q1_org_h = __lasx_vext2xv_hu_bu(q1_org); + + p2_asub_p0 = __lasx_xvabsd_bu(p2_org, p0_org); + is_less_than_beta = __lasx_xvslt_bu(p2_asub_p0, beta); + is_less_than_beta = is_less_than_beta & less_alpha_shift2_add2; + negate_is_less_than_beta = __lasx_xvxori_b(is_less_than_beta, 0xff); + is_less_than_beta = is_less_than_beta & is_less_than; + negate_is_less_than_beta = negate_is_less_than_beta & is_less_than; + + /* combine and store */ + if (__lasx_xbnz_v(is_less_than_beta)) { + __m256i p2_org_h, p3_org_h, p1_h, p2_h; + __m256i p3_org = LASX_LD(src - img_width_2x); + + p2_org_h = __lasx_vext2xv_hu_bu(p2_org); + p3_org_h = __lasx_vext2xv_hu_bu(p3_org); + + AVC_LPF_P0P1P2_OR_Q0Q1Q2(p3_org_h, p0_org_h, q0_org_h, p1_org_h, + p2_org_h, q1_org_h, p0_h, p1_h, p2_h); + + LASX_PCKEV_B(p0_h, p0_h, p0_h); + LASX_PCKEV_B_2(p1_h, p1_h, p2_h, p2_h, p1_h, p2_h); + p0_org = __lasx_xvbitsel_v(p0_org, p0_h, is_less_than_beta); + p1_org = __lasx_xvbitsel_v(p1_org, p1_h, is_less_than_beta); + p2_org = __lasx_xvbitsel_v(p2_org, p2_h, is_less_than_beta); + + LASX_ST(p1_org, src); + LASX_ST(p2_org, src - img_width); + } + + AVC_LPF_P0_OR_Q0(p0_org_h, q1_org_h, p1_org_h, p0_h); + /* combine */ + LASX_PCKEV_B(p0_h, p0_h, p0_h); + p0_org = __lasx_xvbitsel_v(p0_org, p0_h, negate_is_less_than_beta); + LASX_ST(p0_org, data - img_width); + + /* if (tmpFlag && (unsigned)ABS(q2-q0) < thresholds->beta_in) */ + q2_asub_q0 = __lasx_xvabsd_bu(q2_org, q0_org); + is_less_than_beta = __lasx_xvslt_bu(q2_asub_q0, beta); + is_less_than_beta = is_less_than_beta & less_alpha_shift2_add2; + negate_is_less_than_beta = __lasx_xvxori_b(is_less_than_beta, 0xff); + is_less_than_beta = is_less_than_beta & is_less_than; + negate_is_less_than_beta = negate_is_less_than_beta & is_less_than; + + /* combine and store */ + if (__lasx_xbnz_v(is_less_than_beta)) { + __m256i q2_org_h, q3_org_h, q1_h, q2_h; + __m256i q3_org = LASX_LD(data + img_width_2x + img_width); + + q2_org_h = __lasx_vext2xv_hu_bu(q2_org); + q3_org_h = __lasx_vext2xv_hu_bu(q3_org); + + AVC_LPF_P0P1P2_OR_Q0Q1Q2(q3_org_h, q0_org_h, p0_org_h, q1_org_h, + q2_org_h, p1_org_h, q0_h, q1_h, q2_h); + + LASX_PCKEV_B(q0_h, q0_h, q0_h); + LASX_PCKEV_B_2(q1_h, q1_h, q2_h, q2_h, q1_h, q2_h); + q0_org = __lasx_xvbitsel_v(q0_org, q0_h, is_less_than_beta); + q1_org = __lasx_xvbitsel_v(q1_org, q1_h, is_less_than_beta); + q2_org = __lasx_xvbitsel_v(q2_org, q2_h, is_less_than_beta); + + LASX_ST(q1_org, data + img_width); + LASX_ST(q2_org, data + img_width_2x); + } + + AVC_LPF_P0_OR_Q0(q0_org_h, p1_org_h, q1_org_h, q0_h); + + /* combine */ + LASX_PCKEV_B(q0_h, q0_h, q0_h); + q0_org = __lasx_xvbitsel_v(q0_org, q0_h, negate_is_less_than_beta); + + LASX_ST(q0_org, data); + } +} + +void ff_h264_h_lpf_chroma_intra_8_lasx(uint8_t *data, int img_width, + int alpha_in, int beta_in) +{ + uint8_t *src = data - 2; + __m256i p1_org, p0_org, q0_org, q1_org; + __m256i p0_asub_q0, p1_asub_p0, q1_asub_q0, alpha, beta; + __m256i is_less_than, is_less_than_beta, is_less_than_alpha; + + { + __m256i row0, row1, row2, row3, row4, row5, row6, row7; + + LASX_LD_8(src, img_width, row0, row1, row2, row3, + row4, row5, row6, row7); + + /* LASX_TRANSPOSE8x4_B */ + LASX_ILVL_B_4(row2, row0, row3, row1, row6, row4, row7, row5, + p1_org, p0_org, q0_org, q1_org); + LASX_ILVL_B_2(p0_org, p1_org, q1_org, q0_org, row0, row1); + LASX_ILVLH_W_128SV(row1, row0, row3, row2); + p1_org = __lasx_xvpermi_d(row2, 0x00); + p0_org = __lasx_xvpermi_d(row2, 0x55); + q0_org = __lasx_xvpermi_d(row3, 0x00); + q1_org = __lasx_xvpermi_d(row3, 0x55); + } + + alpha = __lasx_xvreplgr2vr_b(alpha_in); + beta = __lasx_xvreplgr2vr_b(beta_in); + + p0_asub_q0 = __lasx_xvabsd_bu(p0_org, q0_org); + p1_asub_p0 = __lasx_xvabsd_bu(p1_org, p0_org); + q1_asub_q0 = __lasx_xvabsd_bu(q1_org, q0_org); + + is_less_than_alpha = __lasx_xvslt_bu(p0_asub_q0, alpha); + is_less_than_beta = __lasx_xvslt_bu(p1_asub_p0, beta); + is_less_than = is_less_than_alpha & is_less_than_beta; + is_less_than_beta = __lasx_xvslt_bu(q1_asub_q0, beta); + is_less_than = is_less_than_beta & is_less_than; + + if (__lasx_xbnz_v(is_less_than)) { + __m256i p0_h, q0_h, p1_org_h, p0_org_h, q0_org_h, q1_org_h; + + p1_org_h = __lasx_vext2xv_hu_bu(p1_org); + p0_org_h = __lasx_vext2xv_hu_bu(p0_org); + q0_org_h = __lasx_vext2xv_hu_bu(q0_org); + q1_org_h = __lasx_vext2xv_hu_bu(q1_org); + + AVC_LPF_P0_OR_Q0(p0_org_h, q1_org_h, p1_org_h, p0_h); + AVC_LPF_P0_OR_Q0(q0_org_h, p1_org_h, q1_org_h, q0_h); + LASX_PCKEV_B_2(p0_h, p0_h, q0_h, q0_h, p0_h, q0_h); + p0_org = __lasx_xvbitsel_v(p0_org, p0_h, is_less_than); + q0_org = __lasx_xvbitsel_v(q0_org, q0_h, is_less_than); + } + p0_org = __lasx_xvilvl_b(q0_org, p0_org); + src = data - 1; + __lasx_xvstelm_h(p0_org, src, 0, 0); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 1); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 2); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 3); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 4); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 5); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 6); + src += img_width; + __lasx_xvstelm_h(p0_org, src, 0, 7); +} + +void ff_h264_v_lpf_chroma_intra_8_lasx(uint8_t *data, int img_width, + int alpha_in, int beta_in) +{ + int img_width_2x = img_width << 1; + __m256i p1_org, p0_org, q0_org, q1_org; + __m256i p0_asub_q0, p1_asub_p0, q1_asub_q0, alpha, beta; + __m256i is_less_than, is_less_than_beta, is_less_than_alpha; + + alpha = __lasx_xvreplgr2vr_b(alpha_in); + beta = __lasx_xvreplgr2vr_b(beta_in); + + p1_org = LASX_LD(data - img_width_2x); + p0_org = LASX_LD(data - img_width); + LASX_LD_2(data, img_width, q0_org, q1_org); + + p0_asub_q0 = __lasx_xvabsd_bu(p0_org, q0_org); + p1_asub_p0 = __lasx_xvabsd_bu(p1_org, p0_org); + q1_asub_q0 = __lasx_xvabsd_bu(q1_org, q0_org); + + is_less_than_alpha = __lasx_xvslt_bu(p0_asub_q0, alpha); + is_less_than_beta = __lasx_xvslt_bu(p1_asub_p0, beta); + is_less_than = is_less_than_alpha & is_less_than_beta; + is_less_than_beta = __lasx_xvslt_bu(q1_asub_q0, beta); + is_less_than = is_less_than_beta & is_less_than; + + if (__lasx_xbnz_v(is_less_than)) { + __m256i p0_h, q0_h, p1_org_h, p0_org_h, q0_org_h, q1_org_h; + + p1_org_h = __lasx_vext2xv_hu_bu(p1_org); + p0_org_h = __lasx_vext2xv_hu_bu(p0_org); + q0_org_h = __lasx_vext2xv_hu_bu(q0_org); + q1_org_h = __lasx_vext2xv_hu_bu(q1_org); + + AVC_LPF_P0_OR_Q0(p0_org_h, q1_org_h, p1_org_h, p0_h); + AVC_LPF_P0_OR_Q0(q0_org_h, p1_org_h, q1_org_h, q0_h); + LASX_PCKEV_B_2(p0_h, p0_h, q0_h, q0_h, p0_h, q0_h); + p0_h = __lasx_xvbitsel_v(p0_org, p0_h, is_less_than); + q0_h = __lasx_xvbitsel_v(q0_org, q0_h, is_less_than); + LASX_ST_D(p0_h, 0, data - img_width); + LASX_ST_D(q0_h, 0, data); + } +} + +void ff_biweight_h264_pixels16_8_lasx(uint8_t *dst, uint8_t *src, + ptrdiff_t stride, int height, + int log2_denom, int weight_dst, + int weight_src, int offset_in) +{ + __m256i wgt; + __m256i src0, src1, src2, src3; + __m256i dst0, dst1, dst2, dst3; + __m256i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m256i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m256i denom, offset; + + offset_in = (unsigned) ((offset_in + 1) | 1) << log2_denom; + offset_in += ((weight_src + weight_dst) << 7); + log2_denom += 1; + + tmp0 = __lasx_xvreplgr2vr_b(weight_src); + tmp1 = __lasx_xvreplgr2vr_b(weight_dst); + wgt = __lasx_xvilvh_b(tmp1, tmp0); + offset = __lasx_xvreplgr2vr_h(offset_in); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_8(src, stride, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + src += 8 * stride; + LASX_PCKEV_Q_4(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + src0, src1, src2, src3); + LASX_LD_8(dst, stride, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + LASX_PCKEV_Q_4(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + dst0, dst1, dst2, dst3); + + LASX_XORI_B_8_128(src0, src1, src2, src3, dst0, dst1, dst2, dst3); + LASX_ILVLH_B_4(dst0, src0, dst1, src1, dst2, src2, dst3, src3, + vec1, vec0, vec3, vec2, vec5, vec4, vec7, vec6); + + LASX_DP2ADD_H_B(offset, wgt, vec0, tmp0); + LASX_DP2ADD_H_B(offset, wgt, vec1, tmp1); + LASX_DP2ADD_H_B(offset, wgt, vec2, tmp2); + LASX_DP2ADD_H_B(offset, wgt, vec3, tmp3); + LASX_DP2ADD_H_B(offset, wgt, vec4, tmp4); + LASX_DP2ADD_H_B(offset, wgt, vec5, tmp5); + LASX_DP2ADD_H_B(offset, wgt, vec6, tmp6); + LASX_DP2ADD_H_B(offset, wgt, vec7, tmp7); + + tmp0 = __lasx_xvsra_h(tmp0, denom); + tmp1 = __lasx_xvsra_h(tmp1, denom); + tmp2 = __lasx_xvsra_h(tmp2, denom); + tmp3 = __lasx_xvsra_h(tmp3, denom); + tmp4 = __lasx_xvsra_h(tmp4, denom); + tmp5 = __lasx_xvsra_h(tmp5, denom); + tmp6 = __lasx_xvsra_h(tmp6, denom); + tmp7 = __lasx_xvsra_h(tmp7, denom); + + LASX_CLIP_H_0_255_4(tmp0, tmp1, tmp2, tmp3, + tmp0, tmp1, tmp2, tmp3); + LASX_CLIP_H_0_255_4(tmp4, tmp5, tmp6, tmp7, + tmp4, tmp5, tmp6, tmp7); + LASX_PCKEV_B_4_128SV(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + dst0, dst1, dst2, dst3); + __lasx_xvstelm_d(dst0, dst, 0, 0); + __lasx_xvstelm_d(dst0, dst, 8, 2); + dst += stride; + __lasx_xvstelm_d(dst0, dst, 0, 1); + __lasx_xvstelm_d(dst0, dst, 8, 3); + dst += stride; + __lasx_xvstelm_d(dst1, dst, 0, 0); + __lasx_xvstelm_d(dst1, dst, 8, 2); + dst += stride; + __lasx_xvstelm_d(dst1, dst, 0, 1); + __lasx_xvstelm_d(dst1, dst, 8, 3); + dst += stride; + __lasx_xvstelm_d(dst2, dst, 0, 0); + __lasx_xvstelm_d(dst2, dst, 8, 2); + dst += stride; + __lasx_xvstelm_d(dst2, dst, 0, 1); + __lasx_xvstelm_d(dst2, dst, 8, 3); + dst += stride; + __lasx_xvstelm_d(dst3, dst, 0, 0); + __lasx_xvstelm_d(dst3, dst, 8, 2); + dst += stride; + __lasx_xvstelm_d(dst3, dst, 0, 1); + __lasx_xvstelm_d(dst3, dst, 8, 3); + dst += stride; + + if (16 == height) { + LASX_LD_8(src, stride, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + LASX_PCKEV_Q_4(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + src0, src1, src2, src3); + LASX_LD_8(dst, stride, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + LASX_PCKEV_Q_4(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + dst0, dst1, dst2, dst3); + + LASX_XORI_B_8_128(src0, src1, src2, src3, dst0, dst1, dst2, dst3); + LASX_ILVLH_B_4(dst0, src0, dst1, src1, dst2, src2, dst3, src3, + vec1, vec0, vec3, vec2, vec5, vec4, vec7, vec6); + + LASX_DP2ADD_H_B(offset, wgt, vec0, tmp0); + LASX_DP2ADD_H_B(offset, wgt, vec1, tmp1); + LASX_DP2ADD_H_B(offset, wgt, vec2, tmp2); + LASX_DP2ADD_H_B(offset, wgt, vec3, tmp3); + LASX_DP2ADD_H_B(offset, wgt, vec4, tmp4); + LASX_DP2ADD_H_B(offset, wgt, vec5, tmp5); + LASX_DP2ADD_H_B(offset, wgt, vec6, tmp6); + LASX_DP2ADD_H_B(offset, wgt, vec7, tmp7); + + tmp0 = __lasx_xvsra_h(tmp0, denom); + tmp1 = __lasx_xvsra_h(tmp1, denom); + tmp2 = __lasx_xvsra_h(tmp2, denom); + tmp3 = __lasx_xvsra_h(tmp3, denom); + tmp4 = __lasx_xvsra_h(tmp4, denom); + tmp5 = __lasx_xvsra_h(tmp5, denom); + tmp6 = __lasx_xvsra_h(tmp6, denom); + tmp7 = __lasx_xvsra_h(tmp7, denom); + + LASX_CLIP_H_0_255_4(tmp0, tmp1, tmp2, tmp3, + tmp0, tmp1, tmp2, tmp3); + LASX_CLIP_H_0_255_4(tmp4, tmp5, tmp6, tmp7, + tmp4, tmp5, tmp6, tmp7); + LASX_PCKEV_B_4_128SV(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + dst0, dst1, dst2, dst3); + __lasx_xvstelm_d(dst0, dst, 0, 0); + __lasx_xvstelm_d(dst0, dst, 8, 2); + dst += stride; + __lasx_xvstelm_d(dst0, dst, 0, 1); + __lasx_xvstelm_d(dst0, dst, 8, 3); + dst += stride; + __lasx_xvstelm_d(dst1, dst, 0, 0); + __lasx_xvstelm_d(dst1, dst, 8, 2); + dst += stride; + __lasx_xvstelm_d(dst1, dst, 0, 1); + __lasx_xvstelm_d(dst1, dst, 8, 3); + dst += stride; + __lasx_xvstelm_d(dst2, dst, 0, 0); + __lasx_xvstelm_d(dst2, dst, 8, 2); + dst += stride; + __lasx_xvstelm_d(dst2, dst, 0, 1); + __lasx_xvstelm_d(dst2, dst, 8, 3); + dst += stride; + __lasx_xvstelm_d(dst3, dst, 0, 0); + __lasx_xvstelm_d(dst3, dst, 8, 2); + dst += stride; + __lasx_xvstelm_d(dst3, dst, 0, 1); + __lasx_xvstelm_d(dst3, dst, 8, 3); + } +} + +static void avc_biwgt_8x4_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t weight_dst, int32_t offset_in) +{ + __m256i wgt, vec0, vec1; + __m256i src0, dst0; + __m256i tmp0, tmp1, tmp2, tmp3, denom, offset; + + offset_in = (unsigned) ((offset_in + 1) | 1) << log2_denom; + offset_in += ((weight_src + weight_dst) << 7); + log2_denom += 1; + + tmp0 = __lasx_xvreplgr2vr_b(weight_src); + tmp1 = __lasx_xvreplgr2vr_b(weight_dst); + wgt = __lasx_xvilvh_b(tmp1, tmp0); + offset = __lasx_xvreplgr2vr_h(offset_in); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src0); + LASX_LD_4(dst, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, dst0); + LASX_XORI_B_2_128(src0, dst0); + LASX_ILVLH_B(dst0, src0, vec1, vec0); + LASX_DP2ADD_H_B(offset, wgt, vec0, tmp0); + LASX_DP2ADD_H_B(offset, wgt, vec1, tmp1); + tmp0 = __lasx_xvsra_h(tmp0, denom); + tmp1 = __lasx_xvsra_h(tmp1, denom); + LASX_CLIP_H_0_255_2(tmp0, tmp1, tmp0, tmp1); + LASX_PCKEV_B_128SV(tmp1, tmp0, dst0) + LASX_ST_D_4(dst0, 0, 2, 1, 3, dst, stride); +} + +static void avc_biwgt_8x8_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t weight_dst, int32_t offset_in) +{ + __m256i wgt, vec0, vec1, vec2, vec3; + __m256i src0, src1, dst0, dst1; + __m256i tmp0, tmp1, tmp2, tmp3, denom, offset; + ptrdiff_t stride_4x = stride << 2; + uint8_t* dst_tmp = dst; + + offset_in = (unsigned) ((offset_in + 1) | 1) << log2_denom; + offset_in += ((weight_src + weight_dst) << 7); + log2_denom += 1; + + tmp0 = __lasx_xvreplgr2vr_b(weight_src); + tmp1 = __lasx_xvreplgr2vr_b(weight_dst); + wgt = __lasx_xvilvh_b(tmp1, tmp0); + offset = __lasx_xvreplgr2vr_h(offset_in); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + src += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src0); + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src1); + + LASX_LD_4(dst_tmp, stride, tmp0, tmp1, tmp2, tmp3); + dst_tmp += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, dst0); + LASX_LD_4(dst_tmp, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, dst1); + + LASX_XORI_B_4_128(src0, src1, dst0, dst1); + LASX_ILVLH_B_2(dst0, src0, dst1, src1, vec1, vec0, vec3, vec2); + LASX_DP2ADD_H_B(offset, wgt, vec0, tmp0); + LASX_DP2ADD_H_B(offset, wgt, vec1, tmp1); + LASX_DP2ADD_H_B(offset, wgt, vec2, tmp2); + LASX_DP2ADD_H_B(offset, wgt, vec3, tmp3); + tmp0 = __lasx_xvsra_h(tmp0, denom); + tmp1 = __lasx_xvsra_h(tmp1, denom); + tmp2 = __lasx_xvsra_h(tmp2, denom); + tmp3 = __lasx_xvsra_h(tmp3, denom); + LASX_CLIP_H_0_255_4(tmp0, tmp1, tmp2, tmp3, tmp0, tmp1, tmp2, tmp3); + LASX_PCKEV_B_2_128SV(tmp1, tmp0, tmp3, tmp2, dst0, dst1) + LASX_ST_D_4(dst0, 0, 2, 1, 3, dst, stride); + dst += stride_4x; + LASX_ST_D_4(dst1, 0, 2, 1, 3, dst, stride); +} + +static void avc_biwgt_8x16_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t weight_dst, int32_t offset_in) +{ + __m256i wgt, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m256i src0, src1, src2, src3, dst0, dst1, dst2, dst3; + __m256i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, denom, offset; + ptrdiff_t stride_4x = stride << 2; + uint8_t* dst_tmp = dst; + + offset_in = (unsigned) ((offset_in + 1) | 1) << log2_denom; + offset_in += ((weight_src + weight_dst) << 7); + log2_denom += 1; + + tmp0 = __lasx_xvreplgr2vr_b(weight_src); + tmp1 = __lasx_xvreplgr2vr_b(weight_dst); + wgt = __lasx_xvilvh_b(tmp1, tmp0); + offset = __lasx_xvreplgr2vr_h(offset_in); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + src += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src0); + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + src += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src1); + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + src += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src2); + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src3); + + LASX_LD_4(dst_tmp, stride, tmp0, tmp1, tmp2, tmp3); + dst_tmp += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, dst0); + LASX_LD_4(dst_tmp, stride, tmp0, tmp1, tmp2, tmp3); + dst_tmp += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, dst1); + LASX_LD_4(dst_tmp, stride, tmp0, tmp1, tmp2, tmp3); + dst_tmp += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, dst2); + LASX_LD_4(dst_tmp, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, dst3); + + LASX_XORI_B_8_128(src0, src1, src2, src3, dst0, dst1, dst2, dst3); + LASX_ILVLH_B_4(dst0, src0, dst1, src1, dst2, src2, dst3, src3, + vec1, vec0, vec3, vec2, vec5, vec4, vec7, vec6); + LASX_DP2ADD_H_B(offset, wgt, vec0, tmp0); + LASX_DP2ADD_H_B(offset, wgt, vec1, tmp1); + LASX_DP2ADD_H_B(offset, wgt, vec2, tmp2); + LASX_DP2ADD_H_B(offset, wgt, vec3, tmp3); + LASX_DP2ADD_H_B(offset, wgt, vec4, tmp4); + LASX_DP2ADD_H_B(offset, wgt, vec5, tmp5); + LASX_DP2ADD_H_B(offset, wgt, vec6, tmp6); + LASX_DP2ADD_H_B(offset, wgt, vec7, tmp7); + tmp0 = __lasx_xvsra_h(tmp0, denom); + tmp1 = __lasx_xvsra_h(tmp1, denom); + tmp2 = __lasx_xvsra_h(tmp2, denom); + tmp3 = __lasx_xvsra_h(tmp3, denom); + tmp4 = __lasx_xvsra_h(tmp4, denom); + tmp5 = __lasx_xvsra_h(tmp5, denom); + tmp6 = __lasx_xvsra_h(tmp6, denom); + tmp7 = __lasx_xvsra_h(tmp7, denom); + LASX_CLIP_H_0_255_4(tmp0, tmp1, tmp2, tmp3, tmp0, tmp1, tmp2, tmp3); + LASX_CLIP_H_0_255_4(tmp4, tmp5, tmp6, tmp7, tmp4, tmp5, tmp6, tmp7); + LASX_PCKEV_B_4_128SV(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + dst0, dst1, dst2, dst3) + LASX_ST_D_4(dst0, 0, 2, 1, 3, dst, stride); + dst += stride_4x; + LASX_ST_D_4(dst1, 0, 2, 1, 3, dst, stride); + dst += stride_4x; + LASX_ST_D_4(dst2, 0, 2, 1, 3, dst, stride); + dst += stride_4x; + LASX_ST_D_4(dst3, 0, 2, 1, 3, dst, stride); +} + +void ff_biweight_h264_pixels8_8_lasx(uint8_t *dst, uint8_t *src, + ptrdiff_t stride, int height, + int log2_denom, int weight_dst, + int weight_src, int offset) +{ + if (4 == height) { + avc_biwgt_8x4_lasx(src, dst, stride, log2_denom, weight_src, weight_dst, + offset); + } else if (8 == height) { + avc_biwgt_8x8_lasx(src, dst, stride, log2_denom, weight_src, weight_dst, + offset); + } else { + avc_biwgt_8x16_lasx(src, dst, stride, log2_denom, weight_src, weight_dst, + offset); + } +} + +static void avc_biwgt_4x2_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t weight_dst, int32_t offset_in) +{ + __m256i wgt, vec0; + __m256i src0, dst0; + __m256i tmp0, tmp1, denom, offset; + + offset_in = (unsigned) ((offset_in + 1) | 1) << log2_denom; + offset_in += ((weight_src + weight_dst) << 7); + log2_denom += 1; + + tmp0 = __lasx_xvreplgr2vr_b(weight_src); + tmp1 = __lasx_xvreplgr2vr_b(weight_dst); + wgt = __lasx_xvilvh_b(tmp1, tmp0); + offset = __lasx_xvreplgr2vr_h(offset_in); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_2(src, stride, tmp0, tmp1); + LASX_ILVL_W_128SV(tmp1, tmp0, src0); + LASX_LD_2(dst, stride, tmp0, tmp1); + LASX_ILVL_W_128SV(tmp1, tmp0, dst0); + LASX_XORI_B_2_128(src0, dst0); + LASX_ILVL_B_128SV(dst0, src0, vec0); + LASX_DP2ADD_H_B(offset, wgt, vec0, tmp0); + tmp0 = __lasx_xvsra_h(tmp0, denom); + LASX_CLIP_H_0_255(tmp0, tmp0); + LASX_PCKEV_B_128SV(tmp0, tmp0, tmp0); + LASX_ST_W_2(tmp0, 0, 1, dst, stride); +} + +static void avc_biwgt_4x4_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t weight_dst, int32_t offset_in) +{ + __m256i wgt, vec0; + __m256i src0, dst0; + __m256i tmp0, tmp1, tmp2, tmp3, denom, offset; + + offset_in = (unsigned) ((offset_in + 1) | 1) << log2_denom; + offset_in += ((weight_src + weight_dst) << 7); + log2_denom += 1; + + tmp0 = __lasx_xvreplgr2vr_b(weight_src); + tmp1 = __lasx_xvreplgr2vr_b(weight_dst); + wgt = __lasx_xvilvh_b(tmp1, tmp0); + offset = __lasx_xvreplgr2vr_h(offset_in); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_W_2_128SV(tmp2, tmp0, tmp3, tmp1, tmp0, tmp1); + LASX_ILVL_W_128SV(tmp1, tmp0, src0); + LASX_LD_4(dst, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_W_2_128SV(tmp2, tmp0, tmp3, tmp1, tmp0, tmp1); + LASX_ILVL_W_128SV(tmp1, tmp0, dst0); + LASX_XORI_B_2_128(src0, dst0); + LASX_ILVL_B(dst0, src0, vec0); + LASX_DP2ADD_H_B(offset, wgt, vec0, tmp0); + tmp0 = __lasx_xvsra_h(tmp0, denom); + LASX_CLIP_H_0_255(tmp0, tmp0); + LASX_PCKEV_B_128SV(tmp0, tmp0, tmp0); + LASX_ST_W_4(tmp0, 0, 1, 4, 5, dst, stride); +} + +static void avc_biwgt_4x8_lasx(uint8_t *src, uint8_t *dst, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t weight_dst, int32_t offset_in) +{ + __m256i wgt, vec0, vec1; + __m256i src0, dst0; + __m256i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, denom, offset; + + offset_in = (unsigned) ((offset_in + 1) | 1) << log2_denom; + offset_in += ((weight_src + weight_dst) << 7); + log2_denom += 1; + + tmp0 = __lasx_xvreplgr2vr_b(weight_src); + tmp1 = __lasx_xvreplgr2vr_b(weight_dst); + wgt = __lasx_xvilvh_b(tmp1, tmp0); + offset = __lasx_xvreplgr2vr_h(offset_in); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_8(src, stride, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + LASX_ILVL_W_4_128SV(tmp2, tmp0, tmp3, tmp1, tmp6, tmp4, tmp7, tmp5, + tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_W_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src0); + LASX_LD_8(dst, stride, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + LASX_ILVL_W_4_128SV(tmp2, tmp0, tmp3, tmp1, tmp6, tmp4, tmp7, tmp5, + tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_W_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, dst0); + LASX_XORI_B_2_128(src0, dst0); + LASX_ILVLH_B(dst0, src0, vec1, vec0); + LASX_DP2ADD_H_B(offset, wgt, vec0, tmp0); + LASX_DP2ADD_H_B(offset, wgt, vec1, tmp1); + tmp0 = __lasx_xvsra_h(tmp0, denom); + tmp1 = __lasx_xvsra_h(tmp1, denom); + LASX_CLIP_H_0_255_2(tmp0, tmp1, tmp0, tmp1); + LASX_PCKEV_B_128SV(tmp1, tmp0, tmp0); + LASX_ST_W_8(tmp0, 0, 1, 4, 5, 2, 3, 6, 7, dst, stride); +} + +void ff_biweight_h264_pixels4_8_lasx(uint8_t *dst, uint8_t *src, + ptrdiff_t stride, int height, + int log2_denom, int weight_dst, + int weight_src, int offset) +{ + if (2 == height) { + avc_biwgt_4x2_lasx(src, dst, stride, log2_denom, weight_src, weight_dst, + offset); + } else if (4 == height) { + avc_biwgt_4x4_lasx(src, dst, stride, log2_denom, weight_src, weight_dst, + offset); + } else { + avc_biwgt_4x8_lasx(src, dst, stride, log2_denom, weight_src, weight_dst, + offset); + } +} + +void ff_weight_h264_pixels16_8_lasx(uint8_t *src, ptrdiff_t stride, + int height, int log2_denom, + int weight_src, int offset_in) +{ + uint32_t offset_val; + __m256i zero = __lasx_xvldi(0); + __m256i src0, src1, src2, src3; + __m256i src0_l, src1_l, src2_l, src3_l, src0_h, src1_h, src2_h, src3_h; + __m256i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m256i wgt, denom, offset; + + offset_val = (unsigned) offset_in << log2_denom; + + wgt = __lasx_xvreplgr2vr_h(weight_src); + offset = __lasx_xvreplgr2vr_h(offset_val); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_8(src, stride, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + LASX_PCKEV_Q_4(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + src0, src1, src2, src3); + LASX_ILVLH_B_4(zero, src0, zero, src1, zero, src2, zero, src3, + src0_h, src0_l, src1_h, src1_l, src2_h, src2_l, src3_h, src3_l); + src0_l = __lasx_xvmul_h(wgt, src0_l); + src0_h = __lasx_xvmul_h(wgt, src0_h); + src1_l = __lasx_xvmul_h(wgt, src1_l); + src1_h = __lasx_xvmul_h(wgt, src1_h); + src2_l = __lasx_xvmul_h(wgt, src2_l); + src2_h = __lasx_xvmul_h(wgt, src2_h); + src3_l = __lasx_xvmul_h(wgt, src3_l); + src3_h = __lasx_xvmul_h(wgt, src3_h); + LASX_SADD_H_4(src0_l, offset, src0_h, offset, src1_l, offset, + src1_h, offset, src0_l, src0_h, src1_l, src1_h); + LASX_SADD_H_4(src2_l, offset, src2_h, offset, src3_l, offset, + src3_h, offset, src2_l, src2_h, src3_l, src3_h); + src0_l = __lasx_xvmaxi_h(src0_l, 0); + src0_h = __lasx_xvmaxi_h(src0_h, 0); + src1_l = __lasx_xvmaxi_h(src1_l, 0); + src1_h = __lasx_xvmaxi_h(src1_h, 0); + src2_l = __lasx_xvmaxi_h(src2_l, 0); + src2_h = __lasx_xvmaxi_h(src2_h, 0); + src3_l = __lasx_xvmaxi_h(src3_l, 0); + src3_h = __lasx_xvmaxi_h(src3_h, 0); + src0_l = __lasx_xvssrlrn_bu_h(src0_l, denom); + src0_h = __lasx_xvssrlrn_bu_h(src0_h, denom); + src1_l = __lasx_xvssrlrn_bu_h(src1_l, denom); + src1_h = __lasx_xvssrlrn_bu_h(src1_h, denom); + src2_l = __lasx_xvssrlrn_bu_h(src2_l, denom); + src2_h = __lasx_xvssrlrn_bu_h(src2_h, denom); + src3_l = __lasx_xvssrlrn_bu_h(src3_l, denom); + src3_h = __lasx_xvssrlrn_bu_h(src3_h, denom); + __lasx_xvstelm_d(src0_l, src, 0, 0); + __lasx_xvstelm_d(src0_l, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src0_h, src, 0, 0); + __lasx_xvstelm_d(src0_h, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src1_l, src, 0, 0); + __lasx_xvstelm_d(src1_l, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src1_h, src, 0, 0); + __lasx_xvstelm_d(src1_h, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src2_l, src, 0, 0); + __lasx_xvstelm_d(src2_l, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src2_h, src, 0, 0); + __lasx_xvstelm_d(src2_h, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src3_l, src, 0, 0); + __lasx_xvstelm_d(src3_l, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src3_h, src, 0, 0); + __lasx_xvstelm_d(src3_h, src, 8, 2); + src += stride; + + if (16 == height) { + LASX_LD_8(src, stride, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + LASX_PCKEV_Q_4(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + src0, src1, src2, src3); + LASX_ILVLH_B_4(zero, src0, zero, src1, zero, src2, zero, src3, + src0_h, src0_l, src1_h, src1_l, src2_h, src2_l, + src3_h, src3_l); + src0_l = __lasx_xvmul_h(wgt, src0_l); + src0_h = __lasx_xvmul_h(wgt, src0_h); + src1_l = __lasx_xvmul_h(wgt, src1_l); + src1_h = __lasx_xvmul_h(wgt, src1_h); + src2_l = __lasx_xvmul_h(wgt, src2_l); + src2_h = __lasx_xvmul_h(wgt, src2_h); + src3_l = __lasx_xvmul_h(wgt, src3_l); + src3_h = __lasx_xvmul_h(wgt, src3_h); + LASX_SADD_H_4(src0_l, offset, src0_h, offset, src1_l, offset, + src1_h, offset, src0_l, src0_h, src1_l, src1_h); + LASX_SADD_H_4(src2_l, offset, src2_h, offset, src3_l, offset, + src3_h, offset, src2_l, src2_h, src3_l, src3_h); + src0_l = __lasx_xvmaxi_h(src0_l, 0); + src0_h = __lasx_xvmaxi_h(src0_h, 0); + src1_l = __lasx_xvmaxi_h(src1_l, 0); + src1_h = __lasx_xvmaxi_h(src1_h, 0); + src2_l = __lasx_xvmaxi_h(src2_l, 0); + src2_h = __lasx_xvmaxi_h(src2_h, 0); + src3_l = __lasx_xvmaxi_h(src3_l, 0); + src3_h = __lasx_xvmaxi_h(src3_h, 0); + src0_l = __lasx_xvssrlrn_bu_h(src0_l, denom); + src0_h = __lasx_xvssrlrn_bu_h(src0_h, denom); + src1_l = __lasx_xvssrlrn_bu_h(src1_l, denom); + src1_h = __lasx_xvssrlrn_bu_h(src1_h, denom); + src2_l = __lasx_xvssrlrn_bu_h(src2_l, denom); + src2_h = __lasx_xvssrlrn_bu_h(src2_h, denom); + src3_l = __lasx_xvssrlrn_bu_h(src3_l, denom); + src3_h = __lasx_xvssrlrn_bu_h(src3_h, denom); + __lasx_xvstelm_d(src0_l, src, 0, 0); + __lasx_xvstelm_d(src0_l, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src0_h, src, 0, 0); + __lasx_xvstelm_d(src0_h, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src1_l, src, 0, 0); + __lasx_xvstelm_d(src1_l, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src1_h, src, 0, 0); + __lasx_xvstelm_d(src1_h, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src2_l, src, 0, 0); + __lasx_xvstelm_d(src2_l, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src2_h, src, 0, 0); + __lasx_xvstelm_d(src2_h, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src3_l, src, 0, 0); + __lasx_xvstelm_d(src3_l, src, 8, 2); + src += stride; + __lasx_xvstelm_d(src3_h, src, 0, 0); + __lasx_xvstelm_d(src3_h, src, 8, 2); + } +} + +static void avc_wgt_8x4_lasx(uint8_t *src, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t offset_in) +{ + uint32_t offset_val; + __m256i wgt, zero = __lasx_xvldi(0); + __m256i src0, src0_h, src0_l; + __m256i tmp0, tmp1, tmp2, tmp3, denom, offset; + + offset_val = (unsigned) offset_in << log2_denom; + + wgt = __lasx_xvreplgr2vr_h(weight_src); + offset = __lasx_xvreplgr2vr_h(offset_val); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src0); + LASX_ILVLH_B(zero, src0, src0_h, src0_l); + src0_l = __lasx_xvmul_h(wgt, src0_l); + src0_h = __lasx_xvmul_h(wgt, src0_h); + src0_l = __lasx_xvsadd_h(src0_l, offset); + src0_h = __lasx_xvsadd_h(src0_h, offset); + src0_l = __lasx_xvmaxi_h(src0_l, 0); + src0_h = __lasx_xvmaxi_h(src0_h, 0); + src0_l = __lasx_xvssrlrn_bu_h(src0_l, denom); + src0_h = __lasx_xvssrlrn_bu_h(src0_h, denom); + + LASX_PCKEV_D(src0_h, src0_l, src0); + LASX_ST_D_4(src0, 0, 1, 2, 3, src, stride); +} + +static void avc_wgt_8x8_lasx(uint8_t *src, ptrdiff_t stride, int32_t log2_denom, + int32_t src_weight, int32_t offset_in) +{ + uint32_t offset_val; + __m256i src0, src1, src0_h, src0_l, src1_h, src1_l, zero = __lasx_xvldi(0); + __m256i tmp0, tmp1, tmp2, tmp3, denom, offset, wgt; + ptrdiff_t stride_4x = stride << 2; + uint8_t* src_tmp = src; + + offset_val = (unsigned) offset_in << log2_denom; + + wgt = __lasx_xvreplgr2vr_h(src_weight); + offset = __lasx_xvreplgr2vr_h(offset_val); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_4(src_tmp, stride, tmp0, tmp1, tmp2, tmp3); + src_tmp += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src0); + LASX_LD_4(src_tmp, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src1); + LASX_ILVLH_B_2(zero, src0, zero, src1, src0_h, src0_l, src1_h, src1_l); + src0_l = __lasx_xvmul_h(wgt, src0_l); + src0_h = __lasx_xvmul_h(wgt, src0_h); + src1_l = __lasx_xvmul_h(wgt, src1_l); + src1_h = __lasx_xvmul_h(wgt, src1_h); + LASX_SADD_H_4(src0_l, offset, src0_h, offset, src1_l, offset, + src1_h, offset, src0_l, src0_h, src1_l, src1_h); + src0_l = __lasx_xvmaxi_h(src0_l, 0); + src0_h = __lasx_xvmaxi_h(src0_h, 0); + src1_l = __lasx_xvmaxi_h(src1_l, 0); + src1_h = __lasx_xvmaxi_h(src1_h, 0); + src0_l = __lasx_xvssrlrn_bu_h(src0_l, denom); + src0_h = __lasx_xvssrlrn_bu_h(src0_h, denom); + src1_l = __lasx_xvssrlrn_bu_h(src1_l, denom); + src1_h = __lasx_xvssrlrn_bu_h(src1_h, denom); + + LASX_PCKEV_D_2(src0_h, src0_l, src1_h, src1_l, src0, src1); + LASX_ST_D_4(src0, 0, 1, 2, 3, src, stride); + src += stride_4x; + LASX_ST_D_4(src1, 0, 1, 2, 3, src, stride); +} + +static void avc_wgt_8x16_lasx(uint8_t *src, ptrdiff_t stride, + int32_t log2_denom, int32_t src_weight, + int32_t offset_in) +{ + uint32_t offset_val; + __m256i src0, src1, src2, src3; + __m256i src0_h, src0_l, src1_h, src1_l, src2_h, src2_l, src3_h, src3_l; + __m256i tmp0, tmp1, tmp2, tmp3, denom, offset, wgt; + __m256i zero = __lasx_xvldi(0); + ptrdiff_t stride_4x = stride << 2; + uint8_t* src_tmp = src; + + offset_val = (unsigned) offset_in << log2_denom; + + wgt = __lasx_xvreplgr2vr_h(src_weight); + offset = __lasx_xvreplgr2vr_h(offset_val); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_4(src_tmp, stride, tmp0, tmp1, tmp2, tmp3); + src_tmp += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src0); + LASX_LD_4(src_tmp, stride, tmp0, tmp1, tmp2, tmp3); + src_tmp += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src1); + LASX_LD_4(src_tmp, stride, tmp0, tmp1, tmp2, tmp3); + src_tmp += stride_4x; + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src2); + LASX_LD_4(src_tmp, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_D_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src3); + + LASX_ILVLH_B_4(zero, src0, zero, src1, zero, src2, zero, src3, + src0_h, src0_l, src1_h, src1_l, src2_h, src2_l, src3_h, src3_l); + src0_l = __lasx_xvmul_h(wgt, src0_l); + src0_h = __lasx_xvmul_h(wgt, src0_h); + src1_l = __lasx_xvmul_h(wgt, src1_l); + src1_h = __lasx_xvmul_h(wgt, src1_h); + src2_l = __lasx_xvmul_h(wgt, src2_l); + src2_h = __lasx_xvmul_h(wgt, src2_h); + src3_l = __lasx_xvmul_h(wgt, src3_l); + src3_h = __lasx_xvmul_h(wgt, src3_h); + + LASX_SADD_H_4(src0_l, offset, src0_h, offset, src1_l, offset, + src1_h, offset, src0_l, src0_h, src1_l, src1_h); + LASX_SADD_H_4(src2_l, offset, src2_h, offset, src3_l, offset, + src3_h, offset, src2_l, src2_h, src3_l, src3_h); + + src0_l = __lasx_xvmaxi_h(src0_l, 0); + src0_h = __lasx_xvmaxi_h(src0_h, 0); + src1_l = __lasx_xvmaxi_h(src1_l, 0); + src1_h = __lasx_xvmaxi_h(src1_h, 0); + src2_l = __lasx_xvmaxi_h(src2_l, 0); + src2_h = __lasx_xvmaxi_h(src2_h, 0); + src3_l = __lasx_xvmaxi_h(src3_l, 0); + src3_h = __lasx_xvmaxi_h(src3_h, 0); + src0_l = __lasx_xvssrlrn_bu_h(src0_l, denom); + src0_h = __lasx_xvssrlrn_bu_h(src0_h, denom); + src1_l = __lasx_xvssrlrn_bu_h(src1_l, denom); + src1_h = __lasx_xvssrlrn_bu_h(src1_h, denom); + src2_l = __lasx_xvssrlrn_bu_h(src2_l, denom); + src2_h = __lasx_xvssrlrn_bu_h(src2_h, denom); + src3_l = __lasx_xvssrlrn_bu_h(src3_l, denom); + src3_h = __lasx_xvssrlrn_bu_h(src3_h, denom); + LASX_PCKEV_D_4(src0_h, src0_l, src1_h, src1_l, src2_h, src2_l, src3_h, src3_l, + src0, src1, src2, src3); + + LASX_ST_D_4(src0, 0, 1, 2, 3, src, stride); + src += stride_4x; + LASX_ST_D_4(src1, 0, 1, 2, 3, src, stride); + src += stride_4x; + LASX_ST_D_4(src2, 0, 1, 2, 3, src, stride); + src += stride_4x; + LASX_ST_D_4(src3, 0, 1, 2, 3, src, stride); +} + +void ff_weight_h264_pixels8_8_lasx(uint8_t *src, ptrdiff_t stride, + int height, int log2_denom, + int weight_src, int offset) +{ + if (4 == height) { + avc_wgt_8x4_lasx(src, stride, log2_denom, weight_src, offset); + } else if (8 == height) { + avc_wgt_8x8_lasx(src, stride, log2_denom, weight_src, offset); + } else { + avc_wgt_8x16_lasx(src, stride, log2_denom, weight_src, offset); + } +} + +static void avc_wgt_4x2_lasx(uint8_t *src, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t offset_in) +{ + uint32_t offset_val; + __m256i wgt, zero = __lasx_xvldi(0); + __m256i src0, tmp0, tmp1, denom, offset; + + offset_val = (unsigned) offset_in << log2_denom; + + wgt = __lasx_xvreplgr2vr_h(weight_src); + offset = __lasx_xvreplgr2vr_h(offset_val); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_2(src, stride, tmp0, tmp1); + LASX_ILVL_W_128SV(tmp1, tmp0, src0); + LASX_ILVL_B_128SV(zero, src0, src0); + src0 = __lasx_xvmul_h(wgt, src0); + src0 = __lasx_xvsadd_h(src0, offset); + src0 = __lasx_xvmaxi_h(src0, 0); + src0 = __lasx_xvssrlrn_bu_h(src0, denom); + LASX_ST_W_2(src0, 0, 1, src, stride); +} + +static void avc_wgt_4x4_lasx(uint8_t *src, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t offset_in) +{ + uint32_t offset_val; + __m256i wgt; + __m256i src0, tmp0, tmp1, tmp2, tmp3, denom, offset; + + offset_val = (unsigned) offset_in << log2_denom; + + wgt = __lasx_xvreplgr2vr_h(weight_src); + offset = __lasx_xvreplgr2vr_h(offset_val); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_4(src, stride, tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_W_2_128SV(tmp2, tmp0, tmp3, tmp1, tmp0, tmp1); + LASX_ILVL_W_128SV(tmp1, tmp0, src0); + LASX_UNPCK_L_HU_BU(src0, src0); + src0 = __lasx_xvmul_h(wgt, src0); + src0 = __lasx_xvsadd_h(src0, offset); + src0 = __lasx_xvmaxi_h(src0, 0); + src0 = __lasx_xvssrlrn_bu_h(src0, denom); + LASX_ST_W_4(src0, 0, 1, 4, 5, src, stride); +} + +static void avc_wgt_4x8_lasx(uint8_t *src, ptrdiff_t stride, + int32_t log2_denom, int32_t weight_src, + int32_t offset_in) +{ + uint32_t offset_val; + __m256i src0, src0_h, src0_l; + __m256i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, denom, offset; + __m256i wgt, zero = __lasx_xvldi(0); + + offset_val = (unsigned) offset_in << log2_denom; + + wgt = __lasx_xvreplgr2vr_h(weight_src); + offset = __lasx_xvreplgr2vr_h(offset_val); + denom = __lasx_xvreplgr2vr_h(log2_denom); + + LASX_LD_8(src, stride, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + LASX_ILVL_W_4_128SV(tmp2, tmp0, tmp3, tmp1, tmp6, tmp4, tmp7, tmp5, + tmp0, tmp1, tmp2, tmp3); + LASX_ILVL_W_2_128SV(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LASX_PCKEV_Q(tmp1, tmp0, src0); + LASX_ILVLH_B(zero, src0, src0_h, src0_l); + src0_l = __lasx_xvmul_h(wgt, src0_l); + src0_h = __lasx_xvmul_h(wgt, src0_h); + src0_l = __lasx_xvsadd_h(src0_l, offset); + src0_h = __lasx_xvsadd_h(src0_h, offset); + src0_l = __lasx_xvmaxi_h(src0_l, 0); + src0_h = __lasx_xvmaxi_h(src0_h, 0); + src0_l = __lasx_xvssrlrn_bu_h(src0_l, denom); + src0_h = __lasx_xvssrlrn_bu_h(src0_h, denom); + LASX_ST_W_4(src0_l, 0, 1, 4, 5, src, stride); + src += (stride << 2); + LASX_ST_W_4(src0_h, 0, 1, 4, 5, src, stride); +} + +void ff_weight_h264_pixels4_8_lasx(uint8_t *src, ptrdiff_t stride, + int height, int log2_denom, + int weight_src, int offset) +{ + if (2 == height) { + avc_wgt_4x2_lasx(src, stride, log2_denom, weight_src, offset); + } else if (4 == height) { + avc_wgt_4x4_lasx(src, stride, log2_denom, weight_src, offset); + } else { + avc_wgt_4x8_lasx(src, stride, log2_denom, weight_src, offset); + } +} diff --git a/libavcodec/loongarch/h264dsp_lasx.h b/libavcodec/loongarch/h264dsp_lasx.h new file mode 100644 index 0000000000..54c57853a2 --- /dev/null +++ b/libavcodec/loongarch/h264dsp_lasx.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * Xiwei Gu + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_H264DSP_LASX_H +#define AVCODEC_LOONGARCH_H264DSP_LASX_H + +#include "libavcodec/h264dec.h" + +void ff_h264_h_lpf_luma_8_lasx(uint8_t *src, int stride, + int alpha, int beta, int8_t *tc0); +void ff_h264_v_lpf_luma_8_lasx(uint8_t *src, int stride, + int alpha, int beta, int8_t *tc0); +void ff_h264_h_lpf_luma_intra_8_lasx(uint8_t *src, int stride, + int alpha, int beta); +void ff_h264_v_lpf_luma_intra_8_lasx(uint8_t *src, int stride, + int alpha, int beta); +void ff_h264_h_lpf_chroma_8_lasx(uint8_t *src, int stride, + int alpha, int beta, int8_t *tc0); +void ff_h264_v_lpf_chroma_8_lasx(uint8_t *src, int stride, + int alpha, int beta, int8_t *tc0); +void ff_h264_h_lpf_chroma_intra_8_lasx(uint8_t *src, int stride, + int alpha, int beta); +void ff_h264_v_lpf_chroma_intra_8_lasx(uint8_t *src, int stride, + int alpha, int beta); +void ff_biweight_h264_pixels16_8_lasx(uint8_t *dst, uint8_t *src, + ptrdiff_t stride, int height, + int log2_denom, int weight_dst, + int weight_src, int offset_in); +void ff_biweight_h264_pixels8_8_lasx(uint8_t *dst, uint8_t *src, + ptrdiff_t stride, int height, + int log2_denom, int weight_dst, + int weight_src, int offset); +void ff_biweight_h264_pixels4_8_lasx(uint8_t *dst, uint8_t *src, + ptrdiff_t stride, int height, + int log2_denom, int weight_dst, + int weight_src, int offset); +void ff_weight_h264_pixels16_8_lasx(uint8_t *src, ptrdiff_t stride, + int height, int log2_denom, + int weight_src, int offset_in); +void ff_weight_h264_pixels8_8_lasx(uint8_t *src, ptrdiff_t stride, + int height, int log2_denom, + int weight_src, int offset); +void ff_weight_h264_pixels4_8_lasx(uint8_t *src, ptrdiff_t stride, + int height, int log2_denom, + int weight_src, int offset); +void ff_h264_idct_add_lasx(uint8_t *dst, int16_t *src, int32_t dst_stride); +void ff_h264_idct8_addblk_lasx(uint8_t *dst, int16_t *src, + int32_t dst_stride); +void ff_h264_idct4x4_addblk_dc_lasx(uint8_t *dst, int16_t *src, + int32_t dst_stride); +void ff_h264_idct8_dc_addblk_lasx(uint8_t *dst, int16_t *src, + int32_t dst_stride); +void ff_h264_idct_add16_lasx(uint8_t *dst, + const int32_t *blk_offset, + int16_t *block, int32_t dst_stride, + const uint8_t nzc[15 * 8]); +void ff_h264_idct8_add4_lasx(uint8_t *dst, const int32_t *blk_offset, + int16_t *block, int32_t dst_stride, + const uint8_t nzc[15 * 8]); +void ff_h264_idct_add8_lasx(uint8_t **dst, + const int32_t *blk_offset, + int16_t *block, int32_t dst_stride, + const uint8_t nzc[15 * 8]); +void ff_h264_idct_add8_422_lasx(uint8_t **dst, + const int32_t *blk_offset, + int16_t *block, int32_t dst_stride, + const uint8_t nzc[15 * 8]); +void ff_h264_idct_add16_intra_lasx(uint8_t *dst, + const int32_t *blk_offset, + int16_t *block, + int32_t dst_stride, + const uint8_t nzc[15 * 8]); +void ff_h264_deq_idct_luma_dc_lasx(int16_t *dst, int16_t *src, + int32_t de_qval); + +void ff_h264_loop_filter_strength_lasx(int16_t bS[2][4][4], uint8_t nnz[40], + int8_t ref[2][40], int16_t mv[2][40][2], + int bidir, int edges, int step, + int mask_mv0, int mask_mv1, int field); +#endif // #ifndef AVCODEC_LOONGARCH_H264DSP_LASX_H diff --git a/libavcodec/loongarch/h264idct_lasx.c b/libavcodec/loongarch/h264idct_lasx.c new file mode 100644 index 0000000000..a4acb2d404 --- /dev/null +++ b/libavcodec/loongarch/h264idct_lasx.c @@ -0,0 +1,461 @@ +/* + * Loongson LASX optimized h264dsp + * + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * Xiwei Gu + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lasx.h" +#include "h264dsp_lasx.h" +#include "libavcodec/bit_depth_template.c" + +#define BUTTERFLY_4_H(tmp0_m, tmp1_m, tmp2_m, tmp3_m, out0, out1, out2, out3) \ +{ \ + out0 = __lasx_xvadd_h(tmp0_m, tmp3_m); \ + out1 = __lasx_xvadd_h(tmp1_m, tmp2_m); \ + out2 = __lasx_xvsub_h(tmp1_m, tmp2_m); \ + out3 = __lasx_xvsub_h(tmp0_m, tmp3_m); \ + } + +#define BUTTERFLY_4_W(tmp0_m, tmp1_m, tmp2_m, tmp3_m, out0, out1, out2, out3) \ +{ \ + out0 = __lasx_xvadd_w(tmp0_m, tmp3_m); \ + out1 = __lasx_xvadd_w(tmp1_m, tmp2_m); \ + out2 = __lasx_xvsub_w(tmp1_m, tmp2_m); \ + out3 = __lasx_xvsub_w(tmp0_m, tmp3_m); \ + } + +#define BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + out0 = __lasx_xvadd_h(in0, in7); \ + out1 = __lasx_xvadd_h(in1, in6); \ + out2 = __lasx_xvadd_h(in2, in5); \ + out3 = __lasx_xvadd_h(in3, in4); \ + out4 = __lasx_xvsub_h(in3, in4); \ + out5 = __lasx_xvsub_h(in2, in5); \ + out6 = __lasx_xvsub_h(in1, in6); \ + out7 = __lasx_xvsub_h(in0, in7); \ +} + +#define AVC_ITRANS_H(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + __m256i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + \ + tmp0_m = __lasx_xvadd_h(in0, in2); \ + tmp1_m = __lasx_xvsub_h(in0, in2); \ + tmp2_m = __lasx_xvsrai_h(in1, 1); \ + tmp2_m = __lasx_xvsub_h(tmp2_m, in3); \ + tmp3_m = __lasx_xvsrai_h(in3, 1); \ + tmp3_m = __lasx_xvadd_h(in1, tmp3_m); \ + \ + BUTTERFLY_4_H(tmp0_m, tmp1_m, tmp2_m, tmp3_m, \ + out0, out1, out2, out3); \ +} + +void ff_h264_idct_add_lasx(uint8_t *dst, int16_t *src, int32_t dst_stride) +{ + __m256i src0_m, src1_m, src2_m, src3_m; + __m256i dst0_m, dst1_m; + __m256i hres0, hres1, hres2, hres3, vres0, vres1, vres2, vres3; + __m256i inp0_m, inp1_m, res0_m, src1, src3; + __m256i src0 = LASX_LD(src); + __m256i src2 = LASX_LD(src + 8); + __m256i zero = __lasx_xvldi(0); + + LASX_ST(zero, src); + LASX_ILVH_D_2_128SV(src0, src0, src2, src2, src1, src3); + AVC_ITRANS_H(src0, src1, src2, src3, hres0, hres1, hres2, hres3); + LASX_TRANSPOSE4x4_H_128SV(hres0, hres1, hres2, hres3, + hres0, hres1, hres2, hres3); + AVC_ITRANS_H(hres0, hres1, hres2, hres3, vres0, vres1, vres2, vres3); + LASX_LD_4(dst, dst_stride, src0_m, src1_m, src2_m, src3_m); + LASX_ILVL_D_2_128SV(vres1, vres0, vres3, vres2, inp0_m, inp1_m); + LASX_PCKEV_Q(inp1_m, inp0_m, inp0_m); + LASX_SRARI_H(inp0_m, inp0_m, 6); + LASX_ILVL_W_2_128SV(src1_m, src0_m, src3_m, src2_m, dst0_m, dst1_m); + LASX_ILVL_D_128SV(dst1_m, dst0_m, dst0_m); + LASX_UNPCK_L_HU_BU(dst0_m, res0_m); + res0_m = __lasx_xvadd_h(res0_m, inp0_m); + LASX_CLIP_H_0_255(res0_m, res0_m); + dst0_m = __lasx_xvpickev_b(res0_m, res0_m); + LASX_ST_W_4(dst0_m, 0, 1, 4, 5, dst, dst_stride); +} + +void ff_h264_idct8_addblk_lasx(uint8_t *dst, int16_t *src, + int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i vec0, vec1, vec2, vec3; + __m256i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m256i res0, res1, res2, res3, res4, res5, res6, res7; + __m256i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m256i zero = __lasx_xvldi(0); + + src[0] += 32; + LASX_LD_8(src, 8, src0, src1, src2, src3, src4, src5, src6, src7); + LASX_ST_4(zero, zero, zero, zero, src, 16); + + vec0 = __lasx_xvadd_h(src0, src4); + vec1 = __lasx_xvsub_h(src0, src4); + vec2 = __lasx_xvsrai_h(src2, 1); + vec2 = __lasx_xvsub_h(vec2, src6); + vec3 = __lasx_xvsrai_h(src6, 1); + vec3 = __lasx_xvadd_h(src2, vec3); + + BUTTERFLY_4_H(vec0, vec1, vec2, vec3, tmp0, tmp1, tmp2, tmp3); + + vec0 = __lasx_xvsrai_h(src7, 1); + vec0 = __lasx_xvsub_h(src5, vec0); + vec0 = __lasx_xvsub_h(vec0, src3); + vec0 = __lasx_xvsub_h(vec0, src7); + + vec1 = __lasx_xvsrai_h(src3, 1); + vec1 = __lasx_xvsub_h(src1, vec1); + vec1 = __lasx_xvadd_h(vec1, src7); + vec1 = __lasx_xvsub_h(vec1, src3); + + vec2 = __lasx_xvsrai_h(src5, 1); + vec2 = __lasx_xvsub_h(vec2, src1); + vec2 = __lasx_xvadd_h(vec2, src7); + vec2 = __lasx_xvadd_h(vec2, src5); + + vec3 = __lasx_xvsrai_h(src1, 1); + vec3 = __lasx_xvadd_h(src3, vec3); + vec3 = __lasx_xvadd_h(vec3, src5); + vec3 = __lasx_xvadd_h(vec3, src1); + + tmp4 = __lasx_xvsrai_h(vec3, 2); + tmp4 = __lasx_xvadd_h(tmp4, vec0); + tmp5 = __lasx_xvsrai_h(vec2, 2); + tmp5 = __lasx_xvadd_h(tmp5, vec1); + tmp6 = __lasx_xvsrai_h(vec1, 2); + tmp6 = __lasx_xvsub_h(tmp6, vec2); + tmp7 = __lasx_xvsrai_h(vec0, 2); + tmp7 = __lasx_xvsub_h(vec3, tmp7); + + BUTTERFLY_8_H(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, + res0, res1, res2, res3, res4, res5, res6, res7); + LASX_TRANSPOSE8x8_H_128SV(res0, res1, res2, res3, res4, res5, res6, res7, + res0, res1, res2, res3, res4, res5, res6, res7); + + LASX_UNPCK_L_W_H_8(res0, res1, res2, res3, res4, res5, res6, res7, + tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + vec0 = __lasx_xvadd_w(tmp0, tmp4); + vec1 = __lasx_xvsub_w(tmp0, tmp4); + + vec2 = __lasx_xvsrai_w(tmp2, 1); + vec2 = __lasx_xvsub_w(vec2, tmp6); + vec3 = __lasx_xvsrai_w(tmp6, 1); + vec3 = __lasx_xvadd_w(vec3, tmp2); + + tmp0 = __lasx_xvadd_w(vec0, vec3); + tmp2 = __lasx_xvadd_w(vec1, vec2); + tmp4 = __lasx_xvsub_w(vec1, vec2); + tmp6 = __lasx_xvsub_w(vec0, vec3); + + vec0 = __lasx_xvsrai_w(tmp7, 1); + vec0 = __lasx_xvsub_w(tmp5, vec0); + vec0 = __lasx_xvsub_w(vec0, tmp3); + vec0 = __lasx_xvsub_w(vec0, tmp7); + + vec1 = __lasx_xvsrai_w(tmp3, 1); + vec1 = __lasx_xvsub_w(tmp1, vec1); + vec1 = __lasx_xvadd_w(vec1, tmp7); + vec1 = __lasx_xvsub_w(vec1, tmp3); + + vec2 = __lasx_xvsrai_w(tmp5, 1); + vec2 = __lasx_xvsub_w(vec2, tmp1); + vec2 = __lasx_xvadd_w(vec2, tmp7); + vec2 = __lasx_xvadd_w(vec2, tmp5); + + vec3 = __lasx_xvsrai_w(tmp1, 1); + vec3 = __lasx_xvadd_w(tmp3, vec3); + vec3 = __lasx_xvadd_w(vec3, tmp5); + vec3 = __lasx_xvadd_w(vec3, tmp1); + + tmp1 = __lasx_xvsrai_w(vec3, 2); + tmp1 = __lasx_xvadd_w(tmp1, vec0); + tmp3 = __lasx_xvsrai_w(vec2, 2); + tmp3 = __lasx_xvadd_w(tmp3, vec1); + tmp5 = __lasx_xvsrai_w(vec1, 2); + tmp5 = __lasx_xvsub_w(tmp5, vec2); + tmp7 = __lasx_xvsrai_w(vec0, 2); + tmp7 = __lasx_xvsub_w(vec3, tmp7); + + BUTTERFLY_4_W(tmp0, tmp2, tmp5, tmp7, res0, res1, res6, res7); + BUTTERFLY_4_W(tmp4, tmp6, tmp1, tmp3, res2, res3, res4, res5); + + LASX_SRAI_W_8(res0, res1, res2, res3, res4, res5, res6, res7, + res0, res1, res2, res3, res4, res5, res6, res7, 6); + LASX_PCKEV_H_4(res1, res0, res3, res2, res5, res4, res7, res6, + res0, res1, res2, res3); + + LASX_LD_8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7); + LASX_ILVL_B_8_128SV(zero, dst0, zero, dst1, zero, dst2, zero, dst3, + zero, dst4, zero, dst5, zero, dst6, zero, dst7, + dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7); + LASX_PCKEV_Q_4(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + dst0, dst1, dst2, dst3); + res0 = __lasx_xvadd_h(res0, dst0); + res1 = __lasx_xvadd_h(res1, dst1); + res2 = __lasx_xvadd_h(res2, dst2); + res3 = __lasx_xvadd_h(res3, dst3); + LASX_CLIP_H_0_255_4(res0, res1, res2, res3, res0, res1, res2, res3); + LASX_PCKEV_B_2_128SV(res1, res0, res3, res2, res0, res1); + LASX_ST_D_4(res0, 0, 2, 1, 3, dst, dst_stride); + LASX_ST_D_4(res1, 0, 2, 1, 3, dst + 4 * dst_stride, dst_stride); +} + +void ff_h264_idct4x4_addblk_dc_lasx(uint8_t *dst, int16_t *src, + int32_t dst_stride) +{ + const int16_t dc = (src[0] + 32) >> 6; + __m256i pred, out; + __m256i src0, src1, src2, src3; + __m256i input_dc = __lasx_xvreplgr2vr_h(dc); + + src[0] = 0; + LASX_LD_4(dst, dst_stride, src0, src1, src2, src3); + LASX_ILVL_W_2_128SV(src1, src0, src3, src2, src0, src1); + pred = __lasx_xvpermi_q(src0, src1, 0x02); + LASX_ADDW_H_H_BU_128SV(input_dc, pred, pred); + LASX_CLIP_H_0_255(pred, pred); + out = __lasx_xvpickev_b(pred, pred); + LASX_ST_W_4(out, 0, 1, 4, 5, dst, dst_stride); +} + +void ff_h264_idct8_dc_addblk_lasx(uint8_t *dst, int16_t *src, + int32_t dst_stride) +{ + int32_t dc_val; + __m256i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m256i dc; + + dc_val = (src[0] + 32) >> 6; + dc = __lasx_xvreplgr2vr_h(dc_val); + + src[0] = 0; + + LASX_LD_8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7); + LASX_UNPCK_L_HU_BU_8(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, + dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7); + LASX_PCKEV_Q_4(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + dst0, dst1, dst2, dst3); + dst0 = __lasx_xvadd_h(dst0, dc); + dst1 = __lasx_xvadd_h(dst1, dc); + dst2 = __lasx_xvadd_h(dst2, dc); + dst3 = __lasx_xvadd_h(dst3, dc); + LASX_CLIP_H_0_255_4(dst0, dst1, dst2, dst3, dst0, dst1, dst2, dst3); + LASX_PCKEV_B_2_128SV(dst1, dst0, dst3, dst2, dst0, dst1); + LASX_ST_D_4(dst0, 0, 2, 1, 3, dst, dst_stride); + LASX_ST_D_4(dst1, 0, 2, 1, 3, dst + 4 * dst_stride, dst_stride); +} + +void ff_h264_idct_add16_lasx(uint8_t *dst, + const int32_t *blk_offset, + int16_t *block, int32_t dst_stride, + const uint8_t nzc[15 * 8]) +{ + int32_t i; + + for (i = 0; i < 16; i++) { + int32_t nnz = nzc[scan8[i]]; + + if (nnz) { + if (nnz == 1 && ((dctcoef *) block)[i * 16]) + ff_h264_idct4x4_addblk_dc_lasx(dst + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + else + ff_h264_idct_add_lasx(dst + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + } + } +} + +void ff_h264_idct8_add4_lasx(uint8_t *dst, const int32_t *blk_offset, + int16_t *block, int32_t dst_stride, + const uint8_t nzc[15 * 8]) +{ + int32_t cnt; + + for (cnt = 0; cnt < 16; cnt += 4) { + int32_t nnz = nzc[scan8[cnt]]; + + if (nnz) { + if (nnz == 1 && ((dctcoef *) block)[cnt * 16]) + ff_h264_idct8_dc_addblk_lasx(dst + blk_offset[cnt], + block + cnt * 16 * sizeof(pixel), + dst_stride); + else + ff_h264_idct8_addblk_lasx(dst + blk_offset[cnt], + block + cnt * 16 * sizeof(pixel), + dst_stride); + } + } +} + + +void ff_h264_idct_add8_lasx(uint8_t **dst, + const int32_t *blk_offset, + int16_t *block, int32_t dst_stride, + const uint8_t nzc[15 * 8]) +{ + int32_t i; + + for (i = 16; i < 20; i++) { + if (nzc[scan8[i]]) + ff_h264_idct_add_lasx(dst[0] + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + else if (((dctcoef *) block)[i * 16]) + ff_h264_idct4x4_addblk_dc_lasx(dst[0] + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + } + for (i = 32; i < 36; i++) { + if (nzc[scan8[i]]) + ff_h264_idct_add_lasx(dst[1] + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + else if (((dctcoef *) block)[i * 16]) + ff_h264_idct4x4_addblk_dc_lasx(dst[1] + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + } +} + +void ff_h264_idct_add8_422_lasx(uint8_t **dst, + const int32_t *blk_offset, + int16_t *block, int32_t dst_stride, + const uint8_t nzc[15 * 8]) +{ + int32_t i; + + for (i = 16; i < 20; i++) { + if (nzc[scan8[i]]) + ff_h264_idct_add_lasx(dst[0] + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + else if (((dctcoef *) block)[i * 16]) + ff_h264_idct4x4_addblk_dc_lasx(dst[0] + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + } + for (i = 32; i < 36; i++) { + if (nzc[scan8[i]]) + ff_h264_idct_add_lasx(dst[1] + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + else if (((dctcoef *) block)[i * 16]) + ff_h264_idct4x4_addblk_dc_lasx(dst[1] + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + } + for (i = 20; i < 24; i++) { + if (nzc[scan8[i + 4]]) + ff_h264_idct_add_lasx(dst[0] + blk_offset[i + 4], + block + i * 16 * sizeof(pixel), + dst_stride); + else if (((dctcoef *) block)[i * 16]) + ff_h264_idct4x4_addblk_dc_lasx(dst[0] + blk_offset[i + 4], + block + i * 16 * sizeof(pixel), + dst_stride); + } + for (i = 36; i < 40; i++) { + if (nzc[scan8[i + 4]]) + ff_h264_idct_add_lasx(dst[1] + blk_offset[i + 4], + block + i * 16 * sizeof(pixel), + dst_stride); + else if (((dctcoef *) block)[i * 16]) + ff_h264_idct4x4_addblk_dc_lasx(dst[1] + blk_offset[i + 4], + block + i * 16 * sizeof(pixel), + dst_stride); + } +} + +void ff_h264_idct_add16_intra_lasx(uint8_t *dst, + const int32_t *blk_offset, + int16_t *block, + int32_t dst_stride, + const uint8_t nzc[15 * 8]) +{ + int32_t i; + + for (i = 0; i < 16; i++) { + if (nzc[scan8[i]]) + ff_h264_idct_add_lasx(dst + blk_offset[i], + block + i * 16 * sizeof(pixel), dst_stride); + else if (((dctcoef *) block)[i * 16]) + ff_h264_idct4x4_addblk_dc_lasx(dst + blk_offset[i], + block + i * 16 * sizeof(pixel), + dst_stride); + } +} + +void ff_h264_deq_idct_luma_dc_lasx(int16_t *dst, int16_t *src, + int32_t de_qval) +{ +#define DC_DEST_STRIDE 16 + + __m256i src0, src1, src2, src3; + __m256i vec0, vec1, vec2, vec3; + __m256i tmp0, tmp1, tmp2, tmp3; + __m256i hres0, hres1, hres2, hres3; + __m256i vres0, vres1, vres2, vres3; + __m256i de_q_vec = __lasx_xvreplgr2vr_w(de_qval); + + LASX_LD_4(src, 4, src0, src1, src2, src3); + LASX_TRANSPOSE4x4_H_128SV(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3); + BUTTERFLY_4_H(tmp0, tmp2, tmp3, tmp1, vec0, vec3, vec2, vec1); + BUTTERFLY_4_H(vec0, vec1, vec2, vec3, hres0, hres3, hres2, hres1); + LASX_TRANSPOSE4x4_H_128SV(hres0, hres1, hres2, hres3, hres0, hres1, hres2, hres3); + BUTTERFLY_4_H(hres0, hres1, hres3, hres2, vec0, vec3, vec2, vec1); + BUTTERFLY_4_H(vec0, vec1, vec2, vec3, vres0, vres1, vres2, vres3); + LASX_UNPCK_L_W_H_4(vres0, vres1, vres2, vres3, vres0, vres1, vres2, vres3); + LASX_PCKEV_Q_2(vres1, vres0, vres3, vres2, vres0, vres1); + + vres0 = __lasx_xvmul_w(vres0, de_q_vec); + vres1 = __lasx_xvmul_w(vres1, de_q_vec); + + vres0 = __lasx_xvsrari_w(vres0, 8); + vres1 = __lasx_xvsrari_w(vres1, 8); + LASX_PCKEV_H(vres1, vres0, vec0); + __lasx_xvstelm_h(vec0, dst + 0 * DC_DEST_STRIDE, 0, 0); + __lasx_xvstelm_h(vec0, dst + 2 * DC_DEST_STRIDE, 0, 1); + __lasx_xvstelm_h(vec0, dst + 8 * DC_DEST_STRIDE, 0, 2); + __lasx_xvstelm_h(vec0, dst + 10 * DC_DEST_STRIDE, 0, 3); + __lasx_xvstelm_h(vec0, dst + 1 * DC_DEST_STRIDE, 0, 4); + __lasx_xvstelm_h(vec0, dst + 3 * DC_DEST_STRIDE, 0, 5); + __lasx_xvstelm_h(vec0, dst + 9 * DC_DEST_STRIDE, 0, 6); + __lasx_xvstelm_h(vec0, dst + 11 * DC_DEST_STRIDE, 0, 7); + __lasx_xvstelm_h(vec0, dst + 4 * DC_DEST_STRIDE, 0, 8); + __lasx_xvstelm_h(vec0, dst + 6 * DC_DEST_STRIDE, 0, 9); + __lasx_xvstelm_h(vec0, dst + 12 * DC_DEST_STRIDE, 0, 10); + __lasx_xvstelm_h(vec0, dst + 14 * DC_DEST_STRIDE, 0, 11); + __lasx_xvstelm_h(vec0, dst + 5 * DC_DEST_STRIDE, 0, 12); + __lasx_xvstelm_h(vec0, dst + 7 * DC_DEST_STRIDE, 0, 13); + __lasx_xvstelm_h(vec0, dst + 13 * DC_DEST_STRIDE, 0, 14); + __lasx_xvstelm_h(vec0, dst + 15 * DC_DEST_STRIDE, 0, 15); + +#undef DC_DEST_STRIDE +} diff --git a/libavcodec/loongarch/h264qpel_init_loongarch.c b/libavcodec/loongarch/h264qpel_init_loongarch.c new file mode 100644 index 0000000000..b98b2dd793 --- /dev/null +++ b/libavcodec/loongarch/h264qpel_init_loongarch.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "h264qpel_lasx.h" +#include "libavutil/loongarch/cpu.h" +#include "libavcodec/h264qpel.h" + +av_cold void ff_h264qpel_init_loongarch(H264QpelContext *c, int bit_depth) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lasx(cpu_flags)) { + if (8 == bit_depth) { + c->put_h264_qpel_pixels_tab[0][0] = ff_put_h264_qpel16_mc00_lasx; + c->put_h264_qpel_pixels_tab[0][1] = ff_put_h264_qpel16_mc10_lasx; + c->put_h264_qpel_pixels_tab[0][2] = ff_put_h264_qpel16_mc20_lasx; + c->put_h264_qpel_pixels_tab[0][3] = ff_put_h264_qpel16_mc30_lasx; + c->put_h264_qpel_pixels_tab[0][4] = ff_put_h264_qpel16_mc01_lasx; + c->put_h264_qpel_pixels_tab[0][5] = ff_put_h264_qpel16_mc11_lasx; + c->put_h264_qpel_pixels_tab[0][6] = ff_put_h264_qpel16_mc21_lasx; + c->put_h264_qpel_pixels_tab[0][7] = ff_put_h264_qpel16_mc31_lasx; + c->put_h264_qpel_pixels_tab[0][8] = ff_put_h264_qpel16_mc02_lasx; + c->put_h264_qpel_pixels_tab[0][9] = ff_put_h264_qpel16_mc12_lasx; + c->put_h264_qpel_pixels_tab[0][10] = ff_put_h264_qpel16_mc22_lasx; + c->put_h264_qpel_pixels_tab[0][11] = ff_put_h264_qpel16_mc32_lasx; + c->put_h264_qpel_pixels_tab[0][12] = ff_put_h264_qpel16_mc03_lasx; + c->put_h264_qpel_pixels_tab[0][13] = ff_put_h264_qpel16_mc13_lasx; + c->put_h264_qpel_pixels_tab[0][14] = ff_put_h264_qpel16_mc23_lasx; + c->put_h264_qpel_pixels_tab[0][15] = ff_put_h264_qpel16_mc33_lasx; + c->avg_h264_qpel_pixels_tab[0][0] = ff_avg_h264_qpel16_mc00_lasx; + c->avg_h264_qpel_pixels_tab[0][1] = ff_avg_h264_qpel16_mc10_lasx; + c->avg_h264_qpel_pixels_tab[0][2] = ff_avg_h264_qpel16_mc20_lasx; + c->avg_h264_qpel_pixels_tab[0][3] = ff_avg_h264_qpel16_mc30_lasx; + c->avg_h264_qpel_pixels_tab[0][4] = ff_avg_h264_qpel16_mc01_lasx; + c->avg_h264_qpel_pixels_tab[0][5] = ff_avg_h264_qpel16_mc11_lasx; + c->avg_h264_qpel_pixels_tab[0][6] = ff_avg_h264_qpel16_mc21_lasx; + c->avg_h264_qpel_pixels_tab[0][7] = ff_avg_h264_qpel16_mc31_lasx; + c->avg_h264_qpel_pixels_tab[0][8] = ff_avg_h264_qpel16_mc02_lasx; + c->avg_h264_qpel_pixels_tab[0][9] = ff_avg_h264_qpel16_mc12_lasx; + c->avg_h264_qpel_pixels_tab[0][10] = ff_avg_h264_qpel16_mc22_lasx; + c->avg_h264_qpel_pixels_tab[0][11] = ff_avg_h264_qpel16_mc32_lasx; + c->avg_h264_qpel_pixels_tab[0][12] = ff_avg_h264_qpel16_mc03_lasx; + c->avg_h264_qpel_pixels_tab[0][13] = ff_avg_h264_qpel16_mc13_lasx; + c->avg_h264_qpel_pixels_tab[0][14] = ff_avg_h264_qpel16_mc23_lasx; + c->avg_h264_qpel_pixels_tab[0][15] = ff_avg_h264_qpel16_mc33_lasx; + + c->put_h264_qpel_pixels_tab[1][0] = ff_put_h264_qpel8_mc00_lasx; + c->put_h264_qpel_pixels_tab[1][1] = ff_put_h264_qpel8_mc10_lasx; + c->put_h264_qpel_pixels_tab[1][2] = ff_put_h264_qpel8_mc20_lasx; + c->put_h264_qpel_pixels_tab[1][3] = ff_put_h264_qpel8_mc30_lasx; + c->put_h264_qpel_pixels_tab[1][4] = ff_put_h264_qpel8_mc01_lasx; + c->put_h264_qpel_pixels_tab[1][5] = ff_put_h264_qpel8_mc11_lasx; + c->put_h264_qpel_pixels_tab[1][6] = ff_put_h264_qpel8_mc21_lasx; + c->put_h264_qpel_pixels_tab[1][7] = ff_put_h264_qpel8_mc31_lasx; + c->put_h264_qpel_pixels_tab[1][8] = ff_put_h264_qpel8_mc02_lasx; + c->put_h264_qpel_pixels_tab[1][9] = ff_put_h264_qpel8_mc12_lasx; + c->put_h264_qpel_pixels_tab[1][10] = ff_put_h264_qpel8_mc22_lasx; + c->put_h264_qpel_pixels_tab[1][11] = ff_put_h264_qpel8_mc32_lasx; + c->put_h264_qpel_pixels_tab[1][12] = ff_put_h264_qpel8_mc03_lasx; + c->put_h264_qpel_pixels_tab[1][13] = ff_put_h264_qpel8_mc13_lasx; + c->put_h264_qpel_pixels_tab[1][14] = ff_put_h264_qpel8_mc23_lasx; + c->put_h264_qpel_pixels_tab[1][15] = ff_put_h264_qpel8_mc33_lasx; + c->avg_h264_qpel_pixels_tab[1][0] = ff_avg_h264_qpel8_mc00_lasx; + c->avg_h264_qpel_pixels_tab[1][1] = ff_avg_h264_qpel8_mc10_lasx; + c->avg_h264_qpel_pixels_tab[1][2] = ff_avg_h264_qpel8_mc20_lasx; + c->avg_h264_qpel_pixels_tab[1][3] = ff_avg_h264_qpel8_mc30_lasx; + c->avg_h264_qpel_pixels_tab[1][5] = ff_avg_h264_qpel8_mc11_lasx; + c->avg_h264_qpel_pixels_tab[1][6] = ff_avg_h264_qpel8_mc21_lasx; + c->avg_h264_qpel_pixels_tab[1][7] = ff_avg_h264_qpel8_mc31_lasx; + c->avg_h264_qpel_pixels_tab[1][8] = ff_avg_h264_qpel8_mc02_lasx; + c->avg_h264_qpel_pixels_tab[1][9] = ff_avg_h264_qpel8_mc12_lasx; + c->avg_h264_qpel_pixels_tab[1][10] = ff_avg_h264_qpel8_mc22_lasx; + c->avg_h264_qpel_pixels_tab[1][11] = ff_avg_h264_qpel8_mc32_lasx; + c->avg_h264_qpel_pixels_tab[1][13] = ff_avg_h264_qpel8_mc13_lasx; + c->avg_h264_qpel_pixels_tab[1][14] = ff_avg_h264_qpel8_mc23_lasx; + c->avg_h264_qpel_pixels_tab[1][15] = ff_avg_h264_qpel8_mc33_lasx; + } + } +} diff --git a/libavcodec/loongarch/h264qpel_lasx.c b/libavcodec/loongarch/h264qpel_lasx.c new file mode 100644 index 0000000000..9bf1f60a4f --- /dev/null +++ b/libavcodec/loongarch/h264qpel_lasx.c @@ -0,0 +1,2053 @@ +/* + * Loongson LASX optimized h264qpel + * + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "h264qpel_lasx.h" +#include "libavutil/loongarch/generic_macros_lasx.h" +#include "libavutil/attributes.h" + +static const uint8_t luma_mask_arr[16 * 6] __attribute__((aligned(0x40))) = { + /* 8 width cases */ + 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7, 12, + 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7, 12, + 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 9, 7, 10, 8, 11, + 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 9, 7, 10, 8, 11, + 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, + 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 +}; + +#define AVC_HORZ_FILTER_SH(in0, in1, mask0, mask1, mask2) \ +( { \ + __m256i out0_m; \ + __m256i tmp0_m; \ + \ + tmp0_m = __lasx_xvshuf_b(in1, in0, mask0); \ + out0_m = __lasx_xvhaddw_h_b(tmp0_m, tmp0_m); \ + tmp0_m = __lasx_xvshuf_b(in1, in0, mask1); \ + LASX_DP2ADD_H_B(out0_m, minus5b, tmp0_m, out0_m); \ + tmp0_m = __lasx_xvshuf_b(in1, in0, mask2); \ + LASX_DP2ADD_H_B(out0_m, plus20b, tmp0_m, out0_m); \ + \ + out0_m; \ +} ) + +#define AVC_DOT_SH3_SH(in0, in1, in2, coeff0, coeff1, coeff2) \ +( { \ + __m256i out0_m; \ + \ + LASX_DP2_H_B(in0, coeff0, out0_m); \ + LASX_DP2ADD_H_B(out0_m, in1, coeff1, out0_m); \ + LASX_DP2ADD_H_B(out0_m, in2, coeff2, out0_m); \ + \ + out0_m; \ +} ) + +static av_always_inline +void avc_luma_hv_qrt_and_aver_dst_16x16_lasx(const uint8_t *src_x, + const uint8_t *src_y, + uint8_t *dst, int32_t stride) +{ + const int16_t filt_const0 = 0xfb01; + const int16_t filt_const1 = 0x1414; + const int16_t filt_const2 = 0x1fb; + uint32_t loop_cnt; + int32_t stride_x2 = stride << 1; + int32_t stride_x3 = stride_x2 + stride; + int32_t stride_x4 = stride << 2; + __m256i tmp0, tmp1; + __m256i src_hz0, src_hz1, src_hz2, src_hz3, mask0, mask1, mask2; + __m256i src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6; + __m256i src_vt7, src_vt8; + __m256i src_vt10_h, src_vt21_h, src_vt32_h, src_vt43_h, src_vt54_h; + __m256i src_vt65_h, src_vt76_h, src_vt87_h, filt0, filt1, filt2; + __m256i hz_out0, hz_out1, hz_out2, hz_out3, vt_out0, vt_out1, vt_out2; + __m256i vt_out3, out0, out1, out2, out3; + __m256i minus5b = __lasx_xvldi(0xFB); + __m256i plus20b = __lasx_xvldi(20); + + filt0 = __lasx_xvreplgr2vr_h(filt_const0); + filt1 = __lasx_xvreplgr2vr_h(filt_const1); + filt2 = __lasx_xvreplgr2vr_h(filt_const2); + + mask0 = LASX_LD(luma_mask_arr); + LASX_LD_2(luma_mask_arr + 32, 32, mask1, mask2); + src_vt0 = LASX_LD(src_y); + LASX_LD_4(src_y + stride, stride, src_vt1, src_vt2, src_vt3, src_vt4); + src_y += stride_x4 + stride; + + LASX_XORI_B_128(src_vt0); + LASX_XORI_B_4_128(src_vt1, src_vt2, src_vt3, src_vt4); + + for (loop_cnt = 4; loop_cnt--;) { + LASX_LD_4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3); + src_x += stride_x4; + src_hz0 = __lasx_xvpermi_d(src_hz0, 0x94); + src_hz1 = __lasx_xvpermi_d(src_hz1, 0x94); + src_hz2 = __lasx_xvpermi_d(src_hz2, 0x94); + src_hz3 = __lasx_xvpermi_d(src_hz3, 0x94); + LASX_XORI_B_4_128(src_hz0, src_hz1, src_hz2, src_hz3); + + hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2); + hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2); + hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2); + hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2); + hz_out0 = __lasx_xvssrarni_b_h(hz_out1, hz_out0, 5); + hz_out2 = __lasx_xvssrarni_b_h(hz_out3, hz_out2, 5); + + LASX_LD_4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8); + src_y += stride_x4; + + LASX_XORI_B_4_128(src_vt5, src_vt6, src_vt7, src_vt8); + LASX_ILVL_B_4(src_vt1, src_vt0, src_vt2, src_vt1, src_vt3, src_vt2, + src_vt4, src_vt3, src_vt10_h, src_vt21_h, src_vt32_h, + src_vt43_h); + LASX_ILVL_B_4(src_vt5, src_vt4, src_vt6, src_vt5, src_vt7, src_vt6, + src_vt8, src_vt7, src_vt54_h, src_vt65_h, src_vt76_h, + src_vt87_h); + vt_out0 = AVC_DOT_SH3_SH(src_vt10_h, src_vt32_h, src_vt54_h, filt0, + filt1, filt2); + vt_out1 = AVC_DOT_SH3_SH(src_vt21_h, src_vt43_h, src_vt65_h, filt0, + filt1, filt2); + vt_out2 = AVC_DOT_SH3_SH(src_vt32_h, src_vt54_h, src_vt76_h, filt0, + filt1, filt2); + vt_out3 = AVC_DOT_SH3_SH(src_vt43_h, src_vt65_h, src_vt87_h, filt0, + filt1, filt2); + vt_out0 = __lasx_xvssrarni_b_h(vt_out1, vt_out0, 5); + vt_out2 = __lasx_xvssrarni_b_h(vt_out3, vt_out2, 5); + + LASX_ADDWL_H_B_2_128SV(hz_out0, vt_out0, hz_out2, vt_out2, out0, out2); + LASX_ADDWH_H_B_2_128SV(hz_out0, vt_out0, hz_out2, vt_out2, out1, out3); + tmp0 = __lasx_xvssrarni_b_h(out1, out0, 1); + tmp1 = __lasx_xvssrarni_b_h(out3, out2, 1); + + LASX_XORI_B_2_128(tmp0, tmp1); + + LASX_LD_4(dst, stride, out0, out1, out2, out3); + LASX_ILVL_D_2(out1, out0, out3, out2, out0, out1); + tmp0 = __lasx_xvavgr_bu(out0, tmp0); + tmp1 = __lasx_xvavgr_bu(out1, tmp1); + + *(int64_t*)dst = __lasx_xvpickve2gr_d(tmp0, 0); + *(int64_t*)(dst + stride) = __lasx_xvpickve2gr_d(tmp0, 1); + *(int64_t*)(dst + stride_x2) = __lasx_xvpickve2gr_d(tmp1, 0); + *(int64_t*)(dst + stride_x3) = __lasx_xvpickve2gr_d(tmp1, 1); + + *(int64_t*)(dst + 8) = __lasx_xvpickve2gr_d(tmp0, 2); + *(int64_t*)(dst + 8 + stride) = __lasx_xvpickve2gr_d(tmp0, 3); + *(int64_t*)(dst + 8 + stride_x2) = __lasx_xvpickve2gr_d(tmp1, 2); + *(int64_t*)(dst + 8 + stride_x3) = __lasx_xvpickve2gr_d(tmp1, 3); + + dst += stride_x4; + src_vt0 = src_vt4; + src_vt1 = src_vt5; + src_vt2 = src_vt6; + src_vt3 = src_vt7; + src_vt4 = src_vt8; + } +} + +static av_always_inline void +avc_luma_hv_qrt_16x16_lasx(const uint8_t *src_x, const uint8_t *src_y, uint8_t *dst, + int32_t stride) +{ + const int16_t filt_const0 = 0xfb01; + const int16_t filt_const1 = 0x1414; + const int16_t filt_const2 = 0x1fb; + uint32_t loop_cnt; + int32_t stride_x2 = stride << 1; + int32_t stride_x3 = stride_x2 + stride; + int32_t stride_x4 = stride << 2; + __m256i tmp0, tmp1; + __m256i src_hz0, src_hz1, src_hz2, src_hz3, mask0, mask1, mask2; + __m256i src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6; + __m256i src_vt7, src_vt8; + __m256i src_vt10_h, src_vt21_h, src_vt32_h, src_vt43_h, src_vt54_h; + __m256i src_vt65_h, src_vt76_h, src_vt87_h, filt0, filt1, filt2; + __m256i hz_out0, hz_out1, hz_out2, hz_out3, vt_out0, vt_out1, vt_out2; + __m256i vt_out3, out0, out1, out2, out3; + __m256i minus5b = __lasx_xvldi(0xFB); + __m256i plus20b = __lasx_xvldi(20); + + filt0 = __lasx_xvreplgr2vr_h(filt_const0); + filt1 = __lasx_xvreplgr2vr_h(filt_const1); + filt2 = __lasx_xvreplgr2vr_h(filt_const2); + + mask0 = LASX_LD(luma_mask_arr); + LASX_LD_2(luma_mask_arr + 32, 32, mask1, mask2); + src_vt0 = LASX_LD(src_y); + LASX_LD_4(src_y + stride, stride, src_vt1, src_vt2, src_vt3, src_vt4); + src_y += stride_x4 + stride; + + LASX_XORI_B_128(src_vt0); + LASX_XORI_B_4_128(src_vt1, src_vt2, src_vt3, src_vt4); + + for (loop_cnt = 4; loop_cnt--;) { + LASX_LD_4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3); + src_x += stride_x4; + src_hz0 = __lasx_xvpermi_d(src_hz0, 0x94); + src_hz1 = __lasx_xvpermi_d(src_hz1, 0x94); + src_hz2 = __lasx_xvpermi_d(src_hz2, 0x94); + src_hz3 = __lasx_xvpermi_d(src_hz3, 0x94); + LASX_XORI_B_4_128(src_hz0, src_hz1, src_hz2, src_hz3); + + hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2); + hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2); + hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2); + hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2); + hz_out0 = __lasx_xvssrarni_b_h(hz_out1, hz_out0, 5); + hz_out2 = __lasx_xvssrarni_b_h(hz_out3, hz_out2, 5); + + LASX_LD_4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8); + src_y += stride_x4; + + LASX_XORI_B_4_128(src_vt5, src_vt6, src_vt7, src_vt8); + LASX_ILVL_B_4(src_vt1, src_vt0, src_vt2, src_vt1, src_vt3, src_vt2, + src_vt4, src_vt3, src_vt10_h, src_vt21_h, src_vt32_h, + src_vt43_h); + LASX_ILVL_B_4(src_vt5, src_vt4, src_vt6, src_vt5, src_vt7, src_vt6, + src_vt8, src_vt7, src_vt54_h, src_vt65_h, src_vt76_h, + src_vt87_h); + vt_out0 = AVC_DOT_SH3_SH(src_vt10_h, src_vt32_h, src_vt54_h, filt0, + filt1, filt2); + vt_out1 = AVC_DOT_SH3_SH(src_vt21_h, src_vt43_h, src_vt65_h, filt0, + filt1, filt2); + vt_out2 = AVC_DOT_SH3_SH(src_vt32_h, src_vt54_h, src_vt76_h, filt0, + filt1, filt2); + vt_out3 = AVC_DOT_SH3_SH(src_vt43_h, src_vt65_h, src_vt87_h, filt0, + filt1, filt2); + vt_out0 = __lasx_xvssrarni_b_h(vt_out1, vt_out0, 5); + vt_out2 = __lasx_xvssrarni_b_h(vt_out3, vt_out2, 5); + + LASX_ADDWL_H_B_2_128SV(hz_out0, vt_out0, hz_out2, vt_out2, out0, out2); + LASX_ADDWH_H_B_2_128SV(hz_out0, vt_out0, hz_out2, vt_out2, out1, out3); + tmp0 = __lasx_xvssrarni_b_h(out1, out0, 1); + tmp1 = __lasx_xvssrarni_b_h(out3, out2, 1); + + LASX_XORI_B_2_128(tmp0, tmp1); + *(int64_t*)dst = __lasx_xvpickve2gr_d(tmp0, 0); + *(int64_t*)(dst + stride) = __lasx_xvpickve2gr_d(tmp0, 1); + *(int64_t*)(dst + stride_x2) = __lasx_xvpickve2gr_d(tmp1, 0); + *(int64_t*)(dst + stride_x3) = __lasx_xvpickve2gr_d(tmp1, 1); + + *(int64_t*)(dst + 8) = __lasx_xvpickve2gr_d(tmp0, 2); + *(int64_t*)(dst + 8 + stride) = __lasx_xvpickve2gr_d(tmp0, 3); + *(int64_t*)(dst + 8 + stride_x2) = __lasx_xvpickve2gr_d(tmp1, 2); + *(int64_t*)(dst + 8 + stride_x3) = __lasx_xvpickve2gr_d(tmp1, 3); + + dst += stride_x4; + src_vt0 = src_vt4; + src_vt1 = src_vt5; + src_vt2 = src_vt6; + src_vt3 = src_vt7; + src_vt4 = src_vt8; + } +} + +/* put_pixels8_8_inline_asm: dst = src */ +static av_always_inline void +put_pixels8_8_inline_asm(uint8_t *dst, const uint8_t *src, ptrdiff_t stride) +{ + uint64_t tmp[8]; + __asm__ volatile ( + "ld.d %[tmp0], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp1], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp2], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp3], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp4], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp5], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp6], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp7], %[src], 0x0 \n\t" + + "st.d %[tmp0], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp1], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp2], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp3], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp4], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp5], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp6], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp7], %[dst], 0x0 \n\t" + : [tmp0]"=&r"(tmp[0]), [tmp1]"=&r"(tmp[1]), + [tmp2]"=&r"(tmp[2]), [tmp3]"=&r"(tmp[3]), + [tmp4]"=&r"(tmp[4]), [tmp5]"=&r"(tmp[5]), + [tmp6]"=&r"(tmp[6]), [tmp7]"=&r"(tmp[7]), + [dst]"+&r"(dst), [src]"+&r"(src) + : [stride]"r"(stride) + : "memory" + ); +} + +/* avg_pixels8_8_lsx : dst = avg(src, dst) + * put_pixels8_l2_8_lsx: dst = avg(src, half) , half stride is 8. + * avg_pixels8_l2_8_lsx: dst = avg(avg(src, half), dst) , half stride is 8.*/ +static av_always_inline void +avg_pixels8_8_lsx(uint8_t *dst, const uint8_t *src, ptrdiff_t stride) +{ + uint8_t *tmp = dst; + __asm__ volatile ( + /* h0~h7 */ + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr7, %[src], 0 \n\t" + + "vld $vr8, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr9, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr10, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr11, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr12, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr13, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr14, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr15, %[tmp], 0 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vstelm.d $vr0, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vstelm.d $vr1, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vstelm.d $vr2, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vstelm.d $vr3, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vstelm.d $vr4, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vstelm.d $vr5, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vstelm.d $vr6, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vstelm.d $vr7, %[dst], 0, 0 \n\t" + : [dst]"+&r"(dst), [tmp]"+&r"(tmp), [src]"+&r"(src) + : [stride]"r"(stride) + : "memory" + ); +} + +/* avg_pixels8_8_lsx : dst = avg(src, dst) + * put_pixels8_l2_8_lsx: dst = avg(src, half) , half stride is 8. + * avg_pixels8_l2_8_lsx: dst = avg(avg(src, half), dst) , half stride is 8.*/ +static av_always_inline void +put_pixels8_l2_8_lsx(uint8_t *dst, const uint8_t *src, const uint8_t *half, + ptrdiff_t dstStride, ptrdiff_t srcStride) +{ + __asm__ volatile ( + /* h0~h7 */ + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr7, %[src], 0 \n\t" + + "vld $vr8, %[half], 0x00 \n\t" + "vld $vr9, %[half], 0x08 \n\t" + "vld $vr10, %[half], 0x10 \n\t" + "vld $vr11, %[half], 0x18 \n\t" + "vld $vr12, %[half], 0x20 \n\t" + "vld $vr13, %[half], 0x28 \n\t" + "vld $vr14, %[half], 0x30 \n\t" + "vld $vr15, %[half], 0x38 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vstelm.d $vr0, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr1, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr2, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr3, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr4, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr5, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr6, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr7, %[dst], 0, 0 \n\t" + : [dst]"+&r"(dst), [half]"+&r"(half), [src]"+&r"(src) + : [srcStride]"r"(srcStride), [dstStride]"r"(dstStride) + : "memory" + ); +} + +/* avg_pixels8_8_lsx : dst = avg(src, dst) + * put_pixels8_l2_8_lsx: dst = avg(src, half) , half stride is 8. + * avg_pixels8_l2_8_lsx: dst = avg(avg(src, half), dst) , half stride is 8.*/ +static av_always_inline void +avg_pixels8_l2_8_lsx(uint8_t *dst, const uint8_t *src, const uint8_t *half, + ptrdiff_t dstStride, ptrdiff_t srcStride) +{ + uint8_t *tmp = dst; + __asm__ volatile ( + /* h0~h7 */ + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr7, %[src], 0 \n\t" + + "vld $vr8, %[half], 0x00 \n\t" + "vld $vr9, %[half], 0x08 \n\t" + "vld $vr10, %[half], 0x10 \n\t" + "vld $vr11, %[half], 0x18 \n\t" + "vld $vr12, %[half], 0x20 \n\t" + "vld $vr13, %[half], 0x28 \n\t" + "vld $vr14, %[half], 0x30 \n\t" + "vld $vr15, %[half], 0x38 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vld $vr8, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr9, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr10, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr11, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr12, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr13, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr14, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr15, %[tmp], 0 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vstelm.d $vr0, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr1, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr2, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr3, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr4, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr5, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr6, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr7, %[dst], 0, 0 \n\t" + : [dst]"+&r"(dst), [tmp]"+&r"(tmp), [half]"+&r"(half), [src]"+&r"(src) + : [dstStride]"r"(dstStride), [srcStride]"r"(srcStride) + : "memory" + ); +} + +/* put_pixels16_8_lsx: dst = src */ +static av_always_inline void +put_pixels16_8_lsx(uint8_t *dst, const uint8_t *src, ptrdiff_t stride) +{ + __asm__ volatile ( + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr7, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + + "vst $vr0, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr1, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr2, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr3, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr4, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr5, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr6, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr7, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr7, %[src], 0 \n\t" + + "vst $vr0, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr1, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr2, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr3, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr4, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr5, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr6, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr7, %[dst], 0 \n\t" + : [dst]"+&r"(dst), [src]"+&r"(src) + : [stride]"r"(stride) + : "memory" + ); +} + +/* avg_pixels16_8_lsx : dst = avg(src, dst) + * put_pixels16_l2_8_lsx: dst = avg(src, half) , half stride is 8. + * avg_pixels16_l2_8_lsx: dst = avg(avg(src, half), dst) , half stride is 8.*/ +static av_always_inline void +avg_pixels16_8_lsx(uint8_t *dst, const uint8_t *src, ptrdiff_t stride) +{ + uint8_t *tmp = dst; + __asm__ volatile ( + /* h0~h7 */ + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr7, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + + "vld $vr8, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr9, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr10, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr11, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr12, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr13, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr14, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr15, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vst $vr0, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr1, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr2, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr3, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr4, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr5, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr6, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr7, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + + /* h8~h15 */ + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr7, %[src], 0 \n\t" + + "vld $vr8, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr9, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr10, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr11, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr12, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr13, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr14, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[stride] \n\t" + "vld $vr15, %[tmp], 0 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vst $vr0, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr1, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr2, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr3, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr4, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr5, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr6, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr7, %[dst], 0 \n\t" + : [dst]"+&r"(dst), [tmp]"+&r"(tmp), [src]"+&r"(src) + : [stride]"r"(stride) + : "memory" + ); +} + +/* avg_pixels16_8_lsx : dst = avg(src, dst) + * put_pixels16_l2_8_lsx: dst = avg(src, half) , half stride is 8. + * avg_pixels16_l2_8_lsx: dst = avg(avg(src, half), dst) , half stride is 8.*/ +static av_always_inline void +put_pixels16_l2_8_lsx(uint8_t *dst, const uint8_t *src, uint8_t *half, + ptrdiff_t dstStride, ptrdiff_t srcStride) +{ + __asm__ volatile ( + /* h0~h7 */ + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr7, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + + "vld $vr8, %[half], 0x00 \n\t" + "vld $vr9, %[half], 0x10 \n\t" + "vld $vr10, %[half], 0x20 \n\t" + "vld $vr11, %[half], 0x30 \n\t" + "vld $vr12, %[half], 0x40 \n\t" + "vld $vr13, %[half], 0x50 \n\t" + "vld $vr14, %[half], 0x60 \n\t" + "vld $vr15, %[half], 0x70 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vst $vr0, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr1, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr2, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr3, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr4, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr5, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr6, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr7, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + + /* h8~h15 */ + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr7, %[src], 0 \n\t" + + "vld $vr8, %[half], 0x80 \n\t" + "vld $vr9, %[half], 0x90 \n\t" + "vld $vr10, %[half], 0xa0 \n\t" + "vld $vr11, %[half], 0xb0 \n\t" + "vld $vr12, %[half], 0xc0 \n\t" + "vld $vr13, %[half], 0xd0 \n\t" + "vld $vr14, %[half], 0xe0 \n\t" + "vld $vr15, %[half], 0xf0 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vst $vr0, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr1, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr2, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr3, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr4, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr5, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr6, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr7, %[dst], 0 \n\t" + : [dst]"+&r"(dst), [half]"+&r"(half), [src]"+&r"(src) + : [dstStride]"r"(dstStride), [srcStride]"r"(srcStride) + : "memory" + ); +} + +/* avg_pixels16_8_lsx : dst = avg(src, dst) + * put_pixels16_l2_8_lsx: dst = avg(src, half) , half stride is 8. + * avg_pixels16_l2_8_lsx: dst = avg(avg(src, half), dst) , half stride is 8.*/ +static av_always_inline void +avg_pixels16_l2_8_lsx(uint8_t *dst, const uint8_t *src, uint8_t *half, + ptrdiff_t dstStride, ptrdiff_t srcStride) +{ + uint8_t *tmp = dst; + __asm__ volatile ( + /* h0~h7 */ + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr7, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + + "vld $vr8, %[half], 0x00 \n\t" + "vld $vr9, %[half], 0x10 \n\t" + "vld $vr10, %[half], 0x20 \n\t" + "vld $vr11, %[half], 0x30 \n\t" + "vld $vr12, %[half], 0x40 \n\t" + "vld $vr13, %[half], 0x50 \n\t" + "vld $vr14, %[half], 0x60 \n\t" + "vld $vr15, %[half], 0x70 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vld $vr8, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr9, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr10, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr11, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr12, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr13, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr14, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr15, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vst $vr0, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr1, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr2, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr3, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr4, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr5, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr6, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr7, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + + /* h8~h15 */ + "vld $vr0, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr1, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr2, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr3, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr4, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr5, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr6, %[src], 0 \n\t" + "add.d %[src], %[src], %[srcStride] \n\t" + "vld $vr7, %[src], 0 \n\t" + + "vld $vr8, %[half], 0x80 \n\t" + "vld $vr9, %[half], 0x90 \n\t" + "vld $vr10, %[half], 0xa0 \n\t" + "vld $vr11, %[half], 0xb0 \n\t" + "vld $vr12, %[half], 0xc0 \n\t" + "vld $vr13, %[half], 0xd0 \n\t" + "vld $vr14, %[half], 0xe0 \n\t" + "vld $vr15, %[half], 0xf0 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vld $vr8, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr9, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr10, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr11, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr12, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr13, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr14, %[tmp], 0 \n\t" + "add.d %[tmp], %[tmp], %[dstStride] \n\t" + "vld $vr15, %[tmp], 0 \n\t" + + "vavgr.bu $vr0, $vr8, $vr0 \n\t" + "vavgr.bu $vr1, $vr9, $vr1 \n\t" + "vavgr.bu $vr2, $vr10, $vr2 \n\t" + "vavgr.bu $vr3, $vr11, $vr3 \n\t" + "vavgr.bu $vr4, $vr12, $vr4 \n\t" + "vavgr.bu $vr5, $vr13, $vr5 \n\t" + "vavgr.bu $vr6, $vr14, $vr6 \n\t" + "vavgr.bu $vr7, $vr15, $vr7 \n\t" + + "vst $vr0, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr1, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr2, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr3, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr4, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr5, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr6, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr7, %[dst], 0 \n\t" + : [dst]"+&r"(dst), [tmp]"+&r"(tmp), [half]"+&r"(half), [src]"+&r"(src) + : [dstStride]"r"(dstStride), [srcStride]"r"(srcStride) + : "memory" + ); +} + +#define QPEL8_H_LOWPASS(out_v) \ + src00 = LASX_LD(src - 2); \ + src += srcStride; \ + src10 = LASX_LD(src - 2); \ + src += srcStride; \ + src00 = __lasx_xvpermi_q(src00, src10, 0x02); \ + src01 = __lasx_xvshuf_b(src00, src00, (__m256i)mask1); \ + src02 = __lasx_xvshuf_b(src00, src00, (__m256i)mask2); \ + src03 = __lasx_xvshuf_b(src00, src00, (__m256i)mask3); \ + src04 = __lasx_xvshuf_b(src00, src00, (__m256i)mask4); \ + src05 = __lasx_xvshuf_b(src00, src00, (__m256i)mask5); \ + LASX_ADDWL_H_BU_2_128SV(src02, src03, src01, src04, src02, src01); \ + LASX_ADDWL_H_BU_128SV(src00, src05, src00); \ + src02 = __lasx_xvmul_h(src02, h_20); \ + src01 = __lasx_xvmul_h(src01, h_5); \ + src02 = __lasx_xvssub_h(src02, src01); \ + src02 = __lasx_xvsadd_h(src02, src00); \ + src02 = __lasx_xvsadd_h(src02, h_16); \ + out_v = __lasx_xvssrani_bu_h(src02, src02, 5); \ + +static av_always_inline void +put_h264_qpel8_h_lowpass_lasx(uint8_t *dst, const uint8_t *src, int dstStride, + int srcStride) +{ + int dstStride_2x = dstStride << 1; + __m256i src00, src01, src02, src03, src04, src05, src10; + __m256i out0, out1, out2, out3; + __m256i zero = {0}; + __m256i h_20 = {20}; + __m256i h_5 = {5}; + __m256i h_16 = {16}; + __m256i mask1 = {0x0807060504030201, 0x0, 0x0807060504030201, 0x0}; + __m256i mask2 = {0x0908070605040302, 0x0, 0x0908070605040302, 0x0}; + __m256i mask3 = {0x0a09080706050403, 0x0, 0x0a09080706050403, 0x0}; + __m256i mask4 = {0x0b0a090807060504, 0x0, 0x0b0a090807060504, 0x0}; + __m256i mask5 = {0x0c0b0a0908070605, 0x0, 0x0c0b0a0908070605, 0x0}; + + zero = __lasx_xvreplve0_h(zero); + h_20 = __lasx_xvreplve0_h(h_20); + h_5 = __lasx_xvreplve0_h(h_5); + h_16 = __lasx_xvreplve0_h(h_16); + + QPEL8_H_LOWPASS(out0) + QPEL8_H_LOWPASS(out1) + QPEL8_H_LOWPASS(out2) + QPEL8_H_LOWPASS(out3) + LASX_ST_D_2(out0, 0, 2, dst, dstStride); + dst += dstStride_2x; + LASX_ST_D_2(out1, 0, 2, dst, dstStride); + dst += dstStride_2x; + LASX_ST_D_2(out2, 0, 2, dst, dstStride); + dst += dstStride_2x; + LASX_ST_D_2(out3, 0, 2, dst, dstStride); +} + +#define QPEL8_V_LOWPASS(src0, src1, src2, src3, src4, src5, src6, \ + tmp0, tmp1, tmp2, tmp3, tmp4, tmp5) \ +{ \ + tmp0 = __lasx_xvpermi_q(src0, src1, 0x02); \ + tmp1 = __lasx_xvpermi_q(src1, src2, 0x02); \ + tmp2 = __lasx_xvpermi_q(src2, src3, 0x02); \ + tmp3 = __lasx_xvpermi_q(src3, src4, 0x02); \ + tmp4 = __lasx_xvpermi_q(src4, src5, 0x02); \ + tmp5 = __lasx_xvpermi_q(src5, src6, 0x02); \ + LASX_ADDWL_H_BU_2_128SV(tmp2, tmp3, tmp1, tmp4, tmp2, tmp1); \ + LASX_ADDWL_H_BU_128SV(tmp0, tmp5, tmp0); \ + tmp2 = __lasx_xvmul_h(tmp2, h_20); \ + tmp1 = __lasx_xvmul_h(tmp1, h_5); \ + tmp2 = __lasx_xvssub_h(tmp2, tmp1); \ + tmp2 = __lasx_xvsadd_h(tmp2, tmp0); \ + tmp2 = __lasx_xvsadd_h(tmp2, h_16); \ + tmp2 = __lasx_xvssrani_bu_h(tmp2, tmp2, 5); \ +} + +static av_always_inline void +put_h264_qpel8_v_lowpass_lasx(uint8_t *dst, const uint8_t *src, int dstStride, + int srcStride) +{ + int srcStride_2x = srcStride << 1; + __m256i src00, src01, src02, src03, src04, src05, src06; + __m256i src07, src08, src09, src10, src11, src12; + __m256i tmp00, tmp01, tmp02, tmp03, tmp04, tmp05; + __m256i zero = {0}; + __m256i h_20 = {20}; + __m256i h_5 = {5}; + __m256i h_16 = {16}; + + zero = __lasx_xvreplve0_h(zero); + h_20 = __lasx_xvreplve0_h(h_20); + h_5 = __lasx_xvreplve0_h(h_5); + h_16 = __lasx_xvreplve0_h(h_16); + + LASX_LD_2(src - srcStride_2x, srcStride, src00, src01); + LASX_LD_8(src, srcStride, src02, src03, src04, src05, + src06, src07, src08, src09); + src += srcStride << 3; + LASX_LD_2(src, srcStride, src10, src11); + src += srcStride_2x; + src12 = LASX_LD(src); + + QPEL8_V_LOWPASS(src00, src01, src02, src03, src04, src05, src06, + tmp00, tmp01, tmp02, tmp03, tmp04, tmp05); + LASX_ST_D_2(tmp02, 0, 2, dst, dstStride); + dst += dstStride << 1; + QPEL8_V_LOWPASS(src02, src03, src04, src05, src06, src07, src08, + tmp00, tmp01, tmp02, tmp03, tmp04, tmp05); + LASX_ST_D_2(tmp02, 0, 2, dst, dstStride); + dst += dstStride << 1; + QPEL8_V_LOWPASS(src04, src05, src06, src07, src08, src09, src10, + tmp00, tmp01, tmp02, tmp03, tmp04, tmp05); + LASX_ST_D_2(tmp02, 0, 2, dst, dstStride); + dst += dstStride << 1; + QPEL8_V_LOWPASS(src06, src07, src08, src09, src10, src11, src12, + tmp00, tmp01, tmp02, tmp03, tmp04, tmp05); + LASX_ST_D_2(tmp02, 0, 2, dst, dstStride); +} + +static av_always_inline void +avg_h264_qpel8_v_lowpass_lasx(uint8_t *dst, const uint8_t *src, int dstStride, + int srcStride) +{ + int srcStride_2x = srcStride << 1; + __m256i src00, src01, src02, src03, src04, src05, src06; + __m256i src07, src08, src09, src10, src11, src12; + __m256i tmp00, tmp01, tmp02, tmp03, tmp04, tmp05, tmp06, tmp07, tmp08, tmp09; + __m256i zero = {0}; + __m256i h_20 = {20}; + __m256i h_5 = {5}; + __m256i h_16 = {16}; + + zero = __lasx_xvreplve0_h(zero); + h_20 = __lasx_xvreplve0_h(h_20); + h_5 = __lasx_xvreplve0_h(h_5); + h_16 = __lasx_xvreplve0_h(h_16); + + LASX_LD_2(src - srcStride_2x, srcStride, src00, src01); + LASX_LD_8(src, srcStride, src02, src03, src04, src05, + src06, src07, src08, src09); + src += srcStride << 3; + LASX_LD_2(src, srcStride, src10, src11); + src += srcStride_2x; + src12 = LASX_LD(src); + + LASX_LD_8(dst, dstStride, tmp06, tmp07, tmp02, tmp03, + tmp04, tmp05, tmp00, tmp01); + tmp06 = __lasx_xvpermi_q(tmp06, tmp07, 0x02); + tmp07 = __lasx_xvpermi_q(tmp02, tmp03, 0x02); + tmp08 = __lasx_xvpermi_q(tmp04, tmp05, 0x02); + tmp09 = __lasx_xvpermi_q(tmp00, tmp01, 0x02); + + QPEL8_V_LOWPASS(src00, src01, src02, src03, src04, src05, src06, + tmp00, tmp01, tmp02, tmp03, tmp04, tmp05); + tmp06 = __lasx_xvavgr_bu(tmp06, tmp02); + LASX_ST_D_2(tmp06, 0, 2, dst, dstStride); + dst += dstStride << 1; + QPEL8_V_LOWPASS(src02, src03, src04, src05, src06, src07, src08, + tmp00, tmp01, tmp02, tmp03, tmp04, tmp05); + tmp07 = __lasx_xvavgr_bu(tmp07, tmp02); + LASX_ST_D_2(tmp07, 0, 2, dst, dstStride); + dst += dstStride << 1; + QPEL8_V_LOWPASS(src04, src05, src06, src07, src08, src09, src10, + tmp00, tmp01, tmp02, tmp03, tmp04, tmp05); + tmp08 = __lasx_xvavgr_bu(tmp08, tmp02); + LASX_ST_D_2(tmp08, 0, 2, dst, dstStride); + dst += dstStride << 1; + QPEL8_V_LOWPASS(src06, src07, src08, src09, src10, src11, src12, + tmp00, tmp01, tmp02, tmp03, tmp04, tmp05); + tmp09 = __lasx_xvavgr_bu(tmp09, tmp02); + LASX_ST_D_2(tmp09, 0, 2, dst, dstStride); +} + +#define QPEL8_HV_LOWPASS_H(tmp) \ +{ \ + src00 = LASX_LD(src - 2); \ + src += srcStride; \ + src10 = LASX_LD(src - 2); \ + src += srcStride; \ + src00 = __lasx_xvpermi_q(src00, src10, 0x02); \ + src01 = __lasx_xvshuf_b(src00, src00, (__m256i)mask1); \ + src02 = __lasx_xvshuf_b(src00, src00, (__m256i)mask2); \ + src03 = __lasx_xvshuf_b(src00, src00, (__m256i)mask3); \ + src04 = __lasx_xvshuf_b(src00, src00, (__m256i)mask4); \ + src05 = __lasx_xvshuf_b(src00, src00, (__m256i)mask5); \ + LASX_ADDWL_H_BU_2_128SV(src02, src03, src01, src04, src02, src01); \ + LASX_ADDWL_H_BU_128SV(src00, src05, src00); \ + src02 = __lasx_xvmul_h(src02, h_20); \ + src01 = __lasx_xvmul_h(src01, h_5); \ + src02 = __lasx_xvssub_h(src02, src01); \ + tmp = __lasx_xvsadd_h(src02, src00); \ +} + +#define QPEL8_HV_LOWPASS_V(src0, src1, src2, src3, \ + src4, src5, temp0, temp1, \ + temp2, temp3, temp4, temp5, \ + out) \ +{ \ + LASX_ADDWL_W_H_2_128SV(src2, src3, src1, src4, temp0, temp2); \ + LASX_ADDWH_W_H_2_128SV(src2, src3, src1, src4, temp1, temp3); \ + LASX_ADDWL_W_H_128SV(src0, src5, temp4); \ + LASX_ADDWH_W_H_128SV(src0, src5, temp5); \ + temp0 = __lasx_xvmul_w(temp0, w_20); \ + temp1 = __lasx_xvmul_w(temp1, w_20); \ + temp2 = __lasx_xvmul_w(temp2, w_5); \ + temp3 = __lasx_xvmul_w(temp3, w_5); \ + temp0 = __lasx_xvssub_w(temp0, temp2); \ + temp1 = __lasx_xvssub_w(temp1, temp3); \ + temp0 = __lasx_xvsadd_w(temp0, temp4); \ + temp1 = __lasx_xvsadd_w(temp1, temp5); \ + temp0 = __lasx_xvsadd_w(temp0, w_512); \ + temp1 = __lasx_xvsadd_w(temp1, w_512); \ + temp0 = __lasx_xvssrani_hu_w(temp0, temp0, 10); \ + temp1 = __lasx_xvssrani_hu_w(temp1, temp1, 10); \ + temp0 = __lasx_xvpackev_d(temp1, temp0); \ + out = __lasx_xvssrani_bu_h(temp0, temp0, 0); \ +} + +static av_always_inline void +put_h264_qpel8_hv_lowpass_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dstStride, ptrdiff_t srcStride) +{ + __m256i src00, src01, src02, src03, src04, src05, src10; + __m256i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6; + __m256i tmp7, tmp8, tmp9, tmp10, tmp11, tmp12; + __m256i h_20 = {20}; + __m256i h_5 = {5}; + __m256i w_20 = {20}; + __m256i w_5 = {5}; + __m256i w_512 = {512}; + __m256i mask1 = {0x0807060504030201, 0x0, 0x0807060504030201, 0x0}; + __m256i mask2 = {0x0908070605040302, 0x0, 0x0908070605040302, 0x0}; + __m256i mask3 = {0x0a09080706050403, 0x0, 0x0a09080706050403, 0x0}; + __m256i mask4 = {0x0b0a090807060504, 0x0, 0x0b0a090807060504, 0x0}; + __m256i mask5 = {0x0c0b0a0908070605, 0x0, 0x0c0b0a0908070605, 0x0}; + + h_20 = __lasx_xvreplve0_h(w_20); + h_5 = __lasx_xvreplve0_h(w_5); + w_20 = __lasx_xvreplve0_w(w_20); + w_5 = __lasx_xvreplve0_w(w_5); + w_512 = __lasx_xvreplve0_w(w_512); + + src -= srcStride << 1; + QPEL8_HV_LOWPASS_H(tmp0) + QPEL8_HV_LOWPASS_H(tmp2) + QPEL8_HV_LOWPASS_H(tmp4) + QPEL8_HV_LOWPASS_H(tmp6) + QPEL8_HV_LOWPASS_H(tmp8) + QPEL8_HV_LOWPASS_H(tmp10) + QPEL8_HV_LOWPASS_H(tmp12) + tmp11 = __lasx_xvpermi_q(tmp12, tmp10, 0x21); + tmp9 = __lasx_xvpermi_q(tmp10, tmp8, 0x21); + tmp7 = __lasx_xvpermi_q(tmp8, tmp6, 0x21); + tmp5 = __lasx_xvpermi_q(tmp6, tmp4, 0x21); + tmp3 = __lasx_xvpermi_q(tmp4, tmp2, 0x21); + tmp1 = __lasx_xvpermi_q(tmp2, tmp0, 0x21); + + QPEL8_HV_LOWPASS_V(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, src00, src01, + src02, src03, src04, src05, tmp0) + QPEL8_HV_LOWPASS_V(tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, src00, src01, + src02, src03, src04, src05, tmp2) + QPEL8_HV_LOWPASS_V(tmp4, tmp5, tmp6, tmp7, tmp8, tmp9, src00, src01, + src02, src03, src04, src05, tmp4) + QPEL8_HV_LOWPASS_V(tmp6, tmp7, tmp8, tmp9, tmp10, tmp11, src00, src01, + src02, src03, src04, src05, tmp6) + LASX_ST_D_2(tmp0, 0, 2, dst, dstStride); + dst += dstStride << 1; + LASX_ST_D_2(tmp2, 0, 2, dst, dstStride); + dst += dstStride << 1; + LASX_ST_D_2(tmp4, 0, 2, dst, dstStride); + dst += dstStride << 1; + LASX_ST_D_2(tmp6, 0, 2, dst, dstStride); +} + +static av_always_inline void +avg_h264_qpel8_h_lowpass_lasx(uint8_t *dst, const uint8_t *src, int dstStride, + int srcStride) +{ + int dstStride_2x = dstStride << 1; + int dstStride_4x = dstStride << 2; + int dstStride_6x = dstStride_2x + dstStride_4x; + __m256i src00, src01, src02, src03, src04, src05, src10; + __m256i dst00, dst01, dst0, dst1, dst2, dst3; + __m256i out0, out1, out2, out3; + __m256i zero = {0}; + __m256i h_20 = {20}; + __m256i h_5 = {5}; + __m256i h_16 = {16}; + __m256i mask1 = {0x0807060504030201, 0x0, 0x0807060504030201, 0x0}; + __m256i mask2 = {0x0908070605040302, 0x0, 0x0908070605040302, 0x0}; + __m256i mask3 = {0x0a09080706050403, 0x0, 0x0a09080706050403, 0x0}; + __m256i mask4 = {0x0b0a090807060504, 0x0, 0x0b0a090807060504, 0x0}; + __m256i mask5 = {0x0c0b0a0908070605, 0x0, 0x0c0b0a0908070605, 0x0}; + + zero = __lasx_xvreplve0_h(zero); + h_20 = __lasx_xvreplve0_h(h_20); + h_5 = __lasx_xvreplve0_h(h_5); + h_16 = __lasx_xvreplve0_h(h_16); + + QPEL8_H_LOWPASS(out0) + QPEL8_H_LOWPASS(out1) + QPEL8_H_LOWPASS(out2) + QPEL8_H_LOWPASS(out3) + LASX_LD_2(dst, dstStride, src00, src01); + LASX_LD_2(dst + dstStride_2x, dstStride, src02, src03); + LASX_LD_2(dst + dstStride_4x, dstStride, src04, src05); + LASX_LD_2(dst + dstStride_6x, dstStride, dst00, dst01); + dst0 = __lasx_xvpermi_q(src00, src01, 0x02); + dst1 = __lasx_xvpermi_q(src02, src03, 0x02); + dst2 = __lasx_xvpermi_q(src04, src05, 0x02); + dst3 = __lasx_xvpermi_q(dst00, dst01, 0x02); + dst0 = __lasx_xvavgr_bu(dst0, out0); + dst1 = __lasx_xvavgr_bu(dst1, out1); + dst2 = __lasx_xvavgr_bu(dst2, out2); + dst3 = __lasx_xvavgr_bu(dst3, out3); + LASX_ST_D_2(dst0, 0, 2, dst, dstStride); + LASX_ST_D_2(dst1, 0, 2, dst + dstStride_2x, dstStride); + LASX_ST_D_2(dst2, 0, 2, dst + dstStride_4x, dstStride); + LASX_ST_D_2(dst3, 0, 2, dst + dstStride_6x, dstStride); +} + +static av_always_inline void +avg_h264_qpel8_hv_lowpass_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dstStride, ptrdiff_t srcStride) +{ + __m256i src00, src01, src02, src03, src04, src05, src10; + __m256i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6; + __m256i tmp7, tmp8, tmp9, tmp10, tmp11, tmp12; + __m256i h_20 = {20}; + __m256i h_5 = {5}; + __m256i w_20 = {20}; + __m256i w_5 = {5}; + __m256i w_512 = {512}; + __m256i mask1 = {0x0807060504030201, 0x0, 0x0807060504030201, 0x0}; + __m256i mask2 = {0x0908070605040302, 0x0, 0x0908070605040302, 0x0}; + __m256i mask3 = {0x0a09080706050403, 0x0, 0x0a09080706050403, 0x0}; + __m256i mask4 = {0x0b0a090807060504, 0x0, 0x0b0a090807060504, 0x0}; + __m256i mask5 = {0x0c0b0a0908070605, 0x0, 0x0c0b0a0908070605, 0x0}; + + h_20 = __lasx_xvreplve0_h(w_20); + h_5 = __lasx_xvreplve0_h(w_5); + w_20 = __lasx_xvreplve0_w(w_20); + w_5 = __lasx_xvreplve0_w(w_5); + w_512 = __lasx_xvreplve0_w(w_512); + + src -= srcStride << 1; + QPEL8_HV_LOWPASS_H(tmp0) + QPEL8_HV_LOWPASS_H(tmp2) + QPEL8_HV_LOWPASS_H(tmp4) + QPEL8_HV_LOWPASS_H(tmp6) + QPEL8_HV_LOWPASS_H(tmp8) + QPEL8_HV_LOWPASS_H(tmp10) + QPEL8_HV_LOWPASS_H(tmp12) + tmp11 = __lasx_xvpermi_q(tmp12, tmp10, 0x21); + tmp9 = __lasx_xvpermi_q(tmp10, tmp8, 0x21); + tmp7 = __lasx_xvpermi_q(tmp8, tmp6, 0x21); + tmp5 = __lasx_xvpermi_q(tmp6, tmp4, 0x21); + tmp3 = __lasx_xvpermi_q(tmp4, tmp2, 0x21); + tmp1 = __lasx_xvpermi_q(tmp2, tmp0, 0x21); + + QPEL8_HV_LOWPASS_V(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, src00, src01, + src02, src03, src04, src05, tmp0) + QPEL8_HV_LOWPASS_V(tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, src00, src01, + src02, src03, src04, src05, tmp2) + QPEL8_HV_LOWPASS_V(tmp4, tmp5, tmp6, tmp7, tmp8, tmp9, src00, src01, + src02, src03, src04, src05, tmp4) + QPEL8_HV_LOWPASS_V(tmp6, tmp7, tmp8, tmp9, tmp10, tmp11, src00, src01, + src02, src03, src04, src05, tmp6) + + LASX_LD_8(dst, dstStride, src00, src01, src02, src03, src04, src05, tmp8, tmp9); + tmp1 = __lasx_xvpermi_q(src00, src01, 0x02); + tmp3 = __lasx_xvpermi_q(src02, src03, 0x02); + tmp5 = __lasx_xvpermi_q(src04, src05, 0x02); + tmp7 = __lasx_xvpermi_q(tmp8, tmp9, 0x02); + tmp0 = __lasx_xvavgr_bu(tmp0, tmp1); + tmp2 = __lasx_xvavgr_bu(tmp2, tmp3); + tmp4 = __lasx_xvavgr_bu(tmp4, tmp5); + tmp6 = __lasx_xvavgr_bu(tmp6, tmp7); + LASX_ST_D_2(tmp0, 0, 2, dst, dstStride); + dst += dstStride << 1; + LASX_ST_D_2(tmp2, 0, 2, dst, dstStride); + dst += dstStride << 1; + LASX_ST_D_2(tmp4, 0, 2, dst, dstStride); + dst += dstStride << 1; + LASX_ST_D_2(tmp6, 0, 2, dst, dstStride); +} + +static av_always_inline void +put_h264_qpel16_h_lowpass_lasx(uint8_t *dst, const uint8_t *src, + int dstStride, int srcStride) +{ + put_h264_qpel8_h_lowpass_lasx(dst, src, dstStride, srcStride); + put_h264_qpel8_h_lowpass_lasx(dst+8, src+8, dstStride, srcStride); + src += srcStride << 3; + dst += dstStride << 3; + put_h264_qpel8_h_lowpass_lasx(dst, src, dstStride, srcStride); + put_h264_qpel8_h_lowpass_lasx(dst+8, src+8, dstStride, srcStride); +} + +static av_always_inline void +avg_h264_qpel16_h_lowpass_lasx(uint8_t *dst, const uint8_t *src, + int dstStride, int srcStride) +{ + avg_h264_qpel8_h_lowpass_lasx(dst, src, dstStride, srcStride); + avg_h264_qpel8_h_lowpass_lasx(dst+8, src+8, dstStride, srcStride); + src += srcStride << 3; + dst += dstStride << 3; + avg_h264_qpel8_h_lowpass_lasx(dst, src, dstStride, srcStride); + avg_h264_qpel8_h_lowpass_lasx(dst+8, src+8, dstStride, srcStride); +} + +static void put_h264_qpel16_v_lowpass_lasx(uint8_t *dst, const uint8_t *src, + int dstStride, int srcStride) +{ + put_h264_qpel8_v_lowpass_lasx(dst, src, dstStride, srcStride); + put_h264_qpel8_v_lowpass_lasx(dst+8, src+8, dstStride, srcStride); + src += 8*srcStride; + dst += 8*dstStride; + put_h264_qpel8_v_lowpass_lasx(dst, src, dstStride, srcStride); + put_h264_qpel8_v_lowpass_lasx(dst+8, src+8, dstStride, srcStride); +} + +static void avg_h264_qpel16_v_lowpass_lasx(uint8_t *dst, const uint8_t *src, + int dstStride, int srcStride) +{ + avg_h264_qpel8_v_lowpass_lasx(dst, src, dstStride, srcStride); + avg_h264_qpel8_v_lowpass_lasx(dst+8, src+8, dstStride, srcStride); + src += 8*srcStride; + dst += 8*dstStride; + avg_h264_qpel8_v_lowpass_lasx(dst, src, dstStride, srcStride); + avg_h264_qpel8_v_lowpass_lasx(dst+8, src+8, dstStride, srcStride); +} + +static void put_h264_qpel16_hv_lowpass_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dstStride, ptrdiff_t srcStride) +{ + put_h264_qpel8_hv_lowpass_lasx(dst, src, dstStride, srcStride); + put_h264_qpel8_hv_lowpass_lasx(dst + 8, src + 8, dstStride, srcStride); + src += srcStride << 3; + dst += dstStride << 3; + put_h264_qpel8_hv_lowpass_lasx(dst, src, dstStride, srcStride); + put_h264_qpel8_hv_lowpass_lasx(dst + 8, src + 8, dstStride, srcStride); +} + +static void avg_h264_qpel16_hv_lowpass_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dstStride, ptrdiff_t srcStride) +{ + avg_h264_qpel8_hv_lowpass_lasx(dst, src, dstStride, srcStride); + avg_h264_qpel8_hv_lowpass_lasx(dst + 8, src + 8, dstStride, srcStride); + src += srcStride << 3; + dst += dstStride << 3; + avg_h264_qpel8_hv_lowpass_lasx(dst, src, dstStride, srcStride); + avg_h264_qpel8_hv_lowpass_lasx(dst + 8, src + 8, dstStride, srcStride); +} + +void ff_put_h264_qpel8_mc00_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + /* In mmi optimization, it used function ff_put_pixels8_8_mmi + * which implemented in hpeldsp_mmi.c */ + put_pixels8_8_inline_asm(dst, src, stride); +} + +void ff_put_h264_qpel8_mc10_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[64]; + + put_h264_qpel8_h_lowpass_lasx(half, src, 8, stride); + /* in qpel8, the stride of half and height of block is 8 */ + put_pixels8_l2_8_lsx(dst, src, half, stride, stride); +} + +void ff_put_h264_qpel8_mc20_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + put_h264_qpel8_h_lowpass_lasx(dst, src, stride, stride); +} + +void ff_put_h264_qpel8_mc30_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[64]; + + put_h264_qpel8_h_lowpass_lasx(half, src, 8, stride); + put_pixels8_l2_8_lsx(dst, src+1, half, stride, stride); +} + +void ff_put_h264_qpel8_mc01_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[64]; + + put_h264_qpel8_v_lowpass_lasx(half, src, 8, stride); + put_pixels8_l2_8_lsx(dst, src, half, stride, stride); +} + +void ff_put_h264_qpel8_mc11_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t halfH[64]; + uint8_t halfV[64]; + + put_h264_qpel8_h_lowpass_lasx(halfH, src, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfV, src, 8, stride); + put_pixels8_l2_8_lsx(dst, halfH, halfV, stride, 8); +} + +void ff_put_h264_qpel8_mc21_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[128]; + uint8_t *const halfH = temp; + uint8_t *const halfHV = temp + 64; + + put_h264_qpel8_h_lowpass_lasx(halfH, src, 8, stride); + put_h264_qpel8_hv_lowpass_lasx(halfHV, src, 8, stride); + put_pixels8_l2_8_lsx(dst, halfH, halfHV, stride, 8); +} + +void ff_put_h264_qpel8_mc31_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t halfH[64]; + uint8_t halfV[64]; + + put_h264_qpel8_h_lowpass_lasx(halfH, src, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfV, src + 1, 8, stride); + put_pixels8_l2_8_lsx(dst, halfH, halfV, stride, 8); +} + +void ff_put_h264_qpel8_mc02_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + put_h264_qpel8_v_lowpass_lasx(dst, src, stride, stride); +} + +void ff_put_h264_qpel8_mc12_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[128]; + uint8_t *const halfHV = temp; + uint8_t *const halfH = temp + 64; + + put_h264_qpel8_hv_lowpass_lasx(halfHV, src, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfH, src, 8, stride); + put_pixels8_l2_8_lsx(dst, halfH, halfHV, stride, 8); +} + +void ff_put_h264_qpel8_mc22_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + put_h264_qpel8_hv_lowpass_lasx(dst, src, stride, stride); +} + +void ff_put_h264_qpel8_mc32_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[128]; + uint8_t *const halfHV = temp; + uint8_t *const halfH = temp + 64; + + put_h264_qpel8_hv_lowpass_lasx(halfHV, src, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfH, src + 1, 8, stride); + put_pixels8_l2_8_lsx(dst, halfH, halfHV, stride, 8); +} + +void ff_put_h264_qpel8_mc03_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[64]; + + put_h264_qpel8_v_lowpass_lasx(half, src, 8, stride); + put_pixels8_l2_8_lsx(dst, src + stride, half, stride, stride); +} + +void ff_put_h264_qpel8_mc13_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t halfH[64]; + uint8_t halfV[64]; + + put_h264_qpel8_h_lowpass_lasx(halfH, src + stride, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfV, src, 8, stride); + put_pixels8_l2_8_lsx(dst, halfH, halfV, stride, 8); +} + +void ff_put_h264_qpel8_mc23_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[128]; + uint8_t *const halfH = temp; + uint8_t *const halfHV = temp + 64; + + put_h264_qpel8_h_lowpass_lasx(halfH, src + stride, 8, stride); + put_h264_qpel8_hv_lowpass_lasx(halfHV, src, 8, stride); + put_pixels8_l2_8_lsx(dst, halfH, halfHV, stride, 8); +} + +void ff_put_h264_qpel8_mc33_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t halfH[64]; + uint8_t halfV[64]; + + put_h264_qpel8_h_lowpass_lasx(halfH, src + stride, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfV, src + 1, 8, stride); + put_pixels8_l2_8_lsx(dst, halfH, halfV, stride, 8); +} + +void ff_avg_h264_qpel8_mc00_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + /* In mmi optimization, it used function ff_avg_pixels8_8_mmi + * which implemented in hpeldsp_mmi.c */ + avg_pixels8_8_lsx(dst, src, stride); +} + +void ff_avg_h264_qpel8_mc10_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[64]; + + put_h264_qpel8_h_lowpass_lasx(half, src, 8, stride); + avg_pixels8_l2_8_lsx(dst, src, half, stride, stride); +} + +void ff_avg_h264_qpel8_mc20_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avg_h264_qpel8_h_lowpass_lasx(dst, src, stride, stride); +} + +void ff_avg_h264_qpel8_mc30_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[64]; + + put_h264_qpel8_h_lowpass_lasx(half, src, 8, stride); + avg_pixels8_l2_8_lsx(dst, src+1, half, stride, stride); +} + +void ff_avg_h264_qpel8_mc11_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t halfH[64]; + uint8_t halfV[64]; + + put_h264_qpel8_h_lowpass_lasx(halfH, src, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfV, src, 8, stride); + avg_pixels8_l2_8_lsx(dst, halfH, halfV, stride, 8); +} + +void ff_avg_h264_qpel8_mc21_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[128]; + uint8_t *const halfH = temp; + uint8_t *const halfHV = temp + 64; + + put_h264_qpel8_h_lowpass_lasx(halfH, src, 8, stride); + put_h264_qpel8_hv_lowpass_lasx(halfHV, src, 8, stride); + avg_pixels8_l2_8_lsx(dst, halfH, halfHV, stride, 8); +} + +void ff_avg_h264_qpel8_mc31_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t halfH[64]; + uint8_t halfV[64]; + + put_h264_qpel8_h_lowpass_lasx(halfH, src, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfV, src + 1, 8, stride); + avg_pixels8_l2_8_lsx(dst, halfH, halfV, stride, 8); +} + +void ff_avg_h264_qpel8_mc02_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avg_h264_qpel8_v_lowpass_lasx(dst, src, stride, stride); +} + +void ff_avg_h264_qpel8_mc12_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[128]; + uint8_t *const halfHV = temp; + uint8_t *const halfH = temp + 64; + + put_h264_qpel8_hv_lowpass_lasx(halfHV, src, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfH, src, 8, stride); + avg_pixels8_l2_8_lsx(dst, halfH, halfHV, stride, 8); +} + +void ff_avg_h264_qpel8_mc22_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avg_h264_qpel8_hv_lowpass_lasx(dst, src, stride, stride); +} + +void ff_avg_h264_qpel8_mc32_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[128]; + uint8_t *const halfHV = temp; + uint8_t *const halfH = temp + 64; + + put_h264_qpel8_hv_lowpass_lasx(halfHV, src, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfH, src + 1, 8, stride); + avg_pixels8_l2_8_lsx(dst, halfH, halfHV, stride, 8); +} + +void ff_avg_h264_qpel8_mc13_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t halfH[64]; + uint8_t halfV[64]; + + put_h264_qpel8_h_lowpass_lasx(halfH, src + stride, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfV, src, 8, stride); + avg_pixels8_l2_8_lsx(dst, halfH, halfV, stride, 8); +} + +void ff_avg_h264_qpel8_mc23_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[128]; + uint8_t *const halfH = temp; + uint8_t *const halfHV = temp + 64; + + put_h264_qpel8_h_lowpass_lasx(halfH, src + stride, 8, stride); + put_h264_qpel8_hv_lowpass_lasx(halfHV, src, 8, stride); + avg_pixels8_l2_8_lsx(dst, halfH, halfHV, stride, 8); +} + +void ff_avg_h264_qpel8_mc33_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t halfH[64]; + uint8_t halfV[64]; + + put_h264_qpel8_h_lowpass_lasx(halfH, src + stride, 8, stride); + put_h264_qpel8_v_lowpass_lasx(halfV, src + 1, 8, stride); + avg_pixels8_l2_8_lsx(dst, halfH, halfV, stride, 8); +} + +void ff_put_h264_qpel16_mc00_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + /* In mmi optimization, it used function ff_put_pixels16_8_mmi + * which implemented in hpeldsp_mmi.c */ + put_pixels16_8_lsx(dst, src, stride); +} + +void ff_put_h264_qpel16_mc10_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[256]; + + put_h264_qpel16_h_lowpass_lasx(half, src, 16, stride); + put_pixels16_l2_8_lsx(dst, src, half, stride, stride); +} + +void ff_put_h264_qpel16_mc20_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + put_h264_qpel16_h_lowpass_lasx(dst, src, stride, stride); +} + +void ff_put_h264_qpel16_mc30_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[256]; + + put_h264_qpel16_h_lowpass_lasx(half, src, 16, stride); + put_pixels16_l2_8_lsx(dst, src+1, half, stride, stride); +} + +void ff_put_h264_qpel16_mc01_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[256]; + + put_h264_qpel16_v_lowpass_lasx(half, src, 16, stride); + put_pixels16_l2_8_lsx(dst, src, half, stride, stride); +} + +void ff_put_h264_qpel16_mc11_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avc_luma_hv_qrt_16x16_lasx(src - 2, src - (stride * 2), dst, stride); +} + +void ff_put_h264_qpel16_mc21_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[512]; + uint8_t *const halfH = temp; + uint8_t *const halfHV = temp + 256; + + put_h264_qpel16_h_lowpass_lasx(halfH, src, 16, stride); + put_h264_qpel16_hv_lowpass_lasx(halfHV, src, 16, stride); + put_pixels16_l2_8_lsx(dst, halfH, halfHV, stride, 16); +} + +void ff_put_h264_qpel16_mc31_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avc_luma_hv_qrt_16x16_lasx(src - 2, src - (stride * 2) + 1, dst, stride); +} + +void ff_put_h264_qpel16_mc02_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + put_h264_qpel16_v_lowpass_lasx(dst, src, stride, stride); +} + +void ff_put_h264_qpel16_mc12_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[512]; + uint8_t *const halfHV = temp; + uint8_t *const halfH = temp + 256; + + put_h264_qpel16_hv_lowpass_lasx(halfHV, src, 16, stride); + put_h264_qpel16_v_lowpass_lasx(halfH, src, 16, stride); + put_pixels16_l2_8_lsx(dst, halfH, halfHV, stride, 16); +} + +void ff_put_h264_qpel16_mc22_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + put_h264_qpel16_hv_lowpass_lasx(dst, src, stride, stride); +} + +void ff_put_h264_qpel16_mc32_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[512]; + uint8_t *const halfHV = temp; + uint8_t *const halfH = temp + 256; + + put_h264_qpel16_hv_lowpass_lasx(halfHV, src, 16, stride); + put_h264_qpel16_v_lowpass_lasx(halfH, src + 1, 16, stride); + put_pixels16_l2_8_lsx(dst, halfH, halfHV, stride, 16); +} + +void ff_put_h264_qpel16_mc03_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[256]; + + put_h264_qpel16_v_lowpass_lasx(half, src, 16, stride); + put_pixels16_l2_8_lsx(dst, src+stride, half, stride, stride); +} + +void ff_put_h264_qpel16_mc13_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avc_luma_hv_qrt_16x16_lasx(src + stride - 2, src - (stride * 2), dst, + stride); +} + +void ff_put_h264_qpel16_mc23_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[512]; + uint8_t *const halfH = temp; + uint8_t *const halfHV = temp + 256; + + put_h264_qpel16_h_lowpass_lasx(halfH, src + stride, 16, stride); + put_h264_qpel16_hv_lowpass_lasx(halfHV, src, 16, stride); + put_pixels16_l2_8_lsx(dst, halfH, halfHV, stride, 16); +} + +void ff_put_h264_qpel16_mc33_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avc_luma_hv_qrt_16x16_lasx(src + stride - 2, src - (stride * 2) + 1, dst, + stride); +} + +void ff_avg_h264_qpel16_mc00_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + /* In mmi optimization, it used function ff_avg_pixels16_8_mmi + * which implemented in hpeldsp_mmi.c */ + avg_pixels16_8_lsx(dst, src, stride); +} + +void ff_avg_h264_qpel16_mc10_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[256]; + + put_h264_qpel16_h_lowpass_lasx(half, src, 16, stride); + avg_pixels16_l2_8_lsx(dst, src, half, stride, stride); +} + +void ff_avg_h264_qpel16_mc20_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avg_h264_qpel16_h_lowpass_lasx(dst, src, stride, stride); +} + +void ff_avg_h264_qpel16_mc30_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[256]; + + put_h264_qpel16_h_lowpass_lasx(half, src, 16, stride); + avg_pixels16_l2_8_lsx(dst, src+1, half, stride, stride); +} + +void ff_avg_h264_qpel16_mc01_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[256]; + + put_h264_qpel16_v_lowpass_lasx(half, src, 16, stride); + avg_pixels16_l2_8_lsx(dst, src, half, stride, stride); +} + +void ff_avg_h264_qpel16_mc11_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avc_luma_hv_qrt_and_aver_dst_16x16_lasx(src - 2, + src - (stride * 2), + dst, stride); +} + +void ff_avg_h264_qpel16_mc21_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[512]; + uint8_t *const halfH = temp; + uint8_t *const halfHV = temp + 256; + + put_h264_qpel16_h_lowpass_lasx(halfH, src, 16, stride); + put_h264_qpel16_hv_lowpass_lasx(halfHV, src, 16, stride); + avg_pixels16_l2_8_lsx(dst, halfH, halfHV, stride, 16); +} + +void ff_avg_h264_qpel16_mc31_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avc_luma_hv_qrt_and_aver_dst_16x16_lasx(src - 2, + src - (stride * 2) + + sizeof(uint8_t), + dst, stride); +} + +void ff_avg_h264_qpel16_mc02_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avg_h264_qpel16_v_lowpass_lasx(dst, src, stride, stride); +} + +void ff_avg_h264_qpel16_mc12_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[512]; + uint8_t *const halfHV = temp; + uint8_t *const halfH = temp + 256; + + put_h264_qpel16_hv_lowpass_lasx(halfHV, src, 16, stride); + put_h264_qpel16_v_lowpass_lasx(halfH, src, 16, stride); + avg_pixels16_l2_8_lsx(dst, halfH, halfHV, stride, 16); +} + +void ff_avg_h264_qpel16_mc22_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avg_h264_qpel16_hv_lowpass_lasx(dst, src, stride, stride); +} + +void ff_avg_h264_qpel16_mc32_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[512]; + uint8_t *const halfHV = temp; + uint8_t *const halfH = temp + 256; + + put_h264_qpel16_hv_lowpass_lasx(halfHV, src, 16, stride); + put_h264_qpel16_v_lowpass_lasx(halfH, src + 1, 16, stride); + avg_pixels16_l2_8_lsx(dst, halfH, halfHV, stride, 16); +} + +void ff_avg_h264_qpel16_mc03_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t half[256]; + + put_h264_qpel16_v_lowpass_lasx(half, src, 16, stride); + avg_pixels16_l2_8_lsx(dst, src + stride, half, stride, stride); +} + +void ff_avg_h264_qpel16_mc13_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avc_luma_hv_qrt_and_aver_dst_16x16_lasx(src + stride - 2, + src - (stride * 2), + dst, stride); +} + +void ff_avg_h264_qpel16_mc23_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + uint8_t temp[512]; + uint8_t *const halfH = temp; + uint8_t *const halfHV = temp + 256; + + put_h264_qpel16_h_lowpass_lasx(halfH, src + stride, 16, stride); + put_h264_qpel16_hv_lowpass_lasx(halfHV, src, 16, stride); + avg_pixels16_l2_8_lsx(dst, halfH, halfHV, stride, 16); +} + +void ff_avg_h264_qpel16_mc33_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride) +{ + avc_luma_hv_qrt_and_aver_dst_16x16_lasx(src + stride - 2, + src - (stride * 2) + + sizeof(uint8_t), + dst, stride); +} diff --git a/libavcodec/loongarch/h264qpel_lasx.h b/libavcodec/loongarch/h264qpel_lasx.h new file mode 100644 index 0000000000..32b6b50917 --- /dev/null +++ b/libavcodec/loongarch/h264qpel_lasx.h @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_H264QPEL_LASX_H +#define AVCODEC_LOONGARCH_H264QPEL_LASX_H + +#include +#include +#include "libavcodec/h264.h" + +void ff_h264_h_lpf_luma_inter_lasx(uint8_t *src, int stride, + int alpha, int beta, int8_t *tc0); +void ff_h264_v_lpf_luma_inter_lasx(uint8_t *src, int stride, + int alpha, int beta, int8_t *tc0); +void ff_put_h264_qpel16_mc00_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc10_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc20_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc30_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc01_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc11_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc21_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc31_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc02_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc12_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc32_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc22_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc03_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc13_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc23_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_put_h264_qpel16_mc33_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc00_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc10_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc20_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc30_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc01_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc11_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc21_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc31_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc02_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc12_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc22_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc32_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc03_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc13_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc23_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel16_mc33_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); + +void ff_put_h264_qpel8_mc00_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc10_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc20_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc30_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc01_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc11_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc21_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc31_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc02_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc12_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc22_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc32_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc03_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc13_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc23_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_put_h264_qpel8_mc33_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride); +void ff_avg_h264_qpel8_mc00_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc10_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc20_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc30_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc11_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc21_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc31_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc02_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc12_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc22_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc32_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc13_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc23_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +void ff_avg_h264_qpel8_mc33_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t dst_stride); +#endif // #ifndef AVCODEC_LOONGARCH_H264QPEL_LASX_H diff --git a/libavcodec/loongarch/hevc_idct_lsx.c b/libavcodec/loongarch/hevc_idct_lsx.c new file mode 100644 index 0000000000..2d166e5790 --- /dev/null +++ b/libavcodec/loongarch/hevc_idct_lsx.c @@ -0,0 +1,874 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "hevcdsp_lsx.h" + +static const int16_t gt8x8_cnst[16] __attribute__ ((aligned (64))) = { + 64, 64, 83, 36, 89, 50, 18, 75, 64, -64, 36, -83, 75, -89, -50, -18 +}; + +static const int16_t gt16x16_cnst[64] __attribute__ ((aligned (64))) = { + 64, 83, 64, 36, 89, 75, 50, 18, 90, 80, 57, 25, 70, 87, 9, 43, + 64, 36, -64, -83, 75, -18, -89, -50, 87, 9, -80, -70, -43, 57, -25, -90, + 64, -36, -64, 83, 50, -89, 18, 75, 80, -70, -25, 90, -87, 9, 43, 57, + 64, -83, 64, -36, 18, -50, 75, -89, 70, -87, 90, -80, 9, -43, -57, 25 +}; + +static const int16_t gt32x32_cnst0[256] __attribute__ ((aligned (64))) = { + 90, 90, 88, 85, 82, 78, 73, 67, 61, 54, 46, 38, 31, 22, 13, 4, + 90, 82, 67, 46, 22, -4, -31, -54, -73, -85, -90, -88, -78, -61, -38, -13, + 88, 67, 31, -13, -54, -82, -90, -78, -46, -4, 38, 73, 90, 85, 61, 22, + 85, 46, -13, -67, -90, -73, -22, 38, 82, 88, 54, -4, -61, -90, -78, -31, + 82, 22, -54, -90, -61, 13, 78, 85, 31, -46, -90, -67, 4, 73, 88, 38, + 78, -4, -82, -73, 13, 85, 67, -22, -88, -61, 31, 90, 54, -38, -90, -46, + 73, -31, -90, -22, 78, 67, -38, -90, -13, 82, 61, -46, -88, -4, 85, 54, + 67, -54, -78, 38, 85, -22, -90, 4, 90, 13, -88, -31, 82, 46, -73, -61, + 61, -73, -46, 82, 31, -88, -13, 90, -4, -90, 22, 85, -38, -78, 54, 67, + 54, -85, -4, 88, -46, -61, 82, 13, -90, 38, 67, -78, -22, 90, -31, -73, + 46, -90, 38, 54, -90, 31, 61, -88, 22, 67, -85, 13, 73, -82, 4, 78, + 38, -88, 73, -4, -67, 90, -46, -31, 85, -78, 13, 61, -90, 54, 22, -82, + 31, -78, 90, -61, 4, 54, -88, 82, -38, -22, 73, -90, 67, -13, -46, 85, + 22, -61, 85, -90, 73, -38, -4, 46, -78, 90, -82, 54, -13, -31, 67, -88, + 13, -38, 61, -78, 88, -90, 85, -73, 54, -31, 4, 22, -46, 67, -82, 90, + 4, -13, 22, -31, 38, -46, 54, -61, 67, -73, 78, -82, 85, -88, 90, -90 +}; + +static const int16_t gt32x32_cnst1[64] __attribute__ ((aligned (64))) = { + 90, 87, 80, 70, 57, 43, 25, 9, 87, 57, 9, -43, -80, -90, -70, -25, + 80, 9, -70, -87, -25, 57, 90, 43, 70, -43, -87, 9, 90, 25, -80, -57, + 57, -80, -25, 90, -9, -87, 43, 70, 43, -90, 57, 25, -87, 70, 9, -80, + 25, -70, 90, -80, 43, 9, -57, 87, 9, -25, 43, -57, 70, -80, 87, -90 +}; + +static const int16_t gt32x32_cnst2[16] __attribute__ ((aligned (64))) = { + 89, 75, 50, 18, 75, -18, -89, -50, 50, -89, 18, 75, 18, -50, 75, -89 +}; + +#define HEVC_IDCT4x4_COL(in_r0, in_l0, in_r1, in_l1, \ + sum0, sum1, sum2, sum3, shift) \ +{ \ + __m128i vec0, vec1, vec2, vec3, vec4, vec5; \ + __m128i cnst64 = __lsx_vldi(0x0840); \ + __m128i cnst83 = __lsx_vldi(0x0853); \ + __m128i cnst36 = __lsx_vldi(0x0824); \ + \ + vec0 = __lsx_dp2_w_h(in_r0, cnst64); \ + vec1 = __lsx_dp2_w_h(in_l0, cnst83); \ + vec2 = __lsx_dp2_w_h(in_r1, cnst64); \ + vec3 = __lsx_dp2_w_h(in_l1, cnst36); \ + vec4 = __lsx_dp2_w_h(in_l0, cnst36); \ + vec5 = __lsx_dp2_w_h(in_l1, cnst83); \ + \ + sum0 = __lsx_vadd_w(vec0, vec2); \ + sum1 = __lsx_vsub_w(vec0, vec2); \ + vec1 = __lsx_vadd_w(vec1, vec3); \ + vec4 = __lsx_vsub_w(vec4, vec5); \ + sum2 = __lsx_vsub_w(sum1, vec4); \ + sum3 = __lsx_vsub_w(sum0, vec1); \ + sum0 = __lsx_vadd_w(sum0, vec1); \ + sum1 = __lsx_vadd_w(sum1, vec4); \ + \ + sum0 = __lsx_vsrari_w(sum0, shift); \ + sum1 = __lsx_vsrari_w(sum1, shift); \ + sum2 = __lsx_vsrari_w(sum2, shift); \ + sum3 = __lsx_vsrari_w(sum3, shift); \ + sum0 = __lsx_vsat_w(sum0, 15); \ + sum1 = __lsx_vsat_w(sum1, 15); \ + sum2 = __lsx_vsat_w(sum2, 15); \ + sum3 = __lsx_vsat_w(sum3, 15); \ +} + +#define HEVC_IDCT8x8_COL(in0, in1, in2, in3, in4, in5, in6, in7, shift) \ +{ \ + __m128i src0_r, src1_r, src2_r, src3_r; \ + __m128i src0_l, src1_l, src2_l, src3_l; \ + __m128i filter0, filter1, filter2, filter3; \ + __m128i temp0_r, temp1_r, temp2_r, temp3_r, temp4_r, temp5_r; \ + __m128i temp0_l, temp1_l, temp2_l, temp3_l, temp4_l, temp5_l; \ + __m128i sum0_r, sum1_r, sum2_r, sum3_r; \ + __m128i sum0_l, sum1_l, sum2_l, sum3_l; \ + \ + LSX_DUP4_ARG2(__lsx_vilvl_h, in4, in0, in6, in2, in5, in1, in3, in7, \ + src0_r, src1_r, src2_r, src3_r); \ + LSX_DUP4_ARG2(__lsx_vilvh_h, in4, in0, in6, in2, in5, in1, in3, in7, \ + src0_l, src1_l, src2_l, src3_l); \ + \ + LSX_DUP4_ARG2(__lsx_vldrepl_w, filter, 0, filter, 4, filter, 8, \ + filter, 12, filter0, filter1, filter2, filter3); \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, src0_r, filter0, src0_l, filter0, \ + src1_r, filter1, src1_l, filter1, temp0_r, temp0_l, \ + temp1_r, temp1_l); \ + \ + BUTTERFLY_4_W(temp0_r, temp0_l, temp1_l, temp1_r, sum0_r, sum0_l, \ + sum1_l, sum1_r); \ + sum2_r = sum1_r; \ + sum2_l = sum1_l; \ + sum3_r = sum0_r; \ + sum3_l = sum0_l; \ + \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, src2_r, filter2, src2_l, filter2, \ + src3_r, filter3, src3_l, filter3, temp2_r, temp2_l, \ + temp3_r, temp3_l); \ + temp2_r = __lsx_vadd_w(temp2_r, temp3_r); \ + temp2_l = __lsx_vadd_w(temp2_l, temp3_l); \ + sum0_r = __lsx_vadd_w(sum0_r, temp2_r); \ + sum0_l = __lsx_vadd_w(sum0_l, temp2_l); \ + sum3_r = __lsx_vsub_w(sum3_r, temp2_r); \ + sum3_l = __lsx_vsub_w(sum3_l, temp2_l); \ + \ + in0 = __lsx_vssrarni_h_w(sum0_l, sum0_r, shift); \ + in7 = __lsx_vssrarni_h_w(sum3_l, sum3_r, shift); \ + \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, src2_r, filter3, src2_l, filter3, \ + src3_r, filter2, src3_l, filter2, temp4_r, temp4_l, \ + temp5_r, temp5_l); \ + temp4_r = __lsx_vsub_w(temp4_r, temp5_r); \ + temp4_l = __lsx_vsub_w(temp4_l, temp5_l); \ + sum1_r = __lsx_vadd_w(sum1_r, temp4_r); \ + sum1_l = __lsx_vadd_w(sum1_l, temp4_l); \ + sum2_r = __lsx_vsub_w(sum2_r, temp4_r); \ + sum2_l = __lsx_vsub_w(sum2_l, temp4_l); \ + \ + in3 = __lsx_vssrarni_h_w(sum1_l, sum1_r, shift); \ + in4 = __lsx_vssrarni_h_w(sum2_l, sum2_r, shift); \ + \ + LSX_DUP4_ARG2(__lsx_vldrepl_w, filter, 16, filter, 20, filter, 24, \ + filter, 28, filter0, filter1, filter2, filter3); \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, src0_r, filter0, src0_l, filter0, \ + src1_r, filter1, src1_l, filter1, temp0_r, temp0_l, \ + temp1_r, temp1_l); \ + \ + BUTTERFLY_4_W(temp0_r, temp0_l, temp1_l, temp1_r, sum0_r, sum0_l, \ + sum1_l, sum1_r); \ + sum2_r = sum1_r; \ + sum2_l = sum1_l; \ + sum3_r = sum0_r; \ + sum3_l = sum0_l; \ + \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, src2_r, filter2, src2_l, filter2, \ + src3_r, filter3, src3_l, filter3, temp2_r, temp2_l, \ + temp3_r, temp3_l); \ + temp2_r = __lsx_vadd_w(temp2_r, temp3_r); \ + temp2_l = __lsx_vadd_w(temp2_l, temp3_l); \ + sum0_r = __lsx_vadd_w(sum0_r, temp2_r); \ + sum0_l = __lsx_vadd_w(sum0_l, temp2_l); \ + sum3_r = __lsx_vsub_w(sum3_r, temp2_r); \ + sum3_l = __lsx_vsub_w(sum3_l, temp2_l); \ + \ + in1 = __lsx_vssrarni_h_w(sum0_l, sum0_r, shift); \ + in6 = __lsx_vssrarni_h_w(sum3_l, sum3_r, shift); \ + \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, src2_r, filter3, src2_l, filter3, \ + src3_r, filter2, src3_l, filter2, temp4_r, temp4_l, \ + temp5_r, temp5_l); \ + temp4_r = __lsx_vsub_w(temp4_r, temp5_r); \ + temp4_l = __lsx_vsub_w(temp4_l, temp5_l); \ + sum1_r = __lsx_vsub_w(sum1_r, temp4_r); \ + sum1_l = __lsx_vsub_w(sum1_l, temp4_l); \ + sum2_r = __lsx_vadd_w(sum2_r, temp4_r); \ + sum2_l = __lsx_vadd_w(sum2_l, temp4_l); \ + \ + in2 = __lsx_vssrarni_h_w(sum1_l, sum1_r, shift); \ + in5 = __lsx_vssrarni_h_w(sum2_l, sum2_r, shift); \ +} + +#define HEVC_IDCT16x16_COL(src0_r, src1_r, src2_r, src3_r, \ + src4_r, src5_r, src6_r, src7_r, \ + src0_l, src1_l, src2_l, src3_l, \ + src4_l, src5_l, src6_l, src7_l, shift) \ +{ \ + int16_t *ptr0, *ptr1; \ + __m128i dst0, dst1; \ + __m128i filter0, filter1, filter2, filter3; \ + __m128i temp0_r, temp1_r, temp0_l, temp1_l; \ + __m128i sum0_r, sum1_r, sum2_r, sum3_r, sum0_l, sum1_l, sum2_l; \ + __m128i sum3_l, res0_r, res1_r, res0_l, res1_l; \ + \ + ptr0 = (buf_ptr + 112); \ + ptr1 = (buf_ptr + 128); \ + k = -1; \ + \ + for (j = 0; j < 4; j++) \ + { \ + LSX_DUP4_ARG2(__lsx_vldrepl_w, filter, 0, filter, 4, filter, 16, \ + filter, 20, filter0, filter1, filter2, filter3); \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, src0_r, filter0, src0_l, filter0, \ + src4_r, filter2, src4_l, filter2, sum0_r, sum0_l, \ + sum2_r, sum2_l); \ + LSX_DUP2_ARG2(__lsx_dp2_w_h, src7_r, filter2, src7_l, filter2, \ + sum3_r, sum3_l); \ + LSX_DUP4_ARG3(__lsx_dp2add_w_h, sum0_r, src1_r, filter1, sum0_l, \ + src1_l, filter1, sum2_r, src5_r, filter3, sum2_l, \ + src5_l, filter3, sum0_r, sum0_l, sum2_r, sum2_l); \ + LSX_DUP2_ARG3(__lsx_dp2add_w_h, sum3_r, src6_r, filter3, sum3_l, \ + src6_l, filter3, sum3_r, sum3_l); \ + \ + sum1_r = sum0_r; \ + sum1_l = sum0_l; \ + \ + LSX_DUP4_ARG2(__lsx_vldrepl_w, filter, 8, filter, 12, filter, 24, \ + filter, 28, filter0, filter1, filter2, filter3); \ + filter += 16; \ + LSX_DUP2_ARG2(__lsx_dp2_w_h, src2_r, filter0, src2_l, filter0, \ + temp0_r, temp0_l); \ + LSX_DUP2_ARG3(__lsx_dp2add_w_h, sum2_r, src6_r, filter2, sum2_l, \ + src6_l, filter2, sum2_r, sum2_l); \ + LSX_DUP2_ARG2(__lsx_dp2_w_h, src5_r, filter2, src5_l, filter2, \ + temp1_r, temp1_l); \ + \ + sum0_r = __lsx_vadd_w(sum0_r, temp0_r); \ + sum0_l = __lsx_vadd_w(sum0_l, temp0_l); \ + sum1_r = __lsx_vsub_w(sum1_r, temp0_r); \ + sum1_l = __lsx_vsub_w(sum1_l, temp0_l); \ + sum3_r = __lsx_vsub_w(temp1_r, sum3_r); \ + sum3_l = __lsx_vsub_w(temp1_l, sum3_l); \ + \ + LSX_DUP2_ARG2(__lsx_dp2_w_h, src3_r, filter1, src3_l, filter1, \ + temp0_r, temp0_l); \ + LSX_DUP4_ARG3(__lsx_dp2add_w_h, sum2_r, src7_r, filter3, sum2_l, \ + src7_l, filter3, sum3_r, src4_r, filter3, sum3_l, \ + src4_l, filter3, sum2_r, sum2_l, sum3_r, sum3_l); \ + \ + sum0_r = __lsx_vadd_w(sum0_r, temp0_r); \ + sum0_l = __lsx_vadd_w(sum0_l, temp0_l); \ + sum1_r = __lsx_vsub_w(sum1_r, temp0_r); \ + sum1_l = __lsx_vsub_w(sum1_l, temp0_l); \ + \ + BUTTERFLY_4_W(sum0_r, sum0_l, sum2_l, sum2_r, res0_r, res0_l, \ + res1_l, res1_r); \ + dst0 = __lsx_vssrarni_h_w(res0_l, res0_r, shift); \ + dst1 = __lsx_vssrarni_h_w(res1_l, res1_r, shift); \ + __lsx_vst(dst0, buf_ptr, 0); \ + __lsx_vst(dst1, (buf_ptr + ((15 - (j * 2)) << 4)), 0); \ + \ + BUTTERFLY_4_W(sum1_r, sum1_l, sum3_l, sum3_r, res0_r, res0_l, \ + res1_l, res1_r); \ + \ + dst0 = __lsx_vssrarni_h_w(res0_l, res0_r, shift); \ + dst1 = __lsx_vssrarni_h_w(res1_l, res1_r, shift); \ + __lsx_vst(dst0, (ptr0 + ((((j + 1) >> 1) * 2 * k) << 4)), 0); \ + __lsx_vst(dst1, (ptr1 - ((((j + 1) >> 1) * 2 * k) << 4)), 0); \ + \ + k *= -1; \ + buf_ptr += 16; \ + } \ +} + +#define HEVC_EVEN16_CALC(input, sum0_r, sum0_l, load_idx, store_idx) \ +{ \ + tmp0_r = __lsx_vld(input + load_idx * 8, 0); \ + tmp0_l = __lsx_vld(input + load_idx * 8, 16); \ + tmp1_r = sum0_r; \ + tmp1_l = sum0_l; \ + sum0_r = __lsx_vadd_w(sum0_r, tmp0_r); \ + sum0_l = __lsx_vadd_w(sum0_l, tmp0_l); \ + __lsx_vst(sum0_r, (input + load_idx * 8), 0); \ + __lsx_vst(sum0_l, (input + load_idx * 8), 16); \ + tmp1_r = __lsx_vsub_w(tmp1_r, tmp0_r); \ + tmp1_l = __lsx_vsub_w(tmp1_l, tmp0_l); \ + __lsx_vst(tmp1_r, (input + store_idx * 8), 0); \ + __lsx_vst(tmp1_l, (input + store_idx * 8), 16); \ +} + +#define HEVC_IDCT_LUMA4x4_COL(in_r0, in_l0, in_r1, in_l1, \ + res0, res1, res2, res3, shift) \ +{ \ + __m128i vec0, vec1, vec2, vec3; \ + __m128i cnst74 = __lsx_vldi(0x84a); \ + __m128i cnst55 = __lsx_vldi(0x837); \ + __m128i cnst29 = __lsx_vldi(0x81d); \ + \ + vec0 = __lsx_vadd_w(in_r0, in_r1); \ + vec2 = __lsx_vsub_w(in_r0, in_l1); \ + res0 = __lsx_vmul_w(vec0, cnst29); \ + res1 = __lsx_vmul_w(vec2, cnst55); \ + res2 = __lsx_vsub_w(in_r0, in_r1); \ + vec1 = __lsx_vadd_w(in_r1, in_l1); \ + res2 = __lsx_vadd_w(res2, in_l1); \ + vec3 = __lsx_vmul_w(in_l0, cnst74); \ + res3 = __lsx_vmul_w(vec0, cnst55); \ + \ + res0 = __lsx_vadd_w(res0, __lsx_vmul_w(vec1, cnst55)); \ + res1 = __lsx_vsub_w(res1, __lsx_vmul_w(vec1, cnst29)); \ + res2 = __lsx_vmul_w(res2, cnst74); \ + res3 = __lsx_vadd_w(res3, __lsx_vmul_w(vec2, cnst29)); \ + \ + res0 = __lsx_vadd_w(res0, vec3); \ + res1 = __lsx_vadd_w(res1, vec3); \ + res3 = __lsx_vsub_w(res3, vec3); \ + \ + res0 = __lsx_vsrari_w(res0, shift); \ + res1 = __lsx_vsrari_w(res1, shift); \ + res2 = __lsx_vsrari_w(res2, shift); \ + res3 = __lsx_vsrari_w(res3, shift); \ + res0 = __lsx_vsat_w(res0, 15); \ + res1 = __lsx_vsat_w(res1, 15); \ + res2 = __lsx_vsat_w(res2, 15); \ + res3 = __lsx_vsat_w(res3, 15); \ +} + +static void hevc_idct_4x4_lsx(int16_t *coeffs) +{ + __m128i in0, in1; + __m128i in_r0, in_l0, in_r1, in_l1; + __m128i sum0, sum1, sum2, sum3; + __m128i zero = __lsx_vldi(0x00); + + in0 = __lsx_vld(coeffs, 0); + in1 = __lsx_vld(coeffs, 16); + in_r0 = __lsx_vilvl_h(zero, in0); + in_l0 = __lsx_vilvh_h(zero, in0); + in_r1 = __lsx_vilvl_h(zero, in1); + in_l1 = __lsx_vilvh_h(zero, in1); + + HEVC_IDCT4x4_COL(in_r0, in_l0, in_r1, in_l1, sum0, sum1, sum2, sum3, 7); + TRANSPOSE4x4_W(sum0, sum1, sum2, sum3, in_r0, in_l0, in_r1, in_l1); + HEVC_IDCT4x4_COL(in_r0, in_l0, in_r1, in_l1, sum0, sum1, sum2, sum3, 12); + + /* Pack and transpose */ + in0 = __lsx_vpickev_h(sum2, sum0); + in1 = __lsx_vpickev_h(sum3, sum1); + sum0 = __lsx_vilvl_h(in1, in0); + sum1 = __lsx_vilvh_h(in1, in0); + in0 = __lsx_vilvl_w(sum1, sum0); + in1 = __lsx_vilvh_w(sum1, sum0); + + __lsx_vst(in0, coeffs, 0); + __lsx_vst(in1, coeffs, 16); +} + +static void hevc_idct_8x8_lsx(int16_t *coeffs) +{ + const int16_t *filter = >8x8_cnst[0]; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + + LSX_DUP4_ARG2(__lsx_vld, coeffs, 0, coeffs, 16, coeffs, 32, + coeffs, 48, in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, coeffs, 64, coeffs, 80, coeffs, 96, + coeffs, 112, in4, in5, in6, in7); + HEVC_IDCT8x8_COL(in0, in1, in2, in3, in4, in5, in6, in7, 7); + TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + HEVC_IDCT8x8_COL(in0, in1, in2, in3, in4, in5, in6, in7, 12); + TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + + __lsx_vst(in0, coeffs,0); + __lsx_vst(in1, coeffs,16); + __lsx_vst(in2, coeffs,32); + __lsx_vst(in3, coeffs,48); + __lsx_vst(in4, coeffs,64); + __lsx_vst(in5, coeffs,80); + __lsx_vst(in6, coeffs,96); + __lsx_vst(in7, coeffs,112); +} + +static void hevc_idct_16x16_lsx(int16_t *coeffs) +{ + int16_t i, j, k; + int16_t buf[256]; + int16_t *buf_ptr = &buf[0]; + int16_t *src = coeffs; + const int16_t *filter = >16x16_cnst[0]; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r; + __m128i src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l, src7_l; + + for (i = 2; i--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 32, src, 64, src, 96, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, src, 128, src, 160, src, 192, src, 224, + in4, in5, in6, in7); + LSX_DUP4_ARG2(__lsx_vld, src, 256, src, 288, src, 320, src, 352, + in8, in9, in10, in11); + LSX_DUP4_ARG2(__lsx_vld, src, 384, src, 416, src, 448, src, 480, + in12, in13, in14, in15); + + LSX_DUP4_ARG2(__lsx_vilvl_h, in4, in0, in12, in8, in6, in2, in14, in10, + src0_r, src1_r, src2_r, src3_r); + LSX_DUP4_ARG2(__lsx_vilvl_h, in5, in1, in13, in9, in3, in7, in11, in15, + src4_r, src5_r, src6_r, src7_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, in4, in0, in12, in8, in6, in2, in14, in10, + src0_l, src1_l, src2_l, src3_l); + LSX_DUP4_ARG2(__lsx_vilvh_h, in5, in1, in13, in9, in3, in7, in11, in15, + src4_l, src5_l, src6_l, src7_l); + + HEVC_IDCT16x16_COL(src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, + src6_r, src7_r, src0_l, src1_l, src2_l, src3_l, + src4_l, src5_l, src6_l, src7_l, 7); + + src += 8; + buf_ptr = (&buf[0] + 8); + filter = >16x16_cnst[0]; + } + + src = &buf[0]; + buf_ptr = coeffs; + filter = >16x16_cnst[0]; + + for (i = 2; i--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, + in0, in8, in1, in9); + LSX_DUP4_ARG2(__lsx_vld, src, 64, src, 80, src, 96, src, 112, + in2, in10, in3, in11); + LSX_DUP4_ARG2(__lsx_vld, src, 128, src, 144, src, 160, src, 176, + in4, in12, in5, in13); + LSX_DUP4_ARG2(__lsx_vld, src, 192, src, 208, src, 224, src, 240, + in6, in14, in7, in15); + TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, + in8, in9, in10, in11, in12, in13, in14, in15); + LSX_DUP4_ARG2(__lsx_vilvl_h, in4, in0, in12, in8, in6, in2, in14, in10, + src0_r, src1_r, src2_r, src3_r); + LSX_DUP4_ARG2(__lsx_vilvl_h, in5, in1, in13, in9, in3, in7, in11, in15, + src4_r, src5_r, src6_r, src7_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, in4, in0, in12, in8, in6, in2, in14, in10, + src0_l, src1_l, src2_l, src3_l); + LSX_DUP4_ARG2(__lsx_vilvh_h, in5, in1, in13, in9, in3, in7, in11, in15, + src4_l, src5_l, src6_l, src7_l); + HEVC_IDCT16x16_COL(src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, + src6_r, src7_r, src0_l, src1_l, src2_l, src3_l, + src4_l, src5_l, src6_l, src7_l, 12); + + src += 128; + buf_ptr = coeffs + 8; + filter = >16x16_cnst[0]; + } + + LSX_DUP4_ARG2(__lsx_vld, coeffs, 0, coeffs, 32, coeffs, 64, coeffs, 96, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, coeffs, 128, coeffs, 160, coeffs, 192, coeffs, 224, + in4, in5, in6, in7); + TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7); + __lsx_vst(vec0, coeffs, 0); + __lsx_vst(vec1, coeffs, 32); + __lsx_vst(vec2, coeffs, 64); + __lsx_vst(vec3, coeffs, 96); + __lsx_vst(vec4, coeffs, 128); + __lsx_vst(vec5, coeffs, 160); + __lsx_vst(vec6, coeffs, 192); + __lsx_vst(vec7, coeffs, 224); + + src = coeffs + 8; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 32, src, 64, src, 96, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, src, 128, src, 160, src, 192, src, 224, + in4, in5, in6, in7); + TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7); + src = coeffs + 128; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 32, src, 64, src, 96, + in8, in9, in10, in11); + LSX_DUP4_ARG2(__lsx_vld, src, 128, src, 160, src, 192, src, 224, + in12, in13, in14, in15); + + __lsx_vst(vec0, src, 0); + __lsx_vst(vec1, src, 32); + __lsx_vst(vec2, src, 64); + __lsx_vst(vec3, src, 96); + __lsx_vst(vec4, src, 128); + __lsx_vst(vec5, src, 160); + __lsx_vst(vec6, src, 192); + __lsx_vst(vec7, src, 224); + TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, + vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7); + src = coeffs + 8; + __lsx_vst(vec0, src, 0); + __lsx_vst(vec1, src, 32); + __lsx_vst(vec2, src, 64); + __lsx_vst(vec3, src, 96); + __lsx_vst(vec4, src, 128); + __lsx_vst(vec5, src, 160); + __lsx_vst(vec6, src, 192); + __lsx_vst(vec7, src, 224); + + src = coeffs + 136; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 32, src, 64, src, 96, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, src, 128, src, 160, src, 192, src, 224, + in4, in5, in6, in7); + TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7); + __lsx_vst(vec0, src, 0); + __lsx_vst(vec1, src, 32); + __lsx_vst(vec2, src, 64); + __lsx_vst(vec3, src, 96); + __lsx_vst(vec4, src, 128); + __lsx_vst(vec5, src, 160); + __lsx_vst(vec6, src, 192); + __lsx_vst(vec7, src, 224); +} + +static void hevc_idct_8x32_column_lsx(int16_t *coeffs, uint8_t buf_pitch, + uint8_t round) +{ + uint8_t i; + uint16_t buf_pitch_2x = (uint16_t)buf_pitch << 1; + uint16_t buf_pitch_4x = (uint16_t)buf_pitch << 2; + uint16_t buf_pitch_8x = (uint16_t)buf_pitch << 3; + uint16_t buf_pitch_16x = (uint16_t)buf_pitch << 4; + uint16_t buf_pitch_32x = (uint16_t)buf_pitch << 5; + const int16_t *filter_ptr0 = >32x32_cnst0[0]; + const int16_t *filter_ptr1 = >32x32_cnst1[0]; + const int16_t *filter_ptr2 = >32x32_cnst2[0]; + const int16_t *filter_ptr3 = >8x8_cnst[0]; + int16_t *src0 = (coeffs + buf_pitch); + int16_t *src1 = (coeffs + 2 * buf_pitch); + int16_t *src2 = (coeffs + 4 * buf_pitch); + int16_t *src3 = (coeffs); + int32_t tmp_buf[8 * 32 + 15]; + int32_t *tmp_buf_ptr = tmp_buf + 15; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r; + __m128i src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l, src7_l; + __m128i filter0, filter1, filter2, filter3; + __m128i sum0_r, sum0_l, sum1_r, sum1_l, tmp0_r, tmp0_l, tmp1_r, tmp1_l; + + /* Align pointer to 64 byte boundary */ + tmp_buf_ptr = (int32_t *)(((uintptr_t) tmp_buf_ptr) & ~(uintptr_t) 63); + + /* process coeff 4, 12, 20, 28 */ + in0 = __lsx_vld(src2, 0); + // TODO: Use vldx once gcc fixed. + //in1 = __lsx_vldx(src2, buf_pitch_16x); + //in2 = __lsx_vldx(src2, buf_pitch_32x); + //in3 = __lsx_vldx(src2, buf_pitch_32x + buf_pitch_16x); + in1 = __lsx_vld(src2 + buf_pitch_8x, 0); + in2 = __lsx_vld(src2 + buf_pitch_16x, 0); + in3 = __lsx_vld(src2 + buf_pitch_16x + buf_pitch_8x, 0); + in4 = __lsx_vld(src3, 0); + // TODO: Use vldx once gcc fixed. + //in5 = __lsx_vldx(src3, buf_pitch_16x); + //in6 = __lsx_vldx(src3, buf_pitch_32x); + //in7 = __lsx_vldx(src3, buf_pitch_32x + buf_pitch_16x); + in5 = __lsx_vld(src3 + buf_pitch_8x, 0); + in6 = __lsx_vld(src3 + buf_pitch_16x, 0); + in7 = __lsx_vld(src3 + buf_pitch_16x + buf_pitch_8x, 0); + LSX_DUP4_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, in6, in4, in7, in5, + src0_r, src1_r, src2_r, src3_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, in1, in0, in3, in2, in6, in4, in7, in5, + src0_l, src1_l, src2_l, src3_l); + + /* loop for all columns of constants */ + for (i = 0; i < 2; i++) { + /* processing single column of constants */ + filter0 = __lsx_vldrepl_w(filter_ptr2, 0); + filter1 = __lsx_vldrepl_w(filter_ptr2, 4); + sum0_r = __lsx_dp2_w_h(src0_r, filter0); + sum0_l = __lsx_dp2_w_h(src0_l, filter0); + sum0_r = __lsx_dp2add_w_h(sum0_r, src1_r, filter1); + sum0_l = __lsx_dp2add_w_h(sum0_l, src1_l, filter1); + __lsx_vst(sum0_r, (tmp_buf_ptr + 2 * i * 8), 0); + __lsx_vst(sum0_l, (tmp_buf_ptr + 2 * i * 8), 16); + + /* processing single column of constants */ + filter0 = __lsx_vldrepl_w(filter_ptr2, 8); + filter1 = __lsx_vldrepl_w(filter_ptr2, 12); + sum0_r = __lsx_dp2_w_h(src0_r, filter0); + sum0_l = __lsx_dp2_w_h(src0_l, filter0); + sum0_r = __lsx_dp2add_w_h(sum0_r, src1_r, filter1); + sum0_l = __lsx_dp2add_w_h(sum0_l, src1_l, filter1); + __lsx_vst(sum0_r, (tmp_buf_ptr + (2 * i + 1) * 8), 0); + __lsx_vst(sum0_l, (tmp_buf_ptr + (2 * i + 1) * 8), 16); + + filter_ptr2 += 8; + } + + /* process coeff 0, 8, 16, 24 */ + /* loop for all columns of constants */ + for (i = 0; i < 2; i++) { + /* processing first column of filter constants */ + filter0 = __lsx_vldrepl_w(filter_ptr3, 0); + filter1 = __lsx_vldrepl_w(filter_ptr3, 4); + + LSX_DUP4_ARG2(__lsx_dp2_w_h, src2_r, filter0, src2_l, filter0, + src3_r, filter1, src3_l, filter1, sum0_r, sum0_l, + tmp1_r, tmp1_l); + sum1_r = __lsx_vsub_w(sum0_r, tmp1_r); + sum1_l = __lsx_vsub_w(sum0_l, tmp1_l); + sum0_r = __lsx_vadd_w(sum0_r, tmp1_r); + sum0_l = __lsx_vadd_w(sum0_l, tmp1_l); + + HEVC_EVEN16_CALC(tmp_buf_ptr, sum0_r, sum0_l, i, (7 - i)); + HEVC_EVEN16_CALC(tmp_buf_ptr, sum1_r, sum1_l, (3 - i), (4 + i)); + + filter_ptr3 += 8; + } + + /* process coeff 2 6 10 14 18 22 26 30 */ + in0 = __lsx_vld(src1, 0); + // TODO: Use vldx once gcc fixed. + //in1 = __lsx_vldx(src1, buf_pitch_8x); + //in2 = __lsx_vldx(src1, buf_pitch_16x); + //in3 = __lsx_vldx(src1, buf_pitch_16x + buf_pitch_8x); + //in4 = __lsx_vldx(src1, buf_pitch_32x); + //in5 = __lsx_vldx(src1, buf_pitch_32x + buf_pitch_8x); + //in6 = __lsx_vldx(src1, buf_pitch_32x + buf_pitch_16x); + //in7 = __lsx_vldx(src1, buf_pitch_32x + buf_pitch_16x + buf_pitch_8x); + in1 = __lsx_vld(src1 + buf_pitch_4x, 0); + in2 = __lsx_vld(src1 + buf_pitch_8x, 0); + in3 = __lsx_vld(src1 + buf_pitch_8x + buf_pitch_4x, 0); + in4 = __lsx_vld(src1 + buf_pitch_16x, 0); + in5 = __lsx_vld(src1 + buf_pitch_16x + buf_pitch_4x, 0); + in6 = __lsx_vld(src1 + buf_pitch_16x + buf_pitch_8x, 0); + in7 = __lsx_vld(src1 + buf_pitch_16x + buf_pitch_8x + buf_pitch_4x, 0); + LSX_DUP4_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, in5, in4, in7, in6, + src0_r, src1_r, src2_r, src3_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, in1, in0, in3, in2, in5, in4, in7, in6, + src0_l, src1_l, src2_l, src3_l); + + /* loop for all columns of constants */ + for (i = 0; i < 8; i++) { + /* processing single column of constants */ + filter0 = __lsx_vldrepl_w(filter_ptr1, 0); + filter1 = __lsx_vldrepl_w(filter_ptr1, 4); + filter2 = __lsx_vldrepl_w(filter_ptr1, 8); + filter3 = __lsx_vldrepl_w(filter_ptr1, 12); + sum0_r = __lsx_dp2_w_h(src0_r, filter0); + sum0_l = __lsx_dp2_w_h(src0_l, filter0); + sum0_r = __lsx_dp2add_w_h(sum0_r, src1_r, filter1); + sum0_l = __lsx_dp2add_w_h(sum0_l, src1_l, filter1); + sum0_r = __lsx_dp2add_w_h(sum0_r, src2_r, filter2); + sum0_l = __lsx_dp2add_w_h(sum0_l, src2_l, filter2); + sum0_r = __lsx_dp2add_w_h(sum0_r, src3_r, filter3); + sum0_l = __lsx_dp2add_w_h(sum0_l, src3_l, filter3); + + tmp0_r = __lsx_vld(tmp_buf_ptr + (i << 3), 0); + tmp0_l = __lsx_vld(tmp_buf_ptr + (i << 3), 16); + tmp1_r = tmp0_r; + tmp1_l = tmp0_l; + tmp0_r = __lsx_vadd_w(tmp0_r, sum0_r); + tmp0_l = __lsx_vadd_w(tmp0_l, sum0_l); + tmp1_r = __lsx_vsub_w(tmp1_r, sum0_r); + tmp1_l = __lsx_vsub_w(tmp1_l, sum0_l); + __lsx_vst(tmp0_r, tmp_buf_ptr + (i << 3), 0); + __lsx_vst(tmp0_l, tmp_buf_ptr + (i << 3), 16); + __lsx_vst(tmp1_r, tmp_buf_ptr + ((15 - i) * 8), 0); + __lsx_vst(tmp1_l, tmp_buf_ptr + ((15 - i) * 8), 16); + + filter_ptr1 += 8; + } + + /* process coeff 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 */ + in0 = __lsx_vld(src0, 0); + // TODO: Use vldx once gcc fixed. + //in1 = __lsx_vldx(src0, buf_pitch_4x); + //in2 = __lsx_vldx(src0, buf_pitch_8x); + //in3 = __lsx_vldx(src0, buf_pitch_8x + buf_pitch_4x); + //in4 = __lsx_vldx(src0, buf_pitch_16x); + //in5 = __lsx_vldx(src0, buf_pitch_16x + buf_pitch_4x); + //in6 = __lsx_vldx(src0, buf_pitch_16x + buf_pitch_8x); + //in7 = __lsx_vldx(src0, buf_pitch_16x + buf_pitch_8x + buf_pitch_4x); + in1 = __lsx_vld(src0 + buf_pitch_2x, 0); + in2 = __lsx_vld(src0 + buf_pitch_4x, 0); + in3 = __lsx_vld(src0 + buf_pitch_4x + buf_pitch_2x, 0); + in4 = __lsx_vld(src0 + buf_pitch_8x, 0); + in5 = __lsx_vld(src0 + buf_pitch_8x + buf_pitch_2x, 0); + in6 = __lsx_vld(src0 + buf_pitch_8x + buf_pitch_4x, 0); + in7 = __lsx_vld(src0 + buf_pitch_8x + buf_pitch_4x + buf_pitch_2x, 0); + src0 += 16 * buf_pitch; + LSX_DUP4_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, in5, in4, in7, in6, + src0_r, src1_r, src2_r, src3_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, in1, in0, in3, in2, in5, in4, in7, in6, + src0_l, src1_l, src2_l, src3_l); + in0 = __lsx_vld(src0, 0); + // TODO: Use vldx once gcc fixed. + //in1 = __lsx_vldx(src0, buf_pitch_4x); + //in2 = __lsx_vldx(src0, buf_pitch_8x); + //in3 = __lsx_vldx(src0, buf_pitch_8x + buf_pitch_4x); + //in4 = __lsx_vldx(src0, buf_pitch_16x); + //in5 = __lsx_vldx(src0, buf_pitch_16x + buf_pitch_4x); + //in6 = __lsx_vldx(src0, buf_pitch_16x + buf_pitch_8x); + //in7 = __lsx_vldx(src0, buf_pitch_16x + buf_pitch_8x + buf_pitch_4x); + in1 = __lsx_vld(src0 + buf_pitch_2x, 0); + in2 = __lsx_vld(src0 + buf_pitch_4x, 0); + in3 = __lsx_vld(src0 + buf_pitch_4x + buf_pitch_2x, 0); + in4 = __lsx_vld(src0 + buf_pitch_8x, 0); + in5 = __lsx_vld(src0 + buf_pitch_8x + buf_pitch_2x, 0); + in6 = __lsx_vld(src0 + buf_pitch_8x + buf_pitch_4x, 0); + in7 = __lsx_vld(src0 + buf_pitch_8x + buf_pitch_4x + buf_pitch_2x, 0); + LSX_DUP4_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, in5, in4, in7, in6, + src4_r, src5_r, src6_r, src7_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, in1, in0, in3, in2, in5, in4, in7, in6, + src4_l, src5_l, src6_l, src7_l); + + /* loop for all columns of filter constants */ + for (i = 0; i < 16; i++) { + /* processing single column of constants */ + filter0 = __lsx_vldrepl_w(filter_ptr0, 0); + filter1 = __lsx_vldrepl_w(filter_ptr0, 4); + filter2 = __lsx_vldrepl_w(filter_ptr0, 8); + filter3 = __lsx_vldrepl_w(filter_ptr0, 12); + sum0_r = __lsx_dp2_w_h(src0_r, filter0); + sum0_l = __lsx_dp2_w_h(src0_l, filter0); + sum0_r = __lsx_dp2add_w_h(sum0_r, src1_r, filter1); + sum0_l = __lsx_dp2add_w_h(sum0_l, src1_l, filter1); + sum0_r = __lsx_dp2add_w_h(sum0_r, src2_r, filter2); + sum0_l = __lsx_dp2add_w_h(sum0_l, src2_l, filter2); + sum0_r = __lsx_dp2add_w_h(sum0_r, src3_r, filter3); + sum0_l = __lsx_dp2add_w_h(sum0_l, src3_l, filter3); + tmp1_r = sum0_r; + tmp1_l = sum0_l; + + filter0 = __lsx_vldrepl_w(filter_ptr0, 16); + filter1 = __lsx_vldrepl_w(filter_ptr0, 20); + filter2 = __lsx_vldrepl_w(filter_ptr0, 24); + filter3 = __lsx_vldrepl_w(filter_ptr0, 28); + sum0_r = __lsx_dp2_w_h(src4_r, filter0); + sum0_l = __lsx_dp2_w_h(src4_l, filter0); + sum0_r = __lsx_dp2add_w_h(sum0_r, src5_r, filter1); + sum0_l = __lsx_dp2add_w_h(sum0_l, src5_l, filter1); + sum0_r = __lsx_dp2add_w_h(sum0_r, src6_r, filter2); + sum0_l = __lsx_dp2add_w_h(sum0_l, src6_l, filter2); + sum0_r = __lsx_dp2add_w_h(sum0_r, src7_r, filter3); + sum0_l = __lsx_dp2add_w_h(sum0_l, src7_l, filter3); + sum0_r = __lsx_vadd_w(sum0_r, tmp1_r); + sum0_l = __lsx_vadd_w(sum0_l, tmp1_l); + + tmp0_r = __lsx_vld(tmp_buf_ptr + i * 8, 0); + tmp0_l = __lsx_vld(tmp_buf_ptr + i * 8, 16); + tmp1_r = tmp0_r; + tmp1_l = tmp0_l; + tmp0_r = __lsx_vadd_w(tmp0_r, sum0_r); + tmp0_l = __lsx_vadd_w(tmp0_l, sum0_l); + sum1_r = __lsx_vreplgr2vr_w(round); + tmp0_r = __lsx_vssrarn_h_w(tmp0_r, sum1_r); + tmp0_l = __lsx_vssrarn_h_w(tmp0_l, sum1_r); + in0 = __lsx_vpackev_d(tmp0_l, tmp0_r); + __lsx_vst(in0, (coeffs + i * buf_pitch), 0); + tmp1_r = __lsx_vsub_w(tmp1_r, sum0_r); + tmp1_l = __lsx_vsub_w(tmp1_l, sum0_l); + tmp1_r = __lsx_vssrarn_h_w(tmp1_r, sum1_r); + tmp1_l = __lsx_vssrarn_h_w(tmp1_l, sum1_r); + in0 = __lsx_vpackev_d(tmp1_l, tmp1_r); + __lsx_vst(in0, (coeffs + (31 - i) * buf_pitch), 0); + + filter_ptr0 += 16; + } +} + +static void hevc_idct_transpose_32x8_to_8x32(int16_t *coeffs, int16_t *tmp_buf) +{ + uint8_t i; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + + for (i = 0; i < 4; i++) { + LSX_DUP4_ARG2(__lsx_vld, coeffs, 0, coeffs, 64, coeffs, 128, + coeffs, 192, in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, coeffs, 256, coeffs, 320, coeffs, 384, + coeffs, 448, in4, in5, in6, in7); + coeffs += 8; + TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + __lsx_vst(in0, tmp_buf, 0); + __lsx_vst(in1, tmp_buf, 16); + __lsx_vst(in2, tmp_buf, 32); + __lsx_vst(in3, tmp_buf, 48); + __lsx_vst(in4, tmp_buf, 64); + __lsx_vst(in5, tmp_buf, 80); + __lsx_vst(in6, tmp_buf, 96); + __lsx_vst(in7, tmp_buf, 112); + tmp_buf += 64; + } +} + +static void hevc_idct_transpose_8x32_to_32x8(int16_t *tmp_buf, int16_t *coeffs) +{ + uint8_t i; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + + for (i = 0; i < 4; i++) { + LSX_DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 16, tmp_buf, 32, + tmp_buf, 48, in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, tmp_buf, 64, tmp_buf, 80, tmp_buf, 96, + tmp_buf, 112, in4, in5, in6, in7); + tmp_buf += 64; + TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + __lsx_vst(in0, coeffs, 0); + __lsx_vst(in1, coeffs, 64); + __lsx_vst(in2, coeffs, 128); + __lsx_vst(in3, coeffs, 192); + __lsx_vst(in4, coeffs, 256); + __lsx_vst(in5, coeffs, 320); + __lsx_vst(in6, coeffs, 384); + __lsx_vst(in7, coeffs, 448); + coeffs += 8; + } +} + +static void hevc_idct_32x32_lsx(int16_t *coeffs) +{ + uint8_t row_cnt, col_cnt; + int16_t *src = coeffs; + int16_t tmp_buf[8 * 32 + 31]; + int16_t *tmp_buf_ptr = tmp_buf + 31; + uint8_t round; + uint8_t buf_pitch; + + /* Align pointer to 64 byte boundary */ + tmp_buf_ptr = (int16_t *)(((uintptr_t) tmp_buf_ptr) & ~(uintptr_t) 63); + + /* column transform */ + round = 7; + buf_pitch = 32; + for (col_cnt = 0; col_cnt < 4; col_cnt++) { + /* process 8x32 blocks */ + hevc_idct_8x32_column_lsx((coeffs + col_cnt * 8), buf_pitch, round); + } + + /* row transform */ + round = 12; + buf_pitch = 8; + for (row_cnt = 0; row_cnt < 4; row_cnt++) { + /* process 32x8 blocks */ + src = (coeffs + 32 * 8 * row_cnt); + + hevc_idct_transpose_32x8_to_8x32(src, tmp_buf_ptr); + hevc_idct_8x32_column_lsx(tmp_buf_ptr, buf_pitch, round); + hevc_idct_transpose_8x32_to_32x8(tmp_buf_ptr, src); + } +} + +void ff_hevc_idct_4x4_lsx(int16_t *coeffs, int col_limit) +{ + hevc_idct_4x4_lsx(coeffs); +} + +void ff_hevc_idct_8x8_lsx(int16_t *coeffs, int col_limit) +{ + hevc_idct_8x8_lsx(coeffs); +} + +void ff_hevc_idct_16x16_lsx(int16_t *coeffs, int col_limit) +{ + hevc_idct_16x16_lsx(coeffs); +} + +void ff_hevc_idct_32x32_lsx(int16_t *coeffs, int col_limit) +{ + hevc_idct_32x32_lsx(coeffs); +} diff --git a/libavcodec/loongarch/hevc_lpf_sao_lsx.c b/libavcodec/loongarch/hevc_lpf_sao_lsx.c new file mode 100644 index 0000000000..01b22085ed --- /dev/null +++ b/libavcodec/loongarch/hevc_lpf_sao_lsx.c @@ -0,0 +1,2422 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "hevcdsp_lsx.h" + +static void hevc_loopfilter_luma_hor_lsx(uint8_t *src, int32_t stride, + int32_t beta, int32_t *tc, + uint8_t *p_is_pcm, uint8_t *q_is_pcm) +{ + uint8_t *p3 = src - (stride << 2); + uint8_t *p2 = src - ((stride << 1) + stride); + uint8_t *p1 = src - (stride << 1); + uint8_t *p0 = src - stride; + uint8_t *q0 = src; + uint8_t *q1 = src + stride; + uint8_t *q2 = src + (stride << 1); + uint8_t *q3 = src + (stride << 1) + stride; + uint8_t flag0, flag1; + int32_t dp00, dq00, dp30, dq30, d00, d30, d0030, d0434; + int32_t dp04, dq04, dp34, dq34, d04, d34; + int32_t tc0, p_is_pcm0, q_is_pcm0, beta30, beta20, tc250; + int32_t tc4, p_is_pcm4, q_is_pcm4, tc254, tmp; + const int32_t stride_2x = (stride << 1); + const int32_t stride_4x = (stride << 2); + const int32_t stride_3x = stride_2x + stride; + + __m128i dst0, dst1, dst2, dst3, dst4, dst5; + __m128i cmp0, cmp1, cmp2, cmp3, p_is_pcm_vec, q_is_pcm_vec; + __m128i temp0, temp1; + __m128i temp2, tc_pos, tc_neg; + __m128i diff0, diff1, delta0, delta1, delta2, abs_delta0; + __m128i zero = {0}; + __m128i p3_src, p2_src, p1_src, p0_src, q0_src, q1_src, q2_src, q3_src; + + dp00 = abs(p2[0] - (p1[0] << 1) + p0[0]); + dq00 = abs(q2[0] - (q1[0] << 1) + q0[0]); + dp30 = abs(p2[3] - (p1[3] << 1) + p0[3]); + dq30 = abs(q2[3] - (q1[3] << 1) + q0[3]); + d00 = dp00 + dq00; + d30 = dp30 + dq30; + dp04 = abs(p2[4] - (p1[4] << 1) + p0[4]); + dq04 = abs(q2[4] - (q1[4] << 1) + q0[4]); + dp34 = abs(p2[7] - (p1[7] << 1) + p0[7]); + dq34 = abs(q2[7] - (q1[7] << 1) + q0[7]); + d04 = dp04 + dq04; + d34 = dp34 + dq34; + + p_is_pcm0 = p_is_pcm[0]; + p_is_pcm4 = p_is_pcm[1]; + q_is_pcm0 = q_is_pcm[0]; + q_is_pcm4 = q_is_pcm[1]; + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, p_is_pcm0, p_is_pcm4, cmp0, cmp1); + p_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + p_is_pcm_vec = __lsx_vseqi_d(p_is_pcm_vec, 0); + d0030 = (d00 + d30) >= beta; + d0434 = (d04 + d34) >= beta; + LSX_DUP2_ARG1(__lsx_vreplgr2vr_w, d0030, d0434, cmp0, cmp1); + cmp3 = __lsx_vpackev_w(cmp1, cmp0); + cmp3 = __lsx_vseqi_w(cmp3, 0); + + if ((!p_is_pcm0 || !p_is_pcm4 || !q_is_pcm0 || !q_is_pcm4) && + (!d0030 || !d0434)) { + LSX_DUP4_ARG2(__lsx_vld, p3, 0, p2, 0, p1, 0, p0, 0, p3_src, p2_src, p1_src, + p0_src); + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, q_is_pcm0, q_is_pcm4, cmp0, cmp1); + q_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + q_is_pcm_vec = __lsx_vseqi_d(q_is_pcm_vec, 0); + + tc0 = tc[0]; + beta30 = beta >> 3; + beta20 = beta >> 2; + tc250 = (((tc0 << 2) + tc0 + 1) >> 1); + tc4 = tc[1]; + tc254 = (((tc4 << 2) + tc4 + 1) >> 1); + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_h, tc0, tc4, cmp0, cmp1); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3_src, zero, p2_src, zero, p1_src, zero, + p0_src, p3_src, p2_src, p1_src, p0_src); + LSX_DUP4_ARG2(__lsx_vld, q0, 0, q1, 0, q2, 0, q3, 0, q0_src, q1_src, q2_src, + q3_src); + flag0 = abs(p3[0] - p0[0]) + abs(q3[0] - q0[0]) < beta30 && abs(p0[0] - q0[0]) + < tc250; + flag0 = flag0 && (abs(p3[3] - p0[3]) + abs(q3[3] - q0[3]) < beta30 && + abs(p0[3] - q0[3]) < tc250 && (d00 << 1) < beta20 && (d30 << 1) < beta20); + tc_pos = __lsx_vpackev_d(cmp1, cmp0); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0_src, zero, q1_src, zero, q2_src, zero, + q3_src, q0_src, q1_src, q2_src, q3_src); + + flag1 = abs(p3[4] - p0[4]) + abs(q3[4] - q0[4]) < beta30 && abs(p0[4] - q0[4]) + < tc254; + flag1 = flag1 && (abs(p3[7] - p0[7]) + abs(q3[7] - q0[7]) < beta30 && + abs(p0[7] - q0[7]) < tc254 && (d04 << 1) < beta20 && (d34 << 1) < beta20); + LSX_DUP2_ARG1(__lsx_vreplgr2vr_w, flag0, flag1, cmp0, cmp1); + cmp2 = __lsx_vpackev_w(cmp1, cmp0); + cmp2 = __lsx_vseqi_w(cmp2, 0); + + if (flag0 && flag1) { /* strong only */ + /* strong filter */ + tc_pos = __lsx_vslli_h(tc_pos, 1); + tc_neg = __lsx_vneg_h(tc_pos); + + /* p part */ + LSX_DUP2_ARG2(__lsx_vadd_h, p1_src, p0_src, temp0, q0_src, temp0, temp0); + temp1 = __lsx_vadd_h(p3_src, p2_src); + temp1 = __lsx_vslli_h(temp1, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p2_src, temp1, temp0, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, p2_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst0 = __lsx_vadd_h(temp2, p2_src); + + temp1 = __lsx_vadd_h(temp0, p2_src); + temp1 = __lsx_vsrari_h(temp1, 2); + temp2 = __lsx_vsub_h(temp1, p1_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst1 = __lsx_vadd_h(temp2, p1_src); + + temp1 = __lsx_vslli_h(temp0, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p2_src, temp1, q1_src, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, p0_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst2 = __lsx_vadd_h(temp2, p0_src); + + p_is_pcm_vec = __lsx_vnor_v(p_is_pcm_vec, p_is_pcm_vec); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst0, p2_src, p_is_pcm_vec, dst1, p1_src, + p_is_pcm_vec, dst0, dst1); + dst2 = __lsx_vbitsel_v(dst2, p0_src, p_is_pcm_vec); + + /* q part */ + LSX_DUP2_ARG2(__lsx_vadd_h, q1_src, p0_src, temp0, q0_src, temp0, temp0); + temp1 = __lsx_vadd_h(q3_src, q2_src); + temp1 = __lsx_vslli_h(temp1, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, q2_src, temp1, temp0, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, q2_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst5 = __lsx_vadd_h(temp2, q2_src); + + temp1 = __lsx_vadd_h(temp0, q2_src); + temp1 = __lsx_vsrari_h(temp1, 2); + temp2 = __lsx_vsub_h(temp1, q1_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst4 = __lsx_vadd_h(temp2, q1_src); + + temp0 = __lsx_vslli_h(temp0, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp0, p1_src, temp1, q2_src, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, q0_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst3 = __lsx_vadd_h(temp2, q0_src); + + q_is_pcm_vec = __lsx_vnor_v(q_is_pcm_vec, q_is_pcm_vec); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst3, q0_src, q_is_pcm_vec, dst4, q1_src, + q_is_pcm_vec, dst3, dst4); + dst5 = __lsx_vbitsel_v(dst5, q2_src, q_is_pcm_vec); + + /* pack results to 8 bit */ + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1); + dst2 = __lsx_vpickev_b(dst5, dst4); + + /* pack src to 8 bit */ + LSX_DUP2_ARG2(__lsx_vpickev_b, p1_src, p2_src, q0_src, p0_src, dst3, dst4); + dst5 = __lsx_vpickev_b(q2_src, q1_src); + + cmp3 = __lsx_vnor_v(cmp3, cmp3); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst0, dst3, cmp3, dst1, dst4, cmp3, dst0, dst1); + dst2 = __lsx_vbitsel_v(dst2, dst5, cmp3); + + __lsx_vstelm_d(dst0, p2, 0, 0); + __lsx_vstelm_d(dst0, p2 + stride, 0, 1); + __lsx_vstelm_d(dst1, p2 + stride_2x, 0, 0); + __lsx_vstelm_d(dst1, p2 + stride_3x, 0, 1); + __lsx_vstelm_d(dst2, p2 + stride_4x, 0, 0); + __lsx_vstelm_d(dst2, p2 + stride_4x + stride, 0, 1); + /* strong filter ends */ + } else if (flag0 == flag1) { /* weak only */ + /* weak filter */ + tc_neg = __lsx_vneg_h(tc_pos); + LSX_DUP2_ARG2(__lsx_vsub_h, q0_src, p0_src, q1_src, p1_src, diff0, diff1); + LSX_DUP2_ARG2(__lsx_vadd_h, __lsx_vslli_h(diff0, 3), diff0, __lsx_vslli_h( + diff1, 1), diff1, diff0, diff1); + delta0 = __lsx_vsub_h(diff0, diff1); + delta0 = __lsx_vsrari_h(delta0, 4); + temp1 = __lsx_vadd_h(__lsx_vslli_h(tc_pos, 3), __lsx_vslli_h(tc_pos, 1)); + abs_delta0 = __lsx_vadda_h(delta0, zero); + abs_delta0 = __lsx_vsle_hu(temp1, abs_delta0); + abs_delta0 = __lsx_vnor_v(abs_delta0, abs_delta0); + + delta0 = __lsx_clip_h(delta0, tc_neg, tc_pos); + temp2 = __lsx_vadd_h(delta0, p0_src); + temp2 = __lsx_clamp255_h(temp2); + temp0 = __lsx_vbitsel_v(temp2, p0_src, __lsx_vnor_v(p_is_pcm_vec, + p_is_pcm_vec)); + temp2 = __lsx_vsub_h(q0_src, delta0); + temp2 = __lsx_clamp255_h(temp2); + temp2 = __lsx_vbitsel_v(temp2, q0_src, __lsx_vnor_v(q_is_pcm_vec, + q_is_pcm_vec)); + LSX_DUP2_ARG2(__lsx_vnor_v, p_is_pcm_vec, p_is_pcm_vec, q_is_pcm_vec, + q_is_pcm_vec, p_is_pcm_vec, q_is_pcm_vec); + + tmp = (beta + (beta >> 1)) >> 3; + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, dp00 + dp30 < tmp, dp04 + dp34 < tmp, + cmp0, cmp1); + cmp0 = __lsx_vpackev_d(cmp1, cmp0); + cmp0 = __lsx_vseqi_d(cmp0, 0); + p_is_pcm_vec = __lsx_vor_v(p_is_pcm_vec, cmp0); + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, dq00 + dq30 < tmp, dq04 + dq34 < tmp, + cmp0, cmp1); + cmp0 = __lsx_vpackev_d(cmp1, cmp0); + cmp0 = __lsx_vseqi_d(cmp0, 0); + q_is_pcm_vec = __lsx_vor_v(q_is_pcm_vec, cmp0); + tc_pos = __lsx_vsrai_h(tc_pos, 1); + tc_neg = __lsx_vneg_h(tc_pos); + + LSX_DUP2_ARG2(__lsx_vavgr_hu, p2_src, p0_src, q0_src, q2_src, delta1, delta2); + LSX_DUP2_ARG2(__lsx_vsub_h, delta1, p1_src, delta2, q1_src, delta1, delta2); + delta1 = __lsx_vadd_h(delta1, delta0); + delta2 = __lsx_vsub_h(delta2, delta0); + LSX_DUP2_ARG2(__lsx_vsrai_h, delta1, 1, delta2, 1, delta1, delta2); + LSX_DUP2_ARG3(__lsx_clip_h, delta1, tc_neg, tc_pos, delta2, tc_neg, tc_pos, + delta1, delta2); + LSX_DUP2_ARG2(__lsx_vadd_h, p1_src, delta1, q1_src, delta2, delta1, delta2); + LSX_DUP2_ARG1(__lsx_clamp255_h, delta1, delta2, delta1, delta2); + LSX_DUP2_ARG3(__lsx_vbitsel_v, delta1, p1_src, p_is_pcm_vec, delta2, q1_src, + q_is_pcm_vec, delta1, delta2); + + abs_delta0 = __lsx_vnor_v(abs_delta0, abs_delta0); + LSX_DUP4_ARG3(__lsx_vbitsel_v, delta1, p1_src, abs_delta0, temp0, p0_src, + abs_delta0, temp2, q0_src, abs_delta0, delta2, q1_src, + abs_delta0, dst1, dst2, dst3, dst4); + /* pack results to 8 bit */ + LSX_DUP2_ARG2(__lsx_vpickev_b, dst2, dst1, dst4, dst3, dst0, dst1); + /* pack src to 8 bit */ + LSX_DUP2_ARG2(__lsx_vpickev_b, p0_src, p1_src, q1_src, q0_src, dst2, dst3); + cmp3 = __lsx_vnor_v(cmp3, cmp3); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst0, dst2, cmp3, dst1, dst3, cmp3, dst0, + dst1); + + p2 += stride; + __lsx_vstelm_d(dst0, p2, 0, 0); + __lsx_vstelm_d(dst0, p2 + stride, 0, 1); + __lsx_vstelm_d(dst1, p2 + stride_2x, 0, 0); + __lsx_vstelm_d(dst1, p2 + stride_3x, 0, 1); + /* weak filter ends */ + } else { /* strong + weak */ + /* strong filter */ + tc_pos = __lsx_vslli_h(tc_pos, 1); + tc_neg = __lsx_vneg_h(tc_pos); + + /* p part */ + LSX_DUP2_ARG2(__lsx_vadd_h, p1_src, p0_src, temp0, q0_src, temp0, temp0); + temp1 = __lsx_vadd_h(p3_src, p2_src); + temp1 = __lsx_vslli_h(temp1, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p2_src, temp1, temp0, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, p2_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst0 = __lsx_vadd_h(temp2, p2_src); + + temp1 = __lsx_vadd_h(temp0, p2_src); + temp1 = __lsx_vsrari_h(temp1, 2); + temp2 = __lsx_vsub_h(temp1, p1_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst1 = __lsx_vadd_h(temp2, p1_src); + + temp1 = __lsx_vslli_h(temp0, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p2_src, temp1, q1_src, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, p0_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst2 = __lsx_vadd_h(temp2, p0_src); + + p_is_pcm_vec = __lsx_vnor_v(p_is_pcm_vec, p_is_pcm_vec); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst0, p2_src, p_is_pcm_vec, dst1, p1_src, + p_is_pcm_vec, dst0, dst1); + dst2 = __lsx_vbitsel_v(dst2, p0_src, p_is_pcm_vec); + + /* q part */ + LSX_DUP2_ARG2(__lsx_vadd_h, q1_src, p0_src, temp0, q0_src, temp0, temp0); + temp1 = __lsx_vadd_h(q3_src, q2_src); + temp1 = __lsx_vslli_h(temp1, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, q2_src, temp1, temp0, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, q2_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst5 = __lsx_vadd_h(temp2, q2_src); + + temp1 = __lsx_vadd_h(temp0, q2_src); + temp1 = __lsx_vsrari_h(temp1, 2); + temp2 = __lsx_vsub_h(temp1, q1_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst4 = __lsx_vadd_h(temp2, q1_src); + + temp1 = __lsx_vslli_h(temp0, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p1_src, temp1, q2_src, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, q0_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst3 = __lsx_vadd_h(temp2, q0_src); + + q_is_pcm_vec = __lsx_vnor_v(q_is_pcm_vec, q_is_pcm_vec); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst3, q0_src, q_is_pcm_vec, dst4, q1_src, + q_is_pcm_vec, dst3, dst4); + dst5 = __lsx_vbitsel_v(dst5, q2_src, q_is_pcm_vec); + + /* pack strong results to 8 bit */ + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1); + dst2 = __lsx_vpickev_b(dst5, dst4); + /* strong filter ends */ + + /* weak filter */ + tc_pos = __lsx_vsrai_h(tc_pos, 1); + tc_neg = __lsx_vneg_h(tc_pos); + + LSX_DUP2_ARG2(__lsx_vsub_h, q0_src, p0_src, q1_src, p1_src, diff0, diff1); + LSX_DUP2_ARG2(__lsx_vadd_h, __lsx_vslli_h(diff0, 3), diff0, __lsx_vslli_h( + diff1, 1), diff1, diff0, diff1); + delta0 = __lsx_vsub_h(diff0, diff1); + delta0 = __lsx_vsrari_h(delta0, 4); + temp1 = __lsx_vadd_h(__lsx_vslli_h(tc_pos, 3), __lsx_vslli_h(tc_pos, 1)); + abs_delta0 = __lsx_vadda_h(delta0, zero); + abs_delta0 = __lsx_vsle_hu(temp1, abs_delta0); + abs_delta0 = __lsx_vnor_v(abs_delta0, abs_delta0); + + delta0 = __lsx_clip_h(delta0, tc_neg, tc_pos); + temp2 = __lsx_vadd_h(delta0, p0_src); + temp2 = __lsx_clamp255_h(temp2); + temp0 = __lsx_vbitsel_v(temp2, p0_src, p_is_pcm_vec); + + temp2 = __lsx_vsub_h(q0_src, delta0); + temp2 = __lsx_clamp255_h(temp2); + temp2 = __lsx_vbitsel_v(temp2, q0_src, q_is_pcm_vec); + + tmp = (beta + (beta >> 1)) >> 3; + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, dp00 + dp30 < tmp, dp04 + dp34 < tmp, + cmp0, cmp1); + cmp0 = __lsx_vpackev_d(cmp1, cmp0); + p_is_pcm_vec = __lsx_vor_v(p_is_pcm_vec, __lsx_vseqi_d(cmp0, 0)); + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, dq00 + dq30 < tmp, dq04 + dq34 < tmp, + cmp0, cmp1); + cmp0 = __lsx_vpackev_d(cmp1, cmp0); + q_is_pcm_vec = __lsx_vor_v(q_is_pcm_vec, __lsx_vseqi_d(cmp0, 0)); + + tc_pos = __lsx_vsrai_h(tc_pos, 1); + tc_neg = __lsx_vneg_h(tc_pos); + + LSX_DUP2_ARG2(__lsx_vavgr_hu, p2_src, p0_src, q0_src, q2_src, delta1, delta2); + LSX_DUP2_ARG2(__lsx_vsub_h, delta1, p1_src, delta2, q1_src, delta1, delta2); + delta1 = __lsx_vadd_h(delta1, delta0); + delta2 = __lsx_vsub_h(delta2, delta0); + LSX_DUP2_ARG2(__lsx_vsrai_h, delta1, 1, delta2, 1, delta1, delta2); + LSX_DUP2_ARG3(__lsx_clip_h, delta1, tc_neg, tc_pos, delta2, tc_neg, tc_pos, + delta1, delta2); + LSX_DUP2_ARG2(__lsx_vadd_h, p1_src, delta1, q1_src, delta2, delta1, delta2); + LSX_DUP2_ARG1(__lsx_clamp255_h, delta1, delta2, delta1, delta2); + LSX_DUP2_ARG3(__lsx_vbitsel_v, delta1, p1_src, p_is_pcm_vec, delta2, q1_src, + q_is_pcm_vec, delta1, delta2); + abs_delta0 = __lsx_vnor_v(abs_delta0, abs_delta0); + LSX_DUP4_ARG3(__lsx_vbitsel_v, delta1, p1_src, abs_delta0, delta2, q1_src, + abs_delta0, temp0, p0_src, abs_delta0, temp2, q0_src, + abs_delta0, delta1, delta2, temp0, temp2); + /* weak filter ends */ + + /* pack weak results to 8 bit */ + LSX_DUP2_ARG2(__lsx_vpickev_b, delta1, p2_src, temp2, temp0, dst3, dst4); + dst5 = __lsx_vpickev_b(q2_src, delta2); + + /* select between weak or strong */ + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst0, dst3, cmp2, dst1, dst4, cmp2, dst0, dst1); + dst2 = __lsx_vbitsel_v(dst2, dst5, cmp2); + + /* pack src to 8 bit */ + LSX_DUP2_ARG2(__lsx_vpickev_b, p1_src, p2_src, q0_src, p0_src, dst3, dst4); + dst5 = __lsx_vpickev_b(q2_src, q1_src); + + cmp3 = __lsx_vnor_v(cmp3, cmp3); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst0, dst3, cmp3, dst1, dst4, cmp3, dst0, dst1); + dst2 = __lsx_vbitsel_v(dst2, dst5, cmp3); + + __lsx_vstelm_d(dst0, p2, 0, 0); + __lsx_vstelm_d(dst0, p2 + stride, 0, 1); + __lsx_vstelm_d(dst1, p2 + stride_2x, 0, 0); + __lsx_vstelm_d(dst1, p2 + stride_3x, 0, 1); + __lsx_vstelm_d(dst2, p2 + stride_4x, 0, 0); + __lsx_vstelm_d(dst2, p2 + stride_4x + stride, 0, 1); + } + } +} + +static void hevc_loopfilter_luma_ver_lsx(uint8_t *src, int32_t stride, + int32_t beta, int32_t *tc, + uint8_t *p_is_pcm, uint8_t *q_is_pcm) +{ + const int32_t stride_2x = (stride << 1); + const int32_t stride_4x = (stride << 2); + const int32_t stride_3x = stride_2x + stride; + uint8_t *p3 = src; + uint8_t *p2 = src + stride_3x; + uint8_t *p1 = src + stride_4x; + uint8_t *p0 = src + stride_4x + stride_3x; + uint8_t flag0, flag1; + int32_t dp00, dq00, dp30, dq30, d00, d30; + int32_t d0030, d0434; + int32_t dp04, dq04, dp34, dq34, d04, d34; + int32_t tc0, p_is_pcm0, q_is_pcm0, beta30, beta20, tc250; + int32_t tc4, p_is_pcm4, q_is_pcm4, tc254, tmp; + + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i cmp0, cmp1, cmp2, p_is_pcm_vec, q_is_pcm_vec; + __m128i cmp3; + __m128i temp0, temp1; + __m128i temp2; + __m128i tc_pos, tc_neg; + __m128i diff0, diff1, delta0, delta1, delta2, abs_delta0; + __m128i zero = {0}; + __m128i p3_src, p2_src, p1_src, p0_src, q0_src, q1_src, q2_src, q3_src; + + dp00 = abs(p3[-3] - (p3[-2] << 1) + p3[-1]); + dq00 = abs(p3[2] - (p3[1] << 1) + p3[0]); + dp30 = abs(p2[-3] - (p2[-2] << 1) + p2[-1]); + dq30 = abs(p2[2] - (p2[1] << 1) + p2[0]); + d00 = dp00 + dq00; + d30 = dp30 + dq30; + p_is_pcm0 = p_is_pcm[0]; + q_is_pcm0 = q_is_pcm[0]; + + dp04 = abs(p1[-3] - (p1[-2] << 1) + p1[-1]); + dq04 = abs(p1[2] - (p1[1] << 1) + p1[0]); + dp34 = abs(p0[-3] - (p0[-2] << 1) + p0[-1]); + dq34 = abs(p0[2] - (p0[1] << 1) + p0[0]); + d04 = dp04 + dq04; + d34 = dp34 + dq34; + p_is_pcm4 = p_is_pcm[1]; + q_is_pcm4 = q_is_pcm[1]; + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, p_is_pcm0, p_is_pcm4, cmp0, cmp1); + p_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + p_is_pcm_vec = __lsx_vseqi_d(p_is_pcm_vec, 0); + + d0030 = (d00 + d30) >= beta; + d0434 = (d04 + d34) >= beta; + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, d0030, d0434, cmp0, cmp1); + cmp3 = __lsx_vpackev_d(cmp1, cmp0); + cmp3 = __lsx_vseqi_d(cmp3, 0); + + if ((!p_is_pcm0 || !p_is_pcm4 || !q_is_pcm0 || !q_is_pcm4) && + (!d0030 || !d0434)) { + src -= 4; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + stride, 0, src + stride_2x, 0, + src + stride_3x, 0, p3_src, p2_src, p1_src, p0_src); + src += stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + stride, 0, src + stride_2x, 0, + src + stride_3x, 0, q0_src, q1_src, q2_src, q3_src); + src -= stride_4x; + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, q_is_pcm0, q_is_pcm4, cmp0, cmp1); + q_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + q_is_pcm_vec = __lsx_vseqi_d(q_is_pcm_vec, 0); + + tc0 = tc[0]; + beta30 = beta >> 3; + beta20 = beta >> 2; + tc250 = (((tc0 << 2) + tc0 + 1) >> 1); + tc4 = tc[1]; + tc254 = (((tc4 << 2) + tc4 + 1) >> 1); + LSX_DUP2_ARG1( __lsx_vreplgr2vr_h, tc0 << 1, tc4 << 1, cmp0, cmp1); + tc_pos = __lsx_vpackev_d(cmp1, cmp0); + TRANSPOSE8x8_B(p3_src, p2_src, p1_src, p0_src, q0_src, q1_src, + q2_src, q3_src, p3_src, p2_src, p1_src, p0_src, + q0_src, q1_src, q2_src, q3_src); + + flag0 = abs(p3[-4] - p3[-1]) + abs(p3[3] - p3[0]) < beta30 && + abs(p3[-1] - p3[0]) < tc250; + flag0 = flag0 && (abs(p2[-4] - p2[-1]) + abs(p2[3] - p2[0]) < beta30 && + abs(p2[-1] - p2[0]) < tc250 && (d00 << 1) < beta20 && + (d30 << 1) < beta20); + cmp0 = __lsx_vreplgr2vr_d(flag0); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3_src, zero, p2_src, zero, p1_src, zero, + p0_src, p3_src, p2_src, p1_src, p0_src); + + flag1 = abs(p1[-4] - p1[-1]) + abs(p1[3] - p1[0]) < beta30 && + abs(p1[-1] - p1[0]) < tc254; + flag1 = flag1 && (abs(p0[-4] - p0[-1]) + abs(p0[3] - p0[0]) < beta30 && + abs(p0[-1] - p0[0]) < tc254 && (d04 << 1) < beta20 && + (d34 << 1) < beta20); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0_src, zero, q1_src, zero, q2_src, zero, + q3_src, q0_src, q1_src, q2_src, q3_src); + + cmp1 = __lsx_vreplgr2vr_d(flag1); + cmp2 = __lsx_vpackev_d(cmp1, cmp0); + cmp2 = __lsx_vseqi_d(cmp2, 0); + + if (flag0 && flag1) { /* strong only */ + /* strong filter */ + tc_neg = __lsx_vneg_h(tc_pos); + /* p part */ + LSX_DUP2_ARG2(__lsx_vadd_h, p1_src, p0_src, temp0, q0_src, temp0, temp0); + + temp1 = __lsx_vadd_h(p3_src, p2_src); + temp1 = __lsx_vslli_h(temp1, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p2_src, temp1, temp0, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, p2_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst0 = __lsx_vadd_h(temp2, p2_src); + + temp1 = __lsx_vadd_h(temp0, p2_src); + temp1 = __lsx_vsrari_h(temp1, 2); + temp2 = __lsx_vsub_h(temp1, p1_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst1 = __lsx_vadd_h(temp2, p1_src); + + temp1 = __lsx_vslli_h(temp0, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p2_src, temp1, q1_src, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, p0_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst2 = __lsx_vadd_h(temp2, p0_src); + + p_is_pcm_vec = __lsx_vnor_v(p_is_pcm_vec, p_is_pcm_vec); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst0, p2_src, p_is_pcm_vec, dst1, p1_src, + p_is_pcm_vec, dst0, dst1); + dst2 = __lsx_vbitsel_v(dst2, p0_src, p_is_pcm_vec); + + /* q part */ + LSX_DUP2_ARG2(__lsx_vadd_h, q1_src, p0_src, temp0, q0_src, temp0, temp0); + temp1 = __lsx_vadd_h(q3_src, q2_src); + temp1 = __lsx_vslli_h(temp1, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, q2_src, temp1, temp0, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, q2_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst5 = __lsx_vadd_h(temp2, q2_src); + + temp1 = __lsx_vadd_h(temp0, q2_src); + temp1 = __lsx_vsrari_h(temp1, 2); + temp2 = __lsx_vsub_h(temp1, q1_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst4 = __lsx_vadd_h(temp2, q1_src); + + temp1 = __lsx_vslli_h(temp0, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p1_src, temp1, q2_src, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, q0_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst3 = __lsx_vadd_h(temp2, q0_src); + + q_is_pcm_vec = __lsx_vnor_v(q_is_pcm_vec, q_is_pcm_vec); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst3, q0_src, q_is_pcm_vec, dst4, q1_src, + q_is_pcm_vec, dst3, dst4); + dst5 = __lsx_vbitsel_v(dst5, q2_src, q_is_pcm_vec); + /* strong filter ends */ + } else if (flag0 == flag1) { /* weak only */ + /* weak filter */ + tc_pos = __lsx_vsrai_h(tc_pos, 1); + tc_neg = __lsx_vneg_h(tc_pos); + + LSX_DUP2_ARG2(__lsx_vsub_h, q0_src, p0_src, q1_src, p1_src, diff0, diff1); + LSX_DUP2_ARG2(__lsx_vadd_h, __lsx_vslli_h(diff0, 3), diff0, __lsx_vslli_h( + diff1, 1), diff1, diff0, diff1); + delta0 = __lsx_vsub_h(diff0, diff1); + delta0 = __lsx_vsrari_h(delta0, 4); + temp1 = __lsx_vadd_h(__lsx_vslli_h(tc_pos, 3), __lsx_vslli_h(tc_pos, 1)); + abs_delta0 = __lsx_vadda_h(delta0, zero); + abs_delta0 = __lsx_vsle_hu(temp1, abs_delta0); + abs_delta0 = __lsx_vnor_v(abs_delta0, abs_delta0); + + delta0 = __lsx_clip_h(delta0, tc_neg, tc_pos); + temp2 = __lsx_vadd_h(delta0, p0_src); + temp2 = __lsx_clamp255_h(temp2); + p_is_pcm_vec = __lsx_vnor_v(p_is_pcm_vec, p_is_pcm_vec); + temp0 = __lsx_vbitsel_v(temp2, p0_src, p_is_pcm_vec); + + temp2 = __lsx_vsub_h(q0_src, delta0); + temp2 = __lsx_clamp255_h(temp2); + q_is_pcm_vec = __lsx_vnor_v(q_is_pcm_vec, q_is_pcm_vec); + temp2 = __lsx_vbitsel_v(temp2, q0_src, q_is_pcm_vec); + + tmp = ((beta + (beta >> 1)) >> 3); + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, !p_is_pcm0 && ((dp00 + dp30) < tmp), + !p_is_pcm4 && ((dp04 + dp34) < tmp), cmp0, cmp1); + p_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + p_is_pcm_vec = __lsx_vseqi_d(p_is_pcm_vec, 0); + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_h, (!q_is_pcm0) && (dq00 + dq30 < tmp), + (!q_is_pcm4) && (dq04 + dq34 < tmp), cmp0, cmp1); + q_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + q_is_pcm_vec = __lsx_vseqi_d(q_is_pcm_vec, 0); + tc_pos = __lsx_vsrai_h(tc_pos, 1); + tc_neg = __lsx_vneg_h(tc_pos); + + LSX_DUP2_ARG2(__lsx_vavgr_hu, p2_src, p0_src, q0_src, q2_src, delta1, delta2); + LSX_DUP2_ARG2(__lsx_vsub_h, delta1, p1_src, delta2, q1_src, delta1, delta2); + delta1 = __lsx_vadd_h(delta1, delta0); + delta2 = __lsx_vsub_h(delta2, delta0); + LSX_DUP2_ARG2(__lsx_vsrai_h, delta1, 1, delta2, 1, delta1, delta2); + LSX_DUP2_ARG3(__lsx_clip_h, delta1, tc_neg, tc_pos, delta2, tc_neg, tc_pos, + delta1, delta2); + LSX_DUP2_ARG2(__lsx_vadd_h, p1_src, delta1, q1_src, delta2, delta1, delta2); + LSX_DUP2_ARG1(__lsx_clamp255_h, delta1, delta2, delta1, delta2); + LSX_DUP2_ARG3(__lsx_vbitsel_v, delta1, p1_src, p_is_pcm_vec, delta2, q1_src, + q_is_pcm_vec, delta1, delta2); + + abs_delta0 = __lsx_vnor_v(abs_delta0, abs_delta0); + LSX_DUP4_ARG3(__lsx_vbitsel_v, delta1, p1_src, abs_delta0, temp0, p0_src, + abs_delta0, temp2, q0_src, abs_delta0, delta2, q1_src, + abs_delta0, dst0, dst1, dst2, dst3); + /* weak filter ends */ + + cmp3 = __lsx_vnor_v(cmp3, cmp3); + LSX_DUP4_ARG3(__lsx_vbitsel_v, dst0, p1_src, cmp3, dst1, p0_src, cmp3, dst2, + q0_src, cmp3, dst3, q1_src, cmp3, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst2, dst0, dst3, dst1, dst0, dst1); + + /* transpose */ + dst4 = __lsx_vilvl_b(dst1, dst0); + dst5 = __lsx_vilvh_b(dst1, dst0); + dst0 = __lsx_vilvl_h(dst5, dst4); + dst1 = __lsx_vilvh_h(dst5, dst4); + + src += 2; + __lsx_vstelm_w(dst0, src, 0, 0); + __lsx_vstelm_w(dst0, src + stride, 0, 1); + __lsx_vstelm_w(dst0, src + stride_2x, 0, 2); + __lsx_vstelm_w(dst0, src + stride_3x, 0, 3); + src += stride_4x; + __lsx_vstelm_w(dst1, src, 0, 0); + __lsx_vstelm_w(dst1, src + stride, 0, 1); + __lsx_vstelm_w(dst1, src + stride_2x, 0, 2); + __lsx_vstelm_w(dst1, src + stride_3x, 0, 3); + return; + } else { /* strong + weak */ + /* strong filter */ + tc_neg = __lsx_vneg_h(tc_pos); + + /* p part */ + LSX_DUP2_ARG2(__lsx_vadd_h, p1_src, p0_src, temp0, q0_src, temp0, temp0); + + temp1 = __lsx_vadd_h(p3_src, p2_src); + temp1 = __lsx_vslli_h(temp1, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p2_src, temp1, temp0, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, p2_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst0 = __lsx_vadd_h(temp2, p2_src); + + temp1 = __lsx_vadd_h(temp0, p2_src); + temp1 = __lsx_vsrari_h(temp1, 2); + temp2 = __lsx_vsub_h(temp1, p1_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst1 = __lsx_vadd_h(temp2, p1_src); + + temp1 = __lsx_vslli_h(temp0, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p2_src, temp1, q1_src, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, p0_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst2 = __lsx_vadd_h(temp2, p0_src); + + p_is_pcm_vec = __lsx_vnor_v(p_is_pcm_vec, p_is_pcm_vec); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst0, p2_src, p_is_pcm_vec, dst1, p1_src, + p_is_pcm_vec, dst0, dst1); + dst2 = __lsx_vbitsel_v(dst2, p0_src, p_is_pcm_vec); + + /* q part */ + LSX_DUP2_ARG2(__lsx_vadd_h, q1_src, p0_src, temp0, q0_src, temp0, temp0); + temp1 = __lsx_vadd_h(q3_src, q2_src); + temp1 = __lsx_vslli_h(temp1, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, q2_src, temp1, temp0, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, q2_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst5 = __lsx_vadd_h(temp2, q2_src); + + temp1 = __lsx_vadd_h(temp0, q2_src); + temp1 = __lsx_vsrari_h(temp1, 2); + temp2 = __lsx_vsub_h(temp1, q1_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst4 = __lsx_vadd_h(temp2, q1_src); + + temp1 = __lsx_vslli_h(temp0, 1); + LSX_DUP2_ARG2(__lsx_vadd_h, temp1, p1_src, temp1, q2_src, temp1, temp1); + temp1 = __lsx_vsrari_h(temp1, 3); + temp2 = __lsx_vsub_h(temp1, q0_src); + temp2 = __lsx_clip_h(temp2, tc_neg, tc_pos); + dst3 = __lsx_vadd_h(temp2, q0_src); + + q_is_pcm_vec = __lsx_vnor_v(q_is_pcm_vec, q_is_pcm_vec); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst3, q0_src, q_is_pcm_vec, dst4, q1_src, + q_is_pcm_vec, dst3, dst4); + dst5 = __lsx_vbitsel_v(dst5, q2_src, q_is_pcm_vec); + /* strong filter ends */ + + /* weak filter */ + tc_pos = __lsx_vsrai_h(tc_pos, 1); + tc_neg = __lsx_vneg_h(tc_pos); + + LSX_DUP2_ARG2(__lsx_vsub_h, q0_src, p0_src, q1_src, p1_src, diff0, diff1); + LSX_DUP2_ARG2(__lsx_vadd_h, __lsx_vslli_h(diff0, 3), diff0, __lsx_vslli_h( + diff1, 1), diff1, diff0, diff1); + delta0 = __lsx_vsub_h(diff0, diff1); + delta0 = __lsx_vsrari_h(delta0, 4); + + temp1 = __lsx_vadd_h(__lsx_vslli_h(tc_pos, 3), __lsx_vslli_h(tc_pos, 1)); + abs_delta0 = __lsx_vadda_h(delta0, zero); + abs_delta0 = __lsx_vsle_hu(temp1, abs_delta0); + abs_delta0 = __lsx_vnor_v(abs_delta0, abs_delta0); + delta0 = __lsx_clip_h(delta0, tc_neg, tc_pos); + temp2 = __lsx_vadd_h(delta0, p0_src); + temp2 = __lsx_clamp255_h(temp2); + temp0 = __lsx_vbitsel_v(temp2, p0_src, p_is_pcm_vec); + temp2 = __lsx_vsub_h(q0_src, delta0); + temp2 = __lsx_clamp255_h(temp2); + temp2 = __lsx_vbitsel_v(temp2, q0_src, q_is_pcm_vec); + + tmp = (beta + (beta >> 1)) >> 3; + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, !p_is_pcm0 && ((dp00 + dp30) < tmp), + !p_is_pcm4 && ((dp04 + dp34) < tmp), cmp0, cmp1); + p_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + p_is_pcm_vec = __lsx_vseqi_d(p_is_pcm_vec, 0); + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_h, (!q_is_pcm0) && (dq00 + dq30 < tmp), + (!q_is_pcm4) && (dq04 + dq34 < tmp), cmp0, cmp1); + q_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + q_is_pcm_vec = __lsx_vseqi_d(q_is_pcm_vec, 0); + tc_pos = __lsx_vsrai_h(tc_pos, 1); + tc_neg = __lsx_vneg_h(tc_pos); + + LSX_DUP2_ARG2(__lsx_vavgr_hu, p2_src, p0_src, q0_src, q2_src, delta1, delta2); + LSX_DUP2_ARG2(__lsx_vsub_h, delta1, p1_src, delta2, q1_src, delta1, delta2); + delta1 = __lsx_vadd_h(delta1, delta0); + delta2 = __lsx_vsub_h(delta2, delta0); + LSX_DUP2_ARG2(__lsx_vsrai_h, delta1, 1, delta2, 1, delta1, delta2); + LSX_DUP2_ARG3(__lsx_clip_h, delta1, tc_neg, tc_pos, delta2, tc_neg, tc_pos, + delta1, delta2); + LSX_DUP2_ARG2(__lsx_vadd_h, p1_src, delta1, q1_src, delta2, delta1, delta2); + LSX_DUP2_ARG1(__lsx_clamp255_h, delta1, delta2, delta1, delta2); + LSX_DUP2_ARG3(__lsx_vbitsel_v, delta1, p1_src, p_is_pcm_vec, delta2, q1_src, + q_is_pcm_vec, delta1, delta2); + + abs_delta0 = __lsx_vnor_v(abs_delta0, abs_delta0); + LSX_DUP4_ARG3(__lsx_vbitsel_v, delta1, p1_src, abs_delta0, delta2, q1_src, + abs_delta0, temp0, p0_src, abs_delta0, temp2, q0_src, + abs_delta0, delta1, delta2, temp0, temp2); + /* weak filter ends*/ + + /* select between weak or strong */ + LSX_DUP4_ARG3(__lsx_vbitsel_v, dst0, p2_src, cmp2, dst1, delta1, cmp2, dst2, + temp0, cmp2, dst3, temp2, cmp2, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst4, delta2, cmp2, dst5, q2_src, cmp2, dst4, + dst5); + } + + cmp3 = __lsx_vnor_v(cmp3, cmp3); + LSX_DUP4_ARG3(__lsx_vbitsel_v, dst0, p2_src, cmp3, dst1, p1_src, cmp3, dst2, + p0_src, cmp3, dst3, q0_src, cmp3, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vbitsel_v, dst4, q1_src, cmp3, dst5, q2_src, cmp3, dst4, dst5); + + /* pack results to 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, dst2, dst0, dst3, dst1, dst4, dst4, dst5, dst5, + dst0, dst1, dst2, dst3); + + /* transpose */ + LSX_DUP2_ARG2(__lsx_vilvl_b, dst1, dst0, dst3, dst2, dst4, dst6); + LSX_DUP2_ARG2(__lsx_vilvh_b, dst1, dst0, dst3, dst2, dst5, dst7); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst7, dst6, dst0, dst2); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst7, dst6, dst1, dst3); + + src += 1; + __lsx_vstelm_w(dst0, src, 0, 0); + __lsx_vstelm_h(dst2, src, 4, 0); + src += stride; + __lsx_vstelm_w(dst0, src, 0, 1); + __lsx_vstelm_h(dst2, src, 4, 2); + src += stride; + + __lsx_vstelm_w(dst0, src, 0, 2); + __lsx_vstelm_h(dst2, src, 4, 4); + src += stride; + __lsx_vstelm_w(dst0, src, 0, 3); + __lsx_vstelm_h(dst2, src, 4, 6); + src += stride; + + __lsx_vstelm_w(dst1, src, 0, 0); + __lsx_vstelm_h(dst3, src, 4, 0); + src += stride; + __lsx_vstelm_w(dst1, src, 0, 1); + __lsx_vstelm_h(dst3, src, 4, 2); + src += stride; + + __lsx_vstelm_w(dst1, src, 0, 2); + __lsx_vstelm_h(dst3, src, 4, 4); + src += stride; + __lsx_vstelm_w(dst1, src, 0, 3); + __lsx_vstelm_h(dst3, src, 4, 6); + } +} + +static void hevc_loopfilter_chroma_hor_lsx(uint8_t *src, int32_t stride, + int32_t *tc, uint8_t *p_is_pcm, + uint8_t *q_is_pcm) +{ + uint8_t *p1_ptr = src - (stride << 1); + uint8_t *p0_ptr = src - stride; + uint8_t *q0_ptr = src; + uint8_t *q1_ptr = src + stride; + __m128i cmp0, cmp1, p_is_pcm_vec, q_is_pcm_vec; + __m128i p1, p0, q0, q1; + __m128i tc_pos, tc_neg; + __m128i zero = {0}; + __m128i temp0, temp1, delta; + + if (!(tc[0] <= 0) || !(tc[1] <= 0)) { + LSX_DUP2_ARG1(__lsx_vreplgr2vr_h, tc[0], tc[1], cmp0, cmp1); + tc_pos = __lsx_vpackev_d(cmp1, cmp0); + tc_neg = __lsx_vneg_h(tc_pos); + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, p_is_pcm[0], p_is_pcm[1], cmp0, cmp1); + p_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + p_is_pcm_vec = __lsx_vseqi_d(p_is_pcm_vec, 0); + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, q_is_pcm[0], q_is_pcm[1], cmp0, cmp1); + q_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + q_is_pcm_vec = __lsx_vseqi_d(q_is_pcm_vec, 0); + + LSX_DUP4_ARG2(__lsx_vld, p1_ptr, 0, p0_ptr, 0, q0_ptr, 0, q1_ptr, 0, p1, + p0, q0, q1); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p1, zero, p0, zero, q0, zero, q1, p1, + p0, q0, q1); + LSX_DUP2_ARG2(__lsx_vsub_h, q0, p0, p1, q1, temp0, temp1); + temp0 = __lsx_vslli_h(temp0, 2); + temp0 = __lsx_vadd_h(temp0, temp1); + delta = __lsx_vsrari_h(temp0, 3); + delta = __lsx_clip_h(delta, tc_neg, tc_pos); + temp0 = __lsx_vadd_h(p0, delta); + temp0 = __lsx_clamp255_h(temp0); + p_is_pcm_vec = __lsx_vnor_v(p_is_pcm_vec, p_is_pcm_vec); + temp0 = __lsx_vbitsel_v(temp0, p0, p_is_pcm_vec); + + temp1 = __lsx_vsub_h(q0, delta); + temp1 = __lsx_clamp255_h(temp1); + q_is_pcm_vec = __lsx_vnor_v(q_is_pcm_vec, q_is_pcm_vec); + temp1 = __lsx_vbitsel_v(temp1, q0, q_is_pcm_vec); + + tc_pos = __lsx_vslei_d(tc_pos, 0); + LSX_DUP2_ARG3(__lsx_vbitsel_v, temp0, p0, tc_pos, temp1, q0, tc_pos, + temp0, temp1); + temp0 = __lsx_vpickev_b(temp1, temp0); + __lsx_vstelm_d(temp0, p0_ptr, 0, 0); + __lsx_vstelm_d(temp0, p0_ptr + stride, 0, 1); + } +} + +static void hevc_loopfilter_chroma_ver_lsx(uint8_t *src, int32_t stride, + int32_t *tc, uint8_t *p_is_pcm, + uint8_t *q_is_pcm) +{ + const int32_t stride_2x = (stride << 1); + const int32_t stride_4x = (stride << 2); + const int32_t stride_3x = stride_2x + stride; + __m128i cmp0, cmp1, p_is_pcm_vec, q_is_pcm_vec; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i p1, p0, q0, q1; + __m128i tc_pos, tc_neg; + __m128i zero = {0}; + __m128i temp0, temp1, delta; + + if (!(tc[0] <= 0) || !(tc[1] <= 0)) { + LSX_DUP2_ARG1(__lsx_vreplgr2vr_h, tc[0], tc[1], cmp0, cmp1); + tc_pos = __lsx_vpackev_d(cmp1, cmp0); + tc_neg = __lsx_vneg_h(tc_pos); + + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, p_is_pcm[0], p_is_pcm[1], cmp0, cmp1); + p_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + p_is_pcm_vec = __lsx_vseqi_d(p_is_pcm_vec, 0); + LSX_DUP2_ARG1(__lsx_vreplgr2vr_d, q_is_pcm[0], q_is_pcm[1], cmp0, cmp1); + q_is_pcm_vec = __lsx_vpackev_d(cmp1, cmp0); + q_is_pcm_vec = __lsx_vseqi_d(q_is_pcm_vec, 0); + + src -= 2; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + stride, 0, src + stride_2x, 0, + src + stride_3x, 0, src0, src1, src2, src3); + src += stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + stride, 0, src + stride_2x, 0, + src + stride_3x, 0, src4, src5, src6, src7); + src -= stride_4x; + TRANSPOSE8x4_B(src0, src1, src2, src3, src4, src5, src6, src7, + p1, p0, q0, q1); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p1, zero, p0, zero, q0, zero, q1, p1, p0, + q0, q1); + + LSX_DUP2_ARG2(__lsx_vsub_h, q0, p0, p1, q1, temp0, temp1); + temp0 = __lsx_vslli_h(temp0, 2); + temp0 = __lsx_vadd_h(temp0, temp1); + delta = __lsx_vsrari_h(temp0, 3); + delta = __lsx_clip_h(delta, tc_neg, tc_pos); + + temp0 = __lsx_vadd_h(p0, delta); + temp1 = __lsx_vsub_h(q0, delta); + LSX_DUP2_ARG1(__lsx_clamp255_h, temp0, temp1, temp0, temp1); + LSX_DUP2_ARG2(__lsx_vnor_v, p_is_pcm_vec, p_is_pcm_vec, q_is_pcm_vec, + q_is_pcm_vec, p_is_pcm_vec, q_is_pcm_vec); + LSX_DUP2_ARG3(__lsx_vbitsel_v, temp0, p0, p_is_pcm_vec, temp1, q0, + q_is_pcm_vec, temp0, temp1); + + tc_pos = __lsx_vslei_d(tc_pos, 0); + LSX_DUP2_ARG3(__lsx_vbitsel_v, temp0, p0, tc_pos, temp1, q0, tc_pos, + temp0, temp1); + temp0 = __lsx_vpackev_b(temp1, temp0); + + src += 1; + __lsx_vstelm_h(temp0, src, 0, 0); + __lsx_vstelm_h(temp0, src + stride, 0, 1); + __lsx_vstelm_h(temp0, src + stride_2x, 0, 2); + __lsx_vstelm_h(temp0, src + stride_3x, 0, 3); + src += stride_4x; + __lsx_vstelm_h(temp0, src, 0, 4); + __lsx_vstelm_h(temp0, src + stride, 0, 5); + __lsx_vstelm_h(temp0, src + stride_2x, 0, 6); + __lsx_vstelm_h(temp0, src + stride_3x, 0, 7); + src -= stride_4x; + } +} + +static void hevc_sao_edge_filter_0degree_4width_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t height) +{ + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + __m128i shuf1 = {0x807060504030201, 0x100F0E0D0C0B0A09}; + __m128i shuf2 = {0x908070605040302, 0x11100F0E0D0C0B0A}; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i cmp_minus10, cmp_minus11, diff_minus10, diff_minus11; + __m128i sao_offset = __lsx_vld(sao_offset_val, 0); + __m128i src_minus10, src_minus11, src_plus10, offset, src0, dst0; + __m128i const1 = __lsx_vldi(1); + __m128i zero = {0}; + + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + src -= 1; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src_minus10, src_minus11); + + for (height -= 2; height; height -= 2) { + src += src_stride_2x; + src_minus10 = __lsx_vpickev_d(src_minus11, src_minus10); + src0 = __lsx_vshuf_b(zero, src_minus10, shuf1); + src_plus10 = __lsx_vshuf_b(zero, src_minus10, shuf2); + + LSX_DUP2_ARG2(__lsx_vseq_b, src0, src_minus10, src0, src_plus10, cmp_minus10, + cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src0, src_minus10, src0, src_plus10, cmp_minus10, + cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + offset = __lsx_vadd_b(diff_minus10, diff_minus11); + offset = __lsx_vaddi_bu(offset, 2); + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src_minus10, src_minus11); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + src0 = __lsx_vxori_b(src0, 128); + dst0 = __lsx_vsadd_b(src0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 2); + dst += dst_stride_2x; + } + + src_minus10 = __lsx_vpickev_d(src_minus11, src_minus10); + src0 = __lsx_vshuf_b(zero, src_minus10, shuf1); + src_plus10 = __lsx_vshuf_b(zero, src_minus10, shuf2); + + LSX_DUP2_ARG2(__lsx_vseq_b, src0, src_minus10, src0, src_plus10, cmp_minus10, + cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src0, src_minus10, src0, src_plus10, cmp_minus10, + cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + offset = __lsx_vadd_b(diff_minus10, diff_minus11); + offset = __lsx_vaddi_bu(offset, 2); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + src0 = __lsx_vxori_b(src0, 128); + dst0 = __lsx_vsadd_b(src0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 2); +} + +static void hevc_sao_edge_filter_0degree_8width_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t height) +{ + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + __m128i shuf1 = {0x807060504030201, 0x100F0E0D0C0B0A09}; + __m128i shuf2 = {0x908070605040302, 0x11100F0E0D0C0B0A}; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i cmp_minus10, cmp_minus11, diff_minus10, diff_minus11; + __m128i src0, src1, dst0, src_minus10, src_minus11, src_plus10, src_plus11; + __m128i offset, sao_offset = __lsx_vld(sao_offset_val, 0); + __m128i zeros = {0}; + + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + src -= 1; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src_minus10, src_minus11); + + for (height -= 2; height; height -= 2) { + src += src_stride_2x; + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus10, shuf1, zeros, src_minus11, + shuf1, src0, src1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus10, shuf2, zeros, src_minus11, + shuf2, src_plus10, src_plus11); + LSX_DUP2_ARG2(__lsx_vpickev_d, src_minus11, src_minus10, src_plus11, src_plus10, + src_minus10, src_plus10); + src0 = __lsx_vpickev_d(src1, src0); + + LSX_DUP2_ARG2(__lsx_vseq_b, src0, src_minus10, src0, src_plus10, cmp_minus10, + cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src0, src_minus10, src0, src_plus10, cmp_minus10, + cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + offset = __lsx_vadd_b(diff_minus10, diff_minus11); + offset = __lsx_vaddi_bu(offset, 2); + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src_minus10, src_minus11); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + src0 = __lsx_vxori_b(src0, 128); + dst0 = __lsx_vsadd_b(src0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + __lsx_vstelm_d(dst0, dst, 0, 0); + __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1); + dst += dst_stride_2x; + } + + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus10, shuf1, zeros, src_minus11, shuf1, + src0, src1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus10, shuf2, zeros, src_minus11, shuf2, + src_plus10, src_plus11); + LSX_DUP2_ARG2(__lsx_vpickev_d, src_minus11, src_minus10, src_plus11, src_plus10, + src_minus10, src_plus10); + src0 = __lsx_vpickev_d(src1, src0); + + LSX_DUP2_ARG2(__lsx_vseq_b, src0, src_minus10, src0, src_plus10, cmp_minus10, + cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src0, src_minus10, src0, src_plus10, cmp_minus10, + cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + offset = __lsx_vadd_b(diff_minus10, diff_minus11); + offset = __lsx_vaddi_bu(offset, 2); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + src0 = __lsx_vxori_b(src0, 128); + dst0 = __lsx_vsadd_b(src0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + __lsx_vstelm_d(dst0, dst, 0, 0); + __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1); +} + +static void hevc_sao_edge_filter_0degree_16multiple_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t width, + int32_t height) +{ + uint8_t *dst_ptr, *src_minus1; + int32_t v_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i shuf1 = {0x807060504030201, 0x100F0E0D0C0B0A09}; + __m128i shuf2 = {0x908070605040302, 0x11100F0E0D0C0B0A}; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i sao_offset; + __m128i cmp_minus10, cmp_plus10, diff_minus10, diff_plus10, cmp_minus11; + __m128i cmp_plus11, diff_minus11, diff_plus11, cmp_minus12, cmp_plus12; + __m128i diff_minus12, diff_plus12, cmp_minus13, cmp_plus13, diff_minus13; + __m128i diff_plus13; + __m128i src10, src11, src12, src13, dst0, dst1, dst2, dst3; + __m128i src_minus10, src_minus11, src_minus12, src_minus13; + __m128i offset_mask0, offset_mask1, offset_mask2, offset_mask3; + __m128i src_zero0, src_zero1, src_zero2, src_zero3; + __m128i src_plus10, src_plus11, src_plus12, src_plus13; + + sao_offset = __lsx_vld(sao_offset_val, 0); + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + + for (; height; height -= 4) { + src_minus1 = src - 1; + LSX_DUP4_ARG2(__lsx_vld, src_minus1, 0, src_minus1 + src_stride, 0, + src_minus1 + src_stride_2x, 0, src_minus1 + src_stride_3x, 0, + src_minus10, src_minus11, src_minus12, src_minus13); + + for (v_cnt = 0; v_cnt < width; v_cnt += 16) { + src_minus1 += 16; + dst_ptr = dst + v_cnt; + LSX_DUP4_ARG2(__lsx_vld, src_minus1, 0, src_minus1 + src_stride, 0, + src_minus1 + src_stride_2x, 0, src_minus1 + src_stride_3x, 0, + src10, src11, src12, src13); + LSX_DUP4_ARG3(__lsx_vshuf_b, src10, src_minus10, shuf1, src11, src_minus11, + shuf1, src12, src_minus12, shuf1, src13, src_minus13, shuf1, + src_zero0, src_zero1, src_zero2, src_zero3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src10, src_minus10, shuf2, src11, src_minus11, + shuf2, src12, src_minus12, shuf2, src13, src_minus13, shuf2, + src_plus10, src_plus11, src_plus12, src_plus13); + + LSX_DUP4_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero0, src_plus10, + src_zero1, src_minus11, src_zero1, src_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vseq_b, src_zero2, src_minus12, src_zero2, src_plus12, + src_zero3, src_minus13, src_zero3, src_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_plus10, cmp_plus10, + cmp_minus11, cmp_minus11, cmp_plus11, cmp_plus11, diff_minus10, + diff_plus10, diff_minus11, diff_plus11); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus12, cmp_minus12, cmp_plus12, cmp_plus12, + cmp_minus13, cmp_minus13, cmp_plus13, cmp_plus13, diff_minus12, + diff_plus12, diff_minus13, diff_plus13); + LSX_DUP4_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero0, src_plus10, + src_zero1, src_minus11, src_zero1, src_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vsle_bu, src_zero2, src_minus12, src_zero2, src_plus12, + src_zero3, src_minus13, src_zero3, src_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_plus10, cmp_plus10, + cmp_minus11, cmp_minus11, cmp_plus11, cmp_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus12, cmp_minus12, cmp_plus12, cmp_plus12, + cmp_minus13, cmp_minus13, cmp_plus13, cmp_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, + diff_plus10, const1, cmp_plus10, diff_minus11, const1, + cmp_minus11, diff_plus11, const1, cmp_plus11, diff_minus10, + diff_plus10, diff_minus11, diff_plus11); + LSX_DUP4_ARG3(__lsx_vbitsel_v, diff_minus12, const1, cmp_minus12, + diff_plus12, const1, cmp_plus12, diff_minus13, const1, + cmp_minus13, diff_plus13, const1, cmp_plus13, diff_minus12, + diff_plus12, diff_minus13, diff_plus13); + + LSX_DUP4_ARG2(__lsx_vadd_b, diff_minus10, diff_plus10, diff_minus11, + diff_plus11, diff_minus12, diff_plus12, diff_minus13, + diff_plus13, offset_mask0, offset_mask1, offset_mask2, + offset_mask3); + LSX_DUP4_ARG2(__lsx_vaddi_bu, offset_mask0, 2, offset_mask1, 2, offset_mask2, + 2, offset_mask3, 2, offset_mask0, offset_mask1, offset_mask2, + offset_mask3); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask0, sao_offset, + sao_offset, offset_mask0, offset_mask0, offset_mask0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask1, sao_offset, + sao_offset, offset_mask1, offset_mask1, offset_mask1); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask2, sao_offset, + sao_offset, offset_mask2, offset_mask2, offset_mask2); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask3, sao_offset, + sao_offset, offset_mask3, offset_mask3, offset_mask3); + + LSX_DUP4_ARG2(__lsx_vxori_b, src_zero0, 128, src_zero1, 128, src_zero2, 128, + src_zero3, 128, src_zero0, src_zero1, src_zero2, src_zero3); + LSX_DUP4_ARG2(__lsx_vsadd_b, src_zero0, offset_mask0, src_zero1, + offset_mask1, src_zero2, offset_mask2, src_zero3, offset_mask3, + dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vxori_b, dst0, 128, dst1, 128, dst2, 128, dst3, 128, + dst0, dst1, dst2, dst3); + + src_minus10 = src10; + src_minus11 = src11; + src_minus12 = src12; + src_minus13 = src13; + + __lsx_vst(dst0, dst_ptr, 0); + __lsx_vst(dst1, dst_ptr + dst_stride, 0); + __lsx_vst(dst2, dst_ptr + dst_stride_2x, 0); + __lsx_vst(dst3, dst_ptr + dst_stride_3x, 0); + } + src += src_stride_4x; + dst += dst_stride_4x; + } +} + +static void hevc_sao_edge_filter_90degree_4width_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t height) +{ + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i dst0; + __m128i sao_offset = __lsx_vld(sao_offset_val, 0); + __m128i cmp_minus10, diff_minus10, cmp_minus11, diff_minus11; + __m128i src_minus10, src_minus11, src10, src11; + __m128i src_zero0, src_zero1; + __m128i offset; + __m128i offset_mask0, offset_mask1; + + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + + /* load in advance */ + LSX_DUP4_ARG2(__lsx_vld, src - src_stride, 0, src, 0, src + src_stride, 0, + src + src_stride_2x, 0, src_minus10, src_minus11, src10, src11); + + for (height -= 2; height; height -= 2) { + src += src_stride_2x; + LSX_DUP4_ARG2(__lsx_vilvl_b, src10, src_minus10, src_minus11, src_minus11, + src11, src_minus11, src10, src10, src_minus10, src_zero0, + src_minus11, src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + src_minus10 = src10; + src_minus11 = src11; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src + src_stride, 0, src + src_stride_2x, 0, + src10, src11); + + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 2); + dst += dst_stride_2x; + } + + LSX_DUP4_ARG2(__lsx_vilvl_b, src10, src_minus10, src_minus11, src_minus11, src11, + src_minus11, src10, src10, src_minus10, src_zero0, src_minus11, + src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_bu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 2); +} + +static void hevc_sao_edge_filter_90degree_8width_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t height) +{ + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i offset, sao_offset = __lsx_vld(sao_offset_val, 0); + __m128i src_zero0, src_zero1, dst0; + __m128i cmp_minus10, diff_minus10, cmp_minus11, diff_minus11; + __m128i src_minus10, src_minus11, src10, src11; + __m128i offset_mask0, offset_mask1; + + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src - src_stride, 0, src, 0, src_minus10, src_minus11); + LSX_DUP2_ARG2(__lsx_vld, src + src_stride, 0, src + src_stride_2x, 0, src10, src11); + + for (height -= 2; height; height -= 2) { + src += src_stride_2x; + LSX_DUP4_ARG2(__lsx_vilvl_b, src10, src_minus10, src_minus11, src_minus11, src11, + src_minus11, src10, src10, src_minus10, src_zero0, src_minus11, + src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + src_minus10 = src10; + src_minus11 = src11; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src + src_stride, 0, src + src_stride_2x, 0, + src10, src11); + + __lsx_vstelm_d(dst0, dst, 0, 0); + __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1); + dst += dst_stride_2x; + } + + LSX_DUP4_ARG2(__lsx_vilvl_b, src10, src_minus10, src_minus11, src_minus11, src11, + src_minus11, src10, src10, src_minus10, src_zero0, src_minus11, + src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + __lsx_vstelm_d(dst0, dst, 0, 0); + __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1); +} + +static void hevc_sao_edge_filter_90degree_16multiple_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t * + sao_offset_val, + int32_t width, + int32_t height) +{ + uint8_t *src_orig = src; + uint8_t *dst_orig = dst; + int32_t h_cnt, v_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i cmp_minus10, cmp_plus10, diff_minus10, diff_plus10, cmp_minus11; + __m128i cmp_plus11, diff_minus11, diff_plus11, cmp_minus12, cmp_plus12; + __m128i diff_minus12, diff_plus12, cmp_minus13, cmp_plus13, diff_minus13; + __m128i diff_plus13; + __m128i src10, src_minus10, dst0, src11, src_minus11, dst1; + __m128i src12, dst2, src13, dst3; + __m128i offset_mask0, offset_mask1, offset_mask2, offset_mask3, sao_offset; + + sao_offset = __lsx_vld(sao_offset_val, 0); + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + + for (v_cnt = 0; v_cnt < width; v_cnt += 16) { + src = src_orig + v_cnt; + dst = dst_orig + v_cnt; + + LSX_DUP2_ARG2(__lsx_vld, src - src_stride, 0, src, 0, src_minus10, src_minus11); + + for (h_cnt = (height >> 2); h_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src + src_stride_4x, 0, + src10, src11, src12, src13); + + LSX_DUP4_ARG2(__lsx_vseq_b, src_minus11, src_minus10, src_minus11, src10, + src10, src_minus11, src10, src11, cmp_minus10, cmp_plus10, + cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vseq_b, src11, src10, src11, src12, src12, src11, src12, + src13, cmp_minus12, cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_plus10, cmp_plus10, + cmp_minus11, cmp_minus11, cmp_plus11, cmp_plus11, diff_minus10, + diff_plus10, diff_minus11, diff_plus11); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus12, cmp_minus12, cmp_plus12, cmp_plus12, + cmp_minus13, cmp_minus13, cmp_plus13, cmp_plus13, diff_minus12, + diff_plus12, diff_minus13, diff_plus13); + LSX_DUP4_ARG2(__lsx_vsle_bu, src_minus11, src_minus10, src_minus11, src10, + src10, src_minus11, src10, src11, cmp_minus10, cmp_plus10, + cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vsle_bu, src11, src10, src11, src12, src12, src11, src12, + src13, cmp_minus12, cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_plus10, cmp_plus10, + cmp_minus11, cmp_minus11, cmp_plus11, cmp_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus12, cmp_minus12, cmp_plus12, cmp_plus12, + cmp_minus13, cmp_minus13, cmp_plus13, cmp_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, + diff_plus10, const1, cmp_plus10, diff_minus11, const1, + cmp_minus11, diff_plus11, const1, cmp_plus11, diff_minus10, + diff_plus10, diff_minus11, diff_plus11); + LSX_DUP4_ARG3(__lsx_vbitsel_v, diff_minus12, const1, cmp_minus12, + diff_plus12, const1, cmp_plus12, diff_minus13, const1, + cmp_minus13, diff_plus13, const1, cmp_plus13, diff_minus12, + diff_plus12, diff_minus13, diff_plus13); + + LSX_DUP4_ARG2(__lsx_vadd_b, diff_minus10, diff_plus10, diff_minus11, + diff_plus11, diff_minus12, diff_plus12, diff_minus13, + diff_plus13, offset_mask0, offset_mask1, offset_mask2, + offset_mask3); + LSX_DUP4_ARG2(__lsx_vaddi_bu, offset_mask0, 2, offset_mask1, 2, offset_mask2, + 2, offset_mask3, 2, offset_mask0, offset_mask1, offset_mask2, + offset_mask3); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask0, sao_offset, + sao_offset, offset_mask0, offset_mask0, offset_mask0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask1, sao_offset, + sao_offset, offset_mask1, offset_mask1, offset_mask1); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask2, sao_offset, + sao_offset, offset_mask2, offset_mask2, offset_mask2); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask3, sao_offset, + sao_offset, offset_mask3, offset_mask3, offset_mask3); + + src_minus10 = src12; + LSX_DUP4_ARG2(__lsx_vxori_b, src_minus11, 128, src10, 128, src11, 128, + src12, 128, src_minus11, src10, src11, src12); + LSX_DUP4_ARG2(__lsx_vsadd_b, src_minus11, offset_mask0, src10, offset_mask1, + src11, offset_mask2, src12, offset_mask3, dst0, dst1, dst2, + dst3); + LSX_DUP4_ARG2(__lsx_vxori_b, dst0, 128, dst1, 128, dst2, 128, dst3, 128, + dst0, dst1, dst2, dst3); + src_minus11 = src13; + + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst + dst_stride, 0); + __lsx_vst(dst2, dst + dst_stride_2x, 0); + __lsx_vst(dst3, dst + dst_stride_3x, 0); + src += src_stride_4x; + dst += dst_stride_4x; + } + } +} + +static void hevc_sao_edge_filter_45degree_4width_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t height) +{ + uint8_t *src_orig; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + __m128i shuf1 = {0x807060504030201, 0x100F0E0D0C0B0A09}; + __m128i shuf2 = {0x908070605040302, 0x11100F0E0D0C0B0A}; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i offset, sao_offset = __lsx_vld(sao_offset_val, 0); + __m128i cmp_minus10, diff_minus10, src_minus10, cmp_minus11, diff_minus11; + __m128i src_minus11, src10, src11; + __m128i src_plus0, src_zero0, src_plus1, src_zero1, dst0; + __m128i offset_mask0, offset_mask1; + __m128i zeros = {0}; + + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + src_orig = src - 1; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src_orig - src_stride, 0, src_orig, 0, src_minus10, + src_minus11); + LSX_DUP2_ARG2(__lsx_vld, src_orig + src_stride, 0, src_orig + src_stride_2x, 0, + src10, src11); + + for (height -= 2; height; height -= 2) { + src_orig += src_stride_2x; + + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus11, shuf1, zeros, src10, shuf1, + src_zero0, src_zero1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src10, shuf2, zeros, src11, shuf2, + src_plus0, src_plus1); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src_plus0, src_minus10, src_plus1, src_minus11, + src_minus10, src_minus11); + LSX_DUP2_ARG2(__lsx_vilvl_b, src_zero0, src_zero0, src_zero1, src_zero1, + src_zero0, src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + src_minus10 = src10; + src_minus11 = src11; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src_orig + src_stride, 0, src_orig + src_stride_2x, + 0, src10, src11); + + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 2); + dst += dst_stride_2x; + } + + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus11, shuf1, zeros, src10, shuf1, + src_zero0, src_zero1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src10, shuf2, zeros, src11, shuf2, src_plus0, + src_plus1); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src_plus0, src_minus10, src_plus1, src_minus11, + src_minus10, src_minus11); + LSX_DUP2_ARG2(__lsx_vilvl_b, src_zero0, src_zero0, src_zero1, src_zero1, src_zero0, + src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 2); +} + +static void hevc_sao_edge_filter_45degree_8width_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t height) +{ + uint8_t *src_orig; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + __m128i shuf1 = {0x807060504030201, 0x100F0E0D0C0B0A09}; + __m128i shuf2 = {0x908070605040302, 0x11100F0E0D0C0B0A}; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i offset, sao_offset = __lsx_vld(sao_offset_val, 0); + __m128i cmp_minus10, diff_minus10, cmp_minus11, diff_minus11; + __m128i src_minus10, src10, src_minus11, src11; + __m128i src_zero0, src_plus10, src_zero1, src_plus11, dst0; + __m128i offset_mask0, offset_mask1; + __m128i zeros = {0}; + + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + src_orig = src - 1; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src_orig - src_stride, 0, src_orig, 0, src_minus10, + src_minus11); + LSX_DUP2_ARG2(__lsx_vld, src_orig + src_stride, 0, src_orig + src_stride_2x, 0, + src10, src11); + + for (height -= 2; height; height -= 2) { + src_orig += src_stride_2x; + + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus11, shuf1, zeros, src10, shuf1, + src_zero0, src_zero1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src10, shuf2, zeros, src11, shuf2, + src_plus10, src_plus11); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src_plus10, src_minus10, src_plus11, src_minus11, + src_minus10, src_minus11); + LSX_DUP2_ARG2(__lsx_vilvl_b, src_zero0, src_zero0, src_zero1, src_zero1, + src_zero0, src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + src_minus10 = src10; + src_minus11 = src11; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src_orig + src_stride, 0, src_orig + src_stride_2x, 0, + src10, src11) + __lsx_vstelm_d(dst0, dst, 0, 0); + __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1); + dst += dst_stride_2x; + } + + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus11, shuf1, zeros, src10, shuf1, + src_zero0, src_zero1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src10, shuf2, zeros, src11, shuf2, src_plus10, + src_plus11); + LSX_DUP2_ARG2(__lsx_vilvl_b, src_plus10, src_minus10, src_plus11, src_minus11, + src_minus10, src_minus11); + LSX_DUP2_ARG2(__lsx_vilvl_b, src_zero0, src_zero0, src_zero1, src_zero1, src_zero0, + src_zero1); + + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + src_minus10 = src10; + src_minus11 = src11; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src_orig + src_stride, 0, src_orig + src_stride_2x, 0, + src10, src11); + + __lsx_vstelm_d(dst0, dst, 0, 0); + __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1); +} + +static void hevc_sao_edge_filter_45degree_16multiple_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t * + sao_offset_val, + int32_t width, + int32_t height) +{ + uint8_t *src_orig = src; + uint8_t *dst_orig = dst; + int32_t v_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i shuf1 = {0x807060504030201, 0x100F0E0D0C0B0A09}; + __m128i shuf2 = {0x908070605040302, 0x11100F0E0D0C0B0A}; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i cmp_minus10, cmp_plus10, diff_minus10, diff_plus10, cmp_minus11; + __m128i cmp_plus11, diff_minus11, diff_plus11, cmp_minus12, cmp_plus12; + __m128i diff_minus12, diff_plus12, cmp_minus13, cmp_plus13, diff_minus13; + __m128i diff_plus13, src_minus14, src_plus13; + __m128i offset_mask0, offset_mask1, offset_mask2, offset_mask3; + __m128i src10, src_minus10, dst0, src11, src_minus11, dst1; + __m128i src12, src_minus12, dst2, src13, src_minus13, dst3; + __m128i src_zero0, src_plus10, src_zero1, src_plus11, src_zero2, src_plus12; + __m128i src_zero3, sao_offset; + + sao_offset = __lsx_vld(sao_offset_val, 0); + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + + for (; height; height -= 4) { + src_orig = src - 1; + dst_orig = dst; + LSX_DUP4_ARG2(__lsx_vld, src_orig, 0, src_orig + src_stride, 0, + src_orig + src_stride_2x, 0, src_orig + src_stride_3x, 0, + src_minus11, src_minus12, src_minus13, src_minus14); + + for (v_cnt = 0; v_cnt < width; v_cnt += 16) { + src_minus10 = __lsx_vld(src_orig - src_stride, 0); + LSX_DUP4_ARG2(__lsx_vld, src_orig, 16, src_orig + src_stride, 16, + src_orig + src_stride_2x, 16, src_orig + src_stride_3x, 16, + src10, src11, src12, src13); + src_plus13 = __lsx_vld(src + v_cnt + src_stride_4x, 1); + src_orig += 16; + + LSX_DUP4_ARG3(__lsx_vshuf_b, src10, src_minus11, shuf1, src11, src_minus12, + shuf1, src12, src_minus13, shuf1, src13, src_minus14, shuf1, + src_zero0, src_zero1, src_zero2, src_zero3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src11, src_minus12, shuf2, src12, src_minus13, + shuf2, src_plus10, src_plus11); + src_plus12 = __lsx_vshuf_b(src13, src_minus14, shuf2); + + LSX_DUP4_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero0, src_plus10, + src_zero1, src_minus11, src_zero1, src_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vseq_b, src_zero2, src_minus12, src_zero2, src_plus12, + src_zero3, src_minus13, src_zero3, src_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_plus10, cmp_plus10, + cmp_minus11, cmp_minus11, cmp_plus11, cmp_plus11, diff_minus10, + diff_plus10, diff_minus11, diff_plus11); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus12, cmp_minus12, cmp_plus12, cmp_plus12, + cmp_minus13, cmp_minus13, cmp_plus13, cmp_plus13, diff_minus12, + diff_plus12, diff_minus13, diff_plus13); + LSX_DUP4_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero0, src_plus10, + src_zero1, src_minus11, src_zero1, src_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vsle_bu, src_zero2, src_minus12, src_zero2, src_plus12, + src_zero3, src_minus13, src_zero3, src_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_plus10, cmp_plus10, + cmp_minus11, cmp_minus11, cmp_plus11, cmp_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus12, cmp_minus12, cmp_plus12, cmp_plus12, + cmp_minus13, cmp_minus13, cmp_plus13, cmp_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, + diff_plus10, const1, cmp_plus10, diff_minus11, const1, + cmp_minus11, diff_plus11, const1, cmp_plus11, diff_minus10, + diff_plus10, diff_minus11, diff_plus11); + LSX_DUP4_ARG3(__lsx_vbitsel_v, diff_minus12, const1, cmp_minus12, + diff_plus12, const1, cmp_plus12, diff_minus13, const1, + cmp_minus13, diff_plus13, const1, cmp_plus13, diff_minus12, + diff_plus12, diff_minus13, diff_plus13); + + LSX_DUP4_ARG2(__lsx_vadd_b, diff_minus10, diff_plus10, diff_minus11, + diff_plus11, diff_minus12, diff_plus12, diff_minus13, + diff_plus13, offset_mask0, offset_mask1, offset_mask2, + offset_mask3); + LSX_DUP4_ARG2(__lsx_vaddi_bu, offset_mask0, 2, offset_mask1, 2, offset_mask2, + 2, offset_mask3, 2, offset_mask0, offset_mask1, offset_mask2, + offset_mask3); + + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask0, sao_offset, + sao_offset, offset_mask0, offset_mask0, offset_mask0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask1, sao_offset, + sao_offset, offset_mask1, offset_mask1, offset_mask1); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask2, sao_offset, + sao_offset, offset_mask2, offset_mask2, offset_mask2); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask3, sao_offset, + sao_offset, offset_mask3, offset_mask3, offset_mask3); + + LSX_DUP4_ARG2(__lsx_vxori_b, src_zero0, 128, src_zero1, 128, src_zero2, 128, + src_zero3, 128, src_zero0, src_zero1, src_zero2, src_zero3); + LSX_DUP4_ARG2(__lsx_vsadd_b, src_zero0, offset_mask0, src_zero1, + offset_mask1, src_zero2, offset_mask2, src_zero3, offset_mask3, + dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vxori_b, dst0, 128, dst1, 128, dst2, 128, dst3, 128, + dst0, dst1, dst2, dst3); + + src_minus11 = src10; + src_minus12 = src11; + src_minus13 = src12; + src_minus14 = src13; + + __lsx_vst(dst0, dst_orig, 0); + __lsx_vst(dst1, dst_orig + dst_stride, 0); + __lsx_vst(dst2, dst_orig + dst_stride_2x, 0); + __lsx_vst(dst3, dst_orig + dst_stride_3x, 0); + dst_orig += 16; + } + src += src_stride_4x; + dst += dst_stride_4x; + } +} + +static void hevc_sao_edge_filter_135degree_4width_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t height) +{ + uint8_t *src_orig; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + + __m128i shuf1 = {0x807060504030201, 0x100F0E0D0C0B0A09}; + __m128i shuf2 = {0x908070605040302, 0x11100F0E0D0C0B0A}; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i offset, sao_offset = __lsx_vld(sao_offset_val, 0); + __m128i src_zero0, src_zero1, dst0; + __m128i cmp_minus10, diff_minus10, cmp_minus11, diff_minus11; + __m128i src_minus10, src10, src_minus11, src11; + __m128i offset_mask0, offset_mask1; + __m128i zeros = {0}; + + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + src_orig = src - 1; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src_orig - src_stride, 0, src_orig, 0, src_minus10, + src_minus11); + LSX_DUP2_ARG2(__lsx_vld, src_orig + src_stride, 0, src_orig + src_stride_2x, 0, + src10, src11); + + for (height -= 2; height; height -= 2) { + src_orig += src_stride_2x; + + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus11, shuf1, zeros, src10, shuf1, + src_zero0, src_zero1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus10, shuf2, zeros, src_minus11, + shuf2, src_minus10, src_minus11); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src10, src_minus10, src11, src_minus11, src_minus10, + src_minus11); + LSX_DUP2_ARG2(__lsx_vilvl_b, src_zero0, src_zero0, src_zero1, src_zero1, + src_zero0, src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + src_minus10 = src10; + src_minus11 = src11; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src_orig + src_stride, 0, src_orig + src_stride_2x, 0, + src10, src11); + + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 2); + dst += dst_stride_2x; + } + + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus11, shuf1, zeros, src10, shuf1, + src_zero0, src_zero1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus10, shuf2, zeros, src_minus11, shuf2, + src_minus10, src_minus11); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src10, src_minus10, src11, src_minus11, src_minus10, + src_minus11); + LSX_DUP2_ARG2(__lsx_vilvl_b, src_zero0, src_zero0, src_zero1, src_zero1, src_zero0, + src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 2); + dst += dst_stride_2x; +} + +static void hevc_sao_edge_filter_135degree_8width_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t height) +{ + uint8_t *src_orig; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + + __m128i shuf1 = {0x807060504030201, 0x100F0E0D0C0B0A09}; + __m128i shuf2 = {0x908070605040302, 0x11100F0E0D0C0B0A}; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i offset, sao_offset = __lsx_vld(sao_offset_val, 0); + __m128i cmp_minus10, diff_minus10, cmp_minus11, diff_minus11; + __m128i src_minus10, src10, src_minus11, src11; + __m128i src_zero0, src_zero1, dst0; + __m128i offset_mask0, offset_mask1; + __m128i zeros = {0}; + + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + src_orig = src - 1; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src_orig - src_stride, 0, src_orig, 0, src_minus10, + src_minus11); + LSX_DUP2_ARG2(__lsx_vld, src_orig + src_stride, 0, src_orig + src_stride_2x, 0, + src10, src11); + + for (height -= 2; height; height -= 2) { + src_orig += src_stride_2x; + + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus11, shuf1, zeros, src10, shuf1, + src_zero0, src_zero1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus10, shuf2, zeros, src_minus11, + shuf2, src_minus10, src_minus11); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src10, src_minus10, src11, src_minus11, src_minus10, + src_minus11); + LSX_DUP2_ARG2(__lsx_vilvl_b, src_zero0, src_zero0, src_zero1, src_zero1, + src_zero0, src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + src_minus10 = src10; + src_minus11 = src11; + + /* load in advance */ + LSX_DUP2_ARG2(__lsx_vld, src_orig + src_stride, 0, src_orig + src_stride_2x, 0, + src10, src11); + + __lsx_vstelm_d(dst0, dst, 0, 0); + __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1); + dst += dst_stride_2x; + } + + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus11, shuf1, zeros, src10, shuf1, + src_zero0, src_zero1); + LSX_DUP2_ARG3(__lsx_vshuf_b, zeros, src_minus10, shuf2, zeros, src_minus11, shuf2, + src_minus10, src_minus11); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src10, src_minus10, src11, src_minus11, src_minus10, + src_minus11); + LSX_DUP2_ARG2(__lsx_vilvl_b, src_zero0, src_zero0, src_zero1, src_zero1, src_zero0, + src_zero1); + LSX_DUP2_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + diff_minus10, diff_minus11); + LSX_DUP2_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero1, src_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_minus11, cmp_minus11, + cmp_minus10, cmp_minus11); + LSX_DUP2_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, diff_minus11, + const1, cmp_minus11, diff_minus10, diff_minus11); + + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, diff_minus10, diff_minus10, diff_minus11, + diff_minus11, offset_mask0, offset_mask1); + LSX_DUP2_ARG2(__lsx_vaddi_hu, offset_mask0, 2, offset_mask1, 2, offset_mask0, + offset_mask1); + LSX_DUP2_ARG2(__lsx_vpickev_b, offset_mask1, offset_mask0, src_zero1, src_zero0, + offset, dst0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset, sao_offset, sao_offset, + offset, offset, offset); + dst0 = __lsx_vxori_b(dst0, 128); + dst0 = __lsx_vsadd_b(dst0, offset); + dst0 = __lsx_vxori_b(dst0, 128); + + __lsx_vstelm_d(dst0, dst, 0, 0); + __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1); +} + +static void hevc_sao_edge_filter_135degree_16multiple_lsx(uint8_t *dst, + int32_t dst_stride, + uint8_t *src, + int32_t src_stride, + int16_t *sao_offset_val, + int32_t width, + int32_t height) +{ + uint8_t *src_orig, *dst_orig; + int32_t v_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i shuf1 = {0x807060504030201, 0x100F0E0D0C0B0A09}; + __m128i shuf2 = {0x908070605040302, 0x11100F0E0D0C0B0A}; + __m128i edge_idx = {0x403000201, 0x0}; + __m128i const1 = __lsx_vldi(1); + __m128i dst0, dst1, dst2, dst3; + __m128i cmp_minus10, cmp_minus11, cmp_minus12, cmp_minus13, cmp_plus10; + __m128i cmp_plus11, cmp_plus12, cmp_plus13, diff_minus10, diff_minus11; + __m128i diff_minus12, diff_minus13, diff_plus10, diff_plus11, diff_plus12; + __m128i diff_plus13, src10, src11, src12, src13, src_minus10, src_minus11; + __m128i src_plus10, src_plus11, src_plus12, src_plus13; + __m128i src_minus12, src_minus13, src_zero0, src_zero1, src_zero2, src_zero3; + __m128i offset_mask0, offset_mask1, offset_mask2, offset_mask3, sao_offset; + + sao_offset = __lsx_vld(sao_offset_val, 0); + sao_offset = __lsx_vpickev_b(sao_offset, sao_offset); + + for (; height; height -= 4) { + src_orig = src - 1; + dst_orig = dst; + + LSX_DUP4_ARG2(__lsx_vld, src_orig, 0, src_orig + src_stride, 0, + src_orig + src_stride_2x, 0, src_orig + src_stride_3x, 0, + src_minus11, src_plus10, src_plus11, src_plus12); + + for (v_cnt = 0; v_cnt < width; v_cnt += 16) { + src_minus10 = __lsx_vld(src_orig - src_stride, 2); + LSX_DUP4_ARG2(__lsx_vld, src_orig, 16, src_orig + src_stride, 16, + src_orig + src_stride_2x, 16, src_orig + src_stride_3x, 16, + src10, src11, src12, src13); + src_plus13 = __lsx_vld(src_orig + src_stride_4x, 0); + src_orig += 16; + + LSX_DUP4_ARG3(__lsx_vshuf_b, src10, src_minus11, shuf1, src11, src_plus10, + shuf1, src12, src_plus11, shuf1, src13, src_plus12, shuf1, + src_zero0, src_zero1, src_zero2, src_zero3); + src_minus11 = __lsx_vshuf_b(src10, src_minus11, shuf2); + LSX_DUP2_ARG3(__lsx_vshuf_b, src11, src_plus10, shuf2, src12, src_plus11, + shuf2, src_minus12, src_minus13); + + LSX_DUP4_ARG2(__lsx_vseq_b, src_zero0, src_minus10, src_zero0, src_plus10, + src_zero1, src_minus11, src_zero1, src_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vseq_b, src_zero2, src_minus12, src_zero2, src_plus12, + src_zero3, src_minus13, src_zero3, src_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_plus10, cmp_plus10, + cmp_minus11, cmp_minus11, cmp_plus11, cmp_plus11, diff_minus10, + diff_plus10, diff_minus11, diff_plus11); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus12, cmp_minus12, cmp_plus12, cmp_plus12, + cmp_minus13, cmp_minus13, cmp_plus13, cmp_plus13, diff_minus12, + diff_plus12, diff_minus13, diff_plus13); + LSX_DUP4_ARG2(__lsx_vsle_bu, src_zero0, src_minus10, src_zero0, src_plus10, + src_zero1, src_minus11, src_zero1, src_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vsle_bu, src_zero2, src_minus12, src_zero2, src_plus12, + src_zero3, src_minus13, src_zero3, src_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus10, cmp_minus10, cmp_plus10, cmp_plus10, + cmp_minus11, cmp_minus11, cmp_plus11, cmp_plus11, cmp_minus10, + cmp_plus10, cmp_minus11, cmp_plus11); + LSX_DUP4_ARG2(__lsx_vnor_v, cmp_minus12, cmp_minus12, cmp_plus12, cmp_plus12, + cmp_minus13, cmp_minus13, cmp_plus13, cmp_plus13, cmp_minus12, + cmp_plus12, cmp_minus13, cmp_plus13); + LSX_DUP4_ARG3(__lsx_vbitsel_v, diff_minus10, const1, cmp_minus10, + diff_plus10, const1, cmp_plus10, diff_minus11, const1, + cmp_minus11, diff_plus11, const1, cmp_plus11, diff_minus10, + diff_plus10, diff_minus11, diff_plus11); + LSX_DUP4_ARG3(__lsx_vbitsel_v, diff_minus12, const1, cmp_minus12, + diff_plus12, const1, cmp_plus12, diff_minus13, const1, + cmp_minus13, diff_plus13, const1, cmp_plus13, diff_minus12, + diff_plus12, diff_minus13, diff_plus13); + + LSX_DUP4_ARG2(__lsx_vadd_b, diff_minus10, diff_plus10, diff_minus11, + diff_plus11, diff_minus12, diff_plus12, diff_minus13, + diff_plus13, offset_mask0, offset_mask1, offset_mask2, + offset_mask3); + LSX_DUP4_ARG2(__lsx_vaddi_bu, offset_mask0, 2, offset_mask1, 2, offset_mask2, + 2, offset_mask3, 2, offset_mask0, offset_mask1, offset_mask2, + offset_mask3); + + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask0, sao_offset, + sao_offset, offset_mask0, offset_mask0, offset_mask0); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask1, sao_offset, + sao_offset, offset_mask1, offset_mask1, offset_mask1); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask2, sao_offset, + sao_offset, offset_mask2, offset_mask2, offset_mask2); + LSX_DUP2_ARG3(__lsx_vshuf_b, edge_idx, edge_idx, offset_mask3, sao_offset, + sao_offset, offset_mask3, offset_mask3, offset_mask3); + + LSX_DUP4_ARG2(__lsx_vxori_b, src_zero0, 128, src_zero1, 128, src_zero2, 128, + src_zero3, 128, src_zero0, src_zero1, src_zero2, src_zero3); + LSX_DUP4_ARG2(__lsx_vsadd_b, src_zero0, offset_mask0, src_zero1, + offset_mask1, src_zero2, offset_mask2, src_zero3, offset_mask3, + dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vxori_b, dst0, 128, dst1, 128, dst2, 128, dst3, 128, + dst0, dst1, dst2, dst3); + + src_minus11 = src10; + src_plus10 = src11; + src_plus11 = src12; + src_plus12 = src13; + + __lsx_vst(dst0, dst_orig, 0); + __lsx_vst(dst1, dst_orig + dst_stride, 0); + __lsx_vst(dst2, dst_orig + dst_stride_2x, 0); + __lsx_vst(dst3, dst_orig + dst_stride_3x, 0); + dst_orig += 16; + } + + src += src_stride_4x; + dst += dst_stride_4x; + } +} + +void ff_hevc_loop_filter_luma_h_8_lsx(uint8_t *src, + ptrdiff_t src_stride, + int32_t beta, int32_t *tc, + uint8_t *no_p, uint8_t *no_q) +{ + hevc_loopfilter_luma_hor_lsx(src, src_stride, beta, tc, no_p, no_q); +} + +void ff_hevc_loop_filter_luma_v_8_lsx(uint8_t *src, + ptrdiff_t src_stride, + int32_t beta, int32_t *tc, + uint8_t *no_p, uint8_t *no_q) +{ + hevc_loopfilter_luma_ver_lsx(src, src_stride, beta, tc, no_p, no_q); +} + +void ff_hevc_loop_filter_chroma_h_8_lsx(uint8_t *src, + ptrdiff_t src_stride, + int32_t *tc, uint8_t *no_p, + uint8_t *no_q) +{ + hevc_loopfilter_chroma_hor_lsx(src, src_stride, tc, no_p, no_q); +} + +void ff_hevc_loop_filter_chroma_v_8_lsx(uint8_t *src, + ptrdiff_t src_stride, + int32_t *tc, uint8_t *no_p, + uint8_t *no_q) +{ + hevc_loopfilter_chroma_ver_lsx(src, src_stride, tc, no_p, no_q); +} + +void ff_hevc_sao_edge_filter_8_lsx(uint8_t *dst, uint8_t *src, + ptrdiff_t stride_dst, + int16_t *sao_offset_val, + int eo, int width, int height) +{ + ptrdiff_t stride_src = (2 * MAX_PB_SIZE + AV_INPUT_BUFFER_PADDING_SIZE) / sizeof(uint8_t); + + switch (eo) { + case 0: + if (width >> 4) { + hevc_sao_edge_filter_0degree_16multiple_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, + width - (width & 0x0F), + height); + dst += width - (width & 0x0F); + src += width - (width & 0x0F); + width &= 0x0F; + } + + if (width >> 3) { + hevc_sao_edge_filter_0degree_8width_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, height); + dst += 8; + src += 8; + width &= 0x07; + } + + if (width) { + hevc_sao_edge_filter_0degree_4width_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, height); + } + break; + + case 1: + if (width >> 4) { + hevc_sao_edge_filter_90degree_16multiple_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, + width - (width & 0x0F), + height); + dst += width - (width & 0x0F); + src += width - (width & 0x0F); + width &= 0x0F; + } + + if (width >> 3) { + hevc_sao_edge_filter_90degree_8width_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, height); + dst += 8; + src += 8; + width &= 0x07; + } + + if (width) { + hevc_sao_edge_filter_90degree_4width_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, height); + } + break; + + case 2: + if (width >> 4) { + hevc_sao_edge_filter_45degree_16multiple_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, + width - (width & 0x0F), + height); + dst += width - (width & 0x0F); + src += width - (width & 0x0F); + width &= 0x0F; + } + + if (width >> 3) { + hevc_sao_edge_filter_45degree_8width_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, height); + dst += 8; + src += 8; + width &= 0x07; + } + + if (width) { + hevc_sao_edge_filter_45degree_4width_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, height); + } + break; + + case 3: + if (width >> 4) { + hevc_sao_edge_filter_135degree_16multiple_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, + width - (width & 0x0F), + height); + dst += width - (width & 0x0F); + src += width - (width & 0x0F); + width &= 0x0F; + } + + if (width >> 3) { + hevc_sao_edge_filter_135degree_8width_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, height); + dst += 8; + src += 8; + width &= 0x07; + } + + if (width) { + hevc_sao_edge_filter_135degree_4width_lsx(dst, stride_dst, + src, stride_src, + sao_offset_val, height); + } + break; + } +} diff --git a/libavcodec/loongarch/hevc_macros_lsx.h b/libavcodec/loongarch/hevc_macros_lsx.h new file mode 100644 index 0000000000..580157a8cf --- /dev/null +++ b/libavcodec/loongarch/hevc_macros_lsx.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_HEVC_MACROS_LSX_H +#define AVCODEC_LOONGARCH_HEVC_MACROS_LSX_H + +static inline __m128i __lsx_hevc_filt_8tap_h(__m128i in0, __m128i in1, __m128i in2, + __m128i in3, __m128i filt0, __m128i filt1, + __m128i filt2, __m128i filt3) +{ + __m128i out_m; + + out_m = __lsx_dp2_h_b(in0, filt0); + out_m = __lsx_dp2add_h_b(out_m, in1, filt1); + out_m = __lsx_dp2add_h_b(out_m, in2, filt2); + out_m = __lsx_dp2add_h_b(out_m, in3, filt3); + return out_m; +} + +static inline __m128i __lsx_hevc_filt_8tap_w(__m128i in0, __m128i in1, __m128i in2, + __m128i in3, __m128i filt0, __m128i filt1, + __m128i filt2, __m128i filt3) +{ + __m128i out_m; + + out_m = __lsx_dp2_w_h(in0, filt0); + out_m = __lsx_dp2add_w_h(out_m, in1, filt1); + out_m = __lsx_dp2add_w_h(out_m, in2, filt2); + out_m = __lsx_dp2add_w_h(out_m, in3, filt3); + return out_m; +} + +static inline __m128i __lsx_hevc_filt_4tap_h(__m128i in0, __m128i in1, __m128i filt0, + __m128i filt1) +{ + __m128i out_m; + + out_m = __lsx_dp2_h_b(in0, filt0); + out_m = __lsx_dp2add_h_b(out_m, in1, filt1); + return out_m; +} + +static inline __m128i __lsx_hevc_filt_4tap_w(__m128i in0, __m128i in1, __m128i filt0, + __m128i filt1) +{ + __m128i out_m; + + out_m = __lsx_dp2_w_h(in0, filt0); + out_m = __lsx_dp2add_w_h(out_m, in1, filt1); + return out_m; +} + +#endif /* AVCODEC_LOONGARCH_HEVC_MACROS_LSX_H */ diff --git a/libavcodec/loongarch/hevc_mc_bi_lsx.c b/libavcodec/loongarch/hevc_mc_bi_lsx.c new file mode 100644 index 0000000000..22dfad90c1 --- /dev/null +++ b/libavcodec/loongarch/hevc_mc_bi_lsx.c @@ -0,0 +1,3075 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "hevcdsp_lsx.h" +#include "hevc_macros_lsx.h" + +static const uint8_t ff_hevc_mask_arr[16 * 2] __attribute__((aligned(0x40))) = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 +}; + +static av_always_inline +void __lsx_hevc_bi_rnd_clip2(__m128i in0, __m128i in1, __m128i vec0, __m128i vec1, + __m128i *out0, __m128i *out1) +{ + *out0 = __lsx_vsadd_h(vec0, in0); + *out1 = __lsx_vsadd_h(vec1, in1); + *out0 = __lsx_vsrari_h(*out0, 7); + *out1 = __lsx_vsrari_h(*out1, 7); + *out0 = __lsx_clamp255_h(*out0); + *out1 = __lsx_clamp255_h(*out1); +} + +static av_always_inline +void __lsx_hevc_bi_rnd_clip4(__m128i in0, __m128i in1, __m128i in2, __m128i in3, + __m128i vec0, __m128i vec1, __m128i vec2, + __m128i vec3, __m128i *out0, __m128i *out1, + __m128i *out2, __m128i *out3) +{ + __lsx_hevc_bi_rnd_clip2(in0, in1, vec0, vec1, out0, out1); + __lsx_hevc_bi_rnd_clip2(in2, in3, vec2, vec3, out2, out3); +} + +static void hevc_bi_copy_4w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt, tp0, tp1, tp2, tp3; + uint64_t tpd0, tpd1, tpd2, tpd3; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i src0, src1; + __m128i zero = {0}; + __m128i in0, in1, in2, in3; + __m128i dst0, dst1, dst2, dst3; + + if (2 == height) { + tp0 = *(uint32_t *)src0_ptr; + tp1 = *(uint32_t *)(src0_ptr + src_stride); + LSX_DUP2_ARG3(__lsx_vinsgr2vr_w, src0, tp0, 0, src0, tp1, 1, src0, src0); + tpd0 = *(uint64_t *)src1_ptr; + tpd1 = *(uint64_t *)(src1_ptr + src2_stride); + LSX_DUP2_ARG3(__lsx_vinsgr2vr_d, in0, tpd0, 0, in0, tpd1, 1, in0, in0); + + dst0 = __lsx_vilvl_b(zero, src0); + dst0 = __lsx_vslli_h(dst0, 6); + dst0 = __lsx_vadd_h(dst0, in0); + dst0 = __lsx_vmaxi_h(dst0, 0); + dst0 = __lsx_vssrlrni_bu_h(dst0, dst0, 7); + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 1); + } else if (4 == height) { + tp0 = *(uint32_t *)src0_ptr; + tp1 = *(uint32_t *)(src0_ptr + src_stride); + tp2 = *(uint32_t *)(src0_ptr + src_stride_2x); + tp3 = *(uint32_t *)(src0_ptr + src_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_w, src0, tp0, 0, src0, tp1, 1, src0, tp2, 2, src0, + tp3, 3, src0, src0, src0, src0); + tpd0 = *(uint64_t *)src1_ptr; + tpd1 = *(uint64_t *)(src1_ptr + src2_stride); + tpd2 = *(uint64_t *)(src1_ptr + src2_stride_2x); + tpd3 = *(uint64_t *)(src1_ptr + src2_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_d, in0, tpd0, 0, in0, tpd1, 1, in1, tpd2, 0, in1, + tpd3, 1, in0, in0, in1, in1); + dst0 = __lsx_vilvl_b(zero, src0); + dst1 = __lsx_vilvh_b(zero, src0); + LSX_DUP2_ARG2(__lsx_vslli_h, dst0, 6, dst1, 6, dst0, dst1); + __lsx_hevc_bi_rnd_clip2(in0, in1, dst0, dst1, &dst0, &dst1); + dst0 = __lsx_vpickev_b(dst1, dst0); + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(dst0, dst + dst_stride_2x, 0, 2); + __lsx_vstelm_w(dst0, dst + dst_stride_3x, 0, 3); + } else if (0 == (height & 0x07)) { + for (loop_cnt = (height >> 3); loop_cnt--;) { + tp0 = *(uint32_t *)src0_ptr; + tp1 = *(uint32_t *)(src0_ptr + src_stride); + tp2 = *(uint32_t *)(src0_ptr + src_stride_2x); + tp3 = *(uint32_t *)(src0_ptr + src_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_w, src0, tp0, 0, src0, tp1, 1, src0, tp2, 2, + src0, tp3, 3, src0, src0, src0, src0); + src0_ptr += src_stride_4x; + tp0 = *(uint32_t *)src0_ptr; + tp1 = *(uint32_t *)(src0_ptr + src_stride); + tp2 = *(uint32_t *)(src0_ptr + src_stride_2x); + tp3 = *(uint32_t *)(src0_ptr + src_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_w, src1, tp0, 0, src1, tp1, 1, src1, tp2, 2, + src1, tp3, 3, src1, src1, src1, src1); + src0_ptr += src_stride_4x; + tpd0 = *(uint64_t *)src1_ptr; + tpd1 = *(uint64_t *)(src1_ptr + src2_stride); + tpd2 = *(uint64_t *)(src1_ptr + src2_stride_2x); + tpd3 = *(uint64_t *)(src1_ptr + src2_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_d, in0, tpd0, 0, in0, tpd1, 1, in1, tpd2, 0, + in1, tpd3, 1, in0, in0, in1, in1); + src1_ptr += src2_stride_4x; + tpd0 = *(uint64_t *)src1_ptr; + tpd1 = *(uint64_t *)(src1_ptr + src2_stride); + tpd2 = *(uint64_t *)(src1_ptr + src2_stride_2x); + tpd3 = *(uint64_t *)(src1_ptr + src2_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_d, in2, tpd0, 0, in2, tpd1, 1, in3, tpd2, 0, + in3, tpd3, 1, in2, in2, in3, in3); + src1_ptr += src2_stride_4x; + LSX_DUP2_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, dst0, dst2); + LSX_DUP2_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, dst1, dst3); + LSX_DUP2_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst1, dst3); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1); + __lsx_vstelm_w(dst0, dst, 0, 0); + __lsx_vstelm_w(dst0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(dst0, dst + dst_stride_2x, 0, 2); + __lsx_vstelm_w(dst0, dst + dst_stride_3x, 0, 3); + dst += dst_stride_4x; + __lsx_vstelm_w(dst1, dst, 0, 0); + __lsx_vstelm_w(dst1, dst + dst_stride, 0, 1); + __lsx_vstelm_w(dst1, dst + dst_stride_2x, 0, 2); + __lsx_vstelm_w(dst1, dst + dst_stride_3x, 0, 3); + dst += dst_stride_4x; + } + } +} + +static void hevc_bi_copy_6w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + uint64_t tp0, tp1, tp2, tp3; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i out0, out1, out2, out3; + __m128i zero = {0}; + __m128i src0, src1, src2, src3; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + tp0 = *(uint64_t *)src0_ptr; + tp1 = *(uint64_t *)(src0_ptr + src_stride); + tp2 = *(uint64_t *)(src0_ptr + src_stride_2x); + tp3 = *(uint64_t *)(src0_ptr + src_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_d, src0, tp0, 0, src0, tp1, 1, src1, tp2, 0, src1, + tp3, 1, src0, src0, src1, src1); + src0_ptr += src_stride_4x; + tp0 = *(uint64_t *)src0_ptr; + tp1 = *(uint64_t *)(src0_ptr + src_stride); + tp2 = *(uint64_t *)(src0_ptr + src_stride_2x); + tp3 = *(uint64_t *)(src0_ptr + src_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_d, src2, tp0, 0, src2, tp1, 1, src3, tp2, 0, src3, + tp3, 1, src2, src2, src3, src3); + src0_ptr += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in1, in2, in3); + src1_ptr += src2_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in4, in5, in6, in7); + src1_ptr += src2_stride_4x; + LSX_DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6, + dst0, dst2, dst4, dst6); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + dst1, dst3, dst5, dst7); + LSX_DUP4_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst5, 6, dst7, 6, dst1, dst3, + dst5, dst7); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + __lsx_hevc_bi_rnd_clip4(in4, in5, in6, in7, dst4, dst5, dst6, dst7, + &dst4, &dst5, &dst6, &dst7); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, out0, out1); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst5, dst4, dst7, dst6, out2, out3); + __lsx_vstelm_w(out0, dst, 0, 0); + __lsx_vstelm_w(out0, dst + dst_stride, 0, 2); + __lsx_vstelm_h(out0, dst, 4, 2); + __lsx_vstelm_h(out0, dst + dst_stride, 4, 6); + __lsx_vstelm_w(out1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_w(out1, dst + dst_stride_3x, 0, 2); + __lsx_vstelm_h(out1, dst + dst_stride_2x, 4, 2); + __lsx_vstelm_h(out1, dst + dst_stride_3x, 4, 6); + dst += dst_stride_4x; + __lsx_vstelm_w(out2, dst, 0, 0); + __lsx_vstelm_w(out2, dst + dst_stride, 0, 2); + __lsx_vstelm_h(out2, dst, 4, 2); + __lsx_vstelm_h(out2, dst + dst_stride, 4, 6); + __lsx_vstelm_w(out3, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_w(out3, dst + dst_stride_3x, 0, 2); + __lsx_vstelm_h(out3, dst + dst_stride_2x, 4, 2); + __lsx_vstelm_h(out3, dst + dst_stride_3x, 4, 6); + dst += dst_stride_4x; + } +} + +static void hevc_bi_copy_8w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + int32_t height) +{ + uint64_t tp0, tp1, tp2, tp3; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i out0, out1, out2, out3; + __m128i src0, src1, src2, src3; + __m128i zero = {0}; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + + if (2 == height) { + tp0 = *(uint64_t *)src0_ptr; + tp1 = *(uint64_t *)(src0_ptr + src_stride); + LSX_DUP2_ARG3(__lsx_vinsgr2vr_d, src0, tp0, 0, src0, tp1, 1, src0, src0); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, in0, in1); + dst0 = __lsx_vsllwil_hu_bu(src0, 6); + dst1 = __lsx_vilvh_b(zero, src0); + dst1 = __lsx_vslli_h(dst1, 6); + __lsx_hevc_bi_rnd_clip2(in0, in1, dst0, dst1, &dst0, &dst1); + out0 = __lsx_vpickev_b(dst1, dst0); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + } else if (4 == height) { + tp0 = *(uint64_t *)src0_ptr; + tp1 = *(uint64_t *)(src0_ptr + src_stride); + tp2 = *(uint64_t *)(src0_ptr + src_stride_2x); + tp3 = *(uint64_t *)(src0_ptr + src_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_d, src0, tp0, 0, src0, tp1, 1, src1, tp2, 0, src1, + tp3, 1, src0, src0, src1, src1); + LSX_DUP2_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, dst0, dst2); + LSX_DUP2_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, dst1, dst3); + LSX_DUP2_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst1, dst3); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, in0, + in1, in2, in3); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_3x, 0, 1); + } else if (6 == height) { + tp0 = *(uint64_t *)src0_ptr; + tp1 = *(uint64_t *)(src0_ptr + src_stride); + tp2 = *(uint64_t *)(src0_ptr + src_stride_2x); + tp3 = *(uint64_t *)(src0_ptr + src_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_d, src0, tp0, 0, src0, tp1, 1, src1, tp2, 0, src1, + tp3, 1, src0, src0, src1, src1); + src0_ptr += src_stride_4x; + tp0 = *(uint64_t *)src0_ptr; + tp1 = *(uint64_t *)(src0_ptr + src_stride); + LSX_DUP2_ARG3(__lsx_vinsgr2vr_d, src2, tp0, 0, src2, tp1, 1, src2, src2); + + LSX_DUP2_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, dst0, dst2); + LSX_DUP2_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, dst1, dst3); + LSX_DUP2_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst1, dst3); + dst4 = __lsx_vsllwil_hu_bu(src2, 6); + dst5 = __lsx_vilvh_b(zero, src2); + dst5 = __lsx_vslli_h(dst5, 6); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in1, in2, in3); + src1_ptr += src2_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, in4, in5); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + __lsx_hevc_bi_rnd_clip2(in4, in5, dst4, dst5, &dst4, &dst5); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, out0, out1); + out2 = __lsx_vpickev_b(dst5, dst4); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_3x, 0, 1); + __lsx_vstelm_d(out2, dst + dst_stride_4x, 0, 0); + __lsx_vstelm_d(out2, dst + dst_stride_4x + dst_stride, 0, 1); + } else if (0 == (height & 0x07)) { + uint32_t loop_cnt; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + tp0 = *(uint64_t *)src0_ptr; + tp1 = *(uint64_t *)(src0_ptr + src_stride); + tp2 = *(uint64_t *)(src0_ptr + src_stride_2x); + tp3 = *(uint64_t *)(src0_ptr + src_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_d, src0, tp0, 0, src0, tp1, 1, src1, tp2, 0, + src1, tp3, 1, src0, src0, src1, src1); + src0_ptr += src_stride_4x; + tp0 = *(uint64_t *)src0_ptr; + tp1 = *(uint64_t *)(src0_ptr + src_stride); + tp2 = *(uint64_t *)(src0_ptr + src_stride_2x); + tp3 = *(uint64_t *)(src0_ptr + src_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_d, src2, tp0, 0, src2, tp1, 1, src3, tp2, 0, + src3, tp3, 1, src2, src2, src3, src3); + src0_ptr += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6, dst0, + dst2, dst4, dst6); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + dst1, dst3, dst5, dst7); + LSX_DUP4_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst5, 6, dst7, 6, dst1, + dst3, dst5, dst7); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in1, in2, in3); + src1_ptr += src2_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in4, in5, in6, in7); + src1_ptr += src2_stride_4x; + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, + dst3, &dst0, &dst1, &dst2, &dst3); + __lsx_hevc_bi_rnd_clip4(in4, in5, in6, in7, dst4, dst5, dst6, + dst7, &dst4, &dst5, &dst6, &dst7); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, out0, out1); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst5, dst4, dst7, dst6, out2, out3); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + __lsx_vstelm_d(out2, dst, 0, 0); + __lsx_vstelm_d(out2, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out3, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out3, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + } + } +} + +static void hevc_bi_copy_12w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i out0, out1, out2; + __m128i src0, src1, src2, src3; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5; + + for (loop_cnt = 4; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, + src0, src1, src2, src3); + src0_ptr += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, + src1_ptr + src2_stride_2x, 16, src1_ptr + src2_stride_3x, 16, + in4, in5, in6, in7); + + src1_ptr += src2_stride_4x; + LSX_DUP2_ARG2(__lsx_vilvl_d, in5, in4, in7, in6, in4, in5); + LSX_DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6, + dst0, dst1, dst2, dst3) + LSX_DUP2_ARG2(__lsx_vilvh_w, src1, src0, src3, src2, src0, src1); + LSX_DUP2_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, dst4, dst5) + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + __lsx_hevc_bi_rnd_clip2(in4, in5, dst4, dst5, &dst4, &dst5); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, out0, out1); + out2 = __lsx_vpickev_b(dst5, dst4); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_3x, 0, 1); + __lsx_vstelm_w(out2, dst, 8, 0); + __lsx_vstelm_w(out2, dst + dst_stride, 8, 1); + __lsx_vstelm_w(out2, dst + dst_stride_2x, 8, 2); + __lsx_vstelm_w(out2, dst + dst_stride_3x, 8, 3); + dst += dst_stride_4x; + } +} + +static void hevc_bi_copy_16w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i out0, out1, out2, out3; + __m128i src0, src1, src2, src3; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l; + __m128i zero = {0}; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, + src0, src1, src2, src3); + src0_ptr += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, + src1_ptr + src2_stride_2x, 16, src1_ptr + src2_stride_3x, 16, + in4, in5, in6, in7); + src1_ptr += src2_stride_4x; + LSX_DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6, + dst0_r, dst1_r, dst2_r, dst3_r) + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + dst0_l, dst1_l, dst2_l, dst3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, dst0_l, 6, dst1_l, 6, dst2_l, 6, dst3_l, 6, dst0_l, + dst1_l, dst2_l, dst3_l); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in4, in5, dst0_r, dst1_r, dst0_l, + dst1_l, &dst0_r, &dst1_r, &dst0_l, &dst1_l); + __lsx_hevc_bi_rnd_clip4(in2, in3, in6, in7, dst2_r, dst3_r, dst2_l, + dst3_l, &dst2_r, &dst3_r, &dst2_l, &dst3_l); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst0_l, dst0_r, dst1_l, dst1_r, out0, out1); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst2_l, dst2_r, dst3_l, dst3_r, out2, out3); + __lsx_vst(out0, dst, 0); + __lsx_vstx(out1, dst, dst_stride); + __lsx_vstx(out2, dst, dst_stride_2x); + __lsx_vstx(out3, dst, dst_stride_3x); + dst += dst_stride_4x; + } +} + +static void hevc_bi_copy_24w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i out0, out1, out2, out3, out4, out5; + __m128i zero = {0}; + __m128i in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9, dst10, dst11; + + for (loop_cnt = 8; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, + in0, in1, in4, in5); + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 16, src0_ptr + src_stride, 16, + src0_ptr + src_stride_2x, 16, src0_ptr + src_stride_3x, 16, + in2, in3, in6, in7); + LSX_DUP4_ARG2(__lsx_vsllwil_hu_bu, in0, 6, in1, 6, in2, 6, in3, 6, + dst0, dst2, dst4, dst5) + + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, in0, zero, in1, zero, in4, zero, in5, + dst1, dst3, dst7, dst9); + LSX_DUP4_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst7, 6, dst9, 6, + dst1, dst3, dst7, dst9); + LSX_DUP4_ARG2(__lsx_vsllwil_hu_bu, in4, 6, in5, 6, in6, 6, in7, 6, + dst6, dst8, dst10, dst11) + + src0_ptr += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, + src1_ptr + src2_stride_2x, 16, src1_ptr + src2_stride_3x, 16, + in4, in5, in6, in7); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 32, src1_ptr + src2_stride, 32, + src1_ptr + src2_stride_2x, 32, src1_ptr + src2_stride_3x, 32, + in8, in9, in10, in11); + src1_ptr += src2_stride_4x; + + __lsx_hevc_bi_rnd_clip4(in0, in4, in1, in5, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + __lsx_hevc_bi_rnd_clip4(in8, in9, in2, in6, dst4, dst5, dst6, dst7, + &dst4, &dst5, &dst6, &dst7); + __lsx_hevc_bi_rnd_clip4(in3, in7, in10, in11, dst8, dst9, dst10, dst11, + &dst8, &dst9, &dst10, &dst11); + LSX_DUP4_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + out0, out1, out2, out3); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst9, dst8, dst11, dst10, out4, out5); + __lsx_vst(out0, dst, 0); + __lsx_vstelm_d(out2, dst, 16, 0); + __lsx_vstx(out1, dst, dst_stride); + __lsx_vstelm_d(out2, dst + dst_stride, 16, 1); + __lsx_vstx(out3, dst, dst_stride_2x); + __lsx_vstelm_d(out5, dst + dst_stride_2x, 16, 0); + __lsx_vstx(out4, dst, dst_stride_3x); + __lsx_vstelm_d(out5, dst + dst_stride_3x, 16, 1); + dst += dst_stride_4x; + } +} + +static void hevc_bi_copy_32w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + __m128i out0, out1, out2, out3; + __m128i src0, src1, src2, src3; + __m128i zero = {0}; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src0, src1); + src0_ptr += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src2, src3); + src0_ptr += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32, src1_ptr, 48, + in0, in1, in2, in3); + src1_ptr += src2_stride; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32, src1_ptr, 48, + in4, in5, in6, in7); + src1_ptr += src2_stride; + + LSX_DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6, + dst0, dst2, dst4, dst6) + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + dst1, dst3, dst5, dst7); + LSX_DUP4_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst5, 6, dst7, 6, dst1, dst3, + dst5, dst7); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + __lsx_hevc_bi_rnd_clip4(in4, in5, in6, in7, dst4, dst5, dst6, dst7, + &dst4, &dst5, &dst6, &dst7); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, out0, out1); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst5, dst4, dst7, dst6, out2, out3); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + dst += dst_stride; + __lsx_vst(out2, dst, 0); + __lsx_vst(out3, dst, 16); + dst += dst_stride; + } +} + +static void hevc_bi_copy_48w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + __m128i out0, out1, out2, out3, out4, out5; + __m128i src0, src1, src2, src3, src4, src5; + __m128i zero = {0}; + __m128i in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9, dst10, dst11; + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src0, src1); + src2 = __lsx_vld(src0_ptr, 32); + src0_ptr += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src3, src4); + src5 = __lsx_vld(src0_ptr, 32); + src0_ptr += src_stride; + + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32, src1_ptr, 48, + in0, in1, in2, in3); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 64, src1_ptr, 80, in4, in5); + src1_ptr += src2_stride; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32, src1_ptr, 48, + in6, in7, in8, in9); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 64, src1_ptr, 80, in10, in11); + src1_ptr += src2_stride; + + LSX_DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6, + dst0, dst2, dst4, dst6); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + dst1, dst3, dst5, dst7); + LSX_DUP4_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst5, 6, dst7, 6, dst1, dst3, + dst5, dst7); + LSX_DUP2_ARG2(__lsx_vsllwil_hu_bu, src4, 6, src5, 6, dst8, dst10); + LSX_DUP2_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, dst9, dst11); + LSX_DUP2_ARG2(__lsx_vslli_h, dst9, 6, dst11, 6, dst9, dst11); + + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + __lsx_hevc_bi_rnd_clip4(in4, in5, in6, in7, dst4, dst5, dst6, dst7, + &dst4, &dst5, &dst6, &dst7); + __lsx_hevc_bi_rnd_clip4(in8, in9, in10, in11, dst8, dst9, dst10, dst11, + &dst8, &dst9, &dst10, &dst11); + LSX_DUP4_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + out0, out1, out2, out3); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst9, dst8, dst11, dst10, out4, out5); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + __lsx_vst(out2, dst, 32); + dst += dst_stride; + __lsx_vst(out3, dst, 0); + __lsx_vst(out4, dst, 16); + __lsx_vst(out5, dst, 32); + dst += dst_stride; + } +} + +static void hevc_bi_copy_64w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + __m128i out0, out1, out2, out3; + __m128i src0, src1, src2, src3; + __m128i zero = {0}; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src0_ptr, 32, src0_ptr, 48, + src0, src1, src2, src3); + src0_ptr += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32, src1_ptr, 48, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 64, src1_ptr, 80, src1_ptr, 96, src1_ptr, 112, + in4, in5, in6, in7); + src1_ptr += src2_stride; + + LSX_DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6, + dst0, dst2, dst4, dst6); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + dst1, dst3, dst5, dst7); + LSX_DUP4_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst5, 6, dst7, 6, dst1, dst3, + dst5, dst7); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + __lsx_hevc_bi_rnd_clip4(in4, in5, in6, in7, dst4, dst5, dst6, dst7, + &dst4, &dst5, &dst6, &dst7); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, out0, out1); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst5, dst4, dst7, dst6, out2, out3); + + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + __lsx_vst(out2, dst, 32); + __lsx_vst(out3, dst, 48); + dst += dst_stride; + } +} + +static void hevc_hz_bi_8t_16w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt; + const int32_t dst_stride_2x = (dst_stride << 1); + __m128i src0, src1, src2, src3; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3; + __m128i in0, in1, in2, in3; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src0_ptr -= 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 8, src0, src1); + src0_ptr += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 8, src2, src3); + src0_ptr += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in0, in1); + src1_ptr += src2_stride; + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in2, in3); + src1_ptr += src2_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt2, dst1, vec1, filt2, dst2, + vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask3, src3, src3, mask3, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt3, dst1, vec1, filt3, dst2, + vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, &dst0, &dst1, + &dst2, &dst3); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1); + __lsx_vst(dst0, dst, 0); + __lsx_vstx(dst1, dst, dst_stride); + dst += dst_stride_2x; + } +} + +static void hevc_hz_bi_8t_24w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, tmp0, tmp1; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2; + __m128i in0, in1, in2; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src0_ptr -= 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1, mask2, + mask3, mask4); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6); + mask7 = __lsx_vaddi_bu(mask0, 14); + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src0, src1); + src0_ptr += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in0, in1); + in2 = __lsx_vld(src1_ptr, 32); + src1_ptr += src2_stride; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask4, src1, src1, + mask0, src0, src0, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, dst0, vec3, filt1, dst0, dst1, dst2, dst0); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask5, src1, src1, mask1, src0, src0, + mask2, src1, src0, mask6, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst1, vec0, filt1, dst2, vec1, filt1, dst0, vec2, + filt2, dst1, vec3, filt2, dst1, dst2, dst0, dst1); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask2, src0, src0, mask3, src1, src0, + mask7, src1, src1, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst2, vec0, filt2, dst0, vec1, filt3, dst1, + vec2, filt3, dst2, vec3, filt3, dst2, dst0, dst1, dst2); + + __lsx_hevc_bi_rnd_clip2(in0, in1, dst0, dst1, &dst0, &dst1); + tmp0 = __lsx_vpickev_b(dst1, dst0); + dst2 = __lsx_vsadd_h(dst2, in2); + dst2 = __lsx_vmaxi_h(dst2, 0); + tmp1 = __lsx_vssrlrni_bu_h(dst2, dst2, 7); + + __lsx_vst(tmp0, dst, 0); + __lsx_vstelm_d(tmp1, dst, 16, 0); + dst += dst_stride; + } +} + +static void hevc_hz_bi_8t_32w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, src2, tmp0, tmp1; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3; + __m128i in0, in1, in2, in3; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src0_ptr -= 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1, mask2, + mask3, mask4); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6); + mask7 = __lsx_vaddi_bu(mask0, 14); + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src0, src1); + src2 = __lsx_vld(src0_ptr, 24); + src0_ptr += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32, src1_ptr, + 48, in0, in1, in2, in3); + src1_ptr += src2_stride; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask4, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src2, src2, mask0, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask5, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask1, src2, src2, mask1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src0, mask6, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask2, src2, src2, mask2, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt2, dst1, vec1, filt2, dst2, vec2, + filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src0, mask7, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask3, src2, src2, mask3, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt3, dst1, vec1, filt3, dst2, vec2, + filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, tmp0, tmp1); + __lsx_vst(tmp0, dst, 0); + __lsx_vst(tmp1, dst, 16); + dst += dst_stride; + } +} + +static void hevc_hz_bi_8t_48w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, src2, src3; + __m128i tmp0, tmp1, tmp2; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3, dst4, dst5; + __m128i in0, in1, in2, in3, in4, in5; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src0_ptr -= 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1, mask2, + mask3, mask4); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6); + mask7 = __lsx_vaddi_bu(mask0, 14); + + for (loop_cnt = 64; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src0, src1); + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 32, src0_ptr, 40, src2, src3); + src0_ptr += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32, src1_ptr, + 48, in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask4, src1, src1, + mask0, src2, src1, mask4, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask5, src1, src1, + mask1, src2, src1, mask5, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src0, mask6, src1, src1, + mask2, src2, src1, mask6, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt2, dst1, vec1, filt2, dst2, + vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src0, mask7, src1, src1, + mask3, src2, src1, mask7, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt3, dst1, vec1, filt3, dst2, + vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3); + __lsx_hevc_bi_rnd_clip2(in0, in1, dst0, dst1, &dst0, &dst1); + __lsx_hevc_bi_rnd_clip2(in2, in3, dst2, dst3, &dst2, &dst3); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, tmp0, tmp1); + __lsx_vst(tmp0, dst, 0); + __lsx_vst(tmp1, dst, 16); + + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 64, src1_ptr, 80, in4, in5); + src1_ptr += src2_stride; + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, src2, src2, + mask1, src3, src3, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + dst4, vec2, filt1, dst5, vec3, filt1, dst4, dst5, dst4, dst5); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2, src2, src2, + mask3, src3, src3, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst4, vec0, filt2, dst5, vec1, filt2, dst4, + vec2, filt3, dst5, vec3, filt3, dst4, dst5, dst4, dst5); + + __lsx_hevc_bi_rnd_clip2(in4, in5, dst4, dst5, &dst4, &dst5); + tmp2 = __lsx_vpickev_b(dst5, dst4); + __lsx_vst(tmp2, dst, 32); + dst += dst_stride; + } +} + +static void hevc_hz_bi_8t_64w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, src2, src3, src4, src5, tmp0, tmp1; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3; + __m128i in0, in1, in2, in3; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src0_ptr -= 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1, mask2, + mask3, mask4); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6); + mask7 = __lsx_vaddi_bu(mask0, 14); + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src0, src1); + src2 = __lsx_vld(src0_ptr, 24); + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 32, src0_ptr, 48, src3, src4); + src5 = __lsx_vld(src0_ptr, 56); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32, src1_ptr, + 48, in0, in1, in2, in3); + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask4, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src2, src2, mask0, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask5, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask1, src2, src2, mask1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src0, mask6, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask2, src2, src2, mask2, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt2, dst1, vec1, filt2, dst2, + vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src0, mask7, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask3, src2, src2, mask3, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt3, dst1, vec1, filt3, dst2, + vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, tmp0, tmp1); + __lsx_vst(tmp0, dst, 0); + __lsx_vst(tmp1, dst, 16); + + src0 = src3; + src1 = src4; + src2 = src5; + + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 64, src1_ptr, 80, src1_ptr, 96, src1_ptr, + 112, in0, in1, in2, in3); + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask4, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src2, src2, mask0, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask5, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask1, src2, src2, mask1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src0, mask6, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask2, src2, src2, mask2, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt2, dst1, vec1, filt2, dst2, + vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src0, mask7, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask3, src2, src2, mask3, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt3, dst1, vec1, filt3, dst2, + vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, tmp0, tmp1); + __lsx_vst(tmp0, dst, 32); + __lsx_vst(tmp1, dst, 48); + src1_ptr += src2_stride; + src0_ptr += src_stride; + dst += dst_stride; + } +} + +static av_always_inline +void hevc_vt_bi_8t_8w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i src0, src1, src2, src3, src4, src5; + __m128i src6, src7, src8, src9, src10; + __m128i in0, in1, in2, in3; + __m128i src10_r, src32_r, src54_r, src76_r, src98_r; + __m128i src21_r, src43_r, src65_r, src87_r, src109_r; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i filt0, filt1, filt2, filt3; + __m128i const_vec; + + src0_ptr -= src_stride_3x; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filt0, filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, src0, src1, + src2, src3); + src0_ptr += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src4, src5); + src6 = __lsx_vld(src0_ptr + src_stride_2x, 0); + src0_ptr += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, + src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_r, src32_r, src54_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, + src7, src8, src9, src10); + src0_ptr += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in1, in2, in3); + src1_ptr += src2_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src76_r, src87_r, src98_r, src109_r); + + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, src54_r, filt2, dst0_r, src76_r, filt3, dst0_r, + dst0_r, dst0_r, dst0_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, src65_r, filt2, dst1_r, src87_r, filt3, dst1_r, + dst1_r, dst1_r, dst1_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src32_r, filt0, dst2_r, src54_r, + filt1, dst2_r, src76_r, filt2, dst2_r, src98_r, filt3, dst2_r, + dst2_r, dst2_r, dst2_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst3_r, src65_r, + filt1, dst3_r, src87_r, filt2, dst3_r, src109_r, filt3, dst3_r, + dst3_r, dst3_r, dst3_r); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0_r, dst1_r, dst2_r, dst3_r, + &dst0_r, &dst1_r, &dst2_r, &dst3_r); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r); + __lsx_vstelm_d(dst0_r, dst, 0, 0); + __lsx_vstelm_d(dst0_r, dst + dst_stride, 0, 1); + __lsx_vstelm_d(dst1_r, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(dst1_r, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + + src6 = src10; + } +} + +static av_always_inline +void hevc_vt_bi_8t_16multx2mult_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height, + int32_t width) +{ + uint8_t *src0_ptr_tmp; + int16_t *src1_ptr_tmp; + uint8_t *dst_tmp; + uint32_t loop_cnt; + uint32_t cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i in0, in1, in2, in3; + __m128i src10_r, src32_r, src54_r, src76_r; + __m128i src21_r, src43_r, src65_r, src87_r; + __m128i dst0_r, dst1_r; + __m128i src10_l, src32_l, src54_l, src76_l; + __m128i src21_l, src43_l, src65_l, src87_l; + __m128i dst0_l, dst1_l; + __m128i filt0, filt1, filt2, filt3; + __m128i const_vec; + + src0_ptr -= src_stride_3x; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filt0, filt1, filt2, filt3); + + for (cnt = (width >> 4); cnt--;) { + src0_ptr_tmp = src0_ptr; + src1_ptr_tmp = src1_ptr; + dst_tmp = dst; + + LSX_DUP4_ARG2(__lsx_vld, src0_ptr_tmp, 0, src0_ptr_tmp + src_stride, 0, + src0_ptr_tmp + src_stride_2x, 0, src0_ptr_tmp + src_stride_3x, + 0, src0, src1, src2, src3); + src0_ptr_tmp += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src0_ptr_tmp, 0, src0_ptr_tmp + src_stride, 0, src4, src5); + src6 = __lsx_vld(src0_ptr_tmp + src_stride_2x, 0); + src0_ptr_tmp += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_r, src32_r, src54_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_l, src32_l, src54_l, src21_l); + LSX_DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, src43_l, src65_l); + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr_tmp, 0, src0_ptr_tmp + src_stride, 0, + src7, src8); + src0_ptr_tmp += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vld, src1_ptr_tmp, 0, src1_ptr_tmp + src2_stride, 0, + in0, in1); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr_tmp, 16, src1_ptr_tmp + src2_stride, + 16, in2, in3); + src1_ptr_tmp += src2_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src7, src8); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src76_l, src87_l); + + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, src54_r, filt2, dst0_r, src76_r, filt3, dst0_r, + dst0_r, dst0_r, dst0_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, src65_r, filt2, dst1_r, src87_r, filt3, dst1_r, + dst1_r, dst1_r, dst1_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src10_l, filt0, dst0_l, src32_l, + filt1, dst0_l, src54_l, filt2, dst0_l, src76_l, filt3, dst0_l, + dst0_l, dst0_l, dst0_l); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src21_l, filt0, dst1_l, src43_l, + filt1, dst1_l, src65_l, filt2, dst1_l, src87_l, filt3, dst1_l, + dst1_l, dst1_l, dst1_l); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0_r, dst1_r, dst0_l, dst1_l, + &dst0_r, &dst1_r, &dst0_l, &dst1_l); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r); + + __lsx_vst(dst0_r, dst_tmp, 0); + __lsx_vst(dst1_r, dst_tmp + dst_stride, 0); + dst_tmp += dst_stride_2x; + + src10_r = src32_r; + src32_r = src54_r; + src54_r = src76_r; + src21_r = src43_r; + src43_r = src65_r; + src65_r = src87_r; + src10_l = src32_l; + src32_l = src54_l; + src54_l = src76_l; + src21_l = src43_l; + src43_l = src65_l; + src65_l = src87_l; + src6 = src8; + } + + src0_ptr += 16; + src1_ptr += 16; + dst += 16; + } +} + +static void hevc_vt_bi_8t_16w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + hevc_vt_bi_8t_16multx2mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter, height, 16); +} + +static void hevc_vt_bi_8t_24w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + hevc_vt_bi_8t_16multx2mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter, height, 16); + hevc_vt_bi_8t_8w_lsx(src0_ptr + 16, src_stride, src1_ptr + 16, src2_stride, + dst + 16, dst_stride, filter, height); +} + +static void hevc_vt_bi_8t_32w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + hevc_vt_bi_8t_16multx2mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter, height, 32); +} + +static void hevc_vt_bi_8t_48w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + hevc_vt_bi_8t_16multx2mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter, height, 48); +} + +static void hevc_vt_bi_8t_64w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + hevc_vt_bi_8t_16multx2mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter, height, 64); +} + +static av_always_inline +void hevc_hv_bi_8t_8multx1mult_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t width) +{ + uint32_t loop_cnt; + uint32_t cnt; + uint8_t *src0_ptr_tmp; + int16_t *src1_ptr_tmp; + uint8_t *dst_tmp; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i out; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0, tmp; + __m128i filt0, filt1, filt2, filt3; + __m128i filt_h0, filt_h1, filt_h2, filt_h3; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1, mask2, mask3; + __m128i const_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i dst0_r, dst0_l; + __m128i dst10_r, dst32_r, dst54_r, dst76_r; + __m128i dst10_l, dst32_l, dst54_l, dst76_l; + + src0_ptr -= src_stride_3x + 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4, filter_x, 6, + filt0, filt1, filt2, filt3); + filt_h3 = __lsx_vld(filter_y, 0); + filt_h3 = __lsx_vsllwil_h_b(filt_h3, 0); + + LSX_DUP4_ARG2(__lsx_vreplvei_w, filt_h3, 0, filt_h3, 1, filt_h3, 2, filt_h3, 3, + filt_h0, filt_h1, filt_h2, filt_h3); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + for (cnt = width >> 3; cnt--;) { + src0_ptr_tmp = src0_ptr; + dst_tmp = dst; + src1_ptr_tmp = src1_ptr; + + LSX_DUP4_ARG2(__lsx_vld, src0_ptr_tmp, 0, src0_ptr_tmp + src_stride, 0, + src0_ptr_tmp + src_stride_2x, 0, src0_ptr_tmp + src_stride_3x, 0, + src0, src1, src2, src3); + src0_ptr_tmp += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src0_ptr_tmp, 0, src0_ptr_tmp + src_stride, 0, src4, src5); + src6 = __lsx_vld(src0_ptr_tmp + src_stride_2x, 0); + src0_ptr_tmp += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + /* row 0 row 1 row 2 row 3 */ + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0, src0, + mask2, src0, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1, + mask2, src1, src1, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2, + mask2, src2, src2, mask3, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3, src3, + mask2, src3, src3, mask3, vec12, vec13, vec14, vec15); + dst0 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, filt2, + filt3); + dst1 = __lsx_hevc_filt_8tap_h(vec4, vec5, vec6, vec7, filt0, filt1, filt2, + filt3); + dst2 = __lsx_hevc_filt_8tap_h(vec8, vec9, vec10, vec11, filt0, filt1, filt2, + filt3); + dst3 = __lsx_hevc_filt_8tap_h(vec12, vec13, vec14, vec15, filt0, filt1, + filt2, filt3); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4, src4, + mask2, src4, src4, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src5, src5, + mask2, src5, src5, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src6, src6, + mask2, src6, src6, mask3, vec8, vec9, vec10, vec11); + dst4 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, filt2, + filt3); + dst5 = __lsx_hevc_filt_8tap_h(vec4, vec5, vec6, vec7, filt0, filt1, filt2, + filt3); + dst6 = __lsx_hevc_filt_8tap_h(vec8, vec9, vec10, vec11, filt0, filt1, filt2, + filt3); + + for (loop_cnt = height; loop_cnt--;) { + src7 = __lsx_vld(src0_ptr_tmp, 0); + src7 = __lsx_vxori_b(src7, 128); + src0_ptr_tmp += src_stride; + + in0 = __lsx_vld(src1_ptr_tmp, 0); + src1_ptr_tmp += src2_stride; + + LSX_DUP4_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, src7, + src7, mask2, src7, src7, mask3, vec0, vec1, vec2, vec3); + dst7 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, + filt2, filt3); + LSX_DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + dst10_r, dst32_r, dst54_r, dst76_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + dst10_l, dst32_l, dst54_l, dst76_l); + + dst0_r = __lsx_hevc_filt_8tap_w(dst10_r, dst32_r, dst54_r, dst76_r, + filt_h0, filt_h1, filt_h2, filt_h3); + dst0_l = __lsx_hevc_filt_8tap_w(dst10_l, dst32_l, dst54_l, dst76_l, + filt_h0, filt_h1, filt_h2, filt_h3); + dst0_r = __lsx_vsrli_w(dst0_r, 6); + dst0_l = __lsx_vsrli_w(dst0_l, 6); + + tmp = __lsx_vpickev_h(dst0_l, dst0_r); + LSX_DUP2_ARG2(__lsx_vsadd_h, tmp, in0, tmp, const_vec, tmp, tmp); + tmp = __lsx_vmaxi_h(tmp, 0); + out = __lsx_vssrlrni_bu_h(tmp, tmp, 7); + __lsx_vstelm_d(out, dst_tmp, 0, 0); + dst_tmp += dst_stride; + + dst0 = dst1; + dst1 = dst2; + dst2 = dst3; + dst3 = dst4; + dst4 = dst5; + dst5 = dst6; + dst6 = dst7; + } + + src0_ptr += 8; + dst += 8; + src1_ptr += 8; + } +} + +static void hevc_hv_bi_8t_8w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_bi_8t_8multx1mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, + height, 8); +} + +static void hevc_hv_bi_8t_16w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_bi_8t_8multx1mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, + height, 16); +} + +static void hevc_hv_bi_8t_24w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_bi_8t_8multx1mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, + height, 24); +} + +static void hevc_hv_bi_8t_32w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_bi_8t_8multx1mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, + height, 32); +} + +static void hevc_hv_bi_8t_48w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_bi_8t_8multx1mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, + height, 48); +} + +static void hevc_hv_bi_8t_64w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_bi_8t_8multx1mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, + height, 64); +} + +static void hevc_hz_bi_4t_24w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int16_t *src1_ptr_tmp; + uint8_t *dst_tmp; + uint32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i filt0, filt1; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1, mask2, mask3; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i const_vec; + + src0_ptr -= 1; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 8, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 10); + + dst_tmp = dst + 16; + src1_ptr_tmp = src1_ptr + 16; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, + src0, src2, src4, src6); + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 16, src0_ptr + src_stride, 16, + src0_ptr + src_stride_2x, 16, src0_ptr + src_stride_3x, 16, + src1, src3, src5, src7); + src0_ptr += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in2, in4, in6); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, + src1_ptr + src2_stride_2x, 16, src1_ptr + src2_stride_3x, 16, + in1, in3, in5, in7); + src1_ptr += src2_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src6, 128, src7, 128, src4, + src5, src6, src7); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask2, src2, src2, + mask0, src3, src2, mask2, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask3, src2, src2, + mask1, src3, src2, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src5, src4, mask2, src6, src6, + mask0, src7, src6, mask2, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst4, dst5, dst6, + dst7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask1, src5, src4, mask3, src6, src6, + mask1, src7, src6, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst4, vec0, filt1, dst5, vec1, filt1, dst6, + vec2, filt1, dst7, vec3, filt1, dst4, dst5, dst6, dst7); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + __lsx_hevc_bi_rnd_clip4(in4, in5, in6, in7, dst4, dst5, dst6, dst7, + &dst4, &dst5, &dst6, &dst7); + + LSX_DUP4_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + dst0, dst1, dst2, dst3); + __lsx_vst(dst0, dst, 0); + __lsx_vstx(dst1, dst, dst_stride); + __lsx_vstx(dst2, dst, dst_stride_2x); + __lsx_vstx(dst3, dst, dst_stride_3x); + dst += dst_stride_4x; + + LSX_DUP4_ARG2(__lsx_vld, src1_ptr_tmp, 0, src1_ptr_tmp + src2_stride, 0, + src1_ptr_tmp + src2_stride_2x, 0, src1_ptr_tmp + src2_stride_3x, 0, + in0, in1, in2, in3); + src1_ptr_tmp += src2_stride_4x; + + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src3, src3, mask0, src5, src5, + mask0, src7, src7, mask0, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask1, src3, src3, mask1, src5, src5, + mask1, src7, src7, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1); + __lsx_vstelm_d(dst0, dst_tmp, 0, 0); + __lsx_vstelm_d(dst0, dst_tmp + dst_stride, 0, 1); + __lsx_vstelm_d(dst1, dst_tmp + dst_stride_2x, 0, 0); + __lsx_vstelm_d(dst1, dst_tmp + dst_stride_3x, 0, 1); + dst_tmp += dst_stride_4x; + } +} + +static void hevc_hz_bi_4t_32w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, src2; + __m128i in0, in1, in2, in3; + __m128i filt0, filt1; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1, mask2, mask3; + __m128i dst0, dst1, dst2, dst3; + __m128i vec0, vec1, vec2, vec3; + __m128i const_vec; + + src0_ptr -= 1; + + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 8, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 10); + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src0, src1); + src2 = __lsx_vld(src0_ptr, 24); + src0_ptr += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32, src1_ptr, + 48, in0, in1, in2, in3); + src1_ptr += src2_stride; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask2, src1, src1, + mask0, src2, src2, mask0, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask3, src1, src1, + mask1, src2, src2, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, vec2, + filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0, dst1, dst2, dst3, + &dst0, &dst1, &dst2, &dst3); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1); + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + dst += dst_stride; + } +} + +static void hevc_vt_bi_4t_12w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i src0, src1, src2, src3, src4, src5, src6; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i src10_r, src32_r, src21_r, src43_r, src54_r, src65_r; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i src10_l, src32_l, src54_l, src21_l, src43_l, src65_l; + __m128i src2110, src4332, src6554; + __m128i dst0_l, dst1_l, filt0, filt1; + __m128i const_vec; + + src0_ptr -= src_stride; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src0, src1); + src2 = __lsx_vld(src0_ptr + src_stride_2x, 0); + src0_ptr += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + src2110 = __lsx_vilvl_d(src21_l, src10_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src3, src4); + src0_ptr += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src5, src6); + src0_ptr += src_stride_2x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, + src1_ptr + src2_stride_2x, 16, src1_ptr + src2_stride_3x, 16, + in4, in5, in6, in7); + src1_ptr += src2_stride_4x; + LSX_DUP2_ARG2(__lsx_vilvl_d, in5, in4, in7, in6, in4, in5); + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l); + src4332 = __lsx_vilvl_d(src43_l, src32_l); + LSX_DUP2_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src54_r, src65_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src5, src4, src6, src5, src54_l, src65_l); + src6554 = __lsx_vilvl_d(src65_l, src54_l); + + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, const_vec, src21_r, filt0, dst1_r, src43_r, filt1, dst0_r, + dst0_r, dst1_r, dst1_r ); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src2110, filt0, dst0_l, src4332, + filt1, const_vec, src32_r, filt0, dst2_r, src54_r, filt1, dst0_l, + dst0_l, dst2_r, dst2_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst3_r, src65_r, + filt1, const_vec, src4332, filt0, dst1_l, src6554, filt1, dst3_r, + dst3_r, dst1_l, dst1_l); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0_r, dst1_r, dst2_r, dst3_r, + &dst0_r, &dst1_r, &dst2_r, &dst3_r); + __lsx_hevc_bi_rnd_clip2(in4, in5, dst0_l, dst1_l, &dst0_l, &dst1_l); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r); + dst0_l = __lsx_vpickev_b(dst1_l, dst0_l); + __lsx_vstelm_d(dst0_r, dst, 0, 0); + __lsx_vstelm_d(dst0_r, dst + dst_stride, 0, 1); + __lsx_vstelm_d(dst1_r, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(dst1_r, dst + dst_stride_3x, 0, 1); + __lsx_vstelm_w(dst0_l, dst, 8, 0); + __lsx_vstelm_w(dst0_l, dst + dst_stride, 8, 1); + __lsx_vstelm_w(dst0_l, dst + dst_stride_2x, 8, 2); + __lsx_vstelm_w(dst0_l, dst + dst_stride_3x, 8, 3); + dst += dst_stride_4x; + + src2 = src6; + src10_r = src54_r; + src21_r = src65_r; + src2110 = src6554; + } +} + +static void hevc_vt_bi_4t_16w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i src0, src1, src2, src3, src4, src5; + __m128i in0, in1, in2, in3; + __m128i src10_r, src32_r, src21_r, src43_r; + __m128i src10_l, src32_l, src21_l, src43_l; + __m128i dst0_r, dst1_r, dst0_l, dst1_l; + __m128i filt0, filt1; + __m128i const_vec; + + src0_ptr -= src_stride; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src0, src1); + src2 = __lsx_vld(src0_ptr + src_stride_2x, 0); + src0_ptr += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src3, src4); + src0_ptr += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, in0, in1); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, in2, in3); + src1_ptr += src2_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src3, src4); + LSX_DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, dst0_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, dst1_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_l, filt0, dst0_l, src32_l, + filt1, dst0_l, dst0_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_l, filt0, dst1_l, src43_l, + filt1, dst1_l, dst1_l); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0_r, dst1_r, dst0_l, dst1_l, + &dst0_r, &dst1_r, &dst0_l, &dst1_l); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r); + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst1_r, dst + dst_stride, 0); + dst += dst_stride_2x; + + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src5, src2); + src0_ptr += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, in0, in1); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, in2, in3); + src1_ptr += src2_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src5, 128, src2, 128, src5, src2); + LSX_DUP2_ARG2(__lsx_vilvl_b, src5, src4, src2, src5, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src5, src4, src2, src5, src10_l, src21_l); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src32_r, filt0, dst0_r, src10_r, + filt1, dst0_r, dst0_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src32_l, filt0, dst0_l, src10_l, + filt1, dst0_l, dst0_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst1_r, src21_r, + filt1, dst1_r, dst1_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src43_l, filt0, dst1_l, src21_l, + filt1, dst1_l, dst1_l); + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0_r, dst1_r, dst0_l, dst1_l, + &dst0_r, &dst1_r, &dst0_l, &dst1_l); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r); + __lsx_vst(dst0_r, dst, 0); + __lsx_vstx(dst1_r, dst, dst_stride); + dst += dst_stride_2x; + } +} + +static void hevc_vt_bi_4t_24w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i src0, src1, src2, src3, src4, src5; + __m128i src6, src7, src8, src9, src10, src11; + __m128i in0, in1, in2, in3, in4, in5; + __m128i src10_r, src32_r, src76_r, src98_r; + __m128i src21_r, src43_r, src87_r, src109_r; + __m128i src10_l, src32_l, src21_l, src43_l; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i dst0_l, dst1_l; + __m128i filt0, filt1; + __m128i const_vec; + + src0_ptr -= src_stride; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + /* 16width */ + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src0, src1); + src2 = __lsx_vld(src0_ptr + src_stride_2x, 0); + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + /* 8width */ + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 16, src0_ptr + src_stride, 16, src6, src7); + src8 = __lsx_vld(src0_ptr + src_stride_2x, 16); + src0_ptr += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src6, 128, src7, 128, src6, src7); + src8 = __lsx_vxori_b(src8, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + /* 16width */ + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src1_ptr, 0, + src1_ptr + src2_stride, 0, src3, src4, in0, in1); + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, src1_ptr, + 32, src1_ptr + src2_stride, 32, in2, in3, in4, in5); + src1_ptr += src2_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src3, src4); + LSX_DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l); + /* 8width */ + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 16, src0_ptr + src_stride, 16, src9, src10); + src0_ptr += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src9, 128, src10, 128, src9, src10); + LSX_DUP2_ARG2(__lsx_vilvl_b, src9, src8, src10, src9, src98_r, src109_r); + /* 16width */ + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, const_vec, src10_l, filt0, dst0_l, src32_l, filt1, dst0_r, + dst0_r, dst0_l, dst0_l); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, const_vec, src21_l, filt0, dst1_l, src43_l, filt1, dst1_r, + dst1_r, dst1_l, dst1_l); + /* 8width */ + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src76_r, filt0, dst2_r, src98_r, + filt1, const_vec, src87_r, filt0, dst3_r, src109_r, filt1, dst2_r, + dst2_r, dst3_r, dst3_r); + /* 16width */ + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0_r, dst1_r, dst0_l, dst1_l, + &dst0_r, &dst1_r, &dst0_l, &dst1_l); + __lsx_hevc_bi_rnd_clip2(in4, in5, dst2_r, dst3_r, &dst2_r, &dst3_r); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r); + dst2_r = __lsx_vpickev_b(dst3_r, dst2_r); + __lsx_vst(dst0_r, dst, 0); + __lsx_vstx(dst1_r, dst, dst_stride); + __lsx_vstelm_d(dst2_r, dst, 16, 0); + __lsx_vstelm_d(dst2_r, dst + dst_stride, 16, 1); + dst += dst_stride_2x; + + /* 16width */ + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src5, src2); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, in0, in1); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, in2, in3); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 32, src1_ptr + src2_stride, 32, in4, in5); + src1_ptr += src2_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src5, 128, src2, 128, src5, src2); + LSX_DUP2_ARG2(__lsx_vilvl_b, src5, src4, src2, src5, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src5, src4, src2, src5, src10_l, src21_l); + /* 8width */ + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 16, src0_ptr + src_stride, 16, src11, src8); + src0_ptr += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src11, 128, src8, 128, src11, src8); + LSX_DUP2_ARG2(__lsx_vilvl_b, src11, src10, src8, src11, src76_r, src87_r); + /* 16width */ + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src32_r, filt0, dst0_r, src10_r, + filt1, const_vec, src32_l, filt0, dst0_l, src10_l, filt1, dst0_r, + dst0_r, dst0_l, dst0_l); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst1_r, src21_r, + filt1, const_vec, src43_l, filt0, dst1_l, src21_l, filt1, dst1_r, + dst1_r, dst1_l, dst1_l); + /* 8width */ + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src98_r, filt0, dst2_r, src76_r, + filt1, const_vec, src109_r, filt0, dst3_r, src87_r, filt1, dst2_r, + dst2_r, dst3_r, dst3_r); + + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0_r, dst1_r, dst0_l, dst1_l, + &dst0_r, &dst1_r, &dst0_l, &dst1_l); + __lsx_hevc_bi_rnd_clip2(in4, in5, dst2_r, dst3_r, &dst2_r, &dst3_r); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r); + dst2_r = __lsx_vpickev_b(dst3_r, dst2_r); + __lsx_vst(dst0_r, dst, 0); + __lsx_vstx(dst1_r, dst, dst_stride); + __lsx_vstelm_d(dst2_r, dst, 16, 0); + __lsx_vstelm_d(dst2_r, dst + dst_stride, 16, 1); + dst += dst_stride_2x; + } +} + +static void hevc_vt_bi_4t_32w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt; + uint8_t *dst_tmp = dst + 16; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i src0, src1, src2, src3, src4, src6, src7, src8, src9, src10; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i src10_r, src32_r, src76_r, src98_r; + __m128i src21_r, src43_r, src87_r, src109_r; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i src10_l, src32_l, src76_l, src98_l; + __m128i src21_l, src43_l, src87_l, src109_l; + __m128i dst0_l, dst1_l, dst2_l, dst3_l; + __m128i filt0, filt1; + __m128i const_vec; + + src0_ptr -= src_stride; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + /* 16width */ + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src0, src1); + src2 = __lsx_vld(src0_ptr + src_stride_2x, 0); + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + + /* next 16width */ + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 16, src0_ptr + src_stride, 16, src6, src7); + src8 = __lsx_vld(src0_ptr + src_stride_2x, 16); + src0_ptr += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src6, 128, src7, 128, src6, src7); + src8 = __lsx_vxori_b(src8, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src76_l, src87_l); + + for (loop_cnt = (height >> 1); loop_cnt--;) { + /* 16width */ + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src3, src4); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, in0, in1); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 16, src1_ptr + src2_stride, 16, in2, in3); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 32, src1_ptr + src2_stride, 32, in4, in5); + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 48, src1_ptr + src2_stride, 48, in6, in7); + src1_ptr += src2_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src3, src4); + LSX_DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l); + /* 16width */ + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, dst0_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_l, filt0, dst0_l, src32_l, + filt1, dst0_l, dst0_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, dst1_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_l, filt0, dst1_l, src43_l, + filt1, dst1_l, dst1_l); + /* 16width */ + __lsx_hevc_bi_rnd_clip4(in0, in1, in2, in3, dst0_r, dst1_r, dst0_l, dst1_l, + &dst0_r, &dst1_r, &dst0_l, &dst1_l); + + src10_r = src32_r; + src21_r = src43_r; + src10_l = src32_l; + src21_l = src43_l; + src2 = src4; + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r); + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst1_r, dst + dst_stride, 0); + dst += dst_stride_2x; + + /* next 16width */ + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 16, src0_ptr + src_stride, 16, src9, src10); + src0_ptr += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src9, 128, src10, 128, src9, src10); + LSX_DUP2_ARG2(__lsx_vilvl_b, src9, src8, src10, src9, src98_r, src109_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src9, src8, src10, src9, src98_l, src109_l); + /* next 16width */ + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src76_r, filt0, dst2_r, src98_r, + filt1, dst2_r, dst2_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src76_l, filt0, dst2_l, src98_l, + filt1, dst2_l, dst2_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src87_r, filt0, dst3_r, src109_r, + filt1, dst3_r, dst3_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src87_l, filt0, dst3_l, src109_l, + filt1, dst3_l, dst3_l); + /* next 16width */ + __lsx_hevc_bi_rnd_clip4(in4, in5, in6, in7, dst2_r, dst3_r, dst2_l, dst3_l, + &dst2_r, &dst3_r, &dst2_l, &dst3_l); + + LSX_DUP2_ARG2(__lsx_vpickev_b, dst2_l, dst2_r, dst3_l, dst3_r, dst2_r, dst3_r); + __lsx_vst(dst2_r, dst_tmp, 0); + __lsx_vstx(dst3_r, dst_tmp, dst_stride); + dst_tmp += dst_stride_2x; + + src76_r = src98_r; + src87_r = src109_r; + src76_l = src98_l; + src87_l = src109_l; + src8 = src10; + } +} + +static void hevc_hv_bi_4t_6w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + uint32_t tpw0, tpw1, tpw2, tpw3; + uint64_t tp0, tp1; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i out0, out1; + __m128i src0, src1, src2, src3, src4, src5, src6; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, mask1; + __m128i filt0, filt1, filt_h0, filt_h1, const_vec; + __m128i dsth0, dsth1, dsth2, dsth3, dsth4, dsth5; + __m128i dsth6, dsth7, dsth8, dsth9, dsth10; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i dst4_r, dst5_r, dst6_r, dst7_r; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + __m128i zero ={0}; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src0_ptr -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filt_h1 = __lsx_vld(filter_y, 0); + filt_h1 = __lsx_vsllwil_h_b(filt_h1, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filt_h1, 0, filt_h1, 1, filt_h0, filt_h1); + + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src0, src1); + src2 = __lsx_vld(src0_ptr + src_stride_2x, 0); + src0_ptr += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + + dsth0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dsth1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dsth2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dsth1, dsth0, dsth2, dsth1, tmp0, tmp2); + LSX_DUP2_ARG2(__lsx_vilvh_h, dsth1, dsth0, dsth2, dsth1, tmp1, tmp3); + + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, src3, src4, + src5, src6); + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, src4, + src5, src6); + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7); + + dsth3 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dsth4 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dsth5 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dsth6 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + + src0_ptr += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, src3, src4, + src5, src6); + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, src4, + src5, src6); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7); + + dsth7 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dsth8 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dsth9 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dsth10 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dsth3, dsth2, dsth4, dsth3, tmp4, tmp6); + LSX_DUP2_ARG2(__lsx_vilvh_h, dsth3, dsth2, dsth4, dsth3, tmp5, tmp7); + LSX_DUP2_ARG2(__lsx_vilvl_h, dsth5, dsth4, dsth6, dsth5, dsth0, dsth2); + LSX_DUP2_ARG2(__lsx_vilvh_h, dsth5, dsth4, dsth6, dsth5, dsth1, dsth3); + dst0_r = __lsx_hevc_filt_4tap_w(tmp0, tmp4, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(tmp2, tmp6, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(tmp4, dsth0, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(tmp6, dsth2, filt_h0, filt_h1); + LSX_DUP2_ARG2(__lsx_vpickev_d, tmp3, tmp1, tmp7, tmp5, tmp0, tmp8); + dst0_l = __lsx_hevc_filt_4tap_w(tmp0, tmp8, filt_h0, filt_h1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dsth7, dsth6, dsth8, dsth7, tmp0, tmp2); + LSX_DUP2_ARG2(__lsx_vilvh_h, dsth7, dsth6, dsth8, dsth7, tmp1, tmp3); + LSX_DUP2_ARG2(__lsx_vilvl_h, dsth9, dsth8, dsth10, dsth9, tmp4, tmp6); + LSX_DUP2_ARG2(__lsx_vilvh_h, dsth9, dsth8, dsth10, dsth9, tmp5, tmp7); + dst4_r = __lsx_hevc_filt_4tap_w(dsth0, tmp0, filt_h0, filt_h1); + dst5_r = __lsx_hevc_filt_4tap_w(dsth2, tmp2, filt_h0, filt_h1); + dst6_r = __lsx_hevc_filt_4tap_w(tmp0, tmp4, filt_h0, filt_h1); + dst7_r = __lsx_hevc_filt_4tap_w(tmp2, tmp6, filt_h0, filt_h1); + LSX_DUP2_ARG2(__lsx_vpickev_d, dsth3, dsth1, tmp3, tmp1, tmp0, tmp1); + tmp2 = __lsx_vpickev_d(tmp7, tmp5); + + dst1_l = __lsx_hevc_filt_4tap_w(tmp8, tmp0, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(tmp0, tmp1, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(tmp1, tmp2, filt_h0, filt_h1); + + LSX_DUP4_ARG2(__lsx_vsrai_d, dst0_r, 6, dst1_r, 6, dst2_r, 6, dst3_r, 6, dst0_r, + dst1_r, dst2_r, dst3_r); + LSX_DUP4_ARG2(__lsx_vsrai_d, dst4_r, 6, dst5_r, 6, dst6_r, 6, dst7_r, 6, dst4_r, + dst5_r, dst6_r, dst7_r); + LSX_DUP4_ARG2(__lsx_vsrai_d, dst0_l, 6, dst1_l, 6, dst2_l, 6, dst3_l, 6, dst0_l, + dst1_l, dst2_l, dst3_l); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst1_r, dst0_r, dst3_r, dst2_r, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst5_r, dst4_r, dst7_r, dst6_r, tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst1_l, dst0_l, dst3_l, dst2_l, tmp4, tmp5); + + tp0 = *(uint64_t *)src1_ptr; + tp1 = *(uint64_t *)(src1_ptr + src2_stride); + LSX_DUP2_ARG3(__lsx_vinsgr2vr_d, zero, tp0, 0, dsth0, tp1, 1, dsth0, dsth0); + tp0 = *(uint64_t *)(src1_ptr + src2_stride_2x); + tp1 = *(uint64_t *)(src1_ptr + src2_stride_3x); + LSX_DUP2_ARG3(__lsx_vinsgr2vr_d, zero, tp0, 0, dsth1, tp1, 1, dsth1, dsth1); + src1_ptr += src2_stride_4x; + tp0 = *(uint64_t *)src1_ptr; + tp1 = *(uint64_t *)(src1_ptr + src2_stride); + LSX_DUP2_ARG3(__lsx_vinsgr2vr_d, zero, tp0, 0, dsth2, tp1, 1, dsth2, dsth2); + tp0 = *(uint64_t *)(src1_ptr + src2_stride_2x); + tp1 = *(uint64_t *)(src1_ptr + src2_stride_3x); + LSX_DUP2_ARG3(__lsx_vinsgr2vr_d, zero, tp0, 0, dsth3, tp1, 1, dsth3, dsth3); + + LSX_DUP4_ARG2(__lsx_vsadd_h, dsth0, const_vec, dsth1, const_vec, dsth2, const_vec, + dsth3, const_vec, dsth0, dsth1, dsth2, dsth3); + LSX_DUP4_ARG2(__lsx_vsadd_h, dsth0, tmp0, dsth1, tmp1, dsth2, tmp2, dsth3, tmp3, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp2, 0, tmp3, 0, tmp0, tmp1, tmp2, + tmp3); + LSX_DUP2_ARG3(__lsx_vssrlrni_bu_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1); + + __lsx_vstelm_w(out0, dst, 0, 0); + __lsx_vstelm_w(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(out0, dst + dst_stride_2x, 0, 2); + __lsx_vstelm_w(out0, dst + dst_stride_3x, 0, 3); + dst += dst_stride_4x; + __lsx_vstelm_w(out1, dst, 0, 0); + __lsx_vstelm_w(out1, dst + dst_stride, 0, 1); + __lsx_vstelm_w(out1, dst + dst_stride_2x, 0, 2); + __lsx_vstelm_w(out1, dst + dst_stride_3x, 0, 3); + dst -= dst_stride_4x; + + src1_ptr -= src2_stride_4x; + tpw0 = *(uint32_t *)(src1_ptr + 4); + tpw1 = *(uint32_t *)(src1_ptr + 4 + src2_stride); + tpw2 = *(uint32_t *)(src1_ptr + 4 + src2_stride_2x); + tpw3 = *(uint32_t *)(src1_ptr + 4 + src2_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_w, zero, tpw0, 0, dsth4, tpw1, 1, dsth4, tpw2, 2, + dsth4, tpw3, 3, dsth4, dsth4, dsth4, dsth4); + src1_ptr += src2_stride_4x; + tpw0 = *(uint32_t *)(src1_ptr + 4); + tpw1 = *(uint32_t *)(src1_ptr + 4 + src2_stride); + tpw2 = *(uint32_t *)(src1_ptr + 4 + src2_stride_2x); + tpw3 = *(uint32_t *)(src1_ptr + 4 + src2_stride_3x); + LSX_DUP4_ARG3(__lsx_vinsgr2vr_w, zero, tpw0, 0, dsth5, tpw1, 1, dsth5, tpw2, 2, + dsth5, tpw3, 3, dsth5, dsth5, dsth5, dsth5); + LSX_DUP2_ARG2(__lsx_vsadd_h, dsth4, const_vec, dsth5, const_vec, dsth4, dsth5); + LSX_DUP2_ARG2(__lsx_vsadd_h, dsth4, tmp4, dsth5, tmp5, tmp4, tmp5); + LSX_DUP2_ARG2(__lsx_vmaxi_h, tmp4, 0, tmp5, 7, tmp4, tmp5); + out0 = __lsx_vssrlrni_bu_h(tmp5, tmp4, 7); + + __lsx_vstelm_h(out0, dst, 4, 0); + __lsx_vstelm_h(out0, dst + dst_stride, 4, 1); + __lsx_vstelm_h(out0, dst + dst_stride_2x, 4, 2); + __lsx_vstelm_h(out0, dst + dst_stride_3x, 4, 3); + dst += dst_stride_4x; + __lsx_vstelm_h(out0, dst, 4, 4); + __lsx_vstelm_h(out0, dst + dst_stride, 4, 5); + __lsx_vstelm_h(out0, dst + dst_stride_2x, 4, 6); + __lsx_vstelm_h(out0, dst + dst_stride_3x, 4, 7); +} + +static av_always_inline +void hevc_hv_bi_4t_8x2_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y) +{ + const int32_t src_stride_2x = (src_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + + __m128i out; + __m128i src0, src1, src2, src3, src4; + __m128i filt0, filt1; + __m128i filt_h0, filt_h1; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1; + __m128i filter_vec, const_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9; + __m128i dst0, dst1, dst2, dst3, dst4; + __m128i dst0_r, dst0_l, dst1_r, dst1_l; + __m128i dst10_r, dst32_r, dst21_r, dst43_r; + __m128i dst10_l, dst32_l, dst21_l, dst43_l; + __m128i tmp0, tmp1; + __m128i in0, in1; + + src0_ptr -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, src0, src1, + src2, src3); + src4 = __lsx_vld(src0_ptr + src_stride_4x, 0); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, + src2, src3); + src4 = __lsx_vxori_b(src4, 128); + + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, in0, in1); + LSX_DUP2_ARG2(__lsx_vsadd_h, in0, const_vec, in1, const_vec, in0, in1); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec6, vec7); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec8, vec9); + + dst0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dst3 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + dst4 = __lsx_hevc_filt_4tap_h(vec8, vec9, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l); + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, dst0_r, + dst0_l, dst1_r, dst1_l); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vsadd_h, in0, tmp0, in1, tmp1, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp0, tmp1); + out = __lsx_vssrlrni_bu_h(tmp1, tmp0, 7); + __lsx_vstelm_d(out, dst, 0, 0); + __lsx_vstelm_d(out, dst + dst_stride, 0, 1); +} + +static av_always_inline +void hevc_hv_bi_4t_8multx4_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t width8mult) +{ + uint32_t cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + + __m128i out0, out1; + __m128i src0, src1, src2, src3, src4, src5, src6, mask0, mask1; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i filt0, filt1, filt_h0, filt_h1, filter_vec, const_vec; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, tmp0, tmp1, tmp2, tmp3; + __m128i in0, in1, in2, in3; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r; + __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l; + + src0_ptr -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + for (cnt = width8mult; cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, src0, + src1, src2, src3); + src0_ptr += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, src4, src5); + src6 = __lsx_vld(src0_ptr + src_stride_2x, 0); + src0_ptr += (8 - src_stride_4x); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, + in0, in1, in2, in3); + src1_ptr += 8; + LSX_DUP4_ARG2(__lsx_vsadd_h, in0, const_vec, in1, const_vec, in2, const_vec, + in3, const_vec, in0, in1, in2, in3); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + + dst0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7); + + dst3 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst4 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst5 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dst6 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + + LSX_DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, + dst0_r, dst0_l, dst1_r, dst1_l); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6, + dst2_r, dst2_l, dst3_r, dst3_l); + LSX_DUP4_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, + dst3_l, dst3_r, tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vsadd_h, in0, tmp0, in1, tmp1, in2, tmp2, in3, tmp3, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp2, 0, tmp3, 0, tmp0, tmp1, + tmp2, tmp3); + LSX_DUP2_ARG3(__lsx_vssrlrni_bu_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_3x, 0, 1); + dst += 8; + } +} + +static av_always_inline +void hevc_hv_bi_4t_8x6_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y) +{ + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + + __m128i out0, out1, out2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i in0, in1, in2, in3, in4, in5; + __m128i filt0, filt1; + __m128i filt_h0, filt_h1; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1; + __m128i filter_vec, const_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9; + __m128i vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i dst4_r, dst4_l, dst5_r, dst5_l; + __m128i dst10_r, dst32_r, dst10_l, dst32_l; + __m128i dst21_r, dst43_r, dst21_l, dst43_l; + __m128i dst54_r, dst54_l, dst65_r, dst65_l; + __m128i dst76_r, dst76_l, dst87_r, dst87_l; + + src0_ptr -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, src0, src1, + src2, src3); + src0_ptr += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr + src_stride, 0, + src0_ptr + src_stride_2x, 0, src0_ptr + src_stride_3x, 0, src4, src5, + src6, src7); + src8 = __lsx_vld(src0_ptr + src_stride_4x, 0); + + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, + src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src6, 128, src7, 128, src4, src5, + src6, src7) + src8 = __lsx_vxori_b(src8, 128); + + LSX_DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, + src1_ptr + src2_stride_2x, 0, src1_ptr + src2_stride_3x, 0, in0, in1, + in2, in3); + src1_ptr += src2_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, in4, in5); + LSX_DUP4_ARG2(__lsx_vsadd_h, in0, const_vec, in1, const_vec, in2, const_vec, in3, + const_vec, in0, in1, in2, in3); + LSX_DUP2_ARG2(__lsx_vsadd_h, in4, const_vec, in5, const_vec, in4, in5); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec6, vec7); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec8, vec9); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec10, vec11); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec12, vec13); + LSX_DUP2_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, vec14, vec15); + LSX_DUP2_ARG3(__lsx_vshuf_b, src8, src8, mask0, src8, src8, mask1, vec16, vec17); + + dst0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dst3 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + dst4 = __lsx_hevc_filt_4tap_h(vec8, vec9, filt0, filt1); + dst5 = __lsx_hevc_filt_4tap_h(vec10, vec11, filt0, filt1); + dst6 = __lsx_hevc_filt_4tap_h(vec12, vec13, filt0, filt1); + dst7 = __lsx_hevc_filt_4tap_h(vec14, vec15, filt0, filt1); + dst8 = __lsx_hevc_filt_4tap_h(vec16, vec17, filt0, filt1); + + LSX_DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3, + dst10_r, dst21_r, dst32_r, dst43_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3, + dst10_l, dst21_l, dst32_l, dst43_l); + LSX_DUP4_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7, + dst54_r, dst65_r, dst76_r, dst87_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7, + dst54_l, dst65_l, dst76_l, dst87_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + dst4_r = __lsx_hevc_filt_4tap_w(dst54_r, dst76_r, filt_h0, filt_h1); + dst4_l = __lsx_hevc_filt_4tap_w(dst54_l, dst76_l, filt_h0, filt_h1); + dst5_r = __lsx_hevc_filt_4tap_w(dst65_r, dst87_r, filt_h0, filt_h1); + dst5_l = __lsx_hevc_filt_4tap_w(dst65_l, dst87_l, filt_h0, filt_h1); + + LSX_DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, dst0_r, + dst0_l, dst1_r, dst1_l); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6, dst2_r, + dst2_l, dst3_r, dst3_l); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst4_r, 6, dst4_l, 6, dst5_r, 6, dst5_l, 6, dst4_r, + dst4_l, dst5_r, dst5_l); + LSX_DUP4_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, + dst3_l, dst3_r, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst4_l, dst4_r, dst5_l, dst5_r, tmp4, tmp5); + LSX_DUP4_ARG2(__lsx_vsadd_h, in0, tmp0, in1, tmp1, in2, tmp2, in3, tmp3, tmp0, tmp1, + tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vsadd_h, in4, tmp4, in5, tmp5, tmp4, tmp5); + LSX_DUP4_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp2, 0, tmp3, 0, tmp0, tmp1, tmp2, + tmp3); + LSX_DUP2_ARG2(__lsx_vmaxi_h, tmp4, 0, tmp5, 0, tmp4, tmp5); + LSX_DUP2_ARG3(__lsx_vssrlrni_bu_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1); + out2 = __lsx_vssrlrni_bu_h(tmp5, tmp4, 7); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + __lsx_vstelm_d(out2, dst, 0, 0); + __lsx_vstelm_d(out2, dst + dst_stride, 0, 1); +} + +static av_always_inline +void hevc_hv_bi_4t_8multx4mult_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t width) +{ + uint32_t loop_cnt, cnt; + uint8_t *src0_ptr_tmp; + int16_t *src1_ptr_tmp; + uint8_t *dst_tmp; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src2_stride_2x = (src2_stride << 1); + const int32_t src2_stride_4x = (src2_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + const int32_t src2_stride_3x = src2_stride_2x + src2_stride; + __m128i out0, out1; + __m128i src0, src1, src2, src3, src4, src5, src6; + __m128i in0, in1, in2, in3; + __m128i filt0, filt1; + __m128i filt_h0, filt_h1; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1; + __m128i filter_vec, const_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i dst10_r, dst32_r, dst21_r, dst43_r; + __m128i dst10_l, dst32_l, dst21_l, dst43_l; + __m128i dst54_r, dst54_l, dst65_r, dst65_l, dst6; + + src0_ptr -= (src_stride + 1); + + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + for (cnt = width >> 3; cnt--;) { + src0_ptr_tmp = src0_ptr; + dst_tmp = dst; + src1_ptr_tmp = src1_ptr; + + LSX_DUP2_ARG2(__lsx_vld, src0_ptr_tmp, 0, src0_ptr_tmp + src_stride, 0, src0, src1); + src2 = __lsx_vld(src0_ptr_tmp + src_stride_2x, 0); + src0_ptr_tmp += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + + dst0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l); + + for (loop_cnt = height >> 2; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src0_ptr_tmp, 0, src0_ptr_tmp + src_stride, 0, + src0_ptr_tmp + src_stride_2x, 0, src0_ptr_tmp + src_stride_3x, + 0, src3, src4, src5, src6); + src0_ptr_tmp += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src1_ptr_tmp, 0, src1_ptr_tmp + src2_stride, 0, + src1_ptr_tmp + src2_stride_2x, 0, src1_ptr_tmp + src2_stride_3x, + 0, in0, in1, in2, in3); + src1_ptr_tmp += src2_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, + src3, src4, src5, src6); + + LSX_DUP4_ARG2(__lsx_vsadd_h, in0, const_vec, in1, const_vec, in2, + const_vec, in3, const_vec, in0, in1, in2, in3); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src4, + src4, mask0, src4, src4, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src6, + src6, mask0, src6, src6, mask1, vec4, vec5, vec6, vec7); + + dst3 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst4 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst5 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dst6 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + + LSX_DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, + dst0_r, dst0_l, dst1_r, dst1_l); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6, + dst2_r, dst2_l, dst3_r, dst3_l); + LSX_DUP4_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, + dst2_r, dst3_l, dst3_r, tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vsadd_h, in0, tmp0, in1, tmp1, in2, tmp2, in3, tmp3, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp2, 0, tmp3, 0, tmp0, + tmp1, tmp2, tmp3); + LSX_DUP2_ARG3(__lsx_vssrlrni_bu_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1); + __lsx_vstelm_d(out0, dst_tmp, 0, 0); + __lsx_vstelm_d(out0, dst_tmp + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst_tmp + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst_tmp + dst_stride_3x, 0, 1); + dst_tmp += dst_stride_4x; + + dst10_r = dst54_r; + dst10_l = dst54_l; + dst21_r = dst65_r; + dst21_l = dst65_l; + dst2 = dst6; + } + + src0_ptr += 8; + dst += 8; + src1_ptr += 8; + } +} + +static void hevc_hv_bi_4t_8w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + if (2 == height) { + hevc_hv_bi_4t_8x2_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y); + } else if (4 == height) { + hevc_hv_bi_4t_8multx4_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, 1); + } else if (6 == height) { + hevc_hv_bi_4t_8x6_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y); + } else { + hevc_hv_bi_4t_8multx4mult_lsx(src0_ptr, src_stride, + src1_ptr, src2_stride, + dst, dst_stride, + filter_x, filter_y, height, 8); + } +} + +static void hevc_hv_bi_4t_16w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + if (4 == height) { + hevc_hv_bi_4t_8multx4_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, 2); + } else { + hevc_hv_bi_4t_8multx4mult_lsx(src0_ptr, src_stride, src1_ptr, + src2_stride, dst, dst_stride, filter_x, + filter_y, height, 16); + } +} + +static void hevc_hv_bi_4t_24w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_bi_4t_8multx4mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, + height, 24); +} + +static void hevc_hv_bi_4t_32w_lsx(uint8_t *src0_ptr, + int32_t src_stride, + int16_t *src1_ptr, + int32_t src2_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_bi_4t_8multx4mult_lsx(src0_ptr, src_stride, src1_ptr, src2_stride, + dst, dst_stride, filter_x, filter_y, + height, 32); +} + +#define BI_MC_COPY(WIDTH) \ +void ff_hevc_put_hevc_bi_pel_pixels##WIDTH##_8_lsx(uint8_t *dst, \ + ptrdiff_t dst_stride, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int16_t *src_16bit, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) \ +{ \ + hevc_bi_copy_##WIDTH##w_lsx(src, src_stride, src_16bit, MAX_PB_SIZE, \ + dst, dst_stride, height); \ +} + +BI_MC_COPY(4); +BI_MC_COPY(6); +BI_MC_COPY(8); +BI_MC_COPY(12); +BI_MC_COPY(16); +BI_MC_COPY(24); +BI_MC_COPY(32); +BI_MC_COPY(48); +BI_MC_COPY(64); + +#undef BI_MC_COPY + +#define BI_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \ +void ff_hevc_put_hevc_bi_##PEL##_##DIR##WIDTH##_8_lsx(uint8_t *dst, \ + ptrdiff_t dst_stride, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int16_t *src_16bit, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) \ +{ \ + const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR - 1]; \ + \ + hevc_##DIR1##_bi_##TAP##t_##WIDTH##w_lsx(src, src_stride, src_16bit, \ + MAX_PB_SIZE, dst, dst_stride, \ + filter, height); \ +} + +BI_MC(qpel, h, 16, 8, hz, mx); +BI_MC(qpel, h, 24, 8, hz, mx); +BI_MC(qpel, h, 32, 8, hz, mx); +BI_MC(qpel, h, 48, 8, hz, mx); +BI_MC(qpel, h, 64, 8, hz, mx); + +BI_MC(qpel, v, 8, 8, vt, my); +BI_MC(qpel, v, 16, 8, vt, my); +BI_MC(qpel, v, 24, 8, vt, my); +BI_MC(qpel, v, 32, 8, vt, my); +BI_MC(qpel, v, 48, 8, vt, my); +BI_MC(qpel, v, 64, 8, vt, my); + +BI_MC(epel, h, 24, 4, hz, mx); +BI_MC(epel, h, 32, 4, hz, mx); + +BI_MC(epel, v, 12, 4, vt, my); +BI_MC(epel, v, 16, 4, vt, my); +BI_MC(epel, v, 24, 4, vt, my); +BI_MC(epel, v, 32, 4, vt, my); + +#undef BI_MC + +#define BI_MC_HV(PEL, WIDTH, TAP) \ +void ff_hevc_put_hevc_bi_##PEL##_hv##WIDTH##_8_lsx(uint8_t *dst, \ + ptrdiff_t dst_stride, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int16_t *src_16bit, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) \ +{ \ + const int8_t *filter_x = ff_hevc_##PEL##_filters[mx - 1]; \ + const int8_t *filter_y = ff_hevc_##PEL##_filters[my - 1]; \ + \ + hevc_hv_bi_##TAP##t_##WIDTH##w_lsx(src, src_stride, src_16bit, \ + MAX_PB_SIZE, dst, dst_stride, \ + filter_x, filter_y, height); \ +} + +BI_MC_HV(qpel, 8, 8); +BI_MC_HV(qpel, 16, 8); +BI_MC_HV(qpel, 24, 8); +BI_MC_HV(qpel, 32, 8); +BI_MC_HV(qpel, 48, 8); +BI_MC_HV(qpel, 64, 8); + +BI_MC_HV(epel, 8, 4); +BI_MC_HV(epel, 6, 4); +BI_MC_HV(epel, 16, 4); +BI_MC_HV(epel, 24, 4); +BI_MC_HV(epel, 32, 4); + +#undef BI_MC_HV diff --git a/libavcodec/loongarch/hevc_mc_uni_lsx.c b/libavcodec/loongarch/hevc_mc_uni_lsx.c new file mode 100644 index 0000000000..bd481c7ff1 --- /dev/null +++ b/libavcodec/loongarch/hevc_mc_uni_lsx.c @@ -0,0 +1,1476 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "hevcdsp_lsx.h" +#include "hevc_macros_lsx.h" + +static const uint8_t ff_hevc_mask_arr[16 * 3] __attribute__((aligned(0x40))) = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static av_always_inline +void common_hz_8t_64w_lsx(uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + int32_t loop_cnt; + __m128i mask0, mask1, mask2, mask3, out; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i filt0, filt1, filt2, filt3; + __m128i res0, res1, res2, res3; + + mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + src -= 3; + + /* rearranging filter */ + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 8, src, 16, src, 24, src0, src1, src2, + src3); + LSX_DUP4_ARG2(__lsx_vld, src, 32, src, 40, src, 48, src, 56, src4, src5, src6, + src7); + src += src_stride; + + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src6, 128, src7, 128, src4, + src5, src6, src7); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec2, vec3); + LSX_DUP4_ARG2(__lsx_dp2_h_b, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + res0, res1, res2, res3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, res0, vec0, filt2, res1, vec1, filt2, res2, + vec2, filt2, res3, vec3, filt2, res0, res1, res2, res3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec6, vec7); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, res0, vec4, filt1, res1, vec5, filt1, res2, + vec6, filt1, res3, vec7, filt1, res0, res1, res2, res3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask3, src3, src3, mask3, vec6, vec7); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, res0, vec4, filt3, res1, vec5, filt3, res2, + vec6, filt3, res3, vec7, filt3, res0, res1, res2, res3); + + out = __lsx_vssrarni_b_h(res1, res0, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 0); + out = __lsx_vssrarni_b_h(res3, res2, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 16); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src5, src5, mask0, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src7, src7, mask0, vec2, vec3); + LSX_DUP4_ARG2(__lsx_dp2_h_b, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, res0, res1, res2, res3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask2, src5, src5, mask2, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask2, src7, src7, mask2, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, res0, vec0, filt2, res1, vec1, filt2, res2, + vec2, filt2, res3, vec3, filt2, res0, res1, res2, res3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask1, src5, src5, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask1, src7, src7, mask1, vec6, vec7); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, res0, vec4, filt1, res1, vec5, filt1, res2, + vec6, filt1, res3, vec7, filt1, res0, res1, res2, res3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask3, src5, src5, mask3, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask3, src7, src7, mask3, vec6, vec7); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, res0, vec4, filt3, res1, vec5, filt3, res2, + vec6, filt3, res3, vec7, filt3, res0, res1, res2, res3); + + LSX_DUP4_ARG2(__lsx_vsrari_h, res0, 6, res1, 6, res2, 6, res3, 6, res0, res1, + res2, res3); + LSX_DUP4_ARG2(__lsx_vsat_h, res0, 7, res1, 7, res2, 7, res3, 7, res0, res1, + res2, res3); + out = __lsx_vpickev_b(res1, res0); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 32); + out = __lsx_vpickev_b(res3, res2); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 48); + dst += dst_stride; + } +} + +static av_always_inline +void common_vt_8t_8w_lsx(uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r; + __m128i src65_r, src87_r, src109_r, filt0, filt1, filt2, filt3; + __m128i tmp0, tmp1; + __m128i out0_r, out1_r, out2_r, out3_r; + + src -= src_stride_3x; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src4, src5); + src6 = __lsx_vld(src + src_stride_2x, 0); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, + src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + src += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, src10_r, + src32_r, src54_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src7, src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + src += src_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src76_r, src87_r, src98_r, src109_r); + LSX_DUP4_ARG2(__lsx_dp2_h_b, src10_r, filt0, src21_r, filt0, src32_r, filt0, + src43_r, filt0, out0_r, out1_r, out2_r, out3_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, out0_r, src32_r, filt1, out1_r, src43_r, filt1, + out2_r, src54_r, filt1, out3_r, src65_r, filt1, out0_r, out1_r, + out2_r, out3_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, out0_r, src54_r, filt2, out1_r, src65_r, filt2, + out2_r, src76_r, filt2, out3_r, src87_r, filt2, out0_r, out1_r, + out2_r, out3_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, out0_r, src76_r, filt3, out1_r, src87_r, filt3, + out2_r, src98_r, filt3, out3_r, src109_r, filt3, out0_r, out1_r, + out2_r, out3_r); + + tmp0 = __lsx_vssrarni_b_h(out1_r, out0_r, 6); + tmp0 = __lsx_vxori_b(tmp0, 128); + tmp1 = __lsx_vssrarni_b_h(out3_r, out2_r, 6); + tmp1 = __lsx_vxori_b(tmp1, 128); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src6 = src10; + } +} + +static av_always_inline +void common_vt_8t_16w_mult_lsx(uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height, + int32_t width) +{ + uint8_t *src_tmp; + uint8_t *dst_tmp; + uint32_t loop_cnt, cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt0, filt1, filt2, filt3; + __m128i src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r; + __m128i src65_r, src87_r, src109_r, src10_l, src32_l, src54_l, src76_l; + __m128i src98_l, src21_l, src43_l, src65_l, src87_l, src109_l; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l; + + src -= src_stride_3x; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + for (cnt = (width >> 4); cnt--;) { + src_tmp = src; + dst_tmp = dst; + + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, src0, + src1, src2, src3); + src_tmp += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src4, src5); + src6 = __lsx_vld(src_tmp + src_stride_2x, 0); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + src_tmp += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_r, src32_r, src54_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_l, src32_l, src54_l, src21_l); + LSX_DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, src43_l, src65_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, src7, + src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + src_tmp += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, + src9, src76_r, src87_r, src98_r, src109_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, + src9, src76_l, src87_l, src98_l, src109_l); + out0_r = __lsx_hevc_filt_8tap_h(src10_r, src32_r, src54_r, src76_r, + filt0, filt1, filt2, filt3); + out1_r = __lsx_hevc_filt_8tap_h(src21_r, src43_r, src65_r, src87_r, + filt0, filt1, filt2, filt3); + out2_r = __lsx_hevc_filt_8tap_h(src32_r, src54_r, src76_r, src98_r, + filt0, filt1, filt2, filt3); + out3_r = __lsx_hevc_filt_8tap_h(src43_r, src65_r, src87_r, src109_r, + filt0, filt1, filt2, filt3); + out0_l = __lsx_hevc_filt_8tap_h(src10_l, src32_l, src54_l, src76_l, + filt0, filt1, filt2, filt3); + out1_l = __lsx_hevc_filt_8tap_h(src21_l, src43_l, src65_l, src87_l, + filt0, filt1, filt2, filt3); + out2_l = __lsx_hevc_filt_8tap_h(src32_l, src54_l, src76_l, src98_l, + filt0, filt1, filt2, filt3); + out3_l = __lsx_hevc_filt_8tap_h(src43_l, src65_l, src87_l, src109_l, + filt0, filt1, filt2, filt3); + LSX_DUP4_ARG3(__lsx_vssrarni_b_h, out0_l, out0_r, 6, out1_l, out1_r, 6, + out2_l, out2_r, 6, out3_l, out3_r, 6, tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp2, 128, tmp3, 128, + tmp0, tmp1, tmp2, tmp3); + __lsx_vst(tmp0, dst_tmp, 0); + __lsx_vstx(tmp1, dst_tmp, dst_stride); + __lsx_vstx(tmp2, dst_tmp, dst_stride_2x); + __lsx_vstx(tmp3, dst_tmp, dst_stride_3x); + dst_tmp += dst_stride_4x; + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src10_l = src54_l; + src32_l = src76_l; + src54_l = src98_l; + src21_l = src65_l; + src43_l = src87_l; + src65_l = src109_l; + src6 = src10; + } + + src += 16; + dst += 16; + } +} + +static void common_vt_8t_24w_lsx(uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, + 16); + common_vt_8t_8w_lsx(src + 16, src_stride, dst + 16, dst_stride, filter, + height); +} + +static void common_vt_8t_32w_lsx(uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, + 32); +} + +static void common_vt_8t_48w_lsx(uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, + 48); +} + +static void common_vt_8t_64w_lsx(uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, + 64); +} + +static av_always_inline +void hevc_hv_uni_8t_8multx2mult_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, int32_t width) +{ + uint32_t loop_cnt, cnt; + uint8_t *src_tmp; + uint8_t *dst_tmp; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + + __m128i out; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i filt0, filt1, filt2, filt3; + __m128i filt_h0, filt_h1, filt_h2, filt_h3; + __m128i mask1, mask2, mask3; + __m128i filter_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8; + __m128i dst0_r, dst0_l, dst1_r, dst1_l; + __m128i dst10_r, dst32_r, dst54_r, dst76_r; + __m128i dst10_l, dst32_l, dst54_l, dst76_l; + __m128i dst21_r, dst43_r, dst65_r, dst87_r; + __m128i dst21_l, dst43_l, dst65_l, dst87_l; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src -= (src_stride_3x + 3); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4, filter_x, 6, + filt0, filt1, filt2, filt3); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP4_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filter_vec, 2, + filter_vec, 3, filt_h0, filt_h1, filt_h2, filt_h3); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + for (cnt = width >> 3; cnt--;) { + src_tmp = src; + dst_tmp = dst; + + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, src0, + src1, src2, src3); + src_tmp += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src4, src5); + src6 = __lsx_vld(src_tmp + src_stride_2x, 0); + src_tmp += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + /* row 0 row 1 row 2 row 3 */ + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0, src0, + mask2, src0, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1, + mask2, src1, src1, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2, + mask2, src2, src2, mask3, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3, src3, + mask2, src3, src3, mask3, vec12, vec13, vec14, vec15); + dst0 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3); + dst1 = __lsx_hevc_filt_8tap_h(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3); + dst2 = __lsx_hevc_filt_8tap_h(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3); + dst3 = __lsx_hevc_filt_8tap_h(vec12, vec13, vec14, vec15, filt0, filt1, filt2, filt3); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4, src4, + mask2, src4, src4, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src5, src5, + mask2, src5, src5, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src6, src6, + mask2, src6, src6, mask3, vec8, vec9, vec10, vec11); + dst4 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3); + dst5 = __lsx_hevc_filt_8tap_h(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3); + dst6 = __lsx_hevc_filt_8tap_h(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3); + + for (loop_cnt = height >> 1; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src7, src8); + LSX_DUP2_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src7, src8); + src_tmp += src_stride_2x; + + LSX_DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1, + dst10_r, dst32_r, dst54_r, dst21_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1, + dst10_l, dst32_l, dst54_l, dst21_l); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst4, dst3, dst6, dst5, dst43_r, dst65_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst4, dst3, dst6, dst5, dst43_l, dst65_l); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, src7, + src7, mask2, src7, src7, mask3, vec0, vec1, vec2, vec3); + dst7 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3); + + dst76_r = __lsx_vilvl_h(dst7, dst6); + dst76_l = __lsx_vilvh_h(dst7, dst6); + dst0_r = __lsx_hevc_filt_8tap_w(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0, + filt_h1, filt_h2, filt_h3); + dst0_l = __lsx_hevc_filt_8tap_w(dst10_l, dst32_l, dst54_l, dst76_l, filt_h0, + filt_h1, filt_h2, filt_h3); + LSX_DUP2_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst0_r, dst0_l); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src8, src8, mask0, src8, src8, mask1, src8, + src8, mask2, src8, src8, mask3, vec0, vec1, vec2, vec3); + dst8 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, + filt2, filt3); + + dst87_r = __lsx_vilvl_h(dst8, dst7); + dst87_l = __lsx_vilvh_h(dst8, dst7); + dst1_r = __lsx_hevc_filt_8tap_w(dst21_r, dst43_r, dst65_r, dst87_r, + filt_h0, filt_h1, filt_h2, filt_h3); + dst1_l = __lsx_hevc_filt_8tap_w(dst21_l, dst43_l, dst65_l, dst87_l, + filt_h0, filt_h1, filt_h2, filt_h3); + LSX_DUP2_ARG2(__lsx_vsrai_w, dst1_r, 6, dst1_l, 6, dst1_r, dst1_l); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, dst0_l, dst0_r, 6, dst1_l, dst1_r, 6, + dst0, dst1); + out = __lsx_vpickev_b(dst1, dst0); + out = __lsx_vxori_b(out, 128); + __lsx_vstelm_d(out, dst_tmp, 0, 0); + __lsx_vstelm_d(out, dst_tmp + dst_stride, 0, 1); + dst_tmp += dst_stride_2x; + + dst0 = dst2; + dst1 = dst3; + dst2 = dst4; + dst3 = dst5; + dst4 = dst6; + dst5 = dst7; + dst6 = dst8; + } + + src += 8; + dst += 8; + } +} + +static void hevc_hv_uni_8t_8w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_uni_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 8); +} + +static void hevc_hv_uni_8t_16w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_uni_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 16); +} + +static void hevc_hv_uni_8t_24w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_uni_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 24); +} + +static void hevc_hv_uni_8t_32w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_uni_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 32); +} + +static void hevc_hv_uni_8t_48w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_uni_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 48); +} + +static void hevc_hv_uni_8t_64w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_uni_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 64); +} + +static av_always_inline +void common_vt_4t_24w_lsx(uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t src_stride_3x = src_stride_2x + src_stride; + + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i src11, filt0, filt1; + __m128i src10_r, src32_r, src76_r, src98_r, src21_r, src43_r, src87_r; + __m128i src109_r, src10_l, src32_l, src21_l, src43_l; + __m128i out, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l; + + src -= src_stride; + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + /* 16 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src2 = __lsx_vld(src + src_stride_2x, 0); + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + + /* 8 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src6, src7); + src8 = __lsx_vld(src + src_stride_2x, 16); + src += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src6, 128, src7, 128, src6, src7); + src8 = __lsx_vxori_b(src8, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r); + + for (loop_cnt = 8; loop_cnt--;) { + /* 16 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src3, src4); + LSX_DUP2_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src3, src4); + LSX_DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l); + + /* 8 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src9, src10); + src += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src9, 128, src10, 128, src9, src10); + LSX_DUP2_ARG2(__lsx_vilvl_b, src9, src8, src10, src9, src98_r, src109_r); + + /* 16 width */ + out0_r = __lsx_hevc_filt_4tap_h(src10_r, src32_r, filt0, filt1); + out0_l = __lsx_hevc_filt_4tap_h(src10_l, src32_l, filt0, filt1); + out1_r = __lsx_hevc_filt_4tap_h(src21_r, src43_r, filt0, filt1); + out1_l = __lsx_hevc_filt_4tap_h(src21_l, src43_l, filt0, filt1); + + /* 8 width */ + out2_r = __lsx_hevc_filt_4tap_h(src76_r, src98_r, filt0, filt1); + out3_r = __lsx_hevc_filt_4tap_h(src87_r, src109_r, filt0, filt1); + + /* 16 + 8 width */ + out = __lsx_vssrarni_b_h(out0_l, out0_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 0); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out2_r, out2_r, 6, out3_r, out3_r, 6, + out2_r, out3_r); + LSX_DUP2_ARG2(__lsx_vxori_b, out2_r, 128, out3_r, 128, out2_r, out3_r); + __lsx_vstelm_d(out2_r, dst, 16, 0); + dst += dst_stride; + out = __lsx_vssrarni_b_h(out1_l, out1_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 0); + __lsx_vstelm_d(out3_r, dst, 16, 0); + dst += dst_stride; + + /* 16 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src5, src2); + LSX_DUP2_ARG2(__lsx_vxori_b, src5, 128, src2, 128, src5, src2); + LSX_DUP2_ARG2(__lsx_vilvl_b, src5, src4, src2, src5, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src5, src4, src2, src5, src10_l, src21_l); + + /* 8 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src11, src8); + src += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src11, 128, src8, 128, src11, src8); + LSX_DUP2_ARG2(__lsx_vilvl_b, src11, src10, src8, src11, src76_r, src87_r); + + /* 16 width */ + out0_r = __lsx_hevc_filt_4tap_h(src32_r, src10_r, filt0, filt1); + out0_l = __lsx_hevc_filt_4tap_h(src32_l, src10_l, filt0, filt1); + out1_r = __lsx_hevc_filt_4tap_h(src43_r, src21_r, filt0, filt1); + out1_l = __lsx_hevc_filt_4tap_h(src43_l, src21_l, filt0, filt1); + + /* 8 width */ + out2_r = __lsx_hevc_filt_4tap_h(src98_r, src76_r, filt0, filt1); + out3_r = __lsx_hevc_filt_4tap_h(src109_r, src87_r, filt0, filt1); + + /* 16 + 8 width */ + out = __lsx_vssrarni_b_h(out0_l, out0_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 0); + out = __lsx_vssrarni_b_h(out2_r, out2_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vstelm_d(out, dst, 16, 0); + dst += dst_stride; + + out = __lsx_vssrarni_b_h(out1_l, out1_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 0); + out = __lsx_vssrarni_b_h(out3_r, out3_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vstelm_d(out, dst, 16, 0); + dst += dst_stride; + } +} + +static av_always_inline +void common_vt_4t_32w_lsx(uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_3x = src_stride_2x + src_stride; + + __m128i src0, src1, src2, src3, src4, src6, src7, src8, src9, src10; + __m128i src10_r, src32_r, src76_r, src98_r; + __m128i src21_r, src43_r, src87_r, src109_r; + __m128i out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l; + __m128i src10_l, src32_l, src76_l, src98_l; + __m128i src21_l, src43_l, src87_l, src109_l; + __m128i filt0, filt1; + __m128i out; + + src -= src_stride; + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + /* 16 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src2 = __lsx_vld(src + src_stride_2x, 0); + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + + /* next 16 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src6, src7); + src8 = __lsx_vld(src + src_stride_2x, 16); + src += src_stride_3x; + + LSX_DUP2_ARG2(__lsx_vxori_b, src6, 128, src7, 128, src6, src7); + src8 = __lsx_vxori_b(src8, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src76_l, src87_l); + + for (loop_cnt = (height >> 1); loop_cnt--;) { + /* 16 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src3, src4); + LSX_DUP2_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src3, src4); + LSX_DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l); + + /* 16 width */ + out0_r = __lsx_hevc_filt_4tap_h(src10_r, src32_r, filt0, filt1); + out0_l = __lsx_hevc_filt_4tap_h(src10_l, src32_l, filt0, filt1); + out1_r = __lsx_hevc_filt_4tap_h(src21_r, src43_r, filt0, filt1); + out1_l = __lsx_hevc_filt_4tap_h(src21_l, src43_l, filt0, filt1); + + /* 16 width */ + out = __lsx_vssrarni_b_h(out0_l, out0_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 0); + out = __lsx_vssrarni_b_h(out1_l, out1_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vstx(out, dst, dst_stride); + + src10_r = src32_r; + src21_r = src43_r; + src10_l = src32_l; + src21_l = src43_l; + src2 = src4; + + /* next 16 width */ + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src9, src10); + src += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src9, 128, src10, 128, src9, src10); + LSX_DUP2_ARG2(__lsx_vilvl_b, src9, src8, src10, src9, src98_r, src109_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src9, src8, src10, src9, src98_l, src109_l); + + /* next 16 width */ + out2_r = __lsx_hevc_filt_4tap_h(src76_r, src98_r, filt0, filt1); + out2_l = __lsx_hevc_filt_4tap_h(src76_l, src98_l, filt0, filt1); + out3_r = __lsx_hevc_filt_4tap_h(src87_r, src109_r, filt0, filt1); + out3_l = __lsx_hevc_filt_4tap_h(src87_l, src109_l, filt0, filt1); + + /* next 16 width */ + out = __lsx_vssrarni_b_h(out2_l, out2_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 16); + out = __lsx_vssrarni_b_h(out3_l, out3_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst + dst_stride, 16); + + dst += dst_stride_2x; + + src76_r = src98_r; + src87_r = src109_r; + src76_l = src98_l; + src87_l = src109_l; + src8 = src10; + } +} + +static av_always_inline +void hevc_hv_uni_4t_8x2_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y) +{ + const int32_t src_stride_2x = (src_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i out; + __m128i src0, src1, src2, src3, src4; + __m128i filt0, filt1; + __m128i filt_h0, filt_h1, filter_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9; + __m128i dst0, dst1, dst2, dst3, dst4; + __m128i dst0_r, dst0_l, dst1_r, dst1_l; + __m128i dst10_r, dst32_r, dst21_r, dst43_r; + __m128i dst10_l, dst32_l, dst21_l, dst43_l; + __m128i out0_r, out1_r; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask1 = __lsx_vaddi_bu(mask0, 2); + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src4 = __lsx_vld(src + src_stride_4x, 0); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src1, src1, + mask0, src1, src1, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src3, src3, + mask0, src3, src3, mask1, vec4, vec5, vec6, vec7); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec8, vec9); + + dst0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dst3 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + dst4 = __lsx_hevc_filt_4tap_h(vec8, vec9, filt0, filt1); + LSX_DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3, + dst10_r, dst21_r, dst32_r, dst43_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3, + dst10_l, dst21_l, dst32_l, dst43_l); + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + LSX_DUP2_ARG3(__lsx_vsrani_h_w, dst0_l, dst0_r, 6, dst1_l, dst1_r, 6, + out0_r, out1_r); + out = __lsx_vssrarni_b_h(out1_r, out0_r, 6); + out = __lsx_vxori_b(out, 128); + __lsx_vstelm_d(out, dst, 0, 0); + __lsx_vstelm_d(out, dst + dst_stride, 0, 1); +} + +static av_always_inline +void hevc_hv_uni_4t_8multx4_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t width8mult) +{ + uint32_t cnt; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i out0, out1; + __m128i src0, src1, src2, src3, src4, src5, src6, mask0, mask1; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i filt0, filt1, filt_h0, filt_h1, filter_vec; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, tmp0, tmp1, tmp2, tmp3; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r; + __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + mask1 = __lsx_vaddi_bu(mask0, 2); + + for (cnt = width8mult; cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src4, src5); + src6 = __lsx_vld(src + src_stride_2x, 0); + src += (8 - src_stride_4x); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + + dst0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7); + + dst3 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst4 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst5 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dst6 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + + LSX_DUP4_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst5, dst4, dst6, dst5, + dst32_r, dst43_r, dst54_r, dst65_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst5, dst4, dst6, dst5, + dst32_l, dst43_l, dst54_l, dst65_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + + LSX_DUP4_ARG3(__lsx_vsrani_h_w, dst0_l, dst0_r, 6, dst1_l, dst1_r, 6, dst2_l, + dst2_r, 6, dst3_l, dst3_r, 6, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 6, tmp3, tmp2, 6, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_3x, 0, 1); + dst += 8; + } +} + +static av_always_inline +void hevc_hv_uni_4t_8x6_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y) +{ + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i out0, out1, out2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i filt0, filt1; + __m128i filt_h0, filt_h1, filter_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9; + __m128i vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i dst4_r, dst4_l, dst5_r, dst5_l; + __m128i dst10_r, dst32_r, dst10_l, dst32_l; + __m128i dst21_r, dst43_r, dst21_l, dst43_l; + __m128i dst54_r, dst54_l, dst65_r, dst65_l; + __m128i dst76_r, dst76_l, dst87_r, dst87_l; + __m128i out0_r, out1_r, out2_r, out3_r, out4_r, out5_r; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask1 = __lsx_vaddi_bu(mask0, 2); + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src4 = __lsx_vld(src + src_stride_4x, 0); + src += (src_stride_4x + src_stride); + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src5, src6, src7, src8); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, + src2, src3) + src4 = __lsx_vxori_b(src4, 128); + LSX_DUP4_ARG2(__lsx_vxori_b, src5, 128, src6, 128, src7, 128, src8, 128, src5, src6, + src7, src8); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src1, src1, + mask0, src1, src1, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src3, src3, + mask0, src3, src3, mask1, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src5, src5, + mask0, src5, src5, mask1, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src7, src7, + mask0, src7, src7, mask1, vec12, vec13, vec14, vec15); + LSX_DUP2_ARG3(__lsx_vshuf_b, src8, src8, mask0, src8, src8, mask1, vec16, vec17); + + dst0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dst3 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + dst4 = __lsx_hevc_filt_4tap_h(vec8, vec9, filt0, filt1); + dst5 = __lsx_hevc_filt_4tap_h(vec10, vec11, filt0, filt1); + dst6 = __lsx_hevc_filt_4tap_h(vec12, vec13, filt0, filt1); + dst7 = __lsx_hevc_filt_4tap_h(vec14, vec15, filt0, filt1); + dst8 = __lsx_hevc_filt_4tap_h(vec16, vec17, filt0, filt1); + + LSX_DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3, + dst10_r, dst21_r, dst32_r, dst43_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3, + dst10_l, dst21_l, dst32_l, dst43_l); + LSX_DUP4_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7, + dst54_r, dst65_r, dst76_r, dst87_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7, + dst54_l, dst65_l, dst76_l, dst87_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + dst4_r = __lsx_hevc_filt_4tap_w(dst54_r, dst76_r, filt_h0, filt_h1); + dst4_l = __lsx_hevc_filt_4tap_w(dst54_l, dst76_l, filt_h0, filt_h1); + dst5_r = __lsx_hevc_filt_4tap_w(dst65_r, dst87_r, filt_h0, filt_h1); + dst5_l = __lsx_hevc_filt_4tap_w(dst65_l, dst87_l, filt_h0, filt_h1); + + LSX_DUP4_ARG3(__lsx_vsrani_h_w, dst0_l, dst0_r, 6, dst1_l, dst1_r, 6, dst2_l, + dst2_r, 6, dst3_l, dst3_r, 6, out0_r, out1_r, out2_r, out3_r); + LSX_DUP2_ARG3(__lsx_vsrani_h_w, dst4_l, dst4_r, 6, dst5_l, dst5_r, 6, + out4_r, out5_r); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1_r, out0_r, 6, out3_r, out2_r, 6, + out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1) + out2 = __lsx_vssrarni_b_h(out5_r, out4_r, 6); + out2 = __lsx_vxori_b(out2, 128); + + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + __lsx_vstelm_d(out2, dst, 0, 0); + __lsx_vstelm_d(out2, dst + dst_stride, 0, 1); +} + +static av_always_inline +void hevc_hv_uni_4t_8multx4mult_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t width8mult) +{ + uint32_t loop_cnt, cnt; + uint8_t *src_tmp; + uint8_t *dst_tmp; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i out0, out1; + __m128i src0, src1, src2, src3, src4, src5, src6; + __m128i filt0, filt1; + __m128i filt_h0, filt_h1, filter_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i dst10_r, dst32_r, dst21_r, dst43_r; + __m128i dst10_l, dst32_l, dst21_l, dst43_l; + __m128i dst54_r, dst54_l, dst65_r, dst65_l, dst6; + __m128i out0_r, out1_r, out2_r, out3_r; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + mask1 = __lsx_vaddi_bu(mask0, 2); + + for (cnt = width8mult; cnt--;) { + src_tmp = src; + dst_tmp = dst; + + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src0, src1); + src2 = __lsx_vld(src_tmp + src_stride_2x, 0); + src_tmp += src_stride_3x; + + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + + dst0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, src3, + src4, src5, src6); + src_tmp += src_stride_4x; + + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, + src3, src4, src5, src6); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src4, + src4, mask0, src4, src4, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src6, + src6, mask0, src6, src6, mask1, vec4, vec5, vec6, vec7); + + dst3 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst4 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst5 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dst6 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + + LSX_DUP4_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst5, dst4, dst6, dst5, + dst32_r, dst43_r, dst54_r, dst65_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst5, dst4, dst6, dst5, + dst32_l, dst43_l, dst54_l, dst65_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + + LSX_DUP4_ARG3(__lsx_vsrani_h_w, dst0_l, dst0_r, 6, dst1_l, dst1_r, 6, + dst2_l, dst2_r, 6, dst3_l, dst3_r, 6, out0_r, out1_r, + out2_r, out3_r); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1_r, out0_r, 6, out3_r, out2_r, 6, + out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst_tmp, 0, 0); + __lsx_vstelm_d(out0, dst_tmp + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst_tmp + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst_tmp + dst_stride_3x, 0, 1); + dst_tmp += dst_stride_4x; + + dst10_r = dst54_r; + dst10_l = dst54_l; + dst21_r = dst65_r; + dst21_l = dst65_l; + dst2 = dst6; + } + + src += 8; + dst += 8; + } +} + +static void hevc_hv_uni_4t_8w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + if (2 == height) { + hevc_hv_uni_4t_8x2_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y); + } else if (4 == height) { + hevc_hv_uni_4t_8multx4_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, 1); + } else if (6 == height) { + hevc_hv_uni_4t_8x6_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y); + } else if (0 == (height & 0x03)) { + hevc_hv_uni_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 1); + } +} + +static av_always_inline +void hevc_hv_uni_4t_12w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + uint32_t loop_cnt; + uint8_t *src_tmp, *dst_tmp; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t dst_stride_4x = (dst_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + const int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i out0, out1; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i mask0, mask1, mask2, mask3; + __m128i filt0, filt1, filt_h0, filt_h1, filter_vec, tmp0, tmp1, tmp2, tmp3; + __m128i dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6; + __m128i dst10, dst21, dst22, dst73, dst84, dst95, dst106; + __m128i dst76_r, dst98_r, dst87_r, dst109_r; + __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r; + __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + mask1 = __lsx_vaddi_bu(mask0, 2); + + src_tmp = src; + dst_tmp = dst; + + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src0, src1); + src2 = __lsx_vld(src_tmp + src_stride_2x, 0); + src_tmp += src_stride_3x; + + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + + dsth0 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dsth1 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dsth2 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dsth1, dsth0, dsth2, dsth1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dsth1, dsth0, dsth2, dsth1, dst10_l, dst21_l); + + for (loop_cnt = 4; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, + src3, src4, src5, src6); + src_tmp += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, + src3, src4, src5, src6); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src4, src4, + mask0, src4, src4, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src6, src6, + mask0, src6, src6, mask1, vec4, vec5, vec6, vec7); + + dsth3 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dsth4 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dsth5 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dsth6 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + + LSX_DUP4_ARG2(__lsx_vilvl_h, dsth3, dsth2, dsth4, dsth3, dsth5, dsth4, dsth6, + dsth5, dst32_r, dst43_r, dst54_r, dst65_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dsth3, dsth2, dsth4, dsth3, dsth5, dsth4, dsth6, + dsth5, dst32_l, dst43_l, dst54_l, dst65_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + + LSX_DUP4_ARG3(__lsx_vsrani_h_w, dst0_l, dst0_r, 6, dst1_l, dst1_r, 6, dst2_l, + dst2_r, 6, dst3_l, dst3_r, 6, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 6, tmp3, tmp2, 6, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + + __lsx_vstelm_d(out0, dst_tmp, 0, 0); + __lsx_vstelm_d(out0, dst_tmp + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst_tmp + dst_stride_2x, 0, 0); + __lsx_vstelm_d(out1, dst_tmp + dst_stride_3x, 0, 1); + dst_tmp += dst_stride_4x; + + dst10_r = dst54_r; + dst10_l = dst54_l; + dst21_r = dst65_r; + dst21_l = dst65_l; + dsth2 = dsth6; + } + + src += 8; + dst += 8; + + mask2 = __lsx_vld(ff_hevc_mask_arr, 16); + mask3 = __lsx_vaddi_bu(mask2, 2); + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src2 = __lsx_vld(src + src_stride_2x, 0); + src += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask2, src1, src0, mask3, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src1, mask2, src2, src1, mask3, vec2, vec3); + + dst10 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst21 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + + dst10_r = __lsx_vilvl_h(dst21, dst10); + dst21_r = __lsx_vilvh_h(dst21, dst10); + dst22 = __lsx_vreplvei_d(dst21, 1); + + for (loop_cnt = 2; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src3, src4, src5, src6); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src7, src8, src9, src10); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6) + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10) + LSX_DUP4_ARG3(__lsx_vshuf_b, src7, src3, mask2, src7, src3, mask3, src8, src4, + mask2, src8, src4, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src9, src5, mask2, src9, src5, mask3, src10, src6, + mask2, src10, src6, mask3, vec4, vec5, vec6, vec7); + + dst73 = __lsx_hevc_filt_4tap_h(vec0, vec1, filt0, filt1); + dst84 = __lsx_hevc_filt_4tap_h(vec2, vec3, filt0, filt1); + dst95 = __lsx_hevc_filt_4tap_h(vec4, vec5, filt0, filt1); + dst106 = __lsx_hevc_filt_4tap_h(vec6, vec7, filt0, filt1); + + dst32_r = __lsx_vilvl_h(dst73, dst22); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst84, dst73, dst95, dst84, dst43_r, dst54_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst84, dst73, dst95, dst84, dst87_r, dst98_r); + dst65_r = __lsx_vilvl_h(dst106, dst95); + dst109_r = __lsx_vilvh_h(dst106, dst95); + dst22 = __lsx_vreplvei_d(dst73, 1); + dst76_r = __lsx_vilvl_h(dst22, dst106); + + dst0 = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst1 = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst2 = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst3 = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst4 = __lsx_hevc_filt_4tap_w(dst54_r, dst76_r, filt_h0, filt_h1); + dst5 = __lsx_hevc_filt_4tap_w(dst65_r, dst87_r, filt_h0, filt_h1); + dst6 = __lsx_hevc_filt_4tap_w(dst76_r, dst98_r, filt_h0, filt_h1); + dst7 = __lsx_hevc_filt_4tap_w(dst87_r, dst109_r, filt_h0, filt_h1); + + LSX_DUP4_ARG3(__lsx_vsrani_h_w, dst1, dst0, 6, dst3, dst2, 6, dst5, dst4, 6, + dst7, dst6, 6, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 6, tmp3, tmp2, 6, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + + __lsx_vstelm_w(out0, dst, 0, 0); + __lsx_vstelm_w(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(out0, dst + dst_stride_2x, 0, 2); + __lsx_vstelm_w(out0, dst + dst_stride_3x, 0, 3); + dst += dst_stride_4x; + __lsx_vstelm_w(out1, dst, 0, 0); + __lsx_vstelm_w(out1, dst + dst_stride, 0, 1); + __lsx_vstelm_w(out1, dst + dst_stride_2x, 0, 2); + __lsx_vstelm_w(out1, dst + dst_stride_3x, 0, 3); + dst += dst_stride_4x; + + dst10_r = dst98_r; + dst21_r = dst109_r; + dst22 = __lsx_vreplvei_d(dst106, 1); + } +} + +static void hevc_hv_uni_4t_16w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + if (4 == height) { + hevc_hv_uni_4t_8multx4_lsx(src, src_stride, dst, dst_stride, filter_x, + filter_y, 2); + } else { + hevc_hv_uni_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 2); + } +} + +static void hevc_hv_uni_4t_24w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_uni_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 3); +} + +static void hevc_hv_uni_4t_32w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_uni_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 4); +} + +#define UNI_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \ +void ff_hevc_put_hevc_uni_##PEL##_##DIR##WIDTH##_8_lsx(uint8_t *dst, \ + ptrdiff_t dst_stride, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) \ +{ \ + const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR - 1]; \ + \ + common_##DIR1##_##TAP##t_##WIDTH##w_lsx(src, src_stride, dst, dst_stride, \ + filter, height); \ +} + +UNI_MC(qpel, h, 64, 8, hz, mx); + +UNI_MC(qpel, v, 24, 8, vt, my); +UNI_MC(qpel, v, 32, 8, vt, my); +UNI_MC(qpel, v, 48, 8, vt, my); +UNI_MC(qpel, v, 64, 8, vt, my); + +UNI_MC(epel, v, 24, 4, vt, my); +UNI_MC(epel, v, 32, 4, vt, my); + +#undef UNI_MC + +#define UNI_MC_HV(PEL, WIDTH, TAP) \ +void ff_hevc_put_hevc_uni_##PEL##_hv##WIDTH##_8_lsx(uint8_t *dst, \ + ptrdiff_t dst_stride, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) \ +{ \ + const int8_t *filter_x = ff_hevc_##PEL##_filters[mx - 1]; \ + const int8_t *filter_y = ff_hevc_##PEL##_filters[my - 1]; \ + \ + hevc_hv_uni_##TAP##t_##WIDTH##w_lsx(src, src_stride, dst, dst_stride, \ + filter_x, filter_y, height); \ +} + +UNI_MC_HV(qpel, 8, 8); +UNI_MC_HV(qpel, 16, 8); +UNI_MC_HV(qpel, 24, 8); +UNI_MC_HV(qpel, 32, 8); +UNI_MC_HV(qpel, 48, 8); +UNI_MC_HV(qpel, 64, 8); + +UNI_MC_HV(epel, 8, 4); +UNI_MC_HV(epel, 12, 4); +UNI_MC_HV(epel, 16, 4); +UNI_MC_HV(epel, 24, 4); +UNI_MC_HV(epel, 32, 4); + +#undef UNI_MC_HV diff --git a/libavcodec/loongarch/hevc_mc_uniw_lsx.c b/libavcodec/loongarch/hevc_mc_uniw_lsx.c new file mode 100644 index 0000000000..619ef95960 --- /dev/null +++ b/libavcodec/loongarch/hevc_mc_uniw_lsx.c @@ -0,0 +1,338 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "hevcdsp_lsx.h" +#include "hevc_macros_lsx.h" + +static const uint8_t ff_hevc_mask_arr[16 * 2] __attribute__((aligned(0x40))) = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 +}; + +static av_always_inline +void hevc_hv_uniwgt_8t_8multx2mult_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t weight, + int32_t offset, + int32_t rnd_val, + int32_t width) +{ + uint32_t loop_cnt, cnt; + uint8_t *src_tmp; + uint8_t *dst_tmp; + const int32_t src_stride_2x = (src_stride << 1); + const int32_t dst_stride_2x = (dst_stride << 1); + const int32_t src_stride_4x = (src_stride << 2); + const int32_t src_stride_3x = src_stride_2x + src_stride; + + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i filt0, filt1, filt2, filt3; + __m128i filt_h0, filt_h1, filt_h2, filt_h3; + __m128i mask1, mask2, mask3; + __m128i filter_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8; + __m128i dst0_r, dst0_l, dst1_r, dst1_l; + __m128i dst10_r, dst32_r, dst54_r, dst76_r; + __m128i dst10_l, dst32_l, dst54_l, dst76_l; + __m128i dst21_r, dst43_r, dst65_r, dst87_r; + __m128i dst21_l, dst43_l, dst65_l, dst87_l; + __m128i weight_vec, offset_vec, rnd_vec, const_128, denom_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src -= (src_stride_3x + 3); + weight_vec = __lsx_vreplgr2vr_w(weight); + offset_vec = __lsx_vreplgr2vr_w(offset); + rnd_vec = __lsx_vreplgr2vr_w(rnd_val); + denom_vec = __lsx_vsubi_wu(rnd_vec, 6); + + const_128 = __lsx_vldi(0x880); + const_128 = __lsx_vmul_w(weight_vec, const_128); + denom_vec = __lsx_vsrar_w(const_128, denom_vec); + offset_vec = __lsx_vadd_w(denom_vec, offset_vec); + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4, filter_x, 6, + filt0, filt1, filt2, filt3); + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP4_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filter_vec, 2, + filter_vec, 3, filt_h0, filt_h1, filt_h2, filt_h3); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + for (cnt = width >> 3; cnt--;) { + src_tmp = src; + dst_tmp = dst; + + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, src0, + src1, src2, src3); + src_tmp += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src4, src5); + src6 = __lsx_vld(src_tmp + src_stride_2x, 0); + src_tmp += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0, src0, + mask2, src0, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1, + mask2, src1, src1, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2, + mask2, src2, src2, mask3, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3, src3, + mask2, src3, src3, mask3, vec12, vec13, vec14, vec15); + dst0 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, filt2, + filt3); + dst1 = __lsx_hevc_filt_8tap_h(vec4, vec5, vec6, vec7, filt0, filt1, filt2, + filt3); + dst2 = __lsx_hevc_filt_8tap_h(vec8, vec9, vec10, vec11, filt0, filt1, filt2, + filt3); + dst3 = __lsx_hevc_filt_8tap_h(vec12, vec13, vec14, vec15, filt0, filt1, + filt2, filt3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4, src4, + mask2, src4, src4, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src5, src5, + mask2, src5, src5, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src6, src6, + mask2, src6, src6, mask3, vec8, vec9, vec10, vec11); + dst4 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, filt2, + filt3); + dst5 = __lsx_hevc_filt_8tap_h(vec4, vec5, vec6, vec7, filt0, filt1, filt2, + filt3); + dst6 = __lsx_hevc_filt_8tap_h(vec8, vec9, vec10, vec11, filt0, filt1, filt2, + filt3); + + LSX_DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1, + dst10_r, dst32_r, dst54_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst4, dst3, dst6, dst5, dst43_r, dst65_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1, + dst10_l, dst32_l, dst54_l, dst21_l); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst4, dst3, dst6, dst5, dst43_l, dst65_l); + + for (loop_cnt = height >> 1; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src7, src8); + src_tmp += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src7, src8); + LSX_DUP4_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, src7, + src7, mask2, src7, src7, mask3, vec0, vec1, vec2, vec3); + dst7 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, + filt2, filt3); + dst76_r = __lsx_vilvl_h(dst7, dst6); + dst76_l = __lsx_vilvh_h(dst7, dst6); + dst0_r = __lsx_hevc_filt_8tap_w(dst10_r, dst32_r, dst54_r, dst76_r, + filt_h0, filt_h1, filt_h2, filt_h3); + dst0_l = __lsx_hevc_filt_8tap_w(dst10_l, dst32_l, dst54_l, dst76_l, + filt_h0, filt_h1, filt_h2, filt_h3); + LSX_DUP2_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst0_r, dst0_l); + + /* row 8 */ + LSX_DUP4_ARG3(__lsx_vshuf_b, src8, src8, mask0, src8, src8, mask1, src8, + src8, mask2, src8, src8, mask3, vec0, vec1, vec2, vec3); + dst8 = __lsx_hevc_filt_8tap_h(vec0, vec1, vec2, vec3, filt0, filt1, + filt2, filt3); + + dst87_r = __lsx_vilvl_h(dst8, dst7); + dst87_l = __lsx_vilvh_h(dst8, dst7); + dst1_r = __lsx_hevc_filt_8tap_w(dst21_r, dst43_r, dst65_r, dst87_r, + filt_h0, filt_h1, filt_h2, filt_h3); + dst1_l = __lsx_hevc_filt_8tap_w(dst21_l, dst43_l, dst65_l, dst87_l, + filt_h0, filt_h1, filt_h2, filt_h3); + LSX_DUP2_ARG2(__lsx_vsrai_w, dst1_r, 6, dst1_l, 6, dst1_r, dst1_l); + + LSX_DUP2_ARG2(__lsx_vmul_w, dst0_r, weight_vec, dst0_l, weight_vec, dst0_r, + dst0_l); + LSX_DUP2_ARG2(__lsx_vmul_w, dst1_r, weight_vec, dst1_l, weight_vec, dst1_r, + dst1_l); + LSX_DUP4_ARG2(__lsx_vsrar_w, dst0_r, rnd_vec, dst1_r, rnd_vec, dst0_l, + rnd_vec, dst1_l, rnd_vec, dst0_r, dst1_r, dst0_l, dst1_l); + + LSX_DUP2_ARG2(__lsx_vadd_w, dst0_r, offset_vec, dst0_l, offset_vec, dst0_r, + dst0_l); + LSX_DUP2_ARG2(__lsx_vadd_w, dst1_r, offset_vec, dst1_l, offset_vec, dst1_r, + dst1_l); + LSX_DUP4_ARG1(__lsx_clamp255_w, dst0_r, dst1_r, dst0_l, dst1_l, dst0_r, + dst1_r, dst0_l, dst1_l); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r); + dst0_r = __lsx_vpickev_b(dst1_r, dst0_r); + + __lsx_vstelm_d(dst0_r, dst_tmp, 0, 0); + __lsx_vstelm_d(dst0_r, dst_tmp + dst_stride, 0, 1); + dst_tmp += dst_stride_2x; + + dst10_r = dst32_r; + dst32_r = dst54_r; + dst54_r = dst76_r; + dst10_l = dst32_l; + dst32_l = dst54_l; + dst54_l = dst76_l; + dst21_r = dst43_r; + dst43_r = dst65_r; + dst65_r = dst87_r; + dst21_l = dst43_l; + dst43_l = dst65_l; + dst65_l = dst87_l; + dst6 = dst8; + } + + src += 8; + dst += 8; + } +} + +static void hevc_hv_uniwgt_8t_8w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t weight, + int32_t offset, + int32_t rnd_val) +{ + hevc_hv_uniwgt_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, weight, + offset, rnd_val, 8); +} + +static void hevc_hv_uniwgt_8t_16w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t weight, + int32_t offset, + int32_t rnd_val) +{ + hevc_hv_uniwgt_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, weight, + offset, rnd_val, 16); +} + +static void hevc_hv_uniwgt_8t_24w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t weight, + int32_t offset, + int32_t rnd_val) +{ + hevc_hv_uniwgt_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, weight, + offset, rnd_val, 24); +} + +static void hevc_hv_uniwgt_8t_32w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t weight, + int32_t offset, + int32_t rnd_val) +{ + hevc_hv_uniwgt_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, weight, + offset, rnd_val, 32); +} + +static void hevc_hv_uniwgt_8t_48w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t weight, + int32_t offset, + int32_t rnd_val) +{ + hevc_hv_uniwgt_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, weight, + offset, rnd_val, 48); +} + +static void hevc_hv_uniwgt_8t_64w_lsx(uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t weight, + int32_t offset, + int32_t rnd_val) +{ + hevc_hv_uniwgt_8t_8multx2mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, weight, + offset, rnd_val, 64); +} + + +#define UNI_W_MC_HV(PEL, WIDTH, TAP) \ +void ff_hevc_put_hevc_uni_w_##PEL##_hv##WIDTH##_8_lsx(uint8_t *dst, \ + ptrdiff_t dst_stride, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int height, \ + int denom, \ + int weight, \ + int offset, \ + intptr_t mx, \ + intptr_t my, \ + int width) \ +{ \ + const int8_t *filter_x = ff_hevc_##PEL##_filters[mx - 1]; \ + const int8_t *filter_y = ff_hevc_##PEL##_filters[my - 1]; \ + int shift = denom + 14 - 8; \ + \ + hevc_hv_uniwgt_##TAP##t_##WIDTH##w_lsx(src, src_stride, dst, dst_stride, \ + filter_x, filter_y, height, \ + weight, offset, shift); \ +} + +UNI_W_MC_HV(qpel, 8, 8); +UNI_W_MC_HV(qpel, 16, 8); +UNI_W_MC_HV(qpel, 24, 8); +UNI_W_MC_HV(qpel, 32, 8); +UNI_W_MC_HV(qpel, 48, 8); +UNI_W_MC_HV(qpel, 64, 8); + +#undef UNI_W_MC_HV diff --git a/libavcodec/loongarch/hevcdsp_init_loongarch.c b/libavcodec/loongarch/hevcdsp_init_loongarch.c new file mode 100644 index 0000000000..94e650d997 --- /dev/null +++ b/libavcodec/loongarch/hevcdsp_init_loongarch.c @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/cpu.h" +#include "libavcodec/loongarch/hevcdsp_lsx.h" + +void ff_hevc_dsp_init_loongarch(HEVCDSPContext *c, const int bit_depth) +{ + int cpu_flags = av_get_cpu_flags(); + + if (have_lsx(cpu_flags)) { + if (bit_depth == 8) { + c->put_hevc_qpel[1][0][0] = ff_hevc_put_hevc_pel_pixels4_8_lsx; + c->put_hevc_qpel[2][0][0] = ff_hevc_put_hevc_pel_pixels6_8_lsx; + c->put_hevc_qpel[3][0][0] = ff_hevc_put_hevc_pel_pixels8_8_lsx; + c->put_hevc_qpel[4][0][0] = ff_hevc_put_hevc_pel_pixels12_8_lsx; + c->put_hevc_qpel[5][0][0] = ff_hevc_put_hevc_pel_pixels16_8_lsx; + c->put_hevc_qpel[6][0][0] = ff_hevc_put_hevc_pel_pixels24_8_lsx; + c->put_hevc_qpel[7][0][0] = ff_hevc_put_hevc_pel_pixels32_8_lsx; + c->put_hevc_qpel[8][0][0] = ff_hevc_put_hevc_pel_pixels48_8_lsx; + c->put_hevc_qpel[9][0][0] = ff_hevc_put_hevc_pel_pixels64_8_lsx; + + c->put_hevc_epel[1][0][0] = ff_hevc_put_hevc_pel_pixels4_8_lsx; + c->put_hevc_epel[2][0][0] = ff_hevc_put_hevc_pel_pixels6_8_lsx; + c->put_hevc_epel[3][0][0] = ff_hevc_put_hevc_pel_pixels8_8_lsx; + c->put_hevc_epel[4][0][0] = ff_hevc_put_hevc_pel_pixels12_8_lsx; + c->put_hevc_epel[5][0][0] = ff_hevc_put_hevc_pel_pixels16_8_lsx; + c->put_hevc_epel[6][0][0] = ff_hevc_put_hevc_pel_pixels24_8_lsx; + c->put_hevc_epel[7][0][0] = ff_hevc_put_hevc_pel_pixels32_8_lsx; + + c->put_hevc_qpel[1][0][1] = ff_hevc_put_hevc_qpel_h4_8_lsx; + c->put_hevc_qpel[3][0][1] = ff_hevc_put_hevc_qpel_h8_8_lsx; + c->put_hevc_qpel[4][0][1] = ff_hevc_put_hevc_qpel_h12_8_lsx; + c->put_hevc_qpel[5][0][1] = ff_hevc_put_hevc_qpel_h16_8_lsx; + c->put_hevc_qpel[6][0][1] = ff_hevc_put_hevc_qpel_h24_8_lsx; + c->put_hevc_qpel[7][0][1] = ff_hevc_put_hevc_qpel_h32_8_lsx; + c->put_hevc_qpel[8][0][1] = ff_hevc_put_hevc_qpel_h48_8_lsx; + c->put_hevc_qpel[9][0][1] = ff_hevc_put_hevc_qpel_h64_8_lsx; + + c->put_hevc_qpel[1][1][0] = ff_hevc_put_hevc_qpel_v4_8_lsx; + c->put_hevc_qpel[3][1][0] = ff_hevc_put_hevc_qpel_v8_8_lsx; + c->put_hevc_qpel[4][1][0] = ff_hevc_put_hevc_qpel_v12_8_lsx; + c->put_hevc_qpel[5][1][0] = ff_hevc_put_hevc_qpel_v16_8_lsx; + c->put_hevc_qpel[6][1][0] = ff_hevc_put_hevc_qpel_v24_8_lsx; + c->put_hevc_qpel[7][1][0] = ff_hevc_put_hevc_qpel_v32_8_lsx; + c->put_hevc_qpel[8][1][0] = ff_hevc_put_hevc_qpel_v48_8_lsx; + c->put_hevc_qpel[9][1][0] = ff_hevc_put_hevc_qpel_v64_8_lsx; + + c->put_hevc_qpel[1][1][1] = ff_hevc_put_hevc_qpel_hv4_8_lsx; + c->put_hevc_qpel[3][1][1] = ff_hevc_put_hevc_qpel_hv8_8_lsx; + c->put_hevc_qpel[4][1][1] = ff_hevc_put_hevc_qpel_hv12_8_lsx; + c->put_hevc_qpel[5][1][1] = ff_hevc_put_hevc_qpel_hv16_8_lsx; + c->put_hevc_qpel[6][1][1] = ff_hevc_put_hevc_qpel_hv24_8_lsx; + c->put_hevc_qpel[7][1][1] = ff_hevc_put_hevc_qpel_hv32_8_lsx; + c->put_hevc_qpel[8][1][1] = ff_hevc_put_hevc_qpel_hv48_8_lsx; + c->put_hevc_qpel[9][1][1] = ff_hevc_put_hevc_qpel_hv64_8_lsx; + + c->put_hevc_epel[7][0][1] = ff_hevc_put_hevc_epel_h32_8_lsx; + + c->put_hevc_epel[5][1][0] = ff_hevc_put_hevc_epel_v16_8_lsx; + c->put_hevc_epel[6][1][0] = ff_hevc_put_hevc_epel_v24_8_lsx; + c->put_hevc_epel[7][1][0] = ff_hevc_put_hevc_epel_v32_8_lsx; + + c->put_hevc_epel[3][1][1] = ff_hevc_put_hevc_epel_hv8_8_lsx; + c->put_hevc_epel[4][1][1] = ff_hevc_put_hevc_epel_hv12_8_lsx; + c->put_hevc_epel[5][1][1] = ff_hevc_put_hevc_epel_hv16_8_lsx; + c->put_hevc_epel[6][1][1] = ff_hevc_put_hevc_epel_hv24_8_lsx; + c->put_hevc_epel[7][1][1] = ff_hevc_put_hevc_epel_hv32_8_lsx; + + c->put_hevc_qpel_uni[9][0][1] = ff_hevc_put_hevc_uni_qpel_h64_8_lsx; + + c->put_hevc_qpel_uni[6][1][0] = ff_hevc_put_hevc_uni_qpel_v24_8_lsx; + c->put_hevc_qpel_uni[7][1][0] = ff_hevc_put_hevc_uni_qpel_v32_8_lsx; + c->put_hevc_qpel_uni[8][1][0] = ff_hevc_put_hevc_uni_qpel_v48_8_lsx; + c->put_hevc_qpel_uni[9][1][0] = ff_hevc_put_hevc_uni_qpel_v64_8_lsx; + + c->put_hevc_qpel_uni[3][1][1] = ff_hevc_put_hevc_uni_qpel_hv8_8_lsx; + c->put_hevc_qpel_uni[5][1][1] = ff_hevc_put_hevc_uni_qpel_hv16_8_lsx; + c->put_hevc_qpel_uni[6][1][1] = ff_hevc_put_hevc_uni_qpel_hv24_8_lsx; + c->put_hevc_qpel_uni[7][1][1] = ff_hevc_put_hevc_uni_qpel_hv32_8_lsx; + c->put_hevc_qpel_uni[8][1][1] = ff_hevc_put_hevc_uni_qpel_hv48_8_lsx; + c->put_hevc_qpel_uni[9][1][1] = ff_hevc_put_hevc_uni_qpel_hv64_8_lsx; + + c->put_hevc_epel_uni[6][1][0] = ff_hevc_put_hevc_uni_epel_v24_8_lsx; + c->put_hevc_epel_uni[7][1][0] = ff_hevc_put_hevc_uni_epel_v32_8_lsx; + + c->put_hevc_epel_uni[3][1][1] = ff_hevc_put_hevc_uni_epel_hv8_8_lsx; + c->put_hevc_epel_uni[4][1][1] = ff_hevc_put_hevc_uni_epel_hv12_8_lsx; + c->put_hevc_epel_uni[5][1][1] = ff_hevc_put_hevc_uni_epel_hv16_8_lsx; + c->put_hevc_epel_uni[6][1][1] = ff_hevc_put_hevc_uni_epel_hv24_8_lsx; + c->put_hevc_epel_uni[7][1][1] = ff_hevc_put_hevc_uni_epel_hv32_8_lsx; + + c->put_hevc_qpel_uni_w[3][1][1] = ff_hevc_put_hevc_uni_w_qpel_hv8_8_lsx; + c->put_hevc_qpel_uni_w[5][1][1] = ff_hevc_put_hevc_uni_w_qpel_hv16_8_lsx; + c->put_hevc_qpel_uni_w[6][1][1] = ff_hevc_put_hevc_uni_w_qpel_hv24_8_lsx; + c->put_hevc_qpel_uni_w[7][1][1] = ff_hevc_put_hevc_uni_w_qpel_hv32_8_lsx; + c->put_hevc_qpel_uni_w[8][1][1] = ff_hevc_put_hevc_uni_w_qpel_hv48_8_lsx; + c->put_hevc_qpel_uni_w[9][1][1] = ff_hevc_put_hevc_uni_w_qpel_hv64_8_lsx; + + c->put_hevc_qpel_bi[1][0][0] = ff_hevc_put_hevc_bi_pel_pixels4_8_lsx; + c->put_hevc_qpel_bi[3][0][0] = ff_hevc_put_hevc_bi_pel_pixels8_8_lsx; + c->put_hevc_qpel_bi[4][0][0] = ff_hevc_put_hevc_bi_pel_pixels12_8_lsx; + c->put_hevc_qpel_bi[5][0][0] = ff_hevc_put_hevc_bi_pel_pixels16_8_lsx; + c->put_hevc_qpel_bi[6][0][0] = ff_hevc_put_hevc_bi_pel_pixels24_8_lsx; + c->put_hevc_qpel_bi[7][0][0] = ff_hevc_put_hevc_bi_pel_pixels32_8_lsx; + c->put_hevc_qpel_bi[8][0][0] = ff_hevc_put_hevc_bi_pel_pixels48_8_lsx; + c->put_hevc_qpel_bi[9][0][0] = ff_hevc_put_hevc_bi_pel_pixels64_8_lsx; + + c->put_hevc_epel_bi[1][0][0] = ff_hevc_put_hevc_bi_pel_pixels4_8_lsx; + c->put_hevc_epel_bi[2][0][0] = ff_hevc_put_hevc_bi_pel_pixels6_8_lsx; + c->put_hevc_epel_bi[3][0][0] = ff_hevc_put_hevc_bi_pel_pixels8_8_lsx; + c->put_hevc_epel_bi[4][0][0] = ff_hevc_put_hevc_bi_pel_pixels12_8_lsx; + c->put_hevc_epel_bi[5][0][0] = ff_hevc_put_hevc_bi_pel_pixels16_8_lsx; + c->put_hevc_epel_bi[6][0][0] = ff_hevc_put_hevc_bi_pel_pixels24_8_lsx; + c->put_hevc_epel_bi[7][0][0] = ff_hevc_put_hevc_bi_pel_pixels32_8_lsx; + + c->put_hevc_qpel_bi[3][1][0] = ff_hevc_put_hevc_bi_qpel_v8_8_lsx; + c->put_hevc_qpel_bi[5][1][0] = ff_hevc_put_hevc_bi_qpel_v16_8_lsx; + c->put_hevc_qpel_bi[6][1][0] = ff_hevc_put_hevc_bi_qpel_v24_8_lsx; + c->put_hevc_qpel_bi[7][1][0] = ff_hevc_put_hevc_bi_qpel_v32_8_lsx; + c->put_hevc_qpel_bi[8][1][0] = ff_hevc_put_hevc_bi_qpel_v48_8_lsx; + c->put_hevc_qpel_bi[9][1][0] = ff_hevc_put_hevc_bi_qpel_v64_8_lsx; + + c->put_hevc_qpel_bi[3][1][1] = ff_hevc_put_hevc_bi_qpel_hv8_8_lsx; + c->put_hevc_qpel_bi[5][1][1] = ff_hevc_put_hevc_bi_qpel_hv16_8_lsx; + c->put_hevc_qpel_bi[6][1][1] = ff_hevc_put_hevc_bi_qpel_hv24_8_lsx; + c->put_hevc_qpel_bi[7][1][1] = ff_hevc_put_hevc_bi_qpel_hv32_8_lsx; + c->put_hevc_qpel_bi[8][1][1] = ff_hevc_put_hevc_bi_qpel_hv48_8_lsx; + c->put_hevc_qpel_bi[9][1][1] = ff_hevc_put_hevc_bi_qpel_hv64_8_lsx; + + c->put_hevc_qpel_bi[5][0][1] = ff_hevc_put_hevc_bi_qpel_h16_8_lsx; + c->put_hevc_qpel_bi[6][0][1] = ff_hevc_put_hevc_bi_qpel_h24_8_lsx; + c->put_hevc_qpel_bi[7][0][1] = ff_hevc_put_hevc_bi_qpel_h32_8_lsx; + c->put_hevc_qpel_bi[8][0][1] = ff_hevc_put_hevc_bi_qpel_h48_8_lsx; + c->put_hevc_qpel_bi[9][0][1] = ff_hevc_put_hevc_bi_qpel_h64_8_lsx; + + c->put_hevc_epel_bi[6][0][1] = ff_hevc_put_hevc_bi_epel_h24_8_lsx; + c->put_hevc_epel_bi[7][0][1] = ff_hevc_put_hevc_bi_epel_h32_8_lsx; + + c->put_hevc_epel_bi[4][1][0] = ff_hevc_put_hevc_bi_epel_v12_8_lsx; + c->put_hevc_epel_bi[5][1][0] = ff_hevc_put_hevc_bi_epel_v16_8_lsx; + c->put_hevc_epel_bi[6][1][0] = ff_hevc_put_hevc_bi_epel_v24_8_lsx; + c->put_hevc_epel_bi[7][1][0] = ff_hevc_put_hevc_bi_epel_v32_8_lsx; + + c->put_hevc_epel_bi[2][1][1] = ff_hevc_put_hevc_bi_epel_hv6_8_lsx; + c->put_hevc_epel_bi[3][1][1] = ff_hevc_put_hevc_bi_epel_hv8_8_lsx; + c->put_hevc_epel_bi[5][1][1] = ff_hevc_put_hevc_bi_epel_hv16_8_lsx; + c->put_hevc_epel_bi[6][1][1] = ff_hevc_put_hevc_bi_epel_hv24_8_lsx; + c->put_hevc_epel_bi[7][1][1] = ff_hevc_put_hevc_bi_epel_hv32_8_lsx; + + c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_8_lsx; + + c->hevc_h_loop_filter_luma = ff_hevc_loop_filter_luma_h_8_lsx; + c->hevc_v_loop_filter_luma = ff_hevc_loop_filter_luma_v_8_lsx; + + c->hevc_h_loop_filter_luma_c = ff_hevc_loop_filter_luma_h_8_lsx; + c->hevc_v_loop_filter_luma_c = ff_hevc_loop_filter_luma_v_8_lsx; + + c->hevc_h_loop_filter_chroma = ff_hevc_loop_filter_chroma_h_8_lsx; + c->hevc_v_loop_filter_chroma = ff_hevc_loop_filter_chroma_v_8_lsx; + + c->hevc_h_loop_filter_chroma_c = ff_hevc_loop_filter_chroma_h_8_lsx; + c->hevc_v_loop_filter_chroma_c = ff_hevc_loop_filter_chroma_v_8_lsx; + + c->idct[0] = ff_hevc_idct_4x4_lsx; + c->idct[1] = ff_hevc_idct_8x8_lsx; + c->idct[2] = ff_hevc_idct_16x16_lsx; + c->idct[3] = ff_hevc_idct_32x32_lsx; + } + } +} diff --git a/libavcodec/loongarch/hevcdsp_lsx.c b/libavcodec/loongarch/hevcdsp_lsx.c new file mode 100644 index 0000000000..9e2d9e29b4 --- /dev/null +++ b/libavcodec/loongarch/hevcdsp_lsx.c @@ -0,0 +1,3338 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "hevcdsp_lsx.h" +#include "hevc_macros_lsx.h" + +static const uint8_t ff_hevc_mask_arr[16 * 2] __attribute__((aligned(0x40))) = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 +}; + +static void hevc_copy_4w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + int32_t height) +{ + __m128i zero = {0}; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + if (2 == height) { + __m128i src0, src1; + __m128i in0; + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src0 = __lsx_vilvl_w(src1, src0); + in0 = __lsx_vilvl_b(zero, src0); + in0 = __lsx_vslli_h(in0, 6); + __lsx_vstelm_d(in0, dst, 0, 0); + __lsx_vstelm_d(in0, dst + dst_stride, 0, 1); + } else if (4 == height) { + __m128i src0, src1, src2, src3; + __m128i in0, in1; + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vilvl_w, src1, src0, src3, src2, src0, src1); + LSX_DUP2_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, in0, in1); + LSX_DUP2_ARG2(__lsx_vslli_h, in0, 6, in1, 6, in0, in1); + __lsx_vstelm_d(in0, dst, 0, 0); + __lsx_vstelm_d(in0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(in1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(in1, dst + dst_stride_3x, 0, 1); + } else if (0 == (height & 0x07)) { + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0, in1, in2, in3; + uint32_t loop_cnt; + for (loop_cnt = (height >> 3); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, + src + src_stride_2x, 0, src + src_stride_3x, 0, + src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0,src + src_stride_2x, + 0, src + src_stride_3x, 0, src4, src5, src6, src7); + src += src_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_w, src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vslli_h, in0, 6, in1, 6, in2, 6, in3, 6, in0, in1, + in2, in3); + + __lsx_vstelm_d(in0, dst, 0, 0); + __lsx_vstelm_d(in0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(in1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(in1, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + __lsx_vstelm_d(in2, dst, 0, 0); + __lsx_vstelm_d(in2, dst + dst_stride, 0, 1); + __lsx_vstelm_d(in3, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(in3, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + } + } +} + +static void hevc_copy_6w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + uint64_t out0_m, out1_m, out2_m, out3_m; + uint64_t out4_m, out5_m, out6_m, out7_m; + uint32_t out8_m, out9_m, out10_m, out11_m; + uint32_t out12_m, out13_m, out14_m, out15_m; + int32_t src_stride_2x = (src_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + + __m128i zero = {0}; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src4, src5, src6, src7); + src += src_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, zero, src6, zero, src7, + in4, in5, in6, in7); + LSX_DUP4_ARG2(__lsx_vslli_h, in0, 6, in1, 6, in2, 6, in3, 6, in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vslli_h, in4, 6, in5, 6, in6, 6, in7, 6, in4, in5, in6, in7); + + LSX_DUP4_ARG2(__lsx_vpickve2gr_du, in0, 0, in1, 0, in2, 0, in3, 0, out0_m, + out1_m, out2_m, out3_m); + LSX_DUP4_ARG2(__lsx_vpickve2gr_du, in4, 0, in5, 0, in6, 0, in7, 0, out4_m, + out5_m, out6_m, out7_m); + + LSX_DUP4_ARG2(__lsx_vpickve2gr_wu, in0, 2, in1, 2, in2, 2, in3, 2, out8_m, + out9_m, out10_m, out11_m); + LSX_DUP4_ARG2(__lsx_vpickve2gr_wu, in4, 2, in5, 2, in6, 2, in7, 2, out12_m, + out13_m, out14_m, out15_m); + + *(uint64_t *)dst = out0_m; + *(uint32_t *)(dst + 4) = out8_m; + dst += dst_stride; + *(uint64_t *)dst = out1_m; + *(uint32_t *)(dst + 4) = out9_m; + dst += dst_stride; + *(uint64_t *)dst = out2_m; + *(uint32_t *)(dst + 4) = out10_m; + dst += dst_stride; + *(uint64_t *)dst = out3_m; + *(uint32_t *)(dst + 4) = out11_m; + dst += dst_stride; + *(uint64_t *)dst = out4_m; + *(uint32_t *)(dst + 4) = out12_m; + dst += dst_stride; + *(uint64_t *)dst = out5_m; + *(uint32_t *)(dst + 4) = out13_m; + dst += dst_stride; + *(uint64_t *)dst = out6_m; + *(uint32_t *)(dst + 4) = out14_m; + dst += dst_stride; + *(uint64_t *)dst = out7_m; + *(uint32_t *)(dst + 4) = out15_m; + dst += dst_stride; + } +} + +static void hevc_copy_8w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + int32_t height) +{ + __m128i zero = {0}; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + if (2 == height) { + __m128i src0, src1; + __m128i in0, in1; + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + LSX_DUP2_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, in0, in1); + LSX_DUP2_ARG2(__lsx_vslli_h, in0, 6, in1, 6, in0, in1); + __lsx_vst(in0, dst, 0); + __lsx_vst(in1, dst + dst_stride, 0); + } else if (4 == height) { + __m128i src0, src1, src2, src3; + __m128i in0, in1, in2, in3; + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vslli_h, in0, 6, in1, 6, in2, 6, in3, 6, in0, in1, in2, in3); + __lsx_vst(in0, dst, 0); + __lsx_vst(in1, dst + dst_stride, 0); + __lsx_vst(in2, dst + dst_stride_2x, 0); + __lsx_vst(in3, dst + dst_stride_3x, 0); + } else if (6 == height) { + __m128i src0, src1, src2, src3, src4, src5; + __m128i in0, in1, in2, in3, in4, in5; + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src4, src5); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, in0, + in1, in2, in3); + LSX_DUP2_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, in4, in5); + LSX_DUP4_ARG2(__lsx_vslli_h, in0, 6, in1, 6, in2, 6, in3, 6, in0, in1, in2, in3); + LSX_DUP2_ARG2(__lsx_vslli_h, in4, 6, in5, 6, in4, in5); + __lsx_vst(in0, dst, 0); + __lsx_vst(in1, dst + dst_stride, 0); + __lsx_vst(in2, dst + dst_stride_2x, 0); + __lsx_vst(in3, dst + dst_stride_3x, 0); + __lsx_vst(in4, dst + dst_stride_4x, 0); + __lsx_vst(in5, dst + dst_stride_4x + dst_stride, 0); + } else if (0 == (height & 0x07)) { + uint32_t loop_cnt; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, + src + src_stride_2x, 0, src + src_stride_3x, 0, src0, + src1, src2, src3); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, + src + src_stride_2x, 0, src + src_stride_3x, 0, src4, + src5, src6, src7); + src += src_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, zero, src6, zero, src7, + in4, in5, in6, in7); + LSX_DUP4_ARG2(__lsx_vslli_h, in0, 6, in1, 6, in2, 6, in3, 6, in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vslli_h, in4, 6, in5, 6, in6, 6, in7, 6, in4, in5, in6, in7); + __lsx_vst(in0, dst, 0); + __lsx_vst(in1, dst + dst_stride, 0); + __lsx_vst(in2, dst + dst_stride_2x, 0); + __lsx_vst(in3, dst + dst_stride_3x, 0); + dst += dst_stride_4x; + __lsx_vst(in4, dst, 0); + __lsx_vst(in5, dst + dst_stride, 0); + __lsx_vst(in6, dst + dst_stride_2x, 0); + __lsx_vst(in7, dst + dst_stride_3x, 0); + dst += dst_stride_4x; + } + } +} + +static void hevc_copy_12w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i zero = {0}; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0, in1, in0_r, in1_r, in2_r, in3_r; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src4, src5, src6, src7); + src += src_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP2_ARG2(__lsx_vilvh_w, src1, src0, src3, src2, src0, src1); + LSX_DUP2_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, in0, in1); + LSX_DUP2_ARG2(__lsx_vslli_h, in0, 6, in1, 6, in0, in1); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in1_r, dst + dst_stride, 0); + __lsx_vst(in2_r, dst + dst_stride_2x, 0); + __lsx_vst(in3_r, dst + dst_stride_3x, 0); + __lsx_vstelm_d(in0, dst, 16, 0); + __lsx_vstelm_d(in0, dst + dst_stride, 16, 1); + __lsx_vstelm_d(in1, dst + dst_stride_2x, 16, 0); + __lsx_vstelm_d(in1, dst + dst_stride_3x, 16, 1); + dst += dst_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP2_ARG2(__lsx_vilvh_w, src5, src4, src7, src6, src0, src1); + LSX_DUP2_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, in0, in1); + LSX_DUP2_ARG2(__lsx_vslli_h, in0, 6, in1, 6, in0, in1); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in1_r, dst + dst_stride, 0); + __lsx_vst(in2_r, dst + dst_stride_2x, 0); + __lsx_vst(in3_r, dst + dst_stride_3x, 0); + __lsx_vstelm_d(in0, dst, 16, 0); + __lsx_vstelm_d(in0, dst + dst_stride, 16, 1); + __lsx_vstelm_d(in1, dst + dst_stride_2x, 16, 0); + __lsx_vstelm_d(in1, dst + dst_stride_3x, 16, 1); + dst += dst_stride_4x; + } +} + +static void hevc_copy_16w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + int32_t height) +{ + __m128i zero = {0}; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + if (4 == height) { + __m128i src0, src1, src2, src3; + __m128i in0_r, in1_r, in2_r, in3_r; + __m128i in0_l, in1_l, in2_l, in3_l; + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src1, src2, src3); + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in1_r, dst + dst_stride, 0); + __lsx_vst(in2_r, dst + dst_stride_2x, 0); + __lsx_vst(in3_r, dst + dst_stride_3x, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_l, dst + dst_stride, 16); + __lsx_vst(in2_l, dst + dst_stride_2x, 16); + __lsx_vst(in3_l, dst + dst_stride_3x, 16); + } else if (12 == height) { + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11; + __m128i in0_r, in1_r, in2_r, in3_r; + __m128i in0_l, in1_l, in2_l, in3_l; + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src4, src5, src6, src7); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src8, src9, src10, src11); + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in1_r, dst + dst_stride, 0); + __lsx_vst(in2_r, dst + dst_stride_2x, 0); + __lsx_vst(in3_r, dst + dst_stride_3x, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_l, dst + dst_stride, 16); + __lsx_vst(in2_l, dst + dst_stride_2x, 16); + __lsx_vst(in3_l, dst + dst_stride_3x, 16); + dst += dst_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in1_r, dst + dst_stride, 0); + __lsx_vst(in2_r, dst + dst_stride_2x, 0); + __lsx_vst(in3_r, dst + dst_stride_3x, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_l, dst + dst_stride, 16); + __lsx_vst(in2_l, dst + dst_stride_2x, 16); + __lsx_vst(in3_l, dst + dst_stride_3x, 16); + dst += dst_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src8, zero, src9, zero, src10, zero, src11, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src8, zero, src9, zero, src10, zero, src11, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in1_r, dst + dst_stride, 0); + __lsx_vst(in2_r, dst + dst_stride_2x, 0); + __lsx_vst(in3_r, dst + dst_stride_3x, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_l, dst + dst_stride, 16); + __lsx_vst(in2_l, dst + dst_stride_2x, 16); + __lsx_vst(in3_l, dst + dst_stride_3x, 16); + } else if (0 == (height & 0x07)) { + uint32_t loop_cnt; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, + src + src_stride_2x, 0, src + src_stride_3x, 0, src0, + src1, src2, src3); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, + src + src_stride_2x, 0, src + src_stride_3x, 0, src4, + src5, src6, src7); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in1_r, dst + dst_stride, 0); + __lsx_vst(in2_r, dst + dst_stride_2x, 0); + __lsx_vst(in3_r, dst + dst_stride_3x, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_l, dst + dst_stride, 16); + __lsx_vst(in2_l, dst + dst_stride_2x, 16); + __lsx_vst(in3_l, dst + dst_stride_3x, 16); + dst += dst_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in1_r, dst + dst_stride, 0); + __lsx_vst(in2_r, dst + dst_stride_2x, 0); + __lsx_vst(in3_r, dst + dst_stride_3x, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_l, dst + dst_stride, 16); + __lsx_vst(in2_l, dst + dst_stride_2x, 16); + __lsx_vst(in3_l, dst + dst_stride_3x, 16); + dst += dst_stride_4x; + } + } +} + +static void hevc_copy_24w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i zero = {0}; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src + src_stride_2x, + 16, src + src_stride_3x, 16, src4, src5, src6, src7); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in1_r, dst + dst_stride, 0); + __lsx_vst(in2_r, dst + dst_stride_2x, 0); + __lsx_vst(in3_r, dst + dst_stride_3x, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_l, dst + dst_stride, 16); + __lsx_vst(in2_l, dst + dst_stride_2x, 16); + __lsx_vst(in3_l, dst + dst_stride_3x, 16); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + __lsx_vst(in0_r, dst, 32); + __lsx_vst(in1_r, dst + dst_stride, 32); + __lsx_vst(in2_r, dst + dst_stride_2x, 32); + __lsx_vst(in3_r, dst + dst_stride_3x, 32); + dst += dst_stride_4x; + } +} + +static void hevc_copy_32w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i zero = {0}; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src2, src4, src6); + LSX_DUP4_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src + src_stride_2x, + 16, src + src_stride_3x, 16, src1, src3, src5, src7); + src += src_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_r, dst, 32); + __lsx_vst(in1_l, dst, 48); + dst += dst_stride; + __lsx_vst(in2_r, dst, 0); + __lsx_vst(in2_l, dst, 16); + __lsx_vst(in3_r, dst, 32); + __lsx_vst(in3_l, dst, 48); + dst += dst_stride; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_r, dst, 32); + __lsx_vst(in1_l, dst, 48); + dst += dst_stride; + __lsx_vst(in2_r, dst, 0); + __lsx_vst(in2_l, dst, 16); + __lsx_vst(in3_r, dst, 32); + __lsx_vst(in3_l, dst, 48); + dst += dst_stride; + } +} + +static void hevc_copy_48w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + __m128i zero = {0}; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11; + __m128i in0_r, in1_r, in2_r, in3_r, in4_r, in5_r; + __m128i in0_l, in1_l, in2_l, in3_l, in4_l, in5_l; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src2 = __lsx_vld(src, 32); + src += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src3, src4); + src5 = __lsx_vld(src, 32); + src += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src6, src7); + src8 = __lsx_vld(src, 32); + src += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src9, src10); + src11 = __lsx_vld(src, 32); + src += src_stride; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP2_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, in4_r, in5_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, in4_l, in5_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in4_r, 6, in5_r, 6, in4_l, 6, in5_l, 6, in4_r, + in5_r, in4_l, in5_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_r, dst, 32); + __lsx_vst(in1_l, dst, 48); + __lsx_vst(in2_r, dst, 64); + __lsx_vst(in2_l, dst, 80); + dst += dst_stride; + __lsx_vst(in3_r, dst, 0); + __lsx_vst(in3_l, dst, 16); + __lsx_vst(in4_r, dst, 32); + __lsx_vst(in4_l, dst, 48); + __lsx_vst(in5_r, dst, 64); + __lsx_vst(in5_l, dst, 80); + dst += dst_stride; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src6, zero, src7, zero, src8, zero, src9, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src6, zero, src7, zero, src8, zero, src9, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP2_ARG2(__lsx_vilvl_b, zero, src10, zero, src11, in4_r, in5_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, zero, src10, zero, src11, in4_l, in5_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in4_r, 6, in5_r, 6, in4_l, 6, in5_l, 6, in4_r, + in5_r, in4_l, in5_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_r, dst, 32); + __lsx_vst(in1_l, dst, 48); + __lsx_vst(in2_r, dst, 64); + __lsx_vst(in2_l, dst, 80); + dst += dst_stride; + __lsx_vst(in3_r, dst, 0); + __lsx_vst(in3_l, dst, 16); + __lsx_vst(in4_r, dst, 32); + __lsx_vst(in4_l, dst, 48); + __lsx_vst(in5_r, dst, 64); + __lsx_vst(in5_l, dst, 80); + dst += dst_stride; + } +} + +static void hevc_copy_64w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + int32_t height) +{ + uint32_t loop_cnt; + __m128i zero = {0}; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l; + + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src4, src5, src6, + src7); + src += src_stride; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_r, dst, 32); + __lsx_vst(in1_l, dst, 48); + __lsx_vst(in2_r, dst, 64); + __lsx_vst(in2_l, dst, 80); + __lsx_vst(in3_r, dst, 96); + __lsx_vst(in3_l, dst, 112); + dst += dst_stride; + + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_r, in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, zero, src6, zero, src7, + in0_l, in1_l, in2_l, in3_l); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_r, 6, in1_r, 6, in2_r, 6, in3_r, 6, in0_r, + in1_r, in2_r, in3_r); + LSX_DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l, + in1_l, in2_l, in3_l); + __lsx_vst(in0_r, dst, 0); + __lsx_vst(in0_l, dst, 16); + __lsx_vst(in1_r, dst, 32); + __lsx_vst(in1_l, dst, 48); + __lsx_vst(in2_r, dst, 64); + __lsx_vst(in2_l, dst, 80); + __lsx_vst(in3_r, dst, 96); + __lsx_vst(in3_l, dst, 112); + dst += dst_stride; + } +} + +static void hevc_hz_8t_4w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 16); + + src -= 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src4, src5, src6, src7); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src6, 128, src7, 128, + src4, src5, src6, src7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask0, src1, src0, mask1, src1, src0, + mask2, src1, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, dst0, + vec2, filt2, dst0, vec3, filt3, dst0, dst0, dst0, dst0); + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src2, mask0, src3, src2, mask1, src3, src2, + mask2, src3, src2, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst1, vec1, filt1, dst1, + vec2, filt2, dst1, vec3, filt3, dst1, dst1, dst1, dst1); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src4, mask0, src5, src4, mask1, src5, src4, + mask2, src5, src4, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst2, vec1, filt1, dst2, + vec2, filt2, dst2, vec3, filt3, dst2, dst2, dst2, dst2); + LSX_DUP4_ARG3(__lsx_vshuf_b, src7, src6, mask0, src7, src6, mask1, src7, src6, + mask2, src7, src6, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst3, vec1, filt1, dst3, + vec2, filt2, dst3, vec3, filt3, dst3, dst3, dst3, dst3); + + __lsx_vstelm_d(dst0, dst, 0, 0); + __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(dst1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(dst1, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + __lsx_vstelm_d(dst2, dst, 0, 0); + __lsx_vstelm_d(dst2, dst + dst_stride, 0, 1); + __lsx_vstelm_d(dst3, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(dst3, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + } +} + +static void hevc_hz_8t_8w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i src0, src1, src2, src3; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src -= 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0, src0, + mask2, src0, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, dst0, + vec2, filt2, dst0, vec3, filt3, dst0, dst0, dst0, dst0); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1, + mask2, src1, src1, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst1, vec1, filt1, dst1, + vec2, filt2, dst1, vec3, filt3, dst1, dst1, dst1, dst1); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2, + mask2, src2, src2, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst2, vec1, filt1, dst2, + vec2, filt2, dst2, vec3, filt3, dst2, dst2, dst2, dst2); + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3, src3, + mask2, src3, src3, mask3, vec0, vec1, vec2, vec3); + dst3 = const_vec; + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst3, vec0, filt0, dst3, vec1, filt1, dst3, + vec2, filt2, dst3, vec3, filt3, dst3, dst3, dst3, dst3); + + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst + dst_stride, 0); + __lsx_vst(dst2, dst + dst_stride_2x, 0); + __lsx_vst(dst3, dst + dst_stride_3x, 0); + dst += dst_stride_4x; + } +} + +static void hevc_hz_8t_12w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3, vec4, vec5; + __m128i filt0, filt1, filt2, filt3, dst0, dst1, dst2, dst3, dst4, dst5; + __m128i const_vec; + + src -= 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + mask4 = __lsx_vld(ff_hevc_mask_arr, 16); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask4, 2, mask4, 4, mask5, mask6); + mask7 = __lsx_vaddi_bu(mask4, 6); + + for (loop_cnt = 4; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vld, src, 8, src + src_stride, 8, src + src_stride_2x, 8, + src + src_stride_3x, 8, src4, src5, src6, src7); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src6, 128, src7, 128, src4, + src5, src6, src7); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask4, src7, src6, mask4, vec4, vec5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, const_vec, vec5, filt0, + dst4, dst5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask5, src7, src6, mask5, vec4, vec5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, dst4, vec4, filt1, dst5, vec5, filt1, dst4, dst5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask6, src7, src6, mask6, vec4, vec5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt2, dst1, vec1, filt2, dst2, + vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, dst4, vec4, filt2, dst5, vec5, filt2, dst4, dst5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask3, src3, src3, mask3, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask7, src7, src6, mask7, vec4, vec5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt3, dst1, vec1, filt3, dst2, + vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, dst4, vec4, filt3, dst5, vec5, filt3, dst4, dst5); + + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst + dst_stride, 0); + __lsx_vst(dst2, dst + dst_stride_2x, 0); + __lsx_vst(dst3, dst + dst_stride_3x, 0); + + __lsx_vstelm_d(dst4, dst, 16, 0); + __lsx_vstelm_d(dst4, dst + dst_stride, 16, 1); + __lsx_vstelm_d(dst5, dst + dst_stride_2x, 16, 0); + __lsx_vstelm_d(dst5, dst + dst_stride_3x, 16, 1); + dst += dst_stride_4x; + } + +} + +static void hevc_hz_8t_16w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + __m128i src0, src1, src2, src3; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3; + __m128i const_vec; + __m128i mask0; + + src -= 3; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, + filter, 6, filt0, filt1, filt2, filt3); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src2); + LSX_DUP2_ARG2(__lsx_vld, src, 8, src + src_stride, 8, src1, src3); + src += src_stride_2x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt2, dst1, vec1, filt2, dst2, + vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask3, src3, src3, mask3, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt3, dst1, vec1, filt3, dst2, + vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3); + + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst2, dst + dst_stride, 0); + __lsx_vst(dst1, dst, 16); + __lsx_vst(dst3, dst + dst_stride, 16); + dst += dst_stride_2x; + } +} + +static void hevc_hz_8t_24w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, src2, src3; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3, vec4, vec5; + __m128i dst0, dst1, dst2, dst3, dst4, dst5; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src -= 3; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1, mask2, + mask3, mask4); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6); + mask7 = __lsx_vaddi_bu(mask0, 14); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + for (loop_cnt = (height >> 1); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src2, src3); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask4, src1, src1, + mask0, src2, src2, mask0, vec0, vec1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src2, mask4, src3, src3, mask0, vec4, vec5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, const_vec, vec5, filt0, + dst4, dst5); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask5, src1, src1, + mask1, src2, src2, mask1, vec0, vec1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src2, mask5, src3, src3, mask1, vec4, vec5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, dst4, vec4, filt1, dst5, vec5, filt1, dst4, dst5); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src0, mask6, src1, src1, + mask2, src2, src2, mask2, vec0, vec1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src2, mask6, src3, src3, mask2, vec4, vec5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt2, dst1, vec1, filt2, dst2, + vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, dst4, vec4, filt2, dst5, vec5, filt2, dst4, dst5); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src0, mask7, src1, src1, + mask3, src2, src2, mask3, vec0, vec1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src2, mask7, src3, src3, mask3, vec4, vec5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt3, dst1, vec1, filt3, dst2, + vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, dst4, vec4, filt3, dst5, vec5, filt3, dst4, dst5); + + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + __lsx_vst(dst2, dst, 32); + dst += dst_stride; + __lsx_vst(dst3, dst, 0); + __lsx_vst(dst4, dst, 16); + __lsx_vst(dst5, dst, 32); + dst += dst_stride; + } +} + +static void hevc_hz_8t_32w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, src2; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src -= 3; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1, mask2, + mask3, mask4); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6); + mask7 = __lsx_vaddi_bu(mask0, 14); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src2 = __lsx_vld(src, 24); + src += src_stride; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0, src0, + mask2, src0, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, dst0, + vec2, filt2, dst0, vec3, filt3, dst0, dst0, dst0, dst0); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask4, src1, src0, mask5, src1, src0, + mask6, src1, src0, mask7, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst1, vec1, filt1, dst1, + vec2, filt2, dst1, vec3, filt3, dst1, dst1, dst1, dst1); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1, + mask2, src1, src1, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst2, vec1, filt1, dst2, + vec2, filt2, dst2, vec3, filt3, dst2, dst2, dst2, dst2); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2, + mask2, src2, src2, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst3, vec1, filt1, dst3, + vec2, filt2, dst3, vec3, filt3, dst3, dst3, dst3, dst3); + + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + __lsx_vst(dst2, dst, 32); + __lsx_vst(dst3, dst, 48); + dst += dst_stride; + } +} + +static void hevc_hz_8t_48w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, src2, src3; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3, vec4, vec5; + __m128i dst0, dst1, dst2, dst3, dst4, dst5; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src -= 3; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1, mask2, + mask3, mask4); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6); + mask7 = __lsx_vaddi_bu(mask0, 14); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src2 = __lsx_vld(src, 32); + src3 = __lsx_vld(src, 40); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask4, src1, src1, + mask0, src2, src1, mask4, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask5, src1, src1, + mask1, src2, src1, mask5, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src0, mask6, src1, src1, + mask2, src2, src1, mask6, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt2, dst1, vec1, filt2, dst2, + vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src0, mask7, src1, src1, + mask3, src2, src1, mask7, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt3, dst1, vec1, filt3, dst2, + vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3); + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + __lsx_vst(dst2, dst, 32); + __lsx_vst(dst3, dst, 48); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec4, vec5); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, const_vec, vec5, filt0, + dst4, dst5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, dst4, vec4, filt1, dst5, vec5, filt1, dst4, dst5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2, vec4, vec5); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, dst4, vec4, filt2, dst5, vec5, filt2, dst4, dst5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask3, src3, src3, mask3, vec4, vec5); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, dst4, vec4, filt3, dst5, vec5, filt3, dst4, dst5); + __lsx_vst(dst4, dst, 64); + __lsx_vst(dst5, dst, 80); + dst += dst_stride; + } +} + +static void hevc_hz_8t_64w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, src2, src3, src4; + __m128i filt0, filt1, filt2, filt3; + __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i const_vec; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + + src -= 3; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1, mask2, + mask3, mask4); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6) + mask7 = __lsx_vaddi_bu(mask0, 14); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src4 = __lsx_vld(src, 56); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0, src0, + mask2, src0, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, dst0, + vec2, filt2, dst0, vec3, filt3, dst0, dst0, dst0, dst0); + __lsx_vst(dst0, dst, 0); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask4, src1, src0, mask5, src1, src0, + mask6, src1, src0, mask7, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst1, vec1, filt1, dst1, + vec2, filt2, dst1, vec3, filt3, dst1, dst1, dst1, dst1); + __lsx_vst(dst1, dst, 16); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1, + mask2, src1, src1, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst2, vec1, filt1, dst2, + vec2, filt2, dst2, vec3, filt3, dst2, dst2, dst2, dst2); + __lsx_vst(dst2, dst, 32); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src1, mask4, src2, src1, mask5, src2, src1, + mask6, src2, src1, mask7, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst3, vec1, filt1, dst3, + vec2, filt2, dst3, vec3, filt3, dst3, dst3, dst3, dst3); + __lsx_vst(dst3, dst, 48); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2, + mask2, src2, src2, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst4, vec1, filt1, dst4, + vec2, filt2, dst4, vec3, filt3, dst4, dst4, dst4, dst4); + __lsx_vst(dst4, dst, 64); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src2, mask4, src3, src2, mask5, src3, src2, + mask6, src3, src2, mask7, vec0, vec1, vec2, vec3); + dst5 = const_vec; + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst5, vec0, filt0, dst5, vec1, filt1, dst5, + vec2, filt2, dst5, vec3, filt3, dst5, dst5, dst5, dst5); + __lsx_vst(dst5, dst, 80); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3, src3, + mask2, src3, src3, mask3, vec0, vec1, vec2, vec3); + dst6 = const_vec; + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst6, vec0, filt0, dst6, vec1, filt1, dst6, + vec2, filt2, dst6, vec3, filt3, dst6, dst6, dst6, dst6); + __lsx_vst(dst6, dst, 96); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4, src4, + mask2, src4, src4, mask3, vec0, vec1, vec2, vec3); + dst7 = const_vec; + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst7, vec0, filt0, dst7, vec1, filt1, dst7, + vec2, filt2, dst7, vec3, filt3, dst7, dst7, dst7, dst7); + __lsx_vst(dst7, dst, 112); + dst += dst_stride; + } +} + +static void hevc_vt_8t_4w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + int32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = (src_stride << 1) + src_stride; + int32_t dst_stride_3x = (dst_stride << 1) + dst_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i src9, src10, src11, src12, src13, src14; + __m128i src10_r, src32_r, src54_r, src76_r, src98_r; + __m128i src21_r, src43_r, src65_r, src87_r, src109_r; + __m128i src1110_r, src1211_r, src1312_r, src1413_r; + __m128i src2110, src4332, src6554, src8776, src10998; + __m128i src12111110, src14131312; + __m128i dst10, dst32, dst54, dst76; + __m128i filt0, filt1, filt2, filt3; + __m128i const_vec; + + src -= src_stride_3x; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src4, src5); + src6 = __lsx_vld(src + src_stride_2x, 0); + src += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_r, src32_r, src54_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r); + LSX_DUP2_ARG2(__lsx_vilvl_d, src21_r, src10_r, src43_r, src32_r, src2110, src4332); + src6554 = __lsx_vilvl_d(src65_r, src54_r); + LSX_DUP2_ARG2(__lsx_vxori_b, src2110, 128, src4332, 128, src2110, src4332); + src6554 = __lsx_vxori_b(src6554, 128); + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src7, src8, src9, src10); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, + 0, src + src_stride_3x, 0, src11, src12, src13, src14); + src += src_stride_4x; + + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src76_r, src87_r, src98_r, src109_r); + LSX_DUP4_ARG2(__lsx_vilvl_b, src11, src10, src12, src11, src13, src12, src14, + src13, src1110_r, src1211_r, src1312_r, src1413_r); + LSX_DUP4_ARG2(__lsx_vilvl_d, src87_r, src76_r, src109_r, src98_r, src1211_r, + src1110_r, src1413_r, src1312_r, src8776, src10998, src12111110, + src14131312); + LSX_DUP4_ARG2(__lsx_vxori_b, src8776, 128, src10998, 128, src12111110, 128, + src14131312, 128, src8776, src10998, src12111110, src14131312); + + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src2110, filt0, dst10, src4332, filt1, + dst10, src6554, filt2, dst10, src8776, filt3, dst10, dst10, dst10, + dst10); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src4332, filt0, dst32, src6554, filt1, + dst32, src8776, filt2, dst32, src10998, filt3, dst32, dst32, + dst32, dst32); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src6554, filt0, dst54, src8776, filt1, + dst54, src10998, filt2, dst54, src12111110, filt3, dst54, dst54, + dst54, dst54); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src8776, filt0, dst76, src10998, + filt1, dst76, src12111110, filt2, dst76, src14131312, filt3, dst76, + dst76, dst76, dst76); + + __lsx_vstelm_d(dst10, dst, 0, 0); + __lsx_vstelm_d(dst10, dst + dst_stride, 0, 1); + __lsx_vstelm_d(dst32, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(dst32, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + __lsx_vstelm_d(dst54, dst, 0, 0); + __lsx_vstelm_d(dst54, dst + dst_stride, 0, 1); + __lsx_vstelm_d(dst76, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(dst76, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + + src2110 = src10998; + src4332 = src12111110; + src6554 = src14131312; + src6 = src14; + } +} + +static void hevc_vt_8t_8w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + int32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = (src_stride << 1) + src_stride; + int32_t dst_stride_3x = (dst_stride << 1) + dst_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i src10_r, src32_r, src54_r, src76_r, src98_r; + __m128i src21_r, src43_r, src65_r, src87_r, src109_r; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i const_vec; + __m128i filt0, filt1, filt2, filt3; + + src -= src_stride_3x; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, + filter, 6, filt0, filt1, filt2, filt3); + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src4, src5); + src6 = __lsx_vld(src + src_stride_2x, 0); + src += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_r, src32_r, src54_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src7, src8, src9, src10); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src76_r, src87_r, src98_r, src109_r); + + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, src54_r, filt2, dst0_r, src76_r, filt3, dst0_r, + dst0_r, dst0_r, dst0_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, src65_r, filt2, dst1_r, src87_r, filt3, dst1_r, + dst1_r, dst1_r, dst1_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src32_r, filt0, dst2_r, src54_r, + filt1, dst2_r, src76_r, filt2, dst2_r, src98_r, filt3, dst2_r, + dst2_r, dst2_r, dst2_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst3_r, src65_r, + filt1, dst3_r, src87_r, filt2, dst3_r, src109_r, filt3, dst3_r, + dst3_r, dst3_r, dst3_r); + + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst1_r, dst + dst_stride, 0); + __lsx_vst(dst2_r, dst + dst_stride_2x, 0); + __lsx_vst(dst3_r, dst + dst_stride_3x, 0); + dst += dst_stride_4x; + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src6 = src10; + } +} + +static void hevc_vt_8t_12w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + int32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = (src_stride << 1) + src_stride; + int32_t dst_stride_3x = (dst_stride << 1) + dst_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i src10_r, src32_r, src54_r, src76_r, src98_r; + __m128i src21_r, src43_r, src65_r, src87_r, src109_r; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i src10_l, src32_l, src54_l, src76_l, src98_l; + __m128i src21_l, src43_l, src65_l, src87_l, src109_l; + __m128i src2110, src4332, src6554, src8776, src10998; + __m128i dst0_l, dst1_l; + __m128i const_vec; + __m128i filt0, filt1, filt2, filt3; + + src -= src_stride_3x; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src4, src5); + src6 = __lsx_vld(src + src_stride_2x, 0); + src += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_r, src32_r, src54_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_l, src32_l, src54_l, src21_l); + LSX_DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, src43_l, src65_l); + LSX_DUP2_ARG2(__lsx_vilvl_d, src21_l, src10_l, src43_l, src32_l, src2110, src4332); + src6554 = __lsx_vilvl_d(src65_l, src54_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src+ src_stride_2x, 0, + src + src_stride_3x, 0, src7, src8, src9, src10); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src76_r, src87_r, src98_r, src109_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src76_l, src87_l, src98_l, src109_l); + LSX_DUP2_ARG2(__lsx_vilvl_d, src87_l, src76_l, src109_l, src98_l, src8776, src10998); + + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, src54_r, filt2, dst0_r, src76_r, filt3, dst0_r, + dst0_r, dst0_r, dst0_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, src65_r, filt2, dst1_r, src87_r, filt3, dst1_r, + dst1_r, dst1_r, dst1_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src32_r, filt0, dst2_r, src54_r, + filt1, dst2_r, src76_r, filt2, dst2_r, src98_r, filt3, dst2_r, + dst2_r, dst2_r, dst2_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst3_r, src65_r, + filt1, dst3_r, src87_r, filt2, dst3_r, src109_r, filt3, dst3_r, + dst3_r, dst3_r, dst3_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src2110, filt0, dst0_l, src4332, + filt1, dst0_l, src6554, filt2, dst0_l, src8776, filt3, dst0_l, + dst0_l, dst0_l, dst0_l); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src4332, filt0, dst1_l, src6554, + filt1, dst1_l, src8776, filt2, dst1_l, src10998, filt3, dst1_l, + dst1_l, dst1_l, dst1_l); + + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst1_r, dst + dst_stride, 0); + __lsx_vst(dst2_r, dst + dst_stride_2x, 0); + __lsx_vst(dst3_r, dst + dst_stride_3x, 0); + __lsx_vstelm_d(dst0_l, dst, 16, 0); + __lsx_vstelm_d(dst0_l, dst + dst_stride, 16, 1); + __lsx_vstelm_d(dst1_l, dst + dst_stride_2x, 16, 0); + __lsx_vstelm_d(dst1_l, dst + dst_stride_3x, 16, 1); + dst += dst_stride_4x; + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src2110 = src6554; + src4332 = src8776; + src6554 = src10998; + src6 = src10; + } +} + +static void hevc_vt_8t_16multx4mult_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height, + int32_t width) +{ + uint8_t *src_tmp; + int16_t *dst_tmp; + int32_t loop_cnt, cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = (src_stride << 1) + src_stride; + int32_t dst_stride_3x = (dst_stride << 1) + dst_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i src10_r, src32_r, src54_r, src76_r, src98_r; + __m128i src21_r, src43_r, src65_r, src87_r, src109_r; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i src10_l, src32_l, src54_l, src76_l, src98_l; + __m128i src21_l, src43_l, src65_l, src87_l, src109_l; + __m128i dst0_l, dst1_l, dst2_l, dst3_l; + __m128i const_vec; + __m128i filt0, filt1, filt2, filt3; + + src -= src_stride_3x; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, filt0, + filt1, filt2, filt3); + + for (cnt = width >> 4; cnt--;) { + src_tmp = src; + dst_tmp = dst; + + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, + src0, src1, src2, src3); + src_tmp += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src4, src5); + src6 = __lsx_vld(src_tmp + src_stride_2x, 0); + src_tmp += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_r, src32_r, src54_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + src10_l, src32_l, src54_l, src21_l); + LSX_DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, src43_l, src65_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, + src7, src8, src9, src10); + src_tmp += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src76_r, src87_r, src98_r, src109_r); + LSX_DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src76_l, src87_l, src98_l, src109_l); + + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, src54_r, filt2, dst0_r, src76_r, filt3, + dst0_r, dst0_r, dst0_r, dst0_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, src65_r, filt2, dst1_r, src87_r, filt3, + dst1_r, dst1_r, dst1_r, dst1_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src32_r, filt0, dst2_r, src54_r, + filt1, dst2_r, src76_r, filt2, dst2_r, src98_r, filt3, + dst2_r, dst2_r, dst2_r, dst2_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst3_r, src65_r, + filt1, dst3_r, src87_r, filt2, dst3_r, src109_r, filt3, + dst3_r, dst3_r, dst3_r, dst3_r); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src10_l, filt0, dst0_l, src32_l, + filt1, dst0_l, src54_l, filt2, dst0_l, src76_l, filt3, + dst0_l, dst0_l, dst0_l, dst0_l); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src21_l, filt0, dst1_l, src43_l, + filt1, dst1_l, src65_l, filt2, dst1_l, src87_l, filt3, + dst1_l, dst1_l, dst1_l, dst1_l); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src32_l, filt0, dst2_l, src54_l, + filt1, dst2_l, src76_l, filt2, dst2_l, src98_l, filt3, + dst2_l, dst2_l, dst2_l, dst2_l); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, src43_l, filt0, dst3_l, src65_l, + filt1, dst3_l, src87_l, filt2, dst3_l, src109_l, filt3, + dst3_l, dst3_l, dst3_l, dst3_l); + + __lsx_vst(dst0_r, dst_tmp, 0); + __lsx_vst(dst1_r, dst_tmp + dst_stride, 0); + __lsx_vst(dst2_r, dst_tmp + dst_stride_2x, 0); + __lsx_vst(dst3_r, dst_tmp + dst_stride_3x, 0); + __lsx_vst(dst0_l, dst_tmp, 16); + __lsx_vst(dst1_l, dst_tmp + dst_stride, 16); + __lsx_vst(dst2_l, dst_tmp + dst_stride_2x, 16); + __lsx_vst(dst3_l, dst_tmp + dst_stride_3x, 16); + dst_tmp += dst_stride_4x; + + src10_r = src54_r; + src32_r = src76_r; + src54_r = src98_r; + src21_r = src65_r; + src43_r = src87_r; + src65_r = src109_r; + src10_l = src54_l; + src32_l = src76_l; + src54_l = src98_l; + src21_l = src65_l; + src43_l = src87_l; + src65_l = src109_l; + src6 = src10; + } + src += 16; + dst += 16; + } +} + +static void hevc_vt_8t_16w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 16); +} + +static void hevc_vt_8t_24w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 16); + hevc_vt_8t_8w_lsx(src + 16, src_stride, dst + 16, dst_stride, + filter, height); +} + +static void hevc_vt_8t_32w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 32); +} + +static void hevc_vt_8t_48w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 48); +} + +static void hevc_vt_8t_64w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 64); +} + +static void hevc_hv_8t_4w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter_x, const int8_t *filter_y, + int32_t height) +{ + uint32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt0, filt1, filt2, filt3; + __m128i filt_h0, filt_h1, filt_h2, filt_h3; + __m128i mask1, mask2, mask3; + __m128i filter_vec, const_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15; + __m128i dst30, dst41, dst52, dst63, dst66, dst97, dst108; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i dst10_r, dst32_r, dst54_r, dst76_r, dst98_r; + __m128i dst21_r, dst43_r, dst65_r, dst87_r, dst109_r; + __m128i mask0; + + mask0 = __lsx_vld(ff_hevc_mask_arr, 16); + + src -= src_stride_3x + 3; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4, + filter_x, 6, filt0, filt1, filt2, filt3); + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + + LSX_DUP4_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filter_vec, 2, + filter_vec, 3, filt_h0, filt_h1, filt_h2, filt_h3); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src4, src5); + src6 = __lsx_vld(src + src_stride_2x, 0); + src += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, + src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src0, mask0, src3, src0, mask1, src3, src0, + mask2, src3, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src1, mask0, src4, src1, mask1, src4, src1, + mask2, src4, src1, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src2, mask0, src5, src2, mask1, src5, src2, + mask2, src5, src2, mask3, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src6, src3, mask0, src6, src3, mask1, src6, src3, + mask2, src6, src3, mask3, vec12, vec13, vec14, vec15); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst30, vec1, filt1, dst30, + vec2, filt2, dst30, vec3, filt3, dst30, dst30, dst30, dst30); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst41, vec5, filt1, dst41, + vec6, filt2, dst41, vec7, filt3, dst41, dst41, dst41, dst41); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec8, filt0, dst52, vec9, filt1, dst52, + vec10, filt2, dst52, vec11, filt3, dst52, dst52, dst52, dst52); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec12, filt0, dst63, vec13, filt1, dst63, + vec14, filt2, dst63, vec15, filt3, dst63, dst63, dst63, dst63); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst41, dst30, dst52, dst41, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst41, dst30, dst52, dst41, dst43_r, dst54_r); + dst32_r = __lsx_vilvl_h(dst63, dst52); + dst65_r = __lsx_vilvh_h(dst63, dst52); + dst66 = __lsx_vreplvei_d(dst63, 1); + + for (loop_cnt = height >> 2; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src7, src8, src9, src10); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src9, src7, mask0, src9, src7, mask1, src9, src7, + mask2, src9, src7, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src10, src8, mask0, src10, src8, mask1, src10, src8, + mask2, src10, src8, mask3, vec4, vec5, vec6, vec7); + + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst97, vec1, filt1, + dst97, vec2, filt2, dst97, vec3, filt3, dst97, dst97, dst97, dst97); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst108, vec5, filt1, + dst108, vec6, filt2, dst108, vec7, filt3, dst108, dst108, dst108, dst108); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst97, dst66, dst108, dst97, dst76_r, dst87_r); + dst109_r = __lsx_vilvh_h(dst108, dst97); + dst66 = __lsx_vreplvei_d(dst97, 1); + dst98_r = __lsx_vilvl_h(dst66, dst108); + + dst0_r = __lsx_hevc_filt_8tap_w(dst10_r, dst32_r, dst54_r, dst76_r, + filt_h0, filt_h1, filt_h2, filt_h3); + dst1_r = __lsx_hevc_filt_8tap_w(dst21_r, dst43_r, dst65_r, dst87_r, + filt_h0, filt_h1, filt_h2, filt_h3); + dst2_r = __lsx_hevc_filt_8tap_w(dst32_r, dst54_r, dst76_r, dst98_r, + filt_h0, filt_h1, filt_h2, filt_h3); + dst3_r = __lsx_hevc_filt_8tap_w(dst43_r, dst65_r, dst87_r, dst109_r, + filt_h0, filt_h1, filt_h2, filt_h3); + LSX_DUP4_ARG2(__lsx_vsrli_w, dst0_r, 6, dst1_r, 6, dst2_r, 6, dst3_r, 6, + dst0_r, dst1_r, dst2_r, dst3_r); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst2_r); + __lsx_vstelm_d(dst0_r, dst, 0, 0); + __lsx_vstelm_d(dst0_r, dst + dst_stride, 0, 1); + __lsx_vstelm_d(dst2_r, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(dst2_r, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + + dst10_r = dst54_r; + dst32_r = dst76_r; + dst54_r = dst98_r; + dst21_r = dst65_r; + dst43_r = dst87_r; + dst65_r = dst109_r; + dst66 = __lsx_vreplvei_d(dst108, 1); + } +} + +static void hevc_hv_8t_8multx1mult_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t width) +{ + uint32_t loop_cnt, cnt; + uint8_t *src_tmp; + int16_t *dst_tmp; + int32_t src_stride_2x = (src_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i filt0, filt1, filt2, filt3; + __m128i filt_h0, filt_h1, filt_h2, filt_h3; + __m128i mask1, mask2, mask3; + __m128i filter_vec, const_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i dst0_r, dst0_l; + __m128i dst10_r, dst32_r, dst54_r, dst76_r; + __m128i dst10_l, dst32_l, dst54_l, dst76_l; + __m128i mask0 = {0x403030202010100, 0x807070606050504}; + + src -= src_stride_3x + 3; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4, + filter_x, 6, filt0, filt1, filt2, filt3); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + + LSX_DUP4_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filter_vec, 2, + filter_vec, 3, filt_h0, filt_h1, filt_h2, filt_h3); + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + for (cnt = width >> 3; cnt--;) { + src_tmp = src; + dst_tmp = dst; + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, + src0, src1, src2, src3); + src_tmp += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src4, src5); + src6 = __lsx_vld(src_tmp + src_stride_2x, 0); + src_tmp += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + /* row 0 row 1 row 2 row 3 */ + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0, src0, + mask2, src0, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1, + mask2, src1, src1, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2, + mask2, src2, src2, mask3, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3, src3, + mask2, src3, src3, mask3, vec12, vec13, vec14, vec15); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, dst0, + vec2, filt2, dst0, vec3, filt3, dst0, dst0, dst0, dst0); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst1, vec5, filt1, dst1, + vec6, filt2, dst1, vec7, filt3, dst1, dst1, dst1, dst1); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec8, filt0, dst2, vec9, filt1, dst2, + vec10, filt2, dst2, vec11, filt3, dst2, dst2, dst2, dst2); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec12, filt0, dst3, vec13, filt1, + dst3, vec14, filt2, dst3, vec15, filt3, dst3, dst3, dst3, dst3); + + /* row 4 row 5 row 6 */ + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4, src4, + mask2, src4, src4, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src5, src5, + mask2, src5, src5, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src6, src6, + mask2, src6, src6, mask3, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst4, filt1, vec1, dst4, + vec2, filt2, dst4, vec3, filt3, dst4, dst4, dst4, dst4); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst5, vec5, filt1, dst5, + vec6, filt2, dst5, vec7, filt3, dst5, dst5, dst5, dst5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec8, filt0, dst6, vec9, filt1, dst6, + vec10, filt2, dst6, vec11, filt3, dst6, dst6, dst6, dst6); + + for (loop_cnt = height; loop_cnt--;) { + src7 = __lsx_vld(src_tmp, 0); + src7 = __lsx_vxori_b(src7, 128); + src_tmp += src_stride; + + LSX_DUP4_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, src7, + src7, mask2, src7, src7, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst7, vec1, filt1, + dst7, vec2, filt2, dst7, vec3, filt3, dst7, dst7, dst7, dst7); + + LSX_DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + dst10_r, dst32_r, dst54_r, dst76_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + dst10_l, dst32_l, dst54_l, dst76_l); + + dst0_r = __lsx_hevc_filt_8tap_w(dst10_r, dst32_r, dst54_r, dst76_r, + filt_h0, filt_h1, filt_h2, filt_h3); + dst0_l = __lsx_hevc_filt_8tap_w(dst10_l, dst32_l, dst54_l, dst76_l, + filt_h0, filt_h1, filt_h2, filt_h3); + dst0_r = __lsx_vsrli_w(dst0_r, 6); + dst0_l = __lsx_vsrli_w(dst0_l, 6); + + dst0_r = __lsx_vpickev_h(dst0_l, dst0_r); + __lsx_vst(dst0_r, dst_tmp, 0); + dst_tmp += dst_stride; + + dst0 = dst1; + dst1 = dst2; + dst2 = dst3; + dst3 = dst4; + dst4 = dst5; + dst5 = dst6; + dst6 = dst7; + } + src += 8; + dst += 8; + } +} + +static void hevc_hv_8t_8w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter_x, const int8_t *filter_y, + int32_t height) +{ + hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 8); +} + +static void hevc_hv_8t_12w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter_x, const int8_t *filter_y, + int32_t height) +{ + uint32_t loop_cnt; + uint8_t *src_tmp; + int16_t *dst_tmp; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15; + __m128i filt0, filt1, filt2, filt3, filt_h0, filt_h1, filt_h2, filt_h3; + __m128i filter_vec, const_vec; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i dst30, dst41, dst52, dst63, dst66, dst97, dst108; + __m128i dst10_r, dst32_r, dst54_r, dst76_r, dst98_r, dst21_r, dst43_r; + __m128i dst65_r, dst87_r, dst109_r, dst10_l, dst32_l, dst54_l, dst76_l; + __m128i dst0_r, dst0_l, dst1_r, dst2_r, dst3_r; + + src -= src_stride_3x + 3; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4, + filter_x, 6, filt0, filt1, filt2, filt3); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + + LSX_DUP4_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filter_vec, 2, + filter_vec, 3, filt_h0, filt_h1, filt_h2, filt_h3); + + mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + src_tmp = src; + dst_tmp = dst; + + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, + src0, src1, src2, src3); + src_tmp += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src4, src5); + src6 = __lsx_vld(src_tmp + src_stride_2x, 0); + src_tmp += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, + src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + /* row 0 row 1 row 2 row 3 */ + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0, src0, + mask2, src0, src0, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1, + mask2, src1, src1, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2, + mask2, src2, src2, mask3, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3, src3, + mask2, src3, src3, mask3, vec12, vec13, vec14, vec15); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, dst0, + vec2, filt2, dst0, vec3, filt3, dst0, dst0, dst0, dst0); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst1, vec5, filt1, dst1, + vec6, filt2, dst1, vec7, filt3, dst1, dst1, dst1, dst1); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec8, filt0, dst2, vec9, filt1, dst2, + vec10, filt2, dst2, vec11, filt3, dst2, dst2, dst2, dst2); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec12, filt0, dst3, vec13, filt1, dst3, + vec14, filt2, dst3, vec15, filt3, dst3, dst3, dst3, dst3); + + /* row 4 row 5 row 6 */ + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4, src4, + mask2, src4, src4, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src5, src5, + mask2, src5, src5, mask3, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src6, src6, + mask2, src6, src6, mask3, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst4, vec1, filt1, dst4, + vec2, filt2, dst4, vec3, filt3, dst4, dst4, dst4, dst4); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst5, vec5, filt1, dst5, + vec6, filt2, dst5, vec7, filt3, dst5, dst5, dst5, dst5); + dst6 = const_vec; + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec8, filt0, dst6, vec9, filt1, dst6, + vec10, filt2, dst6, vec11, filt3, dst6, dst6, dst6, dst6); + + for (loop_cnt = height; loop_cnt--;) { + src7 = __lsx_vld(src_tmp, 0); + src7 = __lsx_vxori_b(src7, 128); + src_tmp += src_stride; + + LSX_DUP4_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, src7, src7, + mask2, src7, src7, mask3, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst7, vec1, filt1, dst7, + vec2, filt2, dst7, vec3, filt3, dst7, dst7, dst7, dst7); + LSX_DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + dst10_r, dst32_r, dst54_r, dst76_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, + dst10_l, dst32_l, dst54_l, dst76_l); + dst0_r = __lsx_hevc_filt_8tap_w(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0, + filt_h1, filt_h2, filt_h3); + dst0_l = __lsx_hevc_filt_8tap_w(dst10_l, dst32_l, dst54_l, dst76_l, filt_h0, + filt_h1, filt_h2, filt_h3); + dst0_r = __lsx_vsrli_w(dst0_r, 6); + dst0_l = __lsx_vsrli_w(dst0_l, 6); + + dst0_r = __lsx_vpickev_h(dst0_l, dst0_r); + __lsx_vst(dst0_r, dst_tmp, 0); + dst_tmp += dst_stride; + + dst0 = dst1; + dst1 = dst2; + dst2 = dst3; + dst3 = dst4; + dst4 = dst5; + dst5 = dst6; + dst6 = dst7; + } + src += 8; + dst += 8; + + mask4 = __lsx_vld(ff_hevc_mask_arr, 16); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask4, 2, mask4, 4, mask5, mask6); + mask7 = __lsx_vaddi_bu(mask4, 6); + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src4, src5); + src6 = __lsx_vld(src + src_stride_2x, 0); + src += src_stride_3x; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3) + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src3, src0, mask4, src3, src0, mask5, src3, src0, + mask6, src3, src0, mask7, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src1, mask4, src4, src1, mask5, src4, src1, + mask6, src4, src1, mask7, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src5, src2, mask4, src5, src2, mask5, src5, src2, + mask6, src5, src2, mask7, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src6, src3, mask4, src6, src3, mask5, src6, src3, + mask6, src6, src3, mask7, vec12, vec13, vec14, vec15); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst30, vec1, filt1, dst30, + vec2, filt2, dst30, vec3, filt3, dst30, dst30, dst30, dst30); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst41, vec5, filt1, dst41, + vec6, filt2, dst41, vec7, filt3, dst41, dst41, dst41, dst41); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec8, filt0, dst52, vec9, filt1, dst52, + vec10, filt2, dst52, vec11, filt3, dst52, dst52, dst52, dst52); + dst63 = const_vec; + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec12, filt0, dst63, vec13, filt1, dst63, + vec14, filt2, dst63, vec15, filt3, dst63, dst63, dst63, dst63); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst41, dst30, dst52, dst41, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst41, dst30, dst52, dst41, dst43_r, dst54_r); + dst32_r = __lsx_vilvl_h(dst63, dst52); + dst65_r = __lsx_vilvh_h(dst63, dst52); + + dst66 = __lsx_vreplvei_d(dst63, 1); + + for (loop_cnt = height >> 2; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src7, src8, src9, src10); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src9, src7, mask4, src9, src7, mask5, src9, src7, + mask6, src9, src7, mask7, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src10, src8, mask4, src10, src8, mask5, src10, + src8, mask6, src10, src8, mask7, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst97, vec1, filt1, + dst97, vec2, filt2, dst97, vec3, filt3, dst97, dst97, dst97, + dst97); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst108, vec5, filt1, + dst108, vec6, filt2, dst108, vec7, filt3, dst108, dst108, dst108, + dst108); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst97, dst66, dst108, dst97, dst76_r, dst87_r); + dst109_r = __lsx_vilvh_h(dst108, dst97); + dst66 = __lsx_vreplvei_d(dst97, 1); + dst98_r = __lsx_vilvl_h(dst66, dst108); + + dst0_r = __lsx_hevc_filt_8tap_w(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0, + filt_h1, filt_h2, filt_h3); + dst1_r = __lsx_hevc_filt_8tap_w(dst21_r, dst43_r, dst65_r, dst87_r, filt_h0, + filt_h1, filt_h2, filt_h3); + dst2_r = __lsx_hevc_filt_8tap_w(dst32_r, dst54_r, dst76_r, dst98_r, filt_h0, + filt_h1, filt_h2, filt_h3); + dst3_r = __lsx_hevc_filt_8tap_w(dst43_r, dst65_r, dst87_r, dst109_r, filt_h0, + filt_h1, filt_h2, filt_h3); + LSX_DUP4_ARG2(__lsx_vsrli_w, dst0_r, 6, dst1_r, 6, dst2_r, 6, dst3_r, 6, + dst0_r, dst1_r, dst2_r, dst3_r); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst2_r); + __lsx_vstelm_d(dst0_r, dst, 0, 0); + __lsx_vstelm_d(dst0_r, dst + dst_stride, 0, 1); + __lsx_vstelm_d(dst2_r, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(dst2_r, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + + dst10_r = dst54_r; + dst32_r = dst76_r; + dst54_r = dst98_r; + dst21_r = dst65_r; + dst43_r = dst87_r; + dst65_r = dst109_r; + dst66 = __lsx_vreplvei_d(dst108, 1); + } +} + +static void hevc_hv_8t_16w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter_x, const int8_t *filter_y, + int32_t height) +{ + hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 16); +} + +static void hevc_hv_8t_24w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter_x, const int8_t *filter_y, + int32_t height) +{ + hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 24); +} + +static void hevc_hv_8t_32w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter_x, const int8_t *filter_y, + int32_t height) +{ + hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 32); +} + +static void hevc_hv_8t_48w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter_x, const int8_t *filter_y, + int32_t height) +{ + hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 48); +} + +static void hevc_hv_8t_64w_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter_x, const int8_t *filter_y, + int32_t height) +{ + hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 64); +} + +static void hevc_hz_4t_32w_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt; + __m128i src0, src1, src2; + __m128i filt0, filt1; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1, mask2, mask3; + __m128i dst0, dst1, dst2, dst3; + __m128i vec0, vec1, vec2, vec3; + __m128i const_vec; + + src -= 1; + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 8, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 10); + + for (loop_cnt = height; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src2 = __lsx_vld(src, 24); + src += src_stride; + + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask2, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src2, src2, mask0, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, const_vec, vec1, filt0, + const_vec, vec2, filt0, const_vec, vec3, filt0, dst0, dst1, dst2, + dst3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask3, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask1, src2, src2, mask1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, dst0, vec0, filt1, dst1, vec1, filt1, dst2, + vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3); + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + __lsx_vst(dst2, dst, 32); + __lsx_vst(dst3, dst, 48); + dst += dst_stride; + } +} + +static void hevc_vt_4t_16w_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i src0, src1, src2, src3, src4, src5; + __m128i src10_r, src32_r, src21_r, src43_r; + __m128i src10_l, src32_l, src21_l, src43_l; + __m128i dst0_r, dst1_r, dst0_l, dst1_l; + __m128i filt0, filt1; + __m128i const_vec; + + src -= src_stride; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src2 = __lsx_vld(src + src_stride_2x, 0); + src += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src3, src4); + src += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src3, src4); + LSX_DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, dst0_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_l, filt0, dst0_l, src32_l, + filt1, dst0_l, dst0_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, dst1_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_l, filt0, dst1_l, src43_l, + filt1, dst1_l, dst1_l); + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst0_l, dst, 16); + dst += dst_stride; + __lsx_vst(dst1_r, dst, 0); + __lsx_vst(dst1_l, dst, 16); + dst += dst_stride; + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src5, src2); + src += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src5, 128, src2, 128, src5, src2); + LSX_DUP2_ARG2(__lsx_vilvl_b, src5, src4, src2, src5, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src5, src4, src2, src5, src10_l, src21_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src32_r, filt0, dst0_r, src10_r, + filt1, dst0_r, dst0_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src32_l, filt0, dst0_l, src10_l, + filt1, dst0_l, dst0_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst1_r, src21_r, + filt1, dst1_r, dst1_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src43_l, filt0, dst1_l, src21_l, + filt1, dst1_l, dst1_l); + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst0_l, dst, 16); + dst += dst_stride; + __lsx_vst(dst1_r, dst, 0); + __lsx_vst(dst1_l, dst, 16); + dst += dst_stride; + } +} + +static void hevc_vt_4t_24w_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t src_stride_3x = src_stride_2x + src_stride; + + __m128i src0, src1, src2, src3, src4, src5; + __m128i src6, src7, src8, src9, src10, src11; + __m128i src10_r, src32_r, src76_r, src98_r; + __m128i src21_r, src43_r, src87_r, src109_r; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i src10_l, src32_l, src21_l, src43_l; + __m128i dst0_l, dst1_l; + __m128i filt0, filt1; + __m128i const_vec; + + src -= src_stride; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src2 = __lsx_vld(src + src_stride_2x, 0); + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src6, src7); + src8 = __lsx_vld(src + src_stride_2x, 16); + src += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src6, 128, src7, 128, src6, src7); + src8 = __lsx_vxori_b(src8, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src3, src4); + LSX_DUP2_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src3, src4); + LSX_DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l); + + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src9, src10); + src += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src9, 128, src10, 128, src9, src10); + LSX_DUP2_ARG2(__lsx_vilvl_b, src9, src8, src10, src9, src98_r, src109_r); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, dst0_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_l, filt0, dst0_l, src32_l, + filt1, dst0_l, dst0_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, dst1_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_l, filt0, dst1_l, src43_l, + filt1, dst1_l, dst1_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src76_r, filt0, dst2_r, src98_r, + filt1, dst2_r, dst2_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src87_r, filt0, dst3_r, src109_r, + filt1, dst3_r, dst3_r); + + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst0_l, dst, 16); + __lsx_vst(dst2_r, dst, 32); + dst += dst_stride; + __lsx_vst(dst1_r, dst, 0); + __lsx_vst(dst1_l, dst, 16); + __lsx_vst(dst3_r, dst, 32); + dst += dst_stride; + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src5, src2); + LSX_DUP2_ARG2(__lsx_vxori_b, src5, 128, src2, 128, src5, src2); + LSX_DUP2_ARG2(__lsx_vilvl_b, src5, src4, src2, src5, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src5, src4, src2, src5, src10_l, src21_l); + + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src11, src8); + src += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src11, 128, src8, 128, src11, src8); + LSX_DUP2_ARG2(__lsx_vilvl_b, src11, src10, src8, src11, src76_r, src87_r); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src32_r, filt0, dst0_r, src10_r, + filt1, dst0_r, dst0_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src32_l, filt0, dst0_l, src10_l, + filt1, dst0_l, dst0_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst1_r, src21_r, + filt1, dst1_r, dst1_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src43_l, filt0, dst1_l, src21_l, + filt1, dst1_l, dst1_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src98_r, filt0, dst2_r, src76_r, + filt1, dst2_r, dst2_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src109_r, filt0, dst3_r, src87_r, + filt1, dst3_r, dst3_r); + + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst0_l, dst, 16); + __lsx_vst(dst2_r, dst, 32); + dst += dst_stride; + __lsx_vst(dst1_r, dst, 0); + __lsx_vst(dst1_l, dst, 16); + __lsx_vst(dst3_r, dst, 32); + dst += dst_stride; + } +} + +static void hevc_vt_4t_32w_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int32_t loop_cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t src_stride_3x = src_stride_2x + src_stride; + + __m128i src0, src1, src2, src3, src4, src5; + __m128i src6, src7, src8, src9, src10, src11; + __m128i src10_r, src32_r, src76_r, src98_r; + __m128i src21_r, src43_r, src87_r, src109_r; + __m128i dst0_r, dst1_r, dst2_r, dst3_r; + __m128i src10_l, src32_l, src76_l, src98_l; + __m128i src21_l, src43_l, src87_l, src109_l; + __m128i dst0_l, dst1_l, dst2_l, dst3_l; + __m128i filt0, filt1; + __m128i const_vec; + + src -= src_stride; + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1); + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src2 = __lsx_vld(src + src_stride_2x, 0); + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1) + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src6, src7); + src8 = __lsx_vld(src + src_stride_2x, 16); + src += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src6, 128, src7, 128, src6, src7); + src8 = __lsx_vxori_b(src8, 128); + LSX_DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src76_l, src87_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src3, src4); + LSX_DUP2_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src3, src4); + LSX_DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l); + + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src9, src10); + src += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src9, 128, src10, 128, src9, src10); + LSX_DUP2_ARG2(__lsx_vilvl_b, src9, src8, src10, src9, src98_r, src109_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src9, src8, src10, src9, src98_l, src109_l); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_r, filt0, dst0_r, src32_r, + filt1, dst0_r, dst0_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src10_l, filt0, dst0_l, src32_l, + filt1, dst0_l, dst0_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_r, filt0, dst1_r, src43_r, + filt1, dst1_r, dst1_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src21_l, filt0, dst1_l,src43_l, + filt1, dst1_l, dst1_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src76_r, filt0, dst2_r, src98_r, + filt1, dst2_r, dst2_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src76_l, filt0, dst2_l, src98_l, + filt1, dst2_l, dst2_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src87_r, filt0, dst3_r, src109_r, + filt1, dst3_r, dst3_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src87_l, filt0, dst3_l, src109_l, + filt1, dst3_l, dst3_l); + + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst0_l, dst, 16); + __lsx_vst(dst2_r, dst, 32); + __lsx_vst(dst2_l, dst, 48); + dst += dst_stride; + __lsx_vst(dst1_r, dst, 0); + __lsx_vst(dst1_l, dst, 16); + __lsx_vst(dst3_r, dst, 32); + __lsx_vst(dst3_l, dst, 48); + dst += dst_stride; + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src5, src2); + LSX_DUP2_ARG2(__lsx_vxori_b, src5, 128, src2, 128, src5, src2); + LSX_DUP2_ARG2(__lsx_vilvl_b, src5, src4, src2, src5, src10_r, src21_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src5, src4, src2, src5, src10_l, src21_l); + + LSX_DUP2_ARG2(__lsx_vld, src, 16, src + src_stride, 16, src11, src8); + src += src_stride_2x; + LSX_DUP2_ARG2(__lsx_vxori_b, src11, 128, src8, 128, src11, src8); + LSX_DUP2_ARG2(__lsx_vilvl_b, src11, src10, src8, src11, src76_r, src87_r); + LSX_DUP2_ARG2(__lsx_vilvh_b, src11, src10, src8, src11, src76_l, src87_l); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src32_r, filt0, dst0_r, src10_r, + filt1, dst0_r, dst0_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src32_l, filt0, dst0_l, src10_l, + filt1, dst0_l, dst0_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src43_r, filt0, dst1_r, src21_r, + filt1, dst1_r, dst1_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src43_l, filt0, dst1_l, src21_l, + filt1, dst1_l, dst1_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src98_r, filt0, dst2_r, src76_r, + filt1, dst2_r, dst2_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src98_l, filt0, dst2_l, src76_l, + filt1, dst2_l, dst2_l); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src109_r, filt0, dst3_r, src87_r, + filt1, dst3_r, dst3_r); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, src109_l, filt0, dst3_l, src87_l, + filt1, dst3_l, dst3_l); + + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst0_l, dst, 16); + __lsx_vst(dst2_r, dst, 32); + __lsx_vst(dst2_l, dst, 48); + dst += dst_stride; + __lsx_vst(dst1_r, dst, 0); + __lsx_vst(dst1_l, dst, 16); + __lsx_vst(dst3_r, dst, 32); + __lsx_vst(dst3_l, dst, 48); + dst += dst_stride; + } +} + +static void hevc_hv_4t_8x2_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y) +{ + int32_t src_stride_2x = (src_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + + __m128i src0, src1, src2, src3, src4; + __m128i filt0, filt1; + __m128i filt_h0, filt_h1; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1; + __m128i filter_vec, const_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9; + __m128i dst0, dst1, dst2, dst3, dst4; + __m128i dst0_r, dst0_l, dst1_r, dst1_l; + __m128i dst10_r, dst32_r, dst21_r, dst43_r; + __m128i dst10_l, dst32_l, dst21_l, dst43_l; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src4 = __lsx_vld(src + src_stride_4x, 0); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec6, vec7); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec8, vec9); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, + dst0, dst0); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec2, filt0, dst1, vec3, filt1, + dst1, dst1); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst2, vec5, filt1, + dst2, dst2); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec6, filt0, dst3, vec7, filt1, + dst3, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec8, filt0, dst4, vec9, filt1, + dst4, dst4); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, dst0_r, + dst0_l, dst1_r, dst1_l); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r); + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst1_r, dst + dst_stride, 0); +} + +static void hevc_hv_4t_8multx4_lsx(uint8_t *src, int32_t src_stride, + int16_t *dst, int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, int32_t width8mult) +{ + int32_t cnt; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i src0, src1, src2, src3, src4, src5, src6, mask0, mask1; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i filt0, filt1, filt_h0, filt_h1, filter_vec, const_vec; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6; + __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r; + __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + for (cnt = width8mult; cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src += src_stride_4x; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src4, src5); + src6 = __lsx_vld(src + src_stride_2x, 0); + src += (8 - src_stride_4x); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3) + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, + dst0, dst0); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec2, filt0, dst1, vec3, filt1, + dst1, dst1); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst2, vec5, filt1, + dst2, dst2); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst3, vec1, filt1, + dst3, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec2, filt0, dst4, vec3, filt1, + dst4, dst4); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst5, vec5, filt1, + dst5, dst5); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec6, filt0, dst6, vec7, filt1, + dst6, dst6); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, + dst0_r, dst0_l, dst1_r, dst1_l); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6, + dst2_r, dst2_l, dst3_r, dst3_l); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst2_l, dst2_r, dst3_l, dst3_r, dst2_r, dst3_r); + + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst1_r, dst + dst_stride, 0); + __lsx_vst(dst2_r, dst + dst_stride_2x, 0); + __lsx_vst(dst3_r, dst + dst_stride_3x, 0); + dst += 8; + } +} + +static void hevc_hv_4t_8x6_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y) +{ + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i filt0, filt1; + __m128i filt_h0, filt_h1; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1; + __m128i filter_vec, const_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9; + __m128i vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i dst4_r, dst4_l, dst5_r, dst5_l; + __m128i dst10_r, dst32_r, dst10_l, dst32_l; + __m128i dst21_r, dst43_r, dst21_l, dst43_l; + __m128i dst54_r, dst54_l, dst65_r, dst65_l; + __m128i dst76_r, dst76_l, dst87_r, dst87_l; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src0, src1, src2, src3); + src4 = __lsx_vld(src + src_stride_4x, 0); + src += (src_stride_4x + src_stride); + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src5, src6, src7, src8); + + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + LSX_DUP4_ARG2(__lsx_vxori_b, src5, 128, src6, 128, src7, 128, src8, 128, src5, + src6, src7, src8); + + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src1, src1, + mask0, src1, src1, mask1, vec0, vec1, vec2, vec3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1,src3, src3, + mask0, src3, src3, mask1, vec4, vec5, vec6, vec7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src5, src5, + mask0, src5, src5, mask1, vec8, vec9, vec10, vec11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src7, src7, + mask0, src7, src7, mask1, vec12, vec13, vec14, vec15); + LSX_DUP2_ARG3(__lsx_vshuf_b, src8, src8, mask0, src8, src8, mask1, vec16, vec17); + + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, + const_vec, vec2, filt0, dst1, vec3, filt1, dst0, dst0, dst1, dst1); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst2, vec5, filt1, + const_vec, vec6, filt0, dst3, vec7, filt1, dst2, dst2, dst3, dst3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec8, filt0, dst4, vec9, filt1, + const_vec, vec10, filt0, dst5, vec11, filt1, dst4, dst4, dst5, dst5); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, const_vec, vec12, filt0, dst6, vec13, filt1, + const_vec, vec14, filt0, dst7, vec15, filt1, dst6, dst6, dst7, dst7); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec16, filt0, dst8, vec17, filt1, dst8, + dst8); + + LSX_DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3, + dst10_r, dst21_r, dst32_r, dst43_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3, + dst10_l, dst21_l, dst32_l, dst43_l); + LSX_DUP4_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7, + dst54_r, dst65_r, dst76_r, dst87_r); + LSX_DUP4_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7, + dst54_l, dst65_l, dst76_l, dst87_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + dst4_r = __lsx_hevc_filt_4tap_w(dst54_r, dst76_r, filt_h0, filt_h1); + dst4_l = __lsx_hevc_filt_4tap_w(dst54_l, dst76_l, filt_h0, filt_h1); + dst5_r = __lsx_hevc_filt_4tap_w(dst65_r, dst87_r, filt_h0, filt_h1); + dst5_l = __lsx_hevc_filt_4tap_w(dst65_l, dst87_l, filt_h0, filt_h1); + + LSX_DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, dst0_r, + dst0_l, dst1_r, dst1_l); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6, dst2_r, + dst2_l, dst3_r, dst3_l); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst4_r, 6, dst4_l, 6, dst5_r, 6, dst5_l, 6, dst4_r, + dst4_l, dst5_r, dst5_l); + + LSX_DUP4_ARG2(__lsx_vpickev_h,dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, + dst3_l, dst3_r, dst0_r, dst1_r, dst2_r, dst3_r); + LSX_DUP2_ARG2(__lsx_vpickev_h, dst4_l, dst4_r, dst5_l, dst5_r, dst4_r, dst5_r); + + __lsx_vst(dst0_r, dst, 0); + __lsx_vst(dst1_r, dst + dst_stride, 0); + dst += dst_stride_2x; + __lsx_vst(dst2_r, dst, 0); + __lsx_vst(dst3_r, dst + dst_stride, 0); + dst += dst_stride_2x; + __lsx_vst(dst4_r, dst, 0); + __lsx_vst(dst5_r, dst + dst_stride, 0); +} + +static void hevc_hv_4t_8multx4mult_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height, + int32_t width8mult) +{ + uint32_t loop_cnt, cnt; + uint8_t *src_tmp; + int16_t *dst_tmp; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i src0, src1, src2, src3, src4, src5, src6; + __m128i filt0, filt1; + __m128i filt_h0, filt_h1; + __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + __m128i mask1; + __m128i filter_vec, const_vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r; + __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + for (cnt = width8mult; cnt--;) { + src_tmp = src; + dst_tmp = dst; + + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src0, src1); + src2 = __lsx_vld(src_tmp + src_stride_2x, 0); + src_tmp += src_stride_3x; + + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, + dst0, dst0); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec2, filt0, dst1, vec3, filt1, + dst1, dst1); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst2, vec5, filt1, + dst2, dst2); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l); + + for (loop_cnt = height >> 2; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, + src3, src4, src5, src6); + src_tmp += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, + src3, src4, src5, src6); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst3, vec1, filt1, + dst3, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec2, filt0, dst4, vec3, filt1, + dst4, dst4); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst5, vec5, filt1, + dst5, dst5); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec6, filt0, dst6, vec7, filt1, + dst6, dst6); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + + LSX_DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, + dst0_r, dst0_l, dst1_r, dst1_l); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6, + dst2_r, dst2_l, dst3_r, dst3_l); + + LSX_DUP4_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, + dst2_r, dst3_l, dst3_r, dst0_r, dst1_r, dst2_r, dst3_r); + + __lsx_vst(dst0_r, dst_tmp, 0); + __lsx_vst(dst1_r, dst_tmp + dst_stride, 0); + __lsx_vst(dst2_r, dst_tmp + dst_stride_2x, 0); + __lsx_vst(dst3_r, dst_tmp + dst_stride_3x, 0); + dst_tmp += dst_stride_4x; + + dst10_r = dst54_r; + dst10_l = dst54_l; + dst21_r = dst65_r; + dst21_l = dst65_l; + dst2 = dst6; + } + src += 8; + dst += 8; + } +} + +static void hevc_hv_4t_8w_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + + if (2 == height) { + hevc_hv_4t_8x2_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y); + } else if (4 == height) { + hevc_hv_4t_8multx4_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, 1); + } else if (6 == height) { + hevc_hv_4t_8x6_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y); + } else if (0 == (height & 0x03)) { + hevc_hv_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 1); + } +} + +static void hevc_hv_4t_12w_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + uint32_t loop_cnt; + uint8_t *src_tmp; + int16_t *dst_tmp; + int32_t src_stride_2x = (src_stride << 1); + int32_t dst_stride_2x = (dst_stride << 1); + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + int32_t src_stride_3x = src_stride_2x + src_stride; + int32_t dst_stride_3x = dst_stride_2x + dst_stride; + + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i mask0, mask1, mask2, mask3; + __m128i filt0, filt1, filt_h0, filt_h1, filter_vec, const_vec; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst10, dst21, dst22, dst73; + __m128i dst84, dst95, dst106, dst76_r, dst98_r, dst87_r, dst109_r; + __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r; + __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l; + __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + src -= (src_stride + 1); + LSX_DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1); + + filter_vec = __lsx_vld(filter_y, 0); + filter_vec = __lsx_vsllwil_h_b(filter_vec, 0); + LSX_DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1); + + mask0 = __lsx_vld(ff_hevc_mask_arr, 0); + mask1 = __lsx_vaddi_bu(mask0, 2); + const_vec = __lsx_vreplgr2vr_h(8192); // 128 << 6 + + src_tmp = src; + dst_tmp = dst; + + LSX_DUP2_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, src0, src1); + src2 = __lsx_vld(src_tmp + src_stride_2x, 0); + src_tmp += src_stride_3x; + + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst0, vec1, filt1, + dst0, dst0); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec2, filt0, dst1, vec3, filt1, + dst1, dst1); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst2, vec5, filt1, + dst2, dst2); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l); + + for (loop_cnt = 4; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src_tmp, 0, src_tmp + src_stride, 0, + src_tmp + src_stride_2x, 0, src_tmp + src_stride_3x, 0, src3, + src4, src5, src6); + src_tmp += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6); + + LSX_DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst3, vec1, filt1, + dst3, dst3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec2, filt0, dst4, vec3, filt1, + dst4, dst4); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst5, vec5, filt1, + dst5, dst5); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec6, filt0, dst6, vec7, filt1, + dst6, dst6); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l); + + dst0_r = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + dst0_l = __lsx_hevc_filt_4tap_w(dst10_l, dst32_l, filt_h0, filt_h1); + dst1_r = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + dst1_l = __lsx_hevc_filt_4tap_w(dst21_l, dst43_l, filt_h0, filt_h1); + dst2_r = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + dst2_l = __lsx_hevc_filt_4tap_w(dst32_l, dst54_l, filt_h0, filt_h1); + dst3_r = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + dst3_l = __lsx_hevc_filt_4tap_w(dst43_l, dst65_l, filt_h0, filt_h1); + + LSX_DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, + dst0_r, dst0_l, dst1_r, dst1_l); + LSX_DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6, + dst2_r, dst2_l, dst3_r, dst3_l); + LSX_DUP4_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, + dst3_l, dst3_r, dst0_r, dst1_r, dst2_r, dst3_r); + __lsx_vst(dst0_r, dst_tmp, 0); + __lsx_vst(dst1_r, dst_tmp + dst_stride, 0); + __lsx_vst(dst2_r, dst_tmp + dst_stride_2x, 0); + __lsx_vst(dst3_r, dst_tmp + dst_stride_3x, 0); + dst_tmp += dst_stride_4x; + + dst10_r = dst54_r; + dst10_l = dst54_l; + dst21_r = dst65_r; + dst21_l = dst65_l; + dst2 = dst6; + } + + src += 8; + dst += 8; + + mask2 = __lsx_vld(ff_hevc_mask_arr, 16); + mask3 = __lsx_vaddi_bu(mask2, 2); + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src2 = __lsx_vld(src + src_stride_2x, 0); + src += src_stride_3x; + LSX_DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + LSX_DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask2, src1, src0, mask3, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src2, src1, mask2, src2, src1, mask3, vec2, vec3); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst10, vec1, filt1, + dst10, dst10); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec2, filt0, dst21, vec3, filt1, + dst21, dst21); + dst10_r = __lsx_vilvl_h(dst21, dst10); + dst21_r = __lsx_vilvh_h(dst21, dst10); + dst22 = __lsx_vreplvei_d(dst21, 1); + + for (loop_cnt = 2; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src3, src4, src5, src6); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src + src_stride_2x, 0, + src + src_stride_3x, 0, src7, src8, src9, src10); + src += src_stride_4x; + LSX_DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6); + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + LSX_DUP2_ARG3(__lsx_vshuf_b, src7, src3, mask2, src7, src3, mask3, vec0, vec1); + LSX_DUP2_ARG3(__lsx_vshuf_b, src8, src4, mask2, src8, src4, mask3, vec2, vec3); + LSX_DUP2_ARG3(__lsx_vshuf_b, src9, src5, mask2, src9, src5, mask3, vec4, vec5); + LSX_DUP2_ARG3(__lsx_vshuf_b, src10, src6, mask2, src10, src6, mask3, vec6, vec7); + + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec0, filt0, dst73, vec1, filt1, + dst73, dst73); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec2, filt0, dst84, vec3, filt1, + dst84, dst84); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec4, filt0, dst95, vec5, filt1, + dst95, dst95); + LSX_DUP2_ARG3(__lsx_dp2add_h_b, const_vec, vec6, filt0, dst106, vec7, filt1, + dst106, dst106); + + LSX_DUP2_ARG2(__lsx_vilvl_h, dst73, dst22, dst84, dst73, dst32_r, dst43_r); + LSX_DUP2_ARG2(__lsx_vilvh_h, dst84, dst73, dst95, dst84, dst87_r, dst98_r); + LSX_DUP2_ARG2(__lsx_vilvl_h, dst95, dst84, dst106, dst95, dst54_r, dst65_r); + dst109_r = __lsx_vilvh_h(dst106, dst95); + dst22 = __lsx_vreplvei_d(dst73, 1); + dst76_r = __lsx_vilvl_h(dst22, dst106); + + tmp0 = __lsx_hevc_filt_4tap_w(dst10_r, dst32_r, filt_h0, filt_h1); + tmp1 = __lsx_hevc_filt_4tap_w(dst21_r, dst43_r, filt_h0, filt_h1); + tmp2 = __lsx_hevc_filt_4tap_w(dst32_r, dst54_r, filt_h0, filt_h1); + tmp3 = __lsx_hevc_filt_4tap_w(dst43_r, dst65_r, filt_h0, filt_h1); + tmp4 = __lsx_hevc_filt_4tap_w(dst54_r, dst76_r, filt_h0, filt_h1); + tmp5 = __lsx_hevc_filt_4tap_w(dst65_r, dst87_r, filt_h0, filt_h1); + tmp6 = __lsx_hevc_filt_4tap_w(dst76_r, dst98_r, filt_h0, filt_h1); + tmp7 = __lsx_hevc_filt_4tap_w(dst87_r, dst109_r, filt_h0, filt_h1); + + LSX_DUP4_ARG2(__lsx_vsrai_w, tmp0, 6, tmp1, 6, tmp2, 6, tmp3, 6, tmp0, tmp1, + tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vsrai_w, tmp4, 6, tmp5, 6, tmp6, 6, tmp7, 6, tmp4, tmp5, + tmp6, tmp7); + LSX_DUP4_ARG2(__lsx_vpickev_h, tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, + tmp0, tmp1, tmp2, tmp3); + + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + __lsx_vstelm_d(tmp2, dst, 0, 0); + __lsx_vstelm_d(tmp2, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp3, dst + dst_stride_2x, 0, 0); + __lsx_vstelm_d(tmp3, dst + dst_stride_3x, 0, 1); + dst += dst_stride_4x; + + dst10_r = dst98_r; + dst21_r = dst109_r; + dst22 = __lsx_vreplvei_d(dst106, 1); + } +} + +static void hevc_hv_4t_16w_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + if (4 == height) { + hevc_hv_4t_8multx4_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, 2); + } else { + hevc_hv_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 2); + } +} + +static void hevc_hv_4t_24w_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 3); +} + +static void hevc_hv_4t_32w_lsx(uint8_t *src, + int32_t src_stride, + int16_t *dst, + int32_t dst_stride, + const int8_t *filter_x, + const int8_t *filter_y, + int32_t height) +{ + hevc_hv_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride, + filter_x, filter_y, height, 4); +} + +#define MC_COPY(WIDTH) \ +void ff_hevc_put_hevc_pel_pixels##WIDTH##_8_lsx(int16_t *dst, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) \ +{ \ + hevc_copy_##WIDTH##w_lsx(src, src_stride, dst, MAX_PB_SIZE, height); \ +} + +MC_COPY(4); +MC_COPY(6); +MC_COPY(8); +MC_COPY(12); +MC_COPY(16); +MC_COPY(24); +MC_COPY(32); +MC_COPY(48); +MC_COPY(64); + +#undef MC_COPY + +#define MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \ +void ff_hevc_put_hevc_##PEL##_##DIR##WIDTH##_8_lsx(int16_t *dst, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) \ +{ \ + const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR - 1]; \ + \ + hevc_##DIR1##_##TAP##t_##WIDTH##w_lsx(src, src_stride, dst, \ + MAX_PB_SIZE, filter, height); \ +} + +MC(qpel, h, 4, 8, hz, mx); +MC(qpel, h, 8, 8, hz, mx); +MC(qpel, h, 12, 8, hz, mx); +MC(qpel, h, 16, 8, hz, mx); +MC(qpel, h, 24, 8, hz, mx); +MC(qpel, h, 32, 8, hz, mx); +MC(qpel, h, 48, 8, hz, mx); +MC(qpel, h, 64, 8, hz, mx); + +MC(qpel, v, 4, 8, vt, my); +MC(qpel, v, 8, 8, vt, my); +MC(qpel, v, 12, 8, vt, my); +MC(qpel, v, 16, 8, vt, my); +MC(qpel, v, 24, 8, vt, my); +MC(qpel, v, 32, 8, vt, my); +MC(qpel, v, 48, 8, vt, my); +MC(qpel, v, 64, 8, vt, my); + +MC(epel, h, 32, 4, hz, mx); + +MC(epel, v, 16, 4, vt, my); +MC(epel, v, 24, 4, vt, my); +MC(epel, v, 32, 4, vt, my); + +#undef MC + +#define MC_HV(PEL, WIDTH, TAP) \ +void ff_hevc_put_hevc_##PEL##_hv##WIDTH##_8_lsx(int16_t *dst, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) \ +{ \ + const int8_t *filter_x = ff_hevc_##PEL##_filters[mx - 1]; \ + const int8_t *filter_y = ff_hevc_##PEL##_filters[my - 1]; \ + \ + hevc_hv_##TAP##t_##WIDTH##w_lsx(src, src_stride, dst, MAX_PB_SIZE, \ + filter_x, filter_y, height); \ +} + +MC_HV(qpel, 4, 8); +MC_HV(qpel, 8, 8); +MC_HV(qpel, 12, 8); +MC_HV(qpel, 16, 8); +MC_HV(qpel, 24, 8); +MC_HV(qpel, 32, 8); +MC_HV(qpel, 48, 8); +MC_HV(qpel, 64, 8); + +MC_HV(epel, 8, 4); +MC_HV(epel, 12, 4); +MC_HV(epel, 16, 4); +MC_HV(epel, 24, 4); +MC_HV(epel, 32, 4); + +#undef MC_HV diff --git a/libavcodec/loongarch/hevcdsp_lsx.h b/libavcodec/loongarch/hevcdsp_lsx.h new file mode 100644 index 0000000000..609cfa081b --- /dev/null +++ b/libavcodec/loongarch/hevcdsp_lsx.h @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_HEVCDSP_LSX_H +#define AVCODEC_LOONGARCH_HEVCDSP_LSX_H + +#include "libavcodec/hevcdsp.h" + +#define MC(PEL, DIR, WIDTH) \ +void ff_hevc_put_hevc_##PEL##_##DIR##WIDTH##_8_lsx(int16_t *dst, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) + +MC(pel, pixels, 4); +MC(pel, pixels, 6); +MC(pel, pixels, 8); +MC(pel, pixels, 12); +MC(pel, pixels, 16); +MC(pel, pixels, 24); +MC(pel, pixels, 32); +MC(pel, pixels, 48); +MC(pel, pixels, 64); + +MC(qpel, h, 4); +MC(qpel, h, 8); +MC(qpel, h, 12); +MC(qpel, h, 16); +MC(qpel, h, 24); +MC(qpel, h, 32); +MC(qpel, h, 48); +MC(qpel, h, 64); + +MC(qpel, v, 4); +MC(qpel, v, 8); +MC(qpel, v, 12); +MC(qpel, v, 16); +MC(qpel, v, 24); +MC(qpel, v, 32); +MC(qpel, v, 48); +MC(qpel, v, 64); + +MC(qpel, hv, 4); +MC(qpel, hv, 8); +MC(qpel, hv, 12); +MC(qpel, hv, 16); +MC(qpel, hv, 24); +MC(qpel, hv, 32); +MC(qpel, hv, 48); +MC(qpel, hv, 64); + +MC(epel, h, 32); + +MC(epel, v, 16); +MC(epel, v, 24); +MC(epel, v, 32); + +MC(epel, hv, 8); +MC(epel, hv, 12); +MC(epel, hv, 16); +MC(epel, hv, 24); +MC(epel, hv, 32); + +#undef MC + +#define UNI_MC(PEL, DIR, WIDTH) \ +void ff_hevc_put_hevc_uni_##PEL##_##DIR##WIDTH##_8_lsx(uint8_t *dst, \ + ptrdiff_t dst_stride, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) + +UNI_MC(qpel, h, 64); + +UNI_MC(qpel, v, 24); +UNI_MC(qpel, v, 32); +UNI_MC(qpel, v, 48); +UNI_MC(qpel, v, 64); + +UNI_MC(qpel, hv, 8); +UNI_MC(qpel, hv, 16); +UNI_MC(qpel, hv, 24); +UNI_MC(qpel, hv, 32); +UNI_MC(qpel, hv, 48); +UNI_MC(qpel, hv, 64); + +UNI_MC(epel, v, 24); +UNI_MC(epel, v, 32); + +UNI_MC(epel, hv, 8); +UNI_MC(epel, hv, 12); +UNI_MC(epel, hv, 16); +UNI_MC(epel, hv, 24); +UNI_MC(epel, hv, 32); + +#undef UNI_MC + +#define UNI_W_MC(PEL, DIR, WIDTH) \ +void ff_hevc_put_hevc_uni_w_##PEL##_##DIR##WIDTH##_8_lsx(uint8_t *dst, \ + ptrdiff_t \ + dst_stride, \ + uint8_t *src, \ + ptrdiff_t \ + src_stride, \ + int height, \ + int denom, \ + int weight, \ + int offset, \ + intptr_t mx, \ + intptr_t my, \ + int width) + +UNI_W_MC(qpel, hv, 8); +UNI_W_MC(qpel, hv, 16); +UNI_W_MC(qpel, hv, 24); +UNI_W_MC(qpel, hv, 32); +UNI_W_MC(qpel, hv, 48); +UNI_W_MC(qpel, hv, 64); + +#undef UNI_W_MC + +#define BI_MC(PEL, DIR, WIDTH) \ +void ff_hevc_put_hevc_bi_##PEL##_##DIR##WIDTH##_8_lsx(uint8_t *dst, \ + ptrdiff_t dst_stride, \ + uint8_t *src, \ + ptrdiff_t src_stride, \ + int16_t *src_16bit, \ + int height, \ + intptr_t mx, \ + intptr_t my, \ + int width) + +BI_MC(pel, pixels, 4); +BI_MC(pel, pixels, 6); +BI_MC(pel, pixels, 8); +BI_MC(pel, pixels, 12); +BI_MC(pel, pixels, 16); +BI_MC(pel, pixels, 24); +BI_MC(pel, pixels, 32); +BI_MC(pel, pixels, 48); +BI_MC(pel, pixels, 64); + +BI_MC(qpel, h, 16); +BI_MC(qpel, h, 24); +BI_MC(qpel, h, 32); +BI_MC(qpel, h, 48); +BI_MC(qpel, h, 64); + +BI_MC(qpel, v, 8); +BI_MC(qpel, v, 16); +BI_MC(qpel, v, 24); +BI_MC(qpel, v, 32); +BI_MC(qpel, v, 48); +BI_MC(qpel, v, 64); + +BI_MC(qpel, hv, 8); +BI_MC(qpel, hv, 16); +BI_MC(qpel, hv, 24); +BI_MC(qpel, hv, 32); +BI_MC(qpel, hv, 48); +BI_MC(qpel, hv, 64); + +BI_MC(epel, h, 24); +BI_MC(epel, h, 32); + +BI_MC(epel, v, 12); +BI_MC(epel, v, 16); +BI_MC(epel, v, 24); +BI_MC(epel, v, 32); + +BI_MC(epel, hv, 6); +BI_MC(epel, hv, 8); +BI_MC(epel, hv, 16); +BI_MC(epel, hv, 24); +BI_MC(epel, hv, 32); + +#undef BI_MC + +void ff_hevc_loop_filter_luma_h_8_lsx(uint8_t *src, + ptrdiff_t src_stride, + int32_t beta, int32_t *tc, + uint8_t *no_p, uint8_t *no_q); + +void ff_hevc_loop_filter_luma_v_8_lsx(uint8_t *src, + ptrdiff_t src_stride, + int32_t beta, int32_t *tc, + uint8_t *no_p, uint8_t *no_q); + +void ff_hevc_loop_filter_chroma_h_8_lsx(uint8_t *src, + ptrdiff_t src_stride, + int32_t *tc, uint8_t *no_p, + uint8_t *no_q); + +void ff_hevc_loop_filter_chroma_v_8_lsx(uint8_t *src, + ptrdiff_t src_stride, + int32_t *tc, uint8_t *no_p, + uint8_t *no_q); + +void ff_hevc_sao_edge_filter_8_lsx(uint8_t *dst, uint8_t *src, + ptrdiff_t stride_dst, + int16_t *sao_offset_val, + int eo, int width, int height); + +void ff_hevc_idct_4x4_lsx(int16_t *coeffs, int col_limit); +void ff_hevc_idct_8x8_lsx(int16_t *coeffs, int col_limit); +void ff_hevc_idct_16x16_lsx(int16_t *coeffs, int col_limit); +void ff_hevc_idct_32x32_lsx(int16_t *coeffs, int col_limit); + +#endif // #ifndef AVCODEC_LOONGARCH_HEVCDSP_LSX_H diff --git a/libavcodec/loongarch/hpeldsp_init_loongarch.c b/libavcodec/loongarch/hpeldsp_init_loongarch.c new file mode 100644 index 0000000000..924ccd0f40 --- /dev/null +++ b/libavcodec/loongarch/hpeldsp_init_loongarch.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/cpu.h" +#include "../hpeldsp.h" +#include "libavcodec/loongarch/hpeldsp_lasx.h" + +void ff_hpeldsp_init_loongarch(HpelDSPContext *c, int flags) +{ + int cpu_flags = av_get_cpu_flags(); + + if (have_lasx(cpu_flags)) { + c->put_pixels_tab[0][0] = ff_put_pixels16_8_lsx; + c->put_pixels_tab[0][1] = ff_put_pixels16_x2_8_lasx; + c->put_pixels_tab[0][2] = ff_put_pixels16_y2_8_lasx; + c->put_pixels_tab[0][3] = ff_put_pixels16_xy2_8_lasx; + + c->put_pixels_tab[1][0] = ff_put_pixels8_8_lasx; + c->put_pixels_tab[1][1] = ff_put_pixels8_x2_8_lasx; + c->put_pixels_tab[1][2] = ff_put_pixels8_y2_8_lasx; + c->put_pixels_tab[1][3] = ff_put_pixels8_xy2_8_lasx; +// +// c->put_pixels_tab[2][0] = ff_put_pixels4_8_lasx; +// c->put_pixels_tab[2][1] = ff_put_pixels4_x2_8_lasx; +// c->put_pixels_tab[2][2] = ff_put_pixels4_y2_8_lasx; +// c->put_pixels_tab[2][3] = ff_put_pixels4_xy2_8_lasx; +// + c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_8_lsx; + c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_8_lasx; + c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_8_lasx; + c->put_no_rnd_pixels_tab[0][3] = ff_put_no_rnd_pixels16_xy2_8_lasx; + + c->put_no_rnd_pixels_tab[1][0] = ff_put_pixels8_8_lasx; + c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_8_lasx; + c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_8_lasx; + c->put_no_rnd_pixels_tab[1][3] = ff_put_no_rnd_pixels8_xy2_8_lasx; + +// c->avg_pixels_tab[0][0] = ff_avg_pixels16_8_lasx; +// c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_8_lasx; +// c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_8_lasx; +// c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_8_lasx; +// +// c->avg_pixels_tab[1][0] = ff_avg_pixels8_8_lasx; +// c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_8_lasx; +// c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_8_lasx; +// c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_8_lasx; +// +// c->avg_pixels_tab[2][0] = ff_avg_pixels4_8_lasx; +// c->avg_pixels_tab[2][1] = ff_avg_pixels4_x2_8_lasx; +// c->avg_pixels_tab[2][2] = ff_avg_pixels4_y2_8_lasx; +// c->avg_pixels_tab[2][3] = ff_avg_pixels4_xy2_8_lasx; + } +} diff --git a/libavcodec/loongarch/hpeldsp_lasx.c b/libavcodec/loongarch/hpeldsp_lasx.c new file mode 100644 index 0000000000..f88e1244a8 --- /dev/null +++ b/libavcodec/loongarch/hpeldsp_lasx.c @@ -0,0 +1,1041 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lasx.h" +#include "libavcodec/loongarch/hpeldsp_lasx.h" + +static inline void +put_pixels8_l2_8_lsx(uint8_t *dst, const uint8_t *src1, const uint8_t *src2, + int dst_stride, int src_stride1, int src_stride2, int h) +{ + __asm__ volatile ( + "1: \n\t" + "vld $vr0, %[src1], 0 \n\t" + "add.d %[src1], %[src1], %[srcStride1] \n\t" + "vld $vr1, %[src1], 0 \n\t" + "add.d %[src1], %[src1], %[srcStride1] \n\t" + "vld $vr2, %[src1], 0 \n\t" + "add.d %[src1], %[src1], %[srcStride1] \n\t" + "vld $vr3, %[src1], 0 \n\t" + "add.d %[src1], %[src1], %[srcStride1] \n\t" + + "vld $vr4, %[src2], 0 \n\t" + "add.d %[src2], %[src2], %[srcStride2] \n\t" + "vld $vr5, %[src2], 0 \n\t" + "add.d %[src2], %[src2], %[srcStride2] \n\t" + "vld $vr6, %[src2], 0 \n\t" + "add.d %[src2], %[src2], %[srcStride2] \n\t" + "vld $vr7, %[src2], 0 \n\t" + "add.d %[src2], %[src2], %[srcStride2] \n\t" + + "addi.d %[h], %[h], -4 \n\t" + + "vavgr.bu $vr0, $vr4, $vr0 \n\t" + "vavgr.bu $vr1, $vr5, $vr1 \n\t" + "vavgr.bu $vr2, $vr6, $vr2 \n\t" + "vavgr.bu $vr3, $vr7, $vr3 \n\t" + "vstelm.d $vr0, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr1, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr2, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vstelm.d $vr3, %[dst], 0, 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "bnez %[h], 1b \n\t" + + : [dst]"+&r"(dst), [src2]"+&r"(src2), [src1]"+&r"(src1), + [h]"+&r"(h) + : [dstStride]"r"(dst_stride), [srcStride1]"r"(src_stride1), + [srcStride2]"r"(src_stride2) + : "memory" + ); +} + +static inline void +put_pixels16_l2_8_lsx(uint8_t *dst, const uint8_t *src1, const uint8_t *src2, + int dst_stride, int src_stride1, int src_stride2, int h) +{ + __asm__ volatile ( + "1: \n\t" + "vld $vr0, %[src1], 0 \n\t" + "add.d %[src1], %[src1], %[srcStride1] \n\t" + "vld $vr1, %[src1], 0 \n\t" + "add.d %[src1], %[src1], %[srcStride1] \n\t" + "vld $vr2, %[src1], 0 \n\t" + "add.d %[src1], %[src1], %[srcStride1] \n\t" + "vld $vr3, %[src1], 0 \n\t" + "add.d %[src1], %[src1], %[srcStride1] \n\t" + + "vld $vr4, %[src2], 0 \n\t" + "add.d %[src2], %[src2], %[srcStride2] \n\t" + "vld $vr5, %[src2], 0 \n\t" + "add.d %[src2], %[src2], %[srcStride2] \n\t" + "vld $vr6, %[src2], 0 \n\t" + "add.d %[src2], %[src2], %[srcStride2] \n\t" + "vld $vr7, %[src2], 0 \n\t" + "add.d %[src2], %[src2], %[srcStride2] \n\t" + + "addi.d %[h], %[h], -4 \n\t" + + "vavgr.bu $vr0, $vr4, $vr0 \n\t" + "vavgr.bu $vr1, $vr5, $vr1 \n\t" + "vavgr.bu $vr2, $vr6, $vr2 \n\t" + "vavgr.bu $vr3, $vr7, $vr3 \n\t" + "vst $vr0, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr1, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr2, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "vst $vr3, %[dst], 0 \n\t" + "add.d %[dst], %[dst], %[dstStride] \n\t" + "bnez %[h], 1b \n\t" + + : [dst]"+&r"(dst), [src2]"+&r"(src2), [src1]"+&r"(src1), + [h]"+&r"(h) + : [dstStride]"r"(dst_stride), [srcStride1]"r"(src_stride1), + [srcStride2]"r"(src_stride2) + : "memory" + ); +} + +void ff_put_pixels8_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + uint64_t tmp[8]; + int h_8 = h >> 3; + int res = h & 7; + + __asm__ volatile ( + "beqz %[h_8], 2f \n\t" + "1: \n\t" + "ld.d %[tmp0], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp1], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp2], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp3], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp4], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp5], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp6], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "ld.d %[tmp7], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + + "addi.d %[h_8], %[h_8], -1 \n\t" + + "st.d %[tmp0], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp1], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp2], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp3], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp4], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp5], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp6], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "st.d %[tmp7], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "bnez %[h_8], 1b \n\t" + + "2: \n\t" + "beqz %[res], 4f \n\t" + "3: \n\t" + "ld.d %[tmp0], %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "addi.d %[res], %[res], -1 \n\t" + "st.d %[tmp0], %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "bnez %[res], 3b \n\t" + "4: \n\t" + : [tmp0]"=&r"(tmp[0]), [tmp1]"=&r"(tmp[1]), + [tmp2]"=&r"(tmp[2]), [tmp3]"=&r"(tmp[3]), + [tmp4]"=&r"(tmp[4]), [tmp5]"=&r"(tmp[5]), + [tmp6]"=&r"(tmp[6]), [tmp7]"=&r"(tmp[7]), + [dst]"+&r"(block), [src]"+&r"(pixels), + [h_8]"+&r"(h_8), [res]"+&r"(res) + : [stride]"r"(line_size) + : "memory" + ); +} + +void ff_put_pixels16_8_lsx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + int h_8 = h >> 3; + int res = h & 7; + + __asm__ volatile ( + "beqz %[h_8], 2f \n\t" + "1: \n\t" + "vld $vr0, %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr1, %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr2, %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr3, %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr4, %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr5, %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr6, %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "vld $vr7, %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + + "addi.d %[h_8], %[h_8], -1 \n\t" + + "vst $vr0, %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr1, %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr2, %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr3, %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr4, %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr5, %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr6, %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "vst $vr7, %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "bnez %[h_8], 1b \n\t" + + "2: \n\t" + "beqz %[res], 4f \n\t" + "3: \n\t" + "vld $vr0, %[src], 0x0 \n\t" + "add.d %[src], %[src], %[stride] \n\t" + "addi.d %[res], %[res], -1 \n\t" + "vst $vr0, %[dst], 0x0 \n\t" + "add.d %[dst], %[dst], %[stride] \n\t" + "bnez %[res], 3b \n\t" + "4: \n\t" + : [dst]"+&r"(block), [src]"+&r"(pixels), + [h_8]"+&r"(h_8), [res]"+&r"(res) + : [stride]"r"(line_size) + : "memory" + ); +} + +void ff_put_pixels8_x2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + put_pixels8_l2_8_lsx(block, pixels, pixels + 1, line_size, line_size, + line_size, h); +} + +void ff_put_pixels8_y2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + put_pixels8_l2_8_lsx(block, pixels, pixels + line_size, line_size, + line_size, line_size, h); +} + +void ff_put_pixels16_x2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + put_pixels16_l2_8_lsx(block, pixels, pixels + 1, line_size, line_size, + line_size, h); +} + +void ff_put_pixels16_y2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + put_pixels16_l2_8_lsx(block, pixels, pixels + line_size, line_size, + line_size, line_size, h); +} + +static void common_hz_bil_no_rnd_16x16_lasx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride_4x = src_stride << 2; + + LASX_LD_4(src, src_stride, src0, src1, src2, src3); + LASX_LD_4((src + 1), src_stride, src4, src5, src6, src7); + src += src_stride_4x; + LASX_PCKEV_Q_4(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + src0 = __lasx_xvavg_bu(src0, src2); + src1 = __lasx_xvavg_bu(src1, src3); + __lasx_xvstelm_d(src0, dst, 0, 0); + __lasx_xvstelm_d(src0, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src0, dst, 0, 2); + __lasx_xvstelm_d(src0, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 0); + __lasx_xvstelm_d(src1, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 2); + __lasx_xvstelm_d(src1, dst + 8, 0, 3); + dst += dst_stride; + + LASX_LD_4(src, src_stride, src0, src1, src2, src3); + LASX_LD_4((src + 1), src_stride, src4, src5, src6, src7); + src += src_stride_4x; + LASX_PCKEV_Q_4(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + src0 = __lasx_xvavg_bu(src0, src2); + src1 = __lasx_xvavg_bu(src1, src3); + __lasx_xvstelm_d(src0, dst, 0, 0); + __lasx_xvstelm_d(src0, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src0, dst, 0, 2); + __lasx_xvstelm_d(src0, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 0); + __lasx_xvstelm_d(src1, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 2); + __lasx_xvstelm_d(src1, dst + 8, 0, 3); + dst += dst_stride; + + LASX_LD_4(src, src_stride, src0, src1, src2, src3); + LASX_LD_4((src + 1), src_stride, src4, src5, src6, src7); + src += src_stride_4x; + LASX_PCKEV_Q_4(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + src0 = __lasx_xvavg_bu(src0, src2); + src1 = __lasx_xvavg_bu(src1, src3); + __lasx_xvstelm_d(src0, dst, 0, 0); + __lasx_xvstelm_d(src0, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src0, dst, 0, 2); + __lasx_xvstelm_d(src0, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 0); + __lasx_xvstelm_d(src1, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 2); + __lasx_xvstelm_d(src1, dst + 8, 0, 3); + dst += dst_stride; + + LASX_LD_4(src, src_stride, src0, src1, src2, src3); + LASX_LD_4((src + 1), src_stride, src4, src5, src6, src7); + src += src_stride_4x; + LASX_PCKEV_Q_4(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + src0 = __lasx_xvavg_bu(src0, src2); + src1 = __lasx_xvavg_bu(src1, src3); + __lasx_xvstelm_d(src0, dst, 0, 0); + __lasx_xvstelm_d(src0, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src0, dst, 0, 2); + __lasx_xvstelm_d(src0, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 0); + __lasx_xvstelm_d(src1, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 2); + __lasx_xvstelm_d(src1, dst + 8, 0, 3); +} + +static void common_hz_bil_no_rnd_8x16_lasx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride_4x = src_stride << 2; + + LASX_LD_4(src, src_stride, src0, src1, src2, src3); + LASX_LD_4((src + 1), src_stride, src4, src5, src6, src7); + src += src_stride_4x; + LASX_PCKEV_Q_4(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + src0 = __lasx_xvavg_bu(src0, src2); + src1 = __lasx_xvavg_bu(src1, src3); + __lasx_xvstelm_d(src0, dst, 0, 0); + __lasx_xvstelm_d(src0, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src0, dst, 0, 2); + __lasx_xvstelm_d(src0, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 0); + __lasx_xvstelm_d(src1, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 2); + __lasx_xvstelm_d(src1, dst + 8, 0, 3); + dst += dst_stride; + + LASX_LD_4(src, src_stride, src0, src1, src2, src3); + LASX_LD_4((src + 1), src_stride, src4, src5, src6, src7); + src += src_stride_4x; + LASX_PCKEV_Q_4(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + src0 = __lasx_xvavg_bu(src0, src2); + src1 = __lasx_xvavg_bu(src1, src3); + __lasx_xvstelm_d(src0, dst, 0, 0); + __lasx_xvstelm_d(src0, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src0, dst, 0, 2); + __lasx_xvstelm_d(src0, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 0); + __lasx_xvstelm_d(src1, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src1, dst, 0, 2); + __lasx_xvstelm_d(src1, dst + 8, 0, 3); +} + +void ff_put_no_rnd_pixels16_x2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + if (h == 16) { + common_hz_bil_no_rnd_16x16_lasx(pixels, line_size, block, line_size); + } else if (h == 8) { + common_hz_bil_no_rnd_8x16_lasx(pixels, line_size, block, line_size); + } +} + +static void common_vt_bil_no_rnd_16x16_lasx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m256i src9, src10, src11, src12, src13, src14, src15, src16; + int32_t src_stride_8x = src_stride << 3; + + LASX_LD_8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += src_stride_8x; + LASX_LD_8(src, src_stride, src8, src9, src10, src11, src12, src13, src14, src15); + src += src_stride_8x; + src16 = LASX_LD(src); + + LASX_PCKEV_Q_8(src1, src0, src2, src1, src3, src2, src4, src3, + src5, src4, src6, src5, src7, src6, src8, src7, + src0, src1, src2, src3, src4, src5, src6, src7); + LASX_PCKEV_Q_8(src9, src8, src10, src9, src11, src10, src12, src11, + src13, src12, src14, src13, src15, src14, src16, src15, + src8, src9, src10, src11, src12, src13, src14, src15); + src0 = __lasx_xvavg_bu(src0, src1); + src2 = __lasx_xvavg_bu(src2, src3); + src4 = __lasx_xvavg_bu(src4, src5); + src6 = __lasx_xvavg_bu(src6, src7); + src8 = __lasx_xvavg_bu(src8, src9); + src10 = __lasx_xvavg_bu(src10, src11); + src12 = __lasx_xvavg_bu(src12, src13); + src14 = __lasx_xvavg_bu(src14, src15); + + __lasx_xvstelm_d(src0, dst, 0, 0); + __lasx_xvstelm_d(src0, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src0, dst, 0, 2); + __lasx_xvstelm_d(src0, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src2, dst, 0, 0); + __lasx_xvstelm_d(src2, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src2, dst, 0, 2); + __lasx_xvstelm_d(src2, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src4, dst, 0, 0); + __lasx_xvstelm_d(src4, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src4, dst, 0, 2); + __lasx_xvstelm_d(src4, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src6, dst, 0, 0); + __lasx_xvstelm_d(src6, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src6, dst, 0, 2); + __lasx_xvstelm_d(src6, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src8, dst, 0, 0); + __lasx_xvstelm_d(src8, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src8, dst, 0, 2); + __lasx_xvstelm_d(src8, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src10, dst, 0, 0); + __lasx_xvstelm_d(src10, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src10, dst, 0, 2); + __lasx_xvstelm_d(src10, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src12, dst, 0, 0); + __lasx_xvstelm_d(src12, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src12, dst, 0, 2); + __lasx_xvstelm_d(src12, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src14, dst, 0, 0); + __lasx_xvstelm_d(src14, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src14, dst, 0, 2); + __lasx_xvstelm_d(src14, dst + 8, 0, 3); +} + +static void common_vt_bil_no_rnd_8x16_lasx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8; + int32_t src_stride_8x = src_stride << 3; + + LASX_LD_8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += src_stride_8x; + src8 = LASX_LD(src); + + LASX_PCKEV_Q_8(src1, src0, src2, src1, src3, src2, src4, src3, + src5, src4, src6, src5, src7, src6, src8, src7, + src0, src1, src2, src3, src4, src5, src6, src7); + src0 = __lasx_xvavg_bu(src0, src1); + src2 = __lasx_xvavg_bu(src2, src3); + src4 = __lasx_xvavg_bu(src4, src5); + src6 = __lasx_xvavg_bu(src6, src7); + + __lasx_xvstelm_d(src0, dst, 0, 0); + __lasx_xvstelm_d(src0, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src0, dst, 0, 2); + __lasx_xvstelm_d(src0, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src2, dst, 0, 0); + __lasx_xvstelm_d(src2, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src2, dst, 0, 2); + __lasx_xvstelm_d(src2, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src4, dst, 0, 0); + __lasx_xvstelm_d(src4, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src4, dst, 0, 2); + __lasx_xvstelm_d(src4, dst + 8, 0, 3); + dst += dst_stride; + __lasx_xvstelm_d(src6, dst, 0, 0); + __lasx_xvstelm_d(src6, dst + 8, 0, 1); + dst += dst_stride; + __lasx_xvstelm_d(src6, dst, 0, 2); + __lasx_xvstelm_d(src6, dst + 8, 0, 3); +} + +void ff_put_no_rnd_pixels16_y2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + if (h == 16) { + common_vt_bil_no_rnd_16x16_lasx(pixels, line_size, block, line_size); + } else if (h == 8) { + common_vt_bil_no_rnd_8x16_lasx(pixels, line_size, block, line_size); + } +} + +static void common_hv_bil_no_rnd_16x16_lasx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; + __m256i src10, src11, src12, src13, src14, src15, src16, src17; + __m256i sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7; + int32_t src_stride_8x = src_stride << 3; + + LASX_LD_8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + LASX_LD_8((src + 1), src_stride, + src9, src10, src11, src12, src13, src14, src15, src16); + src += src_stride_8x; + src8 = LASX_LD(src); + src17 = LASX_LD(src + 1); + + LASX_ILVL_H_8(src9, src0, src10, src1, src11, src2, src12, src3, + src13, src4, src14, src5, src15, src6, src16, src7, + src0, src1, src2, src3, src4, src5, src6, src7); + LASX_ILVL_H(src17, src8, src8); + src0 = __lasx_xvhaddw_hu_bu(src0, src0); + src1 = __lasx_xvhaddw_hu_bu(src1, src1); + src2 = __lasx_xvhaddw_hu_bu(src2, src2); + src3 = __lasx_xvhaddw_hu_bu(src3, src3); + src4 = __lasx_xvhaddw_hu_bu(src4, src4); + src5 = __lasx_xvhaddw_hu_bu(src5, src5); + src6 = __lasx_xvhaddw_hu_bu(src6, src6); + src7 = __lasx_xvhaddw_hu_bu(src7, src7); + src8 = __lasx_xvhaddw_hu_bu(src8, src8); + LASX_ADD_H_8(src0, src1, src1, src2, src2, src3, src3, src4, + src4, src5, src5, src6, src6, src7, src7, src8, + sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7); + sum0 = __lasx_xvaddi_hu(sum0, 1); + sum1 = __lasx_xvaddi_hu(sum1, 1); + sum2 = __lasx_xvaddi_hu(sum2, 1); + sum3 = __lasx_xvaddi_hu(sum3, 1); + sum4 = __lasx_xvaddi_hu(sum4, 1); + sum5 = __lasx_xvaddi_hu(sum5, 1); + sum6 = __lasx_xvaddi_hu(sum6, 1); + sum7 = __lasx_xvaddi_hu(sum7, 1); + sum0 = __lasx_xvsrai_h(sum0, 2); + sum1 = __lasx_xvsrai_h(sum1, 2); + sum2 = __lasx_xvsrai_h(sum2, 2); + sum3 = __lasx_xvsrai_h(sum3, 2); + sum4 = __lasx_xvsrai_h(sum4, 2); + sum5 = __lasx_xvsrai_h(sum5, 2); + sum6 = __lasx_xvsrai_h(sum6, 2); + sum7 = __lasx_xvsrai_h(sum7, 2); + LASX_PCKEV_B_4_128SV(sum1, sum0, sum3, sum2, sum5, sum4, sum7, sum6, + sum0, sum1, sum2, sum3); + LASX_ST_D_2(sum0, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum0, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum1, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum1, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum2, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum2, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum3, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum3, 1, 3, dst, 8); + dst += dst_stride; + + LASX_LD_8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + LASX_LD_8((src + 1), src_stride, + src9, src10, src11, src12, src13, src14, src15, src16); + src += src_stride_8x; + src8 = LASX_LD(src); + src17 = LASX_LD(src + 1); + + LASX_ILVL_H_8(src9, src0, src10, src1, src11, src2, src12, src3, + src13, src4, src14, src5, src15, src6, src16, src7, + src0, src1, src2, src3, src4, src5, src6, src7); + LASX_ILVL_H(src17, src8, src8); + src0 = __lasx_xvhaddw_hu_bu(src0, src0); + src1 = __lasx_xvhaddw_hu_bu(src1, src1); + src2 = __lasx_xvhaddw_hu_bu(src2, src2); + src3 = __lasx_xvhaddw_hu_bu(src3, src3); + src4 = __lasx_xvhaddw_hu_bu(src4, src4); + src5 = __lasx_xvhaddw_hu_bu(src5, src5); + src6 = __lasx_xvhaddw_hu_bu(src6, src6); + src7 = __lasx_xvhaddw_hu_bu(src7, src7); + src8 = __lasx_xvhaddw_hu_bu(src8, src8); + LASX_ADD_H_8(src0, src1, src1, src2, src2, src3, src3, src4, + src4, src5, src5, src6, src6, src7, src7, src8, + sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7); + sum0 = __lasx_xvaddi_hu(sum0, 1); + sum1 = __lasx_xvaddi_hu(sum1, 1); + sum2 = __lasx_xvaddi_hu(sum2, 1); + sum3 = __lasx_xvaddi_hu(sum3, 1); + sum4 = __lasx_xvaddi_hu(sum4, 1); + sum5 = __lasx_xvaddi_hu(sum5, 1); + sum6 = __lasx_xvaddi_hu(sum6, 1); + sum7 = __lasx_xvaddi_hu(sum7, 1); + sum0 = __lasx_xvsrai_h(sum0, 2); + sum1 = __lasx_xvsrai_h(sum1, 2); + sum2 = __lasx_xvsrai_h(sum2, 2); + sum3 = __lasx_xvsrai_h(sum3, 2); + sum4 = __lasx_xvsrai_h(sum4, 2); + sum5 = __lasx_xvsrai_h(sum5, 2); + sum6 = __lasx_xvsrai_h(sum6, 2); + sum7 = __lasx_xvsrai_h(sum7, 2); + LASX_PCKEV_B_4_128SV(sum1, sum0, sum3, sum2, sum5, sum4, sum7, sum6, + sum0, sum1, sum2, sum3); + LASX_ST_D_2(sum0, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum0, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum1, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum1, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum2, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum2, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum3, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum3, 1, 3, dst, 8); +} + +static void common_hv_bil_no_rnd_8x16_lasx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; + __m256i src10, src11, src12, src13, src14, src15, src16, src17; + __m256i sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7; + int32_t src_stride_8x = src_stride << 3; + + LASX_LD_8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + LASX_LD_8((src + 1), src_stride, + src9, src10, src11, src12, src13, src14, src15, src16); + src += src_stride_8x; + src8 = LASX_LD(src); + src17 = LASX_LD(src + 1); + + LASX_ILVL_H_8(src9, src0, src10, src1, src11, src2, src12, src3, + src13, src4, src14, src5, src15, src6, src16, src7, + src0, src1, src2, src3, src4, src5, src6, src7); + LASX_ILVL_H(src17, src8, src8); + src0 = __lasx_xvhaddw_hu_bu(src0, src0); + src1 = __lasx_xvhaddw_hu_bu(src1, src1); + src2 = __lasx_xvhaddw_hu_bu(src2, src2); + src3 = __lasx_xvhaddw_hu_bu(src3, src3); + src4 = __lasx_xvhaddw_hu_bu(src4, src4); + src5 = __lasx_xvhaddw_hu_bu(src5, src5); + src6 = __lasx_xvhaddw_hu_bu(src6, src6); + src7 = __lasx_xvhaddw_hu_bu(src7, src7); + src8 = __lasx_xvhaddw_hu_bu(src8, src8); + LASX_ADD_H_8(src0, src1, src1, src2, src2, src3, src3, src4, + src4, src5, src5, src6, src6, src7, src7, src8, + sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7); + sum0 = __lasx_xvaddi_hu(sum0, 1); + sum1 = __lasx_xvaddi_hu(sum1, 1); + sum2 = __lasx_xvaddi_hu(sum2, 1); + sum3 = __lasx_xvaddi_hu(sum3, 1); + sum4 = __lasx_xvaddi_hu(sum4, 1); + sum5 = __lasx_xvaddi_hu(sum5, 1); + sum6 = __lasx_xvaddi_hu(sum6, 1); + sum7 = __lasx_xvaddi_hu(sum7, 1); + sum0 = __lasx_xvsrai_h(sum0, 2); + sum1 = __lasx_xvsrai_h(sum1, 2); + sum2 = __lasx_xvsrai_h(sum2, 2); + sum3 = __lasx_xvsrai_h(sum3, 2); + sum4 = __lasx_xvsrai_h(sum4, 2); + sum5 = __lasx_xvsrai_h(sum5, 2); + sum6 = __lasx_xvsrai_h(sum6, 2); + sum7 = __lasx_xvsrai_h(sum7, 2); + LASX_PCKEV_B_4_128SV(sum1, sum0, sum3, sum2, sum5, sum4, sum7, sum6, + sum0, sum1, sum2, sum3); + LASX_ST_D_2(sum0, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum0, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum1, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum1, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum2, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum2, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum3, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum3, 1, 3, dst, 8); +} + +void ff_put_no_rnd_pixels16_xy2_8_lasx(uint8_t *block, + const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + if (h == 16) { + common_hv_bil_no_rnd_16x16_lasx(pixels, line_size, block, line_size); + } else if (h == 8) { + common_hv_bil_no_rnd_8x16_lasx(pixels, line_size, block, line_size); + } +} + +static void common_hz_bil_no_rnd_8x8_lasx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i src8, src9, src10, src11, src12, src13, src14, src15; + + LASX_LD_8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + LASX_LD_8((src + 1), src_stride, + src8, src9, src10, src11, src12, src13, src14, src15); + LASX_PCKEV_D_4_128SV(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + LASX_PCKEV_D_4_128SV(src9, src8, src11, src10, src13, src12, src15, src14, + src4, src5, src6, src7); + LASX_PCKEV_Q_4(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + src0 = __lasx_xvavg_bu(src0, src2); + src1 = __lasx_xvavg_bu(src1, src3); + LASX_ST_D_4(src0, 0, 1, 2, 3, dst, dst_stride); + dst += (dst_stride << 2); + LASX_ST_D_4(src1, 0, 1, 2, 3, dst, dst_stride); +} + +static void common_hz_bil_no_rnd_4x8_lasx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + LASX_LD_4(src, src_stride, src0, src1, src2, src3); + LASX_LD_4((src + 1), src_stride, src4, src5, src6, src7); + LASX_PCKEV_D_4_128SV(src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + LASX_PCKEV_Q_2(src1, src0, src3, src2, src0, src1); + src0 = __lasx_xvavg_bu(src0, src1); + LASX_ST_D_4(src0, 0, 1, 2, 3, dst, dst_stride); +} + +void ff_put_no_rnd_pixels8_x2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + if (h == 8) { + common_hz_bil_no_rnd_8x8_lasx(pixels, line_size, block, line_size); + } else if (h == 4) { + common_hz_bil_no_rnd_4x8_lasx(pixels, line_size, block, line_size); + } +} + +static void common_vt_bil_no_rnd_8x8_lasx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8; + + LASX_LD_8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + src += (src_stride << 3); + src8 = LASX_LD(src); + + LASX_PCKEV_D_4_128SV(src1, src0, src2, src1, src3, src2, src4, src3, + src0, src1, src2, src3); + LASX_PCKEV_D_4_128SV(src5, src4, src6, src5, src7, src6, src8, src7, + src4, src5, src6, src7); + LASX_PCKEV_Q_4(src2, src0, src3, src1, src6, src4, src7, src5, + src0, src1, src2, src3); + src0 = __lasx_xvavg_bu(src0, src1); + src1 = __lasx_xvavg_bu(src2, src3); + LASX_ST_D_4(src0, 0, 1, 2, 3, dst, dst_stride); + dst += (dst_stride << 2); + LASX_ST_D_4(src1, 0, 1, 2, 3, dst, dst_stride); +} + +static void common_vt_bil_no_rnd_4x8_lasx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4; + + LASX_LD_4(src, src_stride, src0, src1, src2, src3); + src += (src_stride << 2); + src4 = LASX_LD(src); + LASX_PCKEV_D_4_128SV(src1, src0, src2, src1, src3, src2, src4, src3, + src0, src1, src2, src3); + LASX_PCKEV_Q_2(src2, src0, src3, src1, src0, src1); + src0 = __lasx_xvavg_bu(src0, src1); + LASX_ST_D_4(src0, 0, 1, 2, 3, dst, dst_stride); +} + +void ff_put_no_rnd_pixels8_y2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + if (h == 8) { + common_vt_bil_no_rnd_8x8_lasx(pixels, line_size, block, line_size); + } else if (h == 4) { + common_vt_bil_no_rnd_4x8_lasx(pixels, line_size, block, line_size); + } +} + +static void common_hv_bil_no_rnd_8x8_lasx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i src8, src9, src10, src11, src12, src13, src14, src15, src16, src17; + __m256i sum0, sum1, sum2, sum3; + + LASX_LD_8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + LASX_LD_8((src + 1), src_stride, + src9, src10, src11, src12, src13, src14, src15, src16); + src += (src_stride << 3); + src8 = LASX_LD(src); + src17 = LASX_LD(src + 1); + + LASX_ILVL_B_8_128SV(src9, src0, src10, src1, src11, src2, src12, src3, + src13, src4, src14, src5, src15, src6, src16, src7, + src0, src1, src2, src3, src4, src5, src6, src7); + LASX_ILVL_B_128SV(src17, src8, src8); + LASX_PCKEV_Q_8(src1, src0, src2, src1, src3, src2, src4, src3, + src5, src4, src6, src5, src7, src6, src8, src7, + src0, src1, src2, src3, src4, src5, src6, src7); + src0 = __lasx_xvhaddw_hu_bu(src0, src0); + src1 = __lasx_xvhaddw_hu_bu(src1, src1); + src2 = __lasx_xvhaddw_hu_bu(src2, src2); + src3 = __lasx_xvhaddw_hu_bu(src3, src3); + src4 = __lasx_xvhaddw_hu_bu(src4, src4); + src5 = __lasx_xvhaddw_hu_bu(src5, src5); + src6 = __lasx_xvhaddw_hu_bu(src6, src6); + src7 = __lasx_xvhaddw_hu_bu(src7, src7); + LASX_ADD_H_4(src0, src1, src2, src3, src4, src5, src6, src7, + sum0, sum1, sum2, sum3); + sum0 = __lasx_xvaddi_hu(sum0, 1); + sum1 = __lasx_xvaddi_hu(sum1, 1); + sum2 = __lasx_xvaddi_hu(sum2, 1); + sum3 = __lasx_xvaddi_hu(sum3, 1); + sum0 = __lasx_xvsrai_h(sum0, 2); + sum1 = __lasx_xvsrai_h(sum1, 2); + sum2 = __lasx_xvsrai_h(sum2, 2); + sum3 = __lasx_xvsrai_h(sum3, 2); + LASX_PCKEV_B_2_128SV(sum1, sum0, sum3, sum2, sum0, sum1); + LASX_ST_D_4(sum0, 0, 2, 1, 3, dst, dst_stride); + dst += (dst_stride << 2); + LASX_ST_D_4(sum1, 0, 2, 1, 3, dst, dst_stride); +} + +static void common_hv_bil_no_rnd_4x8_lasx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride) +{ + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i src8, src9, sum0, sum1; + + LASX_LD_4(src, src_stride, src0, src1, src2, src3); + LASX_LD_4((src + 1), src_stride, src5, src6, src7, src8); + src += (src_stride << 2); + src4 = LASX_LD(src); + src9 = LASX_LD(src + 1); + + LASX_ILVL_B_4_128SV(src5, src0, src6, src1, src7, src2, src8, src3, + src0, src1, src2, src3); + LASX_ILVL_B_128SV(src9, src4, src4); + LASX_PCKEV_Q_4(src1, src0, src2, src1, src3, src2, src4, src3, + src0, src1, src2, src3); + src0 = __lasx_xvhaddw_hu_bu(src0, src0); + src1 = __lasx_xvhaddw_hu_bu(src1, src1); + src2 = __lasx_xvhaddw_hu_bu(src2, src2); + src3 = __lasx_xvhaddw_hu_bu(src3, src3); + LASX_ADD_H_2(src0, src1, src2, src3, sum0, sum1); + sum0 = __lasx_xvaddi_hu(sum0, 1); + sum1 = __lasx_xvaddi_hu(sum1, 1); + sum0 = __lasx_xvsrai_h(sum0, 2); + sum1 = __lasx_xvsrai_h(sum1, 2); + LASX_PCKEV_B_128SV(sum1, sum0, sum0); + LASX_ST_D_4(sum0, 0, 2, 1, 3, dst, dst_stride); +} + +void ff_put_no_rnd_pixels8_xy2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + if (h == 8) { + common_hv_bil_no_rnd_8x8_lasx(pixels, line_size, block, line_size); + } else if (h == 4) { + common_hv_bil_no_rnd_4x8_lasx(pixels, line_size, block, line_size); + } +} + +static void common_hv_bil_16w_lasx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + uint8_t height) +{ + uint8_t loop_cnt; + __m256i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; + __m256i src10, src11, src12, src13, src14, src15, src16, src17; + __m256i sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7; + int32_t src_stride_8x = (src_stride << 3); + + for (loop_cnt = (height >> 3); loop_cnt--;) { + LASX_LD_8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7); + LASX_LD_8((src + 1), src_stride, + src9, src10, src11, src12, src13, src14, src15, src16); + src += src_stride_8x; + + src8 = LASX_LD(src); + src17 = LASX_LD(src + 1); + + LASX_ILVL_H_8(src9, src0, src10, src1, src11, src2, src12, src3, + src13, src4, src14, src5, src15, src6, src16, src7, + src0, src1, src2, src3, src4, src5, src6, src7); + LASX_ILVL_H(src17, src8, src8); + src0 = __lasx_xvhaddw_hu_bu(src0, src0); + src1 = __lasx_xvhaddw_hu_bu(src1, src1); + src2 = __lasx_xvhaddw_hu_bu(src2, src2); + src3 = __lasx_xvhaddw_hu_bu(src3, src3); + src4 = __lasx_xvhaddw_hu_bu(src4, src4); + src5 = __lasx_xvhaddw_hu_bu(src5, src5); + src6 = __lasx_xvhaddw_hu_bu(src6, src6); + src7 = __lasx_xvhaddw_hu_bu(src7, src7); + src8 = __lasx_xvhaddw_hu_bu(src8, src8); + LASX_ADD_H_8(src0, src1, src1, src2, src2, src3, src3, src4, + src4, src5, src5, src6, src6, src7, src7, src8, + sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7); + LASX_SRARI_H_4(sum0, sum1, sum2, sum3, sum0, sum1, sum2, sum3, 2); + LASX_SRARI_H_4(sum4, sum5, sum6, sum7, sum4, sum5, sum6, sum7, 2); + LASX_PCKEV_B_4_128SV(sum1, sum0, sum3, sum2, sum5, sum4, sum7, sum6, + sum0, sum1, sum2, sum3); + LASX_ST_D_2(sum0, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum0, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum1, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum1, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum2, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum2, 1, 3, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum3, 0, 2, dst, 8); + dst += dst_stride; + LASX_ST_D_2(sum3, 1, 3, dst, 8); + dst += dst_stride; + } +} + +void ff_put_pixels16_xy2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + common_hv_bil_16w_lasx(pixels, line_size, block, line_size, h); +} + +static void common_hv_bil_8w_lasx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + uint8_t height) +{ + uint8_t loop_cnt; + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i src8, src9, sum0, sum1; + int32_t src_stride_4x = (src_stride << 2); + int32_t dst_stride_4x = (dst_stride << 2); + + src0 = LASX_LD(src); + src5 = LASX_LD(src + 1); + src += src_stride; + + for (loop_cnt = (height >> 2); loop_cnt--;) { + LASX_LD_4(src, src_stride, src1, src2, src3, src4); + LASX_LD_4((src + 1), src_stride, src6, src7, src8, src9); + src += src_stride_4x; + LASX_ILVL_B_4_128SV(src5, src0, src6, src1, src7, src2, src8, src3, + src0, src1, src2, src3); + LASX_ILVL_B_128SV(src9, src4, src5); + LASX_PCKEV_Q_4(src1, src0, src2, src1, src3, src2, src5, src3, + src0, src1, src2, src3); + src0 = __lasx_xvhaddw_hu_bu(src0, src0); + src1 = __lasx_xvhaddw_hu_bu(src1, src1); + src2 = __lasx_xvhaddw_hu_bu(src2, src2); + src3 = __lasx_xvhaddw_hu_bu(src3, src3); + LASX_ADD_H_2(src0, src1, src2, src3, sum0, sum1); + LASX_SRARI_H_2(sum0, sum1, sum0, sum1, 2); + LASX_PCKEV_B_128SV(sum1, sum0, sum0); + LASX_ST_D_4(sum0, 0, 2, 1, 3, dst, dst_stride); + dst += dst_stride_4x; + src0 = src4; + src5 = src9; + } +} + +void ff_put_pixels8_xy2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h) +{ + common_hv_bil_8w_lasx(pixels, line_size, block, line_size, h); +} diff --git a/libavcodec/loongarch/hpeldsp_lasx.h b/libavcodec/loongarch/hpeldsp_lasx.h new file mode 100644 index 0000000000..0477cb54ea --- /dev/null +++ b/libavcodec/loongarch/hpeldsp_lasx.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_HPELDSP_LASX_H +#define AVCODEC_LOONGARCH_HPELDSP_LASX_H + +#include +#include "libavutil/attributes.h" + +void ff_put_pixels8_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h); +void ff_put_pixels8_x2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int32_t h); +void ff_put_pixels8_y2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int32_t h); +void ff_put_pixels16_8_lsx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h); +void ff_put_pixels16_x2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int32_t h); +void ff_put_pixels16_y2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int32_t h); +void ff_put_no_rnd_pixels16_x2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h); +void ff_put_no_rnd_pixels16_y2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h); +void ff_put_no_rnd_pixels16_xy2_8_lasx(uint8_t *block, + const uint8_t *pixels, + ptrdiff_t line_size, int h); +void ff_put_no_rnd_pixels8_x2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h); +void ff_put_no_rnd_pixels8_y2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h); +void ff_put_no_rnd_pixels8_xy2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h); +void ff_put_pixels8_xy2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h); +void ff_put_pixels16_xy2_8_lasx(uint8_t *block, const uint8_t *pixels, + ptrdiff_t line_size, int h); +#endif diff --git a/libavcodec/loongarch/idctdsp_init_loongarch.c b/libavcodec/loongarch/idctdsp_init_loongarch.c new file mode 100644 index 0000000000..9d1d21cc18 --- /dev/null +++ b/libavcodec/loongarch/idctdsp_init_loongarch.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/cpu.h" +#include "idctdsp_loongarch.h" +#include "libavcodec/xvididct.h" + +av_cold void ff_idctdsp_init_loongarch(IDCTDSPContext *c, AVCodecContext *avctx, + unsigned high_bit_depth) +{ + int cpu_flags = av_get_cpu_flags(); + + if (have_lasx(cpu_flags)) { + if ((avctx->lowres != 1) && (avctx->lowres != 2) && (avctx->lowres != 3) && + (avctx->bits_per_raw_sample != 10) && + (avctx->bits_per_raw_sample != 12) && + (avctx->idct_algo == FF_IDCT_AUTO)) { + c->idct_put = ff_simple_idct_put_lasx; + c->idct_add = ff_simple_idct_add_lasx; + c->idct = ff_simple_idct_lasx; + c->perm_type = FF_IDCT_PERM_NONE; + } + c->put_pixels_clamped = ff_put_pixels_clamped_lasx; + c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_lasx; + c->add_pixels_clamped = ff_add_pixels_clamped_lasx; + } +} diff --git a/libavcodec/loongarch/idctdsp_lasx.c b/libavcodec/loongarch/idctdsp_lasx.c new file mode 100644 index 0000000000..78f15f17ee --- /dev/null +++ b/libavcodec/loongarch/idctdsp_lasx.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "idctdsp_loongarch.h" +#include "libavutil/loongarch/generic_macros_lasx.h" + +static void put_pixels_clamped_lasx(const int16_t *block, uint8_t *pixels, + int32_t stride) +{ + __m256i b0, b1, b2, b3; + __m256i temp0, temp1; + + LASX_LD_4(block, 16, b0, b1, b2, b3); + LASX_CLIP_H_0_255_4(b0, b1, b2, b3, b0, b1, b2, b3); + LASX_PCKEV_B_2_128SV(b1, b0, b3, b2, temp0, temp1); + LASX_ST_D_4(temp0, 0, 2, 1, 3, pixels, stride); + pixels += (stride << 2); + LASX_ST_D_4(temp1, 0, 2, 1, 3, pixels, stride); +} + +static void put_signed_pixels_clamped_lasx(const int16_t *block, uint8_t *pixels, + int32_t stride) +{ + __m256i b0, b1, b2, b3; + __m256i temp0, temp1; + __m256i const_128 = {0x0080008000800080, 0x0080008000800080, 0x0080008000800080, 0x0080008000800080}; + + LASX_LD_4(block, 16, b0, b1, b2, b3); + b0 = __lasx_xvadd_h(b0, const_128); + b1 = __lasx_xvadd_h(b1, const_128); + b2 = __lasx_xvadd_h(b2, const_128); + b3 = __lasx_xvadd_h(b3, const_128); + LASX_CLIP_H_0_255_4(b0, b1, b2, b3, b0, b1, b2, b3); + LASX_PCKEV_B_2_128SV(b1, b0, b3, b2, temp0, temp1); + LASX_ST_D_4(temp0, 0, 2, 1, 3, pixels, stride); + pixels += (stride << 2); + LASX_ST_D_4(temp1, 0, 2, 1, 3, pixels, stride); +} + +static void add_pixels_clamped_lasx(const int16_t *block, uint8_t *pixels, + int32_t stride) +{ + __m256i b0, b1, b2, b3; + __m256i p0, p1, p2, p3, p4, p5, p6, p7; + __m256i temp0, temp1, temp2, temp3; + uint8_t *pix = pixels; + + LASX_LD_4(block, 16, b0, b1, b2, b3); + p0 = __lasx_xvldrepl_d(pix, 0); + pix += stride; + p1 = __lasx_xvldrepl_d(pix, 0); + pix += stride; + p2 = __lasx_xvldrepl_d(pix, 0); + pix += stride; + p3 = __lasx_xvldrepl_d(pix, 0); + pix += stride; + p4 = __lasx_xvldrepl_d(pix, 0); + pix += stride; + p5 = __lasx_xvldrepl_d(pix, 0); + pix += stride; + p6 = __lasx_xvldrepl_d(pix, 0); + pix += stride; + p7 = __lasx_xvldrepl_d(pix, 0); + temp0 = __lasx_xvpermi_q(p1, p0, 0x20); + temp1 = __lasx_xvpermi_q(p3, p2, 0x20); + temp2 = __lasx_xvpermi_q(p5, p4, 0x20); + temp3 = __lasx_xvpermi_q(p7, p6, 0x20); + LASX_ADDW_H_H_BU_128SV(b0, temp0, temp0); + LASX_ADDW_H_H_BU_128SV(b1, temp1, temp1); + LASX_ADDW_H_H_BU_128SV(b2, temp2, temp2); + LASX_ADDW_H_H_BU_128SV(b3, temp3, temp3); + LASX_CLIP_H_0_255_4(temp0, temp1, temp2, temp3, temp0, temp1, temp2, temp3); + LASX_PCKEV_B_2_128SV(temp1, temp0, temp3, temp2, temp0, temp1); + LASX_ST_D_4(temp0, 0, 2, 1, 3, pixels, stride); + pixels += (stride << 2); + LASX_ST_D_4(temp1, 0, 2, 1, 3, pixels, stride); +} + +void ff_put_pixels_clamped_lasx(const int16_t *block, + uint8_t *av_restrict pixels, + ptrdiff_t line_size) +{ + put_pixels_clamped_lasx(block, pixels, line_size); +} + +void ff_put_signed_pixels_clamped_lasx(const int16_t *block, + uint8_t *av_restrict pixels, + ptrdiff_t line_size) +{ + put_signed_pixels_clamped_lasx(block, pixels, line_size); +} + +void ff_add_pixels_clamped_lasx(const int16_t *block, + uint8_t *av_restrict pixels, + ptrdiff_t line_size) +{ + add_pixels_clamped_lasx(block, pixels, line_size); +} diff --git a/libavcodec/loongarch/idctdsp_loongarch.h b/libavcodec/loongarch/idctdsp_loongarch.h new file mode 100644 index 0000000000..f36e0f334a --- /dev/null +++ b/libavcodec/loongarch/idctdsp_loongarch.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_IDCTDSP_LOONGARCH_H +#define AVCODEC_LOONGARCH_IDCTDSP_LOONGARCH_H + +#include +#include +#include "../mpegvideo.h" + +void ff_simple_idct_lasx(int16_t *block); +void ff_simple_idct_put_lasx(uint8_t *dest, ptrdiff_t stride_dst, int16_t *block); +void ff_simple_idct_add_lasx(uint8_t *dest, ptrdiff_t stride_dst, int16_t *block); +void ff_put_pixels_clamped_lasx(const int16_t *block, + uint8_t *av_restrict pixels, + ptrdiff_t line_size); +void ff_put_signed_pixels_clamped_lasx(const int16_t *block, + uint8_t *av_restrict pixels, + ptrdiff_t line_size); +void ff_add_pixels_clamped_lasx(const int16_t *block, + uint8_t *av_restrict pixels, + ptrdiff_t line_size); + +#endif /* AVCODEC_LOONGARCH_IDCTDSP_LOONGARCH_H */ diff --git a/libavcodec/loongarch/simple_idct_lasx.c b/libavcodec/loongarch/simple_idct_lasx.c new file mode 100644 index 0000000000..38e879cfa8 --- /dev/null +++ b/libavcodec/loongarch/simple_idct_lasx.c @@ -0,0 +1,320 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/generic_macros_lasx.h" +#include "idctdsp_loongarch.h" + +#define LASX_TRANSPOSE4x16(in_0, in_1, in_2, in_3, out_0, out_1, out_2, out_3) \ +{ \ + __m256i temp_0, temp_1, temp_2, temp_3; \ + __m256i temp_4, temp_5, temp_6, temp_7; \ + temp_0 = __lasx_xvpermi_q(in_2, in_0, 0x20); \ + temp_1 = __lasx_xvpermi_q(in_2, in_0, 0x31); \ + temp_2 = __lasx_xvpermi_q(in_3, in_1, 0x20); \ + temp_3 = __lasx_xvpermi_q(in_3, in_1, 0x31); \ + LASX_ILVLH_H_128SV(temp_1, temp_0, temp_5, temp_4); \ + LASX_ILVLH_H_128SV(temp_3, temp_2, temp_7, temp_6); \ + LASX_ILVLH_W_128SV(temp_6, temp_4, out_1, out_0); \ + LASX_ILVLH_W_128SV(temp_7, temp_5, out_3, out_2); \ +} + +#define LASX_IDCTROWCONDDC \ + const_val = 16383 * ((1 << 19) / 16383); \ + const_val1 = __lasx_xvinsgr2vr_w(const_val0, const_val, 0); \ + const_val1 = __lasx_xvreplve0_w(const_val1); \ + LASX_LD_4(block, 16, in0, in1, in2, in3); \ + LASX_TRANSPOSE4x16(in0, in1, in2, in3, in0, in1, in2, in3); \ + a0 = __lasx_xvpermi_d(in0, 0xD8); \ + a0 = __lasx_vext2xv_w_h(a0); \ + temp = __lasx_xvslli_w(a0, 3); \ + a1 = __lasx_xvpermi_d(in0, 0x8D); \ + a1 = __lasx_vext2xv_w_h(a1); \ + a2 = __lasx_xvpermi_d(in1, 0xD8); \ + a2 = __lasx_vext2xv_w_h(a2); \ + a3 = __lasx_xvpermi_d(in1, 0x8D); \ + a3 = __lasx_vext2xv_w_h(a3); \ + b0 = __lasx_xvpermi_d(in2, 0xD8); \ + b0 = __lasx_vext2xv_w_h(b0); \ + b1 = __lasx_xvpermi_d(in2, 0x8D); \ + b1 = __lasx_vext2xv_w_h(b1); \ + b2 = __lasx_xvpermi_d(in3, 0xD8); \ + b2 = __lasx_vext2xv_w_h(b2); \ + b3 = __lasx_xvpermi_d(in3, 0x8D); \ + b3 = __lasx_vext2xv_w_h(b3); \ + select_vec = a0 | a1 | a2 | a3 | b0 | b1 | b2 | b3; \ + select_vec = __lasx_xvslti_wu(select_vec, 1); \ + \ + w2 = __lasx_xvrepl128vei_h(w1, 2); \ + w3 = __lasx_xvrepl128vei_h(w1, 3); \ + w4 = __lasx_xvrepl128vei_h(w1, 4); \ + w5 = __lasx_xvrepl128vei_h(w1, 5); \ + w6 = __lasx_xvrepl128vei_h(w1, 6); \ + w7 = __lasx_xvrepl128vei_h(w1, 7); \ + w1 = __lasx_xvrepl128vei_h(w1, 1); \ + \ + /* part of FUNC6(idctRowCondDC) */ \ + LASX_MADDWL_W_H_128SV(const_val0, in0, w4, temp0); \ + LASX_MULWL_W_H_2_128SV(in1, w2, in1, w6, temp1, temp2); \ + a0 = __lasx_xvadd_w(temp0, temp1); \ + a1 = __lasx_xvadd_w(temp0, temp2); \ + a2 = __lasx_xvsub_w(temp0, temp2); \ + a3 = __lasx_xvsub_w(temp0, temp1); \ + \ + LASX_ILVH_H_2_128SV(in1, in0, w3, w1, temp0, temp1); \ + LASX_DP2_W_H(temp0, temp1, b0); \ + temp1 = __lasx_xvneg_h(w7); \ + LASX_ILVL_H_128SV(temp1, w3, temp2); \ + LASX_DP2_W_H(temp0, temp2, b1); \ + temp1 = __lasx_xvneg_h(w1); \ + LASX_ILVL_H_128SV(temp1, w5, temp2); \ + LASX_DP2_W_H(temp0, temp2, b2); \ + temp1 = __lasx_xvneg_h(w5); \ + LASX_ILVL_H_128SV(temp1, w7, temp2); \ + LASX_DP2_W_H(temp0, temp2, b3); \ + \ + /* if (AV_RAN64A(row + 4)) */ \ + LASX_ILVL_H_2_128SV(in3, in2, w6, w4, temp0, temp1); \ + LASX_DP2ADD_W_H(a0, temp0, temp1, a0); \ + LASX_ILVL_H_128SV(w2, w4, temp1); \ + LASX_DP2SUB_W_H(a1, temp0, temp1, a1); \ + temp1 = __lasx_xvneg_h(w4); \ + LASX_ILVL_H_128SV(w2, temp1, temp2); \ + LASX_DP2ADD_W_H(a2, temp0, temp2, a2); \ + temp1 = __lasx_xvneg_h(w6); \ + LASX_ILVL_H_128SV(temp1, w4, temp2); \ + LASX_DP2ADD_W_H(a3, temp0, temp2, a3); \ + \ + LASX_ILVH_H_2_128SV(in3, in2, w7, w5, temp0, temp1); \ + LASX_DP2ADD_W_H(b0, temp0, temp1, b0); \ + LASX_ILVL_H_2_128SV(w5, w1, w3, w7, temp1, temp2); \ + LASX_DP2SUB_W_H(b1, temp0, temp1, b1); \ + LASX_DP2ADD_W_H(b2, temp0, temp2, b2); \ + temp1 = __lasx_xvneg_h(w1); \ + LASX_ILVL_H_128SV(temp1, w3, temp2); \ + LASX_DP2ADD_W_H(b3, temp0, temp2, b3); \ + \ + temp0 = __lasx_xvadd_w(a0, b0); \ + temp1 = __lasx_xvadd_w(a1, b1); \ + temp2 = __lasx_xvadd_w(a2, b2); \ + temp3 = __lasx_xvadd_w(a3, b3); \ + a0 = __lasx_xvsub_w(a0, b0); \ + a1 = __lasx_xvsub_w(a1, b1); \ + a2 = __lasx_xvsub_w(a2, b2); \ + a3 = __lasx_xvsub_w(a3, b3); \ + LASX_SRAI_W_8(temp0, temp1, temp2, temp3, a0, a1, a2, a3, \ + temp0, temp1, temp2, temp3, a0, a1, a2, a3, 11); \ + in0 = __lasx_xvbitsel_v(temp0, temp, select_vec); \ + in1 = __lasx_xvbitsel_v(temp1, temp, select_vec); \ + in2 = __lasx_xvbitsel_v(temp2, temp, select_vec); \ + in3 = __lasx_xvbitsel_v(temp3, temp, select_vec); \ + a0 = __lasx_xvbitsel_v(a0, temp, select_vec); \ + a1 = __lasx_xvbitsel_v(a1, temp, select_vec); \ + a2 = __lasx_xvbitsel_v(a2, temp, select_vec); \ + a3 = __lasx_xvbitsel_v(a3, temp, select_vec); \ + in0 = __lasx_xvpickev_h(in1, in0); \ + in1 = __lasx_xvpickev_h(in3, in2); \ + in2 = __lasx_xvpickev_h(a2, a3); \ + in3 = __lasx_xvpickev_h(a0, a1); \ + in0 = __lasx_xvpermi_d(in0, 0xD8); \ + in1 = __lasx_xvpermi_d(in1, 0xD8); \ + in2 = __lasx_xvpermi_d(in2, 0xD8); \ + in3 = __lasx_xvpermi_d(in3, 0xD8); \ + + +#define LASX_IDCTCOLS \ + /* part of FUNC6(idctSparaseCol) */ \ + LASX_TRANSPOSE4x16(in0, in1, in2, in3, in0, in1, in2, in3); \ + LASX_MADDWL_W_H_128SV(const_val1, in0, w4, temp0); \ + LASX_MULWL_W_H_2_128SV(in1, w2, in1, w6, temp1, temp2); \ + a0 = __lasx_xvadd_w(temp0, temp1); \ + a1 = __lasx_xvadd_w(temp0, temp2); \ + a2 = __lasx_xvsub_w(temp0, temp2); \ + a3 = __lasx_xvsub_w(temp0, temp1); \ + \ + LASX_ILVH_H_2_128SV(in1, in0, w3, w1, temp0, temp1); \ + LASX_DP2_W_H(temp0, temp1, b0); \ + temp1 = __lasx_xvneg_h(w7); \ + LASX_ILVL_H_128SV(temp1, w3, temp2); \ + LASX_DP2_W_H(temp0, temp2, b1); \ + temp1 = __lasx_xvneg_h(w1); \ + LASX_ILVL_H_128SV(temp1, w5, temp2); \ + LASX_DP2_W_H(temp0, temp2, b2); \ + temp1 = __lasx_xvneg_h(w5); \ + LASX_ILVL_H_128SV(temp1, w7, temp2); \ + LASX_DP2_W_H(temp0, temp2, b3); \ + \ + /* if (AV_RAN64A(row + 4)) */ \ + LASX_ILVL_H_2_128SV(in3, in2, w6, w4, temp0, temp1); \ + LASX_DP2ADD_W_H(a0, temp0, temp1, a0); \ + LASX_ILVL_H_128SV(w2, w4, temp1); \ + LASX_DP2SUB_W_H(a1, temp0, temp1, a1); \ + temp1 = __lasx_xvneg_h(w4); \ + LASX_ILVL_H_128SV(w2, temp1, temp2); \ + LASX_DP2ADD_W_H(a2, temp0, temp2, a2); \ + temp1 = __lasx_xvneg_h(w6); \ + LASX_ILVL_H_128SV(temp1, w4, temp2); \ + LASX_DP2ADD_W_H(a3, temp0, temp2, a3); \ + \ + LASX_ILVH_H_2_128SV(in3, in2, w7, w5, temp0, temp1); \ + LASX_DP2ADD_W_H(b0, temp0, temp1, b0); \ + LASX_ILVL_H_2_128SV(w5, w1, w3, w7, temp1, temp2); \ + LASX_DP2SUB_W_H(b1, temp0, temp1, b1); \ + LASX_DP2ADD_W_H(b2, temp0, temp2, b2); \ + temp1 = __lasx_xvneg_h(w1); \ + LASX_ILVL_H_128SV(temp1, w3, temp2); \ + LASX_DP2ADD_W_H(b3, temp0, temp2, b3); \ + \ + temp0 = __lasx_xvadd_w(a0, b0); \ + temp1 = __lasx_xvadd_w(a1, b1); \ + temp2 = __lasx_xvadd_w(a2, b2); \ + temp3 = __lasx_xvadd_w(a3, b3); \ + a3 = __lasx_xvsub_w(a3, b3); \ + a2 = __lasx_xvsub_w(a2, b2); \ + a1 = __lasx_xvsub_w(a1, b1); \ + a0 = __lasx_xvsub_w(a0, b0); \ + LASX_SRAI_W_8(temp0, temp1, temp2, temp3, a0, a1, a2, a3, \ + temp0, temp1, temp2, temp3, a0, a1, a2, a3, 20); \ + in0 = __lasx_xvpickev_h(temp1, temp0); \ + in1 = __lasx_xvpickev_h(temp3, temp2); \ + in2 = __lasx_xvpickev_h(a2, a3); \ + in3 = __lasx_xvpickev_h(a0, a1); \ + + +static void simple_idct_lasx(int16_t *block) +{ + int32_t const_val = 1 << 10; + __m256i w1 = {0x4B42539F58C50000, 0x11A822A332493FFF, 0x4B42539F58C50000, 0x11A822A332493FFF}; + __m256i in0, in1, in2, in3; + __m256i w2, w3, w4, w5, w6, w7; + __m256i a0, a1, a2, a3; + __m256i b0, b1, b2, b3; + __m256i temp0, temp1, temp2, temp3; + __m256i const_val0 = __lasx_xvreplgr2vr_w(const_val); + __m256i const_val1, select_vec, temp; + + LASX_IDCTROWCONDDC + LASX_IDCTCOLS + in0 = __lasx_xvpermi_d(in0, 0xD8); + in1 = __lasx_xvpermi_d(in1, 0xD8); + in2 = __lasx_xvpermi_d(in2, 0xD8); + in3 = __lasx_xvpermi_d(in3, 0xD8); + LASX_ST_4(in0, in1, in2, in3, block, 16); +} + +static void simple_idct_put_lasx(uint8_t *dst, int32_t dst_stride, + int16_t *block) +{ + int32_t const_val = 1 << 10; + __m256i w1 = {0x4B42539F58C50000, 0x11A822A332493FFF, 0x4B42539F58C50000, 0x11A822A332493FFF}; + __m256i in0, in1, in2, in3; + __m256i w2, w3, w4, w5, w6, w7; + __m256i a0, a1, a2, a3; + __m256i b0, b1, b2, b3; + __m256i temp0, temp1, temp2, temp3; + __m256i const_val0 = __lasx_xvreplgr2vr_w(const_val); + __m256i const_val1, select_vec, temp; + + LASX_IDCTROWCONDDC + LASX_IDCTCOLS + in0 = __lasx_xvpermi_d(in0, 0xD8); + in1 = __lasx_xvpermi_d(in1, 0xD8); + in2 = __lasx_xvpermi_d(in2, 0xD8); + in3 = __lasx_xvpermi_d(in3, 0xD8); + LASX_CLIP_H_0_255_4(in0, in1, in2, in3, in0, in1, in2, in3); + LASX_PCKEV_B_2_128SV(in1, in0, in3, in2, in0, in1); + LASX_ST_D_4(in0, 0, 2, 1, 3, dst, dst_stride); + dst += (dst_stride << 2); + LASX_ST_D_4(in1, 0, 2, 1, 3, dst, dst_stride); +} + +static void simple_idct_add_lasx(uint8_t *dst, int32_t dst_stride, + int16_t *block) +{ + int32_t const_val = 1 << 10; + uint8_t *dst1 = dst; + __m256i w1 = {0x4B42539F58C50000, 0x11A822A332493FFF, 0x4B42539F58C50000, 0x11A822A332493FFF}; + __m256i sh = {0x0003000200010000, 0x000B000A00090008, 0x0007000600050004, 0x000F000E000D000B}; + __m256i in0, in1, in2, in3; + __m256i w2, w3, w4, w5, w6, w7; + __m256i a0, a1, a2, a3; + __m256i b0, b1, b2, b3; + __m256i temp0, temp1, temp2, temp3; + __m256i const_val0 = __lasx_xvreplgr2vr_w(const_val); + __m256i const_val1, select_vec, temp; + + LASX_IDCTROWCONDDC + LASX_IDCTCOLS + a0 = __lasx_xvldrepl_d(dst1, 0); + a0 = __lasx_vext2xv_hu_bu(a0); + dst1 += dst_stride; + a1 = __lasx_xvldrepl_d(dst1, 0); + a1 = __lasx_vext2xv_hu_bu(a1); + dst1 += dst_stride; + a2 = __lasx_xvldrepl_d(dst1, 0); + a2 = __lasx_vext2xv_hu_bu(a2); + dst1 += dst_stride; + a3 = __lasx_xvldrepl_d(dst1, 0); + a3 = __lasx_vext2xv_hu_bu(a3); + dst1 += dst_stride; + b0 = __lasx_xvldrepl_d(dst1, 0); + b0 = __lasx_vext2xv_hu_bu(b0); + dst1 += dst_stride; + b1 = __lasx_xvldrepl_d(dst1, 0); + b1 = __lasx_vext2xv_hu_bu(b1); + dst1 += dst_stride; + b2 = __lasx_xvldrepl_d(dst1, 0); + b2 = __lasx_vext2xv_hu_bu(b2); + dst1 += dst_stride; + b3 = __lasx_xvldrepl_d(dst1, 0); + b3 = __lasx_vext2xv_hu_bu(b3); + temp0 = __lasx_xvshuf_h(sh, a1, a0); + temp1 = __lasx_xvshuf_h(sh, a3, a2); + temp2 = __lasx_xvshuf_h(sh, b1, b0); + temp3 = __lasx_xvshuf_h(sh, b3, b2); + in0 = __lasx_xvadd_h(temp0, in0); + in1 = __lasx_xvadd_h(temp1, in1); + in2 = __lasx_xvadd_h(temp2, in2); + in3 = __lasx_xvadd_h(temp3, in3); + in0 = __lasx_xvpermi_d(in0, 0xD8); + in1 = __lasx_xvpermi_d(in1, 0xD8); + in2 = __lasx_xvpermi_d(in2, 0xD8); + in3 = __lasx_xvpermi_d(in3, 0xD8); + LASX_CLIP_H_0_255_4(in0, in1, in2, in3, in0, in1, in2, in3); + LASX_PCKEV_B_2_128SV(in1, in0, in3, in2, in0, in1); + LASX_ST_D_4(in0, 0, 2, 1, 3, dst, dst_stride); + dst += (dst_stride << 2); + LASX_ST_D_4(in1, 0, 2, 1, 3, dst, dst_stride); +} + +void ff_simple_idct_lasx(int16_t *block) +{ + simple_idct_lasx(block); +} + +void ff_simple_idct_put_lasx(uint8_t *dst, ptrdiff_t dst_stride, int16_t *block) +{ + simple_idct_put_lasx(dst, dst_stride, block); +} + +void ff_simple_idct_add_lasx(uint8_t *dst, ptrdiff_t dst_stride, int16_t *block) +{ + simple_idct_add_lasx(dst, dst_stride, block); +} diff --git a/libavcodec/loongarch/vc1dsp_init_loongarch.c b/libavcodec/loongarch/vc1dsp_init_loongarch.c new file mode 100644 index 0000000000..e72a4a3203 --- /dev/null +++ b/libavcodec/loongarch/vc1dsp_init_loongarch.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/cpu.h" +#include "libavutil/attributes.h" +#include "libavcodec/vc1dsp.h" +#include "vc1dsp_loongarch.h" + +#define FN_ASSIGN(OP, X, Y, INSN) \ + dsp->OP##vc1_mspel_pixels_tab[1][X+4*Y] = ff_##OP##vc1_mspel_mc##X##Y##INSN; \ + dsp->OP##vc1_mspel_pixels_tab[0][X+4*Y] = ff_##OP##vc1_mspel_mc##X##Y##_16##INSN + +#define FN_ASSIGN_V(OP, Y, INSN) \ + dsp->OP##vc1_mspel_pixels_tab[0][4*Y] = ff_##OP##vc1_mspel_mc0##Y##_16##INSN + +#define FN_ASSIGN_H(OP, X, INSN) \ + dsp->OP##vc1_mspel_pixels_tab[0][X] = ff_##OP##vc1_mspel_mc##X##0_16##INSN + +av_cold void ff_vc1dsp_init_loongarch(VC1DSPContext *dsp) +{ + int cpu_flags = av_get_cpu_flags(); + + if (have_lasx(cpu_flags)) { + dsp->vc1_inv_trans_8x8 = ff_vc1_inv_trans_8x8_lasx; + dsp->vc1_inv_trans_4x8 = ff_vc1_inv_trans_4x8_lasx; + dsp->vc1_inv_trans_8x4 = ff_vc1_inv_trans_8x4_lasx; + dsp->vc1_inv_trans_4x4 = ff_vc1_inv_trans_4x4_lasx; + dsp->vc1_inv_trans_8x8_dc = ff_vc1_inv_trans_8x8_dc_lasx; + dsp->vc1_inv_trans_4x8_dc = ff_vc1_inv_trans_4x8_dc_lasx; + dsp->vc1_inv_trans_8x4_dc = ff_vc1_inv_trans_8x4_dc_lasx; + dsp->vc1_inv_trans_4x4_dc = ff_vc1_inv_trans_4x4_dc_lasx; + FN_ASSIGN(put_, 1, 1, _lasx); + FN_ASSIGN(put_, 1, 2, _lasx); + FN_ASSIGN(put_, 1, 3, _lasx); + FN_ASSIGN(put_, 2, 1, _lasx); + FN_ASSIGN(put_, 2, 2, _lasx); + FN_ASSIGN(put_, 2, 3, _lasx); + FN_ASSIGN(put_, 3, 1, _lasx); + FN_ASSIGN(put_, 3, 2, _lasx); + FN_ASSIGN(put_, 3, 3, _lasx); + FN_ASSIGN_V(put_, 1, _lasx); + FN_ASSIGN_V(put_, 2, _lasx); + FN_ASSIGN_V(put_, 3, _lasx); + FN_ASSIGN_H(put_, 1, _lasx); + FN_ASSIGN_H(put_, 2, _lasx); + FN_ASSIGN_H(put_, 3, _lasx); + dsp->put_no_rnd_vc1_chroma_pixels_tab[0] = ff_put_no_rnd_vc1_chroma_mc8_lasx; + } +} diff --git a/libavcodec/loongarch/vc1dsp_lasx.c b/libavcodec/loongarch/vc1dsp_lasx.c new file mode 100644 index 0000000000..216fcd23fa --- /dev/null +++ b/libavcodec/loongarch/vc1dsp_lasx.c @@ -0,0 +1,997 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "vc1dsp_loongarch.h" +#include "libavutil/loongarch/generic_macros_lasx.h" + +void ff_vc1_inv_trans_8x8_lasx(int16_t block[64]) +{ + int32_t con_4 = 4; + int32_t con_64 = 64; + __m256i in0, in1, in2, in3; + __m256i temp0, temp1, temp2, temp3, t1, t2, t3, t4, t5, t6, t7, t8; + __m256i const_1 = {0x000c000c000c000c, 0x000c000c000c000c, 0x000c000c000c000c, 0x000c000c000c000c}; + __m256i const_2 = {0xfff4000cfff4000c, 0xfff4000cfff4000c, 0xfff4000cfff4000c, 0xfff4000cfff4000c}; + __m256i const_3 = {0x0006001000060010, 0x0006001000060010, 0x0006001000060010, 0x0006001000060010}; + __m256i const_4 = {0xfff00006fff00006, 0xfff00006fff00006, 0xfff00006fff00006, 0xfff00006fff00006}; + __m256i const_5 = {0x000f0010000f0010, 0x000f0010000f0010, 0x000f0010000f0010, 0x000f0010000f0010}; + __m256i const_6 = {0x0004000900040009, 0x0004000900040009, 0x0004000900040009, 0x0004000900040009}; + __m256i const_7 = {0xfffc000ffffc000f, 0xfffc000ffffc000f, 0xfffc000ffffc000f, 0xfffc000ffffc000f}; + __m256i const_8 = {0xfff7fff0fff7fff0, 0xfff7fff0fff7fff0, 0xfff7fff0fff7fff0, 0xfff7fff0fff7fff0}; + __m256i const_9 = {0xfff00009fff00009, 0xfff00009fff00009, 0xfff00009fff00009, 0xfff00009fff00009}; + __m256i const_10 = {0x000f0004000f0004, 0x000f0004000f0004, 0x000f0004000f0004, 0x000f0004000f0004}; + __m256i const_11 = {0xfff70004fff70004, 0xfff70004fff70004, 0xfff70004fff70004, 0xfff70004fff70004}; + __m256i const_12 = {0xfff0000ffff0000f, 0xfff0000ffff0000f, 0xfff0000ffff0000f, 0xfff0000ffff0000f}; + + LASX_LD_4(block, 16, in0, in1, in2, in3); + in0 = __lasx_xvpermi_d(in0, 0xD8); + in1 = __lasx_xvpermi_d(in1, 0xD8); + in2 = __lasx_xvpermi_d(in2, 0xD8); + in3 = __lasx_xvpermi_d(in3, 0xD8); + /* first loops */ + LASX_ILVL_H_2_128SV(in2, in0, in3, in1, temp0, temp1); + t2 = __lasx_xvreplgr2vr_w(con_4); + LASX_DP2ADD_W_H(t2, temp0, const_1, t1); + LASX_DP2ADD_W_H(t2, temp0, const_2, t2); + LASX_DP2_W_H(temp1, const_3, t3); + LASX_DP2_W_H(temp1, const_4, t4); + + t5 = __lasx_xvadd_w(t1, t3); + t6 = __lasx_xvadd_w(t2, t4); + t7 = __lasx_xvsub_w(t2, t4); + t8 = __lasx_xvsub_w(t1, t3); + + LASX_ILVH_H_2_128SV(in1, in0, in3, in2, temp0, temp1); + LASX_DP2_W_H(const_5, temp0, temp2); + LASX_DP2ADD_W_H(temp2, temp1, const_6, t1); + LASX_DP2_W_H(const_7, temp0, temp2); + LASX_DP2ADD_W_H(temp2, temp1, const_8, t2); + LASX_DP2_W_H(const_9, temp0, temp2); + LASX_DP2ADD_W_H(temp2, temp1, const_10, t3); + LASX_DP2_W_H(const_11, temp0, temp2); + LASX_DP2ADD_W_H(temp2, temp1, const_12, t4); + + temp0 = __lasx_xvadd_w(t1, t5); + temp1 = __lasx_xvadd_w(t6, t2); + temp2 = __lasx_xvadd_w(t7, t3); + temp3 = __lasx_xvadd_w(t8, t4); + in0 = __lasx_xvsub_w(t8, t4); + in1 = __lasx_xvsub_w(t7, t3); + in2 = __lasx_xvsub_w(t6, t2); + in3 = __lasx_xvsub_w(t5, t1); + LASX_SRAI_W_8(temp0, temp1, temp2, temp3, in0, in1, in2, in3, + temp0, temp1, temp2, temp3, in0, in1, in2, in3, 3); + + /* second loops */ + temp0 = __lasx_xvpackev_h(temp1, temp0); + temp1 = __lasx_xvpackev_h(temp3, temp2); + temp2 = __lasx_xvpackev_h(in1, in0); + temp3 = __lasx_xvpackev_h(in3, in2); + LASX_ILVL_W_2_128SV(temp1, temp0, temp3, temp2, t1, t3); + LASX_ILVH_W_2_128SV(temp1, temp0, temp3, temp2, t2, t4); + in0 = __lasx_xvpermi_q(t3, t1, 0x20); + in1 = __lasx_xvpermi_q(t3, t1, 0x31); + in2 = __lasx_xvpermi_q(t4, t2, 0x20); + in3 = __lasx_xvpermi_q(t4, t2, 0x31); + LASX_ILVL_H_2_128SV(in1, in0, in3, in2, temp0, temp1); + t3 = __lasx_xvreplgr2vr_w(con_64); + LASX_DP2ADD_W_H(t3, temp0, const_1, t1); + LASX_DP2ADD_W_H(t3, temp0, const_2, t2); + LASX_DP2_W_H(temp1, const_3, t3); + LASX_DP2_W_H(temp1, const_4, t4); + + t5 = __lasx_xvadd_w(t1, t3); + t6 = __lasx_xvadd_w(t2, t4); + t7 = __lasx_xvsub_w(t2, t4); + t8 = __lasx_xvsub_w(t1, t3); + + LASX_ILVH_H_2_128SV(in2, in0, in3, in1, temp0, temp1); + LASX_DP2_W_H(const_5, temp0, temp2); + LASX_DP2ADD_W_H(temp2, temp1, const_6, t1); + LASX_DP2_W_H(const_7, temp0, temp2); + LASX_DP2ADD_W_H(temp2, temp1, const_8, t2); + LASX_DP2_W_H(const_9, temp0, temp2); + LASX_DP2ADD_W_H(temp2, temp1, const_10, t3); + LASX_DP2_W_H(const_11, temp0, temp2); + LASX_DP2ADD_W_H(temp2, temp1, const_12, t4); + + temp0 = __lasx_xvadd_w(t5, t1); + temp1 = __lasx_xvadd_w(t6, t2); + temp2 = __lasx_xvadd_w(t7, t3); + temp3 = __lasx_xvadd_w(t8, t4); + in0 = __lasx_xvsub_w(t8, t4); + in1 = __lasx_xvsub_w(t7, t3); + in2 = __lasx_xvsub_w(t6, t2); + in3 = __lasx_xvsub_w(t5, t1); + in0 = __lasx_xvaddi_wu(in0, 1); + in1 = __lasx_xvaddi_wu(in1, 1); + in2 = __lasx_xvaddi_wu(in2, 1); + in3 = __lasx_xvaddi_wu(in3, 1); + LASX_SRAI_W_8(temp0, temp1, temp2, temp3, in0, in1, in2, in3, + temp0, temp1, temp2, temp3, in0, in1, in2, in3, 7); + t1 = __lasx_xvpickev_h(temp1, temp0); + t2 = __lasx_xvpickev_h(temp3, temp2); + t3 = __lasx_xvpickev_h(in1, in0); + t4 = __lasx_xvpickev_h(in3, in2); + in0 = __lasx_xvpermi_d(t1, 0xD8); + in1 = __lasx_xvpermi_d(t2, 0xD8); + in2 = __lasx_xvpermi_d(t3, 0xD8); + in3 = __lasx_xvpermi_d(t4, 0xD8); + LASX_ST_4(in0, in1, in2, in3, block, 16); +} + +void ff_vc1_inv_trans_8x8_dc_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block) +{ + int dc = block[0]; + uint8_t *dst = dest + stride; + __m256i in0, in1, const_dc, temp0; + __m256i zero = {0}; + + dc = (3 * dc + 1) >> 1; + dc = (3 * dc + 16) >> 5; + + const_dc = __lasx_xvreplgr2vr_h(dc); + in0 = __lasx_xvldrepl_d(dest, 0); + in1 = __lasx_xvldrepl_d(dst, 0); + in0 = __lasx_xvpermi_q(in1, in0, 0x20); + LASX_ILVL_B_128SV(zero, in0, temp0); + temp0 = __lasx_xvadd_h(temp0, const_dc); + LASX_CLIP_H_0_255(temp0, in0); + temp0 = __lasx_xvpickev_b(in0, in0); + __lasx_xvstelm_d(temp0, dest, 0, 0); + __lasx_xvstelm_d(temp0, dst, 0, 2); + dest = dst + stride; + dst = dest + stride; + in0 = __lasx_xvldrepl_d(dest, 0); + in1 = __lasx_xvldrepl_d(dst, 0); + in0 = __lasx_xvpermi_q(in1, in0, 0x20); + LASX_ILVL_B_128SV(zero, in0, temp0); + temp0 = __lasx_xvadd_h(temp0, const_dc); + LASX_CLIP_H_0_255(temp0, in0); + temp0 = __lasx_xvpickev_b(in0, in0); + __lasx_xvstelm_d(temp0, dest, 0, 0); + __lasx_xvstelm_d(temp0, dst, 0, 2); + dest = dst + stride; + dst = dest + stride; + in0 = __lasx_xvldrepl_d(dest, 0); + in1 = __lasx_xvldrepl_d(dst, 0); + in0 = __lasx_xvpermi_q(in1, in0, 0x20); + LASX_ILVL_B_128SV(zero, in0, temp0); + temp0 = __lasx_xvadd_h(temp0, const_dc); + LASX_CLIP_H_0_255(temp0, in0); + temp0 = __lasx_xvpickev_b(in0, in0); + __lasx_xvstelm_d(temp0, dest, 0, 0); + __lasx_xvstelm_d(temp0, dst, 0, 2); + dest = dst + stride; + dst = dest + stride; + in0 = __lasx_xvldrepl_d(dest, 0); + in1 = __lasx_xvldrepl_d(dst, 0); + in0 = __lasx_xvpermi_q(in1, in0, 0x20); + LASX_ILVL_B_128SV(zero, in0, temp0); + temp0 = __lasx_xvadd_h(temp0, const_dc); + LASX_CLIP_H_0_255(temp0, in0); + temp0 = __lasx_xvpickev_b(in0, in0); + __lasx_xvstelm_d(temp0, dest, 0, 0); + __lasx_xvstelm_d(temp0, dst, 0, 2); +} + +void ff_vc1_inv_trans_8x4_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block) +{ + uint8_t *dst = dest; + __m256i shift = {0x0000000400000000, 0x0000000500000001, 0x0000000600000002, 0x0000000700000003}; + __m256i const_64 = {0x0000004000000040, 0x0000004000000040, 0x0000004000000040, 0x0000004000000040}; + __m256i const_1 = {0x00060010000C000C, 0x00060010000C000C, 0x00060010000C000C, 0x00060010000C000C}; + __m256i const_2 = {0xFFF00006FFF4000C, 0xFFF00006FFF4000C, 0xFFF00006FFF4000C, 0xFFF00006FFF4000C}; + __m256i const_3 = {0x0004000F00090010, 0x0004000F00090010, 0x0004000F00090010, 0x0004000F00090010}; + __m256i const_4 = {0xFFF7FFFCFFF0000F, 0xFFF7FFFCFFF0000F, 0xFFF7FFFCFFF0000F, 0xFFF7FFFCFFF0000F}; + __m256i const_5 = {0x000FFFF000040009, 0x000FFFF000040009, 0x000FFFF000040009, 0x000FFFF000040009}; + __m256i const_6 = {0xFFF0FFF7000F0004, 0xFFF0FFF7000F0004, 0xFFF0FFF7000F0004, 0xFFF0FFF7000F0004}; + __m256i const_7 = {0x0000000000000004, 0x0000000000000004, 0x0000000000000004, 0x0000000000000004}; + __m256i const_8 = {0x0011001100110011, 0x0011001100110011, 0x0011001100110011, 0x0011001100110011}; + __m256i const_9 = {0xFFEF0011FFEF0011, 0xFFEF0011FFEF0011, 0xFFEF0011FFEF0011, 0xFFEF0011FFEF0011}; + __m256i const_10 = {0x000A0016000A0016, 0x000A0016000A0016, 0x000A0016000A0016, 0x000A0016000A0016}; + __m256i const_11 = {0x0016FFF60016FFF6, 0x0016FFF60016FFF6, 0x0016FFF60016FFF6, 0x0016FFF60016FFF6}; + __m256i in0, in1; + __m256i temp0, temp1, temp2, temp3, t1, t2, t3, t4; + + LASX_LD_2(block, 16, in0, in1); + /* first loops */ + temp0 = __lasx_xvpermi_d(in0, 0xB1); + temp1 = __lasx_xvpermi_d(in1, 0xB1); + LASX_ILVL_H_2_128SV(temp0, in0, temp1, in1, temp0, temp1); + temp2 = __lasx_xvpickev_w(temp1, temp0); + temp3 = __lasx_xvpickod_w(temp1, temp0); + + LASX_DP2_W_H(temp2, const_1, temp0); + LASX_DP2_W_H(temp2, const_2, temp1); + t1 = __lasx_xvadd_w(temp0, const_7); + t2 = __lasx_xvadd_w(temp1, const_7); + temp0 = __lasx_xvpickev_w(t2, t1); + temp1 = __lasx_xvpickod_w(t2, t1); + t3 = __lasx_xvadd_w(temp0, temp1); + t4 = __lasx_xvsub_w(temp0, temp1); + t4 = __lasx_xvpermi_d(t4, 0xB1); + + LASX_DP4_D_H(temp3, const_3, t1); + LASX_DP4_D_H(temp3, const_4, t2); + LASX_DP4_D_H(temp3, const_5, temp0); + LASX_DP4_D_H(temp3, const_6, temp1); + + temp2 = __lasx_xvpickev_w(t2, t1); + temp3 = __lasx_xvpickev_w(temp1, temp0); + + t1 = __lasx_xvadd_w(temp2, t3); + t2 = __lasx_xvadd_w(temp3, t4); + temp0 = __lasx_xvsub_w(t4, temp3); + temp1 = __lasx_xvsub_w(t3, temp2); + LASX_SRAI_W_4(t1, t2, temp0, temp1, t1, t2, t3, t4, 3); + /* second loops */ + temp2 = __lasx_xvpickev_h(t2, t1); + temp3 = __lasx_xvpickev_h(t4, t3); + temp3 = __lasx_xvshuf4i_h(temp3, 0x4E); + temp0 = __lasx_xvpermi_q(temp3, temp2, 0x20); + temp1 = __lasx_xvpermi_q(temp3, temp2, 0x31); + LASX_DP2ADD_W_H(const_64, temp0, const_8, t1); + LASX_DP2ADD_W_H(const_64, temp0, const_9, t2); + LASX_DP2_W_H(temp1, const_10, t3); + LASX_DP2_W_H(temp1, const_11, t4); + temp0 = __lasx_xvadd_w(t1, t3); + temp1 = __lasx_xvsub_w(t2, t4); + temp2 = __lasx_xvadd_w(t2, t4); + temp3 = __lasx_xvsub_w(t1, t3); + LASX_SRAI_W_4(temp0, temp1, temp2, temp3, t1, t2, t3, t4, 7); + + temp0 = __lasx_xvldrepl_d(dst, 0); + temp0 = __lasx_vext2xv_wu_bu(temp0); + dst += stride; + temp1 = __lasx_xvldrepl_d(dst, 0); + temp1 = __lasx_vext2xv_wu_bu(temp1); + dst += stride; + temp2 = __lasx_xvldrepl_d(dst, 0); + temp2 = __lasx_vext2xv_wu_bu(temp2); + dst += stride; + temp3 = __lasx_xvldrepl_d(dst, 0); + temp3 = __lasx_vext2xv_wu_bu(temp3); + t1 = __lasx_xvadd_w(temp0, t1); + t2 = __lasx_xvadd_w(temp1, t2); + t3 = __lasx_xvadd_w(temp2, t3); + t4 = __lasx_xvadd_w(temp3, t4); + LASX_CLIP_W_0_255_4(t1, t2, t3, t4, t1, t2, t3, t4); + LASX_PCKEV_H_2_128SV(t2, t1, t4, t3, temp0, temp1); + temp2 = __lasx_xvpickev_b(temp1, temp0); + temp0 = __lasx_xvperm_w(temp2, shift); + LASX_ST_D_4(temp0, 0, 1, 2, 3, dest, stride); +} + +void ff_vc1_inv_trans_8x4_dc_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block) +{ + int dc = block[0]; + uint8_t *dst = dest + stride; + __m256i in0, in1, const_dc, temp0; + __m256i zero = {0}; + + dc = (3 * dc + 1) >> 1; + dc = (17 * dc + 64) >> 7; + const_dc = __lasx_xvreplgr2vr_h(dc); + + in0 = __lasx_xvldrepl_d(dest, 0); + in1 = __lasx_xvldrepl_d(dst, 0); + in0 = __lasx_xvpermi_q(in1, in0, 0x20); + LASX_ILVL_B_128SV(zero, in0, temp0); + temp0 = __lasx_xvadd_h(temp0, const_dc); + LASX_CLIP_H_0_255(temp0, in0); + temp0 = __lasx_xvpickev_b(in0, in0); + __lasx_xvstelm_d(temp0, dest, 0, 0); + __lasx_xvstelm_d(temp0, dst, 0, 2); + dest = dst + stride; + dst = dest + stride; + in0 = __lasx_xvldrepl_d(dest, 0); + in1 = __lasx_xvldrepl_d(dst, 0); + in0 = __lasx_xvpermi_q(in1, in0, 0x20); + LASX_ILVL_B_128SV(zero, in0, temp0); + temp0 = __lasx_xvadd_h(temp0, const_dc); + LASX_CLIP_H_0_255(temp0, in0); + temp0 = __lasx_xvpickev_b(in0, in0); + __lasx_xvstelm_d(temp0, dest, 0, 0); + __lasx_xvstelm_d(temp0, dst, 0, 2); +} + +void ff_vc1_inv_trans_4x8_dc_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block) +{ + int dc = block[0]; + uint8_t *dst1 = dest + stride; + uint8_t *dst2 = dst1 + stride; + uint8_t *dst3 = dst2 + stride; + __m256i in0, in1, in2, in3, const_dc, temp0, temp1; + __m256i zero = {0}; + + dc = (17 * dc + 4) >> 3; + dc = (12 * dc + 64) >> 7; + const_dc = __lasx_xvreplgr2vr_h(dc); + + in0 = __lasx_xvldrepl_w(dest, 0); + in1 = __lasx_xvldrepl_w(dst1, 0); + in2 = __lasx_xvldrepl_w(dst2, 0); + in3 = __lasx_xvldrepl_w(dst3, 0); + LASX_ILVL_W_2_128SV(in1, in0, in3, in2, temp0, temp1); + in0 = __lasx_xvpermi_q(temp1, temp0, 0x20); + LASX_ILVL_B_128SV(zero, in0, temp0); + temp0 = __lasx_xvadd_h(temp0, const_dc); + LASX_CLIP_H_0_255(temp0, in0); + temp0 = __lasx_xvpickev_b(in0, in0); + __lasx_xvstelm_w(temp0, dest, 0, 0); + __lasx_xvstelm_w(temp0, dst1, 0, 1); + __lasx_xvstelm_w(temp0, dst2, 0, 4); + __lasx_xvstelm_w(temp0, dst3, 0, 5); + + dest = dst3 + stride; + dst1 = dest + stride; + dst2 = dst1 + stride; + dst3 = dst2 + stride; + + in0 = __lasx_xvldrepl_w(dest, 0); + in1 = __lasx_xvldrepl_w(dst1, 0); + in2 = __lasx_xvldrepl_w(dst2, 0); + in3 = __lasx_xvldrepl_w(dst3, 0); + LASX_ILVL_W_2_128SV(in1, in0, in3, in2, temp0, temp1); + in0 = __lasx_xvpermi_q(temp1, temp0, 0x20); + LASX_ILVL_B_128SV(zero, in0, temp0); + temp0 = __lasx_xvadd_h(temp0, const_dc); + LASX_CLIP_H_0_255(temp0, in0); + temp0 = __lasx_xvpickev_b(in0, in0); + __lasx_xvstelm_w(temp0, dest, 0, 0); + __lasx_xvstelm_w(temp0, dst1, 0, 1); + __lasx_xvstelm_w(temp0, dst2, 0, 4); + __lasx_xvstelm_w(temp0, dst3, 0, 5); + +} + +void ff_vc1_inv_trans_4x8_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block) +{ + uint8_t *dst = dest; + __m256i in0, in1, in2, in3; + __m256i temp0, temp1, temp2, temp3, t1, t2, t3, t4; + + __m256i const_1 = {0x0011001100110011, 0x0011001100110011, 0x0011001100110011, 0x0011001100110011}; + __m256i const_2 = {0xFFEF0011FFEF0011, 0xFFEF0011FFEF0011, 0xFFEF0011FFEF0011, 0xFFEF0011FFEF0011}; + __m256i const_3 = {0x000A0016000A0016, 0x000A0016000A0016, 0x000A0016000A0016, 0x000A0016000A0016}; + __m256i const_4 = {0x0016FFF60016FFF6, 0x0016FFF60016FFF6, 0x0016FFF60016FFF6, 0x0016FFF60016FFF6}; + __m256i const_5 = {0x0000000400000004, 0x0000000400000004, 0x0000000400000004, 0x0000000400000004}; + __m256i const_6 = {0x0000004000000040, 0x0000004000000040, 0x0000004000000040, 0x0000004000000040}; + __m256i const_7 = {0x000C000C000C000C, 0X000C000C000C000C, 0xFFF4000CFFF4000C, 0xFFF4000CFFF4000C}; + __m256i const_8 = {0x0006001000060010, 0x0006001000060010, 0xFFF00006FFF00006, 0xFFF00006FFF00006}; + __m256i const_9 = {0x0009001000090010, 0x0009001000090010, 0x0004000F0004000F, 0x0004000F0004000F}; + __m256i const_10 = {0xFFF0000FFFF0000F, 0xFFF0000FFFF0000F, 0xFFF7FFFCFFF7FFFC, 0xFFF7FFFCFFF7FFFC}; + __m256i const_11 = {0x0004000900040009, 0x0004000900040009, 0x000FFFF0000FFFF0, 0x000FFFF0000FFFF0}; + __m256i const_12 = {0x000F0004000F0004, 0x000F0004000F0004, 0xFFF0FFF7FFF0FFF7, 0xFFF0FFF7FFF0FFF7}; + __m256i shift = {0x0000000400000000, 0x0000000600000002, 0x0000000500000001, 0x0000000700000003}; + + /* first loops */ + LASX_LD_4(block, 16, in0, in1, in2, in3); + in0 = __lasx_xvilvl_d(in1, in0); + in1 = __lasx_xvilvl_d(in3, in2); + temp0 = __lasx_xvpickev_h(in1, in0); + temp1 = __lasx_xvpickod_h(in1, in0); + temp0 = __lasx_xvperm_w(temp0, shift); + temp1 = __lasx_xvperm_w(temp1, shift); + + LASX_DP2ADD_W_H(const_5, temp0, const_1, t1); + LASX_DP2ADD_W_H(const_5, temp0, const_2, t2); + LASX_DP2_W_H(temp1, const_3, t3); + LASX_DP2_W_H(temp1, const_4, t4); + + temp0 = __lasx_xvadd_w(t1, t3); + temp1 = __lasx_xvsub_w(t2, t4); + temp2 = __lasx_xvadd_w(t2, t4); + temp3 = __lasx_xvsub_w(t1, t3); + LASX_SRAI_W_4(temp0, temp1, temp2, temp3, temp0, temp1, temp2, temp3, 3); + + /* second loops */ + t1 = __lasx_xvpickev_w(temp1, temp0); + t2 = __lasx_xvpickev_w(temp3, temp2); + t1 = __lasx_xvpickev_h(t2, t1); + t3 = __lasx_xvpickod_w(temp1, temp0); + t4 = __lasx_xvpickod_w(temp3, temp2); + temp1 = __lasx_xvpickev_h(t4, t3); + temp2 = __lasx_xvpermi_q(t1, t1, 0x00); + temp3 = __lasx_xvpermi_q(t1, t1, 0x11); + LASX_DP2ADD_W_H(const_6, temp2, const_7, t1); + LASX_DP2_W_H(temp3, const_8, t2); + t3 = __lasx_xvadd_w(t1, t2); + t4 = __lasx_xvsub_w(t1, t2); + t4 = __lasx_xvpermi_d(t4, 0x4E); + + LASX_DP2_W_H(temp1, const_9, t1); + LASX_DP2_W_H(temp1, const_10, t2); + LASX_DP2_W_H(temp1, const_11, temp2); + LASX_DP2_W_H(temp1, const_12, temp3); + + temp0 = __lasx_xvpermi_q(t2, t1, 0x20); + temp1 = __lasx_xvpermi_q(t2, t1, 0x31); + t1 = __lasx_xvadd_w(temp0, temp1); + temp0 = __lasx_xvpermi_q(temp3, temp2, 0x20); + temp1 = __lasx_xvpermi_q(temp3, temp2, 0x31); + t2 = __lasx_xvadd_w(temp1, temp0); + temp0 = __lasx_xvadd_w(t1, t3); + temp1 = __lasx_xvadd_w(t2, t4); + temp2 = __lasx_xvsub_w(t4, t2); + temp3 = __lasx_xvsub_w(t3, t1); + temp2 = __lasx_xvaddi_wu(temp2, 1); + temp3 = __lasx_xvaddi_wu(temp3, 1); + LASX_SRAI_W_4(temp0, temp1, temp2, temp3, temp0, temp1, temp2, temp3, 7); + + const_1 = __lasx_xvldrepl_w(dst, 0); + dst += stride; + const_2 = __lasx_xvldrepl_w(dst, 0); + dst += stride; + const_3 = __lasx_xvldrepl_w(dst, 0); + dst += stride; + const_4 = __lasx_xvldrepl_w(dst, 0); + dst += stride; + const_5 = __lasx_xvldrepl_w(dst, 0); + dst += stride; + const_6 = __lasx_xvldrepl_w(dst, 0); + dst += stride; + const_7 = __lasx_xvldrepl_w(dst, 0); + dst += stride; + const_8 = __lasx_xvldrepl_w(dst, 0); + + LASX_ILVL_W_4_128SV(const_2, const_1, const_4, const_3, const_5, const_6, + const_7, const_8, const_1, const_2, const_3, const_4); + const_1 = __lasx_vext2xv_wu_bu(const_1); + const_2 = __lasx_vext2xv_wu_bu(const_2); + const_3 = __lasx_vext2xv_wu_bu(const_3); + const_4 = __lasx_vext2xv_wu_bu(const_4); + + temp0 = __lasx_xvadd_w(temp0, const_1); + temp1 = __lasx_xvadd_w(temp1, const_2); + temp2 = __lasx_xvadd_w(temp2, const_3); + temp3 = __lasx_xvadd_w(temp3, const_4); + LASX_CLIP_W_0_255_4(temp0, temp1, temp2, temp3, temp0, temp1, temp2, temp3); + LASX_PCKEV_H_2_128SV(temp1, temp0, temp3, temp2, temp0, temp1); + temp0 = __lasx_xvpickev_b(temp1, temp0); + LASX_ST_W_8(temp0, 0, 4, 1, 5, 6, 2, 7, 3, dest, stride); +} + +void ff_vc1_inv_trans_4x4_dc_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block) +{ + int dc = block[0]; + uint8_t *dst1 = dest + stride; + uint8_t *dst2 = dst1 + stride; + uint8_t *dst3 = dst2 + stride; + __m256i in0, in1, in2, in3, temp0, temp1, const_dc; + __m256i zero = {0}; + + dc = (17 * dc + 4) >> 3; + dc = (17 * dc + 64) >> 7; + const_dc = __lasx_xvreplgr2vr_h(dc); + + in0 = __lasx_xvldrepl_w(dest, 0); + in1 = __lasx_xvldrepl_w(dst1, 0); + in2 = __lasx_xvldrepl_w(dst2, 0); + in3 = __lasx_xvldrepl_w(dst3, 0); + LASX_ILVL_W_2_128SV(in1, in0, in3, in2, temp0, temp1); + in0 = __lasx_xvpermi_q(temp1, temp0, 0x20); + LASX_ILVL_B_128SV(zero, in0, temp0); + temp0 = __lasx_xvadd_h(temp0, const_dc); + LASX_CLIP_H_0_255(temp0, in0); + temp0 = __lasx_xvpickev_b(in0, in0); + __lasx_xvstelm_w(temp0, dest, 0, 0); + __lasx_xvstelm_w(temp0, dst1, 0, 1); + __lasx_xvstelm_w(temp0, dst2, 0, 4); + __lasx_xvstelm_w(temp0, dst3, 0, 5); +} + +void ff_vc1_inv_trans_4x4_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block) +{ + uint8_t *dst = dest + stride; + __m256i in0, in1, in2, in3; + __m256i temp0, temp1, temp2, temp3, t1, t2; + + __m256i const_1 = {0x0011001100110011, 0xFFEF0011FFEF0011, 0x0011001100110011, 0xFFEF0011FFEF0011}; + __m256i const_2 = {0x000A0016000A0016, 0x0016FFF60016FFF6, 0x000A0016000A0016, 0x0016FFF60016FFF6}; + __m256i const_64 = {0x0000004000000040, 0x0000004000000040, 0x0000004000000040, 0x0000004000000040}; + + LASX_LD_2(block, 16, in0, in1); + /* first loops */ + temp0 = __lasx_xvilvl_d(in1, in0); + temp1 = __lasx_xvpickev_h(temp0, temp0); + temp2 = __lasx_xvpickod_h(temp0, temp0); + LASX_DP2_W_H(temp1, const_1, t1); + LASX_DP2_W_H(temp2, const_2, t2); + t1 = __lasx_xvaddi_wu(t1, 4); + in0 = __lasx_xvadd_w(t1, t2); + in1 = __lasx_xvsub_w(t1, t2); + LASX_SRAI_W_2(in0, in1, in0, in1, 3); + /* second loops */ + temp0 = __lasx_xvpickev_h(in1, in0); + temp1 = __lasx_xvpermi_q(temp0, temp0, 0x00); + temp2 = __lasx_xvpermi_q(temp0, temp0, 0x11); + const_1 = __lasx_xvpermi_d(const_1, 0xD8); + const_2 = __lasx_xvpermi_d(const_2, 0xD8); + LASX_DP2ADD_W_H(const_64, temp1, const_1, t1); + LASX_DP2_W_H(temp2, const_2, t2); + in0 = __lasx_xvadd_w(t1, t2); + in1 = __lasx_xvsub_w(t1, t2); + LASX_SRAI_W_2(in0, in1, in0, in1, 7); + temp0 = __lasx_xvshuf4i_w(in0, 0x9C); + temp1 = __lasx_xvshuf4i_w(in1, 0x9C); + + in0 = __lasx_xvldrepl_w(dest, 0); + in1 = __lasx_xvldrepl_w(dst, 0); + dst += stride; + in2 = __lasx_xvldrepl_w(dst, 0); + dst += stride; + in3 = __lasx_xvldrepl_w(dst, 0); + temp2 = __lasx_xvilvl_w(in2, in0); + temp2 = __lasx_vext2xv_wu_bu(temp2); + temp3 = __lasx_xvilvl_w(in1, in3); + temp3 = __lasx_vext2xv_wu_bu(temp3); + temp0 = __lasx_xvadd_w(temp0, temp2); + temp1 = __lasx_xvadd_w(temp1, temp3); + LASX_CLIP_W_0_255_2(temp0, temp1, temp0, temp1); + temp1 = __lasx_xvpickev_h(temp1, temp0); + temp0 = __lasx_xvpickev_b(temp1, temp1); + LASX_ST_W_4(temp0, 0, 5, 4, 1, dest, stride); +} + +static void put_vc1_mspel_mc_h_v_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride, int hmode, int vmode, + int rnd) +{ + __m256i in0, in1, in2, in3; + __m256i t0, t1, t2, t3, t4, t5, t6, t7; + __m256i temp0, temp1, const_para1_2, const_para0_3; + __m256i const_r, const_sh; + __m256i sh = {0x0000000400000000, 0x0000000500000001, 0x0000000600000002, 0x0000000700000003}; + static const uint8_t para_value[][4] = {{4, 3, 53, 18}, + {1, 1, 9, 9}, + {3, 4, 18, 53}}; + static const int shift_value[] = {0, 5, 1, 5}; + int shift = (shift_value[hmode] + shift_value[vmode]) >> 1; + int r = (1 << (shift - 1)) + rnd - 1; + const uint8_t *para_v = para_value[vmode - 1]; + + const_r = __lasx_xvreplgr2vr_h(r); + const_sh = __lasx_xvreplgr2vr_h(shift); + src -= 1, src -= stride; + const_para0_3 = __lasx_xvldrepl_h(para_v, 0); + const_para1_2 = __lasx_xvldrepl_h(para_v, 2); + LASX_LD_4(src, stride, in0, in1, in2, in3); + in0 = __lasx_xvpermi_d(in0, 0xD8); + in1 = __lasx_xvpermi_d(in1, 0xD8); + in2 = __lasx_xvpermi_d(in2, 0xD8); + in3 = __lasx_xvpermi_d(in3, 0xD8); + LASX_ILVL_B_2_128SV(in2, in1, in3, in0, temp0, temp1); + LASX_DP2_H_BU(temp0, const_para1_2, t0); + LASX_DP2SUB_H_BU(t0, temp1, const_para0_3, t0); + src += (stride << 2); + in0 = LASX_LD(src); + in0 = __lasx_xvpermi_d(in0, 0xD8); + LASX_ILVL_B_2_128SV(in3, in2, in0, in1, temp0, temp1); + LASX_DP2_H_BU(temp0, const_para1_2, t1); + LASX_DP2SUB_H_BU(t1, temp1, const_para0_3, t1); + src += stride; + in1 = LASX_LD(src); + in1 = __lasx_xvpermi_d(in1, 0xD8); + LASX_ILVL_B_2_128SV(in0, in3, in1, in2, temp0, temp1); + LASX_DP2_H_BU(temp0, const_para1_2, t2); + LASX_DP2SUB_H_BU(t2, temp1, const_para0_3, t2); + src += stride; + in2 = LASX_LD(src); + in2 = __lasx_xvpermi_d(in2, 0xD8); + LASX_ILVL_B_2_128SV(in1, in0, in2, in3, temp0, temp1); + LASX_DP2_H_BU(temp0, const_para1_2, t3); + LASX_DP2SUB_H_BU(t3, temp1, const_para0_3, t3); + src += stride; + in3 = LASX_LD(src); + in3 = __lasx_xvpermi_d(in3, 0xD8); + LASX_ILVL_B_2_128SV(in2, in1, in3, in0, temp0, temp1); + LASX_DP2_H_BU(temp0, const_para1_2, t4); + LASX_DP2SUB_H_BU(t4, temp1, const_para0_3, t4); + src += stride; + in0 = LASX_LD(src); + in0 = __lasx_xvpermi_d(in0, 0xD8); + LASX_ILVL_B_2_128SV(in3, in2, in0, in1, temp0, temp1); + LASX_DP2_H_BU(temp0, const_para1_2, t5); + LASX_DP2SUB_H_BU(t5, temp1, const_para0_3, t5); + src += stride; + in1 = LASX_LD(src); + in1 = __lasx_xvpermi_d(in1, 0xD8); + LASX_ILVL_B_2_128SV(in0, in3, in1, in2, temp0, temp1); + LASX_DP2_H_BU(temp0, const_para1_2, t6); + LASX_DP2SUB_H_BU(t6, temp1, const_para0_3, t6); + src += stride; + in2 = LASX_LD(src); + in2 = __lasx_xvpermi_d(in2, 0xD8); + LASX_ILVL_B_2_128SV(in1, in0, in2, in3, temp0, temp1); + LASX_DP2_H_BU(temp0, const_para1_2, t7); + LASX_DP2SUB_H_BU(t7, temp1, const_para0_3, t7); + LASX_ADD_H_8(t0, const_r, t1, const_r, t2, const_r, t3, const_r, + t4, const_r, t5, const_r, t6, const_r, t7, const_r, + t0, t1, t2, t3, t4, t5, t6, t7); + t0 = __lasx_xvsra_h(t0, const_sh); + t1 = __lasx_xvsra_h(t1, const_sh); + t2 = __lasx_xvsra_h(t2, const_sh); + t3 = __lasx_xvsra_h(t3, const_sh); + t4 = __lasx_xvsra_h(t4, const_sh); + t5 = __lasx_xvsra_h(t5, const_sh); + t6 = __lasx_xvsra_h(t6, const_sh); + t7 = __lasx_xvsra_h(t7, const_sh); + LASX_TRANSPOSE8x8_H_128SV(t0, t1, t2, t3, t4, t5, t6, t7, + t0, t1, t2, t3, t4, t5, t6, t7); + para_v = para_value[hmode - 1]; + const_para0_3 = __lasx_xvldrepl_h(para_v, 0); + const_para1_2 = __lasx_xvldrepl_h(para_v, 2); + const_para0_3 = __lasx_vext2xv_h_b(const_para0_3); + const_para1_2 = __lasx_vext2xv_h_b(const_para1_2); + r = 64 - rnd; + const_r = __lasx_xvreplgr2vr_w(r); + in0 = __lasx_xvpermi_d(t0, 0x72); + in1 = __lasx_xvpermi_d(t1, 0x72); + in2 = __lasx_xvpermi_d(t2, 0x72); + t0 = __lasx_xvpermi_d(t0, 0xD8); + t1 = __lasx_xvpermi_d(t1, 0xD8); + t2 = __lasx_xvpermi_d(t2, 0xD8); + t3 = __lasx_xvpermi_d(t3, 0xD8); + t4 = __lasx_xvpermi_d(t4, 0xD8); + t5 = __lasx_xvpermi_d(t5, 0xD8); + t6 = __lasx_xvpermi_d(t6, 0xD8); + t7 = __lasx_xvpermi_d(t7, 0xD8); + LASX_ILVL_H_2_128SV(t2, t1, t3, t0, temp0, temp1); + LASX_DP2_W_H(temp0, const_para1_2, t0); + LASX_DP2SUB_W_H(t0, temp1, const_para0_3, t0); + LASX_ILVL_H_2_128SV(t3, t2, t4, t1, temp0, temp1); + LASX_DP2_W_H(temp0, const_para1_2, t1); + LASX_DP2SUB_W_H(t1, temp1, const_para0_3, t1); + LASX_ILVL_H_2_128SV(t4, t3, t5, t2, temp0, temp1); + LASX_DP2_W_H(temp0, const_para1_2, t2); + LASX_DP2SUB_W_H(t2, temp1, const_para0_3, t2); + LASX_ILVL_H_2_128SV(t5, t4, t6, t3, temp0, temp1); + LASX_DP2_W_H(temp0, const_para1_2, t3); + LASX_DP2SUB_W_H(t3, temp1, const_para0_3, t3); + LASX_ILVL_H_2_128SV(t6, t5, t7, t4, temp0, temp1); + LASX_DP2_W_H(temp0, const_para1_2, t4); + LASX_DP2SUB_W_H(t4, temp1, const_para0_3, t4); + LASX_ILVL_H_2_128SV(t7, t6, in0, t5, temp0, temp1); + LASX_DP2_W_H(temp0, const_para1_2, t5); + LASX_DP2SUB_W_H(t5, temp1, const_para0_3, t5); + LASX_ILVL_H_2_128SV(in0, t7, in1, t6, temp0, temp1); + LASX_DP2_W_H(temp0, const_para1_2, t6); + LASX_DP2SUB_W_H(t6, temp1, const_para0_3, t6); + LASX_ILVL_H_2_128SV(in1, in0, in2, t7, temp0, temp1); + LASX_DP2_W_H(temp0, const_para1_2, t7); + LASX_DP2SUB_W_H(t7, temp1, const_para0_3, t7); + t0 = __lasx_xvadd_w(t0, const_r); + t1 = __lasx_xvadd_w(t1, const_r); + t2 = __lasx_xvadd_w(t2, const_r); + t3 = __lasx_xvadd_w(t3, const_r); + t4 = __lasx_xvadd_w(t4, const_r); + t5 = __lasx_xvadd_w(t5, const_r); + t6 = __lasx_xvadd_w(t6, const_r); + t7 = __lasx_xvadd_w(t7, const_r); + LASX_SRAI_W_8(t0, t1, t2, t3, t4, t5, t6, t7, + t0, t1, t2, t3, t4, t5, t6, t7, 7); + LASX_TRANSPOSE8x8_W(t0, t1, t2, t3, t4, t5, t6, t7, + t0, t1, t2, t3, t4, t5, t6, t7); + LASX_CLIP_W_0_255_4(t0, t1, t2, t3, t0, t1, t2, t3); + LASX_CLIP_W_0_255_4(t4, t5, t6, t7, t4, t5, t6, t7); + LASX_PCKEV_H_4_128SV(t1, t0, t3, t2, t5, t4, t7, t6, + t0, t1, t2, t3); + LASX_PCKEV_B_2_128SV(t1, t0, t3, t2, t0, t1); + t0 = __lasx_xvperm_w(t0, sh); + t1 = __lasx_xvperm_w(t1, sh); + LASX_ST_D_4(t0, 0, 1, 2, 3, dst, stride); + dst += (stride << 2); + LASX_ST_D_4(t1, 0, 1, 2, 3, dst, stride); +} + +#define PUT_VC1_MSPEL_MC_LASX(hmode, vmode) \ +void ff_put_vc1_mspel_mc ## hmode ## vmode ## _lasx(uint8_t *dst, \ + const uint8_t *src, \ + ptrdiff_t stride, int rnd) \ +{ \ + put_vc1_mspel_mc_h_v_lasx(dst, src, stride, hmode, vmode, rnd); \ +} \ +void ff_put_vc1_mspel_mc ## hmode ## vmode ## _16_lasx(uint8_t *dst, \ + const uint8_t *src, \ + ptrdiff_t stride, int rnd) \ +{ \ + put_vc1_mspel_mc_h_v_lasx(dst, src, stride, hmode, vmode, rnd); \ + put_vc1_mspel_mc_h_v_lasx(dst + 8, src + 8, stride, hmode, vmode, rnd); \ + dst += 8 * stride, src += 8 * stride; \ + put_vc1_mspel_mc_h_v_lasx(dst, src, stride, hmode, vmode, rnd); \ + put_vc1_mspel_mc_h_v_lasx(dst + 8, src + 8, stride, hmode, vmode, rnd); \ +} + +PUT_VC1_MSPEL_MC_LASX(1, 1); +PUT_VC1_MSPEL_MC_LASX(1, 2); +PUT_VC1_MSPEL_MC_LASX(1, 3); + +PUT_VC1_MSPEL_MC_LASX(2, 1); +PUT_VC1_MSPEL_MC_LASX(2, 2); +PUT_VC1_MSPEL_MC_LASX(2, 3); + +PUT_VC1_MSPEL_MC_LASX(3, 1); +PUT_VC1_MSPEL_MC_LASX(3, 2); +PUT_VC1_MSPEL_MC_LASX(3, 3); + +void ff_put_no_rnd_vc1_chroma_mc8_lasx(uint8_t *dst /* align 8 */, + uint8_t *src /* align 1 */, + ptrdiff_t stride, int h, int x, int y) +{ + const int intA = (8 - x) * (8 - y); + const int intB = (x) * (8 - y); + const int intC = (8 - x) * (y); + const int intD = (x) * (y); + __m256i src00, src01, src10, src11; + __m256i A, B, C, D; + int i; + + av_assert2(x < 8 && y < 8 && x >= 0 && y >= 0); + + A = __lasx_xvreplgr2vr_h(intA); + B = __lasx_xvreplgr2vr_h(intB); + C = __lasx_xvreplgr2vr_h(intC); + D = __lasx_xvreplgr2vr_h(intD); + for(i = 0; i < h; i++){ + LASX_LD_2(src, 1, src00, src01); + src += stride; + LASX_LD_2(src, 1, src10, src11); + + LASX_UNPCK_L_HU_BU_4(src00, src01, src10, src11, + src00, src01, src10, src11); + src00 = __lasx_xvmul_h(src00, A); + src01 = __lasx_xvmul_h(src01, B); + src10 = __lasx_xvmul_h(src10, C); + src11 = __lasx_xvmul_h(src11, D); + src00 = __lasx_xvadd_h(src00, src01); + src10 = __lasx_xvadd_h(src10, src11); + src00 = __lasx_xvadd_h(src00, src10); + src00 = __lasx_xvaddi_hu(src00, 28); + src00 = __lasx_xvsrli_h(src00, 6); + LASX_PCKEV_B_128SV(src00, src00, src00); + LASX_ST_D(src00, 0, dst); + dst += stride; + } +} + +static void put_vc1_mspel_mc_v_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride, int vmode, int rnd) +{ + __m256i in0, in1, in2, in3, temp0, temp1, t0; + __m256i const_para0_3, const_para1_2, const_r, const_sh; + static const uint16_t para_value[][2] = {{0x0304, 0x1235}, + {0x0101, 0x0909}, + {0x0403, 0x3512}}; + const uint16_t *para_v = para_value[vmode - 1]; + static const int shift_value[] = {0, 6, 4, 6}; + static int add_value[3]; + ptrdiff_t stride_2x = stride << 1; + int i = 0; + add_value[2] = add_value[0] = 31 + rnd, add_value[1] = 7 + rnd; + + const_r = __lasx_xvreplgr2vr_h(add_value[vmode - 1]); + const_sh = __lasx_xvreplgr2vr_h(shift_value[vmode]); + const_para0_3 = __lasx_xvreplgr2vr_h(*para_v); + const_para1_2 = __lasx_xvreplgr2vr_h(*(para_v + 1)); + + LASX_LD_2((src - stride), stride, in0, in1); + in2 = LASX_LD(src + stride); + in0 = __lasx_xvpermi_d(in0, 0xD8); + in1 = __lasx_xvpermi_d(in1, 0xD8); + in2 = __lasx_xvpermi_d(in2, 0xD8); + for (; i < 16; i++) { + in3 = LASX_LD(src + stride_2x); + in3 = __lasx_xvpermi_d(in3, 0xD8); + LASX_ILVL_B_2_128SV(in2, in1, in3, in0, temp0, temp1); + LASX_DP2_H_BU(temp0, const_para1_2, t0); + LASX_DP2SUB_H_BU(t0, temp1, const_para0_3, t0); + LASX_ADD_H(t0, const_r, t0); + t0 = __lasx_xvsra_h(t0, const_sh); + LASX_CLIP_H_0_255(t0, t0); + LASX_PCKEV_B_128SV(t0, t0, t0); + LASX_ST_D_2(t0, 0, 2, dst, 8); + dst += stride; + src += stride; + in0 = in1; + in1 = in2; + in2 = in3; + } +} + +#define PUT_VC1_MSPEL_MC_V_LASX(vmode) \ +void ff_put_vc1_mspel_mc0 ## vmode ## _16_lasx(uint8_t *dst, \ + const uint8_t *src, \ + ptrdiff_t stride, int rnd) \ +{ \ + put_vc1_mspel_mc_v_lasx(dst, src, stride, vmode, rnd); \ +} + +PUT_VC1_MSPEL_MC_V_LASX(1); +PUT_VC1_MSPEL_MC_V_LASX(2); +PUT_VC1_MSPEL_MC_V_LASX(3); + +#define ROW_LASX(in0, in1, in2, in3, out0) \ + LASX_ILVL_B_2_128SV(in2, in1, in3, in0, tmp0_m, tmp1_m); \ + LASX_DP2_H_BU(tmp0_m, const_para1_2, out0); \ + LASX_DP2SUB_H_BU(out0, tmp1_m, const_para0_3, out0); \ + LASX_ADD_H(out0, const_r, out0); \ + out0 = __lasx_xvsra_h(out0, const_sh); \ + LASX_CLIP_H_0_255(out0, out0); \ + LASX_PCKEV_B(out0, out0, out0); + +static void put_vc1_mspel_mc_h_lasx(uint8_t *dst, const uint8_t *src, + ptrdiff_t stride, int hmode, int rnd) +{ + __m256i in0, in1, in2, in3, in4, in5, in6, in7, + in8, in9, in10, in11, in12, in13, in14, in15; + __m256i out0, out1, out2, out3, out4, out5, out6, out7, out8, out9, + out10, out11, out12, out13, out14, out15, out16, out17, out18; + __m256i const_para0_3, const_para1_2, const_r, const_sh; + __m256i tmp0_m, tmp1_m, tmp2_m, tmp3_m; + __m256i tmp4_m, tmp5_m, tmp6_m, tmp7_m; + __m256i t0, t1, t2, t3, t4, t5, t6, t7; + static const uint16_t para_value[][2] = {{0x0304, 0x1235}, + {0x0101, 0x0909}, + {0x0403, 0x3512}}; + const uint16_t *para_v = para_value[hmode - 1]; + static const int shift_value[] = {0, 6, 4, 6}; + static int add_value[3]; + add_value[2] = add_value[0] = 32 - rnd, add_value[1] = 8 - rnd; + + const_r = __lasx_xvreplgr2vr_h(add_value[hmode - 1]); + const_sh = __lasx_xvreplgr2vr_h(shift_value[hmode]); + const_para0_3 = __lasx_xvreplgr2vr_h(*para_v); + const_para1_2 = __lasx_xvreplgr2vr_h(*(para_v + 1)); + src -= 1; + + LASX_LD_8(src, stride, in0, in1, in2, in3, in4, in5, in6, in7); + src += stride << 3; + LASX_LD_8(src, stride, in8, in9, in10, in11, in12, in13, in14, in15); + LASX_ILVL_B_8_128SV(in2, in0, in3, in1, in6, in4, in7, in5, + in10, in8, in11, in9, in14, in12, in15, in13, + tmp0_m, tmp1_m, tmp2_m, tmp3_m, + tmp4_m, tmp5_m, tmp6_m, tmp7_m); + LASX_ILVLH_B_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, t1, t0, t3, t2); + LASX_ILVLH_B_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, t5, t4, t7, t6); + LASX_ILVLH_W_2_128SV(t2, t0, t3, t1, tmp2_m, tmp0_m, tmp6_m, tmp4_m); + LASX_ILVLH_W_2_128SV(t6, t4, t7, t5, tmp3_m, tmp1_m, tmp7_m, tmp5_m); + LASX_ILVLH_D_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, out1, out0, out3, out2); + LASX_ILVLH_D_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, out5, out4, out7, out6); + + LASX_ILVH_B_8_128SV(in2, in0, in3, in1, in6, in4, in7, in5, + in10, in8, in11, in9, in14, in12, in15, in13, + tmp0_m, tmp1_m, tmp2_m, tmp3_m, + tmp4_m, tmp5_m, tmp6_m, tmp7_m); + LASX_ILVLH_B_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, t1, t0, t3, t2); + LASX_ILVLH_B_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, t5, t4, t7, t6); + LASX_ILVLH_W_2_128SV(t2, t0, t3, t1, tmp2_m, tmp0_m, tmp6_m, tmp4_m); + LASX_ILVLH_W_2_128SV(t6, t4, t7, t5, tmp3_m, tmp1_m, tmp7_m, tmp5_m); + LASX_ILVLH_D_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, out9, out8, out11, out10); + LASX_ILVLH_D_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, + out13, out12, out15, out14); + LASX_PCKOD_Q_2(out0, out0, out1, out1, out16, out17); + LASX_PCKOD_Q(out2, out2, out18); + + out0 = __lasx_xvpermi_d(out0, 0xD8); + out1 = __lasx_xvpermi_d(out1, 0xD8); + out2 = __lasx_xvpermi_d(out2, 0xD8); + out3 = __lasx_xvpermi_d(out3, 0xD8); + out4 = __lasx_xvpermi_d(out4, 0xD8); + out5 = __lasx_xvpermi_d(out5, 0xD8); + out6 = __lasx_xvpermi_d(out6, 0xD8); + out7 = __lasx_xvpermi_d(out7, 0xD8); + out8 = __lasx_xvpermi_d(out8, 0xD8); + out9 = __lasx_xvpermi_d(out9, 0xD8); + out10 = __lasx_xvpermi_d(out10, 0xD8); + out11 = __lasx_xvpermi_d(out11, 0xD8); + out12 = __lasx_xvpermi_d(out12, 0xD8); + out13 = __lasx_xvpermi_d(out13, 0xD8); + out14 = __lasx_xvpermi_d(out14, 0xD8); + out15 = __lasx_xvpermi_d(out15, 0xD8); + out16 = __lasx_xvpermi_d(out16, 0xD8); + out17 = __lasx_xvpermi_d(out17, 0xD8); + out18 = __lasx_xvpermi_d(out18, 0xD8); + + ROW_LASX(out0, out1, out2, out3, in0); + ROW_LASX(out1, out2, out3, out4, in1); + ROW_LASX(out2, out3, out4, out5, in2); + ROW_LASX(out3, out4, out5, out6, in3); + ROW_LASX(out4, out5, out6, out7, in4); + ROW_LASX(out5, out6, out7, out8, in5); + ROW_LASX(out6, out7, out8, out9, in6); + ROW_LASX(out7, out8, out9, out10, in7); + ROW_LASX(out8, out9, out10, out11, in8); + ROW_LASX(out9, out10, out11, out12, in9); + ROW_LASX(out10, out11, out12, out13, in10); + ROW_LASX(out11, out12, out13, out14, in11); + ROW_LASX(out12, out13, out14, out15, in12); + ROW_LASX(out13, out14, out15, out16, in13); + ROW_LASX(out14, out15, out16, out17, in14); + ROW_LASX(out15, out16, out17, out18, in15); + + LASX_ILVL_B_8_128SV(in2, in0, in3, in1, in6, in4, in7, in5, + in10, in8, in11, in9, in14, in12, in15, in13, + tmp0_m, tmp1_m, tmp2_m, tmp3_m, + tmp4_m, tmp5_m, tmp6_m, tmp7_m); + LASX_ILVLH_B_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, t1, t0, t3, t2); + LASX_ILVLH_B_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, t5, t4, t7, t6); + LASX_ILVLH_W_2_128SV(t2, t0, t3, t1, tmp2_m, tmp0_m, tmp6_m, tmp4_m); + LASX_ILVLH_W_2_128SV(t6, t4, t7, t5, tmp3_m, tmp1_m, tmp7_m, tmp5_m); + LASX_ILVLH_D_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, out1, out0, out3, out2); + LASX_ILVLH_D_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, out5, out4, out7, out6); + + LASX_ILVH_B_8_128SV(in2, in0, in3, in1, in6, in4, in7, in5, + in10, in8, in11, in9, in14, in12, in15, in13, + tmp0_m, tmp1_m, tmp2_m, tmp3_m, + tmp4_m, tmp5_m, tmp6_m, tmp7_m); + LASX_ILVLH_B_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, t1, t0, t3, t2); + LASX_ILVLH_B_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, t5, t4, t7, t6); + LASX_ILVLH_W_2_128SV(t2, t0, t3, t1, tmp2_m, tmp0_m, tmp6_m, tmp4_m); + LASX_ILVLH_W_2_128SV(t6, t4, t7, t5, tmp3_m, tmp1_m, tmp7_m, tmp5_m); + LASX_ILVLH_D_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, out9, out8, out11, out10); + LASX_ILVLH_D_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, + out13, out12, out15, out14); + LASX_ST_D_2(out0, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out1, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out2, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out3, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out4, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out5, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out6, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out7, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out8, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out9, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out10, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out11, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out12, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out13, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out14, 0, 1, dst, 8); + dst += stride; + LASX_ST_D_2(out15, 0, 1, dst, 8); +} + +#define PUT_VC1_MSPEL_MC_H_LASX(hmode) \ +void ff_put_vc1_mspel_mc ## hmode ## 0_16_lasx(uint8_t *dst, \ + const uint8_t *src, \ + ptrdiff_t stride, int rnd) \ +{ \ + put_vc1_mspel_mc_h_lasx(dst, src, stride, hmode, rnd); \ +} + +PUT_VC1_MSPEL_MC_H_LASX(1); +PUT_VC1_MSPEL_MC_H_LASX(2); +PUT_VC1_MSPEL_MC_H_LASX(3); diff --git a/libavcodec/loongarch/vc1dsp_loongarch.h b/libavcodec/loongarch/vc1dsp_loongarch.h new file mode 100644 index 0000000000..398631aecc --- /dev/null +++ b/libavcodec/loongarch/vc1dsp_loongarch.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_VC1DSP_LOONGARCH_H +#define AVCODEC_LOONGARCH_VC1DSP_LOONGARCH_H + +#include "libavcodec/vc1dsp.h" +#include "libavutil/avassert.h" + +void ff_vc1_inv_trans_8x8_lasx(int16_t block[64]); +void ff_vc1_inv_trans_8x8_dc_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block); +void ff_vc1_inv_trans_8x4_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block); +void ff_vc1_inv_trans_8x4_dc_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block); +void ff_vc1_inv_trans_4x8_dc_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block); +void ff_vc1_inv_trans_4x8_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *blokc); +void ff_vc1_inv_trans_4x4_dc_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block); +void ff_vc1_inv_trans_4x4_lasx(uint8_t *dest, ptrdiff_t stride, int16_t *block); + +#define FF_PUT_VC1_MSPEL_MC_LASX(hmode, vmode) \ +void ff_put_vc1_mspel_mc ## hmode ## vmode ## _lasx(uint8_t *dst, \ + const uint8_t *src, \ + ptrdiff_t stride, int rnd); \ +void ff_put_vc1_mspel_mc ## hmode ## vmode ## _16_lasx(uint8_t *dst, \ + const uint8_t *src, \ + ptrdiff_t stride, int rnd); + +FF_PUT_VC1_MSPEL_MC_LASX(1, 1); +FF_PUT_VC1_MSPEL_MC_LASX(1, 2); +FF_PUT_VC1_MSPEL_MC_LASX(1, 3); + +FF_PUT_VC1_MSPEL_MC_LASX(2, 1); +FF_PUT_VC1_MSPEL_MC_LASX(2, 2); +FF_PUT_VC1_MSPEL_MC_LASX(2, 3); + +FF_PUT_VC1_MSPEL_MC_LASX(3, 1); +FF_PUT_VC1_MSPEL_MC_LASX(3, 2); +FF_PUT_VC1_MSPEL_MC_LASX(3, 3); + +#define FF_PUT_VC1_MSPEL_MC_V_LASX(vmode) \ +void ff_put_vc1_mspel_mc0 ## vmode ## _16_lasx(uint8_t *dst, \ + const uint8_t *src, \ + ptrdiff_t stride, int rnd); + +FF_PUT_VC1_MSPEL_MC_V_LASX(1); +FF_PUT_VC1_MSPEL_MC_V_LASX(2); +FF_PUT_VC1_MSPEL_MC_V_LASX(3); + +#define FF_PUT_VC1_MSPEL_MC_H_LASX(hmode) \ +void ff_put_vc1_mspel_mc ## hmode ## 0_16_lasx(uint8_t *dst, \ + const uint8_t *src, \ + ptrdiff_t stride, int rnd); + +FF_PUT_VC1_MSPEL_MC_H_LASX(1); +FF_PUT_VC1_MSPEL_MC_H_LASX(2); +FF_PUT_VC1_MSPEL_MC_H_LASX(3); + +void ff_put_no_rnd_vc1_chroma_mc8_lasx(uint8_t *dst /* align 8 */, + uint8_t *src /* align 1 */, + ptrdiff_t stride, int h, int x, int y); + +#endif /* AVCODEC_LOONGARCH_VC1DSP_LOONGARCH_H */ diff --git a/libavcodec/loongarch/videodsp_init.c b/libavcodec/loongarch/videodsp_init.c new file mode 100644 index 0000000000..bfe35ad90f --- /dev/null +++ b/libavcodec/loongarch/videodsp_init.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Xiwei Gu + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" +#include "libavutil/attributes.h" +#include "libavcodec/videodsp.h" + +static void prefetch_loongarch(uint8_t *mem, ptrdiff_t stride, int h) +{ + register const uint8_t *p = mem; + + __asm__ volatile ( + "1: \n\t" + "preld 0, %[p], 0 \n\t" + "preld 0, %[p], 32 \n\t" + "addi.d %[h], %[h], -1 \n\t" + "add.d %[p], %[p], %[stride] \n\t" + + "blt $r0, %[h], 1b \n\t" + : [p] "+r" (p), [h] "+r" (h) + : [stride] "r" (stride) + ); +} + +av_cold void ff_videodsp_init_loongarch(VideoDSPContext *ctx, int bpc) +{ + ctx->prefetch = prefetch_loongarch; +} diff --git a/libavcodec/loongarch/vp8_lpf_lsx.c b/libavcodec/loongarch/vp8_lpf_lsx.c new file mode 100644 index 0000000000..d16d5dca4f --- /dev/null +++ b/libavcodec/loongarch/vp8_lpf_lsx.c @@ -0,0 +1,437 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hecai Yuan + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavcodec/vp8dsp.h" +#include "vp8dsp_loongarch.h" +#include "libavutil/loongarch/generic_macros_lsx.h" + +#define VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev) \ +{ \ + __m128i p2_m, p1_m, p0_m, q2_m, q1_m, q0_m; \ + __m128i filt, q0_sub_p0, cnst4b, cnst3b; \ + __m128i u, filt1, filt2, filt_sign, q0_sub_p0_sign; \ + __m128i q0_sub_p0_l, q0_sub_p0_h, filt_l, u_l, u_h, filt_h; \ + __m128i cnst3h, cnst27h, cnst18h, cnst63h; \ + \ + cnst3h = __lsx_vreplgr2vr_h(3); \ + \ + p2_m = __lsx_vxori_b(p2, 0x80); \ + p1_m = __lsx_vxori_b(p1, 0x80); \ + p0_m = __lsx_vxori_b(p0, 0x80); \ + q0_m = __lsx_vxori_b(q0, 0x80); \ + q1_m = __lsx_vxori_b(q1, 0x80); \ + q2_m = __lsx_vxori_b(q2, 0x80); \ + \ + filt = __lsx_vssub_b(p1_m, q1_m); \ + q0_sub_p0 = __lsx_vsub_b(q0_m, p0_m); \ + q0_sub_p0_sign = __lsx_vslti_b(q0_sub_p0, 0); \ + filt_sign = __lsx_vslti_b(filt, 0); \ + \ + /* right part */ \ + q0_sub_p0_l = __lsx_vilvl_b(q0_sub_p0_sign, q0_sub_p0); \ + q0_sub_p0_l = __lsx_vmul_h(q0_sub_p0_l, cnst3h); \ + filt_l = __lsx_vilvl_b(filt_sign, filt); \ + filt_l = __lsx_vadd_h(filt_l, q0_sub_p0_l); \ + filt_l = __lsx_vsat_h(filt_l, 7); \ + \ + /* left part */ \ + q0_sub_p0_h = __lsx_vilvh_b(q0_sub_p0_sign, q0_sub_p0); \ + q0_sub_p0_h = __lsx_vmul_h(q0_sub_p0_h, cnst3h); \ + filt_h = __lsx_vilvh_b(filt_sign, filt); \ + filt_h = __lsx_vadd_h(filt_h, q0_sub_p0_h); \ + filt_h = __lsx_vsat_h(filt_h, 7); \ + \ + /* combine left and right part */ \ + filt = __lsx_vpickev_b(filt_h, filt_l); \ + filt = filt & mask; \ + filt2 = filt & hev; \ + /* filt_val &= ~hev */ \ + hev = __lsx_vxori_b(hev, 0xff); \ + filt = filt & hev; \ + cnst4b = __lsx_vreplgr2vr_b(4); \ + filt1 = __lsx_vsadd_b(filt2, cnst4b); \ + filt1 = __lsx_vsrai_b(filt1, 3); \ + cnst3b = __lsx_vreplgr2vr_b(3); \ + filt2 = __lsx_vsadd_b(filt2, cnst3b); \ + filt2 = __lsx_vsrai_b(filt2, 3); \ + q0_m = __lsx_vssub_b(q0_m, filt1); \ + p0_m = __lsx_vsadd_b(p0_m, filt2); \ + \ + filt_sign = __lsx_vslti_b(filt, 0); \ + filt_l = __lsx_vilvl_b(filt_sign, filt); \ + filt_h = __lsx_vilvh_b(filt_sign, filt); \ + \ + cnst27h = __lsx_vreplgr2vr_h(27); \ + cnst63h = __lsx_vreplgr2vr_h(63); \ + \ + /* right part */ \ + u_l = __lsx_vmul_h(filt_l, cnst27h); \ + u_l = __lsx_vadd_h(u_l, cnst63h); \ + u_l = __lsx_vsrai_h(u_l, 7); \ + u_l = __lsx_vsat_h(u_l, 7); \ + /* left part */ \ + u_h = __lsx_vmul_h(filt_h, cnst27h); \ + u_h = __lsx_vadd_h(u_h, cnst63h); \ + u_h = __lsx_vsrai_h(u_h, 7); \ + u_h = __lsx_vsat_h(u_h, 7); \ + /* combine left and right part */ \ + u = __lsx_vpickev_b(u_h, u_l); \ + q0_m = __lsx_vssub_b(q0_m, u); \ + q0 = __lsx_vxori_b(q0_m, 0x80); \ + p0_m = __lsx_vsadd_b(p0_m, u); \ + p0 = __lsx_vxori_b(p0_m, 0x80); \ + cnst18h = __lsx_vreplgr2vr_h(18); \ + u_l = __lsx_vmul_h(filt_l, cnst18h); \ + u_l = __lsx_vadd_h(u_l, cnst63h); \ + u_l = __lsx_vsrai_h(u_l, 7); \ + u_l = __lsx_vsat_h(u_l, 7); \ + \ + /* left part */ \ + u_h = __lsx_vmul_h(filt_h, cnst18h); \ + u_h = __lsx_vadd_h(u_h, cnst63h); \ + u_h = __lsx_vsrai_h(u_h, 7); \ + u_h = __lsx_vsat_h(u_h, 7); \ + /* combine left and right part */ \ + u = __lsx_vpickev_b(u_h, u_l); \ + q1_m = __lsx_vssub_b(q1_m, u); \ + q1 = __lsx_vxori_b(q1_m, 0x80); \ + p1_m = __lsx_vsadd_b(p1_m, u); \ + p1 = __lsx_vxori_b(p1_m, 0x80); \ + u_l = __lsx_vslli_h(filt_l, 3); \ + u_l = __lsx_vadd_h(u_l, filt_l); \ + u_l = __lsx_vadd_h(u_l, cnst63h); \ + u_l = __lsx_vsrai_h(u_l, 7); \ + u_l = __lsx_vsat_h(u_l, 7); \ + \ + /* left part */ \ + u_h = __lsx_vslli_h(filt_h, 3); \ + u_h = __lsx_vadd_h(u_h, filt_h); \ + u_h = __lsx_vadd_h(u_h, cnst63h); \ + u_h = __lsx_vsrai_h(u_h, 7); \ + u_h = __lsx_vsat_h(u_h, 7); \ + /* combine left and right part */ \ + u = __lsx_vpickev_b(u_h, u_l); \ + q2_m = __lsx_vssub_b(q2_m, u); \ + q2 = __lsx_vxori_b(q2_m, 0x80); \ + p2_m = __lsx_vsadd_b(p2_m, u); \ + p2 = __lsx_vxori_b(p2_m, 0x80); \ +} + +#define LPF_MASK_HEV(p3_src, p2_src, p1_src, p0_src, \ + q0_src, q1_src, q2_src, q3_src, \ + limit_src, b_limit_src, thresh_src, \ + hev_dst, mask_dst, flat_dst) \ +{ \ + __m128i p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \ + __m128i p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \ + \ + /* absolute subtraction of pixel values */ \ + p3_asub_p2_m = __lsx_vabsd_bu(p3_src, p2_src); \ + p2_asub_p1_m = __lsx_vabsd_bu(p2_src, p1_src); \ + p1_asub_p0_m = __lsx_vabsd_bu(p1_src, p0_src); \ + q1_asub_q0_m = __lsx_vabsd_bu(q1_src, q0_src); \ + q2_asub_q1_m = __lsx_vabsd_bu(q2_src, q1_src); \ + q3_asub_q2_m = __lsx_vabsd_bu(q3_src, q2_src); \ + p0_asub_q0_m = __lsx_vabsd_bu(p0_src, q0_src); \ + p1_asub_q1_m = __lsx_vabsd_bu(p1_src, q1_src); \ + \ + /* calculation of hev */ \ + flat_dst = __lsx_vmax_bu(p1_asub_p0_m, q1_asub_q0_m); \ + hev_dst = __lsx_vslt_bu(thresh_src, flat_dst); \ + /* calculation of mask */ \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p0_asub_q0_m); \ + p1_asub_q1_m = __lsx_vsrli_b(p1_asub_q1_m, 1); \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p1_asub_q1_m); \ + mask_dst = __lsx_vslt_bu(b_limit_src, p0_asub_q0_m); \ + mask_dst = __lsx_vmax_bu(flat_dst, mask_dst); \ + p3_asub_p2_m = __lsx_vmax_bu(p3_asub_p2_m, p2_asub_p1_m); \ + mask_dst = __lsx_vmax_bu(p3_asub_p2_m, mask_dst); \ + q2_asub_q1_m = __lsx_vmax_bu(q2_asub_q1_m, q3_asub_q2_m); \ + mask_dst = __lsx_vmax_bu(q2_asub_q1_m, mask_dst); \ + mask_dst = __lsx_vslt_bu(limit_src, mask_dst); \ + mask_dst = __lsx_vxori_b(mask_dst, 0xff); \ +} + +#define VP8_ST6x1_UB(in0, in0_idx, in1, in1_idx, pdst, stride) \ +{ \ + \ + __lsx_vstelm_w(in0, pdst, 0, in0_idx); \ + __lsx_vstelm_h(in1, pdst + stride, 0, in1_idx); \ +} + +void ff_vp8_v_loop_filter16_lsx(uint8_t *dst, ptrdiff_t stride, int b_limit_in, + int limit_in, int thresh_in) +{ + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + + b_limit = __lsx_vreplgr2vr_b(b_limit_in); + limit = __lsx_vreplgr2vr_b(limit_in); + thresh = __lsx_vreplgr2vr_b(thresh_in); + + /*load vector elements*/ + LSX_DUP4_ARG2(__lsx_vld, dst - stride4, 0, dst - stride3, 0, dst - stride2, 0, + dst - stride, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, dst + stride3, 0, + q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + /*store vector elements*/ + __lsx_vst(p2, dst - stride3, 0); + __lsx_vst(p1, dst - stride2, 0); + __lsx_vst(p0, dst - stride, 0); + __lsx_vst(q0, dst, 0); + + __lsx_vst(q1, dst + stride, 0); + __lsx_vst(q2, dst + stride2, 0); +} + +void ff_vp8_v_loop_filter8uv_lsx(uint8_t *dst_u, uint8_t *dst_v, + ptrdiff_t stride, int b_limit_in, + int limit_in, int thresh_in) +{ + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u; + __m128i p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v; + + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + + b_limit = __lsx_vreplgr2vr_b(b_limit_in); + limit = __lsx_vreplgr2vr_b(limit_in); + thresh = __lsx_vreplgr2vr_b(thresh_in); + + LSX_DUP4_ARG2(__lsx_vld, dst_u - stride4, 0, dst_u - stride3, 0, dst_u - stride2, 0, + dst_u - stride, 0, p3_u, p2_u, p1_u, p0_u); + LSX_DUP4_ARG2(__lsx_vld, dst_u, 0, dst_u + stride, 0, dst_u + stride2, 0, + dst_u + stride3, 0, q0_u, q1_u, q2_u, q3_u); + + LSX_DUP4_ARG2(__lsx_vld, dst_v - stride4, 0, dst_v - stride3, 0, dst_v - stride2, 0, + dst_v - stride, 0, p3_v, p2_v, p1_v, p0_v); + LSX_DUP4_ARG2(__lsx_vld, dst_v, 0, dst_v + stride, 0, dst_v + stride2, 0, + dst_v + stride3, 0, q0_v, q1_v, q2_v, q3_v); + + /* rht 8 element of p3 are u pixel and left 8 element of p3 are v pixei */ + LSX_DUP4_ARG2(__lsx_vilvl_d, p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vilvl_d, q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + __lsx_vstelm_d(p2, dst_u - stride3, 0, 0); + __lsx_vstelm_d(p1, dst_u - stride2, 0, 0); + __lsx_vstelm_d(p0, dst_u - stride , 0, 0); + __lsx_vstelm_d(q0, dst_u, 0, 0); + + __lsx_vstelm_d(q1, dst_u + stride, 0, 0); + __lsx_vstelm_d(q2, dst_u + stride2, 0, 0); + + __lsx_vstelm_d(p2, dst_v - stride3, 0, 1); + __lsx_vstelm_d(p1, dst_v - stride2, 0, 1); + __lsx_vstelm_d(p0, dst_v - stride , 0, 1); + __lsx_vstelm_d(q0, dst_v, 0, 1); + + __lsx_vstelm_d(q1, dst_v + stride, 0, 1); + __lsx_vstelm_d(q2, dst_v + stride2, 0, 1); +} + +void ff_vp8_h_loop_filter16_lsx(uint8_t *dst, ptrdiff_t stride, int b_limit_in, + int limit_in, int thresh_in) +{ + uint8_t *temp_src; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i row0, row1, row2, row3, row4, row5, row6, row7, row8; + __m128i row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + + b_limit = __lsx_vreplgr2vr_b(b_limit_in); + limit = __lsx_vreplgr2vr_b(limit_in); + thresh = __lsx_vreplgr2vr_b(thresh_in); + + temp_src = dst - 4; + LSX_DUP4_ARG2(__lsx_vld, temp_src, 0, temp_src + stride, 0, temp_src + stride2, 0, + temp_src + stride3, 0, row0, row1, row2, row3); + temp_src += stride4; + LSX_DUP4_ARG2(__lsx_vld, temp_src, 0, temp_src + stride, 0, + temp_src + stride2, 0, temp_src + stride3, 0, row4, row5, row6, row7); + + temp_src += stride4; + LSX_DUP4_ARG2(__lsx_vld, temp_src, 0, temp_src + stride, 0, temp_src + stride2, 0, + temp_src + stride3, 0, row8, row9, row10, row11); + temp_src += stride4; + LSX_DUP4_ARG2(__lsx_vld, temp_src, 0, temp_src + stride, 0, + temp_src + stride2, 0, temp_src + stride3, 0, row12, row13, row14, row15); + TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, row9, row10, row11, + row12, row13, row14, row15, p3, p2, p1, p0, q0, q1, q2, q3); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + tmp0 = __lsx_vilvl_b(p1, p2); + tmp1 = __lsx_vilvl_b(q0, p0); + + tmp3 = __lsx_vilvl_h(tmp1, tmp0); + tmp4 = __lsx_vilvh_h(tmp1, tmp0); + + tmp0 = __lsx_vilvh_b(p1, p2); + tmp1 = __lsx_vilvh_b(q0, p0); + + tmp6 = __lsx_vilvl_h(tmp1, tmp0); + tmp7 = __lsx_vilvh_h(tmp1, tmp0); + + tmp2 = __lsx_vilvl_b(q2, q1); + tmp5 = __lsx_vilvh_b(q2, q1); + + temp_src = dst - 3; + VP8_ST6x1_UB(tmp3, 0, tmp2, 0, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp3, 1, tmp2, 1, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp3, 2, tmp2, 2, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp3, 3, tmp2, 3, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp4, 0, tmp2, 4, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp4, 1, tmp2, 5, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp4, 2, tmp2, 6, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp4, 3, tmp2, 7, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp6, 0, tmp5, 0, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp6, 1, tmp5, 1, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp6, 2, tmp5, 2, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp6, 3, tmp5, 3, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp7, 0, tmp5, 4, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp7, 1, tmp5, 5, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp7, 2, tmp5, 6, temp_src, 4); + temp_src += stride; + VP8_ST6x1_UB(tmp7, 3, tmp5, 7, temp_src, 4); +} + +void ff_vp8_h_loop_filter8uv_lsx(uint8_t *dst_u, uint8_t *dst_v, + ptrdiff_t stride, int b_limit_in, + int limit_in, int thresh_in) +{ + uint8_t *temp_src; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i row0, row1, row2, row3, row4, row5, row6, row7, row8; + __m128i row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + + b_limit = __lsx_vreplgr2vr_b(b_limit_in); + limit = __lsx_vreplgr2vr_b(limit_in); + thresh = __lsx_vreplgr2vr_b(thresh_in); + + temp_src = dst_u - 4; + LSX_DUP4_ARG2(__lsx_vld, temp_src, 0, temp_src + stride, 0, temp_src + stride2, 0, + temp_src + stride3, 0, row0, row1, row2, row3); + temp_src += stride4; + LSX_DUP4_ARG2(__lsx_vld, temp_src, 0, temp_src + stride, 0, + temp_src + stride2, 0, temp_src + stride3, 0, row4, row5, row6, row7); + + temp_src = dst_v - 4; + LSX_DUP4_ARG2(__lsx_vld, temp_src, 0, temp_src + stride, 0, temp_src + stride2, 0, + temp_src + stride3, 0, row8, row9, row10, row11); + temp_src += stride4; + LSX_DUP4_ARG2(__lsx_vld, temp_src, 0, temp_src + stride, 0, + temp_src + stride2, 0, temp_src + stride3, 0, row12, row13, row14, row15); + + TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, + row8, row9, row10, row11, row12, row13, row14, row15, + p3, p2, p1, p0, q0, q1, q2, q3); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + tmp0 = __lsx_vilvl_b(p1, p2); + tmp1 = __lsx_vilvl_b(q0, p0); + + tmp3 = __lsx_vilvl_h(tmp1, tmp0); + tmp4 = __lsx_vilvh_h(tmp1, tmp0); + + tmp0 = __lsx_vilvh_b(p1, p2); + tmp1 = __lsx_vilvh_b(q0, p0); + + tmp6 = __lsx_vilvl_h(tmp1, tmp0); + tmp7 = __lsx_vilvh_h(tmp1, tmp0); + + tmp2 = __lsx_vilvl_b(q2, q1); + tmp5 = __lsx_vilvh_b(q2, q1); + + dst_u -= 3; + VP8_ST6x1_UB(tmp3, 0, tmp2, 0, dst_u, 4); + dst_u += stride; + VP8_ST6x1_UB(tmp3, 1, tmp2, 1, dst_u, 4); + dst_u += stride; + VP8_ST6x1_UB(tmp3, 2, tmp2, 2, dst_u, 4); + dst_u += stride; + VP8_ST6x1_UB(tmp3, 3, tmp2, 3, dst_u, 4); + dst_u += stride; + VP8_ST6x1_UB(tmp4, 0, tmp2, 4, dst_u, 4); + dst_u += stride; + VP8_ST6x1_UB(tmp4, 1, tmp2, 5, dst_u, 4); + dst_u += stride; + VP8_ST6x1_UB(tmp4, 2, tmp2, 6, dst_u, 4); + dst_u += stride; + VP8_ST6x1_UB(tmp4, 3, tmp2, 7, dst_u, 4); + + dst_v -= 3; + VP8_ST6x1_UB(tmp6, 0, tmp5, 0, dst_v, 4); + dst_v += stride; + VP8_ST6x1_UB(tmp6, 1, tmp5, 1, dst_v, 4); + dst_v += stride; + VP8_ST6x1_UB(tmp6, 2, tmp5, 2, dst_v, 4); + dst_v += stride; + VP8_ST6x1_UB(tmp6, 3, tmp5, 3, dst_v, 4); + dst_v += stride; + VP8_ST6x1_UB(tmp7, 0, tmp5, 4, dst_v, 4); + dst_v += stride; + VP8_ST6x1_UB(tmp7, 1, tmp5, 5, dst_v, 4); + dst_v += stride; + VP8_ST6x1_UB(tmp7, 2, tmp5, 6, dst_v, 4); + dst_v += stride; + VP8_ST6x1_UB(tmp7, 3, tmp5, 7, dst_v, 4); +} diff --git a/libavcodec/loongarch/vp8dsp_init_loongarch.c b/libavcodec/loongarch/vp8dsp_init_loongarch.c new file mode 100644 index 0000000000..17a7c4065c --- /dev/null +++ b/libavcodec/loongarch/vp8dsp_init_loongarch.c @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hecai Yuan + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * VP8 compatible video decoder + */ + +#include "libavutil/loongarch/cpu.h" +#include "libavcodec/vp8dsp.h" +#include "vp8dsp_loongarch.h" + +av_cold void ff_vp8dsp_init_loongarch(VP8DSPContext *dsp) +{ + int cpu_flags = av_get_cpu_flags(); + + if (have_lsx(cpu_flags)) { + dsp->vp8_v_loop_filter16y = ff_vp8_v_loop_filter16_lsx; + dsp->vp8_h_loop_filter16y = ff_vp8_h_loop_filter16_lsx; + dsp->vp8_v_loop_filter8uv = ff_vp8_v_loop_filter8uv_lsx; + dsp->vp8_h_loop_filter8uv = ff_vp8_h_loop_filter8uv_lsx; + } +} diff --git a/libavcodec/loongarch/vp8dsp_loongarch.h b/libavcodec/loongarch/vp8dsp_loongarch.h new file mode 100644 index 0000000000..f73ab784cc --- /dev/null +++ b/libavcodec/loongarch/vp8dsp_loongarch.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hecai Yuan + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_VP8DSP_LOONGARCH_H +#define AVCODEC_LOONGARCH_VP8DSP_LOONGARCH_H + +#include "libavcodec/vp8dsp.h" + +/* loop filter */ +void ff_vp8_v_loop_filter16_lsx(uint8_t *dst, ptrdiff_t stride, + int flim_e, int flim_i, int hev_thresh); +void ff_vp8_h_loop_filter16_lsx(uint8_t *dst, ptrdiff_t stride, + int flim_e, int flim_i, int hev_thresh); +void ff_vp8_h_loop_filter8uv_lsx(uint8_t *dst_u, uint8_t *dst_v, + ptrdiff_t stride, + int flim_e, int flim_i, int hev_thresh); +void ff_vp8_v_loop_filter8uv_lsx(uint8_t *dst_u, uint8_t *dst_v, + ptrdiff_t stride, + int flim_e, int flim_i, int hev_thresh); + +#endif // #ifndef AVCODEC_LOONGARCH_VP8DSP_LOONGARCH_H diff --git a/libavcodec/loongarch/vp9_idct_lsx.c b/libavcodec/loongarch/vp9_idct_lsx.c new file mode 100644 index 0000000000..b9412c0b5d --- /dev/null +++ b/libavcodec/loongarch/vp9_idct_lsx.c @@ -0,0 +1,1513 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Jin Bo + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavcodec/vp9dsp.h" +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "vp9dsp_loongarch.h" +#include "libavutil/attributes.h" + +#define VP9_DCT_CONST_BITS 14 +#define ALLOC_ALIGNED(align) __attribute__ ((aligned(align))) +#define ROUND_POWER_OF_TWO(value, n) (((value) + (1 << ((n) - 1))) >> (n)) + +static const int32_t cospi_1_64 = 16364; +static const int32_t cospi_2_64 = 16305; +static const int32_t cospi_3_64 = 16207; +static const int32_t cospi_4_64 = 16069; +static const int32_t cospi_5_64 = 15893; +static const int32_t cospi_6_64 = 15679; +static const int32_t cospi_7_64 = 15426; +static const int32_t cospi_8_64 = 15137; +static const int32_t cospi_9_64 = 14811; +static const int32_t cospi_10_64 = 14449; +static const int32_t cospi_11_64 = 14053; +static const int32_t cospi_12_64 = 13623; +static const int32_t cospi_13_64 = 13160; +static const int32_t cospi_14_64 = 12665; +static const int32_t cospi_15_64 = 12140; +static const int32_t cospi_16_64 = 11585; +static const int32_t cospi_17_64 = 11003; +static const int32_t cospi_18_64 = 10394; +static const int32_t cospi_19_64 = 9760; +static const int32_t cospi_20_64 = 9102; +static const int32_t cospi_21_64 = 8423; +static const int32_t cospi_22_64 = 7723; +static const int32_t cospi_23_64 = 7005; +static const int32_t cospi_24_64 = 6270; +static const int32_t cospi_25_64 = 5520; +static const int32_t cospi_26_64 = 4756; +static const int32_t cospi_27_64 = 3981; +static const int32_t cospi_28_64 = 3196; +static const int32_t cospi_29_64 = 2404; +static const int32_t cospi_30_64 = 1606; +static const int32_t cospi_31_64 = 804; + +static const int32_t sinpi_1_9 = 5283; +static const int32_t sinpi_2_9 = 9929; +static const int32_t sinpi_3_9 = 13377; +static const int32_t sinpi_4_9 = 15212; + +#define VP9_DOTP_CONST_PAIR(reg0, reg1, cnst0, cnst1, out0, out1) \ +{ \ + __m128i k0_m = __lsx_vreplgr2vr_h(cnst0); \ + __m128i s0_m, s1_m, s2_m, s3_m; \ + \ + s0_m = __lsx_vreplgr2vr_h(cnst1); \ + k0_m = __lsx_vpackev_h(s0_m, k0_m); \ + \ + s1_m = __lsx_vilvl_h(__lsx_vneg_h(reg1), reg0); \ + s0_m = __lsx_vilvh_h(__lsx_vneg_h(reg1), reg0); \ + s3_m = __lsx_vilvl_h(reg0, reg1); \ + s2_m = __lsx_vilvh_h(reg0, reg1); \ + LSX_DUP2_ARG2(__lsx_dp2_w_h, s1_m, k0_m, s0_m, k0_m, s1_m, \ + s0_m); \ + LSX_DUP2_ARG2(__lsx_vsrari_w, s1_m, VP9_DCT_CONST_BITS, \ + s0_m, VP9_DCT_CONST_BITS, s1_m, s0_m); \ + out0 = __lsx_vpickev_h(s0_m, s1_m); \ + LSX_DUP2_ARG2(__lsx_dp2_w_h, s3_m, k0_m, s2_m, k0_m, s1_m, \ + s0_m); \ + LSX_DUP2_ARG2(__lsx_vsrari_w, s1_m, VP9_DCT_CONST_BITS, \ + s0_m, VP9_DCT_CONST_BITS, s1_m, s0_m); \ + out1 = __lsx_vpickev_h(s0_m, s1_m); \ +} + +#define VP9_SET_COSPI_PAIR(c0_h, c1_h) \ +( { \ + __m128i out0_m, r0_m, r1_m; \ + \ + r0_m = __lsx_vreplgr2vr_h(c0_h); \ + r1_m = __lsx_vreplgr2vr_h(c1_h); \ + out0_m = __lsx_vpackev_h(r1_m, r0_m); \ + \ + out0_m; \ +} ) + +#define VP9_ADDBLK_ST8x4_UB(dst, dst_stride, in0, in1, in2, in3) \ +{ \ + uint8_t *dst_m = (uint8_t *) (dst); \ + __m128i dst0_m, dst1_m, dst2_m, dst3_m; \ + __m128i tmp0_m, tmp1_m; \ + __m128i res0_m, res1_m, res2_m, res3_m; \ + __m128i zero_m = __lsx_vldi(0); \ + LSX_DUP4_ARG2(__lsx_vld, \ + dst_m, 0, \ + dst_m + dst_stride, 0, \ + dst_m + 2 * dst_stride, 0, \ + dst_m + 3 * dst_stride, 0, \ + dst0_m, dst1_m, dst2_m, dst3_m); \ + LSX_DUP4_ARG2(__lsx_vilvl_b, \ + zero_m, dst0_m, zero_m, dst1_m, zero_m, dst2_m, \ + zero_m, dst3_m, res0_m, res1_m, res2_m, res3_m); \ + LSX_DUP4_ARG2(__lsx_vadd_h, \ + res0_m, in0, res1_m, in1, res2_m, in2, res3_m, \ + in3, res0_m, res1_m, res2_m, res3_m); \ + LSX_DUP4_ARG1(__lsx_clamp255_h, \ + res0_m, res1_m, res2_m, res3_m, \ + res0_m, res1_m, res2_m, res3_m); \ + LSX_DUP2_ARG2(__lsx_vpickev_b, \ + res1_m, res0_m, res3_m, res2_m, tmp0_m, tmp1_m); \ + __lsx_vstelm_d(tmp0_m, dst_m, 0, 0); \ + __lsx_vstelm_d(tmp0_m, dst_m + dst_stride, 0, 1); \ + __lsx_vstelm_d(tmp1_m, dst_m + 2 * dst_stride, 0, 0); \ + __lsx_vstelm_d(tmp1_m, dst_m + 3 * dst_stride, 0, 1); \ +} + +#define VP9_UNPCK_UB_SH(in, out_h, out_l) \ +{ \ + __m128i zero = __lsx_vldi(0); \ + out_l = __lsx_vilvl_b(zero, in); \ + out_h = __lsx_vilvh_b(zero, in); \ +} + +#define VP9_ILVLTRANS4x8_H(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + __m128i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + __m128i tmp0_n, tmp1_n, tmp2_n, tmp3_n; \ + __m128i zero_m = __lsx_vldi(0); \ + \ + LSX_DUP4_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, in5, in4, in7, in6, \ + tmp0_n, tmp1_n, tmp2_n, tmp3_n); \ + tmp0_m = __lsx_vilvl_w(tmp1_n, tmp0_n); \ + tmp2_m = __lsx_vilvh_w(tmp1_n, tmp0_n); \ + tmp1_m = __lsx_vilvl_w(tmp3_n, tmp2_n); \ + tmp3_m = __lsx_vilvh_w(tmp3_n, tmp2_n); \ + \ + out0 = __lsx_vilvl_d(tmp1_m, tmp0_m); \ + out1 = __lsx_vilvh_d(tmp1_m, tmp0_m); \ + out2 = __lsx_vilvl_d(tmp3_m, tmp2_m); \ + out3 = __lsx_vilvh_d(tmp3_m, tmp2_m); \ + \ + out4 = zero_m; \ + out5 = zero_m; \ + out6 = zero_m; \ + out7 = zero_m; \ +} + +/* multiply and add macro */ +#define VP9_MADD(inp0, inp1, inp2, inp3, cst0, cst1, cst2, cst3, \ + out0, out1, out2, out3) \ +{ \ + __m128i madd_s0_m, madd_s1_m, madd_s2_m, madd_s3_m; \ + __m128i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + \ + madd_s1_m = __lsx_vilvl_h(inp1, inp0); \ + madd_s0_m = __lsx_vilvh_h(inp1, inp0); \ + madd_s3_m = __lsx_vilvl_h(inp3, inp2); \ + madd_s2_m = __lsx_vilvh_h(inp3, inp2); \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, \ + madd_s1_m, cst0, madd_s0_m, cst0, \ + madd_s1_m, cst1, madd_s0_m, cst1, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + LSX_DUP4_ARG2(__lsx_vsrari_w, \ + tmp0_m, VP9_DCT_CONST_BITS, \ + tmp1_m, VP9_DCT_CONST_BITS, \ + tmp2_m, VP9_DCT_CONST_BITS, \ + tmp3_m, VP9_DCT_CONST_BITS, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + LSX_DUP2_ARG2(__lsx_vpickev_h, tmp1_m, tmp0_m, tmp3_m, tmp2_m, \ + out0, out1); \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, \ + madd_s3_m, cst2, madd_s2_m, cst2, \ + madd_s3_m, cst3, madd_s2_m, cst3, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + LSX_DUP4_ARG2(__lsx_vsrari_w, \ + tmp0_m, VP9_DCT_CONST_BITS, \ + tmp1_m, VP9_DCT_CONST_BITS, \ + tmp2_m, VP9_DCT_CONST_BITS, \ + tmp3_m, VP9_DCT_CONST_BITS, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + LSX_DUP2_ARG2(__lsx_vpickev_h, tmp1_m, tmp0_m, tmp3_m, tmp2_m, \ + out2, out3); \ +} + +#define VP9_SET_CONST_PAIR(mask_h, idx1_h, idx2_h) \ +( { \ + __m128i c0_m, c1_m; \ + \ + LSX_DUP2_ARG2(__lsx_vreplvei_h, \ + mask_h, idx1_h, mask_h, idx2_h, \ + c0_m, c1_m); \ + c0_m = __lsx_vpackev_h(c1_m, c0_m); \ + \ + c0_m; \ +} ) + +/* idct 8x8 macro */ +#define VP9_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + __m128i tp0_m, tp1_m, tp2_m, tp3_m, tp4_m, tp5_m, tp6_m, tp7_m; \ + __m128i k0_m, k1_m, k2_m, k3_m, res0_m, res1_m, res2_m, res3_m; \ + __m128i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + v8i16 mask_m = { cospi_28_64, cospi_4_64, cospi_20_64, cospi_12_64, \ + cospi_16_64, -cospi_4_64, -cospi_20_64, -cospi_16_64 }; \ + \ + k0_m = VP9_SET_CONST_PAIR(mask_m, 0, 5); \ + k1_m = VP9_SET_CONST_PAIR(mask_m, 1, 0); \ + k2_m = VP9_SET_CONST_PAIR(mask_m, 6, 3); \ + k3_m = VP9_SET_CONST_PAIR(mask_m, 3, 2); \ + VP9_MADD(in1, in7, in3, in5, k0_m, k1_m, k2_m, k3_m, in1, in7, in3, in5); \ + LSX_DUP2_ARG2(__lsx_vsub_h, in1, in3, in7, in5, res0_m, res1_m); \ + k0_m = VP9_SET_CONST_PAIR(mask_m, 4, 7); \ + k1_m = __lsx_vreplvei_h(mask_m, 4); \ + \ + res2_m = __lsx_vilvl_h(res0_m, res1_m); \ + res3_m = __lsx_vilvh_h(res0_m, res1_m); \ + LSX_DUP4_ARG2(__lsx_dp2_w_h, \ + res2_m, k0_m, res3_m, k0_m, res2_m, k1_m, res3_m, k1_m, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + LSX_DUP4_ARG2(__lsx_vsrari_w, \ + tmp0_m, VP9_DCT_CONST_BITS, tmp1_m, VP9_DCT_CONST_BITS, \ + tmp2_m, VP9_DCT_CONST_BITS, tmp3_m, VP9_DCT_CONST_BITS, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + tp4_m = __lsx_vadd_h(in1, in3); \ + LSX_DUP2_ARG2(__lsx_vpickev_h, tmp1_m, tmp0_m, tmp3_m, tmp2_m, \ + tp5_m, tp6_m); \ + tp7_m = __lsx_vadd_h(in7, in5); \ + k2_m = VP9_SET_COSPI_PAIR(cospi_24_64, -cospi_8_64); \ + k3_m = VP9_SET_COSPI_PAIR(cospi_8_64, cospi_24_64); \ + VP9_MADD(in0, in4, in2, in6, k1_m, k0_m, k2_m, k3_m, \ + in0, in4, in2, in6); \ + BUTTERFLY_4_H(in0, in4, in2, in6, tp0_m, tp1_m, tp2_m, tp3_m); \ + BUTTERFLY_8_H(tp0_m, tp1_m, tp2_m, tp3_m, tp4_m, tp5_m, tp6_m, tp7_m, \ + out0, out1, out2, out3, out4, out5, out6, out7); \ +} + +static av_always_inline +void vp9_idct8x8_1_add_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + int16_t out; + int32_t val; + __m128i vec; + + out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), VP9_DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO((out * cospi_16_64), VP9_DCT_CONST_BITS); + val = ROUND_POWER_OF_TWO(out, 5); + vec = __lsx_vreplgr2vr_h(val); + input[0] = 0; + + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, vec, vec, vec, vec); + dst += (4 * dst_stride); + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, vec, vec, vec, vec); +} + +static void vp9_idct8x8_12_colcol_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i s0, s1, s2, s3, s4, s5, s6, s7, k0, k1, k2, k3, m0, m1, m2, m3; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i zero = __lsx_vldi(0); + + /* load vector elements of 8x8 block */ + LSX_DUP4_ARG2(__lsx_vld, + input, 0, input, 16, input, 32, input, 48, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, + input, 64, input, 80, input, 96, input, 112, + in4, in5, in6, in7); + __lsx_vst(zero, input, 0); + __lsx_vst(zero, input, 16); + __lsx_vst(zero, input, 32); + __lsx_vst(zero, input, 48); + __lsx_vst(zero, input, 64); + __lsx_vst(zero, input, 80); + __lsx_vst(zero, input, 96); + __lsx_vst(zero, input, 112); + LSX_DUP4_ARG2(__lsx_vilvl_d, + in1, in0, in3, in2, in5, in4, in7, in6, + in0, in1, in2, in3); + + /* stage1 */ + LSX_DUP2_ARG2(__lsx_vilvh_h, in3, in0, in2, in1, s0, s1); + k0 = VP9_SET_COSPI_PAIR(cospi_28_64, -cospi_4_64); + k1 = VP9_SET_COSPI_PAIR(cospi_4_64, cospi_28_64); + k2 = VP9_SET_COSPI_PAIR(-cospi_20_64, cospi_12_64); + k3 = VP9_SET_COSPI_PAIR(cospi_12_64, cospi_20_64); + LSX_DUP4_ARG2(__lsx_dp2_w_h, s0, k0, s0, k1, s1, k2, s1, k3, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vsrari_w, + tmp0, VP9_DCT_CONST_BITS, + tmp1, VP9_DCT_CONST_BITS, + tmp2, VP9_DCT_CONST_BITS, + tmp3, VP9_DCT_CONST_BITS, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vpickev_h, + zero, tmp0, zero, tmp1, + zero, tmp2, zero, tmp3, + s0, s1, s2, s3); + BUTTERFLY_4_H(s0, s1, s3, s2, s4, s7, s6, s5); + + /* stage2 */ + LSX_DUP2_ARG2(__lsx_vilvl_h, in3, in1, in2, in0, s1, s0); + k0 = VP9_SET_COSPI_PAIR(cospi_16_64, cospi_16_64); + k1 = VP9_SET_COSPI_PAIR(cospi_16_64, -cospi_16_64); + k2 = VP9_SET_COSPI_PAIR(cospi_24_64, -cospi_8_64); + k3 = VP9_SET_COSPI_PAIR(cospi_8_64, cospi_24_64); + LSX_DUP4_ARG2(__lsx_dp2_w_h, s0, k0, s0, k1, s1, k2, s1, k3, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vsrari_w, + tmp0, VP9_DCT_CONST_BITS, + tmp1, VP9_DCT_CONST_BITS, + tmp2, VP9_DCT_CONST_BITS, + tmp3, VP9_DCT_CONST_BITS, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vpickev_h, + zero, tmp0, zero, tmp1, + zero, tmp2, zero, tmp3, + s0, s1, s2, s3); + BUTTERFLY_4_H(s0, s1, s2, s3, m0, m1, m2, m3); + + /* stage3 */ + s0 = __lsx_vilvl_h(s6, s5); + + k1 = VP9_SET_COSPI_PAIR(-cospi_16_64, cospi_16_64); + LSX_DUP2_ARG2(__lsx_dp2_w_h, s0, k1, s0, k0, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vsrari_w, + tmp0, VP9_DCT_CONST_BITS, tmp1, + VP9_DCT_CONST_BITS, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vpickev_h, zero, tmp0, zero, tmp1, s2, s3); + + /* stage4 */ + BUTTERFLY_8_H(m0, m1, m2, m3, s4, s2, s3, s7, + in0, in1, in2, in3, in4, in5, in6, in7); + VP9_ILVLTRANS4x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + VP9_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + + /* final rounding (add 2^4, divide by 2^5) and shift */ + LSX_DUP4_ARG2(__lsx_vsrari_h, in0 , 5, in1, 5, in2, 5, in3, 5, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vsrari_h, in4 , 5, in5, 5, in6, 5, in7, 5, + in4, in5, in6, in7); + + /* add block and store 8x8 */ + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, in0, in1, in2, in3); + dst += (4 * dst_stride); + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, in4, in5, in6, in7); +} + +static void vp9_idct8x8_colcol_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i zero = __lsx_vldi(0); + + /* load vector elements of 8x8 block */ + LSX_DUP4_ARG2(__lsx_vld, + input, 0, input, 16, input, 32, input, 48, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vld, + input, 64, input, 80, input, 96, input, 112, + in4, in5, in6, in7); + __lsx_vst(zero, input, 0); + __lsx_vst(zero, input, 16); + __lsx_vst(zero, input, 32); + __lsx_vst(zero, input, 48); + __lsx_vst(zero, input, 64); + __lsx_vst(zero, input, 80); + __lsx_vst(zero, input, 96); + __lsx_vst(zero, input, 112); + /* 1D idct8x8 */ + VP9_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + /* columns transform */ + TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + /* 1D idct8x8 */ + VP9_IDCT8x8_1D(in0, in1, in2, in3, in4, in5, in6, in7, + in0, in1, in2, in3, in4, in5, in6, in7); + /* final rounding (add 2^4, divide by 2^5) and shift */ + LSX_DUP4_ARG2(__lsx_vsrari_h, + in0, 5, in1, 5, in2, 5, in3, 5, + in0, in1, in2, in3); + LSX_DUP4_ARG2(__lsx_vsrari_h, + in4, 5, in5, 5, in6, 5, in7, 5, + in4, in5, in6, in7); + /* add block and store 8x8 */ + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, in0, in1, in2, in3); + dst += (4 * dst_stride); + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, in4, in5, in6, in7); +} + +static void vp9_idct16_1d_columns_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + __m128i loc0, loc1, loc2, loc3; + __m128i reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14; + __m128i reg1, reg3, reg5, reg7, reg9, reg11, reg13, reg15; + __m128i tmp5, tmp6, tmp7; + __m128i zero = __lsx_vldi(0); + int32_t offset = dst_stride << 2; + + LSX_DUP4_ARG2(__lsx_vld, + input, 32*0, input, 32*1, + input, 32*2, input, 32*3, + reg0, reg1, reg2, reg3); + LSX_DUP4_ARG2(__lsx_vld, + input, 32*4, input, 32*5, + input, 32*6, input, 32*7, + reg4, reg5, reg6, reg7); + LSX_DUP4_ARG2(__lsx_vld, + input, 32*8, input, 32*9, + input, 32*10, input, 32*11, + reg8, reg9, reg10, reg11); + LSX_DUP4_ARG2(__lsx_vld, + input, 32*12, input, 32*13, + input, 32*14, input, 32*15, + reg12, reg13, reg14, reg15); + + __lsx_vst(zero, input, 32*0); + __lsx_vst(zero, input, 32*1); + __lsx_vst(zero, input, 32*2); + __lsx_vst(zero, input, 32*3); + __lsx_vst(zero, input, 32*4); + __lsx_vst(zero, input, 32*5); + __lsx_vst(zero, input, 32*6); + __lsx_vst(zero, input, 32*7); + __lsx_vst(zero, input, 32*8); + __lsx_vst(zero, input, 32*9); + __lsx_vst(zero, input, 32*10); + __lsx_vst(zero, input, 32*11); + __lsx_vst(zero, input, 32*12); + __lsx_vst(zero, input, 32*13); + __lsx_vst(zero, input, 32*14); + __lsx_vst(zero, input, 32*15); + + VP9_DOTP_CONST_PAIR(reg2, reg14, cospi_28_64, cospi_4_64, reg2, reg14); + VP9_DOTP_CONST_PAIR(reg10, reg6, cospi_12_64, cospi_20_64, reg10, reg6); + BUTTERFLY_4_H(reg2, reg14, reg6, reg10, loc0, loc1, reg14, reg2); + VP9_DOTP_CONST_PAIR(reg14, reg2, cospi_16_64, cospi_16_64, loc2, loc3); + VP9_DOTP_CONST_PAIR(reg0, reg8, cospi_16_64, cospi_16_64, reg0, reg8); + VP9_DOTP_CONST_PAIR(reg4, reg12, cospi_24_64, cospi_8_64, reg4, reg12); + BUTTERFLY_4_H(reg8, reg0, reg4, reg12, reg2, reg6, reg10, reg14); + + reg0 = __lsx_vsub_h(reg2, loc1); + reg2 = __lsx_vadd_h(reg2, loc1); + reg12 = __lsx_vsub_h(reg14, loc0); + reg14 = __lsx_vadd_h(reg14, loc0); + reg4 = __lsx_vsub_h(reg6, loc3); + reg6 = __lsx_vadd_h(reg6, loc3); + reg8 = __lsx_vsub_h(reg10, loc2); + reg10 = __lsx_vadd_h(reg10, loc2); + + /* stage2 */ + VP9_DOTP_CONST_PAIR(reg1, reg15, cospi_30_64, cospi_2_64, reg1, reg15); + VP9_DOTP_CONST_PAIR(reg9, reg7, cospi_14_64, cospi_18_64, loc2, loc3); + + reg9 = __lsx_vsub_h(reg1, loc2); + reg1 = __lsx_vadd_h(reg1, loc2); + reg7 = __lsx_vsub_h(reg15, loc3); + reg15 = __lsx_vadd_h(reg15, loc3); + + VP9_DOTP_CONST_PAIR(reg5, reg11, cospi_22_64, cospi_10_64, reg5, reg11); + VP9_DOTP_CONST_PAIR(reg13, reg3, cospi_6_64, cospi_26_64, loc0, loc1); + BUTTERFLY_4_H(loc0, loc1, reg11, reg5, reg13, reg3, reg11, reg5); + + loc1 = __lsx_vadd_h(reg15, reg3); + reg3 = __lsx_vsub_h(reg15, reg3); + loc2 = __lsx_vadd_h(reg2, loc1); + reg15 = __lsx_vsub_h(reg2, loc1); + + loc1 = __lsx_vadd_h(reg1, reg13); + reg13 = __lsx_vsub_h(reg1, reg13); + loc0 = __lsx_vadd_h(reg0, loc1); + loc1 = __lsx_vsub_h(reg0, loc1); + tmp6 = loc0; + tmp7 = loc1; + reg0 = loc2; + + VP9_DOTP_CONST_PAIR(reg7, reg9, cospi_24_64, cospi_8_64, reg7, reg9); + VP9_DOTP_CONST_PAIR(__lsx_vneg_h(reg5), __lsx_vneg_h(reg11), cospi_8_64, + cospi_24_64, reg5, reg11); + + loc0 = __lsx_vadd_h(reg9, reg5); + reg5 = __lsx_vsub_h(reg9, reg5); + reg2 = __lsx_vadd_h(reg6, loc0); + reg1 = __lsx_vsub_h(reg6, loc0); + + loc0 = __lsx_vadd_h(reg7, reg11); + reg11 = __lsx_vsub_h(reg7, reg11); + loc1 = __lsx_vadd_h(reg4, loc0); + loc2 = __lsx_vsub_h(reg4, loc0); + tmp5 = loc1; + + VP9_DOTP_CONST_PAIR(reg5, reg11, cospi_16_64, cospi_16_64, reg5, reg11); + BUTTERFLY_4_H(reg8, reg10, reg11, reg5, loc0, reg4, reg9, loc1); + + reg10 = loc0; + reg11 = loc1; + + VP9_DOTP_CONST_PAIR(reg3, reg13, cospi_16_64, cospi_16_64, reg3, reg13); + BUTTERFLY_4_H(reg12, reg14, reg13, reg3, reg8, reg6, reg7, reg5); + reg13 = loc2; + + /* Transpose and store the output */ + reg12 = tmp5; + reg14 = tmp6; + reg3 = tmp7; + + LSX_DUP4_ARG2(__lsx_vsrari_h, + reg0, 6, reg2, 6, reg4, 6, reg6, 6, + reg0, reg2, reg4, reg6); + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, reg0, reg2, reg4, reg6); + dst += offset; + LSX_DUP4_ARG2(__lsx_vsrari_h, + reg8, 6, reg10, 6, reg12, 6, reg14, 6, + reg8, reg10, reg12, reg14); + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, reg8, reg10, reg12, reg14); + dst += offset; + LSX_DUP4_ARG2(__lsx_vsrari_h, + reg3, 6, reg5, 6, reg11, 6, reg13, 6, + reg3, reg5, reg11, reg13); + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, reg3, reg13, reg11, reg5); + dst += offset; + LSX_DUP4_ARG2(__lsx_vsrari_h, + reg1, 6, reg7, 6, reg9, 6, reg15, 6, + reg1, reg7, reg9, reg15); + VP9_ADDBLK_ST8x4_UB(dst, dst_stride, reg7, reg9, reg1, reg15); +} + +static void vp9_idct16_1d_columns_lsx(int16_t *input, int16_t *output) +{ + __m128i loc0, loc1, loc2, loc3; + __m128i reg1, reg3, reg5, reg7, reg9, reg11, reg13, reg15; + __m128i reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14; + __m128i tmp5, tmp6, tmp7; + __m128i zero = __lsx_vldi(0); + int16_t *offset; + + LSX_DUP4_ARG2(__lsx_vld, + input, 32*0, input, 32*1, + input, 32*2, input, 32*3, + reg0, reg1, reg2, reg3); + LSX_DUP4_ARG2(__lsx_vld, + input, 32*4, input, 32*5, + input, 32*6, input, 32*7, + reg4, reg5, reg6, reg7); + LSX_DUP4_ARG2(__lsx_vld, + input, 32*8, input, 32*9, + input, 32*10, input, 32*11, + reg8, reg9, reg10, reg11); + LSX_DUP4_ARG2(__lsx_vld, + input, 32*12, input, 32*13, + input, 32*14, input, 32*15, + reg12, reg13, reg14, reg15); + + __lsx_vst(zero, input, 32*0); + __lsx_vst(zero, input, 32*1); + __lsx_vst(zero, input, 32*2); + __lsx_vst(zero, input, 32*3); + __lsx_vst(zero, input, 32*4); + __lsx_vst(zero, input, 32*5); + __lsx_vst(zero, input, 32*6); + __lsx_vst(zero, input, 32*7); + __lsx_vst(zero, input, 32*8); + __lsx_vst(zero, input, 32*9); + __lsx_vst(zero, input, 32*10); + __lsx_vst(zero, input, 32*11); + __lsx_vst(zero, input, 32*12); + __lsx_vst(zero, input, 32*13); + __lsx_vst(zero, input, 32*14); + __lsx_vst(zero, input, 32*15); + + VP9_DOTP_CONST_PAIR(reg2, reg14, cospi_28_64, cospi_4_64, reg2, reg14); + VP9_DOTP_CONST_PAIR(reg10, reg6, cospi_12_64, cospi_20_64, reg10, reg6); + BUTTERFLY_4_H(reg2, reg14, reg6, reg10, loc0, loc1, reg14, reg2); + VP9_DOTP_CONST_PAIR(reg14, reg2, cospi_16_64, cospi_16_64, loc2, loc3); + VP9_DOTP_CONST_PAIR(reg0, reg8, cospi_16_64, cospi_16_64, reg0, reg8); + VP9_DOTP_CONST_PAIR(reg4, reg12, cospi_24_64, cospi_8_64, reg4, reg12); + BUTTERFLY_4_H(reg8, reg0, reg4, reg12, reg2, reg6, reg10, reg14); + + reg0 = __lsx_vsub_h(reg2, loc1); + reg2 = __lsx_vadd_h(reg2, loc1); + reg12 = __lsx_vsub_h(reg14, loc0); + reg14 = __lsx_vadd_h(reg14, loc0); + reg4 = __lsx_vsub_h(reg6, loc3); + reg6 = __lsx_vadd_h(reg6, loc3); + reg8 = __lsx_vsub_h(reg10, loc2); + reg10 = __lsx_vadd_h(reg10, loc2); + + /* stage2 */ + VP9_DOTP_CONST_PAIR(reg1, reg15, cospi_30_64, cospi_2_64, reg1, reg15); + VP9_DOTP_CONST_PAIR(reg9, reg7, cospi_14_64, cospi_18_64, loc2, loc3); + + reg9 = __lsx_vsub_h(reg1, loc2); + reg1 = __lsx_vadd_h(reg1, loc2); + reg7 = __lsx_vsub_h(reg15, loc3); + reg15 = __lsx_vadd_h(reg15, loc3); + + VP9_DOTP_CONST_PAIR(reg5, reg11, cospi_22_64, cospi_10_64, reg5, reg11); + VP9_DOTP_CONST_PAIR(reg13, reg3, cospi_6_64, cospi_26_64, loc0, loc1); + BUTTERFLY_4_H(loc0, loc1, reg11, reg5, reg13, reg3, reg11, reg5); + + loc1 = __lsx_vadd_h(reg15, reg3); + reg3 = __lsx_vsub_h(reg15, reg3); + loc2 = __lsx_vadd_h(reg2, loc1); + reg15 = __lsx_vsub_h(reg2, loc1); + + loc1 = __lsx_vadd_h(reg1, reg13); + reg13 = __lsx_vsub_h(reg1, reg13); + loc0 = __lsx_vadd_h(reg0, loc1); + loc1 = __lsx_vsub_h(reg0, loc1); + tmp6 = loc0; + tmp7 = loc1; + reg0 = loc2; + + VP9_DOTP_CONST_PAIR(reg7, reg9, cospi_24_64, cospi_8_64, reg7, reg9); + VP9_DOTP_CONST_PAIR(__lsx_vneg_h(reg5), __lsx_vneg_h(reg11), cospi_8_64, + cospi_24_64, reg5, reg11); + + loc0 = __lsx_vadd_h(reg9, reg5); + reg5 = __lsx_vsub_h(reg9, reg5); + reg2 = __lsx_vadd_h(reg6, loc0); + reg1 = __lsx_vsub_h(reg6, loc0); + + loc0 = __lsx_vadd_h(reg7, reg11); + reg11 = __lsx_vsub_h(reg7, reg11); + loc1 = __lsx_vadd_h(reg4, loc0); + loc2 = __lsx_vsub_h(reg4, loc0); + + tmp5 = loc1; + + VP9_DOTP_CONST_PAIR(reg5, reg11, cospi_16_64, cospi_16_64, reg5, reg11); + BUTTERFLY_4_H(reg8, reg10, reg11, reg5, loc0, reg4, reg9, loc1); + + reg10 = loc0; + reg11 = loc1; + + VP9_DOTP_CONST_PAIR(reg3, reg13, cospi_16_64, cospi_16_64, reg3, reg13); + BUTTERFLY_4_H(reg12, reg14, reg13, reg3, reg8, reg6, reg7, reg5); + reg13 = loc2; + + /* Transpose and store the output */ + reg12 = tmp5; + reg14 = tmp6; + reg3 = tmp7; + + /* transpose block */ + TRANSPOSE8x8_H(reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14, + reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14); + + __lsx_vst(reg0, output, 32*0); + __lsx_vst(reg2, output, 32*1); + __lsx_vst(reg4, output, 32*2); + __lsx_vst(reg6, output, 32*3); + __lsx_vst(reg8, output, 32*4); + __lsx_vst(reg10, output, 32*5); + __lsx_vst(reg12, output, 32*6); + __lsx_vst(reg14, output, 32*7); + + /* transpose block */ + TRANSPOSE8x8_H(reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15, + reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15); + + offset = output + 8; + __lsx_vst(reg3, offset, 32*0); + __lsx_vst(reg13, offset, 32*1); + __lsx_vst(reg11, offset, 32*2); + __lsx_vst(reg5, offset, 32*3); + + offset = output + 8 + 4 * 16; + __lsx_vst(reg7, offset, 32*0); + __lsx_vst(reg9, offset, 32*1); + __lsx_vst(reg1, offset, 32*2); + __lsx_vst(reg15, offset, 32*3); +} + +static void vp9_idct16x16_1_add_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + uint8_t i; + int16_t out; + __m128i vec, res0, res1, res2, res3, res4, res5, res6, res7; + __m128i dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3; + + out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), VP9_DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO((out * cospi_16_64), VP9_DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO(out, 6); + input[0] = 0; + vec = __lsx_vreplgr2vr_h(out); + + for (i = 4; i--;) { + LSX_DUP4_ARG2(__lsx_vld, + dst, 0, + dst + dst_stride, 0, + dst + dst_stride * 2, 0, + dst + dst_stride * 3, 0, + dst0, dst1, dst2, dst3); + VP9_UNPCK_UB_SH(dst0, res4, res0); + VP9_UNPCK_UB_SH(dst1, res5, res1); + VP9_UNPCK_UB_SH(dst2, res6, res2); + VP9_UNPCK_UB_SH(dst3, res7, res3); + LSX_DUP4_ARG2(__lsx_vadd_h, + res0, vec, res1, vec, res2, vec, res3, vec, + res0, res1, res2, res3); + LSX_DUP4_ARG2(__lsx_vadd_h, + res4, vec, res5, vec, res6, vec, res7, vec, + res4, res5, res6, res7); + LSX_DUP4_ARG1(__lsx_clamp255_h, + res0, res1, res2, res3, + res0, res1, res2, res3); + LSX_DUP4_ARG1(__lsx_clamp255_h, + res4, res5, res6, res7, + res4, res5, res6, res7); + LSX_DUP4_ARG2(__lsx_vpickev_b, + res4, res0, res5, res1, res6, res2, res7, res3, + tmp0, tmp1, tmp2, tmp3); + __lsx_vst(tmp0, dst, 0); + __lsx_vst(tmp1, dst + dst_stride, 0); + __lsx_vst(tmp2, dst + dst_stride * 2, 0); + __lsx_vst(tmp3, dst + dst_stride * 3, 0); + dst += dst_stride << 2; + } +} + +static void vp9_idct16x16_10_colcol_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + int32_t i; + int16_t out_arr[16 * 16] ALLOC_ALIGNED(16); + int16_t *out = out_arr; + __m128i zero = __lsx_vldi(0); + + /* transform rows */ + vp9_idct16_1d_columns_lsx(input, out); + + /* short case just considers top 4 rows as valid output */ + out += 4 * 16; + for (i = 3; i--;) { + __lsx_vst(zero, out, 0); + __lsx_vst(zero, out, 16); + __lsx_vst(zero, out, 32); + __lsx_vst(zero, out, 48); + __lsx_vst(zero, out, 64); + __lsx_vst(zero, out, 80); + __lsx_vst(zero, out, 96); + __lsx_vst(zero, out, 112); + out += 64; + } + + out = out_arr; + + /* transform columns */ + for (i = 0; i < 2; i++) { + /* process 8 * 16 block */ + vp9_idct16_1d_columns_addblk_lsx((out + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +static void vp9_idct16x16_colcol_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + int32_t i; + int16_t out_arr[16 * 16] ALLOC_ALIGNED(16); + int16_t *out = out_arr; + + /* transform rows */ + for (i = 0; i < 2; i++) { + /* process 8 * 16 block */ + vp9_idct16_1d_columns_lsx((input + (i << 3)), (out + (i << 7))); + } + + /* transform columns */ + for (i = 0; i < 2; i++) { + /* process 8 * 16 block */ + vp9_idct16_1d_columns_addblk_lsx((out + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +static void vp9_idct_butterfly_transpose_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf, + int16_t *tmp_odd_buf, + int16_t *dst) +{ + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i m0, m1, m2, m3, m4, m5, m6, m7, n0, n1, n2, n3, n4, n5, n6, n7; + + /* FINAL BUTTERFLY : Dependency on Even & Odd */ + vec0 = __lsx_vld(tmp_odd_buf, 0); + vec1 = __lsx_vld(tmp_odd_buf, 9 * 16); + vec2 = __lsx_vld(tmp_odd_buf, 14 * 16); + vec3 = __lsx_vld(tmp_odd_buf, 6 * 16); + loc0 = __lsx_vld(tmp_eve_buf, 0); + loc1 = __lsx_vld(tmp_eve_buf, 8 * 16); + loc2 = __lsx_vld(tmp_eve_buf, 4 * 16); + loc3 = __lsx_vld(tmp_eve_buf, 12 * 16); + + LSX_DUP4_ARG2(__lsx_vadd_h, + loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + m0, m4, m2, m6); + + #define SUB(a, b) __lsx_vsub_h(a, b) + + __lsx_vst(SUB(loc0, vec3), tmp_buf, 31 * 16); + __lsx_vst(SUB(loc1, vec2), tmp_buf, 23 * 16); + __lsx_vst(SUB(loc2, vec1), tmp_buf, 27 * 16); + __lsx_vst(SUB(loc3, vec0), tmp_buf, 19 * 16); + + /* Load 8 & Store 8 */ + vec0 = __lsx_vld(tmp_odd_buf, 4 * 16); + vec1 = __lsx_vld(tmp_odd_buf, 13 * 16); + vec2 = __lsx_vld(tmp_odd_buf, 10 * 16); + vec3 = __lsx_vld(tmp_odd_buf, 3 * 16); + loc0 = __lsx_vld(tmp_eve_buf, 2 * 16); + loc1 = __lsx_vld(tmp_eve_buf, 10 * 16); + loc2 = __lsx_vld(tmp_eve_buf, 6 * 16); + loc3 = __lsx_vld(tmp_eve_buf, 14 * 16); + + LSX_DUP4_ARG2(__lsx_vadd_h, + loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + m1, m5, m3, m7); + + __lsx_vst(SUB(loc0, vec3), tmp_buf, 29 * 16); + __lsx_vst(SUB(loc1, vec2), tmp_buf, 21 * 16); + __lsx_vst(SUB(loc2, vec1), tmp_buf, 25 * 16); + __lsx_vst(SUB(loc3, vec0), tmp_buf, 17 * 16); + + /* Load 8 & Store 8 */ + vec0 = __lsx_vld(tmp_odd_buf, 2 * 16); + vec1 = __lsx_vld(tmp_odd_buf, 11 * 16); + vec2 = __lsx_vld(tmp_odd_buf, 12 * 16); + vec3 = __lsx_vld(tmp_odd_buf, 7 * 16); + loc0 = __lsx_vld(tmp_eve_buf, 1 * 16); + loc1 = __lsx_vld(tmp_eve_buf, 9 * 16); + loc2 = __lsx_vld(tmp_eve_buf, 5 * 16); + loc3 = __lsx_vld(tmp_eve_buf, 13 * 16); + + LSX_DUP4_ARG2(__lsx_vadd_h, + loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + n0, n4, n2, n6); + + __lsx_vst(SUB(loc0, vec3), tmp_buf, 30 * 16); + __lsx_vst(SUB(loc1, vec2), tmp_buf, 22 * 16); + __lsx_vst(SUB(loc2, vec1), tmp_buf, 26 * 16); + __lsx_vst(SUB(loc3, vec0), tmp_buf, 18 * 16); + + /* Load 8 & Store 8 */ + vec0 = __lsx_vld(tmp_odd_buf, 5 * 16); + vec1 = __lsx_vld(tmp_odd_buf, 15 * 16); + vec2 = __lsx_vld(tmp_odd_buf, 8 * 16); + vec3 = __lsx_vld(tmp_odd_buf, 1 * 16); + loc0 = __lsx_vld(tmp_eve_buf, 3 * 16); + loc1 = __lsx_vld(tmp_eve_buf, 11 * 16); + loc2 = __lsx_vld(tmp_eve_buf, 7 * 16); + loc3 = __lsx_vld(tmp_eve_buf, 15 * 16); + + LSX_DUP4_ARG2(__lsx_vadd_h, + loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + n1, n5, n3, n7); + + __lsx_vst(SUB(loc0, vec3), tmp_buf, 28 * 16); + __lsx_vst(SUB(loc1, vec2), tmp_buf, 20 * 16); + __lsx_vst(SUB(loc2, vec1), tmp_buf, 24 * 16); + __lsx_vst(SUB(loc3, vec0), tmp_buf, 16 * 16); + + /* Transpose : 16 vectors */ + /* 1st & 2nd 8x8 */ + TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, + m0, n0, m1, n1, m2, n2, m3, n3); + __lsx_vst(m0, dst, 0); + __lsx_vst(n0, dst, 32 * 2); + __lsx_vst(m1, dst, 32 * 4); + __lsx_vst(n1, dst, 32 * 6); + __lsx_vst(m2, dst, 32 * 8); + __lsx_vst(n2, dst, 32 * 10); + __lsx_vst(m3, dst, 32 * 12); + __lsx_vst(n3, dst, 32 * 14); + + TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, + m4, n4, m5, n5, m6, n6, m7, n7); + + __lsx_vst(m4, dst, 16); + __lsx_vst(n4, dst, 16 + 32 * 2); + __lsx_vst(m5, dst, 16 + 32 * 4); + __lsx_vst(n5, dst, 16 + 32 * 6); + __lsx_vst(m6, dst, 16 + 32 * 8); + __lsx_vst(n6, dst, 16 + 32 * 10); + __lsx_vst(m7, dst, 16 + 32 * 12); + __lsx_vst(n7, dst, 16 + 32 * 14); + + /* 3rd & 4th 8x8 */ + LSX_DUP4_ARG2(__lsx_vld, + tmp_buf, 16 * 16, tmp_buf, 16 * 17, + tmp_buf, 16 * 18, tmp_buf, 16 * 19, + m0, n0, m1, n1); + LSX_DUP4_ARG2(__lsx_vld, + tmp_buf, 16 * 20, tmp_buf, 16 * 21, + tmp_buf, 16 * 22, tmp_buf, 16 * 23, + m2, n2, m3, n3); + + LSX_DUP4_ARG2(__lsx_vld, + tmp_buf, 16 * 24, tmp_buf, 16 * 25, + tmp_buf, 16 * 26, tmp_buf, 16 * 27, + m4, n4, m5, n5); + LSX_DUP4_ARG2(__lsx_vld, + tmp_buf, 16 * 28, tmp_buf, 16 * 29, + tmp_buf, 16 * 30, tmp_buf, 16 * 31, + m6, n6, m7, n7); + + TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, + m0, n0, m1, n1, m2, n2, m3, n3); + + __lsx_vst(m0, dst, 32); + __lsx_vst(n0, dst, 32 + 32 * 2); + __lsx_vst(m1, dst, 32 + 32 * 4); + __lsx_vst(n1, dst, 32 + 32 * 6); + __lsx_vst(m2, dst, 32 + 32 * 8); + __lsx_vst(n2, dst, 32 + 32 * 10); + __lsx_vst(m3, dst, 32 + 32 * 12); + __lsx_vst(n3, dst, 32 + 32 * 14); + + TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, + m4, n4, m5, n5, m6, n6, m7, n7); + + __lsx_vst(m4, dst, 48); + __lsx_vst(n4, dst, 48 + 32 * 2); + __lsx_vst(m5, dst, 48 + 32 * 4); + __lsx_vst(n5, dst, 48 + 32 * 6); + __lsx_vst(m6, dst, 48 + 32 * 8); + __lsx_vst(n6, dst, 48 + 32 * 10); + __lsx_vst(m7, dst, 48 + 32 * 12); + __lsx_vst(n7, dst, 48 + 32 * 14); +} + +static void vp9_idct8x32_column_even_process_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf) +{ + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + __m128i stp0, stp1, stp2, stp3, stp4, stp5, stp6, stp7; + __m128i zero = __lsx_vldi(0); + + /* Even stage 1 */ + LSX_DUP4_ARG2(__lsx_vld, + tmp_buf, 0, tmp_buf, 32 * 8, + tmp_buf, 32 * 16, tmp_buf, 32 * 24, + reg0, reg1, reg2, reg3); + LSX_DUP4_ARG2(__lsx_vld, + tmp_buf, 32 * 32, tmp_buf, 32 * 40, + tmp_buf, 32 * 48, tmp_buf, 32 * 56, + reg4, reg5, reg6, reg7); + + __lsx_vst(zero, tmp_buf, 0); + __lsx_vst(zero, tmp_buf, 32 * 8); + __lsx_vst(zero, tmp_buf, 32 * 16); + __lsx_vst(zero, tmp_buf, 32 * 24); + __lsx_vst(zero, tmp_buf, 32 * 32); + __lsx_vst(zero, tmp_buf, 32 * 40); + __lsx_vst(zero, tmp_buf, 32 * 48); + __lsx_vst(zero, tmp_buf, 32 * 56); + + tmp_buf += (2 * 32); + + VP9_DOTP_CONST_PAIR(reg1, reg7, cospi_28_64, cospi_4_64, reg1, reg7); + VP9_DOTP_CONST_PAIR(reg5, reg3, cospi_12_64, cospi_20_64, reg5, reg3); + BUTTERFLY_4_H(reg1, reg7, reg3, reg5, vec1, vec3, vec2, vec0); + VP9_DOTP_CONST_PAIR(vec2, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + + loc1 = vec3; + loc0 = vec1; + + VP9_DOTP_CONST_PAIR(reg0, reg4, cospi_16_64, cospi_16_64, reg0, reg4); + VP9_DOTP_CONST_PAIR(reg2, reg6, cospi_24_64, cospi_8_64, reg2, reg6); + BUTTERFLY_4_H(reg4, reg0, reg2, reg6, vec1, vec3, vec2, vec0); + BUTTERFLY_4_H(vec0, vec1, loc1, loc0, stp3, stp0, stp7, stp4); + BUTTERFLY_4_H(vec2, vec3, loc3, loc2, stp2, stp1, stp6, stp5); + + /* Even stage 2 */ + /* Load 8 */ + LSX_DUP4_ARG2(__lsx_vld, + tmp_buf, 0, tmp_buf, 32 * 8, + tmp_buf, 32 * 16, tmp_buf, 32 * 24, + reg0, reg1, reg2, reg3); + LSX_DUP4_ARG2(__lsx_vld, + tmp_buf, 32 * 32, tmp_buf, 32 * 40, + tmp_buf, 32 * 48, tmp_buf, 32 * 56, + reg4, reg5, reg6, reg7); + + __lsx_vst(zero, tmp_buf, 0); + __lsx_vst(zero, tmp_buf, 32 * 8); + __lsx_vst(zero, tmp_buf, 32 * 16); + __lsx_vst(zero, tmp_buf, 32 * 24); + __lsx_vst(zero, tmp_buf, 32 * 32); + __lsx_vst(zero, tmp_buf, 32 * 40); + __lsx_vst(zero, tmp_buf, 32 * 48); + __lsx_vst(zero, tmp_buf, 32 * 56); + + VP9_DOTP_CONST_PAIR(reg0, reg7, cospi_30_64, cospi_2_64, reg0, reg7); + VP9_DOTP_CONST_PAIR(reg4, reg3, cospi_14_64, cospi_18_64, reg4, reg3); + VP9_DOTP_CONST_PAIR(reg2, reg5, cospi_22_64, cospi_10_64, reg2, reg5); + VP9_DOTP_CONST_PAIR(reg6, reg1, cospi_6_64, cospi_26_64, reg6, reg1); + + vec0 = __lsx_vadd_h(reg0, reg4); + reg0 = __lsx_vsub_h(reg0, reg4); + reg4 = __lsx_vadd_h(reg6, reg2); + reg6 = __lsx_vsub_h(reg6, reg2); + reg2 = __lsx_vadd_h(reg1, reg5); + reg1 = __lsx_vsub_h(reg1, reg5); + reg5 = __lsx_vadd_h(reg7, reg3); + reg7 = __lsx_vsub_h(reg7, reg3); + reg3 = vec0; + + vec1 = reg2; + reg2 = __lsx_vadd_h(reg3, reg4); + reg3 = __lsx_vsub_h(reg3, reg4); + reg4 = __lsx_vsub_h(reg5, vec1); + reg5 = __lsx_vadd_h(reg5, vec1); + + VP9_DOTP_CONST_PAIR(reg7, reg0, cospi_24_64, cospi_8_64, reg0, reg7); + VP9_DOTP_CONST_PAIR(__lsx_vneg_h(reg6), reg1, cospi_24_64, cospi_8_64, + reg6, reg1); + + vec0 = __lsx_vsub_h(reg0, reg6); + reg0 = __lsx_vadd_h(reg0, reg6); + vec1 = __lsx_vsub_h(reg7, reg1); + reg7 = __lsx_vadd_h(reg7, reg1); + + VP9_DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, reg6, reg1); + VP9_DOTP_CONST_PAIR(reg4, reg3, cospi_16_64, cospi_16_64, reg3, reg4); + + /* Even stage 3 : Dependency on Even stage 1 & Even stage 2 */ + /* Store 8 */ + BUTTERFLY_4_H(stp0, stp1, reg7, reg5, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 0); + __lsx_vst(loc3, tmp_eve_buf, 16); + __lsx_vst(loc2, tmp_eve_buf, 14 * 16); + __lsx_vst(loc0, tmp_eve_buf, 14 * 16 + 16); + BUTTERFLY_4_H(stp2, stp3, reg4, reg1, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 2 * 16); + __lsx_vst(loc3, tmp_eve_buf, 2 * 16 + 16); + __lsx_vst(loc2, tmp_eve_buf, 12 * 16); + __lsx_vst(loc0, tmp_eve_buf, 12 * 16 + 16); + + /* Store 8 */ + BUTTERFLY_4_H(stp4, stp5, reg6, reg3, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 4 * 16); + __lsx_vst(loc3, tmp_eve_buf, 4 * 16 + 16); + __lsx_vst(loc2, tmp_eve_buf, 10 * 16); + __lsx_vst(loc0, tmp_eve_buf, 10 * 16 + 16); + + BUTTERFLY_4_H(stp6, stp7, reg2, reg0, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 6 * 16); + __lsx_vst(loc3, tmp_eve_buf, 6 * 16 + 16); + __lsx_vst(loc2, tmp_eve_buf, 8 * 16); + __lsx_vst(loc0, tmp_eve_buf, 8 * 16 + 16); +} + +static void vp9_idct8x32_column_odd_process_store(int16_t *tmp_buf, + int16_t *tmp_odd_buf) +{ + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + __m128i zero = __lsx_vldi(0); + + /* Odd stage 1 */ + reg0 = __lsx_vld(tmp_buf, 64); + reg1 = __lsx_vld(tmp_buf, 7 * 64); + reg2 = __lsx_vld(tmp_buf, 9 * 64); + reg3 = __lsx_vld(tmp_buf, 15 * 64); + reg4 = __lsx_vld(tmp_buf, 17 * 64); + reg5 = __lsx_vld(tmp_buf, 23 * 64); + reg6 = __lsx_vld(tmp_buf, 25 * 64); + reg7 = __lsx_vld(tmp_buf, 31 * 64); + + __lsx_vst(zero, tmp_buf, 64); + __lsx_vst(zero, tmp_buf, 7 * 64); + __lsx_vst(zero, tmp_buf, 9 * 64); + __lsx_vst(zero, tmp_buf, 15 * 64); + __lsx_vst(zero, tmp_buf, 17 * 64); + __lsx_vst(zero, tmp_buf, 23 * 64); + __lsx_vst(zero, tmp_buf, 25 * 64); + __lsx_vst(zero, tmp_buf, 31 * 64); + + VP9_DOTP_CONST_PAIR(reg0, reg7, cospi_31_64, cospi_1_64, reg0, reg7); + VP9_DOTP_CONST_PAIR(reg4, reg3, cospi_15_64, cospi_17_64, reg3, reg4); + VP9_DOTP_CONST_PAIR(reg2, reg5, cospi_23_64, cospi_9_64, reg2, reg5); + VP9_DOTP_CONST_PAIR(reg6, reg1, cospi_7_64, cospi_25_64, reg1, reg6); + + vec0 = __lsx_vadd_h(reg0, reg3); + reg0 = __lsx_vsub_h(reg0, reg3); + reg3 = __lsx_vadd_h(reg7, reg4); + reg7 = __lsx_vsub_h(reg7, reg4); + reg4 = __lsx_vadd_h(reg1, reg2); + reg1 = __lsx_vsub_h(reg1, reg2); + reg2 = __lsx_vadd_h(reg6, reg5); + reg6 = __lsx_vsub_h(reg6, reg5); + reg5 = vec0; + + /* 4 Stores */ + LSX_DUP2_ARG2(__lsx_vadd_h, reg5, reg4, reg3, reg2, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 4 * 16); + __lsx_vst(vec1, tmp_odd_buf, 4 * 16 + 16); + LSX_DUP2_ARG2(__lsx_vsub_h, reg5, reg4, reg3, reg2, vec0, vec1); + VP9_DOTP_CONST_PAIR(vec1, vec0, cospi_24_64, cospi_8_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 0); + __lsx_vst(vec1, tmp_odd_buf, 16); + + /* 4 Stores */ + VP9_DOTP_CONST_PAIR(reg7, reg0, cospi_28_64, cospi_4_64, reg0, reg7); + VP9_DOTP_CONST_PAIR(reg6, reg1, -cospi_4_64, cospi_28_64, reg1, reg6); + BUTTERFLY_4_H(reg0, reg7, reg6, reg1, vec0, vec1, vec2, vec3); + __lsx_vst(vec0, tmp_odd_buf, 6 * 16); + __lsx_vst(vec1, tmp_odd_buf, 6 * 16 + 16); + VP9_DOTP_CONST_PAIR(vec2, vec3, cospi_24_64, cospi_8_64, vec2, vec3); + __lsx_vst(vec2, tmp_odd_buf, 2 * 16); + __lsx_vst(vec3, tmp_odd_buf, 2 * 16 + 16); + + /* Odd stage 2 */ + /* 8 loads */ + reg0 = __lsx_vld(tmp_buf, 3 * 64); + reg1 = __lsx_vld(tmp_buf, 5 * 64); + reg2 = __lsx_vld(tmp_buf, 11 * 64); + reg3 = __lsx_vld(tmp_buf, 13 * 64); + reg4 = __lsx_vld(tmp_buf, 19 * 64); + reg5 = __lsx_vld(tmp_buf, 21 * 64); + reg6 = __lsx_vld(tmp_buf, 27 * 64); + reg7 = __lsx_vld(tmp_buf, 29 * 64); + + __lsx_vst(zero, tmp_buf, 3 * 64); + __lsx_vst(zero, tmp_buf, 5 * 64); + __lsx_vst(zero, tmp_buf, 11 * 64); + __lsx_vst(zero, tmp_buf, 13 * 64); + __lsx_vst(zero, tmp_buf, 19 * 64); + __lsx_vst(zero, tmp_buf, 21 * 64); + __lsx_vst(zero, tmp_buf, 27 * 64); + __lsx_vst(zero, tmp_buf, 29 * 64); + + VP9_DOTP_CONST_PAIR(reg1, reg6, cospi_27_64, cospi_5_64, reg1, reg6); + VP9_DOTP_CONST_PAIR(reg5, reg2, cospi_11_64, cospi_21_64, reg2, reg5); + VP9_DOTP_CONST_PAIR(reg3, reg4, cospi_19_64, cospi_13_64, reg3, reg4); + VP9_DOTP_CONST_PAIR(reg7, reg0, cospi_3_64, cospi_29_64, reg0, reg7); + + /* 4 Stores */ + LSX_DUP4_ARG2(__lsx_vsub_h,reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, + vec0, vec1, vec2, vec3); + VP9_DOTP_CONST_PAIR(vec1, vec0, cospi_12_64, cospi_20_64, loc0, loc1); + VP9_DOTP_CONST_PAIR(vec3, vec2, -cospi_20_64, cospi_12_64, loc2, loc3); + BUTTERFLY_4_H(loc2, loc3, loc1, loc0, vec0, vec1, vec3, vec2); + __lsx_vst(vec0, tmp_odd_buf, 12 * 16); + __lsx_vst(vec1, tmp_odd_buf, 12 * 16 + 3 * 16); + VP9_DOTP_CONST_PAIR(vec3, vec2, -cospi_8_64, cospi_24_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 10 * 16); + __lsx_vst(vec1, tmp_odd_buf, 10 * 16 + 16); + + /* 4 Stores */ + LSX_DUP4_ARG2(__lsx_vadd_h, reg0, reg3, reg1, reg2, reg5, reg6, reg4, reg7, + vec0, vec1, vec2, vec3); + BUTTERFLY_4_H(vec0, vec3, vec2, vec1, reg0, reg1, reg3, reg2); + __lsx_vst(reg0, tmp_odd_buf, 13 * 16); + __lsx_vst(reg1, tmp_odd_buf, 13 * 16 + 16); + VP9_DOTP_CONST_PAIR(reg3, reg2, -cospi_8_64, cospi_24_64, + reg0, reg1); + __lsx_vst(reg0, tmp_odd_buf, 8 * 16); + __lsx_vst(reg1, tmp_odd_buf, 8 * 16 + 16); + + /* Odd stage 3 : Dependency on Odd stage 1 & Odd stage 2 */ + /* Load 8 & Store 8 */ + LSX_DUP4_ARG2(__lsx_vld, + tmp_odd_buf, 0, tmp_odd_buf, 16, + tmp_odd_buf, 32, tmp_odd_buf, 48, + reg0, reg1, reg2, reg3); + LSX_DUP4_ARG2(__lsx_vld, + tmp_odd_buf, 8 * 16, tmp_odd_buf, 8 * 16 + 16, + tmp_odd_buf, 8 * 16 + 32, tmp_odd_buf, 8 * 16 + 48, + reg4, reg5, reg6, reg7); + + LSX_DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, + loc0, loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 0); + __lsx_vst(loc1, tmp_odd_buf, 16); + __lsx_vst(loc2, tmp_odd_buf, 32); + __lsx_vst(loc3, tmp_odd_buf, 48); + LSX_DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg1, reg5, vec0, vec1); + VP9_DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + + LSX_DUP2_ARG2(__lsx_vsub_h, reg2, reg6, reg3, reg7, vec0, vec1); + VP9_DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 8 * 16); + __lsx_vst(loc1, tmp_odd_buf, 8 * 16 + 16); + __lsx_vst(loc2, tmp_odd_buf, 8 * 16 + 32); + __lsx_vst(loc3, tmp_odd_buf, 8 * 16 + 48); + + /* Load 8 & Store 8 */ + LSX_DUP4_ARG2(__lsx_vld, + tmp_odd_buf, 4 * 16, tmp_odd_buf, 4 * 16 + 16, + tmp_odd_buf, 4 * 16 + 32, tmp_odd_buf, 4 * 16 + 48, + reg1, reg2, reg0, reg3); + LSX_DUP4_ARG2(__lsx_vld, + tmp_odd_buf, 12 * 16, tmp_odd_buf, 12 * 16 + 16, + tmp_odd_buf, 12 * 16 + 32, tmp_odd_buf, 12 * 16 + 48, + reg4, reg5, reg6, reg7); + + LSX_DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, + loc0, loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 4 * 16); + __lsx_vst(loc1, tmp_odd_buf, 4 * 16 + 16); + __lsx_vst(loc2, tmp_odd_buf, 4 * 16 + 32); + __lsx_vst(loc3, tmp_odd_buf, 4 * 16 + 48); + + LSX_DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg3, reg7, vec0, vec1); + VP9_DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + + LSX_DUP2_ARG2(__lsx_vsub_h, reg1, reg5, reg2, reg6, vec0, vec1); + VP9_DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 12 * 16); + __lsx_vst(loc1, tmp_odd_buf, 12 * 16 + 16); + __lsx_vst(loc2, tmp_odd_buf, 12 * 16 + 32); + __lsx_vst(loc3, tmp_odd_buf, 12 * 16 + 48); +} + +static void vp9_idct8x32_column_butterfly_addblk(int16_t *tmp_eve_buf, + int16_t *tmp_odd_buf, + uint8_t *dst, + int32_t dst_stride) +{ + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i m0, m1, m2, m3, m4, m5, m6, m7, n0, n1, n2, n3, n4, n5, n6, n7; + + /* FINAL BUTTERFLY : Dependency on Even & Odd */ + vec0 = __lsx_vld(tmp_odd_buf, 0); + vec1 = __lsx_vld(tmp_odd_buf, 9 * 16); + vec2 = __lsx_vld(tmp_odd_buf, 14 * 16); + vec3 = __lsx_vld(tmp_odd_buf, 6 * 16); + loc0 = __lsx_vld(tmp_eve_buf, 0); + loc1 = __lsx_vld(tmp_eve_buf, 8 * 16); + loc2 = __lsx_vld(tmp_eve_buf, 4 * 16); + loc3 = __lsx_vld(tmp_eve_buf, 12 * 16); + + LSX_DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + m0, m4, m2, m6); + LSX_DUP4_ARG2(__lsx_vsrari_h, m0, 6, m2, 6, m4, 6, m6, 6, m0, m2, m4, m6); + VP9_ADDBLK_ST8x4_UB(dst, (4 * dst_stride), m0, m2, m4, m6); + + LSX_DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + m6, m2, m4, m0); + LSX_DUP4_ARG2(__lsx_vsrari_h, m0, 6, m2, 6, m4, 6, m6, 6, m0, m2, m4, m6); + VP9_ADDBLK_ST8x4_UB((dst + 19 * dst_stride), (4 * dst_stride), + m0, m2, m4, m6); + + /* Load 8 & Store 8 */ + vec0 = __lsx_vld(tmp_odd_buf, 4 * 16); + vec1 = __lsx_vld(tmp_odd_buf, 13 * 16); + vec2 = __lsx_vld(tmp_odd_buf, 10 * 16); + vec3 = __lsx_vld(tmp_odd_buf, 3 * 16); + loc0 = __lsx_vld(tmp_eve_buf, 2 * 16); + loc1 = __lsx_vld(tmp_eve_buf, 10 * 16); + loc2 = __lsx_vld(tmp_eve_buf, 6 * 16); + loc3 = __lsx_vld(tmp_eve_buf, 14 * 16); + + LSX_DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + m1, m5, m3, m7); + LSX_DUP4_ARG2(__lsx_vsrari_h, m1, 6, m3, 6, m5, 6, m7, 6, m1, m3, m5, m7); + VP9_ADDBLK_ST8x4_UB((dst + 2 * dst_stride), (4 * dst_stride), + m1, m3, m5, m7); + + LSX_DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + m7, m3, m5, m1); + LSX_DUP4_ARG2(__lsx_vsrari_h, m1, 6, m3, 6, m5, 6, m7, 6, m1, m3, m5, m7); + VP9_ADDBLK_ST8x4_UB((dst + 17 * dst_stride), (4 * dst_stride), + m1, m3, m5, m7); + + /* Load 8 & Store 8 */ + vec0 = __lsx_vld(tmp_odd_buf, 2 * 16); + vec1 = __lsx_vld(tmp_odd_buf, 11 * 16); + vec2 = __lsx_vld(tmp_odd_buf, 12 * 16); + vec3 = __lsx_vld(tmp_odd_buf, 7 * 16); + loc0 = __lsx_vld(tmp_eve_buf, 1 * 16); + loc1 = __lsx_vld(tmp_eve_buf, 9 * 16); + loc2 = __lsx_vld(tmp_eve_buf, 5 * 16); + loc3 = __lsx_vld(tmp_eve_buf, 13 * 16); + + LSX_DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + n0, n4, n2, n6); + LSX_DUP4_ARG2(__lsx_vsrari_h, n0, 6, n2, 6, n4, 6, n6, 6, n0, n2, n4, n6); + VP9_ADDBLK_ST8x4_UB((dst + 1 * dst_stride), (4 * dst_stride), + n0, n2, n4, n6); + LSX_DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + n6, n2, n4, n0); + LSX_DUP4_ARG2(__lsx_vsrari_h, n0, 6, n2, 6, n4, 6, n6, 6, n0, n2, n4, n6); + VP9_ADDBLK_ST8x4_UB((dst + 18 * dst_stride), (4 * dst_stride), + n0, n2, n4, n6); + + /* Load 8 & Store 8 */ + vec0 = __lsx_vld(tmp_odd_buf, 5 * 16); + vec1 = __lsx_vld(tmp_odd_buf, 15 * 16); + vec2 = __lsx_vld(tmp_odd_buf, 8 * 16); + vec3 = __lsx_vld(tmp_odd_buf, 1 * 16); + loc0 = __lsx_vld(tmp_eve_buf, 3 * 16); + loc1 = __lsx_vld(tmp_eve_buf, 11 * 16); + loc2 = __lsx_vld(tmp_eve_buf, 7 * 16); + loc3 = __lsx_vld(tmp_eve_buf, 15 * 16); + + LSX_DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + n1, n5, n3, n7); + LSX_DUP4_ARG2(__lsx_vsrari_h, n1, 6, n3, 6, n5, 6, n7, 6, n1, n3, n5, n7); + VP9_ADDBLK_ST8x4_UB((dst + 3 * dst_stride), (4 * dst_stride), + n1, n3, n5, n7); + LSX_DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, + n7, n3, n5, n1); + LSX_DUP4_ARG2(__lsx_vsrari_h, n1, 6, n3, 6, n5, 6, n7, 6, n1, n3, n5, n7); + VP9_ADDBLK_ST8x4_UB((dst + 16 * dst_stride), (4 * dst_stride), + n1, n3, n5, n7); +} + +static void vp9_idct8x32_1d_columns_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + int16_t tmp_odd_buf[16 * 8] ALLOC_ALIGNED(16); + int16_t tmp_eve_buf[16 * 8] ALLOC_ALIGNED(16); + + vp9_idct8x32_column_even_process_store(input, &tmp_eve_buf[0]); + vp9_idct8x32_column_odd_process_store(input, &tmp_odd_buf[0]); + vp9_idct8x32_column_butterfly_addblk(&tmp_eve_buf[0], &tmp_odd_buf[0], + dst, dst_stride); +} + +static void vp9_idct8x32_1d_columns_lsx(int16_t *input, int16_t *output, + int16_t *tmp_buf) +{ + int16_t tmp_odd_buf[16 * 8] ALLOC_ALIGNED(16); + int16_t tmp_eve_buf[16 * 8] ALLOC_ALIGNED(16); + + vp9_idct8x32_column_even_process_store(input, &tmp_eve_buf[0]); + vp9_idct8x32_column_odd_process_store(input, &tmp_odd_buf[0]); + vp9_idct_butterfly_transpose_store(tmp_buf, &tmp_eve_buf[0], + &tmp_odd_buf[0], output); +} + +static void vp9_idct32x32_1_add_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + int32_t i; + int16_t out; + __m128i zero = __lsx_vldi(0); + __m128i dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3; + __m128i res0, res1, res2, res3, res4, res5, res6, res7, vec; + + out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), VP9_DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO((out * cospi_16_64), VP9_DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO(out, 6); + input[0] = 0; + + vec = __lsx_vreplgr2vr_h(out); + + for (i = 16; i--;) { + LSX_DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + LSX_DUP2_ARG2(__lsx_vld, dst + dst_stride, 0, + dst + dst_stride, 16, dst2, dst3); + + LSX_DUP4_ARG2(__lsx_vilvl_b, + zero, dst0, zero, dst1, zero, dst2, zero, dst3, + res0, res1, res2, res3); + LSX_DUP4_ARG2(__lsx_vilvh_b, + zero, dst0, zero, dst1, zero, dst2, zero, dst3, + res4, res5, res6, res7); + LSX_DUP4_ARG2(__lsx_vadd_h, + res0, vec, res1, vec, res2, vec, res3, vec, + res0, res1, res2, res3); + LSX_DUP4_ARG2(__lsx_vadd_h, + res4, vec, res5, vec, res6, vec, res7, vec, + res4, res5, res6, res7); + LSX_DUP4_ARG1(__lsx_clamp255_h, + res0, res1, res2, res3, + res0, res1, res2, res3); + LSX_DUP4_ARG1(__lsx_clamp255_h, + res4, res5, res6, res7, + res4, res5, res6, res7); + LSX_DUP4_ARG2(__lsx_vpickev_b, + res4, res0, res5, res1, res6, res2, res7, res3, + tmp0, tmp1, tmp2, tmp3); + + __lsx_vst(tmp0, dst, 0); + __lsx_vst(tmp1, dst, 16); + dst += dst_stride; + __lsx_vst(tmp2, dst, 0); + __lsx_vst(tmp3, dst, 16); + dst += dst_stride; + } +} + +static void vp9_idct32x32_34_colcol_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + int32_t i; + int16_t out_arr[32 * 32] ALLOC_ALIGNED(16); + int16_t *out_ptr = out_arr; + int16_t tmp_buf[8 * 32] ALLOC_ALIGNED(16); + __m128i zero = __lsx_vldi(0); + + for (i = 16; i--;) { + __lsx_vst(zero, out_ptr, 0); + __lsx_vst(zero, out_ptr, 16); + __lsx_vst(zero, out_ptr, 32); + __lsx_vst(zero, out_ptr, 48); + __lsx_vst(zero, out_ptr, 64); + __lsx_vst(zero, out_ptr, 80); + __lsx_vst(zero, out_ptr, 96); + __lsx_vst(zero, out_ptr, 112); + out_ptr += 64; + } + + out_ptr = out_arr; + + /* process 8*32 block */ + vp9_idct8x32_1d_columns_lsx(input, out_ptr, &tmp_buf[0]); + + /* transform columns */ + for (i = 0; i < 4; i++) { + /* process 8*32 block */ + vp9_idct8x32_1d_columns_addblk_lsx((out_ptr + (i << 3)), + (dst + (i << 3)), dst_stride); + } +} + +static void vp9_idct32x32_colcol_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) +{ + int32_t i; + int16_t out_arr[32 * 32] ALLOC_ALIGNED(16); + int16_t *out_ptr = out_arr; + int16_t tmp_buf[8 * 32] ALLOC_ALIGNED(16); + + /* transform rows */ + for (i = 0; i < 4; i++) { + /* process 8*32 block */ + vp9_idct8x32_1d_columns_lsx((input + (i << 3)), (out_ptr + (i << 8)), + &tmp_buf[0]); + } + + /* transform columns */ + for (i = 0; i < 4; i++) { + /* process 8*32 block */ + vp9_idct8x32_1d_columns_addblk_lsx((out_ptr + (i << 3)), + (dst + (i << 3)), dst_stride); + } +} + +void ff_idct_idct_8x8_add_lsx(uint8_t *dst, ptrdiff_t stride, + int16_t *block, int eob) +{ + if (eob == 1) { + vp9_idct8x8_1_add_lsx(block, dst, stride); + } + else if (eob <= 12) { + vp9_idct8x8_12_colcol_addblk_lsx(block, dst, stride); + } + else { + vp9_idct8x8_colcol_addblk_lsx(block, dst, stride); + } +} + +void ff_idct_idct_16x16_add_lsx(uint8_t *dst, ptrdiff_t stride, + int16_t *block, int eob) +{ + if (eob == 1) { + /* DC only DCT coefficient. */ + vp9_idct16x16_1_add_lsx(block, dst, stride); + } + else if (eob <= 10) { + vp9_idct16x16_10_colcol_addblk_lsx(block, dst, stride); + } + else { + vp9_idct16x16_colcol_addblk_lsx(block, dst, stride); + } +} + +void ff_idct_idct_32x32_add_lsx(uint8_t *dst, ptrdiff_t stride, + int16_t *block, int eob) +{ + if (eob == 1) { + vp9_idct32x32_1_add_lsx(block, dst, stride); + } + else if (eob <= 34) { + vp9_idct32x32_34_colcol_addblk_lsx(block, dst, stride); + } + else { + vp9_idct32x32_colcol_addblk_lsx(block, dst, stride); + } +} + diff --git a/libavcodec/loongarch/vp9_intra_lsx.c b/libavcodec/loongarch/vp9_intra_lsx.c new file mode 100644 index 0000000000..6adb3f9e3c --- /dev/null +++ b/libavcodec/loongarch/vp9_intra_lsx.c @@ -0,0 +1,612 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavcodec/vp9dsp.h" +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "vp9dsp_loongarch.h" + +#define LSX_ST_8(_dst0, _dst1, _dst2, _dst3, _dst4, \ + _dst5, _dst6, _dst7, _dst, _stride) \ +{ \ + __lsx_vst(_dst0, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst1, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst2, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst3, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst4, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst5, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst6, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst7, _dst, 0); \ + _dst += _stride; \ +} + +#define LSX_ST_8X16(_dst0, _dst1, _dst2, _dst3, _dst4, \ + _dst5, _dst6, _dst7, _dst, _stride) \ +{ \ + __lsx_vst(_dst0, _dst, 0); \ + __lsx_vst(_dst0, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst1, _dst, 0); \ + __lsx_vst(_dst1, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst2, _dst, 0); \ + __lsx_vst(_dst2, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst3, _dst, 0); \ + __lsx_vst(_dst3, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst4, _dst, 0); \ + __lsx_vst(_dst4, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst5, _dst, 0); \ + __lsx_vst(_dst5, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst6, _dst, 0); \ + __lsx_vst(_dst6, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst7, _dst, 0); \ + __lsx_vst(_dst7, _dst, 16); \ + _dst += _stride; \ +} + +void ff_vert_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *left, + const uint8_t *src) +{ + __m128i src0; + + src0 = __lsx_vld(src, 0); + LSX_ST_8(src0, src0, src0, src0, src0, src0, src0, src0, dst, dst_stride); + LSX_ST_8(src0, src0, src0, src0, src0, src0, src0, src0, dst, dst_stride); +} + +void ff_vert_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *left, + const uint8_t *src) +{ + uint32_t row; + __m128i src0, src1; + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + for (row = 32; row--;) { + __lsx_vst(src0, dst, 0); + __lsx_vst(src1, dst, 16); + dst += dst_stride; + } +} + +void ff_hor_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, + const uint8_t *top) +{ + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11, src12, src13, src14, src15; + + src15 = __lsx_vldrepl_b(src, 0); + src14 = __lsx_vldrepl_b(src, 1); + src13 = __lsx_vldrepl_b(src, 2); + src12 = __lsx_vldrepl_b(src, 3); + src11 = __lsx_vldrepl_b(src, 4); + src10 = __lsx_vldrepl_b(src, 5); + src9 = __lsx_vldrepl_b(src, 6); + src8 = __lsx_vldrepl_b(src, 7); + src7 = __lsx_vldrepl_b(src, 8); + src6 = __lsx_vldrepl_b(src, 9); + src5 = __lsx_vldrepl_b(src, 10); + src4 = __lsx_vldrepl_b(src, 11); + src3 = __lsx_vldrepl_b(src, 12); + src2 = __lsx_vldrepl_b(src, 13); + src1 = __lsx_vldrepl_b(src, 14); + src0 = __lsx_vldrepl_b(src, 15); + LSX_ST_8(src0, src1, src2, src3, src4, src5, src6, src7, dst, dst_stride); + LSX_ST_8(src8, src9, src10, src11, src12, src13, src14, src15, dst, dst_stride); +} + +void ff_hor_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, + const uint8_t *top) +{ + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11, src12, src13, src14, src15; + __m128i src16, src17, src18, src19, src20, src21, src22, src23; + __m128i src24, src25, src26, src27, src28, src29, src30, src31; + + src31 = __lsx_vldrepl_b(src, 0); + src30 = __lsx_vldrepl_b(src, 1); + src29 = __lsx_vldrepl_b(src, 2); + src28 = __lsx_vldrepl_b(src, 3); + src27 = __lsx_vldrepl_b(src, 4); + src26 = __lsx_vldrepl_b(src, 5); + src25 = __lsx_vldrepl_b(src, 6); + src24 = __lsx_vldrepl_b(src, 7); + src23 = __lsx_vldrepl_b(src, 8); + src22 = __lsx_vldrepl_b(src, 9); + src21 = __lsx_vldrepl_b(src, 10); + src20 = __lsx_vldrepl_b(src, 11); + src19 = __lsx_vldrepl_b(src, 12); + src18 = __lsx_vldrepl_b(src, 13); + src17 = __lsx_vldrepl_b(src, 14); + src16 = __lsx_vldrepl_b(src, 15); + src15 = __lsx_vldrepl_b(src, 16); + src14 = __lsx_vldrepl_b(src, 17); + src13 = __lsx_vldrepl_b(src, 18); + src12 = __lsx_vldrepl_b(src, 19); + src11 = __lsx_vldrepl_b(src, 20); + src10 = __lsx_vldrepl_b(src, 21); + src9 = __lsx_vldrepl_b(src, 22); + src8 = __lsx_vldrepl_b(src, 23); + src7 = __lsx_vldrepl_b(src, 24); + src6 = __lsx_vldrepl_b(src, 25); + src5 = __lsx_vldrepl_b(src, 26); + src4 = __lsx_vldrepl_b(src, 27); + src3 = __lsx_vldrepl_b(src, 28); + src2 = __lsx_vldrepl_b(src, 29); + src1 = __lsx_vldrepl_b(src, 30); + src0 = __lsx_vldrepl_b(src, 31); + LSX_ST_8X16(src0, src1, src2, src3, src4, src5, src6, src7, dst, dst_stride); + LSX_ST_8X16(src8, src9, src10, src11, src12, src13, src14, src15, dst, dst_stride); + LSX_ST_8X16(src16, src17, src18, src19, src20, src21, src22, src23, dst, dst_stride); + LSX_ST_8X16(src24, src25, src26, src27, src28, src29, src30, src31, dst, dst_stride); +} + +void ff_dc_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, + const uint8_t *src_top) +{ + __m128i tmp0, tmp1, dst0; + + tmp0 = __lsx_vldrepl_w(src_top, 0); + tmp1 = __lsx_vldrepl_w(src_left, 0); + dst0 = __lsx_vilvl_w(tmp1, tmp0); + dst0 = __lsx_vhaddw_hu_bu(dst0, dst0); + dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); + dst0 = __lsx_vhaddw_du_wu(dst0, dst0); + dst0 = __lsx_vsrari_w(dst0, 3); + dst0 = __lsx_vshuf4i_b(dst0, 0); + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 0); +} + +#define INTRA_DC_TL_4X4(dir) \ +void ff_dc_##dir##_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ + const uint8_t *left, \ + const uint8_t *top) \ +{ \ + __m128i tmp0, dst0; \ + \ + tmp0 = __lsx_vldrepl_w(dir, 0); \ + dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0); \ + dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ + dst0 = __lsx_vsrari_w(dst0, 2); \ + dst0 = __lsx_vshuf4i_b(dst0, 0); \ + __lsx_vstelm_w(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_w(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_w(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_w(dst0, dst, 0, 0); \ +} +INTRA_DC_TL_4X4(top); +INTRA_DC_TL_4X4(left); + +void ff_dc_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, + const uint8_t *src_top) +{ + __m128i tmp0, tmp1, dst0; + + tmp0 = __lsx_vldrepl_d(src_top, 0); + tmp1 = __lsx_vldrepl_d(src_left, 0); + dst0 = __lsx_vilvl_d(tmp1, tmp0); + dst0 = __lsx_vhaddw_hu_bu(dst0, dst0); + dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); + dst0 = __lsx_vhaddw_du_wu(dst0, dst0); + dst0 = __lsx_vhaddw_qu_du(dst0, dst0); + dst0 = __lsx_vsrari_w(dst0, 4); + dst0 = __lsx_vreplvei_b(dst0, 0); + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 0); +} + +#define INTRA_DC_TL_8X8(dir) \ +void ff_dc_##dir##_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ + const uint8_t *left, \ + const uint8_t *top) \ +{ \ + __m128i tmp0, dst0; \ + \ + tmp0 = __lsx_vldrepl_d(dir, 0); \ + dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0); \ + dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ + dst0 = __lsx_vhaddw_du_wu(dst0, dst0); \ + dst0 = __lsx_vsrari_w(dst0, 3); \ + dst0 = __lsx_vreplvei_b(dst0, 0); \ + __lsx_vstelm_d(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_d(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_d(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_d(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_d(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_d(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_d(dst0, dst, 0, 0); \ + dst += dst_stride; \ + __lsx_vstelm_d(dst0, dst, 0, 0); \ +} + +INTRA_DC_TL_8X8(top); +INTRA_DC_TL_8X8(left); + +void ff_dc_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, + const uint8_t *src_left, const uint8_t *src_top) +{ + __m128i tmp0, tmp1, dst0; + + tmp0 = __lsx_vld(src_top, 0); + tmp1 = __lsx_vld(src_left, 0); + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp0, tmp1); + dst0 = __lsx_vadd_h(tmp0, tmp1); + dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); + dst0 = __lsx_vhaddw_du_wu(dst0, dst0); + dst0 = __lsx_vhaddw_qu_du(dst0, dst0); + dst0 = __lsx_vsrari_w(dst0, 5); + dst0 = __lsx_vreplvei_b(dst0, 0); + LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); + LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); +} + +#define INTRA_DC_TL_16X16(dir) \ +void ff_dc_##dir##_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ + const uint8_t *left, \ + const uint8_t *top) \ +{ \ + __m128i tmp0, dst0; \ + \ + tmp0 = __lsx_vld(dir, 0); \ + dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0); \ + dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ + dst0 = __lsx_vhaddw_du_wu(dst0, dst0); \ + dst0 = __lsx_vhaddw_qu_du(dst0, dst0); \ + dst0 = __lsx_vsrari_w(dst0, 4); \ + dst0 = __lsx_vreplvei_b(dst0, 0); \ + LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride);\ + LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride);\ +} + +INTRA_DC_TL_16X16(top); +INTRA_DC_TL_16X16(left); + +void ff_dc_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, + const uint8_t *src_left, const uint8_t *src_top) +{ + __m128i tmp0, tmp1, tmp2, tmp3, dst0; + + LSX_DUP2_ARG2(__lsx_vld, src_top, 0, src_top, 16, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vld, src_left, 0, src_left, 16, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp2, tmp2, + tmp3, tmp3, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vadd_h, tmp0, tmp1, tmp2, tmp3, tmp0, tmp1); + dst0 = __lsx_vadd_h(tmp0, tmp1); + dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); + dst0 = __lsx_vhaddw_du_wu(dst0, dst0); + dst0 = __lsx_vhaddw_qu_du(dst0, dst0); + dst0 = __lsx_vsrari_w(dst0, 6); + dst0 = __lsx_vreplvei_b(dst0, 0); + LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); + LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); + LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); + LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); +} + +#define INTRA_DC_TL_32X32(dir) \ +void ff_dc_##dir##_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ + const uint8_t *left, \ + const uint8_t *top) \ +{ \ + __m128i tmp0, tmp1, dst0; \ + \ + LSX_DUP2_ARG2(__lsx_vld, dir, 0, dir, 16, tmp0, tmp1); \ + LSX_DUP2_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp0, tmp1); \ + dst0 = __lsx_vadd_h(tmp0, tmp1); \ + dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ + dst0 = __lsx_vhaddw_du_wu(dst0, dst0); \ + dst0 = __lsx_vhaddw_qu_du(dst0, dst0); \ + dst0 = __lsx_vsrari_w(dst0, 5); \ + dst0 = __lsx_vreplvei_b(dst0, 0); \ + LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride);\ + LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride);\ + LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride);\ + LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride);\ +} + +INTRA_DC_TL_32X32(top); +INTRA_DC_TL_32X32(left); + +#define INTRA_PREDICT_VALDC_16X16_LSX(val) \ +void ff_dc_##val##_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ + const uint8_t *left, const uint8_t *top) \ +{ \ + __m128i out = __lsx_vldi(val); \ + \ + LSX_ST_8(out, out, out, out, out, out, out, out, dst, dst_stride); \ + LSX_ST_8(out, out, out, out, out, out, out, out, dst, dst_stride); \ +} + +INTRA_PREDICT_VALDC_16X16_LSX(127); +INTRA_PREDICT_VALDC_16X16_LSX(128); +INTRA_PREDICT_VALDC_16X16_LSX(129); + +#define INTRA_PREDICT_VALDC_32X32_LSX(val) \ +void ff_dc_##val##_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ + const uint8_t *left, const uint8_t *top) \ +{ \ + __m128i out = __lsx_vldi(val); \ + \ + LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ + LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ + LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ + LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ +} + +INTRA_PREDICT_VALDC_32X32_LSX(127); +INTRA_PREDICT_VALDC_32X32_LSX(128); +INTRA_PREDICT_VALDC_32X32_LSX(129); + +void ff_tm_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, + const uint8_t *src_left, const uint8_t *src_top_ptr) +{ + uint8_t top_left = src_top_ptr[-1]; + __m128i tmp0, tmp1, tmp2, tmp3, reg0, reg1; + __m128i src0, src1, src2, src3; + __m128i dst0, dst1, dst2, dst3; + + reg0 = __lsx_vreplgr2vr_h(top_left); + reg1 = __lsx_vld(src_top_ptr, 0); + LSX_DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, + 3, tmp3, tmp2, tmp1, tmp0); + LSX_DUP4_ARG2(__lsx_vilvl_b, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vhaddw_hu_bu, src0, src0, src1, src1, src2, src2, src3, + src3, dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vssub_hu, dst0, reg0, dst1, reg0, dst2, reg0, dst3, reg0, + dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vsat_hu, dst0, 7, dst1, 7, dst2, 7, dst3, 7, + dst0, dst1, dst2, dst3); + LSX_DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1); + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 2); +} + +void ff_tm_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, + const uint8_t *src_left, const uint8_t *src_top_ptr) +{ + uint8_t top_left = src_top_ptr[-1]; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i reg0, reg1; + + reg0 = __lsx_vreplgr2vr_h(top_left); + reg1 = __lsx_vld(src_top_ptr, 0); + LSX_DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, + 3, tmp7, tmp6, tmp5, tmp4); + LSX_DUP4_ARG2(__lsx_vldrepl_b, src_left, 4, src_left, 5, src_left, 6, src_left, + 7, tmp3, tmp2, tmp1, tmp0); + LSX_DUP4_ARG2(__lsx_vilvl_b, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vilvl_b, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, reg1, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vhaddw_hu_bu, src0, src0, src1, src1, src2, src2, src3, + src3, src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vhaddw_hu_bu, src4, src4, src5, src5, src6, src6, src7, + src7, src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vpickev_b, src1, src0, src3, src2, src5, src4, src7, src6, + src0, src1, src2, src3); + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 1); +} + +void ff_tm_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, + const uint8_t *src_left, const uint8_t *src_top_ptr) +{ + uint8_t top_left = src_top_ptr[-1]; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i reg0, reg1; + + reg0 = __lsx_vreplgr2vr_h(top_left); + reg1 = __lsx_vld(src_top_ptr, 0); + LSX_DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, + 3, tmp15, tmp14, tmp13, tmp12); + LSX_DUP4_ARG2(__lsx_vldrepl_b, src_left, 4, src_left, 5, src_left, 6, src_left, + 7, tmp11, tmp10, tmp9, tmp8); + LSX_DUP4_ARG2(__lsx_vldrepl_b, src_left, 8, src_left, 9, src_left, 10, src_left, + 11, tmp7, tmp6, tmp5, tmp4); + LSX_DUP4_ARG2(__lsx_vldrepl_b, src_left, 12, src_left, 13, src_left, 14, src_left, + 15, tmp3, tmp2, tmp1, tmp0); + LSX_DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_vaddwev_h_bu, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, reg1, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vaddwod_h_bu, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, reg1, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, + tmp4, tmp5, tmp6, tmp7); + LSX_DUP4_ARG2(__lsx_vaddwev_h_bu, tmp8, reg1, tmp9, reg1, tmp10, reg1, tmp11, reg1, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vaddwod_h_bu, tmp8, reg1, tmp9, reg1, tmp10, reg1, tmp11, reg1, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, + tmp8, tmp9, tmp10, tmp11); + LSX_DUP4_ARG2(__lsx_vaddwev_h_bu, tmp12, reg1, tmp13, reg1, tmp14, reg1, tmp15, reg1, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vaddwod_h_bu, tmp12, reg1, tmp13, reg1, tmp14, reg1, tmp15, reg1, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, + tmp12, tmp13, tmp14, tmp15); + LSX_ST_8(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, dst, dst_stride); + LSX_ST_8(tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15, dst, dst_stride); +} + +void ff_tm_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, + const uint8_t *src_left, const uint8_t *src_top_ptr) +{ + uint8_t top_left = src_top_ptr[-1]; + uint32_t loop_cnt; + __m128i tmp0, tmp1, tmp2, tmp3, reg0, reg1, reg2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + + reg0 = __lsx_vreplgr2vr_h(top_left); + LSX_DUP2_ARG2(__lsx_vld, src_top_ptr, 0, src_top_ptr, 16, reg1, reg2); + + src_left += 28; + for (loop_cnt = 8; loop_cnt--;) { + LSX_DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, 3, + tmp3, tmp2, tmp1, tmp0); + src_left -= 4; + LSX_DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg2, tmp1, reg2, tmp2, reg2, tmp3, reg2, + dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg2, tmp1, reg2, tmp2, reg2, tmp3, reg2, + dst4, dst5, dst6, dst7); + LSX_DUP4_ARG2(__lsx_vssub_hu, dst0, reg0, dst1, reg0, dst2, reg0, dst3, reg0, + dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vssub_hu, dst4, reg0, dst5, reg0, dst6, reg0, dst7, reg0, + dst4, dst5, dst6, dst7); + LSX_DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, + src4, src5, src6, src7); + LSX_DUP4_ARG2(__lsx_vsat_hu, dst0, 7, dst1, 7, dst2, 7, dst3, 7, + dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vsat_hu, dst4, 7, dst5, 7, dst6, 7, dst7, 7, + dst4, dst5, dst6, dst7); + LSX_DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vpackev_b, dst4, dst0, dst5, dst1, dst6, dst2, dst7, dst3, + dst0, dst1, dst2, dst3); + __lsx_vst(src0, dst, 0); + __lsx_vst(dst0, dst, 16); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + __lsx_vst(dst1, dst, 16); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + __lsx_vst(dst2, dst, 16); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + __lsx_vst(dst3, dst, 16); + dst += dst_stride; + } +} diff --git a/libavcodec/loongarch/vp9_lpf_lsx.c b/libavcodec/loongarch/vp9_lpf_lsx.c new file mode 100644 index 0000000000..f8bde7e300 --- /dev/null +++ b/libavcodec/loongarch/vp9_lpf_lsx.c @@ -0,0 +1,3140 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Jin Bo + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavcodec/vp9dsp.h" +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "libavutil/common.h" +#include "vp9dsp_loongarch.h" + +#define LSX_LD_8(_src, _stride, _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7) \ +{ \ + _in0 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _in1 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _in2 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _in3 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _in4 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _in5 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _in6 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _in7 = __lsx_vld(_src, 0); \ +} + +#define LSX_ST_8(_dst0, _dst1, _dst2, _dst3, _dst4, _dst5, \ + _dst6, _dst7, _dst, _stride) \ +{ \ + __lsx_vst(_dst0, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst1, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst2, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst3, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst4, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst5, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst6, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst7, _dst, 0); \ +} + +#define VP9_LPF_FILTER4_4W(p1_src, p0_src, q0_src, q1_src, mask_src, hev_src, \ + p1_dst, p0_dst, q0_dst, q1_dst) \ +{ \ + __m128i p1_tmp, p0_tmp, q0_tmp, q1_tmp, q0_sub_p0, filt, filt1, filt2; \ + const __m128i cnst3b = __lsx_vldi(3); \ + const __m128i cnst4b = __lsx_vldi(4); \ + \ + p1_tmp = __lsx_vxori_b(p1_src, 0x80); \ + p0_tmp = __lsx_vxori_b(p0_src, 0x80); \ + q0_tmp = __lsx_vxori_b(q0_src, 0x80); \ + q1_tmp = __lsx_vxori_b(q1_src, 0x80); \ + \ + filt = __lsx_vssub_b(p1_tmp, q1_tmp); \ + \ + filt = filt & hev_src; \ + \ + q0_sub_p0 = __lsx_vssub_b(q0_tmp, p0_tmp); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = filt & mask_src; \ + \ + filt1 = __lsx_vsadd_b(filt, cnst4b); \ + filt1 = __lsx_vsrai_b(filt1, 3); \ + \ + filt2 = __lsx_vsadd_b(filt, cnst3b); \ + filt2 = __lsx_vsrai_b(filt2, 3); \ + \ + q0_tmp = __lsx_vssub_b(q0_tmp, filt1); \ + q0_dst = __lsx_vxori_b(q0_tmp, 0x80); \ + p0_tmp = __lsx_vsadd_b(p0_tmp, filt2); \ + p0_dst = __lsx_vxori_b(p0_tmp, 0x80); \ + \ + filt = __lsx_vsrari_b(filt1, 1); \ + hev_src = __lsx_vxori_b(hev_src, 0xff); \ + filt = filt & hev_src; \ + \ + q1_tmp = __lsx_vssub_b(q1_tmp, filt); \ + q1_dst = __lsx_vxori_b(q1_tmp, 0x80); \ + p1_tmp = __lsx_vsadd_b(p1_tmp, filt); \ + p1_dst = __lsx_vxori_b(p1_tmp, 0x80); \ +} + +#define VP9_FLAT4(p3_src, p2_src, p0_src, q0_src, q2_src, q3_src, flat_dst) \ +{ \ + __m128i f_tmp = __lsx_vldi(1); \ + __m128i p2_a_sub_p0, q2_a_sub_q0, p3_a_sub_p0, q3_a_sub_q0; \ + \ + p2_a_sub_p0 = __lsx_vabsd_bu(p2_src, p0_src); \ + q2_a_sub_q0 = __lsx_vabsd_bu(q2_src, q0_src); \ + p3_a_sub_p0 = __lsx_vabsd_bu(p3_src, p0_src); \ + q3_a_sub_q0 = __lsx_vabsd_bu(q3_src, q0_src); \ + \ + p2_a_sub_p0 = __lsx_vmax_bu(p2_a_sub_p0, q2_a_sub_q0); \ + flat_dst = __lsx_vmax_bu(p2_a_sub_p0, flat_dst); \ + p3_a_sub_p0 = __lsx_vmax_bu(p3_a_sub_p0, q3_a_sub_q0); \ + flat_dst = __lsx_vmax_bu(p3_a_sub_p0, flat_dst); \ + \ + flat_dst = __lsx_vslt_bu(f_tmp, flat_dst); \ + flat_dst = __lsx_vxori_b(flat_dst, 0xff); \ + flat_dst = flat_dst & mask; \ +} + +#define VP9_FLAT5(p7_src, p6_src, p5_src, p4_src, p0_src, q0_src, q4_src, \ + q5_src, q6_src, q7_src, flat_src, flat2_dst) \ +{ \ + __m128i f_tmp = __lsx_vldi(1); \ + __m128i p4_a_sub_p0, q4_a_sub_q0, p5_a_sub_p0, q5_a_sub_q0; \ + __m128i p6_a_sub_p0, q6_a_sub_q0, p7_a_sub_p0, q7_a_sub_q0; \ + \ + p4_a_sub_p0 = __lsx_vabsd_bu(p4_src, p0_src); \ + q4_a_sub_q0 = __lsx_vabsd_bu(q4_src, q0_src); \ + p5_a_sub_p0 = __lsx_vabsd_bu(p5_src, p0_src); \ + q5_a_sub_q0 = __lsx_vabsd_bu(q5_src, q0_src); \ + p6_a_sub_p0 = __lsx_vabsd_bu(p6_src, p0_src); \ + q6_a_sub_q0 = __lsx_vabsd_bu(q6_src, q0_src); \ + p7_a_sub_p0 = __lsx_vabsd_bu(p7_src, p0_src); \ + q7_a_sub_q0 = __lsx_vabsd_bu(q7_src, q0_src); \ + \ + p4_a_sub_p0 = __lsx_vmax_bu(p4_a_sub_p0, q4_a_sub_q0); \ + flat2_dst = __lsx_vmax_bu(p5_a_sub_p0, q5_a_sub_q0); \ + flat2_dst = __lsx_vmax_bu(p4_a_sub_p0, flat2_dst); \ + p6_a_sub_p0 = __lsx_vmax_bu(p6_a_sub_p0, q6_a_sub_q0); \ + flat2_dst = __lsx_vmax_bu(p6_a_sub_p0, flat2_dst); \ + p7_a_sub_p0 = __lsx_vmax_bu(p7_a_sub_p0, q7_a_sub_q0); \ + flat2_dst = __lsx_vmax_bu(p7_a_sub_p0, flat2_dst); \ + \ + flat2_dst = __lsx_vslt_bu(f_tmp, flat2_dst); \ + flat2_dst = __lsx_vxori_b(flat2_dst, 0xff); \ + flat2_dst = flat2_dst & flat_src; \ +} + +#define VP9_FILTER8(p3_src, p2_src, p1_src, p0_src, \ + q0_src, q1_src, q2_src, q3_src, \ + p2_filt8_dst, p1_filt8_dst, p0_filt8_dst, \ + q0_filt8_dst, q1_filt8_dst, q2_filt8_dst) \ +{ \ + __m128i tmp0, tmp1, tmp2; \ + \ + tmp2 = __lsx_vadd_h(p2_src, p1_src); \ + tmp2 = __lsx_vadd_h(tmp2, p0_src); \ + tmp0 = __lsx_vslli_h(p3_src, 1); \ + \ + tmp0 = __lsx_vadd_h(tmp0, tmp2); \ + tmp0 = __lsx_vadd_h(tmp0, q0_src); \ + tmp1 = __lsx_vadd_h(tmp0, p3_src); \ + tmp1 = __lsx_vadd_h(tmp1, p2_src); \ + p2_filt8_dst = __lsx_vsrari_h(tmp1, 3); \ + \ + tmp1 = __lsx_vadd_h(tmp0, p1_src); \ + tmp1 = __lsx_vadd_h(tmp1, q1_src); \ + p1_filt8_dst = __lsx_vsrari_h(tmp1, 3); \ + \ + tmp1 = __lsx_vadd_h(q2_src, q1_src); \ + tmp1 = __lsx_vadd_h(tmp1, q0_src); \ + tmp2 = __lsx_vadd_h(tmp2, tmp1); \ + tmp0 = __lsx_vadd_h(tmp2, p0_src); \ + tmp0 = __lsx_vadd_h(tmp0, p3_src); \ + p0_filt8_dst = __lsx_vsrari_h(tmp0, 3); \ + \ + tmp0 = __lsx_vadd_h(q2_src, q3_src); \ + tmp0 = __lsx_vadd_h(tmp0, p0_src); \ + tmp0 = __lsx_vadd_h(tmp0, tmp1); \ + tmp1 = __lsx_vadd_h(q3_src, q3_src); \ + tmp1 = __lsx_vadd_h(tmp1, tmp0); \ + q2_filt8_dst = __lsx_vsrari_h(tmp1, 3); \ + \ + tmp0 = __lsx_vadd_h(tmp2, q3_src); \ + tmp1 = __lsx_vadd_h(tmp0, q0_src); \ + q0_filt8_dst = __lsx_vsrari_h(tmp1, 3); \ + \ + tmp1 = __lsx_vsub_h(tmp0, p2_src); \ + tmp0 = __lsx_vadd_h(q1_src, q3_src); \ + tmp1 = __lsx_vadd_h(tmp0, tmp1); \ + q1_filt8_dst = __lsx_vsrari_h(tmp1, 3); \ +} + +#define LPF_MASK_HEV(p3_src, p2_src, p1_src, p0_src, q0_src, q1_src, \ + q2_src, q3_src, limit_src, b_limit_src, thresh_src, \ + hev_dst, mask_dst, flat_dst) \ +{ \ + __m128i p3_asub_p2_tmp, p2_asub_p1_tmp, p1_asub_p0_tmp, q1_asub_q0_tmp; \ + __m128i p1_asub_q1_tmp, p0_asub_q0_tmp, q3_asub_q2_tmp, q2_asub_q1_tmp; \ + \ + /* absolute subtraction of pixel values */ \ + p3_asub_p2_tmp = __lsx_vabsd_bu(p3_src, p2_src); \ + p2_asub_p1_tmp = __lsx_vabsd_bu(p2_src, p1_src); \ + p1_asub_p0_tmp = __lsx_vabsd_bu(p1_src, p0_src); \ + q1_asub_q0_tmp = __lsx_vabsd_bu(q1_src, q0_src); \ + q2_asub_q1_tmp = __lsx_vabsd_bu(q2_src, q1_src); \ + q3_asub_q2_tmp = __lsx_vabsd_bu(q3_src, q2_src); \ + p0_asub_q0_tmp = __lsx_vabsd_bu(p0_src, q0_src); \ + p1_asub_q1_tmp = __lsx_vabsd_bu(p1_src, q1_src); \ + \ + /* calculation of hev */ \ + flat_dst = __lsx_vmax_bu(p1_asub_p0_tmp, q1_asub_q0_tmp); \ + hev_dst = __lsx_vslt_bu(thresh_src, flat_dst); \ + \ + /* calculation of mask */ \ + p0_asub_q0_tmp = __lsx_vsadd_bu(p0_asub_q0_tmp, p0_asub_q0_tmp); \ + p1_asub_q1_tmp = __lsx_vsrli_b(p1_asub_q1_tmp, 1); \ + p0_asub_q0_tmp = __lsx_vsadd_bu(p0_asub_q0_tmp, p1_asub_q1_tmp); \ + \ + mask_dst = __lsx_vslt_bu(b_limit_src, p0_asub_q0_tmp); \ + mask_dst = __lsx_vmax_bu(flat_dst, mask_dst); \ + p3_asub_p2_tmp = __lsx_vmax_bu(p3_asub_p2_tmp, p2_asub_p1_tmp); \ + mask_dst = __lsx_vmax_bu(p3_asub_p2_tmp, mask_dst); \ + q2_asub_q1_tmp = __lsx_vmax_bu(q2_asub_q1_tmp, q3_asub_q2_tmp); \ + mask_dst = __lsx_vmax_bu(q2_asub_q1_tmp, mask_dst); \ + \ + mask_dst = __lsx_vslt_bu(limit_src, mask_dst); \ + mask_dst = __lsx_vxori_b(mask_dst, 0xff); \ +} + +#define LSX_TRANSPOSE8x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, _out7) \ +{ \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + LSX_DUP4_ARG2(__lsx_vilvl_b, _in2, _in0, _in3, _in1, _in6, _in4, _in7, _in5, \ + _tmp0, _tmp1, _tmp2, _tmp3); \ + LSX_DUP2_ARG2(__lsx_vilvl_b, _tmp1, _tmp0, _tmp3, _tmp2, _tmp4, _tmp6); \ + LSX_DUP2_ARG2(__lsx_vilvh_b, _tmp1, _tmp0, _tmp3, _tmp2, _tmp5, _tmp7); \ + LSX_DUP2_ARG2(__lsx_vilvl_w, _tmp6, _tmp4, _tmp7, _tmp5, _out0, _out4); \ + LSX_DUP2_ARG2(__lsx_vilvh_w, _tmp6, _tmp4, _tmp7, _tmp5, _out2, _out6); \ + LSX_DUP4_ARG2(__lsx_vbsrl_v, _out0, 8, _out2, 8, _out4, 8, _out6, 8, \ + _out1, _out3, _out5, _out7); \ +} + +#define LSX_TRANSPOSE16x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, _in8, \ + _in9, _in10, _in11, _in12, _in13, _in14, _in15, _out0, \ + _out1, _out2, _out3, _out4, _out5, _out6, _out7) \ +{ \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + LSX_DUP4_ARG2(__lsx_vilvl_b, _in2, _in0, _in3, _in1, _in6, _in4, _in7, _in5, \ + _tmp0, _tmp1, _tmp2, _tmp3); \ + LSX_DUP4_ARG2(__lsx_vilvl_b, _in10, _in8, _in11, _in9, _in14, _in12, _in15, \ + _in13, _tmp4, _tmp5, _tmp6, _tmp7); \ + LSX_DUP2_ARG2(__lsx_vilvl_b, _tmp1, _tmp0, _tmp3, _tmp2, _t0, _t2); \ + LSX_DUP2_ARG2(__lsx_vilvh_b, _tmp1, _tmp0, _tmp3, _tmp2, _t1, _t3); \ + LSX_DUP2_ARG2(__lsx_vilvl_b, _tmp5, _tmp4, _tmp7, _tmp6, _t4, _t6); \ + LSX_DUP2_ARG2(__lsx_vilvh_b, _tmp5, _tmp4, _tmp7, _tmp6, _t5, _t7); \ + LSX_DUP2_ARG2(__lsx_vilvl_w, _t2, _t0, _t3, _t1, _tmp0, _tmp4); \ + LSX_DUP2_ARG2(__lsx_vilvh_w, _t2, _t0, _t3, _t1, _tmp2, _tmp6); \ + LSX_DUP2_ARG2(__lsx_vilvl_w, _t6, _t4, _t7, _t5, _tmp1, _tmp5); \ + LSX_DUP2_ARG2(__lsx_vilvh_w, _t6, _t4, _t7, _t5, _tmp3, _tmp7); \ + LSX_DUP2_ARG2(__lsx_vilvl_d, _tmp1, _tmp0, _tmp3, _tmp2, _out0, _out2); \ + LSX_DUP2_ARG2(__lsx_vilvh_d, _tmp1, _tmp0, _tmp3, _tmp2, _out1, _out3); \ + LSX_DUP2_ARG2(__lsx_vilvl_d, _tmp5, _tmp4, _tmp7, _tmp6, _out4, _out6); \ + LSX_DUP2_ARG2(__lsx_vilvh_d, _tmp5, _tmp4, _tmp7, _tmp6, _out5, _out7); \ +} + + +void ff_loop_filter_v_4_8_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + __m128i mask, hev, flat, thresh, b_limit, limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0, p1_out, p0_out, q0_out, q1_out; + + LSX_DUP4_ARG2(__lsx_vld, dst - stride4, 0, dst - stride3, 0, dst - stride2, 0, + dst - stride, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, + dst + stride3, 0, q0, q1, q2, q3); + + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + limit = __lsx_vreplgr2vr_b(limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + __lsx_vstelm_d(p1_out, dst - stride2, 0, 0); + __lsx_vstelm_d(p0_out, dst - stride, 0, 0); + __lsx_vstelm_d(q0_out, dst , 0, 0); + __lsx_vstelm_d(q1_out, dst + stride, 0, 0); +} + +void ff_loop_filter_v_44_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + __m128i mask, hev, flat, thresh0, b_limit0; + __m128i limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + + LSX_DUP4_ARG2(__lsx_vld, dst - stride4, 0, dst - stride3, 0, dst - stride2, 0, + dst - stride, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, + dst + stride3, 0, q0, q1, q2, q3); + + thresh0 = __lsx_vreplgr2vr_b(thresh_ptr); + thresh1 = __lsx_vreplgr2vr_b(thresh_ptr >> 8); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vreplgr2vr_b(b_limit_ptr); + b_limit1 = __lsx_vreplgr2vr_b(b_limit_ptr >> 8); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vreplgr2vr_b(limit_ptr); + limit1 = __lsx_vreplgr2vr_b(limit_ptr >> 8); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, + hev, mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + + __lsx_vst(p1, dst - stride2, 0); + __lsx_vst(p0, dst - stride, 0); + __lsx_vst(q0, dst , 0); + __lsx_vst(q1, dst + stride, 0); +} + +void ff_loop_filter_v_8_8_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + __m128i mask, hev, flat, thresh, b_limit, limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i p2_filter8, p1_filter8, p0_filter8; + __m128i q0_filter8, q1_filter8, q2_filter8; + __m128i p3_l, p2_l, p1_l, p0_l, q3_l, q2_l, q1_l, q0_l; + __m128i zero = __lsx_vldi(0); + + LSX_DUP4_ARG2(__lsx_vld, dst - stride4, 0, dst - stride3, 0, dst - stride2, 0, + dst - stride, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, + dst + stride3, 0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + limit = __lsx_vreplgr2vr_b(limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + __lsx_vstelm_d(p1_out, dst - stride2, 0, 0); + __lsx_vstelm_d(p0_out, dst - stride, 0, 0); + __lsx_vstelm_d(q0_out, dst , 0, 0); + __lsx_vstelm_d(q1_out, dst + stride, 0, 0); + } else { + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filter8, + p1_filter8, p0_filter8, q0_filter8, q1_filter8, q2_filter8); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, zero, p2_filter8, zero, p1_filter8, + zero, p0_filter8, zero, q0_filter8, p2_filter8, + p1_filter8, p0_filter8, q0_filter8); + LSX_DUP2_ARG2(__lsx_vpickev_b, zero, q1_filter8, zero, q2_filter8, + q1_filter8, q2_filter8); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filter8, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filter8, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filter8, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filter8, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filter8, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filter8, flat); + + __lsx_vstelm_d(p2_out, dst - stride3, 0, 0); + __lsx_vstelm_d(p1_out, dst - stride2, 0, 0); + __lsx_vstelm_d(p0_out, dst - stride, 0, 0); + __lsx_vstelm_d(q0_out, dst, 0, 0); + __lsx_vstelm_d(q1_out, dst + stride, 0, 0); + __lsx_vstelm_d(q2_out, dst + stride2, 0, 0); + } +} + +void ff_loop_filter_v_88_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, tmp, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i zero = __lsx_vldi(0); + + /* load vector elements */ + LSX_DUP4_ARG2(__lsx_vld, dst - stride4, 0, dst - stride3, 0, dst - stride2, 0, + dst - stride, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, + dst + stride3, 0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + tmp = __lsx_vreplgr2vr_b(thresh_ptr >> 8); + thresh = __lsx_vilvl_d(tmp, thresh); + + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + tmp = __lsx_vreplgr2vr_b(b_limit_ptr >> 8); + b_limit = __lsx_vilvl_d(tmp, b_limit); + + limit = __lsx_vreplgr2vr_b(limit_ptr); + tmp = __lsx_vreplgr2vr_b(limit_ptr >> 8); + limit = __lsx_vilvl_d(tmp, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + } else { + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_h, p2_h, p1_h, p0_h); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_h, q1_h, q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, + p1_filt8_l, p0_filt8_h, p0_filt8_l, q0_filt8_h, + q0_filt8_l, p2_filt8_l, p1_filt8_l, p0_filt8_l, + q0_filt8_l); + LSX_DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, + q2_filt8_l, q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + + __lsx_vst(p2_out, dst - stride3, 0); + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + __lsx_vst(q2_out, dst + stride2, 0); + } +} + +void ff_loop_filter_v_84_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, tmp, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i zero = __lsx_vldi(0); + + /* load vector elements */ + LSX_DUP4_ARG2(__lsx_vld, dst - stride4, 0, dst - stride3, 0, dst - stride2, 0, + dst - stride, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, + dst + stride3, 0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + tmp = __lsx_vreplgr2vr_b(thresh_ptr >> 8); + thresh = __lsx_vilvl_d(tmp, thresh); + + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + tmp = __lsx_vreplgr2vr_b(b_limit_ptr >> 8); + b_limit = __lsx_vilvl_d(tmp, b_limit); + + limit = __lsx_vreplgr2vr_b(limit_ptr); + tmp = __lsx_vreplgr2vr_b(limit_ptr >> 8); + limit = __lsx_vilvl_d(tmp, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + } else { + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, p2_filt8_l, p2_filt8_l, p1_filt8_l, + p1_filt8_l, p0_filt8_l, p0_filt8_l, q0_filt8_l, + q0_filt8_l, p2_filt8_l, p1_filt8_l, p0_filt8_l, + q0_filt8_l); + LSX_DUP2_ARG2(__lsx_vpickev_b, q1_filt8_l, q1_filt8_l, q2_filt8_l, + q2_filt8_l, q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + __lsx_vst(p2_out, dst - stride3, 0); + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + __lsx_vst(q2_out, dst + stride2, 0); + } +} + +void ff_loop_filter_v_48_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, tmp, thresh, b_limit, limit; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i zero = { 0 }; + + /* load vector elements */ + LSX_DUP4_ARG2(__lsx_vld, dst - stride4, 0, dst - stride3, 0, dst - stride2, 0, + dst - stride, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, + dst + stride3, 0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + tmp = __lsx_vreplgr2vr_b(thresh_ptr >> 8); + thresh = __lsx_vilvl_d(tmp, thresh); + + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + tmp = __lsx_vreplgr2vr_b(b_limit_ptr >> 8); + b_limit = __lsx_vilvl_d(tmp, b_limit); + + limit = __lsx_vreplgr2vr_b(limit_ptr); + tmp = __lsx_vreplgr2vr_b(limit_ptr >> 8); + limit = __lsx_vilvl_d(tmp, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + flat = __lsx_vilvh_d(flat, zero); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + } else { + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_h, p2_h, p1_h, p0_h); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_h, q1_h, q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_h, p1_filt8_h, + p1_filt8_h, p0_filt8_h, p0_filt8_h, q0_filt8_h, + q0_filt8_h, p2_filt8_h, p1_filt8_h, p0_filt8_h, + q0_filt8_h); + LSX_DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_h, q2_filt8_h, + q2_filt8_h, q1_filt8_h, q2_filt8_h); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filt8_h, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_h, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_h, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_h, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_h, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filt8_h, flat); + + __lsx_vst(p2_out, dst - stride3, 0); + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + __lsx_vst(q2_out, dst + stride2, 0); + } +} + +static int32_t vp9_hz_lpf_t4_and_t8_16w(uint8_t *dst, ptrdiff_t stride, + uint8_t *filter48, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i zero = __lsx_vldi(0); + + /* load vector elements */ + LSX_DUP4_ARG2(__lsx_vld, dst - stride4, 0, dst - stride3, 0, dst - stride2, 0, + dst - stride, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, + dst + stride3, 0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + limit = __lsx_vreplgr2vr_b(limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + return 1; + } else { + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_h, p2_h, p1_h, p0_h); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_h, q1_h, q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, + p1_filt8_l, p0_filt8_h, p0_filt8_l, q0_filt8_h, + q0_filt8_l, p2_filt8_l, p1_filt8_l, p0_filt8_l, + q0_filt8_l); + LSX_DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, + q2_filt8_l, q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + __lsx_vst(p2_out, filter48, 0); + __lsx_vst(p1_out, filter48, 16); + __lsx_vst(p0_out, filter48, 32); + __lsx_vst(q0_out, filter48, 48); + __lsx_vst(q1_out, filter48, 64); + __lsx_vst(q2_out, filter48, 80); + __lsx_vst(flat, filter48, 96); + + return 0; + } +} + +static void vp9_hz_lpf_t16_16w(uint8_t *dst, ptrdiff_t stride, uint8_t *filter48) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst - stride4; + uint8_t *dst_tmp1 = dst + stride4; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + __m128i flat, flat2, filter8; + __m128i zero = __lsx_vldi(0); + __m128i out_h, out_l; + v8u16 p7_l_in, p6_l_in, p5_l_in, p4_l_in; + v8u16 p3_l_in, p2_l_in, p1_l_in, p0_l_in; + v8u16 q7_l_in, q6_l_in, q5_l_in, q4_l_in; + v8u16 q3_l_in, q2_l_in, q1_l_in, q0_l_in; + v8u16 p7_h_in, p6_h_in, p5_h_in, p4_h_in; + v8u16 p3_h_in, p2_h_in, p1_h_in, p0_h_in; + v8u16 q7_h_in, q6_h_in, q5_h_in, q4_h_in; + v8u16 q3_h_in, q2_h_in, q1_h_in, q0_h_in; + v8u16 tmp0_l, tmp1_l, tmp0_h, tmp1_h; + + flat = __lsx_vld(filter48, 96); + + LSX_DUP4_ARG2(__lsx_vld, dst_tmp - stride4, 0, dst_tmp - stride3, 0, + dst_tmp - stride2, 0, dst_tmp - stride, 0, p7, p6, p5, p4); + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp + stride, 0, dst_tmp + stride2, 0, + dst_tmp + stride3, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, dst + stride3, + 0, q0, q1, q2, q3); + LSX_DUP4_ARG2(__lsx_vld, dst_tmp1, 0, dst_tmp1 + stride, 0, dst_tmp1 + stride2, 0, + dst_tmp1 + stride3, 0, q4, q5, q6, q7); + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + /* if flat2 is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat2)) { + LSX_DUP4_ARG2(__lsx_vld, filter48, 0, filter48, 16, filter48, 32, filter48, + 48, p2, p1, p0, q0); + LSX_DUP2_ARG2(__lsx_vld, filter48, 64, filter48, 80, q1, q2); + + __lsx_vst(p2, dst - stride3, 0); + __lsx_vst(p1, dst - stride2, 0); + __lsx_vst(p0, dst - stride, 0); + __lsx_vst(q0, dst, 0); + __lsx_vst(q1, dst + stride, 0); + __lsx_vst(q2, dst + stride2, 0); + } else { + dst = dst_tmp - stride3; + + p7_l_in = (v8u16)__lsx_vilvl_b(zero, p7); + p6_l_in = (v8u16)__lsx_vilvl_b(zero, p6); + p5_l_in = (v8u16)__lsx_vilvl_b(zero, p5); + p4_l_in = (v8u16)__lsx_vilvl_b(zero, p4); + p3_l_in = (v8u16)__lsx_vilvl_b(zero, p3); + p2_l_in = (v8u16)__lsx_vilvl_b(zero, p2); + p1_l_in = (v8u16)__lsx_vilvl_b(zero, p1); + p0_l_in = (v8u16)__lsx_vilvl_b(zero, p0); + + q0_l_in = (v8u16)__lsx_vilvl_b(zero, q0); + + tmp0_l = p7_l_in << 3; + tmp0_l -= p7_l_in; + tmp0_l += p6_l_in; + tmp0_l += q0_l_in; + tmp1_l = p6_l_in + p5_l_in; + tmp1_l += p4_l_in; + tmp1_l += p3_l_in; + tmp1_l += p2_l_in; + tmp1_l += p1_l_in; + tmp1_l += p0_l_in; + tmp1_l += tmp0_l; + + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + p7_h_in = (v8u16)__lsx_vilvh_b(zero, p7); + p6_h_in = (v8u16)__lsx_vilvh_b(zero, p6); + p5_h_in = (v8u16)__lsx_vilvh_b(zero, p5); + p4_h_in = (v8u16)__lsx_vilvh_b(zero, p4); + + p3_h_in = (v8u16)__lsx_vilvh_b(zero, p3); + p2_h_in = (v8u16)__lsx_vilvh_b(zero, p2); + p1_h_in = (v8u16)__lsx_vilvh_b(zero, p1); + p0_h_in = (v8u16)__lsx_vilvh_b(zero, p0); + q0_h_in = (v8u16)__lsx_vilvh_b(zero, q0); + + tmp0_h = p7_h_in << 3; + tmp0_h -= p7_h_in; + tmp0_h += p6_h_in; + tmp0_h += q0_h_in; + tmp1_h = p6_h_in + p5_h_in; + tmp1_h += p4_h_in; + tmp1_h += p3_h_in; + tmp1_h += p2_h_in; + tmp1_h += p1_h_in; + tmp1_h += p0_h_in; + tmp1_h += tmp0_h; + + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p6 = __lsx_vbitsel_v(p6, out_l, flat2); + __lsx_vst(p6, dst, 0); + dst += stride; + + /* p5 */ + q1_l_in = (v8u16)__lsx_vilvl_b(zero, q1); + tmp0_l = p5_l_in - p6_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q1_h_in = (v8u16)__lsx_vilvh_b(zero, q1); + tmp0_h = p5_h_in - p6_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p5 = __lsx_vbitsel_v(p5, out_l, flat2); + __lsx_vst(p5, dst, 0); + dst += stride; + + /* p4 */ + q2_l_in = (v8u16)__lsx_vilvl_b(zero, q2); + tmp0_l = p4_l_in - p5_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q2_h_in = (v8u16)__lsx_vilvh_b(zero, q2); + tmp0_h = p4_h_in - p5_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p4 = __lsx_vbitsel_v(p4, out_l, flat2); + __lsx_vst(p4, dst, 0); + dst += stride; + + /* p3 */ + q3_l_in = (v8u16)__lsx_vilvl_b(zero, q3); + tmp0_l = p3_l_in - p4_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q3_h_in = (v8u16)__lsx_vilvh_b(zero, q3); + tmp0_h = p3_h_in - p4_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p3 = __lsx_vbitsel_v(p3, out_l, flat2); + __lsx_vst(p3, dst, 0); + dst += stride; + + /* p2 */ + q4_l_in = (v8u16)__lsx_vilvl_b(zero, q4); + filter8 = __lsx_vld(filter48, 0); + tmp0_l = p2_l_in - p3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q4_h_in = (v8u16)__lsx_vilvh_b(zero, q4); + tmp0_h = p2_h_in - p3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* p1 */ + q5_l_in = (v8u16)__lsx_vilvl_b(zero, q5); + filter8 = __lsx_vld(filter48, 16); + tmp0_l = p1_l_in - p2_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q5_h_in = (v8u16)__lsx_vilvh_b(zero, q5); + tmp0_h = p1_h_in - p2_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* p0 */ + q6_l_in = (v8u16)__lsx_vilvl_b(zero, q6); + filter8 = __lsx_vld(filter48, 32); + tmp0_l = p0_l_in - p1_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q6_h_in = (v8u16)__lsx_vilvh_b(zero, q6); + tmp0_h = p0_h_in - p1_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q0 */ + q7_l_in = (v8u16)__lsx_vilvl_b(zero, q7); + filter8 = __lsx_vld(filter48, 48); + tmp0_l = q7_l_in - p0_l_in; + tmp0_l += q0_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q7_h_in = (v8u16)__lsx_vilvh_b(zero, q7); + tmp0_h = q7_h_in - p0_h_in; + tmp0_h += q0_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q1 */ + filter8 = __lsx_vld(filter48, 64); + tmp0_l = q7_l_in - q0_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p6_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q0_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p6_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q2 */ + filter8 = __lsx_vld(filter48, 80); + tmp0_l = q7_l_in - q1_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p5_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q1_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p5_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q3 */ + tmp0_l = q7_l_in - q2_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p4_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q2_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p4_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q3 = __lsx_vbitsel_v(q3, out_l, flat2); + __lsx_vst(q3, dst, 0); + dst += stride; + + /* q4 */ + tmp0_l = q7_l_in - q3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p3_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p3_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q4 = __lsx_vbitsel_v(q4, out_l, flat2); + __lsx_vst(q4, dst, 0); + dst += stride; + + /* q5 */ + tmp0_l = q7_l_in - q4_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p2_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q4_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p2_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q5 = __lsx_vbitsel_v(q5, out_l, flat2); + __lsx_vst(q5, dst, 0); + dst += stride; + + /* q6 */ + tmp0_l = q7_l_in - q5_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p1_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q5_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p1_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q6 = __lsx_vbitsel_v(q6, out_l, flat2); + __lsx_vst(q6, dst, 0); + } +} + +void ff_loop_filter_v_16_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + uint8_t filter48[16 * 8] __attribute__ ((aligned(16))); + uint8_t early_exit = 0; + + early_exit = vp9_hz_lpf_t4_and_t8_16w(dst, stride, &filter48[0], + b_limit_ptr, limit_ptr, thresh_ptr); + + if (0 == early_exit) { + vp9_hz_lpf_t16_16w(dst, stride, filter48); + } +} + +void ff_loop_filter_v_16_8_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst - stride4; + uint8_t *dst_tmp1 = dst + stride4; + __m128i zero = __lsx_vldi(0); + __m128i flat2, mask, hev, flat, thresh, b_limit, limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0, p7, p6, p5, p4, q4, q5, q6, q7; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i p0_filter16, p1_filter16; + __m128i p2_filter8, p1_filter8, p0_filter8; + __m128i q0_filter8, q1_filter8, q2_filter8; + __m128i p7_l, p6_l, p5_l, p4_l, q7_l, q6_l, q5_l, q4_l; + __m128i p3_l, p2_l, p1_l, p0_l, q3_l, q2_l, q1_l, q0_l; + __m128i tmp0, tmp1, tmp2; + + /* load vector elements */ + LSX_DUP4_ARG2(__lsx_vld, dst - stride4, 0, dst - stride3, 0, dst - stride2, 0, + dst - stride, 0, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst + stride, 0, dst + stride2, 0, + dst + stride3, 0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + limit = __lsx_vreplgr2vr_b(limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + __lsx_vstelm_d(p1_out, dst - stride2, 0, 0); + __lsx_vstelm_d(p0_out, dst - stride, 0, 0); + __lsx_vstelm_d(q0_out, dst , 0, 0); + __lsx_vstelm_d(q1_out, dst + stride, 0, 0); + } else { + /* convert 8 bit input data into 16 bit */ + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, + p2_filter8, p1_filter8, p0_filter8, q0_filter8, + q1_filter8, q2_filter8); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, zero, p2_filter8, zero, p1_filter8, + zero, p0_filter8, zero, q0_filter8, p2_filter8, + p1_filter8, p0_filter8, q0_filter8); + LSX_DUP2_ARG2(__lsx_vpickev_b, zero, q1_filter8, zero, q2_filter8, + q1_filter8, q2_filter8); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filter8, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filter8, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filter8, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filter8, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filter8, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filter8, flat); + + /* load 16 vector elements */ + LSX_DUP4_ARG2(__lsx_vld, dst_tmp - stride4, 0, dst_tmp - stride3, 0, + dst_tmp - stride2, 0, dst_tmp - stride, 0, p7, p6, p5, p4); + LSX_DUP4_ARG2(__lsx_vld, dst_tmp1, 0, dst_tmp1 + stride, 0, dst_tmp1 + stride2, + 0, dst_tmp1 + stride3, 0, q4, q5, q6, q7); + + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + /* if flat2 is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat2)) { + dst -= stride3; + __lsx_vstelm_d(p2_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p0_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q0_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q1_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q2_out, dst, 0, 0); + } else { + /* LSB(right) 8 pixel operation */ + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p7, zero, p6, zero, p5, zero, p4, + p7_l, p6_l, p5_l, p4_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q4, zero, q5, zero, q6, zero, q7, + q4_l, q5_l, q6_l, q7_l); + + tmp0 = __lsx_vslli_h(p7_l, 3); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp0 = __lsx_vadd_h(tmp0, p6_l); + tmp0 = __lsx_vadd_h(tmp0, q0_l); + + dst = dst_tmp - stride3; + + /* calculation of p6 and p5 */ + tmp1 = __lsx_vadd_h(p6_l, p5_l); + tmp1 = __lsx_vadd_h(tmp1, p4_l); + tmp1 = __lsx_vadd_h(tmp1, p3_l); + tmp1 = __lsx_vadd_h(tmp1, p2_l); + tmp1 = __lsx_vadd_h(tmp1, p1_l); + tmp1 = __lsx_vadd_h(tmp1, p0_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp0 = __lsx_vsub_h(p5_l, p6_l); + tmp0 = __lsx_vadd_h(tmp0, q1_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + LSX_DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, + p1_filter16, p0_filter16, p1_filter16); + p0_filter16 = __lsx_vbitsel_v(p6, p0_filter16, flat2); + p1_filter16 = __lsx_vbitsel_v(p5, p1_filter16, flat2); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of p4 and p3 */ + tmp0 = __lsx_vsub_h(p4_l, p5_l); + tmp0 = __lsx_vadd_h(tmp0, q2_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp2 = __lsx_vsub_h(p3_l, p4_l); + tmp2 = __lsx_vadd_h(tmp2, q3_l); + tmp2 = __lsx_vsub_h(tmp2, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + LSX_DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, + p1_filter16, p0_filter16, p1_filter16); + p0_filter16 = __lsx_vbitsel_v(p4, p0_filter16, flat2); + p1_filter16 = __lsx_vbitsel_v(p3, p1_filter16, flat2); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of p2 and p1 */ + tmp0 = __lsx_vsub_h(p2_l, p3_l); + tmp0 = __lsx_vadd_h(tmp0, q4_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp2 = __lsx_vsub_h(p1_l, p2_l); + tmp2 = __lsx_vadd_h(tmp2, q5_l); + tmp2 = __lsx_vsub_h(tmp2, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + LSX_DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, + p1_filter16, p0_filter16, p1_filter16); + p0_filter16 = __lsx_vbitsel_v(p2_out, p0_filter16, flat2); + p1_filter16 = __lsx_vbitsel_v(p1_out, p1_filter16, flat2); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of p0 and q0 */ + tmp0 = __lsx_vsub_h(p0_l, p1_l); + tmp0 = __lsx_vadd_h(tmp0, q6_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp2 = __lsx_vsub_h(q7_l, p0_l); + tmp2 = __lsx_vadd_h(tmp2, q0_l); + tmp2 = __lsx_vsub_h(tmp2, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h((__m128i)tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h((__m128i)tmp1, 4); + LSX_DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, + p1_filter16, p0_filter16, p1_filter16); + p0_filter16 = __lsx_vbitsel_v(p0_out, p0_filter16, flat2); + p1_filter16 = __lsx_vbitsel_v(q0_out, p1_filter16, flat2); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of q1 and q2 */ + tmp0 = __lsx_vsub_h(q7_l, q0_l); + tmp0 = __lsx_vadd_h(tmp0, q1_l); + tmp0 = __lsx_vsub_h(tmp0, p6_l); + tmp2 = __lsx_vsub_h(q7_l, q1_l); + tmp2 = __lsx_vadd_h(tmp2, q2_l); + tmp2 = __lsx_vsub_h(tmp2, p5_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + LSX_DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, + p1_filter16, p0_filter16, p1_filter16); + p0_filter16 = __lsx_vbitsel_v(q1_out, p0_filter16, flat2); + p1_filter16 = __lsx_vbitsel_v(q2_out, p1_filter16, flat2); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of q3 and q4 */ + tmp0 = __lsx_vsub_h(q7_l, q2_l); + tmp0 = __lsx_vadd_h(tmp0, q3_l); + tmp0 = __lsx_vsub_h(tmp0, p4_l); + tmp2 = __lsx_vsub_h(q7_l, q3_l); + tmp2 = __lsx_vadd_h(tmp2, q4_l); + tmp2 = __lsx_vsub_h(tmp2, p3_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + LSX_DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, + p1_filter16, p0_filter16, p1_filter16); + p0_filter16 = __lsx_vbitsel_v(q3, p0_filter16, flat2); + p1_filter16 = __lsx_vbitsel_v(q4, p1_filter16, flat2); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of q5 and q6 */ + tmp0 = __lsx_vsub_h(q7_l, q4_l); + tmp0 = __lsx_vadd_h(tmp0, q5_l); + tmp0 = __lsx_vsub_h(tmp0, p2_l); + tmp2 = __lsx_vsub_h(q7_l, q5_l); + tmp2 = __lsx_vadd_h(tmp2, q6_l); + tmp2 = __lsx_vsub_h(tmp2, p1_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + LSX_DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, + p1_filter16, p0_filter16, p1_filter16); + p0_filter16 = __lsx_vbitsel_v(q5, p0_filter16, flat2); + p1_filter16 = __lsx_vbitsel_v(q6, p1_filter16, flat2); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + } + } +} + +void ff_loop_filter_h_4_8_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst + stride4; + __m128i mask, hev, flat, limit, thresh, b_limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i vec0, vec1, vec2, vec3; + + LSX_DUP4_ARG2(__lsx_vld, dst, -4, dst + stride, -4, dst + stride2, -4, + dst + stride3, -4, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + limit = __lsx_vreplgr2vr_b(limit_ptr); + + LSX_TRANSPOSE8x8_B(p3, p2, p1, p0, q0, q1, q2, q3, + p3, p2, p1, p0, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + LSX_DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + + dst -= 2; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_w(vec2, dst + stride, 0, 1); + __lsx_vstelm_w(vec2, dst + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_w(vec3, dst + stride, 0, 1); + __lsx_vstelm_w(vec3, dst + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst + stride3, 0, 3); +} + +void ff_loop_filter_h_44_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst; + __m128i mask, hev, flat; + __m128i thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i row8, row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row0, row1, row2, row3); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row4, row5, row6, row7); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row8, row9, row10, row11); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row12, row13, row14, row15); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, + row8, row9, row10, row11, row12, row13, row14, row15, + p3, p2, p1, p0, q0, q1, q2, q3); + + thresh0 = __lsx_vreplgr2vr_b(thresh_ptr); + thresh1 = __lsx_vreplgr2vr_b(thresh_ptr >> 8); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vreplgr2vr_b(b_limit_ptr); + b_limit1 = __lsx_vreplgr2vr_b(b_limit_ptr >> 8); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vreplgr2vr_b(limit_ptr); + limit1 = __lsx_vreplgr2vr_b(limit_ptr >> 8); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, + hev, mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + LSX_DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, tmp0, tmp1); + tmp2 = __lsx_vilvl_h(tmp1, tmp0); + tmp3 = __lsx_vilvh_h(tmp1, tmp0); + LSX_DUP2_ARG2(__lsx_vilvh_b, p0, p1, q1, q0, tmp0, tmp1); + tmp4 = __lsx_vilvl_h(tmp1, tmp0); + tmp5 = __lsx_vilvh_h(tmp1, tmp0); + + dst -= 2; + __lsx_vstelm_w(tmp2, dst, 0, 0); + __lsx_vstelm_w(tmp2, dst + stride, 0, 1); + __lsx_vstelm_w(tmp2, dst + stride2, 0, 2); + __lsx_vstelm_w(tmp2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(tmp3, dst, 0, 0); + __lsx_vstelm_w(tmp3, dst + stride, 0, 1); + __lsx_vstelm_w(tmp3, dst + stride2, 0, 2); + __lsx_vstelm_w(tmp3, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(tmp4, dst, 0, 0); + __lsx_vstelm_w(tmp4, dst + stride, 0, 1); + __lsx_vstelm_w(tmp4, dst + stride2, 0, 2); + __lsx_vstelm_w(tmp4, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(tmp5, dst, 0, 0); + __lsx_vstelm_w(tmp5, dst + stride, 0, 1); + __lsx_vstelm_w(tmp5, dst + stride2, 0, 2); + __lsx_vstelm_w(tmp5, dst + stride3, 0, 3); +} + +void ff_loop_filter_h_8_8_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p1_out, p0_out, q0_out, q1_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i vec0, vec1, vec2, vec3, vec4; + __m128i zero = __lsx_vldi(0); + + /* load vector elements */ + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, p3, p2, p1, p0); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, q0, q1, q2, q3); + + LSX_TRANSPOSE8x8_B(p3, p2, p1, p0, q0, q1, q2, q3, + p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + limit = __lsx_vreplgr2vr_b(limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + /* Store 4 pixels p1-_q1 */ + LSX_DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + + dst -= 2; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_w(vec2, dst + stride, 0, 1); + __lsx_vstelm_w(vec2, dst + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_w(vec3, dst + stride, 0, 1); + __lsx_vstelm_w(vec3, dst + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst + stride3, 0, 3); + } else { + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, p2_filt8_l, p2_filt8_l, p1_filt8_l, + p1_filt8_l, p0_filt8_l, p0_filt8_l, q0_filt8_l, + q0_filt8_l, p2_filt8_l, p1_filt8_l, p0_filt8_l, + q0_filt8_l); + LSX_DUP2_ARG2(__lsx_vpickev_b, q1_filt8_l, q1_filt8_l, q2_filt8_l, + q2_filt8_l, q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2 = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1 = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0 = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0 = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1 = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2 = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + /* Store 6 pixels p2-_q2 */ + LSX_DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + vec4 = __lsx_vilvl_b(q2, q1); + + dst -= 3; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_h(vec4, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec2, dst, 0, 1); + __lsx_vstelm_h(vec4, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec2, dst, 0, 2); + __lsx_vstelm_h(vec4, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec2, dst, 0, 3); + __lsx_vstelm_h(vec4, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_h(vec4, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 1); + __lsx_vstelm_h(vec4, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 2); + __lsx_vstelm_h(vec4, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 3); + __lsx_vstelm_h(vec4, dst, 4, 7); + } +} + +void ff_loop_filter_h_88_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p1_out, p0_out, q0_out, q1_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i row4, row5, row6, row7, row12, row13, row14, row15; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i zero = __lsx_vldi(0); + + + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, p0, p1, p2, p3); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row4, row5, row6, row7); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, q3, q2, q1, q0); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row12, row13, row14, row15); + + /* transpose 16x8 matrix into 8x16 */ + LSX_TRANSPOSE16x8_B(p0, p1, p2, p3, row4, row5, row6, row7, + q3, q2, q1, q0, row12, row13, row14, row15, + p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + vec0 = __lsx_vreplgr2vr_b(thresh_ptr >> 8); + thresh = __lsx_vilvl_d(vec0, thresh); + + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + vec0 = __lsx_vreplgr2vr_b(b_limit_ptr >> 8); + b_limit = __lsx_vilvl_d(vec0, b_limit); + + limit = __lsx_vreplgr2vr_b(limit_ptr); + vec0 = __lsx_vreplgr2vr_b(limit_ptr >> 8); + limit = __lsx_vilvl_d(vec0, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + LSX_DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, + vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + LSX_DUP2_ARG2(__lsx_vilvh_b, p0_out, p1_out, q1_out, q0_out, + vec0, vec1); + vec4 = __lsx_vilvl_h(vec1, vec0); + vec5 = __lsx_vilvh_h(vec1, vec0); + + dst -= 2; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_w(vec2, dst + stride, 0, 1); + __lsx_vstelm_w(vec2, dst + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_w(vec3, dst + stride, 0, 1); + __lsx_vstelm_w(vec3, dst + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_w(vec4, dst + stride, 0, 1); + __lsx_vstelm_w(vec4, dst + stride2, 0, 2); + __lsx_vstelm_w(vec4, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec5, dst, 0, 0); + __lsx_vstelm_w(vec5, dst + stride, 0, 1); + __lsx_vstelm_w(vec5, dst + stride2, 0, 2); + __lsx_vstelm_w(vec5, dst + stride3, 0, 3); + } else { + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_h, p2_h, p1_h, p0_h); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_h, q1_h, q2_h, q3_h); + + /* filter8 */ + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, + p1_filt8_l, p0_filt8_h, p0_filt8_l, q0_filt8_h, + q0_filt8_l, p2_filt8_l, p1_filt8_l, p0_filt8_l, + q0_filt8_l); + LSX_DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, + q2_filt8_l, q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2 = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1 = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0 = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0 = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1 = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2 = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + LSX_DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec3 = __lsx_vilvl_h(vec1, vec0); + vec4 = __lsx_vilvh_h(vec1, vec0); + LSX_DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, vec0, vec1); + vec6 = __lsx_vilvl_h(vec1, vec0); + vec7 = __lsx_vilvh_h(vec1, vec0); + vec2 = __lsx_vilvl_b(q2, q1); + vec5 = __lsx_vilvh_b(q2, q1); + + dst -= 3; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 7); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 7); + } +} + +void ff_loop_filter_h_84_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p1_out, p0_out, q0_out, q1_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i row4, row5, row6, row7, row12, row13, row14, row15; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i zero = __lsx_vldi(0); + + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, p0, p1, p2, p3); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row4, row5, row6, row7); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, q3, q2, q1, q0); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row12, row13, row14, row15); + + /* transpose 16x8 matrix into 8x16 */ + LSX_TRANSPOSE16x8_B(p0, p1, p2, p3, row4, row5, row6, row7, + q3, q2, q1, q0, row12, row13, row14, row15, + p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + vec0 = __lsx_vreplgr2vr_b(thresh_ptr >> 8); + thresh = __lsx_vilvl_d(vec0, thresh); + + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + vec0 = __lsx_vreplgr2vr_b(b_limit_ptr >> 8); + b_limit = __lsx_vilvl_d(vec0, b_limit); + + limit = __lsx_vreplgr2vr_b(limit_ptr); + vec0 = __lsx_vreplgr2vr_b(limit_ptr >> 8); + limit = __lsx_vilvl_d(vec0, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + LSX_DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + LSX_DUP2_ARG2(__lsx_vilvh_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec4 = __lsx_vilvl_h(vec1, vec0); + vec5 = __lsx_vilvh_h(vec1, vec0); + + dst -= 2; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_w(vec2, dst + stride, 0, 1); + __lsx_vstelm_w(vec2, dst + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_w(vec3, dst + stride, 0, 1); + __lsx_vstelm_w(vec3, dst + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_w(vec4, dst + stride, 0, 1); + __lsx_vstelm_w(vec4, dst + stride2, 0, 2); + __lsx_vstelm_w(vec4, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec5, dst, 0, 0); + __lsx_vstelm_w(vec5, dst + stride, 0, 1); + __lsx_vstelm_w(vec5, dst + stride2, 0, 2); + __lsx_vstelm_w(vec5, dst + stride3, 0, 3); + } else { + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, p2_filt8_l, p2_filt8_l, p1_filt8_l, + p1_filt8_l, p0_filt8_l, p0_filt8_l, q0_filt8_l, + q0_filt8_l, p2_filt8_l, p1_filt8_l, p0_filt8_l, + q0_filt8_l); + LSX_DUP2_ARG2(__lsx_vpickev_b, q1_filt8_l, q1_filt8_l, q2_filt8_l, + q2_filt8_l, q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2 = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1 = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0 = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0 = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1 = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2 = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + LSX_DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec3 = __lsx_vilvl_h(vec1, vec0); + vec4 = __lsx_vilvh_h(vec1, vec0); + LSX_DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, vec0, vec1); + vec6 = __lsx_vilvl_h(vec1, vec0); + vec7 = __lsx_vilvh_h(vec1, vec0); + vec2 = __lsx_vilvl_b(q2, q1); + vec5 = __lsx_vilvh_b(q2, q1); + + dst -= 3; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 7); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 7); + } +} + +void ff_loop_filter_h_48_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p1_out, p0_out, q0_out, q1_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i row4, row5, row6, row7, row12, row13, row14, row15; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i zero = __lsx_vldi(0); + + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, p0, p1, p2, p3); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row4, row5, row6, row7); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, q3, q2, q1, q0); + dst_tmp += stride4; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, -4, dst_tmp + stride, -4, dst_tmp + stride2, -4, + dst_tmp + stride3, -4, row12, row13, row14, row15); + + /* transpose 16x8 matrix into 8x16 */ + LSX_TRANSPOSE16x8_B(p0, p1, p2, p3, row4, row5, row6, row7, + q3, q2, q1, q0, row12, row13, row14, row15, + p3, p2, p1, p0, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + vec0 = __lsx_vreplgr2vr_b(thresh_ptr >> 8); + thresh = __lsx_vilvl_d(vec0, thresh); + + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + vec0 = __lsx_vreplgr2vr_b(b_limit_ptr >> 8); + b_limit = __lsx_vilvl_d(vec0, b_limit); + + limit = __lsx_vreplgr2vr_b(limit_ptr); + vec0 = __lsx_vreplgr2vr_b(limit_ptr >> 8); + limit = __lsx_vilvl_d(vec0, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + flat = __lsx_vilvh_d(flat, zero); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + LSX_DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + LSX_DUP2_ARG2(__lsx_vilvh_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec4 = __lsx_vilvl_h(vec1, vec0); + vec5 = __lsx_vilvh_h(vec1, vec0); + + dst -= 2; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_w(vec2, dst + stride, 0, 1); + __lsx_vstelm_w(vec2, dst + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_w(vec3, dst + stride, 0, 1); + __lsx_vstelm_w(vec3, dst + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_w(vec4, dst + stride, 0, 1); + __lsx_vstelm_w(vec4, dst + stride2, 0, 2); + __lsx_vstelm_w(vec4, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec5, dst, 0, 0); + __lsx_vstelm_w(vec5, dst + stride, 0, 1); + __lsx_vstelm_w(vec5, dst + stride2, 0, 2); + __lsx_vstelm_w(vec5, dst + stride3, 0, 3); + } else { + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_h, p2_h, p1_h, p0_h); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_h, q1_h, q2_h, q3_h); + + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_h, p1_filt8_h, + p1_filt8_h, p0_filt8_h, p0_filt8_h, q0_filt8_h, + q0_filt8_h, p2_filt8_h, p1_filt8_h, p0_filt8_h, + q0_filt8_h); + LSX_DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_h, q2_filt8_h, + q2_filt8_h, q1_filt8_h, q2_filt8_h); + + /* store pixel values */ + p2 = __lsx_vbitsel_v(p2, p2_filt8_h, flat); + p1 = __lsx_vbitsel_v(p1_out, p1_filt8_h, flat); + p0 = __lsx_vbitsel_v(p0_out, p0_filt8_h, flat); + q0 = __lsx_vbitsel_v(q0_out, q0_filt8_h, flat); + q1 = __lsx_vbitsel_v(q1_out, q1_filt8_h, flat); + q2 = __lsx_vbitsel_v(q2, q2_filt8_h, flat); + + LSX_DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec3 = __lsx_vilvl_h(vec1, vec0); + vec4 = __lsx_vilvh_h(vec1, vec0); + LSX_DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, vec0, vec1); + vec6 = __lsx_vilvl_h(vec1, vec0); + vec7 = __lsx_vilvh_h(vec1, vec0); + vec2 = __lsx_vilvl_b(q2, q1); + vec5 = __lsx_vilvh_b(q2, q1); + + dst -= 3; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 7); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 7); + } +} + +static void vp9_transpose_16x8_to_8x16(uint8_t *input, int32_t in_pitch, + uint8_t *output, int32_t out_pitch) +{ + __m128i p7_org, p6_org, p5_org, p4_org, p3_org, p2_org, p1_org, p0_org; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + + LSX_LD_8(input, in_pitch, + p7_org, p6_org, p5_org, p4_org, p3_org, p2_org, p1_org, p0_org); + /* 8x8 transpose */ + LSX_TRANSPOSE8x8_B(p7_org, p6_org, p5_org, p4_org, p3_org, p2_org, p1_org, + p0_org, p7, p6, p5, p4, p3, p2, p1, p0); + /* 8x8 transpose */ + LSX_DUP4_ARG2(__lsx_vilvh_b, p5_org, p7_org, p4_org, p6_org, p1_org, + p3_org, p0_org, p2_org, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vilvl_b, tmp1, tmp0, tmp3, tmp2, tmp4, tmp6); + LSX_DUP2_ARG2(__lsx_vilvh_b, tmp1, tmp0, tmp3, tmp2, tmp5, tmp7); + LSX_DUP2_ARG2(__lsx_vilvl_w, tmp6, tmp4, tmp7, tmp5, q0, q4); + LSX_DUP2_ARG2(__lsx_vilvh_w, tmp6, tmp4, tmp7, tmp5, q2, q6); + LSX_DUP4_ARG2(__lsx_vbsrl_v, q0, 8, q2, 8, q4, 8, q6, 8, q1, q3, q5, q7); + + LSX_ST_8(p7, p6, p5, p4, p3, p2, p1, p0, output, out_pitch); + output += out_pitch; + LSX_ST_8(q0, q1, q2, q3, q4, q5, q6, q7, output, out_pitch); +} + +static void vp9_transpose_8x16_to_16x8(uint8_t *input, int32_t in_pitch, + uint8_t *output, int32_t out_pitch) +{ + __m128i p7_o, p6_o, p5_o, p4_o, p3_o, p2_o, p1_o, p0_o; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + + LSX_LD_8(input, in_pitch, p7, p6, p5, p4, p3, p2, p1, p0); + input += in_pitch; + LSX_LD_8(input, in_pitch, q0, q1, q2, q3, q4, q5, q6, q7); + LSX_TRANSPOSE16x8_B(p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, + q6, q7, p7_o, p6_o, p5_o, p4_o, p3_o, p2_o, p1_o, p0_o); + LSX_ST_8(p7_o, p6_o, p5_o, p4_o, p3_o, p2_o, p1_o, p0_o, output, out_pitch); +} + +static void vp9_transpose_16x16(uint8_t *input, int32_t in_stride, + uint8_t *output, int32_t out_stride) +{ + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i row8, row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp4, tmp5, tmp6, tmp7; + __m128i tmp2, tmp3; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + + LSX_LD_8(input, in_stride, row0, row1, row2, row3, row4, row5, row6, row7); + input += in_stride; + LSX_LD_8(input, in_stride, + row8, row9, row10, row11, row12, row13, row14, row15); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, + row8, row9, row10, row11, row12, row13, row14, row15, + p7, p6, p5, p4, p3, p2, p1, p0); + + /* transpose 16x8 matrix into 8x16 */ + /* total 8 intermediate register and 32 instructions */ + q7 = __lsx_vpackod_d(row8, row0); + q6 = __lsx_vpackod_d(row9, row1); + q5 = __lsx_vpackod_d(row10, row2); + q4 = __lsx_vpackod_d(row11, row3); + q3 = __lsx_vpackod_d(row12, row4); + q2 = __lsx_vpackod_d(row13, row5); + q1 = __lsx_vpackod_d(row14, row6); + q0 = __lsx_vpackod_d(row15, row7); + + LSX_DUP2_ARG2(__lsx_vpackev_b, q6, q7, q4, q5, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vpackod_b, q6, q7, q4, q5, tmp4, tmp5); + + LSX_DUP2_ARG2(__lsx_vpackev_b, q2, q3, q0, q1, q5, q7); + LSX_DUP2_ARG2(__lsx_vpackod_b, q2, q3, q0, q1, tmp6, tmp7); + + LSX_DUP2_ARG2(__lsx_vpackev_h, tmp1, tmp0, q7, q5, tmp2, tmp3); + q0 = __lsx_vpackev_w(tmp3, tmp2); + q4 = __lsx_vpackod_w(tmp3, tmp2); + + tmp2 = __lsx_vpackod_h(tmp1, tmp0); + tmp3 = __lsx_vpackod_h(q7, q5); + q2 = __lsx_vpackev_w(tmp3, tmp2); + q6 = __lsx_vpackod_w(tmp3, tmp2); + + LSX_DUP2_ARG2(__lsx_vpackev_h, tmp5, tmp4, tmp7, tmp6, tmp2, tmp3); + q1 = __lsx_vpackev_w(tmp3, tmp2); + q5 = __lsx_vpackod_w(tmp3, tmp2); + + tmp2 = __lsx_vpackod_h(tmp5, tmp4); + tmp3 = __lsx_vpackod_h(tmp7, tmp6); + q3 = __lsx_vpackev_w(tmp3, tmp2); + q7 = __lsx_vpackod_w(tmp3, tmp2); + + LSX_ST_8(p7, p6, p5, p4, p3, p2, p1, p0, output, out_stride); + output += out_stride; + LSX_ST_8(q0, q1, q2, q3, q4, q5, q6, q7, output, out_stride); +} + +static int32_t vp9_vt_lpf_t4_and_t8_8w(uint8_t *src, uint8_t *filter48, + uint8_t *src_org, int32_t pitch_org, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i vec0, vec1, vec2, vec3; + __m128i zero = __lsx_vldi(0); + + /* load vector elements */ + LSX_DUP4_ARG2(__lsx_vld, src, -64, src, -48, src, -32, + src, -16, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, + q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + limit = __lsx_vreplgr2vr_b(limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + LSX_DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + + src_org -= 2; + __lsx_vstelm_w(vec2, src_org, 0, 0); + src_org += pitch_org; + __lsx_vstelm_w(vec2, src_org, 0, 1); + src_org += pitch_org; + __lsx_vstelm_w(vec2, src_org, 0, 2); + src_org += pitch_org; + __lsx_vstelm_w(vec2, src_org, 0, 3); + src_org += pitch_org; + __lsx_vstelm_w(vec3, src_org, 0, 0); + src_org += pitch_org; + __lsx_vstelm_w(vec3, src_org, 0, 1); + src_org += pitch_org; + __lsx_vstelm_w(vec3, src_org, 0, 2); + src_org += pitch_org; + __lsx_vstelm_w(vec3, src_org, 0, 3); + return 1; + } else { + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + /* convert 16 bit output data into 8 bit */ + p2_l = __lsx_vpickev_b(p2_filt8_l, p2_filt8_l); + p1_l = __lsx_vpickev_b(p1_filt8_l, p1_filt8_l); + p0_l = __lsx_vpickev_b(p0_filt8_l, p0_filt8_l); + q0_l = __lsx_vpickev_b(q0_filt8_l, q0_filt8_l); + q1_l = __lsx_vpickev_b(q1_filt8_l, q1_filt8_l); + q2_l = __lsx_vpickev_b(q2_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_l, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_l, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_l, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_l, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_l, flat); + q2_out = __lsx_vbitsel_v(q2, q2_l, flat); + + __lsx_vst(p2_out, filter48, 0); + __lsx_vst(p1_out, filter48, 16); + __lsx_vst(p0_out, filter48, 32); + __lsx_vst(q0_out, filter48, 48); + __lsx_vst(q1_out, filter48, 64); + __lsx_vst(q2_out, filter48, 80); + __lsx_vst(flat, filter48, 96); + + return 0; + } +} + +static int32_t vp9_vt_lpf_t16_8w(uint8_t *dst, uint8_t *dst_org, + ptrdiff_t stride, + uint8_t *filter48) +{ + __m128i zero = __lsx_vldi(0); + __m128i filter8, flat, flat2; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + v8u16 p7_l_in, p6_l_in, p5_l_in, p4_l_in; + v8u16 p3_l_in, p2_l_in, p1_l_in, p0_l_in; + v8u16 q7_l_in, q6_l_in, q5_l_in, q4_l_in; + v8u16 q3_l_in, q2_l_in, q1_l_in, q0_l_in; + v8u16 tmp0_l, tmp1_l; + __m128i out_l; + uint8_t *dst_tmp = dst - 128; + + /* load vector elements */ + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, + dst_tmp, 48, p7, p6, p5, p4); + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, 64, dst_tmp, 80, dst_tmp, 96, + dst_tmp, 112, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, + dst, 48, q0, q1, q2, q3); + LSX_DUP4_ARG2(__lsx_vld, dst, 64, dst, 80, dst, 96, + dst, 112, q4, q5, q6, q7); + + flat = __lsx_vld(filter48, 96); + + + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + /* if flat2 is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat2)) { + __m128i vec0, vec1, vec2, vec3, vec4; + + LSX_DUP4_ARG2(__lsx_vld, filter48, 0, filter48, 16, filter48, 32, + filter48, 48, p2, p1, p0, q0); + LSX_DUP2_ARG2(__lsx_vld, filter48, 64, filter48, 80, q1, q2); + + LSX_DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec3 = __lsx_vilvl_h(vec1, vec0); + vec4 = __lsx_vilvh_h(vec1, vec0); + vec2 = __lsx_vilvl_b(q2, q1); + + dst_org -= 3; + __lsx_vstelm_w(vec3, dst_org, 0, 0); + __lsx_vstelm_h(vec2, dst_org, 4, 0); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 1); + __lsx_vstelm_h(vec2, dst_org, 4, 1); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 2); + __lsx_vstelm_h(vec2, dst_org, 4, 2); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 3); + __lsx_vstelm_h(vec2, dst_org, 4, 3); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 0); + __lsx_vstelm_h(vec2, dst_org, 4, 4); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 1); + __lsx_vstelm_h(vec2, dst_org, 4, 5); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 2); + __lsx_vstelm_h(vec2, dst_org, 4, 6); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 3); + __lsx_vstelm_h(vec2, dst_org, 4, 7); + return 1; + } else { + dst -= 7 * 16; + + p7_l_in = (v8u16)__lsx_vilvl_b(zero, p7); + p6_l_in = (v8u16)__lsx_vilvl_b(zero, p6); + p5_l_in = (v8u16)__lsx_vilvl_b(zero, p5); + p4_l_in = (v8u16)__lsx_vilvl_b(zero, p4); + p3_l_in = (v8u16)__lsx_vilvl_b(zero, p3); + p2_l_in = (v8u16)__lsx_vilvl_b(zero, p2); + p1_l_in = (v8u16)__lsx_vilvl_b(zero, p1); + p0_l_in = (v8u16)__lsx_vilvl_b(zero, p0); + q0_l_in = (v8u16)__lsx_vilvl_b(zero, q0); + + tmp0_l = p7_l_in << 3; + tmp0_l -= p7_l_in; + tmp0_l += p6_l_in; + tmp0_l += q0_l_in; + tmp1_l = p6_l_in + p5_l_in; + tmp1_l += p4_l_in; + tmp1_l += p3_l_in; + tmp1_l += p2_l_in; + tmp1_l += p1_l_in; + tmp1_l += p0_l_in; + tmp1_l += tmp0_l; + + out_l =__lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l =__lsx_vpickev_b(out_l, out_l); + p6 = __lsx_vbitsel_v(p6, out_l, flat2); + __lsx_vstelm_d(p6, dst, 0, 0); + dst += 16; + + /* p5 */ + q1_l_in = (v8u16)__lsx_vilvl_b(zero, q1); + tmp0_l = p5_l_in - p6_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + p5 = __lsx_vbitsel_v(p5, out_l, flat2); + __lsx_vstelm_d(p5, dst, 0, 0); + dst += 16; + + /* p4 */ + q2_l_in = (v8u16)__lsx_vilvl_b(zero, q2); + tmp0_l = p4_l_in - p5_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + p4 = __lsx_vbitsel_v(p4, out_l, flat2); + __lsx_vstelm_d(p4, dst, 0, 0); + dst += 16; + + /* p3 */ + q3_l_in = (v8u16)__lsx_vilvl_b(zero, q3); + tmp0_l = p3_l_in - p4_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + p3 = __lsx_vbitsel_v(p3, out_l, flat2); + __lsx_vstelm_d(p3, dst, 0, 0); + dst += 16; + + /* p2 */ + q4_l_in = (v8u16)__lsx_vilvl_b(zero, q4); + filter8 = __lsx_vld(filter48, 0); + tmp0_l = p2_l_in - p3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vstelm_d(filter8, dst, 0, 0); + dst += 16; + + /* p1 */ + q5_l_in = (v8u16)__lsx_vilvl_b(zero, q5); + filter8 = __lsx_vld(filter48, 16); + tmp0_l = p1_l_in - p2_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vstelm_d(filter8, dst, 0, 0); + dst += 16; + + /* p0 */ + q6_l_in = (v8u16)__lsx_vilvl_b(zero, q6); + filter8 = __lsx_vld(filter48, 32); + tmp0_l = p0_l_in - p1_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vstelm_d(filter8, dst, 0, 0); + dst += 16; + + /* q0 */ + q7_l_in = (v8u16)__lsx_vilvl_b(zero, q7); + filter8 = __lsx_vld(filter48, 48); + tmp0_l = q7_l_in - p0_l_in; + tmp0_l += q0_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((v8i16) tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vstelm_d(filter8, dst, 0, 0); + dst += 16; + + /* q1 */ + filter8 = __lsx_vld(filter48, 64); + tmp0_l = q7_l_in - q0_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p6_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vstelm_d(filter8, dst, 0, 0); + dst += 16; + + /* q2 */ + filter8 = __lsx_vld(filter48, 80); + tmp0_l = q7_l_in - q1_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p5_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vstelm_d(filter8, dst, 0, 0); + dst += 16; + + /* q3 */ + tmp0_l = q7_l_in - q2_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p4_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + q3 = __lsx_vbitsel_v(q3, out_l, flat2); + __lsx_vstelm_d(q3, dst, 0, 0); + dst += 16; + + /* q4 */ + tmp0_l = q7_l_in - q3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p3_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + q4 = __lsx_vbitsel_v(q4, out_l, flat2); + __lsx_vstelm_d(q4, dst, 0, 0); + dst += 16; + + /* q5 */ + tmp0_l = q7_l_in - q4_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p2_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + q5 = __lsx_vbitsel_v(q5, out_l, flat2); + __lsx_vstelm_d(q5, dst, 0, 0); + dst += 16; + + /* q6 */ + tmp0_l = q7_l_in - q5_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p1_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + out_l = __lsx_vpickev_b(out_l, out_l); + q6 = __lsx_vbitsel_v(q6, out_l, flat2); + __lsx_vstelm_d(q6, dst, 0, 0); + + return 0; + } +} + +void ff_loop_filter_h_16_8_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + uint8_t early_exit = 0; + uint8_t transposed_input[16 * 24] __attribute__ ((aligned(16))); + uint8_t *filter48 = &transposed_input[16 * 16]; + + vp9_transpose_16x8_to_8x16(dst - 8, stride, transposed_input, 16); + + early_exit = vp9_vt_lpf_t4_and_t8_8w((transposed_input + 16 * 8), + &filter48[0], dst, stride, + b_limit_ptr, limit_ptr, thresh_ptr); + + if (0 == early_exit) { + early_exit = vp9_vt_lpf_t16_8w((transposed_input + 16 * 8), dst, stride, + &filter48[0]); + + if (0 == early_exit) { + vp9_transpose_8x16_to_16x8(transposed_input, 16, dst - 8, stride); + } + } +} + +static int32_t vp9_vt_lpf_t4_and_t8_16w(uint8_t *dst, uint8_t *filter48, + uint8_t *dst_org, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + ptrdiff_t stride2 = stride << 1; + ptrdiff_t stride3 = stride2 + stride; + ptrdiff_t stride4 = stride2 << 1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i vec0, vec1, vec2, vec3, vec4, vec5; + __m128i zero = __lsx_vldi(0); + + /* load vector elements */ + LSX_DUP4_ARG2(__lsx_vld, dst, -64, dst, -48, dst, -32, + dst, -16, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, + dst, 48, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(b_limit_ptr); + limit = __lsx_vreplgr2vr_b(limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, + hev, mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + LSX_DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + LSX_DUP2_ARG2(__lsx_vilvh_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec4 = __lsx_vilvl_h(vec1, vec0); + vec5 = __lsx_vilvh_h(vec1, vec0); + + dst_org -= 2; + __lsx_vstelm_w(vec2, dst_org, 0, 0); + __lsx_vstelm_w(vec2, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec2, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst_org + stride3, 0, 3); + dst_org += stride4; + __lsx_vstelm_w(vec3, dst_org, 0, 0); + __lsx_vstelm_w(vec3, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec3, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst_org + stride3, 0, 3); + dst_org += stride4; + __lsx_vstelm_w(vec4, dst_org, 0, 0); + __lsx_vstelm_w(vec4, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec4, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec4, dst_org + stride3, 0, 3); + dst_org += stride4; + __lsx_vstelm_w(vec5, dst_org, 0, 0); + __lsx_vstelm_w(vec5, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec5, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec5, dst_org + stride3, 0, 3); + + return 1; + } else { + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_l, p2_l, p1_l, p0_l); + LSX_DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_l, q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, + p3_h, p2_h, p1_h, p0_h); + LSX_DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, + q0_h, q1_h, q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + LSX_DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, + p1_filt8_l, p0_filt8_h, p0_filt8_l, q0_filt8_h, + q0_filt8_l, p2_filt8_l, p1_filt8_l, p0_filt8_l, + q0_filt8_l); + LSX_DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, + q2_filt8_l, q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + __lsx_vst(p2_out, filter48, 0); + __lsx_vst(p1_out, filter48, 16); + __lsx_vst(p0_out, filter48, 32); + __lsx_vst(q0_out, filter48, 48); + __lsx_vst(q1_out, filter48, 64); + __lsx_vst(q2_out, filter48, 80); + __lsx_vst(flat, filter48, 96); + + return 0; + } +} + +static int32_t vp9_vt_lpf_t16_16w(uint8_t *dst, uint8_t *dst_org, + ptrdiff_t stride, + uint8_t *filter48) +{ + __m128i zero = __lsx_vldi(0); + __m128i flat, flat2, filter8; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + v8u16 p7_l_in, p6_l_in, p5_l_in, p4_l_in; + v8u16 p3_l_in, p2_l_in, p1_l_in, p0_l_in; + v8u16 q7_l_in, q6_l_in, q5_l_in, q4_l_in; + v8u16 q3_l_in, q2_l_in, q1_l_in, q0_l_in; + v8u16 p7_h_in, p6_h_in, p5_h_in, p4_h_in; + v8u16 p3_h_in, p2_h_in, p1_h_in, p0_h_in; + v8u16 q7_h_in, q6_h_in, q5_h_in, q4_h_in; + v8u16 q3_h_in, q2_h_in, q1_h_in, q0_h_in; + v8u16 tmp0_l, tmp1_l, tmp0_h, tmp1_h; + __m128i out_l, out_h; + uint8_t *dst_tmp = dst - 128; + + flat = __lsx_vld(filter48, 96); + + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, + dst_tmp, 48, p7, p6, p5, p4); + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, 64, dst_tmp, 80, dst_tmp, 96, + dst_tmp, 112, p3, p2, p1, p0); + LSX_DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, + dst, 48, q0, q1, q2, q3); + LSX_DUP4_ARG2(__lsx_vld, dst, 64, dst, 80, dst, 96, + dst, 112, q4, q5, q6, q7); + + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + /* if flat2 is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat2)) { + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + + LSX_DUP4_ARG2(__lsx_vld, filter48, 0, filter48, 16, filter48, 32, + filter48, 48, p2, p1, p0, q0); + LSX_DUP2_ARG2(__lsx_vld, filter48, 64, filter48, 80, q1, q2); + + LSX_DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec3 = __lsx_vilvl_h(vec1, vec0); + vec4 = __lsx_vilvh_h(vec1, vec0); + LSX_DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, vec0, vec1); + vec6 = __lsx_vilvl_h(vec1, vec0); + vec7 = __lsx_vilvh_h(vec1, vec0); + vec2 = __lsx_vilvl_b(q2, q1); + vec5 = __lsx_vilvh_b(q2, q1); + + dst_org -= 3; + __lsx_vstelm_w(vec3, dst_org, 0, 0); + __lsx_vstelm_h(vec2, dst_org, 4, 0); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 1); + __lsx_vstelm_h(vec2, dst_org, 4, 1); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 2); + __lsx_vstelm_h(vec2, dst_org, 4, 2); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 3); + __lsx_vstelm_h(vec2, dst_org, 4, 3); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 0); + __lsx_vstelm_h(vec2, dst_org, 4, 4); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 1); + __lsx_vstelm_h(vec2, dst_org, 4, 5); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 2); + __lsx_vstelm_h(vec2, dst_org, 4, 6); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 3); + __lsx_vstelm_h(vec2, dst_org, 4, 7); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 0); + __lsx_vstelm_h(vec5, dst_org, 4, 0); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 1); + __lsx_vstelm_h(vec5, dst_org, 4, 1); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 2); + __lsx_vstelm_h(vec5, dst_org, 4, 2); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 3); + __lsx_vstelm_h(vec5, dst_org, 4, 3); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 0); + __lsx_vstelm_h(vec5, dst_org, 4, 4); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 1); + __lsx_vstelm_h(vec5, dst_org, 4, 5); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 2); + __lsx_vstelm_h(vec5, dst_org, 4, 6); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 3); + __lsx_vstelm_h(vec5, dst_org, 4, 7); + + return 1; + } else { + dst -= 7 * 16; + + p7_l_in = (v8u16)__lsx_vilvl_b(zero, p7); + p6_l_in = (v8u16)__lsx_vilvl_b(zero, p6); + p5_l_in = (v8u16)__lsx_vilvl_b(zero, p5); + p4_l_in = (v8u16)__lsx_vilvl_b(zero, p4); + p3_l_in = (v8u16)__lsx_vilvl_b(zero, p3); + p2_l_in = (v8u16)__lsx_vilvl_b(zero, p2); + p1_l_in = (v8u16)__lsx_vilvl_b(zero, p1); + p0_l_in = (v8u16)__lsx_vilvl_b(zero, p0); + q0_l_in = (v8u16)__lsx_vilvl_b(zero, q0); + + tmp0_l = p7_l_in << 3; + tmp0_l -= p7_l_in; + tmp0_l += p6_l_in; + tmp0_l += q0_l_in; + tmp1_l = p6_l_in + p5_l_in; + tmp1_l += p4_l_in; + tmp1_l += p3_l_in; + tmp1_l += p2_l_in; + tmp1_l += p1_l_in; + tmp1_l += p0_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + p7_h_in = (v8u16)__lsx_vilvh_b(zero, p7); + p6_h_in = (v8u16)__lsx_vilvh_b(zero, p6); + p5_h_in = (v8u16)__lsx_vilvh_b(zero, p5); + p4_h_in = (v8u16)__lsx_vilvh_b(zero, p4); + p3_h_in = (v8u16)__lsx_vilvh_b(zero, p3); + p2_h_in = (v8u16)__lsx_vilvh_b(zero, p2); + p1_h_in = (v8u16)__lsx_vilvh_b(zero, p1); + p0_h_in = (v8u16)__lsx_vilvh_b(zero, p0); + q0_h_in = (v8u16)__lsx_vilvh_b(zero, q0); + + tmp0_h = p7_h_in << 3; + tmp0_h -= p7_h_in; + tmp0_h += p6_h_in; + tmp0_h += q0_h_in; + tmp1_h = p6_h_in + p5_h_in; + tmp1_h += p4_h_in; + tmp1_h += p3_h_in; + tmp1_h += p2_h_in; + tmp1_h += p1_h_in; + tmp1_h += p0_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p6 = __lsx_vbitsel_v(p6, out_l, flat2); + __lsx_vst(p6, dst, 0); + + /* p5 */ + q1_l_in = (v8u16)__lsx_vilvl_b(zero, q1); + tmp0_l = p5_l_in - p6_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q1_h_in = (v8u16)__lsx_vilvh_b(zero, q1); + tmp0_h = p5_h_in - p6_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + p5 = __lsx_vbitsel_v(p5, out_l, flat2); + __lsx_vst(p5, dst, 16); + + /* p4 */ + q2_l_in = (v8u16)__lsx_vilvl_b(zero, q2); + tmp0_l = p4_l_in - p5_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q2_h_in = (v8u16)__lsx_vilvh_b(zero, q2); + tmp0_h = p4_h_in - p5_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + p4 = __lsx_vbitsel_v(p4, out_l, flat2); + __lsx_vst(p4, dst, 16*2); + + /* p3 */ + q3_l_in = (v8u16)__lsx_vilvl_b(zero, q3); + tmp0_l = p3_l_in - p4_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q3_h_in = (v8u16)__lsx_vilvh_b(zero, q3); + tmp0_h = p3_h_in - p4_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + p3 = __lsx_vbitsel_v(p3, out_l, flat2); + __lsx_vst(p3, dst, 16*3); + + /* p2 */ + q4_l_in = (v8u16)__lsx_vilvl_b(zero, q4); + filter8 = __lsx_vld(filter48, 0); + tmp0_l = p2_l_in - p3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q4_h_in = (v8u16)__lsx_vilvh_b(zero, q4); + tmp0_h = p2_h_in - p3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16*4); + + /* p1 */ + q5_l_in = (v8u16)__lsx_vilvl_b(zero, q5); + filter8 = __lsx_vld(filter48, 16); + tmp0_l = p1_l_in - p2_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q5_h_in = (v8u16)__lsx_vilvh_b(zero, q5); + tmp0_h = p1_h_in - p2_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)(tmp1_h), 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16*5); + + /* p0 */ + q6_l_in = (v8u16)__lsx_vilvl_b(zero, q6); + filter8 = __lsx_vld(filter48, 32); + tmp0_l = p0_l_in - p1_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q6_h_in = (v8u16)__lsx_vilvh_b(zero, q6); + tmp0_h = p0_h_in - p1_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16*6); + + /* q0 */ + q7_l_in = (v8u16)__lsx_vilvl_b(zero, q7); + filter8 = __lsx_vld(filter48, 48); + tmp0_l = q7_l_in - p0_l_in; + tmp0_l += q0_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q7_h_in = (v8u16)__lsx_vilvh_b(zero, q7); + tmp0_h = q7_h_in - p0_h_in; + tmp0_h += q0_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16*7); + + /* q1 */ + filter8 = __lsx_vld(filter48, 64); + tmp0_l = q7_l_in - q0_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p6_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q0_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p6_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16*8); + + /* q2 */ + filter8 = __lsx_vld(filter48, 80); + tmp0_l = q7_l_in - q1_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p5_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q1_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p5_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16*9); + + /* q3 */ + tmp0_l = q7_l_in - q2_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p4_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q2_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p4_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q3 = __lsx_vbitsel_v(q3, out_l, flat2); + __lsx_vst(q3, dst, 16*10); + + /* q4 */ + tmp0_l = q7_l_in - q3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p3_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p3_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q4 = __lsx_vbitsel_v(q4, out_l, flat2); + __lsx_vst(q4, dst, 16*11); + + /* q5 */ + tmp0_l = q7_l_in - q4_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p2_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q4_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p2_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q5 = __lsx_vbitsel_v(q5, out_l, flat2); + __lsx_vst(q5, dst, 16*12); + + /* q6 */ + tmp0_l = q7_l_in - q5_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p1_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q5_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p1_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q6 = __lsx_vbitsel_v(q6, out_l, flat2); + __lsx_vst(q6, dst, 16*13); + + return 0; + } +} + +void ff_loop_filter_h_16_16_lsx(uint8_t *dst, ptrdiff_t stride, + int32_t b_limit_ptr, + int32_t limit_ptr, + int32_t thresh_ptr) +{ + uint8_t early_exit = 0; + uint8_t transposed_input[16 * 24] __attribute__ ((aligned(16))); + uint8_t *filter48 = &transposed_input[16 * 16]; + + vp9_transpose_16x16((dst - 8), stride, &transposed_input[0], 16); + + early_exit = vp9_vt_lpf_t4_and_t8_16w((transposed_input + 16 * 8), + &filter48[0], dst, stride, + b_limit_ptr, limit_ptr, thresh_ptr); + + if (0 == early_exit) { + early_exit = vp9_vt_lpf_t16_16w((transposed_input + 16 * 8), dst, stride, + &filter48[0]); + + if (0 == early_exit) { + vp9_transpose_16x16(transposed_input, 16, (dst - 8), stride); + } + } +} + diff --git a/libavcodec/loongarch/vp9_mc_lsx.c b/libavcodec/loongarch/vp9_mc_lsx.c new file mode 100644 index 0000000000..02ee916447 --- /dev/null +++ b/libavcodec/loongarch/vp9_mc_lsx.c @@ -0,0 +1,2318 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavcodec/vp9dsp.h" +#include "libavutil/loongarch/generic_macros_lsx.h" +#include "vp9dsp_loongarch.h" + +static const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + + +#define HORIZ_8TAP_4WID_4VECS_FILT(_src0, _src1, _src2, _src3, \ + _mask0, _mask1, _mask2, _mask3, \ + _filter0, _filter1, _filter2, _filter3, \ + _out0, _out1) \ +{ \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _reg0, _reg1, _reg2, _reg3; \ + \ + LSX_DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask0, \ + _src3, _src2, _mask0, _tmp0, _tmp1); \ + LSX_DUP2_ARG2(__lsx_dp2_h_b, _tmp0, _filter0, _tmp1, _filter0, _reg0, _reg1); \ + LSX_DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask1, \ + _src3, _src2, _mask1, _tmp2, _tmp3); \ + LSX_DUP2_ARG3(__lsx_dp2add_h_b, _reg0, _tmp2, _filter1, \ + _reg1, _tmp3, _filter1, _reg0, _reg1); \ + LSX_DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask2, \ + _src3, _src2, _mask2, _tmp4, _tmp5); \ + LSX_DUP2_ARG2(__lsx_dp2_h_b, _tmp4, _filter2, _tmp5, _filter2, _reg2, _reg3); \ + LSX_DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask3, \ + _src3, _src2, _mask3, _tmp6, _tmp7); \ + LSX_DUP2_ARG3(__lsx_dp2add_h_b, _reg2, _tmp6, _filter3, \ + _reg3, _tmp7, _filter3, _reg2, _reg3); \ + LSX_DUP2_ARG2(__lsx_vsadd_h, _reg0, _reg2, _reg1, _reg3, _out0, _out1); \ +} + +#define HORIZ_8TAP_8WID_4VECS_FILT(_src0, _src1, _src2, _src3, \ + _mask0, _mask1, _mask2, _mask3, \ + _filter0, _filter1, _filter2, _filter3, \ + _out0, _out1, _out2, _out3) \ +{ \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _reg0, _reg1, _reg2, _reg3, _reg4, _reg5, _reg6, _reg7; \ + \ + LSX_DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask0, _src1, _src1, _mask0, _src2, \ + _src2, _mask0, _src3, _src3, _mask0, _tmp0, _tmp1, _tmp2, _tmp3); \ + LSX_DUP4_ARG2(__lsx_dp2_h_b, _tmp0, _filter0, _tmp1, _filter0, _tmp2, _filter0, \ + _tmp3, _filter0, _reg0, _reg1, _reg2, _reg3); \ + LSX_DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask2, _src1, _src1, _mask2, _src2, \ + _src2, _mask2, _src3, _src3, _mask2, _tmp0, _tmp1, _tmp2, _tmp3); \ + LSX_DUP4_ARG2(__lsx_dp2_h_b, _tmp0, _filter2, _tmp1, _filter2, _tmp2, _filter2, \ + _tmp3, _filter2, _reg4, _reg5, _reg6, _reg7); \ + LSX_DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask1, _src1, _src1, _mask1, _src2, \ + _src2, _mask1, _src3, _src3, _mask1, _tmp4, _tmp5, _tmp6, _tmp7); \ + LSX_DUP4_ARG3(__lsx_dp2add_h_b, _reg0, _tmp4, _filter1, _reg1, _tmp5, _filter1, \ + _reg2, _tmp6, _filter1, _reg3, _tmp7, _filter1, _reg0, _reg1, _reg2, _reg3);\ + LSX_DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask3, _src1, _src1, _mask3, _src2, \ + _src2, _mask3, _src3, _src3, _mask3, _tmp4, _tmp5, _tmp6, _tmp7); \ + LSX_DUP4_ARG3(__lsx_dp2add_h_b, _reg4, _tmp4, _filter3, _reg5, _tmp5, _filter3, \ + _reg6, _tmp6, _filter3, _reg7, _tmp7, _filter3, _reg4, _reg5, _reg6, _reg7);\ + LSX_DUP4_ARG2(__lsx_vsadd_h, _reg0, _reg4, _reg1, _reg5, _reg2, _reg6, _reg3, _reg7, \ + _out0, _out1, _out2, _out3); \ +} + +#define FILT_8TAP_DPADD_S_H(_reg0, _reg1, _reg2, _reg3, \ + _filter0, _filter1, _filter2, _filter3) \ +( { \ + __m128i _vec0, _vec1; \ + \ + _vec0 = __lsx_dp2_h_b(_reg0, _filter0); \ + _vec0 = __lsx_dp2add_h_b(_vec0, _reg1, _filter1); \ + _vec1 = __lsx_dp2_h_b(_reg2, _filter2); \ + _vec1 = __lsx_dp2add_h_b(_vec1, _reg3, _filter3); \ + _vec0 = __lsx_vsadd_h(_vec0, _vec1); \ + \ + _vec0; \ +} ) + +#define HORIZ_8TAP_FILT(_src0, _src1, _mask0, _mask1, _mask2, _mask3, \ + _filt_h0, _filt_h1, _filt_h2, _filt_h3) \ +( { \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3; \ + __m128i _out; \ + \ + LSX_DUP4_ARG3(__lsx_vshuf_b, _src1, _src0, _mask0, _src1, _src0, _mask1, _src1,\ + _src0, _mask2, _src1, _src0, _mask3, _tmp0, _tmp1, _tmp2, _tmp3);\ + _out = FILT_8TAP_DPADD_S_H(_tmp0, _tmp1, _tmp2, _tmp3, _filt_h0, _filt_h1, \ + _filt_h2, _filt_h3); \ + _out = __lsx_vsrari_h(_out, 7); \ + _out = __lsx_vsat_h(_out, 7); \ + \ + _out; \ +} ) + +#define LSX_LD_4(_src, _stride, _src0, _src1, _src2, _src3) \ +{ \ + _src0 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _src1 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _src2 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _src3 = __lsx_vld(_src, 0); \ +} + +#define LSX_ST_4(_dst0, _dst1, _dst2, _dst3, _dst, _stride) \ +{ \ + __lsx_vst(_dst0, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst1, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst2, _dst, 0); \ + _dst += _stride; \ + __lsx_vst(_dst3, _dst, 0); \ +} + +#define LSX_LD_4_16(_src, _stride, _src0, _src1, _src2, _src3) \ +{ \ + _src0 = __lsx_vld(_src, 16); \ + _src += _stride; \ + _src1 = __lsx_vld(_src, 16); \ + _src += _stride; \ + _src2 = __lsx_vld(_src, 16); \ + _src += _stride; \ + _src3 = __lsx_vld(_src, 16); \ +} + +#define LSX_ST_4_16(_dst0, _dst1, _dst2, _dst3, _dst, _stride) \ +{ \ + __lsx_vst(_dst0, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst1, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst2, _dst, 16); \ + _dst += _stride; \ + __lsx_vst(_dst3, _dst, 16); \ +} + +static void common_hz_8t_4x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) +{ + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out, out0, out1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1); + out = __lsx_vssrarni_b_h(out1, out0, 7); + out = __lsx_vxori_b(out, 128); + __lsx_vstelm_w(out, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 3); +} + +static void common_hz_8t_4x8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) +{ + int32_t stride = src_stride << 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src += stride; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1); + src += stride; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src += stride; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 3); +} + +static void common_hz_8t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + if (height == 4) { + common_hz_8t_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_8t_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_8t_8x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) +{ + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); +} + +static void common_hz_8t_8x8mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt = height >> 2; + int32_t stride = src_stride << 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src1); + src += stride; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + src += stride; + } +} + +static void common_hz_8t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + if (height == 4) { + common_hz_8t_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_8t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_hz_8t_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt = height >> 1; + int32_t stride = src_stride << 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src + src_stride, 0, src0, src2); + LSX_DUP2_ARG2(__lsx_vld, src, 8, src + src_stride, 8, src1, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + dst += dst_stride; + __lsx_vst(out1, dst, 0); + dst += dst_stride; + src += stride; + } +} + +static void common_hz_8t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt = height >> 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + __m128i shuff = {0x0F0E0D0C0B0A0908, 0x1716151413121110}; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + + dst += dst_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_8t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + int32_t loop_cnt = height; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + __m128i shuff = {0x0F0E0D0C0B0A0908, 0x1716151413121110}; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + + LSX_DUP2_ARG2(__lsx_vld, src, 32, src, 48, src0, src2); + src3 = __lsx_vld(src, 56); + src1 = __lsx_vshuf_b(src2, src0, shuff); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 32); + __lsx_vst(out1, dst, 48); + src += src_stride; + dst += dst_stride; + } +} + +static void common_vt_8t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i reg0, reg1, reg2, reg3, reg4; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1; + + src -= (3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, tmp4, tmp5); + LSX_DUP2_ARG2(__lsx_vilvl_d, tmp3, tmp0, tmp4, tmp1, reg0, reg1); + reg2 = __lsx_vilvl_d(tmp5, tmp2); + LSX_DUP2_ARG2(__lsx_vxori_b, reg0, 128, reg1, 128, reg0, reg1); + reg2 = __lsx_vxori_b(reg2, 128); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, reg3, reg4); + LSX_DUP2_ARG2(__lsx_vxori_b, reg3, 128, reg4, 128, reg3, reg4); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, reg3, filter0, filter1, filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg1, reg2, reg3, reg4, filter0, filter1, filter2, filter3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + + reg0 = reg2; + reg1 = reg3; + reg2 = reg4; + src6 = src10; + } +} + +static void common_vt_8t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1, out2, out3; + + src -= (3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0, reg1, reg2, reg3); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, tmp0, filter0, filter1, filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, tmp1, filter0, filter1, filter2, filter3); + out2 = FILT_8TAP_DPADD_S_H(reg1, reg2, tmp0, tmp2, filter0, filter1, filter2, filter3); + out3 = FILT_8TAP_DPADD_S_H(reg4, reg5, tmp1, tmp3, filter0, filter1, filter2, filter3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + reg0 = reg2; + reg1 = tmp0; + reg2 = tmp2; + reg3 = reg5; + reg4 = tmp1; + reg5 = tmp3; + src6 = src10; + } +} + +static void common_vt_8t_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + uint32_t loop_cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filter0, filter1, filter2, filter3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i reg6, reg7, reg8, reg9, reg10, reg11; + __m128i tmp0, tmp1, tmp2, tmp3; + + src -= (3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0, reg1, reg2, reg3); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + LSX_DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, reg6, reg7, reg8, reg9); + LSX_DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src4, src5, src7, src8); + tmp0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, src0, filter0, filter1, filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, src1, filter0, filter1, filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg6, reg7, reg8, src4, filter0, filter1, filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg9, reg10, reg11, src5, filter0, filter1, filter2, filter3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + __lsx_vst(tmp1, dst, 0); + dst += dst_stride; + tmp0 = FILT_8TAP_DPADD_S_H(reg1, reg2, src0, src2, filter0, filter1, filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg4, reg5, src1, src3, filter0, filter1, filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg7, reg8, src4, src7, filter0, filter1, filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg10, reg11, src5, src8, filter0, filter1, filter2, filter3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + __lsx_vst(tmp1, dst, 0); + dst += dst_stride; + + reg0 = reg2; + reg1 = src0; + reg2 = src2; + reg3 = reg5; + reg4 = src1; + reg5 = src3; + reg6 = reg8; + reg7 = src4; + reg8 = src7; + reg9 = reg11; + reg10 = src5; + reg11 = src8; + src6 = src10; + } +} + +static void common_vt_8t_16w_mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height, + int32_t width) +{ + const uint8_t *src_tmp; + uint8_t *dst_tmp; + uint32_t cnt = width >> 4; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filter0, filter1, filter2, filter3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i reg6, reg7, reg8, reg9, reg10, reg11; + __m128i tmp0, tmp1, tmp2, tmp3; + + src -= (3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (;cnt--;) { + uint32_t loop_cnt = height >> 2; + + src_tmp = src; + dst_tmp = dst; + + LSX_LD_4(src_tmp, src_stride, src0, src1, src2, src3); + src_tmp += src_stride; + src4 = __lsx_vld(src_tmp, 0); + src_tmp += src_stride; + src5 = __lsx_vld(src_tmp, 0); + src_tmp += src_stride; + src6 = __lsx_vld(src_tmp, 0); + src_tmp += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg0, reg1, reg2, reg3); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + LSX_DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg6, reg7, reg8, reg9); + LSX_DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11); + + for (;loop_cnt--;) { + LSX_LD_4(src_tmp, src_stride, src7, src8, src9, src10); + src_tmp += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src4, src5, src7, src8); + tmp0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, src0, filter0, filter1, filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, src1, filter0, filter1, filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg6, reg7, reg8, src4, filter0, filter1, filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg9, reg10, reg11, src5, filter0, filter1, filter2, filter3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(tmp1, dst_tmp, 0); + dst_tmp += dst_stride; + tmp0 = FILT_8TAP_DPADD_S_H(reg1, reg2, src0, src2, filter0, filter1, filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg4, reg5, src1, src3, filter0, filter1, filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg7, reg8, src4, src7, filter0, filter1, filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg10, reg11, src5, src8, filter0, filter1, filter2, filter3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(tmp1, dst_tmp, 0); + dst_tmp += dst_stride; + + reg0 = reg2; + reg1 = src0; + reg2 = src2; + reg3 = reg5; + reg4 = src1; + reg5 = src3; + reg6 = reg8; + reg7 = src4; + reg8 = src7; + reg9 = reg11; + reg10 = src5; + reg11 = src8; + src6 = src10; + } + src += 16; + dst += 16; + } +} + +static void common_vt_8t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, 32); +} + +static void common_vt_8t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) +{ + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, 64); +} + +static void common_hv_8ht_8vt_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + uint32_t loop_cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i out0, out1; + __m128i shuff = {0x0F0E0D0C0B0A0908, 0x1716151413121110}; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= (3 + 3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + tmp0 = HORIZ_8TAP_FILT(src0, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp2 = HORIZ_8TAP_FILT(src2, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = HORIZ_8TAP_FILT(src4, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp5 = HORIZ_8TAP_FILT(src5, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + LSX_DUP2_ARG3(__lsx_vshuf_b, tmp2, tmp0, shuff, tmp4, tmp2, shuff, tmp1, tmp3); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + LSX_DUP2_ARG2(__lsx_vpackev_b, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + tmp2 = __lsx_vpackev_b(tmp5, tmp4); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + tmp3 = HORIZ_8TAP_FILT(src7, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = __lsx_vshuf_b(tmp3, tmp5, shuff); + tmp4 = __lsx_vpackev_b(tmp3, tmp4); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp4, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src1 = HORIZ_8TAP_FILT(src9, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vshuf_b(src1, tmp3, shuff); + src0 = __lsx_vpackev_b(src1, src0); + out1 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp4, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + + tmp5 = src1; + tmp0 = tmp2; + tmp1 = tmp4; + tmp2 = src0; + } +} + +static void common_hv_8ht_8vt_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + uint32_t loop_cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6; + __m128i out0, out1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= (3 + 3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + src0 = HORIZ_8TAP_FILT(src0, src0, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = HORIZ_8TAP_FILT(src1, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = HORIZ_8TAP_FILT(src2, src2, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src3 = HORIZ_8TAP_FILT(src3, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src4 = HORIZ_8TAP_FILT(src4, src4, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src5 = HORIZ_8TAP_FILT(src5, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src6 = HORIZ_8TAP_FILT(src6, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + LSX_DUP4_ARG2(__lsx_vpackev_b, src1, src0, src3, src2, src5, src4, + src2, src1, tmp0, tmp1, tmp2, tmp4); + LSX_DUP2_ARG2(__lsx_vpackev_b, src4, src3, src6, src5, tmp5, tmp6); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + src7 = HORIZ_8TAP_FILT(src7, src7, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp3 = __lsx_vpackev_b(src7, src6); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src8 = HORIZ_8TAP_FILT(src8, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vpackev_b(src8, src7); + out1 = FILT_8TAP_DPADD_S_H(tmp4, tmp5, tmp6, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src9 = HORIZ_8TAP_FILT(src9, src9, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = __lsx_vpackev_b(src9, src8); + src3 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp3, src1, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src10 = HORIZ_8TAP_FILT(src10, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = __lsx_vpackev_b(src10, src9); + src4 = FILT_8TAP_DPADD_S_H(tmp5, tmp6, src0, src2, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, src4, src3, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + src6 = src10; + tmp0 = tmp2; + tmp1 = tmp3; + tmp2 = src1; + tmp4 = tmp6; + tmp5 = src0; + tmp6 = src2; + } +} + +static void common_hv_8ht_8vt_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + int32_t multiple8_cnt; + + for (multiple8_cnt = 2; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + int32_t multiple8_cnt; + + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + int32_t multiple8_cnt; + + for (multiple8_cnt = 8; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void copy_width8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height) +{ + int32_t cnt = height >> 2; + __m128i src0, src1, src2, src3; + + for (;cnt--;) { + src0 = __lsx_vldrepl_d(src, 0); + src += src_stride; + src1 = __lsx_vldrepl_d(src, 0); + src += src_stride; + src2 = __lsx_vldrepl_d(src, 0); + src += src_stride; + src3 = __lsx_vldrepl_d(src, 0); + src += src_stride; + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + } +} + +static void copy_width16_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height) +{ + int32_t cnt = height >> 2; + __m128i src0, src1, src2, src3; + + for (;cnt--;) { + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + LSX_ST_4(src0, src1, src2, src3, dst, dst_stride); + dst += dst_stride; + } +} + +static void copy_width32_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height) +{ + int32_t cnt = height >> 2; + const uint8_t *src_tmp = src; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + + for (;cnt--;) { + LSX_LD_4(src_tmp, src_stride, src0, src1, src2, src3); + src_tmp += src_stride; + + LSX_LD_4_16(src, src_stride, src4, src5, src6, src7); + src += src_stride; + LSX_ST_4(src0, src1, src2, src3, dst_tmp, dst_stride); + dst_tmp += dst_stride; + LSX_ST_4_16(src4, src5, src6, src7, dst, dst_stride); + dst += dst_stride; + } +} + +static void copy_width64_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height) +{ + int32_t cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11, src12, src13, src14, src15; + + for (;cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, src3); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src4, src5, src6, src7); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src8, src9, src10, src11); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src12, src13, src14, src15); + src += src_stride; + __lsx_vst(src0, dst, 0); + __lsx_vst(src1, dst, 16); + __lsx_vst(src2, dst, 32); + __lsx_vst(src3, dst, 48); + dst += dst_stride; + __lsx_vst(src4, dst, 0); + __lsx_vst(src5, dst, 16); + __lsx_vst(src6, dst, 32); + __lsx_vst(src7, dst, 48); + dst += dst_stride; + __lsx_vst(src8, dst, 0); + __lsx_vst(src9, dst, 16); + __lsx_vst(src10, dst, 32); + __lsx_vst(src11, dst, 48); + dst += dst_stride; + __lsx_vst(src12, dst, 0); + __lsx_vst(src13, dst, 16); + __lsx_vst(src14, dst, 32); + __lsx_vst(src15, dst, 48); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_4x4_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) +{ + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1; + __m128i dst0, dst1, dst2, dst3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp0, tmp1); + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + tmp0 = __lsx_vssrarni_b_h(tmp1, tmp0, 7); + tmp0 = __lsx_vxori_b(tmp0, 128); + dst0 = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 3); +} + +static void common_hz_8t_and_aver_dst_4x8_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) +{ + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3, tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + tmp0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp0 = __lsx_vilvl_w(tmp1, tmp0); + tmp1 = __lsx_vilvl_w(tmp3, tmp2); + dst0 = __lsx_vilvl_d(tmp1, tmp0); + + tmp0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp3 = __lsx_vldrepl_w(dst_tmp, 0); + tmp0 = __lsx_vilvl_w(tmp1, tmp0); + tmp1 = __lsx_vilvl_w(tmp3, tmp2); + dst1 = __lsx_vilvl_d(tmp1, tmp0); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp0, tmp1); + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp2, tmp3); + LSX_DUP4_ARG3(__lsx_vssrarni_b_h, tmp0, tmp0, 7, tmp1, tmp1, 7, tmp2, tmp2, 7, + tmp3, tmp3, 7, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp1, dst1, dst0, dst1); + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 3); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 3); +} + +static void common_hz_8t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + if (height == 4) { + common_hz_8t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_8t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_8t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1, dst2, dst3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp0, tmp1, tmp2, tmp3); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + LSX_DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp1, dst1, dst0, dst1); + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 1); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int32_t loop_cnt = height >> 1; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3, dst0, dst1, dst2, dst3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (;loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3); + src += src_stride; + dst0 = __lsx_vld(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vld(dst_tmp, 0); + dst_tmp += dst_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, src2, src2, + mask0, src3, src3, mask0, tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, src2, src2, + mask1, src3, src3, mask1, tmp4, tmp5, tmp6, tmp7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, src2, src2, + mask2, src3, src3, mask2, tmp8, tmp9, tmp10, tmp11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, src2, src2, + mask3, src3, src3, mask3, tmp12, tmp13, tmp14, tmp15); + LSX_DUP4_ARG2(__lsx_dp2_h_b, tmp0, filter0, tmp1, filter0, tmp2, filter0, tmp3, + filter0, tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_dp2_h_b, tmp8, filter2, tmp9, filter2, tmp10, filter2, tmp11, + filter2, tmp8, tmp9, tmp10, tmp11); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, tmp0, tmp4, filter1, tmp1, tmp5, filter1, tmp2, + tmp6, filter1, tmp3, tmp7, filter1, tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, tmp8, tmp12, filter3, tmp9, tmp13, filter3, tmp10, + tmp14, filter3, tmp11, tmp15, filter3, tmp4, tmp5, tmp6, tmp7); + LSX_DUP4_ARG2(__lsx_vsadd_h, tmp0, tmp4, tmp1, tmp5, tmp2, tmp6, tmp3, tmp7, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, dst2, dst3); + LSX_DUP2_ARG2(__lsx_vxori_b, dst2, 128, dst3, 128, dst2, dst3); + LSX_DUP2_ARG2(__lsx_vavgr_bu, dst0, dst2, dst1, dst3, dst0, dst1); + __lsx_vst(dst0, dst, 0); + dst += dst_stride; + __lsx_vst(dst1, dst, 0); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt = height; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3, dst0, dst1; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; + __m128i shuff = {0x0F0E0D0C0B0A0908, 0x1716151413121110}; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (;loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + LSX_DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst, 16, dst0, dst1); + dst_tmp += dst_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, src2, src2, + mask0, src3, src3, mask0, tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, src2, src2, + mask1, src3, src3, mask1, tmp4, tmp5, tmp6, tmp7); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, src2, src2, + mask2, src3, src3, mask2, tmp8, tmp9, tmp10, tmp11); + LSX_DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, src2, src2, + mask3, src3, src3, mask3, tmp12, tmp13, tmp14, tmp15); + LSX_DUP4_ARG2(__lsx_dp2_h_b, tmp0, filter0, tmp1, filter0, tmp2, filter0, tmp3, + filter0, tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG2(__lsx_dp2_h_b, tmp8, filter2, tmp9, filter2, tmp10, filter2, tmp11, + filter2, tmp8, tmp9, tmp10, tmp11); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, tmp0, tmp4, filter1, tmp1, tmp5, filter1, tmp2, + tmp6, filter1, tmp3, tmp7, filter1, tmp0, tmp1, tmp2, tmp3); + LSX_DUP4_ARG3(__lsx_dp2add_h_b, tmp8, tmp12, filter3, tmp9, tmp13, filter3, tmp10, + tmp14, filter3, tmp11, tmp15, filter3, tmp4, tmp5, tmp6, tmp7); + LSX_DUP4_ARG2(__lsx_vsadd_h, tmp0, tmp4, tmp1, tmp5, tmp2, tmp6, tmp3, tmp7, + tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vavgr_bu, dst0, tmp0, dst1, tmp1, dst0, dst1); + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + int32_t loop_cnt = height; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3, dst0, dst1; + __m128i shuff = {0x0F0E0D0C0B0A0908, 0x1716151413121110}; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (;loop_cnt--;) { + LSX_DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + LSX_DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + LSX_DUP2_ARG2(__lsx_vavgr_bu, out0, dst0, out1, dst1, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + + LSX_DUP2_ARG2(__lsx_vld, src, 32, src, 48, src0, src2); + src3 = __lsx_vld(src, 56); + src1 = __lsx_vshuf_b(src2, src0, shuff); + LSX_DUP2_ARG2(__lsx_vld, dst, 32, dst, 48, dst0, dst1); + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, + src0, src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, out2, out3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + LSX_DUP2_ARG2(__lsx_vavgr_bu, out0, dst0, out1, dst1, out0, out1); + __lsx_vst(out0, dst, 32); + __lsx_vst(out1, dst, 48); + src += src_stride; + dst += dst_stride; + } +} + +static void common_vt_8t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i reg0, reg1, reg2, reg3, reg4; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1; + + src -= (3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, tmp4, tmp5); + LSX_DUP2_ARG2(__lsx_vilvl_d, tmp3, tmp0, tmp4, tmp1, reg0, reg1); + reg2 = __lsx_vilvl_d(tmp5, tmp2); + LSX_DUP2_ARG2(__lsx_vxori_b, reg0, 128, reg1, 128, reg0, reg1); + reg2 = __lsx_vxori_b(reg2, 128); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + src0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + LSX_DUP2_ARG2(__lsx_vilvl_w, src1, src0, src3, src2, src0, src1); + src0 = __lsx_vilvl_d(src1, src0); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, tmp0, tmp1, tmp2, tmp3); + LSX_DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, reg3, reg4); + LSX_DUP2_ARG2(__lsx_vxori_b, reg3, 128, reg4, 128, reg3, reg4); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, reg3, filter0, filter1, filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg1, reg2, reg3, reg4, filter0, filter1, filter2, filter3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + out0 = __lsx_vavgr_bu(out0, src0); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + reg0 = reg2; + reg1 = reg3; + reg2 = reg4; + src6 = src10; + } +} + +static void common_vt_8t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1, out2, out3; + + src -= (3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0, reg1, reg2, reg3); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + src0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + LSX_DUP2_ARG2(__lsx_vilvl_d, src1, src0, src3, src2, src0, src1); + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, tmp0, filter0, filter1, filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, tmp1, filter0, filter1, filter2, filter3); + out2 = FILT_8TAP_DPADD_S_H(reg1, reg2, tmp0, tmp2, filter0, filter1, filter2, filter3); + out3 = FILT_8TAP_DPADD_S_H(reg4, reg5, tmp1, tmp3, filter0, filter1, filter2, filter3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + LSX_DUP2_ARG2(__lsx_vavgr_bu, out0, src0, out1, src1, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + reg0 = reg2; + reg1 = tmp0; + reg2 = tmp2; + reg3 = reg5; + reg4 = tmp1; + reg5 = tmp3; + src6 = src10; + } +} + +static void common_vt_8t_and_aver_dst_16w_mult_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height, + int32_t width) +{ + const uint8_t *src_tmp; + uint8_t *dst_tmp; + uint32_t cnt = width >> 4; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filter0, filter1, filter2, filter3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i reg6, reg7, reg8, reg9, reg10, reg11; + __m128i tmp0, tmp1, tmp2, tmp3; + + src -= (3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (;cnt--;) { + uint32_t loop_cnt = height >> 2; + uint8_t *dst_reg = dst; + + src_tmp = src; + dst_tmp = dst; + + LSX_LD_4(src_tmp, src_stride, src0, src1, src2, src3); + src_tmp += src_stride; + src4 = __lsx_vld(src_tmp, 0); + src_tmp += src_stride; + src5 = __lsx_vld(src_tmp, 0); + src_tmp += src_stride; + src6 = __lsx_vld(src_tmp, 0); + src_tmp += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + LSX_DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg0, reg1, reg2, reg3); + LSX_DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + LSX_DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg6, reg7, reg8, reg9); + LSX_DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11); + + for (;loop_cnt--;) { + LSX_LD_4(src_tmp, src_stride, src7, src8, src9, src10); + src_tmp += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + LSX_DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src0, src1, src2, src3); + LSX_DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src4, src5, src7, src8); + tmp0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, src0, filter0, filter1, filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, src1, filter0, filter1, filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg6, reg7, reg8, src4, filter0, filter1, filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg9, reg10, reg11, src5, filter0, filter1, filter2, filter3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + tmp2 = __lsx_vld(dst_reg, 0); + dst_reg += dst_stride; + tmp3 = __lsx_vld(dst_reg, 0); + dst_reg += dst_stride; + LSX_DUP2_ARG2(__lsx_vavgr_bu, tmp0, tmp2, tmp1, tmp3, tmp0, tmp1); + __lsx_vst(tmp0, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(tmp1, dst_tmp, 0); + dst_tmp += dst_stride; + tmp0 = FILT_8TAP_DPADD_S_H(reg1, reg2, src0, src2, filter0, filter1, filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg4, reg5, src1, src3, filter0, filter1, filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg7, reg8, src4, src7, filter0, filter1, filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg10, reg11, src5, src8, filter0, filter1, filter2, filter3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + LSX_DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + tmp2 = __lsx_vld(dst_reg, 0); + dst_reg += dst_stride; + tmp3 = __lsx_vld(dst_reg, 0); + dst_reg += dst_stride; + LSX_DUP2_ARG2(__lsx_vavgr_bu, tmp0, tmp2, tmp1, tmp3, tmp0, tmp1); + __lsx_vst(tmp0, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(tmp1, dst_tmp, 0); + dst_tmp += dst_stride; + + reg0 = reg2; + reg1 = src0; + reg2 = src2; + reg3 = reg5; + reg4 = src1; + reg5 = src3; + reg6 = reg8; + reg7 = src4; + reg8 = src7; + reg9 = reg11; + reg10 = src5; + reg11 = src8; + src6 = src10; + } + src += 16; + dst += 16; + } +} + +static void common_vt_8t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 16); +} + +static void common_vt_8t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 32); +} + +static void common_vt_8t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, + int32_t height) +{ + common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 64); +} + +static void common_hv_8ht_8vt_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i out0, out1; + __m128i shuff = {0x0F0E0D0C0B0A0908, 0x1716151413121110}; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= (3 + 3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + tmp0 = HORIZ_8TAP_FILT(src0, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp2 = HORIZ_8TAP_FILT(src2, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = HORIZ_8TAP_FILT(src4, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp5 = HORIZ_8TAP_FILT(src5, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + LSX_DUP2_ARG3(__lsx_vshuf_b, tmp2, tmp0, shuff, tmp4, tmp2, shuff, tmp1, tmp3); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + LSX_DUP2_ARG2(__lsx_vpackev_b, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + tmp2 = __lsx_vpackev_b(tmp5, tmp4); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + src2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src4 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src5 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + LSX_DUP2_ARG2(__lsx_vilvl_w, src3, src2, src5, src4, src2, src3); + src2 = __lsx_vilvl_d(src3, src2); + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + tmp3 = HORIZ_8TAP_FILT(src7, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = __lsx_vshuf_b(tmp3, tmp5, shuff); + tmp4 = __lsx_vpackev_b(tmp3, tmp4); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp4, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src1 = HORIZ_8TAP_FILT(src9, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vshuf_b(src1, tmp3, shuff); + src0 = __lsx_vpackev_b(src1, src0); + out1 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp4, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + out0 = __lsx_vavgr_bu(out0, src2); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + + tmp5 = src1; + tmp0 = tmp2; + tmp1 = tmp4; + tmp2 = src0; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6; + __m128i out0, out1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= (3 + 3 * src_stride); + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + LSX_DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, src1, src2, src3); + LSX_DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + src0 = HORIZ_8TAP_FILT(src0, src0, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = HORIZ_8TAP_FILT(src1, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = HORIZ_8TAP_FILT(src2, src2, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src3 = HORIZ_8TAP_FILT(src3, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src4 = HORIZ_8TAP_FILT(src4, src4, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src5 = HORIZ_8TAP_FILT(src5, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src6 = HORIZ_8TAP_FILT(src6, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + + LSX_DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + LSX_DUP4_ARG2(__lsx_vpackev_b, src1, src0, src3, src2, src5, src4, + src2, src1, tmp0, tmp1, tmp2, tmp4); + LSX_DUP2_ARG2(__lsx_vpackev_b, src4, src3, src6, src5, tmp5, tmp6); + + for (;loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + + LSX_DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + src7 = HORIZ_8TAP_FILT(src7, src7, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp3 = __lsx_vpackev_b(src7, src6); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src8 = HORIZ_8TAP_FILT(src8, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vpackev_b(src8, src7); + out1 = FILT_8TAP_DPADD_S_H(tmp4, tmp5, tmp6, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src9 = HORIZ_8TAP_FILT(src9, src9, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = __lsx_vpackev_b(src9, src8); + src3 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp3, src1, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src10 = HORIZ_8TAP_FILT(src10, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = __lsx_vpackev_b(src10, src9); + src4 = FILT_8TAP_DPADD_S_H(tmp5, tmp6, src0, src2, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + LSX_DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, src4, src3, 7, out0, out1); + LSX_DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + src5 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src7 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src8 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src9 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + LSX_DUP2_ARG2(__lsx_vilvl_d, src7, src5, src9, src8, src5, src7); + LSX_DUP2_ARG2(__lsx_vavgr_bu, out0, src5, out1, src7, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + src6 = src10; + tmp0 = tmp2; + tmp1 = tmp3; + tmp2 = src1; + tmp4 = tmp6; + tmp5 = src0; + tmp6 = src2; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + int32_t multiple8_cnt; + + for (multiple8_cnt = 2; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, + height); + + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + int32_t multiple8_cnt; + + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, + height); + + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, + uint8_t *dst, + int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) +{ + int32_t multiple8_cnt; + + for (multiple8_cnt = 8; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, + height); + + src += 8; + dst += 8; + } +} + +static void avg_width8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height) +{ + int32_t cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, dst0, dst1; + __m128i tmp0, tmp1, tmp2, tmp3; + + for (;cnt--;) { + tmp0 = __lsx_vldrepl_d(src, 0); + src += src_stride; + tmp1 = __lsx_vldrepl_d(src, 0); + src += src_stride; + tmp2 = __lsx_vldrepl_d(src, 0); + src += src_stride; + tmp3 = __lsx_vldrepl_d(src, 0); + src += src_stride; + LSX_DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, src0, src1); + tmp0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + tmp1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + tmp2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + tmp3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + LSX_DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, dst0, dst1); + LSX_DUP2_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, dst0, dst1); + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 1); + dst += dst_stride; + } +} + +static void avg_width16_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height) +{ + int32_t cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3; + __m128i dst0, dst1, dst2, dst3; + + for (;cnt--;) { + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + LSX_LD_4(dst_tmp, dst_stride, dst0, dst1, dst2, dst3); + dst_tmp += dst_stride; + LSX_DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, + src2, dst2, src3, dst3, dst0, dst1, dst2, dst3); + LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride); + dst += dst_stride; + } +} + +static void avg_width32_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height) +{ + int32_t cnt = height >> 2; + const uint8_t *src_tmp = src; + uint8_t *dst_tmp, *dst_tmp1, *dst_tmp2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + + dst_tmp = dst_tmp1 = dst_tmp2 = dst; + for (;cnt--;) { + LSX_LD_4(src, src_stride, src0, src2, src4, src6); + src += src_stride; + LSX_LD_4_16(src_tmp, src_stride, src1, src3, src5, src7); + src_tmp += src_stride; + LSX_LD_4(dst_tmp1, dst_stride, dst0, dst2, dst4, dst6); + dst_tmp1 += dst_stride; + LSX_LD_4_16(dst_tmp2, dst_stride, dst1, dst3, dst5, dst7); + dst_tmp2 += dst_stride; + LSX_DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, + src2, dst2, src3, dst3, dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, + src6, dst6, src7, dst7, dst4, dst5, dst6, dst7); + LSX_ST_4(dst0, dst2, dst4, dst6, dst_tmp, dst_stride); + dst_tmp += dst_stride; + LSX_ST_4_16(dst1, dst3, dst5, dst7, dst, dst_stride); + dst += dst_stride; + } +} + +static void avg_width64_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height) +{ + int32_t cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11, src12, src13, src14, src15; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15; + + for (;cnt--;) { + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, + src0, src1, src2, src3); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, + src4, src5, src6, src7); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, + src8, src9, src10, src11); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, + src12, src13, src14, src15); + src += src_stride; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst0, dst1, dst2, dst3); + dst_tmp += dst_stride; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst4, dst5, dst6, dst7); + dst_tmp += dst_stride; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst8, dst9, dst10, dst11); + dst_tmp += dst_stride; + LSX_DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst12, dst13, dst14, dst15); + dst_tmp += dst_stride; + LSX_DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, + src2, dst2, src3, dst3, dst0, dst1, dst2, dst3); + LSX_DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, + src6, dst6, src7, dst7, dst4, dst5, dst6, dst7); + LSX_DUP4_ARG2(__lsx_vavgr_bu, src8, dst8, src9, dst9, src10, + dst10, src11, dst11, dst8, dst9, dst10, dst11); + LSX_DUP4_ARG2(__lsx_vavgr_bu, src12, dst12, src13, dst13, src14, + dst14, src15, dst15, dst12, dst13, dst14, dst15); + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + __lsx_vst(dst2, dst, 32); + __lsx_vst(dst3, dst, 48); + dst += dst_stride; + __lsx_vst(dst4, dst, 0); + __lsx_vst(dst5, dst, 16); + __lsx_vst(dst6, dst, 32); + __lsx_vst(dst7, dst, 48); + dst += dst_stride; + __lsx_vst(dst8, dst, 0); + __lsx_vst(dst9, dst, 16); + __lsx_vst(dst10, dst, 32); + __lsx_vst(dst11, dst, 48); + dst += dst_stride; + __lsx_vst(dst12, dst, 0); + __lsx_vst(dst13, dst, 16); + __lsx_vst(dst14, dst, 32); + __lsx_vst(dst15, dst, 48); + dst += dst_stride; + } +} + +static const int8_t vp9_subpel_filters_lsx[3][15][8] = { + [FILTER_8TAP_REGULAR] = { + {0, 1, -5, 126, 8, -3, 1, 0}, + {-1, 3, -10, 122, 18, -6, 2, 0}, + {-1, 4, -13, 118, 27, -9, 3, -1}, + {-1, 4, -16, 112, 37, -11, 4, -1}, + {-1, 5, -18, 105, 48, -14, 4, -1}, + {-1, 5, -19, 97, 58, -16, 5, -1}, + {-1, 6, -19, 88, 68, -18, 5, -1}, + {-1, 6, -19, 78, 78, -19, 6, -1}, + {-1, 5, -18, 68, 88, -19, 6, -1}, + {-1, 5, -16, 58, 97, -19, 5, -1}, + {-1, 4, -14, 48, 105, -18, 5, -1}, + {-1, 4, -11, 37, 112, -16, 4, -1}, + {-1, 3, -9, 27, 118, -13, 4, -1}, + {0, 2, -6, 18, 122, -10, 3, -1}, + {0, 1, -3, 8, 126, -5, 1, 0}, + }, [FILTER_8TAP_SHARP] = { + {-1, 3, -7, 127, 8, -3, 1, 0}, + {-2, 5, -13, 125, 17, -6, 3, -1}, + {-3, 7, -17, 121, 27, -10, 5, -2}, + {-4, 9, -20, 115, 37, -13, 6, -2}, + {-4, 10, -23, 108, 48, -16, 8, -3}, + {-4, 10, -24, 100, 59, -19, 9, -3}, + {-4, 11, -24, 90, 70, -21, 10, -4}, + {-4, 11, -23, 80, 80, -23, 11, -4}, + {-4, 10, -21, 70, 90, -24, 11, -4}, + {-3, 9, -19, 59, 100, -24, 10, -4}, + {-3, 8, -16, 48, 108, -23, 10, -4}, + {-2, 6, -13, 37, 115, -20, 9, -4}, + {-2, 5, -10, 27, 121, -17, 7, -3}, + {-1, 3, -6, 17, 125, -13, 5, -2}, + {0, 1, -3, 8, 127, -7, 3, -1}, + }, [FILTER_8TAP_SMOOTH] = { + {-3, -1, 32, 64, 38, 1, -3, 0}, + {-2, -2, 29, 63, 41, 2, -3, 0}, + {-2, -2, 26, 63, 43, 4, -4, 0}, + {-2, -3, 24, 62, 46, 5, -4, 0}, + {-2, -3, 21, 60, 49, 7, -4, 0}, + {-1, -4, 18, 59, 51, 9, -4, 0}, + {-1, -4, 16, 57, 53, 12, -4, -1}, + {-1, -4, 14, 55, 55, 14, -4, -1}, + {-1, -4, 12, 53, 57, 16, -4, -1}, + {0, -4, 9, 51, 59, 18, -4, -1}, + {0, -4, 7, 49, 60, 21, -3, -2}, + {0, -4, 5, 46, 62, 24, -3, -2}, + {0, -4, 4, 43, 63, 26, -2, -2}, + {0, -3, 2, 41, 63, 29, -2, -2}, + {0, -3, 1, 38, 64, 32, -1, -3}, + } +}; + +#define VP9_8TAP_LOONGARCH_LSX_FUNC(SIZE, type, type_idx) \ +void ff_put_8tap_##type##_##SIZE##h_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my) \ +{ \ + const int8_t *filter = vp9_subpel_filters_lsx[type_idx][mx-1]; \ + \ + common_hz_8t_##SIZE##w_lsx(src, srcstride, dst, dststride, filter, h); \ +} \ + \ +void ff_put_8tap_##type##_##SIZE##v_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my) \ +{ \ + const int8_t *filter = vp9_subpel_filters_lsx[type_idx][my-1]; \ + \ + common_vt_8t_##SIZE##w_lsx(src, srcstride, dst, dststride, filter, h); \ +} \ + \ +void ff_put_8tap_##type##_##SIZE##hv_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my) \ +{ \ + const int8_t *hfilter = vp9_subpel_filters_lsx[type_idx][mx-1]; \ + const int8_t *vfilter = vp9_subpel_filters_lsx[type_idx][my-1]; \ + \ + common_hv_8ht_8vt_##SIZE##w_lsx(src, srcstride, dst, dststride, hfilter, \ + vfilter, h); \ +} \ + \ +void ff_avg_8tap_##type##_##SIZE##h_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my) \ +{ \ + const int8_t *filter = vp9_subpel_filters_lsx[type_idx][mx-1]; \ + \ + common_hz_8t_and_aver_dst_##SIZE##w_lsx(src, srcstride, dst, \ + dststride, filter, h); \ +} \ + \ +void ff_avg_8tap_##type##_##SIZE##v_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my) \ +{ \ + const int8_t *filter = vp9_subpel_filters_lsx[type_idx][my-1]; \ + \ + common_vt_8t_and_aver_dst_##SIZE##w_lsx(src, srcstride, dst, dststride, \ + filter, h); \ +} \ + \ +void ff_avg_8tap_##type##_##SIZE##hv_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my) \ +{ \ + const int8_t *hfilter = vp9_subpel_filters_lsx[type_idx][mx-1]; \ + const int8_t *vfilter = vp9_subpel_filters_lsx[type_idx][my-1]; \ + \ + common_hv_8ht_8vt_and_aver_dst_##SIZE##w_lsx(src, srcstride, dst, \ + dststride, hfilter, \ + vfilter, h); \ +} + +#define VP9_COPY_LOONGARCH_LSX_FUNC(SIZE) \ +void ff_copy##SIZE##_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, ptrdiff_t srcstride, \ + int h, int mx, int my) \ +{ \ + \ + copy_width##SIZE##_lsx(src, srcstride, dst, dststride, h); \ +} \ +void ff_avg##SIZE##_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, ptrdiff_t srcstride, \ + int h, int mx, int my) \ +{ \ + \ + avg_width##SIZE##_lsx(src, srcstride, dst, dststride, h); \ +} + +VP9_8TAP_LOONGARCH_LSX_FUNC(64, regular, FILTER_8TAP_REGULAR); +VP9_8TAP_LOONGARCH_LSX_FUNC(32, regular, FILTER_8TAP_REGULAR); +VP9_8TAP_LOONGARCH_LSX_FUNC(16, regular, FILTER_8TAP_REGULAR); +VP9_8TAP_LOONGARCH_LSX_FUNC(8, regular, FILTER_8TAP_REGULAR); +VP9_8TAP_LOONGARCH_LSX_FUNC(4, regular, FILTER_8TAP_REGULAR); + +VP9_8TAP_LOONGARCH_LSX_FUNC(64, sharp, FILTER_8TAP_SHARP); +VP9_8TAP_LOONGARCH_LSX_FUNC(32, sharp, FILTER_8TAP_SHARP); +VP9_8TAP_LOONGARCH_LSX_FUNC(16, sharp, FILTER_8TAP_SHARP); +VP9_8TAP_LOONGARCH_LSX_FUNC(8, sharp, FILTER_8TAP_SHARP); +VP9_8TAP_LOONGARCH_LSX_FUNC(4, sharp, FILTER_8TAP_SHARP); + +VP9_8TAP_LOONGARCH_LSX_FUNC(64, smooth, FILTER_8TAP_SMOOTH); +VP9_8TAP_LOONGARCH_LSX_FUNC(32, smooth, FILTER_8TAP_SMOOTH); +VP9_8TAP_LOONGARCH_LSX_FUNC(16, smooth, FILTER_8TAP_SMOOTH); +VP9_8TAP_LOONGARCH_LSX_FUNC(8, smooth, FILTER_8TAP_SMOOTH); +VP9_8TAP_LOONGARCH_LSX_FUNC(4, smooth, FILTER_8TAP_SMOOTH); + +VP9_COPY_LOONGARCH_LSX_FUNC(64); +VP9_COPY_LOONGARCH_LSX_FUNC(32); +VP9_COPY_LOONGARCH_LSX_FUNC(16); +VP9_COPY_LOONGARCH_LSX_FUNC(8); + +#undef VP9_8TAP_LOONGARCH_LSX_FUNC +#undef VP9_COPY_LOONGARCH_LSX_FUNC diff --git a/libavcodec/loongarch/vp9dsp_init_loongarch.c b/libavcodec/loongarch/vp9dsp_init_loongarch.c new file mode 100644 index 0000000000..f85abde11d --- /dev/null +++ b/libavcodec/loongarch/vp9dsp_init_loongarch.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/loongarch/cpu.h" +#include "libavutil/attributes.h" +#include "libavcodec/vp9dsp.h" +#include "vp9dsp_loongarch.h" + +#define init_subpel1(idx1, idx2, idxh, idxv, sz, dir, type) \ + dsp->mc[idx1][FILTER_8TAP_SMOOTH ][idx2][idxh][idxv] = \ + ff_##type##_8tap_smooth_##sz##dir##_lsx; \ + dsp->mc[idx1][FILTER_8TAP_REGULAR][idx2][idxh][idxv] = \ + ff_##type##_8tap_regular_##sz##dir##_lsx; \ + dsp->mc[idx1][FILTER_8TAP_SHARP ][idx2][idxh][idxv] = \ + ff_##type##_8tap_sharp_##sz##dir##_lsx; + +#define init_subpel2(idx, idxh, idxv, dir, type) \ + init_subpel1(0, idx, idxh, idxv, 64, dir, type); \ + init_subpel1(1, idx, idxh, idxv, 32, dir, type); \ + init_subpel1(2, idx, idxh, idxv, 16, dir, type); \ + init_subpel1(3, idx, idxh, idxv, 8, dir, type); \ + init_subpel1(4, idx, idxh, idxv, 4, dir, type) + +#define init_subpel3(idx, type) \ + init_subpel2(idx, 1, 0, h, type) \ + init_subpel2(idx, 0, 1, v, type); \ + init_subpel2(idx, 1, 1, hv, type); + +#define init_fpel(idx1, idx2, sz, type) \ + dsp->mc[idx1][FILTER_8TAP_SMOOTH ][idx2][0][0] = ff_##type##sz##_lsx; \ + dsp->mc[idx1][FILTER_8TAP_REGULAR][idx2][0][0] = ff_##type##sz##_lsx; \ + dsp->mc[idx1][FILTER_8TAP_SHARP ][idx2][0][0] = ff_##type##sz##_lsx; \ + dsp->mc[idx1][FILTER_BILINEAR ][idx2][0][0] = ff_##type##sz##_lsx + +#define init_copy(idx, sz) \ + init_fpel(idx, 0, sz, copy); \ + init_fpel(idx, 1, sz, avg) + +static av_cold void vp9dsp_mc_init_lsx(VP9DSPContext *dsp, int bpp) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lsx(cpu_flags)) { + if (bpp == 8) { + init_subpel3(0, put); + init_subpel3(1, avg); + init_copy(0, 64); + init_copy(1, 32); + init_copy(2, 16); + init_copy(3, 8); + } + } +} + +#undef init_subpel1 +#undef init_subpel2 +#undef init_subpel3 +#undef init_copy +#undef init_fpel + +#define init_intra_pred1_lsx(tx, sz) \ + dsp->intra_pred[tx][VERT_PRED] = ff_vert_##sz##_lsx; \ + dsp->intra_pred[tx][HOR_PRED] = ff_hor_##sz##_lsx; \ + dsp->intra_pred[tx][DC_PRED] = ff_dc_##sz##_lsx; \ + dsp->intra_pred[tx][LEFT_DC_PRED] = ff_dc_left_##sz##_lsx; \ + dsp->intra_pred[tx][TOP_DC_PRED] = ff_dc_top_##sz##_lsx; \ + dsp->intra_pred[tx][DC_128_PRED] = ff_dc_128_##sz##_lsx; \ + dsp->intra_pred[tx][DC_127_PRED] = ff_dc_127_##sz##_lsx; \ + dsp->intra_pred[tx][DC_129_PRED] = ff_dc_129_##sz##_lsx; \ + dsp->intra_pred[tx][TM_VP8_PRED] = ff_tm_##sz##_lsx; \ + +#define init_intra_pred2_lsx(tx, sz) \ + dsp->intra_pred[tx][DC_PRED] = ff_dc_##sz##_lsx; \ + dsp->intra_pred[tx][LEFT_DC_PRED] = ff_dc_left_##sz##_lsx; \ + dsp->intra_pred[tx][TOP_DC_PRED] = ff_dc_top_##sz##_lsx; \ + dsp->intra_pred[tx][TM_VP8_PRED] = ff_tm_##sz##_lsx; \ + +static av_cold void vp9dsp_intrapred_init_lsx(VP9DSPContext *dsp, int bpp) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lsx(cpu_flags)) { + if (bpp == 8) { + init_intra_pred1_lsx(TX_16X16, 16x16); + init_intra_pred1_lsx(TX_32X32, 32x32); + init_intra_pred2_lsx(TX_4X4, 4x4); + init_intra_pred2_lsx(TX_8X8, 8x8); + } + } +} + +#undef init_intra_pred_lsx +#undef init_intra_pred_lsx + +#define init_idct(tx, nm) \ + dsp->itxfm_add[tx][DCT_DCT] = \ + dsp->itxfm_add[tx][ADST_DCT] = \ + dsp->itxfm_add[tx][DCT_ADST] = \ + dsp->itxfm_add[tx][ADST_ADST] = nm##_add_lsx + +#define init_itxfm(tx, sz) \ + dsp->itxfm_add[tx][DCT_DCT] = ff_idct_idct_##sz##_add_lsx + +static av_cold void vp9dsp_itxfm_init_lsx(VP9DSPContext *dsp, int bpp) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lsx(cpu_flags)) { + if (bpp == 8) { + init_itxfm(TX_8X8, 8x8); + init_itxfm(TX_16X16, 16x16); + init_idct(TX_32X32, ff_idct_idct_32x32); + } + } +} + +#undef init_idct +#undef init_itxfm + +static av_cold void vp9dsp_loopfilter_init_lsx(VP9DSPContext *dsp, int bpp) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lsx(cpu_flags)) { + if (bpp == 8) { + dsp->loop_filter_8[0][0] = ff_loop_filter_h_4_8_lsx; + dsp->loop_filter_8[0][1] = ff_loop_filter_v_4_8_lsx; + dsp->loop_filter_8[1][0] = ff_loop_filter_h_8_8_lsx; + dsp->loop_filter_8[1][1] = ff_loop_filter_v_8_8_lsx; + dsp->loop_filter_8[2][0] = ff_loop_filter_h_16_8_lsx; + dsp->loop_filter_8[2][1] = ff_loop_filter_v_16_8_lsx; + + dsp->loop_filter_16[0] = ff_loop_filter_h_16_16_lsx; + dsp->loop_filter_16[1] = ff_loop_filter_v_16_16_lsx; + + dsp->loop_filter_mix2[0][0][0] = ff_loop_filter_h_44_16_lsx; + dsp->loop_filter_mix2[0][0][1] = ff_loop_filter_v_44_16_lsx; + dsp->loop_filter_mix2[0][1][0] = ff_loop_filter_h_48_16_lsx; + dsp->loop_filter_mix2[0][1][1] = ff_loop_filter_v_48_16_lsx; + dsp->loop_filter_mix2[1][0][0] = ff_loop_filter_h_84_16_lsx; + dsp->loop_filter_mix2[1][0][1] = ff_loop_filter_v_84_16_lsx; + dsp->loop_filter_mix2[1][1][0] = ff_loop_filter_h_88_16_lsx; + dsp->loop_filter_mix2[1][1][1] = ff_loop_filter_v_88_16_lsx; + } + } +} + +static av_cold void vp9dsp_init_lsx(VP9DSPContext *dsp, int bpp) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lsx(cpu_flags)) { + vp9dsp_mc_init_lsx(dsp, bpp); + vp9dsp_intrapred_init_lsx(dsp, bpp); + vp9dsp_loopfilter_init_lsx(dsp, bpp); + vp9dsp_itxfm_init_lsx(dsp, bpp); + } +} + +av_cold void ff_vp9dsp_init_loongarch(VP9DSPContext *dsp, int bpp) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lsx(cpu_flags)) + vp9dsp_init_lsx(dsp, bpp); +} diff --git a/libavcodec/loongarch/vp9dsp_loongarch.h b/libavcodec/loongarch/vp9dsp_loongarch.h new file mode 100644 index 0000000000..3cc918a18c --- /dev/null +++ b/libavcodec/loongarch/vp9dsp_loongarch.h @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_LOONGARCH_VP9DSP_LOONGARCH_H +#define AVCODEC_LOONGARCH_VP9DSP_LOONGARCH_H + +#define VP9_8TAP_LOONGARCH_LSX_FUNC(SIZE, type, type_idx) \ +void ff_put_8tap_##type##_##SIZE##h_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my); \ + \ +void ff_put_8tap_##type##_##SIZE##v_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my); \ + \ +void ff_put_8tap_##type##_##SIZE##hv_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my); \ + \ +void ff_avg_8tap_##type##_##SIZE##h_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my); \ + \ +void ff_avg_8tap_##type##_##SIZE##v_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my); \ + \ +void ff_avg_8tap_##type##_##SIZE##hv_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, \ + ptrdiff_t srcstride, \ + int h, int mx, int my); + +#define VP9_COPY_LOONGARCH_LSX_FUNC(SIZE) \ +void ff_copy##SIZE##_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, ptrdiff_t srcstride, \ + int h, int mx, int my); \ + \ +void ff_avg##SIZE##_lsx(uint8_t *dst, ptrdiff_t dststride, \ + const uint8_t *src, ptrdiff_t srcstride, \ + int h, int mx, int my); + +VP9_8TAP_LOONGARCH_LSX_FUNC(64, regular, FILTER_8TAP_REGULAR); +VP9_8TAP_LOONGARCH_LSX_FUNC(32, regular, FILTER_8TAP_REGULAR); +VP9_8TAP_LOONGARCH_LSX_FUNC(16, regular, FILTER_8TAP_REGULAR); +VP9_8TAP_LOONGARCH_LSX_FUNC(8, regular, FILTER_8TAP_REGULAR); +VP9_8TAP_LOONGARCH_LSX_FUNC(4, regular, FILTER_8TAP_REGULAR); + +VP9_8TAP_LOONGARCH_LSX_FUNC(64, sharp, FILTER_8TAP_SHARP); +VP9_8TAP_LOONGARCH_LSX_FUNC(32, sharp, FILTER_8TAP_SHARP); +VP9_8TAP_LOONGARCH_LSX_FUNC(16, sharp, FILTER_8TAP_SHARP); +VP9_8TAP_LOONGARCH_LSX_FUNC(8, sharp, FILTER_8TAP_SHARP); +VP9_8TAP_LOONGARCH_LSX_FUNC(4, sharp, FILTER_8TAP_SHARP); + +VP9_8TAP_LOONGARCH_LSX_FUNC(64, smooth, FILTER_8TAP_SMOOTH); +VP9_8TAP_LOONGARCH_LSX_FUNC(32, smooth, FILTER_8TAP_SMOOTH); +VP9_8TAP_LOONGARCH_LSX_FUNC(16, smooth, FILTER_8TAP_SMOOTH); +VP9_8TAP_LOONGARCH_LSX_FUNC(8, smooth, FILTER_8TAP_SMOOTH); +VP9_8TAP_LOONGARCH_LSX_FUNC(4, smooth, FILTER_8TAP_SMOOTH); + +VP9_COPY_LOONGARCH_LSX_FUNC(64); +VP9_COPY_LOONGARCH_LSX_FUNC(32); +VP9_COPY_LOONGARCH_LSX_FUNC(16); +VP9_COPY_LOONGARCH_LSX_FUNC(8); + +#undef VP9_8TAP_LOONGARCH_LSX_FUNC +#undef VP9_COPY_LOONGARCH_LSX_FUNC + +void ff_vert_16x16_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_vert_32x32_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_hor_16x16_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_hor_32x32_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_dc_4x4_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_dc_8x8_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_dc_16x16_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_dc_32x32_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_dc_left_4x4_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_dc_left_8x8_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_dc_left_16x16_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_dc_left_32x32_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_dc_top_4x4_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_dc_top_8x8_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_dc_top_16x16_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_dc_top_32x32_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_dc_128_16x16_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_dc_128_32x32_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_dc_127_16x16_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_dc_127_32x32_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_dc_129_16x16_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_dc_129_32x32_lsx(uint8_t *dst, ptrdiff_t stride, + const uint8_t *left, const uint8_t *top); +void ff_tm_4x4_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_tm_8x8_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_tm_16x16_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_tm_32x32_lsx(uint8_t *dst, ptrdiff_t stride, const uint8_t *left, + const uint8_t *top); +void ff_loop_filter_h_16_8_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_v_16_8_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_h_4_8_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_v_4_8_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_h_44_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_v_44_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_h_8_8_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_v_8_8_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_h_88_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_v_88_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_h_84_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_v_84_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_h_48_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_v_48_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_h_16_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_loop_filter_v_16_16_lsx(uint8_t *dst, ptrdiff_t stride, int32_t e, + int32_t i, int32_t h); +void ff_idct_idct_8x8_add_lsx(uint8_t *dst, ptrdiff_t stride, + int16_t *block, int eob); +void ff_idct_idct_16x16_add_lsx(uint8_t *dst, ptrdiff_t stride, + int16_t *block, int eob); +void ff_idct_idct_32x32_add_lsx(uint8_t *dst, ptrdiff_t stride, + int16_t *block, int eob); + +#endif /* AVCODEC_LOONGARCH_VP9DSP_LOONGARCH_H */ diff --git a/libavcodec/mips/Makefile b/libavcodec/mips/Makefile index 2be4d9b8a2..81a73a4d0e 100644 --- a/libavcodec/mips/Makefile +++ b/libavcodec/mips/Makefile @@ -57,7 +57,8 @@ MSA-OBJS-$(CONFIG_VP8_DECODER) += mips/vp8_mc_msa.o \ mips/vp8_lpf_msa.o MSA-OBJS-$(CONFIG_VP3DSP) += mips/vp3dsp_idct_msa.o MSA-OBJS-$(CONFIG_H264DSP) += mips/h264dsp_msa.o \ - mips/h264idct_msa.o + mips/h264idct_msa.o \ + mips/h264_deblock_msa.o MSA-OBJS-$(CONFIG_H264QPEL) += mips/h264qpel_msa.o MSA-OBJS-$(CONFIG_H264CHROMA) += mips/h264chroma_msa.o MSA-OBJS-$(CONFIG_H264PRED) += mips/h264pred_msa.o diff --git a/libavcodec/mips/blockdsp_mmi.c b/libavcodec/mips/blockdsp_mmi.c index 68641e2544..8b5c7e955c 100644 --- a/libavcodec/mips/blockdsp_mmi.c +++ b/libavcodec/mips/blockdsp_mmi.c @@ -76,8 +76,8 @@ void ff_clear_block_mmi(int16_t *block) double ftmp[2]; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp1] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp1] \n\t" MMI_SQC1(%[ftmp0], %[ftmp1], %[block], 0x00) MMI_SQC1(%[ftmp0], %[ftmp1], %[block], 0x10) MMI_SQC1(%[ftmp0], %[ftmp1], %[block], 0x20) @@ -97,8 +97,8 @@ void ff_clear_blocks_mmi(int16_t *block) double ftmp[2]; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp1] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp1] \n\t" MMI_SQC1(%[ftmp0], %[ftmp1], %[block], 0x00) MMI_SQC1(%[ftmp0], %[ftmp1], %[block], 0x10) MMI_SQC1(%[ftmp0], %[ftmp1], %[block], 0x20) diff --git a/libavcodec/mips/cabac.h b/libavcodec/mips/cabac.h index 6a13b777ba..2f3a3dc0a5 100644 --- a/libavcodec/mips/cabac.h +++ b/libavcodec/mips/cabac.h @@ -2,7 +2,8 @@ * Loongson SIMD optimized h264chroma * * Copyright (c) 2018 Loongson Technology Corporation Limited - * Copyright (c) 2018 Shiyou Yin + * Contributed by Shiyou Yin + * Gu Xiwei(guxiwei-hf@loongson.cn) * * This file is part of FFmpeg. * @@ -25,7 +26,7 @@ #define AVCODEC_MIPS_CABAC_H #include "libavcodec/cabac.h" -#include "libavutil/mips/asmdefs.h" +#include "libavutil/mips/mmiutils.h" #include "config.h" #define get_cabac_inline get_cabac_inline_mips @@ -36,7 +37,7 @@ static av_always_inline int get_cabac_inline(CABACContext *c, __asm__ volatile ( "lbu %[bit], 0(%[state]) \n\t" "and %[tmp0], %[c_range], 0xC0 \n\t" - PTR_ADDU "%[tmp0], %[tmp0], %[tmp0] \n\t" + PTR_SLL "%[tmp0], %[tmp0], 0x01 \n\t" PTR_ADDU "%[tmp0], %[tmp0], %[tables] \n\t" PTR_ADDU "%[tmp0], %[tmp0], %[bit] \n\t" /* tmp1: RangeLPS */ @@ -44,18 +45,11 @@ static av_always_inline int get_cabac_inline(CABACContext *c, PTR_SUBU "%[c_range], %[c_range], %[tmp1] \n\t" PTR_SLL "%[tmp0], %[c_range], 0x11 \n\t" - PTR_SUBU "%[tmp0], %[tmp0], %[c_low] \n\t" - - /* tmp2: lps_mask */ - PTR_SRA "%[tmp2], %[tmp0], 0x1F \n\t" - /* If tmp0 < 0, lps_mask == 0xffffffff*/ - /* If tmp0 >= 0, lps_mask == 0x00000000*/ + "slt %[tmp2], %[tmp0], %[c_low] \n\t" "beqz %[tmp2], 1f \n\t" - PTR_SLL "%[tmp0], %[c_range], 0x11 \n\t" + "move %[c_range], %[tmp1] \n\t" + "not %[bit], %[bit] \n\t" PTR_SUBU "%[c_low], %[c_low], %[tmp0] \n\t" - PTR_SUBU "%[tmp0], %[tmp1], %[c_range] \n\t" - PTR_ADDU "%[c_range], %[c_range], %[tmp0] \n\t" - "xor %[bit], %[bit], %[tmp2] \n\t" "1: \n\t" /* tmp1: *state */ @@ -70,23 +64,18 @@ static av_always_inline int get_cabac_inline(CABACContext *c, PTR_SLL "%[c_range], %[c_range], %[tmp2] \n\t" PTR_SLL "%[c_low], %[c_low], %[tmp2] \n\t" - "and %[tmp0], %[c_low], %[cabac_mask] \n\t" - "bnez %[tmp0], 1f \n\t" - PTR_ADDI "%[tmp0], %[c_low], -0X01 \n\t" + "and %[tmp1], %[c_low], %[cabac_mask] \n\t" + "bnez %[tmp1], 1f \n\t" + PTR_ADDIU "%[tmp0], %[c_low], -0X01 \n\t" "xor %[tmp0], %[c_low], %[tmp0] \n\t" PTR_SRA "%[tmp0], %[tmp0], 0x0f \n\t" PTR_ADDU "%[tmp0], %[tmp0], %[tables] \n\t" + /* tmp2: ff_h264_norm_shift[x >> (CABAC_BITS - 1)] */ "lbu %[tmp2], %[norm_off](%[tmp0]) \n\t" -#if CABAC_BITS == 16 - "lbu %[tmp0], 0(%[c_bytestream]) \n\t" - "lbu %[tmp1], 1(%[c_bytestream]) \n\t" - PTR_SLL "%[tmp0], %[tmp0], 0x09 \n\t" - PTR_SLL "%[tmp1], %[tmp1], 0x01 \n\t" - PTR_ADDU "%[tmp0], %[tmp0], %[tmp1] \n\t" -#else - "lbu %[tmp0], 0(%[c_bytestream]) \n\t" + + "lhu %[tmp0], 0(%[c_bytestream]) \n\t" + "wsbh %[tmp0], %[tmp0] \n\t" PTR_SLL "%[tmp0], %[tmp0], 0x01 \n\t" -#endif PTR_SUBU "%[tmp0], %[tmp0], %[cabac_mask] \n\t" "li %[tmp1], 0x07 \n\t" @@ -94,10 +83,13 @@ static av_always_inline int get_cabac_inline(CABACContext *c, PTR_SLL "%[tmp0], %[tmp0], %[tmp1] \n\t" PTR_ADDU "%[c_low], %[c_low], %[tmp0] \n\t" -#if !UNCHECKED_BITSTREAM_READER - "bge %[c_bytestream], %[c_bytestream_end], 1f \n\t" +#if UNCHECKED_BITSTREAM_READER + PTR_ADDIU "%[c_bytestream], %[c_bytestream], 0x02 \n\t" +#else + "slt %[tmp0], %[c_bytestream], %[c_bytestream_end] \n\t" + PTR_ADDIU "%[tmp2], %[c_bytestream], 0x02 \n\t" + "movn %[c_bytestream], %[tmp2], %[tmp0] \n\t" #endif - PTR_ADDIU "%[c_bytestream], %[c_bytestream], 0X02 \n\t" "1: \n\t" : [bit]"=&r"(bit), [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [tmp2]"=&r"(tmp2), [c_range]"+&r"(c->range), [c_low]"+&r"(c->low), @@ -116,4 +108,85 @@ static av_always_inline int get_cabac_inline(CABACContext *c, return bit; } +#define get_cabac_bypass get_cabac_bypass_mips +static av_always_inline int get_cabac_bypass_mips(CABACContext *c) +{ + mips_reg tmp0, tmp1; + int res = 0; + __asm__ volatile( + PTR_SLL "%[c_low], %[c_low], 0x01 \n\t" + "and %[tmp0], %[c_low], %[cabac_mask] \n\t" + "bnez %[tmp0], 1f \n\t" + "lhu %[tmp1], 0(%[c_bytestream]) \n\t" + "wsbh %[tmp1], %[tmp1] \n\t" + PTR_SLL "%[tmp1], %[tmp1], 0x01 \n\t" + PTR_SUBU "%[tmp1], %[tmp1], %[cabac_mask] \n\t" + PTR_ADDU "%[c_low], %[c_low], %[tmp1] \n\t" +#if UNCHECKED_BITSTREAM_READER + PTR_ADDIU "%[c_bytestream], %[c_bytestream], 0x02 \n\t" +#else + "slt %[tmp0], %[c_bytestream], %[c_bytestream_end] \n\t" + PTR_ADDIU "%[tmp1], %[c_bytestream], 0x02 \n\t" + "movn %[c_bytestream], %[tmp1], %[tmp0] \n\t" +#endif + "1: \n\t" + PTR_SLL "%[tmp1], %[c_range], 0x11 \n\t" + "slt %[tmp0], %[c_low], %[tmp1] \n\t" + PTR_SUBU "%[tmp1], %[c_low], %[tmp1] \n\t" + "movz %[res], %[one], %[tmp0] \n\t" + "movz %[c_low], %[tmp1], %[tmp0] \n\t" + : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [res]"+&r"(res), + [c_range]"+&r"(c->range), [c_low]"+&r"(c->low), + [c_bytestream]"+&r"(c->bytestream) + : [cabac_mask]"r"(CABAC_MASK), +#if !UNCHECKED_BITSTREAM_READER + [c_bytestream_end]"r"(c->bytestream_end), +#endif + [one]"r"(0x01) + : "memory" + ); + return res; +} + +#define get_cabac_bypass_sign get_cabac_bypass_sign_mips +static av_always_inline int get_cabac_bypass_sign_mips(CABACContext *c, int val) +{ + mips_reg tmp0, tmp1; + int res = val; + __asm__ volatile( + PTR_SLL "%[c_low], %[c_low], 0x01 \n\t" + "and %[tmp0], %[c_low], %[cabac_mask] \n\t" + "bnez %[tmp0], 1f \n\t" + "lhu %[tmp1], 0(%[c_bytestream]) \n\t" + "wsbh %[tmp1], %[tmp1] \n\t" + PTR_SLL "%[tmp1], %[tmp1], 0x01 \n\t" + PTR_SUBU "%[tmp1], %[tmp1], %[cabac_mask] \n\t" + PTR_ADDU "%[c_low], %[c_low], %[tmp1] \n\t" +#if UNCHECKED_BITSTREAM_READER + PTR_ADDIU "%[c_bytestream], %[c_bytestream], 0x02 \n\t" +#else + "slt %[tmp0], %[c_bytestream], %[c_bytestream_end] \n\t" + PTR_ADDIU "%[tmp1], %[c_bytestream], 0x02 \n\t" + "movn %[c_bytestream], %[tmp1], %[tmp0] \n\t" +#endif + "1: \n\t" + PTR_SLL "%[tmp1], %[c_range], 0x11 \n\t" + "slt %[tmp0], %[c_low], %[tmp1] \n\t" + PTR_SUBU "%[tmp1], %[c_low], %[tmp1] \n\t" + "movz %[c_low], %[tmp1], %[tmp0] \n\t" + PTR_SUBU "%[tmp1], %[zero], %[res] \n\t" + "movn %[res], %[tmp1], %[tmp0] \n\t" + : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [res]"+&r"(res), + [c_range]"+&r"(c->range), [c_low]"+&r"(c->low), + [c_bytestream]"+&r"(c->bytestream) + : [cabac_mask]"r"(CABAC_MASK), +#if !UNCHECKED_BITSTREAM_READER + [c_bytestream_end]"r"(c->bytestream_end), +#endif + [zero]"r"(0x0) + : "memory" + ); + + return res; +} #endif /* AVCODEC_MIPS_CABAC_H */ diff --git a/libavcodec/mips/constants.c b/libavcodec/mips/constants.c index a7c4a5ccf6..6a8f1a522b 100644 --- a/libavcodec/mips/constants.c +++ b/libavcodec/mips/constants.c @@ -19,50 +19,49 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -#include "config.h" -#include "libavutil/mem.h" +#include "libavutil/intfloat.h" #include "constants.h" -DECLARE_ALIGNED(8, const uint64_t, ff_pw_1) = {0x0001000100010001ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_2) = {0x0002000200020002ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_3) = {0x0003000300030003ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_4) = {0x0004000400040004ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_5) = {0x0005000500050005ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_6) = {0x0006000600060006ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_8) = {0x0008000800080008ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_9) = {0x0009000900090009ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_10) = {0x000A000A000A000AULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_12) = {0x000C000C000C000CULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_15) = {0x000F000F000F000FULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_16) = {0x0010001000100010ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_17) = {0x0011001100110011ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_18) = {0x0012001200120012ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_20) = {0x0014001400140014ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_22) = {0x0016001600160016ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_28) = {0x001C001C001C001CULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_32) = {0x0020002000200020ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_53) = {0x0035003500350035ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_64) = {0x0040004000400040ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_128) = {0x0080008000800080ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_512) = {0x0200020002000200ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_m8tom5) = {0xFFFBFFFAFFF9FFF8ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_m4tom1) = {0xFFFFFFFEFFFDFFFCULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_1to4) = {0x0004000300020001ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_5to8) = {0x0008000700060005ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_0to3) = {0x0003000200010000ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_4to7) = {0x0007000600050004ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_8tob) = {0x000b000a00090008ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pw_ctof) = {0x000f000e000d000cULL}; - -DECLARE_ALIGNED(8, const uint64_t, ff_pb_1) = {0x0101010101010101ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pb_3) = {0x0303030303030303ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pb_80) = {0x8080808080808080ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pb_A1) = {0xA1A1A1A1A1A1A1A1ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_pb_FE) = {0xFEFEFEFEFEFEFEFEULL}; - -DECLARE_ALIGNED(8, const uint64_t, ff_rnd) = {0x0004000400040004ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_rnd2) = {0x0040004000400040ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_rnd3) = {0x0020002000200020ULL}; - -DECLARE_ALIGNED(8, const uint64_t, ff_wm1010) = {0xFFFF0000FFFF0000ULL}; -DECLARE_ALIGNED(8, const uint64_t, ff_d40000) = {0x0000000000040000ULL}; +union av_intfloat64 ff_pw_1 = {0x0001000100010001ULL}; +union av_intfloat64 ff_pw_2 = {0x0002000200020002ULL}; +union av_intfloat64 ff_pw_3 = {0x0003000300030003ULL}; +union av_intfloat64 ff_pw_4 = {0x0004000400040004ULL}; +union av_intfloat64 ff_pw_5 = {0x0005000500050005ULL}; +union av_intfloat64 ff_pw_6 = {0x0006000600060006ULL}; +union av_intfloat64 ff_pw_8 = {0x0008000800080008ULL}; +union av_intfloat64 ff_pw_9 = {0x0009000900090009ULL}; +union av_intfloat64 ff_pw_10 = {0x000A000A000A000AULL}; +union av_intfloat64 ff_pw_12 = {0x000C000C000C000CULL}; +union av_intfloat64 ff_pw_15 = {0x000F000F000F000FULL}; +union av_intfloat64 ff_pw_16 = {0x0010001000100010ULL}; +union av_intfloat64 ff_pw_17 = {0x0011001100110011ULL}; +union av_intfloat64 ff_pw_18 = {0x0012001200120012ULL}; +union av_intfloat64 ff_pw_20 = {0x0014001400140014ULL}; +union av_intfloat64 ff_pw_22 = {0x0016001600160016ULL}; +union av_intfloat64 ff_pw_28 = {0x001C001C001C001CULL}; +union av_intfloat64 ff_pw_32 = {0x0020002000200020ULL}; +union av_intfloat64 ff_pw_53 = {0x0035003500350035ULL}; +union av_intfloat64 ff_pw_64 = {0x0040004000400040ULL}; +union av_intfloat64 ff_pw_128 = {0x0080008000800080ULL}; +union av_intfloat64 ff_pw_512 = {0x0200020002000200ULL}; +union av_intfloat64 ff_pw_m8tom5 = {0xFFFBFFFAFFF9FFF8ULL}; +union av_intfloat64 ff_pw_m4tom1 = {0xFFFFFFFEFFFDFFFCULL}; +union av_intfloat64 ff_pw_1to4 = {0x0004000300020001ULL}; +union av_intfloat64 ff_pw_5to8 = {0x0008000700060005ULL}; +union av_intfloat64 ff_pw_0to3 = {0x0003000200010000ULL}; +union av_intfloat64 ff_pw_4to7 = {0x0007000600050004ULL}; +union av_intfloat64 ff_pw_8tob = {0x000b000a00090008ULL}; +union av_intfloat64 ff_pw_ctof = {0x000f000e000d000cULL}; +union av_intfloat64 ff_pw_32_1 = {0x0000000100000001ULL}; +union av_intfloat64 ff_pw_32_4 = {0x0000000400000004ULL}; +union av_intfloat64 ff_pw_32_64 = {0x0000004000000040ULL}; +union av_intfloat64 ff_pb_1 = {0x0101010101010101ULL}; +union av_intfloat64 ff_pb_3 = {0x0303030303030303ULL}; +union av_intfloat64 ff_pb_80 = {0x8080808080808080ULL}; +union av_intfloat64 ff_pb_A1 = {0xA1A1A1A1A1A1A1A1ULL}; +union av_intfloat64 ff_pb_FE = {0xFEFEFEFEFEFEFEFEULL}; +union av_intfloat64 ff_rnd = {0x0004000400040004ULL}; +union av_intfloat64 ff_rnd2 = {0x0040004000400040ULL}; +union av_intfloat64 ff_rnd3 = {0x0020002000200020ULL}; +union av_intfloat64 ff_ff_wm1010 = {0xFFFF0000FFFF0000ULL}; +union av_intfloat64 ff_d40000 = {0x0000000000040000ULL}; diff --git a/libavcodec/mips/constants.h b/libavcodec/mips/constants.h index 2604559950..bd86cd1bf1 100644 --- a/libavcodec/mips/constants.h +++ b/libavcodec/mips/constants.h @@ -22,50 +22,48 @@ #ifndef AVCODEC_MIPS_CONSTANTS_H #define AVCODEC_MIPS_CONSTANTS_H -#include - -extern const uint64_t ff_pw_1; -extern const uint64_t ff_pw_2; -extern const uint64_t ff_pw_3; -extern const uint64_t ff_pw_4; -extern const uint64_t ff_pw_5; -extern const uint64_t ff_pw_6; -extern const uint64_t ff_pw_8; -extern const uint64_t ff_pw_9; -extern const uint64_t ff_pw_10; -extern const uint64_t ff_pw_12; -extern const uint64_t ff_pw_15; -extern const uint64_t ff_pw_16; -extern const uint64_t ff_pw_17; -extern const uint64_t ff_pw_18; -extern const uint64_t ff_pw_20; -extern const uint64_t ff_pw_22; -extern const uint64_t ff_pw_28; -extern const uint64_t ff_pw_32; -extern const uint64_t ff_pw_53; -extern const uint64_t ff_pw_64; -extern const uint64_t ff_pw_128; -extern const uint64_t ff_pw_512; -extern const uint64_t ff_pw_m8tom5; -extern const uint64_t ff_pw_m4tom1; -extern const uint64_t ff_pw_1to4; -extern const uint64_t ff_pw_5to8; -extern const uint64_t ff_pw_0to3; -extern const uint64_t ff_pw_4to7; -extern const uint64_t ff_pw_8tob; -extern const uint64_t ff_pw_ctof; - -extern const uint64_t ff_pb_1; -extern const uint64_t ff_pb_3; -extern const uint64_t ff_pb_80; -extern const uint64_t ff_pb_A1; -extern const uint64_t ff_pb_FE; - -extern const uint64_t ff_rnd; -extern const uint64_t ff_rnd2; -extern const uint64_t ff_rnd3; - -extern const uint64_t ff_wm1010; -extern const uint64_t ff_d40000; +extern union av_intfloat64 ff_pw_1; +extern union av_intfloat64 ff_pw_2; +extern union av_intfloat64 ff_pw_3; +extern union av_intfloat64 ff_pw_4; +extern union av_intfloat64 ff_pw_5; +extern union av_intfloat64 ff_pw_6; +extern union av_intfloat64 ff_pw_8; +extern union av_intfloat64 ff_pw_9; +extern union av_intfloat64 ff_pw_10; +extern union av_intfloat64 ff_pw_12; +extern union av_intfloat64 ff_pw_15; +extern union av_intfloat64 ff_pw_16; +extern union av_intfloat64 ff_pw_17; +extern union av_intfloat64 ff_pw_18; +extern union av_intfloat64 ff_pw_20; +extern union av_intfloat64 ff_pw_22; +extern union av_intfloat64 ff_pw_28; +extern union av_intfloat64 ff_pw_32; +extern union av_intfloat64 ff_pw_53; +extern union av_intfloat64 ff_pw_64; +extern union av_intfloat64 ff_pw_128; +extern union av_intfloat64 ff_pw_512; +extern union av_intfloat64 ff_pw_m8tom5; +extern union av_intfloat64 ff_pw_m4tom1; +extern union av_intfloat64 ff_pw_1to4; +extern union av_intfloat64 ff_pw_5to8; +extern union av_intfloat64 ff_pw_0to3; +extern union av_intfloat64 ff_pw_4to7; +extern union av_intfloat64 ff_pw_8tob; +extern union av_intfloat64 ff_pw_ctof; +extern union av_intfloat64 ff_pw_32_1; +extern union av_intfloat64 ff_pw_32_4; +extern union av_intfloat64 ff_pw_32_64; +extern union av_intfloat64 ff_pb_1; +extern union av_intfloat64 ff_pb_3; +extern union av_intfloat64 ff_pb_80; +extern union av_intfloat64 ff_pb_A1; +extern union av_intfloat64 ff_pb_FE; +extern union av_intfloat64 ff_rnd; +extern union av_intfloat64 ff_rnd2; +extern union av_intfloat64 ff_rnd3; +extern union av_intfloat64 ff_wm1010; +extern union av_intfloat64 ff_d40000; #endif /* AVCODEC_MIPS_CONSTANTS_H */ diff --git a/libavcodec/mips/h264_deblock_msa.c b/libavcodec/mips/h264_deblock_msa.c new file mode 100644 index 0000000000..4fed55c504 --- /dev/null +++ b/libavcodec/mips/h264_deblock_msa.c @@ -0,0 +1,153 @@ +/* + * MIPS SIMD optimized H.264 deblocking code + * + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Gu Xiwei + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavcodec/bit_depth_template.c" +#include "h264dsp_mips.h" +#include "libavutil/mips/generic_macros_msa.h" +#include "libavcodec/mips/h264dsp_mips.h" + +#define h264_loop_filter_strength_iteration_msa(edges, step, mask_mv, dir, \ + d_idx, mask_dir) \ +do { \ + int b_idx = 0; \ + int step_x4 = step << 2; \ + int d_idx_12 = d_idx + 12; \ + int d_idx_52 = d_idx + 52; \ + int d_idx_x4 = d_idx << 2; \ + int d_idx_x4_48 = d_idx_x4 + 48; \ + int dir_x32 = dir * 32; \ + uint8_t *ref_t = (uint8_t*)ref; \ + uint8_t *mv_t = (uint8_t*)mv; \ + uint8_t *nnz_t = (uint8_t*)nnz; \ + uint8_t *bS_t = (uint8_t*)bS; \ + mask_mv <<= 3; \ + for (; b_idx < edges; b_idx += step) { \ + out &= mask_dir; \ + if (!(mask_mv & b_idx)) { \ + if (bidir) { \ + ref_2 = LD_SB(ref_t + d_idx_12); \ + ref_3 = LD_SB(ref_t + d_idx_52); \ + ref_0 = LD_SB(ref_t + 12); \ + ref_1 = LD_SB(ref_t + 52); \ + ref_2 = (v16i8)__msa_ilvr_w((v4i32)ref_3, (v4i32)ref_2); \ + ref_0 = (v16i8)__msa_ilvr_w((v4i32)ref_0, (v4i32)ref_0); \ + ref_1 = (v16i8)__msa_ilvr_w((v4i32)ref_1, (v4i32)ref_1); \ + ref_3 = (v16i8)__msa_shf_h((v8i16)ref_2, 0x4e); \ + ref_0 -= ref_2; \ + ref_1 -= ref_3; \ + ref_0 = (v16i8)__msa_or_v((v16u8)ref_0, (v16u8)ref_1); \ +\ + tmp_2 = LD_SH(mv_t + d_idx_x4_48); \ + tmp_3 = LD_SH(mv_t + 48); \ + tmp_4 = LD_SH(mv_t + 208); \ + tmp_5 = tmp_2 - tmp_3; \ + tmp_6 = tmp_2 - tmp_4; \ + SAT_SH2_SH(tmp_5, tmp_6, 7); \ + tmp_0 = __msa_pckev_b((v16i8)tmp_6, (v16i8)tmp_5); \ + tmp_0 += cnst_1; \ + tmp_0 = (v16i8)__msa_subs_u_b((v16u8)tmp_0, (v16u8)cnst_0);\ + tmp_0 = (v16i8)__msa_sat_s_h((v8i16)tmp_0, 7); \ + tmp_0 = __msa_pckev_b(tmp_0, tmp_0); \ + out = (v16i8)__msa_or_v((v16u8)ref_0, (v16u8)tmp_0); \ +\ + tmp_2 = LD_SH(mv_t + 208 + d_idx_x4); \ + tmp_5 = tmp_2 - tmp_3; \ + tmp_6 = tmp_2 - tmp_4; \ + SAT_SH2_SH(tmp_5, tmp_6, 7); \ + tmp_1 = __msa_pckev_b((v16i8)tmp_6, (v16i8)tmp_5); \ + tmp_1 += cnst_1; \ + tmp_1 = (v16i8)__msa_subs_u_b((v16u8)tmp_1, (v16u8)cnst_0); \ + tmp_1 = (v16i8)__msa_sat_s_h((v8i16)tmp_1, 7); \ + tmp_1 = __msa_pckev_b(tmp_1, tmp_1); \ +\ + tmp_1 = (v16i8)__msa_shf_h((v8i16)tmp_1, 0x4e); \ + out = (v16i8)__msa_or_v((v16u8)out, (v16u8)tmp_1); \ + tmp_0 = (v16i8)__msa_shf_h((v8i16)out, 0x4e); \ + out = (v16i8)__msa_min_u_b((v16u8)out, (v16u8)tmp_0); \ + } else { \ + ref_0 = LD_SB(ref_t + d_idx_12); \ + ref_3 = LD_SB(ref_t + 12); \ + tmp_2 = LD_SH(mv_t + d_idx_x4_48); \ + tmp_3 = LD_SH(mv_t + 48); \ + tmp_4 = tmp_3 - tmp_2; \ + tmp_1 = (v16i8)__msa_sat_s_h(tmp_4, 7); \ + tmp_1 = __msa_pckev_b(tmp_1, tmp_1); \ + tmp_1 += cnst_1; \ + out = (v16i8)__msa_subs_u_b((v16u8)tmp_1, (v16u8)cnst_0); \ + out = (v16i8)__msa_sat_s_h((v8i16)out, 7); \ + out = __msa_pckev_b(out, out); \ + ref_0 = ref_3 - ref_0; \ + out = (v16i8)__msa_or_v((v16u8)out, (v16u8)ref_0); \ + } \ + } \ + tmp_0 = LD_SB(nnz_t + 12); \ + tmp_1 = LD_SB(nnz_t + d_idx_12); \ + tmp_0 = (v16i8)__msa_or_v((v16u8)tmp_0, (v16u8)tmp_1); \ + tmp_0 = (v16i8)__msa_min_u_b((v16u8)tmp_0, (v16u8)cnst_2); \ + out = (v16i8)__msa_min_u_b((v16u8)out, (v16u8)cnst_2); \ + tmp_0 = (v16i8)((v8i16)tmp_0 << 1); \ + tmp_0 = (v16i8)__msa_max_u_b((v16u8)out, (v16u8)tmp_0); \ + tmp_0 = __msa_ilvr_b(zero, tmp_0); \ + ST_D1(tmp_0, 0, bS_t + dir_x32); \ + ref_t += step; \ + mv_t += step_x4; \ + nnz_t += step; \ + bS_t += step; \ + } \ +} while(0) + +void ff_h264_loop_filter_strength_msa(int16_t bS[2][4][4], uint8_t nnz[40], + int8_t ref[2][40], int16_t mv[2][40][2], + int bidir, int edges, int step, + int mask_mv0, int mask_mv1, int field) +{ + v16i8 out; + v16i8 ref_0, ref_1, ref_2, ref_3; + v16i8 tmp_0, tmp_1; + v8i16 tmp_2, tmp_3, tmp_4, tmp_5, tmp_6; + v16i8 cnst_0, cnst_1, cnst_2; + v16i8 zero = { 0 }; + v16i8 one = __msa_fill_b(0xff); + if (field) { + cnst_0 = (v16i8)__msa_fill_h(0x206); + cnst_1 = (v16i8)__msa_fill_h(0x103); + cnst_2 = (v16i8)__msa_fill_h(0x101); + } else { + cnst_0 = __msa_fill_b(0x6); + cnst_1 = __msa_fill_b(0x3); + cnst_2 = __msa_fill_b(0x1); + } + step <<= 3; + edges <<= 3; + + h264_loop_filter_strength_iteration_msa(edges, step, mask_mv1, 1, -8, zero); + h264_loop_filter_strength_iteration_msa(32, 8, mask_mv0, 0, -1, one); + + LD_SB2((int8_t*)bS, 16, tmp_0, tmp_1); + tmp_2 = (v8i16)__msa_ilvl_d((v2i64)tmp_0, (v2i64)tmp_0); + tmp_3 = (v8i16)__msa_ilvl_d((v2i64)tmp_1, (v2i64)tmp_1); + TRANSPOSE4x4_SH_SH(tmp_0, tmp_2, tmp_1, tmp_3, tmp_2, tmp_3, tmp_4, tmp_5); + tmp_0 = (v16i8)__msa_ilvr_d((v2i64)tmp_3, (v2i64)tmp_2); + tmp_1 = (v16i8)__msa_ilvr_d((v2i64)tmp_5, (v2i64)tmp_4); + ST_SB2(tmp_0, tmp_1, (int8_t*)bS, 16); +} diff --git a/libavcodec/mips/h264chroma_init_mips.c b/libavcodec/mips/h264chroma_init_mips.c index 6bb19d3ddd..755cc0401f 100644 --- a/libavcodec/mips/h264chroma_init_mips.c +++ b/libavcodec/mips/h264chroma_init_mips.c @@ -28,7 +28,15 @@ av_cold void ff_h264chroma_init_mips(H264ChromaContext *c, int bit_depth) int cpu_flags = av_get_cpu_flags(); int high_bit_depth = bit_depth > 8; - /* MMI apears to be faster than MSA here */ + if (have_mmi(cpu_flags)) { + if (!high_bit_depth) { + c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_mmi; + c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_mmi; + c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_mmi; + c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_mmi; + } + } + if (have_msa(cpu_flags)) { if (!high_bit_depth) { c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_msa; @@ -40,13 +48,4 @@ av_cold void ff_h264chroma_init_mips(H264ChromaContext *c, int bit_depth) c->avg_h264_chroma_pixels_tab[2] = ff_avg_h264_chroma_mc2_msa; } } - - if (have_mmi(cpu_flags)) { - if (!high_bit_depth) { - c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_mmi; - c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_mmi; - c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_mmi; - c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_mmi; - } - } } diff --git a/libavcodec/mips/h264chroma_mmi.c b/libavcodec/mips/h264chroma_mmi.c index 739dd7d4d6..cc2d7cb7e9 100644 --- a/libavcodec/mips/h264chroma_mmi.c +++ b/libavcodec/mips/h264chroma_mmi.c @@ -29,12 +29,12 @@ void ff_put_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, int h, int x, int y) { - int A = 64, B, C, D, E; double ftmp[12]; - uint64_t tmp[1]; + union mmi_intfloat64 A, B, C, D, E; + A.i = 64; if (!(x || y)) { - /* x=0, y=0, A=64 */ + /* x=0, y=0, A.i=64 */ __asm__ volatile ( "1: \n\t" MMI_ULDC1(%[ftmp0], %[src], 0x00) @@ -66,14 +66,13 @@ void ff_put_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, ); } else if (x && y) { /* x!=0, y!=0 */ - D = x * y; - B = (x << 3) - D; - C = (y << 3) - D; - A = 64 - D - B - C; + D.i = x * y; + B.i = (x << 3) - D.i; + C.i = (y << 3) - D.i; + A.i = 64 - D.i - B.i - C.i; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[B], %[B], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp9] \n\t" @@ -158,22 +157,21 @@ void ff_put_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, [ftmp6]"=&f"(ftmp[6]), [ftmp7]"=&f"(ftmp[7]), [ftmp8]"=&f"(ftmp[8]), [ftmp9]"=&f"(ftmp[9]), [ftmp10]"=&f"(ftmp[10]), [ftmp11]"=&f"(ftmp[11]), - [tmp0]"=&r"(tmp[0]), [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) - : [stride]"r"((mips_reg)stride),[ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [B]"f"(B), - [C]"f"(C), [D]"f"(D) + : [stride]"r"((mips_reg)stride),[ff_pw_32]"f"(ff_pw_32.f), + [A]"f"(A.f), [B]"f"(B.f), + [C]"f"(C.f), [D]"f"(D.f), + [tmp0]"r"(0x06) : "memory" ); } else if (x) { /* x!=0, y==0 */ - E = x << 3; - A = 64 - E; + E.i = x << 3; + A.i = 64 - E.i; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[E], %[E], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp7] \n\t" @@ -207,22 +205,20 @@ void ff_put_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, [ftmp2]"=&f"(ftmp[2]), [ftmp3]"=&f"(ftmp[3]), [ftmp4]"=&f"(ftmp[4]), [ftmp5]"=&f"(ftmp[5]), [ftmp6]"=&f"(ftmp[6]), [ftmp7]"=&f"(ftmp[7]), - [tmp0]"=&r"(tmp[0]), [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) : [stride]"r"((mips_reg)stride), - [ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [E]"f"(E) + [ff_pw_32]"f"(ff_pw_32.f), [tmp0]"r"(0x06), + [A]"f"(A.f), [E]"f"(E.f) : "memory" ); } else { /* x==0, y!=0 */ - E = y << 3; - A = 64 - E; + E.i = y << 3; + A.i = 64 - E.i; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[E], %[E], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp7] \n\t" @@ -276,12 +272,12 @@ void ff_put_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, [ftmp2]"=&f"(ftmp[2]), [ftmp3]"=&f"(ftmp[3]), [ftmp4]"=&f"(ftmp[4]), [ftmp5]"=&f"(ftmp[5]), [ftmp6]"=&f"(ftmp[6]), [ftmp7]"=&f"(ftmp[7]), - [ftmp8]"=&f"(ftmp[8]), [tmp0]"=&r"(tmp[0]), + [ftmp8]"=&f"(ftmp[8]), [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) : [stride]"r"((mips_reg)stride), - [ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [E]"f"(E) + [ff_pw_32]"f"(ff_pw_32.f), [A]"f"(A.f), + [E]"f"(E.f), [tmp0]"r"(0x06) : "memory" ); } @@ -290,12 +286,12 @@ void ff_put_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, void ff_avg_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, int h, int x, int y) { - int A = 64, B, C, D, E; double ftmp[10]; - uint64_t tmp[1]; + union mmi_intfloat64 A, B, C, D, E; + A.i = 64; if(!(x || y)){ - /* x=0, y=0, A=64 */ + /* x=0, y=0, A.i=64 */ __asm__ volatile ( "1: \n\t" MMI_ULDC1(%[ftmp0], %[src], 0x00) @@ -323,13 +319,12 @@ void ff_avg_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, ); } else if (x && y) { /* x!=0, y!=0 */ - D = x * y; - B = (x << 3) - D; - C = (y << 3) - D; - A = 64 - D - B - C; + D.i = x * y; + B.i = (x << 3) - D.i; + C.i = (y << 3) - D.i; + A.i = 64 - D.i - B.i - C.i; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[B], %[B], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp9] \n\t" @@ -383,21 +378,20 @@ void ff_avg_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, [ftmp4]"=&f"(ftmp[4]), [ftmp5]"=&f"(ftmp[5]), [ftmp6]"=&f"(ftmp[6]), [ftmp7]"=&f"(ftmp[7]), [ftmp8]"=&f"(ftmp[8]), [ftmp9]"=&f"(ftmp[9]), - [tmp0]"=&r"(tmp[0]), [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) - : [stride]"r"((mips_reg)stride),[ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [B]"f"(B), - [C]"f"(C), [D]"f"(D) + : [stride]"r"((mips_reg)stride),[ff_pw_32]"f"(ff_pw_32.f), + [A]"f"(A.f), [B]"f"(B.f), + [C]"f"(C.f), [D]"f"(D.f), + [tmp0]"r"(0x06) : "memory" ); } else if (x) { /* x!=0, y==0 */ - E = x << 3; - A = 64 - E; + E.i = x << 3; + A.i = 64 - E.i; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[E], %[E], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp7] \n\t" @@ -433,21 +427,19 @@ void ff_avg_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, [ftmp2]"=&f"(ftmp[2]), [ftmp3]"=&f"(ftmp[3]), [ftmp4]"=&f"(ftmp[4]), [ftmp5]"=&f"(ftmp[5]), [ftmp6]"=&f"(ftmp[6]), [ftmp7]"=&f"(ftmp[7]), - [tmp0]"=&r"(tmp[0]), [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) : [stride]"r"((mips_reg)stride), - [ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [E]"f"(E) + [ff_pw_32]"f"(ff_pw_32.f), [tmp0]"r"(0x06), + [A]"f"(A.f), [E]"f"(E.f) : "memory" ); } else { /* x==0, y!=0 */ - E = y << 3; - A = 64 - E; + E.i = y << 3; + A.i = 64 - E.i; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[E], %[E], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp7] \n\t" @@ -469,8 +461,8 @@ void ff_avg_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, "pmullh %[ftmp6], %[ftmp6], %[E] \n\t" "paddh %[ftmp2], %[ftmp4], %[ftmp6] \n\t" - "paddh %[ftmp1], %[ftmp1], %[ff_pw_32] \n\t" - "paddh %[ftmp2], %[ftmp2], %[ff_pw_32] \n\t" + "paddh %[ftmp1], %[ftmp1], %[ff_pw_32] \n\t" + "paddh %[ftmp2], %[ftmp2], %[ff_pw_32] \n\t" "psrlh %[ftmp1], %[ftmp1], %[ftmp7] \n\t" "psrlh %[ftmp2], %[ftmp2], %[ftmp7] \n\t" "packushb %[ftmp1], %[ftmp1], %[ftmp2] \n\t" @@ -483,12 +475,11 @@ void ff_avg_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, [ftmp2]"=&f"(ftmp[2]), [ftmp3]"=&f"(ftmp[3]), [ftmp4]"=&f"(ftmp[4]), [ftmp5]"=&f"(ftmp[5]), [ftmp6]"=&f"(ftmp[6]), [ftmp7]"=&f"(ftmp[7]), - [tmp0]"=&r"(tmp[0]), [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) : [stride]"r"((mips_reg)stride), - [ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [E]"f"(E) + [ff_pw_32]"f"(ff_pw_32.f), [tmp0]"r"(0x06), + [A]"f"(A.f), [E]"f"(E.f) : "memory" ); } @@ -497,20 +488,19 @@ void ff_avg_h264_chroma_mc8_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, void ff_put_h264_chroma_mc4_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, int h, int x, int y) { - const int A = (8 - x) * (8 - y); - const int B = x * (8 - y); - const int C = (8 - x) * y; - const int D = x * y; - const int E = B + C; double ftmp[8]; - uint64_t tmp[1]; mips_reg addr[1]; + union mmi_intfloat64 A, B, C, D, E; DECLARE_VAR_LOW32; + A.i = (8 - x) * (8 - y); + B.i = x * (8 - y); + C.i = (8 - x) * y; + D.i = x * y; + E.i = B.i + C.i; - if (D) { + if (D.i) { __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[B], %[B], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp7] \n\t" @@ -547,20 +537,19 @@ void ff_put_h264_chroma_mc4_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, [ftmp2]"=&f"(ftmp[2]), [ftmp3]"=&f"(ftmp[3]), [ftmp4]"=&f"(ftmp[4]), [ftmp5]"=&f"(ftmp[5]), [ftmp6]"=&f"(ftmp[6]), [ftmp7]"=&f"(ftmp[7]), - [tmp0]"=&r"(tmp[0]), RESTRICT_ASM_LOW32 [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) - : [stride]"r"((mips_reg)stride),[ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [B]"f"(B), - [C]"f"(C), [D]"f"(D) + : [stride]"r"((mips_reg)stride),[ff_pw_32]"f"(ff_pw_32.f), + [A]"f"(A.f), [B]"f"(B.f), + [C]"f"(C.f), [D]"f"(D.f), + [tmp0]"r"(0x06) : "memory" ); - } else if (E) { - const int step = C ? stride : 1; + } else if (E.i) { + const int step = C.i ? stride : 1; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[E], %[E], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp5] \n\t" @@ -585,14 +574,13 @@ void ff_put_h264_chroma_mc4_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, : [ftmp0]"=&f"(ftmp[0]), [ftmp1]"=&f"(ftmp[1]), [ftmp2]"=&f"(ftmp[2]), [ftmp3]"=&f"(ftmp[3]), [ftmp4]"=&f"(ftmp[4]), [ftmp5]"=&f"(ftmp[5]), - [tmp0]"=&r"(tmp[0]), RESTRICT_ASM_LOW32 [addr0]"=&r"(addr[0]), [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) : [stride]"r"((mips_reg)stride),[step]"r"((mips_reg)step), - [ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [E]"f"(E) + [ff_pw_32]"f"(ff_pw_32.f), [tmp0]"r"(0x06), + [A]"f"(A.f), [E]"f"(E.f) : "memory" ); } else { @@ -621,20 +609,19 @@ void ff_put_h264_chroma_mc4_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, void ff_avg_h264_chroma_mc4_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, int h, int x, int y) { - const int A = (8 - x) *(8 - y); - const int B = x * (8 - y); - const int C = (8 - x) * y; - const int D = x * y; - const int E = B + C; double ftmp[8]; - uint64_t tmp[1]; mips_reg addr[1]; + union mmi_intfloat64 A, B, C, D, E; DECLARE_VAR_LOW32; + A.i = (8 - x) *(8 - y); + B.i = x * (8 - y); + C.i = (8 - x) * y; + D.i = x * y; + E.i = B.i + C.i; - if (D) { + if (D.i) { __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[B], %[B], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp7] \n\t" @@ -673,20 +660,19 @@ void ff_avg_h264_chroma_mc4_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, [ftmp2]"=&f"(ftmp[2]), [ftmp3]"=&f"(ftmp[3]), [ftmp4]"=&f"(ftmp[4]), [ftmp5]"=&f"(ftmp[5]), [ftmp6]"=&f"(ftmp[6]), [ftmp7]"=&f"(ftmp[7]), - [tmp0]"=&r"(tmp[0]), RESTRICT_ASM_LOW32 [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) - : [stride]"r"((mips_reg)stride),[ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [B]"f"(B), - [C]"f"(C), [D]"f"(D) + : [stride]"r"((mips_reg)stride),[ff_pw_32]"f"(ff_pw_32.f), + [A]"f"(A.f), [B]"f"(B.f), + [C]"f"(C.f), [D]"f"(D.f), + [tmp0]"r"(0x06) : "memory" ); - } else if (E) { - const int step = C ? stride : 1; + } else if (E.i) { + const int step = C.i ? stride : 1; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "dli %[tmp0], 0x06 \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[E], %[E], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp5] \n\t" @@ -713,14 +699,13 @@ void ff_avg_h264_chroma_mc4_mmi(uint8_t *dst, uint8_t *src, ptrdiff_t stride, : [ftmp0]"=&f"(ftmp[0]), [ftmp1]"=&f"(ftmp[1]), [ftmp2]"=&f"(ftmp[2]), [ftmp3]"=&f"(ftmp[3]), [ftmp4]"=&f"(ftmp[4]), [ftmp5]"=&f"(ftmp[5]), - [tmp0]"=&r"(tmp[0]), RESTRICT_ASM_LOW32 [addr0]"=&r"(addr[0]), [dst]"+&r"(dst), [src]"+&r"(src), [h]"+&r"(h) : [stride]"r"((mips_reg)stride),[step]"r"((mips_reg)step), - [ff_pw_32]"f"(ff_pw_32), - [A]"f"(A), [E]"f"(E) + [ff_pw_32]"f"(ff_pw_32.f), [tmp0]"r"(0x06), + [A]"f"(A.f), [E]"f"(E.f) : "memory" ); } else { diff --git a/libavcodec/mips/h264dsp_init_mips.c b/libavcodec/mips/h264dsp_init_mips.c index 9cd05e0f2f..02669e0059 100644 --- a/libavcodec/mips/h264dsp_init_mips.c +++ b/libavcodec/mips/h264dsp_init_mips.c @@ -78,8 +78,12 @@ av_cold void ff_h264dsp_init_mips(H264DSPContext *c, const int bit_depth, } if (have_msa(cpu_flags)) { + if (chroma_format_idc <= 1) + c->h264_loop_filter_strength = ff_h264_loop_filter_strength_msa; if (bit_depth == 8) { - c->h264_v_loop_filter_luma = ff_h264_v_lpf_luma_inter_msa; + /* TODO: Presently, MMI version of h264_v_loop_filter_luma are + more effective than MSA. To be refined. */ + //c->h264_v_loop_filter_luma = ff_h264_v_lpf_luma_inter_msa; c->h264_h_loop_filter_luma = ff_h264_h_lpf_luma_inter_msa; c->h264_h_loop_filter_luma_mbaff = ff_h264_h_loop_filter_luma_mbaff_msa; diff --git a/libavcodec/mips/h264dsp_mips.h b/libavcodec/mips/h264dsp_mips.h index 35e16c41b3..5847ef36fe 100644 --- a/libavcodec/mips/h264dsp_mips.h +++ b/libavcodec/mips/h264dsp_mips.h @@ -319,6 +319,10 @@ void ff_vp8_pred8x8_129_dc_8_msa(uint8_t *src, ptrdiff_t stride); void ff_vp8_pred16x16_127_dc_8_msa(uint8_t *src, ptrdiff_t stride); void ff_vp8_pred16x16_129_dc_8_msa(uint8_t *src, ptrdiff_t stride); +void ff_h264_loop_filter_strength_msa(int16_t bS[2][4][4], uint8_t nnz[40], + int8_t ref[2][40], int16_t mv[2][40][2], int bidir, int edges, + int step, int mask_mv0, int mask_mv1, int field); + void ff_h264_add_pixels4_8_mmi(uint8_t *_dst, int16_t *_src, int stride); void ff_h264_idct_add_8_mmi(uint8_t *dst, int16_t *block, int stride); void ff_h264_idct8_add_8_mmi(uint8_t *dst, int16_t *block, int stride); diff --git a/libavcodec/mips/h264dsp_mmi.c b/libavcodec/mips/h264dsp_mmi.c index 173e191c77..66a82c131e 100644 --- a/libavcodec/mips/h264dsp_mmi.c +++ b/libavcodec/mips/h264dsp_mmi.c @@ -33,7 +33,7 @@ void ff_h264_add_pixels4_8_mmi(uint8_t *dst, int16_t *src, int stride) DECLARE_VAR_LOW32; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" MMI_LDC1(%[ftmp1], %[src], 0x00) MMI_LDC1(%[ftmp2], %[src], 0x08) MMI_LDC1(%[ftmp3], %[src], 0x10) @@ -88,7 +88,7 @@ void ff_h264_idct_add_8_mmi(uint8_t *dst, int16_t *block, int stride) MMI_LDC1(%[ftmp2], %[block], 0x10) MMI_LDC1(%[ftmp3], %[block], 0x18) /* memset(block, 0, 32) */ - "xor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" + "pxor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" "gssqc1 %[ftmp4], %[ftmp4], 0x00(%[block]) \n\t" "gssqc1 %[ftmp4], %[ftmp4], 0x10(%[block]) \n\t" "dli %[tmp0], 0x01 \n\t" @@ -126,7 +126,7 @@ void ff_h264_idct_add_8_mmi(uint8_t *dst, int16_t *block, int stride) "psubh %[ftmp5], %[ftmp5], %[ftmp4] \n\t" MMI_ULWC1(%[ftmp2], %[dst], 0x00) MMI_LWXC1(%[ftmp0], %[dst], %[stride], 0x00) - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" "psrah %[ftmp3], %[ftmp10], %[ftmp9] \n\t" "psrah %[ftmp4], %[ftmp11], %[ftmp9] \n\t" "punpcklbh %[ftmp2], %[ftmp2], %[ftmp7] \n\t" @@ -161,7 +161,7 @@ void ff_h264_idct_add_8_mmi(uint8_t *dst, int16_t *block, int stride) RESTRICT_ASM_ADDRT [tmp0]"=&r"(tmp[0]) : [dst]"r"(dst), [block]"r"(block), - [stride]"r"((mips_reg)stride), [ff_pw_32]"f"(ff_pw_32) + [stride]"r"((mips_reg)stride), [ff_pw_32]"f"(ff_pw_32.f) : "memory" ); @@ -418,7 +418,7 @@ void ff_h264_idct8_add_8_mmi(uint8_t *dst, int16_t *block, int stride) "psubh %[ftmp7], %[ftmp7], %[ftmp1] \n\t" MMI_SDC1(%[ftmp0], $sp, 0x10) "dmfc1 %[tmp1], %[ftmp2] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp2] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp2] \n\t" MMI_SDC1(%[ftmp2], %[block], 0x00) MMI_SDC1(%[ftmp2], %[block], 0x08) MMI_SDC1(%[ftmp2], %[block], 0x10) @@ -554,7 +554,7 @@ void ff_h264_idct8_add_8_mmi(uint8_t *dst, int16_t *block, int stride) "psubh %[ftmp3], %[ftmp3], %[ftmp4] \n\t" MMI_SDC1(%[ftmp7], $sp, 0x18) "dmfc1 %[tmp2], %[ftmp0] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" MMI_ULWC1(%[ftmp6], %[addr0], 0x00) MMI_LWXC1(%[ftmp7], %[addr0], %[stride], 0x00) "psrah %[ftmp2], %[ftmp2], %[ftmp10] \n\t" @@ -645,7 +645,7 @@ void ff_h264_idct_dc_add_8_mmi(uint8_t *dst, int16_t *block, int stride) __asm__ volatile ( "mtc1 %[dc], %[ftmp5] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[ftmp5], %[ftmp5], %[ftmp0] \n\t" MMI_ULWC1(%[ftmp1], %[dst0], 0x00) MMI_ULWC1(%[ftmp2], %[dst1], 0x00) @@ -689,7 +689,7 @@ void ff_h264_idct8_dc_add_8_mmi(uint8_t *dst, int16_t *block, int stride) __asm__ volatile ( "mtc1 %[dc], %[ftmp5] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[ftmp5], %[ftmp5], %[ftmp0] \n\t" MMI_LDC1(%[ftmp1], %[dst0], 0x00) MMI_LDC1(%[ftmp2], %[dst1], 0x00) @@ -928,7 +928,7 @@ void ff_h264_luma_dc_dequant_idct_8_mmi(int16_t *output, int16_t *input, "packsswh %[ftmp0], %[ftmp0], %[ftmp1] \n\t" "packsswh %[ftmp2], %[ftmp2], %[ftmp5] \n\t" "dmfc1 %[tmp1], %[ftmp0] \n\t" - "dsrl %[ftmp0], %[ftmp0], %[ftmp9] \n\t" + "ssrld %[ftmp0], %[ftmp0], %[ftmp9] \n\t" "mfc1 %[input], %[ftmp0] \n\t" "sh %[tmp1], 0x00(%[output]) \n\t" "sh %[input], 0x80(%[output]) \n\t" @@ -937,7 +937,7 @@ void ff_h264_luma_dc_dequant_idct_8_mmi(int16_t *output, int16_t *input, "sh %[tmp1], 0x20(%[output]) \n\t" "sh %[input], 0xa0(%[output]) \n\t" "dmfc1 %[tmp1], %[ftmp2] \n\t" - "dsrl %[ftmp2], %[ftmp2], %[ftmp9] \n\t" + "ssrld %[ftmp2], %[ftmp2], %[ftmp9] \n\t" "mfc1 %[input], %[ftmp2] \n\t" "sh %[tmp1], 0x40(%[output]) \n\t" "sh %[input], 0xc0(%[output]) \n\t" @@ -962,7 +962,7 @@ void ff_h264_luma_dc_dequant_idct_8_mmi(int16_t *output, int16_t *input, "packsswh %[ftmp3], %[ftmp3], %[ftmp1] \n\t" "packsswh %[ftmp4], %[ftmp4], %[ftmp5] \n\t" "dmfc1 %[tmp1], %[ftmp3] \n\t" - "dsrl %[ftmp3], %[ftmp3], %[ftmp9] \n\t" + "ssrld %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "mfc1 %[input], %[ftmp3] \n\t" "sh %[tmp1], 0x100(%[output]) \n\t" "sh %[input], 0x180(%[output]) \n\t" @@ -971,7 +971,7 @@ void ff_h264_luma_dc_dequant_idct_8_mmi(int16_t *output, int16_t *input, "sh %[tmp1], 0x120(%[output]) \n\t" "sh %[input], 0x1a0(%[output]) \n\t" "dmfc1 %[tmp1], %[ftmp4] \n\t" - "dsrl %[ftmp4], %[ftmp4], %[ftmp9] \n\t" + "ssrld %[ftmp4], %[ftmp4], %[ftmp9] \n\t" "mfc1 %[input], %[ftmp4] \n\t" "sh %[tmp1], 0x140(%[output]) \n\t" "sh %[input], 0x1c0(%[output]) \n\t" @@ -1015,7 +1015,7 @@ void ff_h264_luma_dc_dequant_idct_8_mmi(int16_t *output, int16_t *input, "packsswh %[ftmp0], %[ftmp0], %[ftmp1] \n\t" "packsswh %[ftmp2], %[ftmp2], %[ftmp5] \n\t" "dmfc1 %[tmp1], %[ftmp0] \n\t" - "dsrl %[ftmp0], %[ftmp0], %[ftmp9] \n\t" + "ssrld %[ftmp0], %[ftmp0], %[ftmp9] \n\t" "sh %[tmp1], 0x00(%[output]) \n\t" "mfc1 %[input], %[ftmp0] \n\t" "dsrl %[tmp1], %[tmp1], 0x10 \n\t" @@ -1024,7 +1024,7 @@ void ff_h264_luma_dc_dequant_idct_8_mmi(int16_t *output, int16_t *input, PTR_SRL "%[input], %[input], 0x10 \n\t" "dmfc1 %[tmp1], %[ftmp2] \n\t" "sh %[input], 0xa0(%[output]) \n\t" - "dsrl %[ftmp2], %[ftmp2], %[ftmp9] \n\t" + "ssrld %[ftmp2], %[ftmp2], %[ftmp9] \n\t" "sh %[tmp1], 0x40(%[output]) \n\t" "mfc1 %[input], %[ftmp2] \n\t" "dsrl %[tmp1], %[tmp1], 0x10 \n\t" @@ -1049,7 +1049,7 @@ void ff_h264_luma_dc_dequant_idct_8_mmi(int16_t *output, int16_t *input, "packsswh %[ftmp3], %[ftmp3], %[ftmp1] \n\t" "packsswh %[ftmp4], %[ftmp4], %[ftmp5] \n\t" "dmfc1 %[tmp1], %[ftmp3] \n\t" - "dsrl %[ftmp3], %[ftmp3], %[ftmp9] \n\t" + "ssrld %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "mfc1 %[input], %[ftmp3] \n\t" "sh %[tmp1], 0x100(%[output]) \n\t" "sh %[input], 0x180(%[output]) \n\t" @@ -1058,7 +1058,7 @@ void ff_h264_luma_dc_dequant_idct_8_mmi(int16_t *output, int16_t *input, "sh %[tmp1], 0x120(%[output]) \n\t" "sh %[input], 0x1a0(%[output]) \n\t" "dmfc1 %[tmp1], %[ftmp4] \n\t" - "dsrl %[ftmp4], %[ftmp4], %[ftmp9] \n\t" + "ssrld %[ftmp4], %[ftmp4], %[ftmp9] \n\t" "mfc1 %[input], %[ftmp4] \n\t" "sh %[tmp1], 0x140(%[output]) \n\t" "sh %[input], 0x1c0(%[output]) \n\t" @@ -1077,7 +1077,7 @@ void ff_h264_luma_dc_dequant_idct_8_mmi(int16_t *output, int16_t *input, RESTRICT_ASM_ALL64 [output]"+&r"(output), [input]"+&r"(input), [qmul]"+&r"(qmul) - : [ff_pw_1]"f"(ff_pw_1) + : [ff_pw_1]"f"(ff_pw_1.f) : "memory" ); } @@ -1143,7 +1143,7 @@ void ff_h264_weight_pixels16_8_mmi(uint8_t *block, ptrdiff_t stride, int height, for (y=0; y> 2], pTc[1 >> 2], pTc[2 >> 2], pTc[3 >> 2], + pTc[4 >> 2], pTc[5 >> 2], pTc[6 >> 2], pTc[7 >> 2], + pTc[8 >> 2], pTc[9 >> 2], pTc[10 >> 2], pTc[11 >> 2], + pTc[12 >> 2], pTc[13 >> 2], pTc[14 >> 2], pTc[15 >> 2] }; + negTc = zero - tc; + iTc = tc; + + // Load data from pPix + LD_SH8(pPix - 3, iStride, t0, t1, t2, t3, q1_l, q1_r, q2_l, q2_r); + LD_SH8(pPix + 8 * iStride - 3, iStride, p0_l, p0_r, p1_l, p1_r, + p2_l, p2_r, q0_l, q0_r); + TRANSPOSE16x8_UB_UB(t0, t1, t2, t3, q1_l, q1_r, q2_l, q2_r, + p0_l, p0_r, p1_l, p1_r, p2_l, p2_r, q0_l, q0_r, + p2, p1, p0, q0, q1, q2, alpha, beta); + + alpha = (v16u8)__msa_fill_b(iAlpha); + beta = (v16u8)__msa_fill_b(iBeta); + + bDetaP0Q0 = __msa_asub_u_b(p0, q0); + bDetaP1P0 = __msa_asub_u_b(p1, p0); + bDetaQ1Q0 = __msa_asub_u_b(q1, q0); + bDetaP2P0 = __msa_asub_u_b(p2, p0); + bDetaQ2Q0 = __msa_asub_u_b(q2, q0); + bDetaP0Q0 = (v16u8)__msa_clt_u_b(bDetaP0Q0, alpha); + bDetaP1P0 = (v16u8)__msa_clt_u_b(bDetaP1P0, beta); + bDetaQ1Q0 = (v16u8)__msa_clt_u_b(bDetaQ1Q0, beta); + bDetaP2P0 = (v16u8)__msa_clt_u_b(bDetaP2P0, beta); + bDetaQ2Q0 = (v16u8)__msa_clt_u_b(bDetaQ2Q0, beta); + + // Unsigned extend p0, p1, p2, q0, q1, q2 from 8 bits to 16 bits + ILVRL_B2_SH(zero, p0, p0_r, p0_l); + ILVRL_B2_SH(zero, p1, p1_r, p1_l); + ILVRL_B2_SH(zero, p2, p2_r, p2_l); + ILVRL_B2_SH(zero, q0, q0_r, q0_l); + ILVRL_B2_SH(zero, q1, q1_r, q1_l); + ILVRL_B2_SH(zero, q2, q2_r, q2_l); + // Signed extend tc, negTc from 8 bits to 16 bits + flags = __msa_clt_s_b(tc, zero); + ILVRL_B2(v8i16, flags, tc, tc_r, tc_l); + flags = __msa_clt_s_b(negTc, zero); + ILVRL_B2(v8i16, flags, negTc, negTc_r, negTc_l); + + f = (v16i8)bDetaP0Q0 & (v16i8)bDetaP1P0 & (v16i8)bDetaQ1Q0; + flags = f & (v16i8)bDetaP2P0; + flags = __msa_ceq_b(flags, zero); + iTc += ((~flags) & const_1_b); + flags = f & (v16i8)bDetaQ2Q0; + flags = __msa_ceq_b(flags, zero); + iTc += ((~flags) & const_1_b); + negiTc = zero - iTc; + // Signed extend iTc, negiTc from 8 bits to 16 bits + flags = __msa_clt_s_b(iTc, zero); + ILVRL_B2(v8i16, flags, iTc, iTc_r, iTc_l); + flags = __msa_clt_s_b(negiTc, zero); + ILVRL_B2(v8i16, flags, negiTc, negiTc_r, negiTc_l); + + // Calculate the left part + // p1 + t0 = (p2_l + ((p0_l + q0_l + const_1_h) >> 1) - (p1_l << 1)) >> 1; + t0 = __msa_max_s_h(negTc_l, t0); + t0 = __msa_min_s_h(tc_l, t0); + t1 = p1_l + t0; + // q1 + t0 = (q2_l + ((p0_l + q0_l + const_1_h) >> 1) - (q1_l << 1)) >> 1; + t0 = __msa_max_s_h(negTc_l, t0); + t0 = __msa_min_s_h(tc_l, t0); + t2 = q1_l + t0; + // iDeta + t0 = (((q0_l - p0_l) << 2) + (p1_l - q1_l) + const_4_h) >> 3; + t0 = __msa_max_s_h(negiTc_l, t0); + t0 = __msa_min_s_h(iTc_l, t0); + p1_l = t1; + q1_l = t2; + // p0 + t1 = p0_l + t0; + t2 = t1 & const_not_255_h; + t3 = __msa_cle_s_h((v8i16)zero, t1); + flags = (v16i8)__msa_ceq_h(t2, (v8i16)zero); + p0_l = (t1 & (v8i16)flags) + (t3 & (v8i16)(~flags)); + // q0 + t1 = q0_l - t0; + t2 = t1 & const_not_255_h; + t3 = __msa_cle_s_h((v8i16)zero, t1); + flags = (v16i8)__msa_ceq_h(t2, (v8i16)zero); + q0_l = (t1 & (v8i16)flags) + (t3 & (v8i16)(~flags)); + + // Calculate the right part + // p1 + t0 = (p2_r + ((p0_r + q0_r + const_1_h) >> 1) - (p1_r << 1)) >> 1; + t0 = __msa_max_s_h(negTc_r, t0); + t0 = __msa_min_s_h(tc_r, t0); + t1 = p1_r + t0; + // q1 + t0 = (q2_r + ((p0_r + q0_r + const_1_h) >> 1) - (q1_r << 1)) >> 1; + t0 = __msa_max_s_h(negTc_r, t0); + t0 = __msa_min_s_h(tc_r, t0); + t2 = q1_r + t0; + // iDeta + t0 = (((q0_r - p0_r) << 2) + (p1_r - q1_r) + const_4_h) >> 3; + t0 = __msa_max_s_h(negiTc_r, t0); + t0 = __msa_min_s_h(iTc_r, t0); + p1_r = t1; + q1_r = t2; + // p0 + t1 = p0_r + t0; + t2 = t1 & const_not_255_h; + t3 = __msa_cle_s_h((v8i16)zero, t1); + flags = (v16i8)__msa_ceq_h(t2, (v8i16)zero); + p0_r = (t1 & (v8i16)flags) + (t3 & (v8i16)(~flags)); + // q0 + t1 = q0_r - t0; + t2 = t1 & const_not_255_h; + t3 = __msa_cle_s_h((v8i16)zero, t1); + flags = (v16i8)__msa_ceq_h(t2, (v8i16)zero); + q0_r = (t1 & (v8i16)flags) + (t3 & (v8i16)(~flags)); + + // Combined left and right + PCKEV_B4(v8i16, p1_l, p1_r, p0_l, p0_r, q0_l, q0_r, q1_l, q1_r, + t0, t1, t2, t3); + flags = (v16i8)__msa_cle_s_b(zero, tc); + flags &= f; + p0 = (v16u8)(((v16i8)t1 & flags) + (p0 & (~flags))); + q0 = (v16u8)(((v16i8)t2 & flags) + (q0 & (~flags))); + // Using t1, t2 as temporary flags + t1 = (v8i16)(flags & (~(__msa_ceq_b((v16i8)bDetaP2P0, zero)))); + p1 = (v16u8)(t0 & t1) + (p1 & (v16u8)(~t1)); + t2 = (v8i16)(flags & (~(__msa_ceq_b((v16i8)bDetaQ2Q0, zero)))); + q1 = (v16u8)(t3 & t2) + (q1 & (v16u8)(~t2)); + + ILVRL_B2_SH(p0, p1, t0, t1); + ILVRL_B2_SH(q1, q0, t2, t3); + ILVRL_H2_UB(t2, t0, p1, p0); + ILVRL_H2_UB(t3, t1, q0, q1); + // Store data to pPix + ST_W8(p1, p0, 0, 1, 2, 3, 0, 1, 2, 3, pPix - 2, iStride); + ST_W8(q0, q1, 0, 1, 2, 3, 0, 1, 2, 3, pPix + 8 * iStride - 2, iStride); } static void avc_loopfilter_luma_inter_edge_hor_msa(uint8_t *data, @@ -2180,23 +2056,24 @@ static void avc_h_loop_filter_chroma422_mbaff_msa(uint8_t *src, void ff_h264_h_lpf_luma_inter_msa(uint8_t *data, ptrdiff_t img_width, int alpha, int beta, int8_t *tc) { - uint8_t bs0 = 1; - uint8_t bs1 = 1; - uint8_t bs2 = 1; - uint8_t bs3 = 1; - - if (tc[0] < 0) - bs0 = 0; - if (tc[1] < 0) - bs1 = 0; - if (tc[2] < 0) - bs2 = 0; - if (tc[3] < 0) - bs3 = 0; - - avc_loopfilter_luma_inter_edge_ver_msa(data, bs0, bs1, bs2, bs3, - tc[0], tc[1], tc[2], tc[3], - alpha, beta, img_width); +// uint8_t bs0 = 1; +// uint8_t bs1 = 1; +// uint8_t bs2 = 1; +// uint8_t bs3 = 1; +// +// if (tc[0] < 0) +// bs0 = 0; +// if (tc[1] < 0) +// bs1 = 0; +// if (tc[2] < 0) +// bs2 = 0; +// if (tc[3] < 0) +// bs3 = 0; +// +// avc_loopfilter_luma_inter_edge_ver_msa(data, bs0, bs1, bs2, bs3, +// tc[0], tc[1], tc[2], tc[3], +// alpha, beta, img_width); + avc_loopfilter_luma_inter_edge_ver_msa(data, img_width, alpha, beta, tc); } void ff_h264_v_lpf_luma_inter_msa(uint8_t *data, ptrdiff_t img_width, diff --git a/libavcodec/mips/h264pred_mmi.c b/libavcodec/mips/h264pred_mmi.c index 0209c2e43f..480411f5b5 100644 --- a/libavcodec/mips/h264pred_mmi.c +++ b/libavcodec/mips/h264pred_mmi.c @@ -155,14 +155,14 @@ void ff_pred16x16_dc_8_mmi(uint8_t *src, ptrdiff_t stride) void ff_pred8x8l_top_dc_8_mmi(uint8_t *src, int has_topleft, int has_topright, ptrdiff_t stride) { - uint32_t dc; double ftmp[11]; mips_reg tmp[3]; + union av_intfloat64 dc; DECLARE_VAR_ALL64; DECLARE_VAR_ADDRT; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" MMI_ULDC1(%[ftmp10], %[srcA], 0x00) MMI_ULDC1(%[ftmp9], %[src0], 0x00) MMI_ULDC1(%[ftmp8], %[src1], 0x00) @@ -209,12 +209,12 @@ void ff_pred8x8l_top_dc_8_mmi(uint8_t *src, int has_topleft, [ftmp10]"=&f"(ftmp[10]), [tmp0]"=&r"(tmp[0]), [tmp1]"=&r"(tmp[1]), RESTRICT_ASM_ALL64 - [dc]"=r"(dc) + [dc]"=r"(dc.i) : [srcA]"r"((mips_reg)(src-stride-1)), [src0]"r"((mips_reg)(src-stride)), [src1]"r"((mips_reg)(src-stride+1)), [has_topleft]"r"(has_topleft), [has_topright]"r"(has_topright), - [ff_pb_1]"r"(ff_pb_1), [ff_pw_2]"f"(ff_pw_2) + [ff_pb_1]"r"(ff_pb_1.i), [ff_pw_2]"f"(ff_pw_2.f) : "memory" ); @@ -238,7 +238,7 @@ void ff_pred8x8l_top_dc_8_mmi(uint8_t *src, int has_topleft, RESTRICT_ASM_ALL64 RESTRICT_ASM_ADDRT [src]"+&r"(src) - : [dc]"f"(dc), [stride]"r"((mips_reg)stride) + : [dc]"f"(dc.f), [stride]"r"((mips_reg)stride) : "memory" ); } @@ -246,9 +246,10 @@ void ff_pred8x8l_top_dc_8_mmi(uint8_t *src, int has_topleft, void ff_pred8x8l_dc_8_mmi(uint8_t *src, int has_topleft, int has_topright, ptrdiff_t stride) { - uint32_t dc, dc1, dc2; + uint32_t dc1, dc2; double ftmp[14]; mips_reg tmp[1]; + union av_intfloat64 dc; const int l0 = ((has_topleft ? src[-1+-1*stride] : src[-1+0*stride]) + 2*src[-1+0*stride] + src[-1+1*stride] + 2) >> 2; const int l1 = (src[-1+0*stride] + 2*src[-1+1*stride] + src[-1+2*stride] + 2) >> 2; @@ -266,7 +267,7 @@ void ff_pred8x8l_dc_8_mmi(uint8_t *src, int has_topleft, int has_topright, MMI_ULDC1(%[ftmp4], %[srcA], 0x00) MMI_ULDC1(%[ftmp5], %[src0], 0x00) MMI_ULDC1(%[ftmp6], %[src1], 0x00) - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "dli %[tmp0], 0x03 \n\t" "punpcklbh %[ftmp7], %[ftmp4], %[ftmp0] \n\t" "punpckhbh %[ftmp8], %[ftmp4], %[ftmp0] \n\t" @@ -322,7 +323,7 @@ void ff_pred8x8l_dc_8_mmi(uint8_t *src, int has_topleft, int has_topright, ); dc1 = l0+l1+l2+l3+l4+l5+l6+l7; - dc = ((dc1+dc2+8)>>4)*0x01010101U; + dc.i = ((dc1+dc2+8)>>4)*0x01010101U; __asm__ volatile ( "dli %[tmp0], 0x02 \n\t" @@ -344,7 +345,7 @@ void ff_pred8x8l_dc_8_mmi(uint8_t *src, int has_topleft, int has_topright, RESTRICT_ASM_ALL64 RESTRICT_ASM_ADDRT [src]"+&r"(src) - : [dc]"f"(dc), [stride]"r"((mips_reg)stride) + : [dc]"f"(dc.f), [stride]"r"((mips_reg)stride) : "memory" ); } @@ -357,7 +358,7 @@ void ff_pred8x8l_vertical_8_mmi(uint8_t *src, int has_topleft, DECLARE_VAR_ALL64; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" MMI_LDC1(%[ftmp3], %[srcA], 0x00) MMI_LDC1(%[ftmp4], %[src0], 0x00) MMI_LDC1(%[ftmp5], %[src1], 0x00) @@ -530,7 +531,7 @@ void ff_pred8x8_top_dc_8_mmi(uint8_t *src, ptrdiff_t stride) __asm__ volatile ( "dli %[tmp0], 0x02 \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" PTR_SUBU "%[addr0], %[src], %[stride] \n\t" MMI_LDC1(%[ftmp1], %[addr0], 0x00) "punpcklbh %[ftmp2], %[ftmp1], %[ftmp0] \n\t" @@ -640,7 +641,7 @@ void ff_pred8x8_dc_8_mmi(uint8_t *src, ptrdiff_t stride) PTR_SRL "%[addr4], 0x02 \n\t" PTR_SRL "%[addr1], 0x02 \n\t" PTR_SRL "%[addr2], 0x03 \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "dmtc1 %[addr3], %[ftmp1] \n\t" "pshufh %[ftmp1], %[ftmp1], %[ftmp0] \n\t" "dmtc1 %[addr4], %[ftmp2] \n\t" @@ -757,9 +758,9 @@ static inline void pred16x16_plane_compat_mmi(uint8_t *src, int stride, "dmtc1 %[tmp0], %[ftmp4] \n\t" MMI_ULDC1(%[ftmp0], %[addr0], -0x01) MMI_ULDC1(%[ftmp2], %[addr0], 0x08) - "dsrl %[ftmp1], %[ftmp0], %[ftmp4] \n\t" - "dsrl %[ftmp3], %[ftmp2], %[ftmp4] \n\t" - "xor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" + "ssrld %[ftmp1], %[ftmp0], %[ftmp4] \n\t" + "ssrld %[ftmp3], %[ftmp2], %[ftmp4] \n\t" + "pxor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" "punpcklbh %[ftmp0], %[ftmp0], %[ftmp4] \n\t" "punpcklbh %[ftmp1], %[ftmp1], %[ftmp4] \n\t" "punpcklbh %[ftmp2], %[ftmp2], %[ftmp4] \n\t" @@ -915,7 +916,7 @@ static inline void pred16x16_plane_compat_mmi(uint8_t *src, int stride, "dmul %[tmp3], %[tmp3], %[tmp2] \n\t" "dsubu %[tmp5], %[tmp5], %[tmp3] \n\t" - "xor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" + "pxor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" "dmtc1 %[tmp0], %[ftmp0] \n\t" "pshufh %[ftmp0], %[ftmp0], %[ftmp4] \n\t" "dmtc1 %[tmp1], %[ftmp5] \n\t" @@ -965,10 +966,10 @@ static inline void pred16x16_plane_compat_mmi(uint8_t *src, int stride, [addr0]"=&r"(addr[0]) : [src]"r"(src), [stride]"r"((mips_reg)stride), [svq3]"r"(svq3), [rv40]"r"(rv40), - [ff_pw_m8tom5]"f"(ff_pw_m8tom5), [ff_pw_m4tom1]"f"(ff_pw_m4tom1), - [ff_pw_1to4]"f"(ff_pw_1to4), [ff_pw_5to8]"f"(ff_pw_5to8), - [ff_pw_0to3]"f"(ff_pw_0to3), [ff_pw_4to7]"r"(ff_pw_4to7), - [ff_pw_8tob]"r"(ff_pw_8tob), [ff_pw_ctof]"r"(ff_pw_ctof) + [ff_pw_m8tom5]"f"(ff_pw_m8tom5.f),[ff_pw_m4tom1]"f"(ff_pw_m4tom1.f), + [ff_pw_1to4]"f"(ff_pw_1to4.f), [ff_pw_5to8]"f"(ff_pw_5to8.f), + [ff_pw_0to3]"f"(ff_pw_0to3.f), [ff_pw_4to7]"r"(ff_pw_4to7.i), + [ff_pw_8tob]"r"(ff_pw_8tob.i), [ff_pw_ctof]"r"(ff_pw_ctof.i) : "memory" ); } diff --git a/libavcodec/mips/h264qpel_mmi.c b/libavcodec/mips/h264qpel_mmi.c index 13fbebf7f5..3482956e13 100644 --- a/libavcodec/mips/h264qpel_mmi.c +++ b/libavcodec/mips/h264qpel_mmi.c @@ -114,7 +114,7 @@ static void put_h264_qpel4_h_lowpass_mmi(uint8_t *dst, const uint8_t *src, DECLARE_VAR_LOW32; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "dli %[tmp0], 0x04 \n\t" "1: \n\t" MMI_ULWC1(%[ftmp1], %[src], -0x02) @@ -155,8 +155,8 @@ static void put_h264_qpel4_h_lowpass_mmi(uint8_t *dst, const uint8_t *src, [dst]"+&r"(dst), [src]"+&r"(src) : [dstStride]"r"((mips_reg)dstStride), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_20]"f"(ff_pw_20), [ff_pw_5]"f"(ff_pw_5), - [ff_pw_16]"f"(ff_pw_16) + [ff_pw_20]"f"(ff_pw_20.f), [ff_pw_5]"f"(ff_pw_5.f), + [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); } @@ -169,7 +169,7 @@ static void put_h264_qpel8_h_lowpass_mmi(uint8_t *dst, const uint8_t *src, DECLARE_VAR_ALL64; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "dli %[tmp0], 0x08 \n\t" "1: \n\t" MMI_ULDC1(%[ftmp1], %[src], -0x02) @@ -225,8 +225,8 @@ static void put_h264_qpel8_h_lowpass_mmi(uint8_t *dst, const uint8_t *src, [dst]"+&r"(dst), [src]"+&r"(src) : [dstStride]"r"((mips_reg)dstStride), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_20]"f"(ff_pw_20), [ff_pw_5]"f"(ff_pw_5), - [ff_pw_16]"f"(ff_pw_16) + [ff_pw_20]"f"(ff_pw_20.f), [ff_pw_5]"f"(ff_pw_5.f), + [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); } @@ -250,7 +250,7 @@ static void avg_h264_qpel4_h_lowpass_mmi(uint8_t *dst, const uint8_t *src, DECLARE_VAR_LOW32; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "dli %[tmp0], 0x04 \n\t" "1: \n\t" MMI_ULWC1(%[ftmp1], %[src], -0x02) @@ -293,8 +293,8 @@ static void avg_h264_qpel4_h_lowpass_mmi(uint8_t *dst, const uint8_t *src, [dst]"+&r"(dst), [src]"+&r"(src) : [dstStride]"r"((mips_reg)dstStride), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_20]"f"(ff_pw_20), [ff_pw_5]"f"(ff_pw_5), - [ff_pw_16]"f"(ff_pw_16) + [ff_pw_20]"f"(ff_pw_20.f), [ff_pw_5]"f"(ff_pw_5.f), + [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); } @@ -307,7 +307,7 @@ static void avg_h264_qpel8_h_lowpass_mmi(uint8_t *dst, const uint8_t *src, DECLARE_VAR_ALL64; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "dli %[tmp0], 0x08 \n\t" "1: \n\t" MMI_ULDC1(%[ftmp1], %[src], -0x02) @@ -365,8 +365,8 @@ static void avg_h264_qpel8_h_lowpass_mmi(uint8_t *dst, const uint8_t *src, [dst]"+&r"(dst), [src]"+&r"(src) : [dstStride]"r"((mips_reg)dstStride), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_20]"f"(ff_pw_20), [ff_pw_5]"f"(ff_pw_5), - [ff_pw_16]"f"(ff_pw_16) + [ff_pw_20]"f"(ff_pw_20.f), [ff_pw_5]"f"(ff_pw_5.f), + [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); } @@ -394,7 +394,7 @@ static void put_h264_qpel4_v_lowpass_mmi(uint8_t *dst, const uint8_t *src, __asm__ volatile ( ".set push \n\t" ".set noreorder \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "dli %[tmp0], 0x02 \n\t" MMI_LWC1(%[ftmp1], %[src], 0x00) "mtc1 %[tmp0], %[ftmp10] \n\t" @@ -486,7 +486,7 @@ static void put_h264_qpel4_v_lowpass_mmi(uint8_t *dst, const uint8_t *src, [dst]"+&r"(dst), [src]"+&r"(src) : [dstStride]"r"((mips_reg)dstStride), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_5]"f"(ff_pw_5), [ff_pw_16]"f"(ff_pw_16) + [ff_pw_5]"f"(ff_pw_5.f), [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); } @@ -516,7 +516,7 @@ static void put_h264_qpel8_v_lowpass_mmi(uint8_t *dst, const uint8_t *src, PTR_ADDU "%[src], %[src], %[srcStride] \n\t" MMI_LWC1(%[ftmp2], %[src], 0x00) PTR_ADDU "%[src], %[src], %[srcStride] \n\t" - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" MMI_LWC1(%[ftmp3], %[src], 0x00) PTR_ADDU "%[src], %[src], %[srcStride] \n\t" MMI_LWC1(%[ftmp4], %[src], 0x00) @@ -780,7 +780,7 @@ static void put_h264_qpel8_v_lowpass_mmi(uint8_t *dst, const uint8_t *src, [h]"+&r"(h) : [dstStride]"r"((mips_reg)dstStride), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_5]"f"(ff_pw_5), [ff_pw_16]"f"(ff_pw_16) + [ff_pw_5]"f"(ff_pw_5.f), [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); @@ -812,7 +812,7 @@ static void avg_h264_qpel4_v_lowpass_mmi(uint8_t *dst, const uint8_t *src, ".set push \n\t" ".set noreorder \n\t" "dli %[tmp0], 0x02 \n\t" - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" "mtc1 %[tmp0], %[ftmp9] \n\t" "dli %[tmp0], 0x05 \n\t" MMI_LWC1(%[ftmp0], %[src], 0x00) @@ -909,7 +909,7 @@ static void avg_h264_qpel4_v_lowpass_mmi(uint8_t *dst, const uint8_t *src, [src]"+&r"(src), [dst]"+&r"(dst) : [dstStride]"r"((mips_reg)dstStride), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_5]"f"(ff_pw_5), [ff_pw_16]"f"(ff_pw_16) + [ff_pw_5]"f"(ff_pw_5.f), [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); } @@ -930,7 +930,7 @@ static void avg_h264_qpel8_v_lowpass_mmi(uint8_t *dst, const uint8_t *src, ".set push \n\t" ".set noreorder \n\t" "dli %[tmp0], 0x02 \n\t" - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" "mtc1 %[tmp0], %[ftmp9] \n\t" "dli %[tmp0], 0x05 \n\t" MMI_LWC1(%[ftmp0], %[src], 0x00) @@ -1235,7 +1235,7 @@ static void avg_h264_qpel8_v_lowpass_mmi(uint8_t *dst, const uint8_t *src, [h]"+&r"(h) : [dstStride]"r"((mips_reg)dstStride), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_5]"f"(ff_pw_5), [ff_pw_16]"f"(ff_pw_16) + [ff_pw_5]"f"(ff_pw_5.f), [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); @@ -1269,7 +1269,7 @@ static void put_h264_qpel4_hv_lowpass_mmi(uint8_t *dst, const uint8_t *src, src -= 2*srcStride; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "dli %[tmp0], 0x09 \n\t" "1: \n\t" MMI_ULWC1(%[ftmp1], %[src], -0x02) @@ -1306,7 +1306,7 @@ static void put_h264_qpel4_hv_lowpass_mmi(uint8_t *dst, const uint8_t *src, [tmp]"+&r"(tmp), [src]"+&r"(src) : [tmpStride]"r"(8), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_20]"f"(ff_pw_20), [ff_pw_5]"f"(ff_pw_5) + [ff_pw_20]"f"(ff_pw_20.f), [ff_pw_5]"f"(ff_pw_5.f) : "memory" ); @@ -1347,7 +1347,7 @@ static void put_h264_qpel8or16_hv1_lowpass_mmi(int16_t *tmp, MMI_ULWC1(%[ftmp0], %[src], 0x00) "mtc1 %[tmp0], %[ftmp10] \n\t" PTR_ADDU "%[src], %[src], %[srcStride] \n\t" - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" MMI_ULWC1(%[ftmp1], %[src], 0x00) PTR_ADDU "%[src], %[src], %[srcStride] \n\t" MMI_ULWC1(%[ftmp2], %[src], 0x00) @@ -1567,7 +1567,7 @@ static void put_h264_qpel8or16_hv1_lowpass_mmi(int16_t *tmp, [src]"+&r"(src) : [tmp]"r"(tmp), [size]"r"(size), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_5]"f"(ff_pw_5), [ff_pw_16]"f"(ff_pw_16) + [ff_pw_5]"f"(ff_pw_5.f), [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); @@ -1684,7 +1684,7 @@ static void put_h264_qpel8_h_lowpass_l2_mmi(uint8_t *dst, const uint8_t *src, "dli %[tmp0], 0x02 \n\t" "mtc1 %[tmp0], %[ftmp7] \n\t" "dli %[tmp0], 0x05 \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp8] \n\t" "1: \n\t" MMI_ULDC1(%[ftmp1], %[src], 0x00) @@ -1742,7 +1742,7 @@ static void put_h264_qpel8_h_lowpass_l2_mmi(uint8_t *dst, const uint8_t *src, [src2]"+&r"(src2), [h]"+&r"(h) : [src2Stride]"r"((mips_reg)src2Stride), [dstStride]"r"((mips_reg)dstStride), - [ff_pw_5]"f"(ff_pw_5), [ff_pw_16]"f"(ff_pw_16) + [ff_pw_5]"f"(ff_pw_5.f), [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); } @@ -1833,7 +1833,7 @@ static void avg_h264_qpel4_hv_lowpass_mmi(uint8_t *dst, const uint8_t *src, src -= 2*srcStride; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "dli %[tmp0], 0x09 \n\t" "1: \n\t" MMI_ULWC1(%[ftmp1], %[src], -0x02) @@ -1870,7 +1870,7 @@ static void avg_h264_qpel4_hv_lowpass_mmi(uint8_t *dst, const uint8_t *src, [tmp]"+&r"(tmp), [src]"+&r"(src) : [tmpStride]"r"(8), [srcStride]"r"((mips_reg)srcStride), - [ff_pw_20]"f"(ff_pw_20), [ff_pw_5]"f"(ff_pw_5) + [ff_pw_20]"f"(ff_pw_20.f), [ff_pw_5]"f"(ff_pw_5.f) : "memory" ); @@ -2005,7 +2005,7 @@ static void avg_h264_qpel8_h_lowpass_l2_mmi(uint8_t *dst, const uint8_t *src, "ori %[tmp0], $0, 0x8 \n\t" "mtc1 %[tmp1], %[ftmp7] \n\t" "dli %[tmp1], 0x05 \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "mtc1 %[tmp1], %[ftmp8] \n\t" "1: \n\t" MMI_ULDC1(%[ftmp1], %[src], 0x00) @@ -2065,7 +2065,7 @@ static void avg_h264_qpel8_h_lowpass_l2_mmi(uint8_t *dst, const uint8_t *src, [src2]"+&r"(src2) : [dstStride]"r"((mips_reg)dstStride), [src2Stride]"r"((mips_reg)src2Stride), - [ff_pw_5]"f"(ff_pw_5), [ff_pw_16]"f"(ff_pw_16) + [ff_pw_5]"f"(ff_pw_5.f), [ff_pw_16]"f"(ff_pw_16.f) : "memory" ); } diff --git a/libavcodec/mips/hevcdsp_mmi.c b/libavcodec/mips/hevcdsp_mmi.c index aa83e1f9ad..87fc2555a4 100644 --- a/libavcodec/mips/hevcdsp_mmi.c +++ b/libavcodec/mips/hevcdsp_mmi.c @@ -32,7 +32,7 @@ void ff_hevc_put_hevc_qpel_h##w##_8_mmi(int16_t *dst, uint8_t *_src, \ int x, y; \ pixel *src = (pixel*)_src - 3; \ ptrdiff_t srcstride = _srcstride / sizeof(pixel); \ - uint64_t ftmp[15]; \ + double ftmp[15]; \ uint64_t rtmp[1]; \ const int8_t *filter = ff_hevc_qpel_filters[mx - 1]; \ \ @@ -46,7 +46,7 @@ void ff_hevc_put_hevc_qpel_h##w##_8_mmi(int16_t *dst, uint8_t *_src, \ "punpcklbh %[ftmp1], %[ftmp0], %[ftmp1] \n\t" \ "psrah %[ftmp1], %[ftmp1], %[ftmp0] \n\t" \ "psrah %[ftmp2], %[ftmp2], %[ftmp0] \n\t" \ - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ \ "1: \n\t" \ "2: \n\t" \ @@ -132,7 +132,7 @@ void ff_hevc_put_hevc_qpel_hv##w##_8_mmi(int16_t *dst, uint8_t *_src, \ ptrdiff_t srcstride = _srcstride / sizeof(pixel); \ int16_t tmp_array[(MAX_PB_SIZE + QPEL_EXTRA) * MAX_PB_SIZE]; \ int16_t *tmp = tmp_array; \ - uint64_t ftmp[15]; \ + double ftmp[15]; \ uint64_t rtmp[1]; \ \ src -= (QPEL_EXTRA_BEFORE * srcstride + 3); \ @@ -147,7 +147,7 @@ void ff_hevc_put_hevc_qpel_hv##w##_8_mmi(int16_t *dst, uint8_t *_src, \ "punpcklbh %[ftmp1], %[ftmp0], %[ftmp1] \n\t" \ "psrah %[ftmp1], %[ftmp1], %[ftmp0] \n\t" \ "psrah %[ftmp2], %[ftmp2], %[ftmp0] \n\t" \ - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ \ "1: \n\t" \ "2: \n\t" \ @@ -329,10 +329,12 @@ void ff_hevc_put_hevc_qpel_bi_h##w##_8_mmi(uint8_t *_dst, \ pixel *dst = (pixel *)_dst; \ ptrdiff_t dststride = _dststride / sizeof(pixel); \ const int8_t *filter = ff_hevc_qpel_filters[mx - 1]; \ - uint64_t ftmp[20]; \ + double ftmp[20]; \ uint64_t rtmp[1]; \ - int shift = 7; \ - int offset = 64; \ + union av_intfloat64 shift; \ + union av_intfloat64 offset; \ + shift.i = 7; \ + offset.i = 64; \ \ x = width >> 2; \ y = height; \ @@ -344,7 +346,7 @@ void ff_hevc_put_hevc_qpel_bi_h##w##_8_mmi(uint8_t *_dst, \ "punpcklbh %[ftmp1], %[ftmp0], %[ftmp1] \n\t" \ "psrah %[ftmp1], %[ftmp1], %[ftmp0] \n\t" \ "psrah %[ftmp2], %[ftmp2], %[ftmp0] \n\t" \ - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ "punpcklhw %[offset], %[offset], %[offset] \n\t" \ "punpcklwd %[offset], %[offset], %[offset] \n\t" \ \ @@ -403,7 +405,7 @@ void ff_hevc_put_hevc_qpel_bi_h##w##_8_mmi(uint8_t *_dst, \ "psraw %[ftmp6], %[ftmp6], %[shift] \n\t" \ "packsswh %[ftmp5], %[ftmp5], %[ftmp6] \n\t" \ "pcmpgth %[ftmp7], %[ftmp5], %[ftmp0] \n\t" \ - "and %[ftmp3], %[ftmp5], %[ftmp7] \n\t" \ + "pand %[ftmp3], %[ftmp5], %[ftmp7] \n\t" \ "packushb %[ftmp3], %[ftmp3], %[ftmp3] \n\t" \ "gsswlc1 %[ftmp3], 0x03(%[dst]) \n\t" \ "gsswrc1 %[ftmp3], 0x00(%[dst]) \n\t" \ @@ -430,9 +432,9 @@ void ff_hevc_put_hevc_qpel_bi_h##w##_8_mmi(uint8_t *_dst, \ [ftmp10]"=&f"(ftmp[10]), [ftmp11]"=&f"(ftmp[11]), \ [ftmp12]"=&f"(ftmp[12]), [src2]"+&r"(src2), \ [dst]"+&r"(dst), [src]"+&r"(src), [y]"+&r"(y), [x]"=&r"(x), \ - [offset]"+&f"(offset), [rtmp0]"=&r"(rtmp[0]) \ + [offset]"+&f"(offset.f), [rtmp0]"=&r"(rtmp[0]) \ : [src_stride]"r"(srcstride), [dst_stride]"r"(dststride), \ - [filter]"r"(filter), [shift]"f"(shift) \ + [filter]"r"(filter), [shift]"f"(shift.f) \ : "memory" \ ); \ } @@ -463,10 +465,12 @@ void ff_hevc_put_hevc_qpel_bi_hv##w##_8_mmi(uint8_t *_dst, \ ptrdiff_t dststride = _dststride / sizeof(pixel); \ int16_t tmp_array[(MAX_PB_SIZE + QPEL_EXTRA) * MAX_PB_SIZE]; \ int16_t *tmp = tmp_array; \ - uint64_t ftmp[20]; \ + double ftmp[20]; \ uint64_t rtmp[1]; \ - int shift = 7; \ - int offset = 64; \ + union av_intfloat64 shift; \ + union av_intfloat64 offset; \ + shift.i = 7; \ + offset.i = 64; \ \ src -= (QPEL_EXTRA_BEFORE * srcstride + 3); \ filter = ff_hevc_qpel_filters[mx - 1]; \ @@ -480,7 +484,7 @@ void ff_hevc_put_hevc_qpel_bi_hv##w##_8_mmi(uint8_t *_dst, \ "punpcklbh %[ftmp1], %[ftmp0], %[ftmp1] \n\t" \ "psrah %[ftmp1], %[ftmp1], %[ftmp0] \n\t" \ "psrah %[ftmp2], %[ftmp2], %[ftmp0] \n\t" \ - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ \ "1: \n\t" \ "2: \n\t" \ @@ -612,7 +616,7 @@ void ff_hevc_put_hevc_qpel_bi_hv##w##_8_mmi(uint8_t *_dst, \ "packsswh %[ftmp3], %[ftmp3], %[ftmp5] \n\t" \ "gsldlc1 %[ftmp4], 0x07(%[src2]) \n\t" \ "gsldrc1 %[ftmp4], 0x00(%[src2]) \n\t" \ - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" \ + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" \ "li %[rtmp0], 0x10 \n\t" \ "dmtc1 %[rtmp0], %[ftmp8] \n\t" \ "punpcklhw %[ftmp5], %[ftmp7], %[ftmp3] \n\t" \ @@ -631,7 +635,7 @@ void ff_hevc_put_hevc_qpel_bi_hv##w##_8_mmi(uint8_t *_dst, \ "psraw %[ftmp6], %[ftmp6], %[shift] \n\t" \ "packsswh %[ftmp5], %[ftmp5], %[ftmp6] \n\t" \ "pcmpgth %[ftmp7], %[ftmp5], %[ftmp7] \n\t" \ - "and %[ftmp3], %[ftmp5], %[ftmp7] \n\t" \ + "pand %[ftmp3], %[ftmp5], %[ftmp7] \n\t" \ "packushb %[ftmp3], %[ftmp3], %[ftmp3] \n\t" \ "gsswlc1 %[ftmp3], 0x03(%[dst]) \n\t" \ "gsswrc1 %[ftmp3], 0x00(%[dst]) \n\t" \ @@ -659,9 +663,9 @@ void ff_hevc_put_hevc_qpel_bi_hv##w##_8_mmi(uint8_t *_dst, \ [ftmp12]"=&f"(ftmp[12]), [ftmp13]"=&f"(ftmp[13]), \ [ftmp14]"=&f"(ftmp[14]), [src2]"+&r"(src2), \ [dst]"+&r"(dst), [tmp]"+&r"(tmp), [y]"+&r"(y), [x]"=&r"(x), \ - [offset]"+&f"(offset), [rtmp0]"=&r"(rtmp[0]) \ + [offset]"+&f"(offset.f), [rtmp0]"=&r"(rtmp[0]) \ : [filter]"r"(filter), [stride]"r"(dststride), \ - [shift]"f"(shift) \ + [shift]"f"(shift.f) \ : "memory" \ ); \ } @@ -692,10 +696,12 @@ void ff_hevc_put_hevc_epel_bi_hv##w##_8_mmi(uint8_t *_dst, \ const int8_t *filter = ff_hevc_epel_filters[mx - 1]; \ int16_t tmp_array[(MAX_PB_SIZE + EPEL_EXTRA) * MAX_PB_SIZE]; \ int16_t *tmp = tmp_array; \ - uint64_t ftmp[12]; \ + double ftmp[12]; \ uint64_t rtmp[1]; \ - int shift = 7; \ - int offset = 64; \ + union av_intfloat64 shift; \ + union av_intfloat64 offset; \ + shift.i = 7; \ + offset.i = 64; \ \ src -= (EPEL_EXTRA_BEFORE * srcstride + 1); \ x = width >> 2; \ @@ -706,7 +712,7 @@ void ff_hevc_put_hevc_epel_bi_hv##w##_8_mmi(uint8_t *_dst, \ "dmtc1 %[rtmp0], %[ftmp0] \n\t" \ "punpcklbh %[ftmp1], %[ftmp0], %[ftmp1] \n\t" \ "psrah %[ftmp1], %[ftmp1], %[ftmp0] \n\t" \ - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ \ "1: \n\t" \ "2: \n\t" \ @@ -771,7 +777,7 @@ void ff_hevc_put_hevc_epel_bi_hv##w##_8_mmi(uint8_t *_dst, \ "li %[rtmp0], 0x06 \n\t" \ "dmtc1 %[rtmp0], %[ftmp0] \n\t" \ "punpcklwd %[offset], %[offset], %[offset] \n\t" \ - "xor %[ftmp2], %[ftmp2], %[ftmp2] \n\t" \ + "pxor %[ftmp2], %[ftmp2], %[ftmp2] \n\t" \ \ "1: \n\t" \ "li %[x], " #x_step " \n\t" \ @@ -821,7 +827,7 @@ void ff_hevc_put_hevc_epel_bi_hv##w##_8_mmi(uint8_t *_dst, \ "psraw %[ftmp6], %[ftmp6], %[shift] \n\t" \ "packsswh %[ftmp5], %[ftmp5], %[ftmp6] \n\t" \ "pcmpgth %[ftmp7], %[ftmp5], %[ftmp2] \n\t" \ - "and %[ftmp3], %[ftmp5], %[ftmp7] \n\t" \ + "pand %[ftmp3], %[ftmp5], %[ftmp7] \n\t" \ "packushb %[ftmp3], %[ftmp3], %[ftmp3] \n\t" \ "gsswlc1 %[ftmp3], 0x03(%[dst]) \n\t" \ "gsswrc1 %[ftmp3], 0x00(%[dst]) \n\t" \ @@ -847,9 +853,9 @@ void ff_hevc_put_hevc_epel_bi_hv##w##_8_mmi(uint8_t *_dst, \ [ftmp8]"=&f"(ftmp[8]), [ftmp9]"=&f"(ftmp[9]), \ [ftmp10]"=&f"(ftmp[10]), [src2]"+&r"(src2), \ [dst]"+&r"(dst), [tmp]"+&r"(tmp), [y]"+&r"(y), [x]"=&r"(x), \ - [offset]"+&f"(offset), [rtmp0]"=&r"(rtmp[0]) \ + [offset]"+&f"(offset.f), [rtmp0]"=&r"(rtmp[0]) \ : [filter]"r"(filter), [stride]"r"(dststride), \ - [shift]"f"(shift) \ + [shift]"f"(shift.f) \ : "memory" \ ); \ } @@ -875,14 +881,15 @@ void ff_hevc_put_hevc_pel_bi_pixels##w##_8_mmi(uint8_t *_dst, \ ptrdiff_t srcstride = _srcstride / sizeof(pixel); \ pixel *dst = (pixel *)_dst; \ ptrdiff_t dststride = _dststride / sizeof(pixel); \ - uint64_t ftmp[12]; \ + double ftmp[12]; \ uint64_t rtmp[1]; \ - int shift = 7; \ + union av_intfloat64 shift; \ + shift.i = 7; \ \ y = height; \ x = width >> 3; \ __asm__ volatile( \ - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ "li %[rtmp0], 0x06 \n\t" \ "dmtc1 %[rtmp0], %[ftmp1] \n\t" \ "li %[rtmp0], 0x10 \n\t" \ @@ -930,8 +937,8 @@ void ff_hevc_put_hevc_pel_bi_pixels##w##_8_mmi(uint8_t *_dst, \ "packsswh %[ftmp4], %[ftmp4], %[ftmp5] \n\t" \ "pcmpgth %[ftmp3], %[ftmp2], %[ftmp0] \n\t" \ "pcmpgth %[ftmp5], %[ftmp4], %[ftmp0] \n\t" \ - "and %[ftmp2], %[ftmp2], %[ftmp3] \n\t" \ - "and %[ftmp4], %[ftmp4], %[ftmp5] \n\t" \ + "pand %[ftmp2], %[ftmp2], %[ftmp3] \n\t" \ + "pand %[ftmp4], %[ftmp4], %[ftmp5] \n\t" \ "packushb %[ftmp2], %[ftmp2], %[ftmp4] \n\t" \ "gssdlc1 %[ftmp2], 0x07(%[dst]) \n\t" \ "gssdrc1 %[ftmp2], 0x00(%[dst]) \n\t" \ @@ -959,7 +966,7 @@ void ff_hevc_put_hevc_pel_bi_pixels##w##_8_mmi(uint8_t *_dst, \ [ftmp10]"=&f"(ftmp[10]), [offset]"=&f"(ftmp[11]), \ [src2]"+&r"(src2), [dst]"+&r"(dst), [src]"+&r"(src), \ [x]"+&r"(x), [y]"+&r"(y), [rtmp0]"=&r"(rtmp[0]) \ - : [dststride]"r"(dststride), [shift]"f"(shift), \ + : [dststride]"r"(dststride), [shift]"f"(shift.f), \ [srcstride]"r"(srcstride) \ : "memory" \ ); \ @@ -989,10 +996,12 @@ void ff_hevc_put_hevc_qpel_uni_hv##w##_8_mmi(uint8_t *_dst, \ ptrdiff_t dststride = _dststride / sizeof(pixel); \ int16_t tmp_array[(MAX_PB_SIZE + QPEL_EXTRA) * MAX_PB_SIZE]; \ int16_t *tmp = tmp_array; \ - uint64_t ftmp[20]; \ + double ftmp[20]; \ uint64_t rtmp[1]; \ - int shift = 6; \ - int offset = 32; \ + union av_intfloat64 shift; \ + union av_intfloat64 offset; \ + shift.i = 6; \ + offset.i = 32; \ \ src -= (QPEL_EXTRA_BEFORE * srcstride + 3); \ filter = ff_hevc_qpel_filters[mx - 1]; \ @@ -1006,7 +1015,7 @@ void ff_hevc_put_hevc_qpel_uni_hv##w##_8_mmi(uint8_t *_dst, \ "punpcklbh %[ftmp1], %[ftmp0], %[ftmp1] \n\t" \ "psrah %[ftmp1], %[ftmp1], %[ftmp0] \n\t" \ "psrah %[ftmp2], %[ftmp2], %[ftmp0] \n\t" \ - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ \ "1: \n\t" \ "2: \n\t" \ @@ -1139,9 +1148,9 @@ void ff_hevc_put_hevc_qpel_uni_hv##w##_8_mmi(uint8_t *_dst, \ "packsswh %[ftmp3], %[ftmp3], %[ftmp5] \n\t" \ "paddh %[ftmp3], %[ftmp3], %[offset] \n\t" \ "psrah %[ftmp3], %[ftmp3], %[shift] \n\t" \ - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" \ + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" \ "pcmpgth %[ftmp7], %[ftmp3], %[ftmp7] \n\t" \ - "and %[ftmp3], %[ftmp3], %[ftmp7] \n\t" \ + "pand %[ftmp3], %[ftmp3], %[ftmp7] \n\t" \ "packushb %[ftmp3], %[ftmp3], %[ftmp3] \n\t" \ "gsswlc1 %[ftmp3], 0x03(%[dst]) \n\t" \ "gsswrc1 %[ftmp3], 0x00(%[dst]) \n\t" \ @@ -1166,9 +1175,9 @@ void ff_hevc_put_hevc_qpel_uni_hv##w##_8_mmi(uint8_t *_dst, \ [ftmp12]"=&f"(ftmp[12]), [ftmp13]"=&f"(ftmp[13]), \ [ftmp14]"=&f"(ftmp[14]), \ [dst]"+&r"(dst), [tmp]"+&r"(tmp), [y]"+&r"(y), [x]"=&r"(x), \ - [offset]"+&f"(offset), [rtmp0]"=&r"(rtmp[0]) \ + [offset]"+&f"(offset.f), [rtmp0]"=&r"(rtmp[0]) \ : [filter]"r"(filter), [stride]"r"(dststride), \ - [shift]"f"(shift) \ + [shift]"f"(shift.f) \ : "memory" \ ); \ } diff --git a/libavcodec/mips/hpeldsp_mmi.c b/libavcodec/mips/hpeldsp_mmi.c index e69b2bd980..bf3e4636aa 100644 --- a/libavcodec/mips/hpeldsp_mmi.c +++ b/libavcodec/mips/hpeldsp_mmi.c @@ -676,14 +676,14 @@ inline void ff_put_no_rnd_pixels8_l2_8_mmi(uint8_t *dst, const uint8_t *src1, PTR_ADDU "%[addr1], %[src2], %[src_stride2] \n\t" MMI_ULDC1(%[ftmp3], %[addr1], 0x00) PTR_ADDU "%[src1], %[src1], %[addr2] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp4] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp4] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp4] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp4] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp4] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp4] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp4] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp4] \n\t" "pavgb %[ftmp0], %[ftmp0], %[ftmp2] \n\t" "pavgb %[ftmp1], %[ftmp1], %[ftmp3] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp4] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp4] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp4] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp4] \n\t" MMI_SDC1(%[ftmp0], %[dst], 0x00) MMI_SDXC1(%[ftmp1], %[dst], %[dst_stride], 0x00) PTR_ADDU "%[src2], %[src2], %[addr3] \n\t" @@ -696,14 +696,14 @@ inline void ff_put_no_rnd_pixels8_l2_8_mmi(uint8_t *dst, const uint8_t *src1, PTR_ADDU "%[addr1], %[src2], %[src_stride2] \n\t" MMI_ULDC1(%[ftmp3], %[addr1], 0x00) PTR_ADDU "%[src1], %[src1], %[addr2] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp4] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp4] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp4] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp4] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp4] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp4] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp4] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp4] \n\t" "pavgb %[ftmp0], %[ftmp0], %[ftmp2] \n\t" "pavgb %[ftmp1], %[ftmp1], %[ftmp3] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp4] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp4] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp4] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp4] \n\t" MMI_SDC1(%[ftmp0], %[dst], 0x00) MMI_SDXC1(%[ftmp1], %[dst], %[dst_stride], 0x00) PTR_ADDU "%[src2], %[src2], %[addr3] \n\t" @@ -846,7 +846,7 @@ void ff_put_pixels8_xy2_8_mmi(uint8_t *block, const uint8_t *pixels, DECLARE_VAR_ADDRT; __asm__ volatile ( - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" "dli %[addr0], 0x0f \n\t" "pcmpeqw %[ftmp6], %[ftmp6], %[ftmp6] \n\t" "dmtc1 %[addr0], %[ftmp8] \n\t" diff --git a/libavcodec/mips/idctdsp_mmi.c b/libavcodec/mips/idctdsp_mmi.c index a96dac4704..d22e5eedd7 100644 --- a/libavcodec/mips/idctdsp_mmi.c +++ b/libavcodec/mips/idctdsp_mmi.c @@ -142,7 +142,7 @@ void ff_put_signed_pixels_clamped_mmi(const int16_t *block, [pixels]"+&r"(pixels) : [block]"r"(block), [line_size]"r"((mips_reg)line_size), - [ff_pb_80]"f"(ff_pb_80) + [ff_pb_80]"f"(ff_pb_80.f) : "memory" ); } @@ -154,7 +154,7 @@ void ff_add_pixels_clamped_mmi(const int16_t *block, uint64_t tmp[1]; __asm__ volatile ( "li %[tmp0], 0x04 \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "1: \n\t" MMI_LDC1(%[ftmp5], %[pixels], 0x00) PTR_ADDU "%[pixels], %[pixels], %[line_size] \n\t" diff --git a/libavcodec/mips/mpegvideo_mmi.c b/libavcodec/mips/mpegvideo_mmi.c index e4aba08661..3d5b5e20ab 100644 --- a/libavcodec/mips/mpegvideo_mmi.c +++ b/libavcodec/mips/mpegvideo_mmi.c @@ -28,12 +28,13 @@ void ff_dct_unquantize_h263_intra_mmi(MpegEncContext *s, int16_t *block, int n, int qscale) { - int64_t level, qmul, qadd, nCoeffs; + int64_t level, nCoeffs; double ftmp[6]; mips_reg addr[1]; + union mmi_intfloat64 qmul_u, qadd_u; DECLARE_VAR_ALL64; - qmul = qscale << 1; + qmul_u.i = qscale << 1; av_assert2(s->block_last_index[n]>=0 || s->h263_aic); if (!s->h263_aic) { @@ -41,9 +42,9 @@ void ff_dct_unquantize_h263_intra_mmi(MpegEncContext *s, int16_t *block, level = block[0] * s->y_dc_scale; else level = block[0] * s->c_dc_scale; - qadd = (qscale-1) | 1; + qadd_u.i = (qscale-1) | 1; } else { - qadd = 0; + qadd_u.i = 0; level = block[0]; } @@ -53,13 +54,13 @@ void ff_dct_unquantize_h263_intra_mmi(MpegEncContext *s, int16_t *block, nCoeffs = s->inter_scantable.raster_end[s->block_last_index[n]]; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "packsswh %[qmul], %[qmul], %[qmul] \n\t" "packsswh %[qmul], %[qmul], %[qmul] \n\t" "packsswh %[qadd], %[qadd], %[qadd] \n\t" "packsswh %[qadd], %[qadd], %[qadd] \n\t" "psubh %[ftmp0], %[ftmp0], %[qadd] \n\t" - "xor %[ftmp5], %[ftmp5], %[ftmp5] \n\t" + "pxor %[ftmp5], %[ftmp5], %[ftmp5] \n\t" ".p2align 4 \n\t" "1: \n\t" @@ -72,12 +73,12 @@ void ff_dct_unquantize_h263_intra_mmi(MpegEncContext *s, int16_t *block, "pmullh %[ftmp2], %[ftmp2], %[qmul] \n\t" "pcmpgth %[ftmp3], %[ftmp3], %[ftmp5] \n\t" "pcmpgth %[ftmp4], %[ftmp4], %[ftmp5] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp3] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp4] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp3] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp4] \n\t" "paddh %[ftmp1], %[ftmp1], %[ftmp0] \n\t" "paddh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp1] \n\t" - "xor %[ftmp4], %[ftmp4], %[ftmp2] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp1] \n\t" + "pxor %[ftmp4], %[ftmp4], %[ftmp2] \n\t" "pcmpeqh %[ftmp1], %[ftmp1], %[ftmp0] \n\t" "pcmpeqh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" "pandn %[ftmp1], %[ftmp1], %[ftmp3] \n\t" @@ -93,7 +94,7 @@ void ff_dct_unquantize_h263_intra_mmi(MpegEncContext *s, int16_t *block, [addr0]"=&r"(addr[0]) : [block]"r"((mips_reg)(block+nCoeffs)), [nCoeffs]"r"((mips_reg)(2*(-nCoeffs))), - [qmul]"f"(qmul), [qadd]"f"(qadd) + [qmul]"f"(qmul_u.f), [qadd]"f"(qadd_u.f) : "memory" ); @@ -103,24 +104,25 @@ void ff_dct_unquantize_h263_intra_mmi(MpegEncContext *s, int16_t *block, void ff_dct_unquantize_h263_inter_mmi(MpegEncContext *s, int16_t *block, int n, int qscale) { - int64_t qmul, qadd, nCoeffs; + int64_t nCoeffs; double ftmp[6]; mips_reg addr[1]; + union mmi_intfloat64 qmul_u, qadd_u; DECLARE_VAR_ALL64; - qmul = qscale << 1; - qadd = (qscale - 1) | 1; + qmul_u.i = qscale << 1; + qadd_u.i = (qscale - 1) | 1; av_assert2(s->block_last_index[n]>=0 || s->h263_aic); nCoeffs = s->inter_scantable.raster_end[s->block_last_index[n]]; __asm__ volatile ( "packsswh %[qmul], %[qmul], %[qmul] \n\t" "packsswh %[qmul], %[qmul], %[qmul] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "packsswh %[qadd], %[qadd], %[qadd] \n\t" "packsswh %[qadd], %[qadd], %[qadd] \n\t" "psubh %[ftmp0], %[ftmp0], %[qadd] \n\t" - "xor %[ftmp5], %[ftmp5], %[ftmp5] \n\t" + "pxor %[ftmp5], %[ftmp5], %[ftmp5] \n\t" ".p2align 4 \n\t" "1: \n\t" PTR_ADDU "%[addr0], %[block], %[nCoeffs] \n\t" @@ -132,12 +134,12 @@ void ff_dct_unquantize_h263_inter_mmi(MpegEncContext *s, int16_t *block, "pmullh %[ftmp2], %[ftmp2], %[qmul] \n\t" "pcmpgth %[ftmp3], %[ftmp3], %[ftmp5] \n\t" "pcmpgth %[ftmp4], %[ftmp4], %[ftmp5] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp3] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp4] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp3] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp4] \n\t" "paddh %[ftmp1], %[ftmp1], %[ftmp0] \n\t" "paddh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp1] \n\t" - "xor %[ftmp4], %[ftmp4], %[ftmp2] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp1] \n\t" + "pxor %[ftmp4], %[ftmp4], %[ftmp2] \n\t" "pcmpeqh %[ftmp1], %[ftmp1], %[ftmp0] \n\t" "pcmpeqh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" "pandn %[ftmp1], %[ftmp1], %[ftmp3] \n\t" @@ -153,7 +155,7 @@ void ff_dct_unquantize_h263_inter_mmi(MpegEncContext *s, int16_t *block, [addr0]"=&r"(addr[0]) : [block]"r"((mips_reg)(block+nCoeffs)), [nCoeffs]"r"((mips_reg)(2*(-nCoeffs))), - [qmul]"f"(qmul), [qadd]"f"(qadd) + [qmul]"f"(qmul_u.f), [qadd]"f"(qadd_u.f) : "memory" ); } @@ -201,18 +203,18 @@ void ff_dct_unquantize_mpeg1_intra_mmi(MpegEncContext *s, int16_t *block, MMI_LDXC1(%[ftmp7], %[addr0], %[quant], 0x08) "pmullh %[ftmp6], %[ftmp6], %[ftmp1] \n\t" "pmullh %[ftmp7], %[ftmp7], %[ftmp1] \n\t" - "xor %[ftmp8], %[ftmp8], %[ftmp8] \n\t" - "xor %[ftmp9], %[ftmp9], %[ftmp9] \n\t" + "pxor %[ftmp8], %[ftmp8], %[ftmp8] \n\t" + "pxor %[ftmp9], %[ftmp9], %[ftmp9] \n\t" "pcmpgth %[ftmp8], %[ftmp8], %[ftmp2] \n\t" "pcmpgth %[ftmp9], %[ftmp9], %[ftmp3] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp9] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "psubh %[ftmp2], %[ftmp2], %[ftmp8] \n\t" "psubh %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "pmullh %[ftmp2], %[ftmp2], %[ftmp6] \n\t" "pmullh %[ftmp3], %[ftmp3], %[ftmp7] \n\t" - "xor %[ftmp6], %[ftmp6], %[ftmp6] \n\t" - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" + "pxor %[ftmp6], %[ftmp6], %[ftmp6] \n\t" + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" "pcmpeqh %[ftmp6], %[ftmp6], %[ftmp4] \n\t" "dli %[tmp0], 0x03 \n\t" "pcmpeqh %[ftmp7], %[ftmp7], %[ftmp5] \n\t" @@ -221,10 +223,10 @@ void ff_dct_unquantize_mpeg1_intra_mmi(MpegEncContext *s, int16_t *block, "psrah %[ftmp3], %[ftmp3], %[ftmp4] \n\t" "psubh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" "psubh %[ftmp3], %[ftmp3], %[ftmp0] \n\t" - "or %[ftmp2], %[ftmp2], %[ftmp0] \n\t" - "or %[ftmp3], %[ftmp3], %[ftmp0] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp9] \n\t" + "por %[ftmp2], %[ftmp2], %[ftmp0] \n\t" + "por %[ftmp3], %[ftmp3], %[ftmp0] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "psubh %[ftmp2], %[ftmp2], %[ftmp8] \n\t" "psubh %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "pandn %[ftmp6], %[ftmp6], %[ftmp2] \n\t" @@ -287,12 +289,12 @@ void ff_dct_unquantize_mpeg1_inter_mmi(MpegEncContext *s, int16_t *block, MMI_LDXC1(%[ftmp7], %[addr0], %[quant], 0x08) "pmullh %[ftmp6], %[ftmp6], %[ftmp1] \n\t" "pmullh %[ftmp7], %[ftmp7], %[ftmp1] \n\t" - "xor %[ftmp8], %[ftmp8], %[ftmp8] \n\t" - "xor %[ftmp9], %[ftmp9], %[ftmp9] \n\t" + "pxor %[ftmp8], %[ftmp8], %[ftmp8] \n\t" + "pxor %[ftmp9], %[ftmp9], %[ftmp9] \n\t" "pcmpgth %[ftmp8], %[ftmp8], %[ftmp2] \n\t" "pcmpgth %[ftmp9], %[ftmp9], %[ftmp3] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp9] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "psubh %[ftmp2], %[ftmp2], %[ftmp8] \n\t" "psubh %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "paddh %[ftmp2], %[ftmp2], %[ftmp2] \n\t" @@ -301,8 +303,8 @@ void ff_dct_unquantize_mpeg1_inter_mmi(MpegEncContext *s, int16_t *block, "paddh %[ftmp3], %[ftmp3], %[ftmp0] \n\t" "pmullh %[ftmp2], %[ftmp2], %[ftmp6] \n\t" "pmullh %[ftmp3], %[ftmp3], %[ftmp7] \n\t" - "xor %[ftmp6], %[ftmp6], %[ftmp6] \n\t" - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" + "pxor %[ftmp6], %[ftmp6], %[ftmp6] \n\t" + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" "pcmpeqh %[ftmp6], %[ftmp6], %[ftmp4] \n\t" "dli %[tmp0], 0x04 \n\t" "pcmpeqh %[ftmp7], %[ftmp7], %[ftmp5] \n\t" @@ -311,10 +313,10 @@ void ff_dct_unquantize_mpeg1_inter_mmi(MpegEncContext *s, int16_t *block, "psrah %[ftmp3], %[ftmp3], %[ftmp4] \n\t" "psubh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" "psubh %[ftmp3], %[ftmp3], %[ftmp0] \n\t" - "or %[ftmp2], %[ftmp2], %[ftmp0] \n\t" - "or %[ftmp3], %[ftmp3], %[ftmp0] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp9] \n\t" + "por %[ftmp2], %[ftmp2], %[ftmp0] \n\t" + "por %[ftmp3], %[ftmp3], %[ftmp0] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "psubh %[ftmp2], %[ftmp2], %[ftmp8] \n\t" "psubh %[ftmp3], %[ftmp3], %[ftmp9] \n\t" "pandn %[ftmp6], %[ftmp6], %[ftmp2] \n\t" @@ -386,26 +388,26 @@ void ff_dct_unquantize_mpeg2_intra_mmi(MpegEncContext *s, int16_t *block, MMI_LDXC1(%[ftmp6], %[addr0], %[quant], 0x08) "pmullh %[ftmp5], %[ftmp5], %[ftmp9] \n\t" "pmullh %[ftmp6], %[ftmp6], %[ftmp9] \n\t" - "xor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" - "xor %[ftmp8], %[ftmp8], %[ftmp8] \n\t" + "pxor %[ftmp7], %[ftmp7], %[ftmp7] \n\t" + "pxor %[ftmp8], %[ftmp8], %[ftmp8] \n\t" "pcmpgth %[ftmp7], %[ftmp7], %[ftmp1] \n\t" "pcmpgth %[ftmp8], %[ftmp8], %[ftmp2] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp7] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp7] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" "psubh %[ftmp1], %[ftmp1], %[ftmp7] \n\t" "psubh %[ftmp2], %[ftmp2], %[ftmp8] \n\t" "pmullh %[ftmp1], %[ftmp1], %[ftmp5] \n\t" "pmullh %[ftmp2], %[ftmp2], %[ftmp6] \n\t" - "xor %[ftmp5], %[ftmp5], %[ftmp5] \n\t" - "xor %[ftmp6], %[ftmp6], %[ftmp6] \n\t" + "pxor %[ftmp5], %[ftmp5], %[ftmp5] \n\t" + "pxor %[ftmp6], %[ftmp6], %[ftmp6] \n\t" "pcmpeqh %[ftmp5], %[ftmp5], %[ftmp3] \n\t" "dli %[tmp0], 0x03 \n\t" "pcmpeqh %[ftmp6] , %[ftmp6], %[ftmp4] \n\t" "mtc1 %[tmp0], %[ftmp3] \n\t" "psrah %[ftmp1], %[ftmp1], %[ftmp3] \n\t" "psrah %[ftmp2], %[ftmp2], %[ftmp3] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp7] \n\t" - "xor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp7] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp8] \n\t" "psubh %[ftmp1], %[ftmp1], %[ftmp7] \n\t" "psubh %[ftmp2], %[ftmp2], %[ftmp8] \n\t" "pandn %[ftmp5], %[ftmp5], %[ftmp1] \n\t" @@ -445,16 +447,16 @@ void ff_denoise_dct_mmi(MpegEncContext *s, int16_t *block) s->dct_count[intra]++; __asm__ volatile( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "1: \n\t" MMI_LDC1(%[ftmp1], %[block], 0x00) - "xor %[ftmp2], %[ftmp2], %[ftmp2] \n\t" + "pxor %[ftmp2], %[ftmp2], %[ftmp2] \n\t" MMI_LDC1(%[ftmp3], %[block], 0x08) - "xor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" + "pxor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" "pcmpgth %[ftmp2], %[ftmp2], %[ftmp1] \n\t" "pcmpgth %[ftmp4], %[ftmp4], %[ftmp3] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp2] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp4] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp2] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp4] \n\t" "psubh %[ftmp1], %[ftmp1], %[ftmp2] \n\t" "psubh %[ftmp3], %[ftmp3], %[ftmp4] \n\t" MMI_LDC1(%[ftmp6], %[offset], 0x00) @@ -463,8 +465,8 @@ void ff_denoise_dct_mmi(MpegEncContext *s, int16_t *block) MMI_LDC1(%[ftmp6], %[offset], 0x08) "mov.d %[ftmp7], %[ftmp3] \n\t" "psubush %[ftmp3], %[ftmp3], %[ftmp6] \n\t" - "xor %[ftmp1], %[ftmp1], %[ftmp2] \n\t" - "xor %[ftmp3], %[ftmp3], %[ftmp4] \n\t" + "pxor %[ftmp1], %[ftmp1], %[ftmp2] \n\t" + "pxor %[ftmp3], %[ftmp3], %[ftmp4] \n\t" "psubh %[ftmp1], %[ftmp1], %[ftmp2] \n\t" "psubh %[ftmp3], %[ftmp3], %[ftmp4] \n\t" MMI_SDC1(%[ftmp1], %[block], 0x00) diff --git a/libavcodec/mips/pixblockdsp_mmi.c b/libavcodec/mips/pixblockdsp_mmi.c index a915a3c28b..1230f5de88 100644 --- a/libavcodec/mips/pixblockdsp_mmi.c +++ b/libavcodec/mips/pixblockdsp_mmi.c @@ -33,7 +33,7 @@ void ff_get_pixels_8_mmi(int16_t *av_restrict block, const uint8_t *pixels, DECLARE_VAR_ADDRT; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" MMI_LDC1(%[ftmp1], %[pixels], 0x00) MMI_LDXC1(%[ftmp2], %[pixels], %[stride], 0x00) @@ -103,12 +103,12 @@ void ff_diff_pixels_mmi(int16_t *av_restrict block, const uint8_t *src1, __asm__ volatile ( "li %[tmp0], 0x08 \n\t" - "xor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" + "pxor %[ftmp4], %[ftmp4], %[ftmp4] \n\t" "1: \n\t" MMI_LDC1(%[ftmp0], %[src1], 0x00) - "or %[ftmp1], %[ftmp0], %[ftmp0] \n\t" + "por %[ftmp1], %[ftmp0], %[ftmp0] \n\t" MMI_LDC1(%[ftmp2], %[src2], 0x00) - "or %[ftmp3], %[ftmp2], %[ftmp2] \n\t" + "por %[ftmp3], %[ftmp2], %[ftmp2] \n\t" "punpcklbh %[ftmp0], %[ftmp0], %[ftmp4] \n\t" "punpckhbh %[ftmp1], %[ftmp1], %[ftmp4] \n\t" "punpcklbh %[ftmp2], %[ftmp2], %[ftmp4] \n\t" diff --git a/libavcodec/mips/simple_idct_mmi.c b/libavcodec/mips/simple_idct_mmi.c index 73d797ffbc..fc2a84f863 100644 --- a/libavcodec/mips/simple_idct_mmi.c +++ b/libavcodec/mips/simple_idct_mmi.c @@ -132,7 +132,7 @@ void ff_simple_idct_8_mmi(int16_t *block) "psllh $f28, "#src1", $f30 \n\t" \ "dmtc1 $9, $f31 \n\t" \ "punpcklhw $f29, $f28, $f28 \n\t" \ - "and $f29, $f29, $f31 \n\t" \ + "pand $f29, $f29, $f31 \n\t" \ "paddw $f28, $f28, $f29 \n\t" \ "punpcklwd "#src1", $f28, $f28 \n\t" \ "punpcklwd "#src2", $f28, $f28 \n\t" \ @@ -267,9 +267,9 @@ void ff_simple_idct_8_mmi(int16_t *block) "punpcklwd $f8, $f27, $f29 \n\t" "punpckhwd $f12, $f27, $f29 \n\t" - "or $f26, $f2, $f6 \n\t" - "or $f26, $f26, $f10 \n\t" - "or $f26, $f26, $f14 \n\t" + "por $f26, $f2, $f6 \n\t" + "por $f26, $f26, $f10 \n\t" + "por $f26, $f26, $f14 \n\t" "dmfc1 $10, $f26 \n\t" "bnez $10, 1f \n\t" /* case1: In this case, row[1,3,5,7] are all zero */ @@ -337,9 +337,9 @@ void ff_simple_idct_8_mmi(int16_t *block) "punpcklwd $f9, $f27, $f29 \n\t" "punpckhwd $f13, $f27, $f29 \n\t" - "or $f26, $f3, $f7 \n\t" - "or $f26, $f26, $f11 \n\t" - "or $f26, $f26, $f15 \n\t" + "por $f26, $f3, $f7 \n\t" + "por $f26, $f26, $f11 \n\t" + "por $f26, $f26, $f15 \n\t" "dmfc1 $10, $f26 \n\t" "bnez $10, 1f \n\t" /* case1: In this case, row[1,3,5,7] are all zero */ diff --git a/libavcodec/mips/vc1dsp_mmi.c b/libavcodec/mips/vc1dsp_mmi.c index 98378683b8..693f1013fe 100644 --- a/libavcodec/mips/vc1dsp_mmi.c +++ b/libavcodec/mips/vc1dsp_mmi.c @@ -126,12 +126,14 @@ void ff_vc1_inv_trans_8x8_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo double ftmp[9]; mips_reg addr[1]; int count; + union mmi_intfloat64 dc_u; dc = (3 * dc + 1) >> 1; dc = (3 * dc + 16) >> 5; + dc_u.i = dc; __asm__ volatile( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[dc], %[dc], %[ftmp0] \n\t" "li %[count], 0x02 \n\t" @@ -186,7 +188,7 @@ void ff_vc1_inv_trans_8x8_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo [addr0]"=&r"(addr[0]), [count]"=&r"(count), [dest]"+&r"(dest) : [linesize]"r"((mips_reg)linesize), - [dc]"f"(dc) + [dc]"f"(dc_u.f) : "memory" ); } @@ -195,9 +197,6 @@ void ff_vc1_inv_trans_8x8_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo void ff_vc1_inv_trans_8x8_mmi(int16_t block[64]) { DECLARE_ALIGNED(16, int16_t, temp[64]); - DECLARE_ALIGNED(8, const uint64_t, ff_pw_1_local) = {0x0000000100000001ULL}; - DECLARE_ALIGNED(8, const uint64_t, ff_pw_4_local) = {0x0000000400000004ULL}; - DECLARE_ALIGNED(8, const uint64_t, ff_pw_64_local)= {0x0000004000000040ULL}; double ftmp[23]; uint64_t tmp[1]; @@ -404,8 +403,8 @@ void ff_vc1_inv_trans_8x8_mmi(int16_t block[64]) [ftmp20]"=&f"(ftmp[20]), [ftmp21]"=&f"(ftmp[21]), [ftmp22]"=&f"(ftmp[22]), [tmp0]"=&r"(tmp[0]) - : [ff_pw_1]"f"(ff_pw_1_local), [ff_pw_64]"f"(ff_pw_64_local), - [ff_pw_4]"f"(ff_pw_4_local), [block]"r"(block), + : [ff_pw_1]"f"(ff_pw_32_1.f), [ff_pw_64]"f"(ff_pw_32_64.f), + [ff_pw_4]"f"(ff_pw_32_4.f), [block]"r"(block), [temp]"r"(temp) : "memory" ); @@ -417,12 +416,14 @@ void ff_vc1_inv_trans_8x4_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo { int dc = block[0]; double ftmp[9]; + union mmi_intfloat64 dc_u; dc = ( 3 * dc + 1) >> 1; dc = (17 * dc + 64) >> 7; + dc_u.i = dc; __asm__ volatile( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[dc], %[dc], %[ftmp0] \n\t" MMI_LDC1(%[ftmp1], %[dest0], 0x00) @@ -464,7 +465,7 @@ void ff_vc1_inv_trans_8x4_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo [ftmp8]"=&f"(ftmp[8]) : [dest0]"r"(dest+0*linesize), [dest1]"r"(dest+1*linesize), [dest2]"r"(dest+2*linesize), [dest3]"r"(dest+3*linesize), - [dc]"f"(dc) + [dc]"f"(dc_u.f) : "memory" ); } @@ -477,8 +478,6 @@ void ff_vc1_inv_trans_8x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) double ftmp[16]; uint32_t tmp[1]; int16_t count = 4; - DECLARE_ALIGNED(16, const uint64_t, ff_pw_4_local) = {0x0000000400000004ULL}; - DECLARE_ALIGNED(16, const uint64_t, ff_pw_64_local)= {0x0000004000000040ULL}; int16_t coeff[64] = {12, 16, 16, 15, 12, 9, 6, 4, 12, 15, 6, -4, -12, -16, -16, -9, 12, 9, -6, -16, -12, 4, 16, 15, @@ -588,7 +587,7 @@ void ff_vc1_inv_trans_8x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) [ftmp12]"=&f"(ftmp[12]), [ftmp13]"=&f"(ftmp[13]), [ftmp14]"=&f"(ftmp[14]), [tmp0]"=&r"(tmp[0]), [src]"+&r"(src), [dst]"+&r"(dst), [count]"+&r"(count) - : [ff_pw_4]"f"(ff_pw_4_local), [coeff]"r"(coeff) + : [ff_pw_4]"f"(ff_pw_32_4.f), [coeff]"r"(coeff) : "memory" ); @@ -702,7 +701,7 @@ void ff_vc1_inv_trans_8x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) MMI_LWC1(%[ftmp3], %[tmp0], 0x00) PTR_ADDU "%[tmp0], %[tmp0], %[linesize] \n\t" MMI_LWC1(%[ftmp4], %[tmp0], 0x00) - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "punpcklbh %[ftmp1], %[ftmp1], %[ftmp0] \n\t" "punpcklbh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" "punpcklbh %[ftmp3], %[ftmp3], %[ftmp0] \n\t" @@ -826,7 +825,7 @@ void ff_vc1_inv_trans_8x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) MMI_LWC1(%[ftmp3], %[tmp0], 0x04) PTR_ADDU "%[tmp0], %[tmp0], %[linesize] \n\t" MMI_LWC1(%[ftmp4], %[tmp0], 0x04) - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "punpcklbh %[ftmp1], %[ftmp1], %[ftmp0] \n\t" "punpcklbh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" "punpcklbh %[ftmp3], %[ftmp3], %[ftmp0] \n\t" @@ -856,7 +855,7 @@ void ff_vc1_inv_trans_8x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) [ftmp12]"=&f"(ftmp[12]), [ftmp13]"=&f"(ftmp[13]), [ftmp14]"=&f"(ftmp[14]), [ftmp15]"=&f"(ftmp[15]), [tmp0]"=&r"(tmp[0]) - : [ff_pw_64]"f"(ff_pw_64_local), + : [ff_pw_64]"f"(ff_pw_32_64.f), [src]"r"(src), [dest]"r"(dest), [linesize]"r"(linesize) :"memory" ); @@ -868,13 +867,15 @@ void ff_vc1_inv_trans_4x8_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo { int dc = block[0]; double ftmp[9]; + union mmi_intfloat64 dc_u; DECLARE_VAR_LOW32; dc = (17 * dc + 4) >> 3; dc = (12 * dc + 64) >> 7; + dc_u.i = dc; __asm__ volatile( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[dc], %[dc], %[ftmp0] \n\t" MMI_LWC1(%[ftmp1], %[dest0], 0x00) @@ -931,7 +932,7 @@ void ff_vc1_inv_trans_4x8_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo [dest2]"r"(dest+2*linesize), [dest3]"r"(dest+3*linesize), [dest4]"r"(dest+4*linesize), [dest5]"r"(dest+5*linesize), [dest6]"r"(dest+6*linesize), [dest7]"r"(dest+7*linesize), - [dc]"f"(dc) + [dc]"f"(dc_u.f) : "memory" ); } @@ -942,14 +943,11 @@ void ff_vc1_inv_trans_4x8_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) int16_t *src = block; int16_t *dst = block; double ftmp[23]; - uint32_t count = 8, tmp[1]; + uint64_t count = 8, tmp[1]; int16_t coeff[16] = {17, 22, 17, 10, 17, 10,-17,-22, 17,-10,-17, 22, 17,-22, 17,-10}; - DECLARE_ALIGNED(8, const uint64_t, ff_pw_1_local) = {0x0000000100000001ULL}; - DECLARE_ALIGNED(8, const uint64_t, ff_pw_4_local) = {0x0000000400000004ULL}; - DECLARE_ALIGNED(8, const uint64_t, ff_pw_64_local)= {0x0000004000000040ULL}; // 1st loop __asm__ volatile ( @@ -995,7 +993,7 @@ void ff_vc1_inv_trans_4x8_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) [ftmp10]"=&f"(ftmp[10]), [ftmp11]"=&f"(ftmp[11]), [tmp0]"=&r"(tmp[0]), [count]"+&r"(count), [src]"+&r"(src), [dst]"+&r"(dst) - : [ff_pw_4]"f"(ff_pw_4_local), [coeff]"r"(coeff) + : [ff_pw_4]"f"(ff_pw_32_4.f), [coeff]"r"(coeff) : "memory" ); @@ -1055,7 +1053,7 @@ void ff_vc1_inv_trans_4x8_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) MMI_LWC1(%[ftmp7], %[tmp0], 0x00) PTR_ADDU "%[tmp0], %[tmp0], %[linesize] \n\t" MMI_LWC1(%[ftmp8], %[tmp0], 0x00) - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "punpcklbh %[ftmp1], %[ftmp1], %[ftmp0] \n\t" "punpcklbh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" "punpcklbh %[ftmp3], %[ftmp3], %[ftmp0] \n\t" @@ -1112,7 +1110,7 @@ void ff_vc1_inv_trans_4x8_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) [ftmp20]"=&f"(ftmp[20]), [ftmp21]"=&f"(ftmp[21]), [ftmp22]"=&f"(ftmp[22]), [tmp0]"=&r"(tmp[0]) - : [ff_pw_1]"f"(ff_pw_1_local), [ff_pw_64]"f"(ff_pw_64_local), + : [ff_pw_1]"f"(ff_pw_32_1.f), [ff_pw_64]"f"(ff_pw_32_64.f), [src]"r"(src), [dest]"r"(dest), [linesize]"r"(linesize) : "memory" ); @@ -1124,13 +1122,15 @@ void ff_vc1_inv_trans_4x4_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo { int dc = block[0]; double ftmp[5]; + union mmi_intfloat64 dc_u; DECLARE_VAR_LOW32; dc = (17 * dc + 4) >> 3; dc = (17 * dc + 64) >> 7; + dc_u.i = dc; __asm__ volatile( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "pshufh %[dc], %[dc], %[ftmp0] \n\t" MMI_LWC1(%[ftmp1], %[dest0], 0x00) @@ -1163,7 +1163,7 @@ void ff_vc1_inv_trans_4x4_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo [ftmp4]"=&f"(ftmp[4]) : [dest0]"r"(dest+0*linesize), [dest1]"r"(dest+1*linesize), [dest2]"r"(dest+2*linesize), [dest3]"r"(dest+3*linesize), - [dc]"f"(dc) + [dc]"f"(dc_u.f) : "memory" ); } @@ -1178,8 +1178,6 @@ void ff_vc1_inv_trans_4x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) 17, 10,-17,-22, 17,-10,-17, 22, 17,-22, 17,-10}; - DECLARE_ALIGNED(8, const uint64_t, ff_pw_4_local) = {0x0000000400000004ULL}; - DECLARE_ALIGNED(8, const uint64_t, ff_pw_64_local)= {0x0000004000000040ULL}; // 1st loop __asm__ volatile ( @@ -1223,7 +1221,7 @@ void ff_vc1_inv_trans_4x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) [ftmp10]"=&f"(ftmp[10]), [ftmp11]"=&f"(ftmp[11]), [tmp0]"=&r"(tmp[0]), [count]"+&r"(count), [src]"+&r"(src), [dst]"+&r"(dst) - : [ff_pw_4]"f"(ff_pw_4_local), [coeff]"r"(coeff) + : [ff_pw_4]"f"(ff_pw_32_4.f), [coeff]"r"(coeff) : "memory" ); @@ -1336,7 +1334,7 @@ void ff_vc1_inv_trans_4x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) MMI_LWC1(%[ftmp3], %[tmp0], 0x00) PTR_ADDU "%[tmp0], %[tmp0], %[linesize] \n\t" MMI_LWC1(%[ftmp4], %[tmp0], 0x00) - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "punpcklbh %[ftmp1], %[ftmp1], %[ftmp0] \n\t" "punpcklbh %[ftmp2], %[ftmp2], %[ftmp0] \n\t" "punpcklbh %[ftmp3], %[ftmp3], %[ftmp0] \n\t" @@ -1367,7 +1365,7 @@ void ff_vc1_inv_trans_4x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) [ftmp12]"=&f"(ftmp[12]), [ftmp13]"=&f"(ftmp[13]), [ftmp14]"=&f"(ftmp[14]), [ftmp15]"=&f"(ftmp[15]), [tmp0]"=&r"(tmp[0]) - : [ff_pw_64]"f"(ff_pw_64_local), + : [ff_pw_64]"f"(ff_pw_32_64.f), [src]"r"(src), [dest]"r"(dest), [linesize]"r"(linesize) :"memory" ); @@ -1657,14 +1655,15 @@ static void vc1_put_ver_16b_shift2_mmi(int16_t *dst, const uint8_t *src, mips_reg stride, int rnd, int64_t shift) { + union mmi_intfloat64 shift_u; DECLARE_VAR_LOW32; DECLARE_VAR_ADDRT; + shift_u.i = shift; __asm__ volatile( - "xor $f0, $f0, $f0 \n\t" + "pxor $f0, $f0, $f0 \n\t" "li $8, 0x03 \n\t" LOAD_ROUNDER_MMI("%[rnd]") - "ldc1 $f12, %[ff_pw_9] \n\t" "1: \n\t" MMI_ULWC1($f4, %[src], 0x00) PTR_ADDU "%[src], %[src], %[stride] \n\t" @@ -1686,9 +1685,9 @@ static void vc1_put_ver_16b_shift2_mmi(int16_t *dst, : RESTRICT_ASM_LOW32 RESTRICT_ASM_ADDRT [src]"+r"(src), [dst]"+r"(dst) : [stride]"r"(stride), [stride1]"r"(-2*stride), - [shift]"f"(shift), [rnd]"m"(rnd), - [stride2]"r"(9*stride-4), [ff_pw_9]"m"(ff_pw_9) - : "$8", "$9", "$f0", "$f2", "$f4", "$f6", "$f8", "$f10", "$f12", + [shift]"f"(shift_u.f), [rnd]"m"(rnd), + [stride2]"r"(9*stride-4) + : "$8", "$9", "$f0", "$f2", "$f4", "$f6", "$f8", "$f10", "$f14", "$f16", "memory" ); } @@ -1710,8 +1709,6 @@ static void OPNAME ## vc1_hor_16b_shift2_mmi(uint8_t *dst, mips_reg stride, \ \ __asm__ volatile( \ LOAD_ROUNDER_MMI("%[rnd]") \ - "ldc1 $f12, %[ff_pw_128] \n\t" \ - "ldc1 $f10, %[ff_pw_9] \n\t" \ "1: \n\t" \ MMI_ULDC1($f2, %[src], 0x00) \ MMI_ULDC1($f4, %[src], 0x08) \ @@ -1725,16 +1722,16 @@ static void OPNAME ## vc1_hor_16b_shift2_mmi(uint8_t *dst, mips_reg stride, \ "paddh $f6, $f6, $f0 \n\t" \ MMI_ULDC1($f0, %[src], 0x0b) \ "paddh $f8, $f8, $f0 \n\t" \ - "pmullh $f6, $f6, $f10 \n\t" \ - "pmullh $f8, $f8, $f10 \n\t" \ + "pmullh $f6, $f6, %[ff_pw_9] \n\t" \ + "pmullh $f8, $f8, %[ff_pw_9] \n\t" \ "psubh $f6, $f6, $f2 \n\t" \ "psubh $f8, $f8, $f4 \n\t" \ "li $8, 0x07 \n\t" \ "mtc1 $8, $f16 \n\t" \ NORMALIZE_MMI("$f16") \ /* Remove bias */ \ - "paddh $f6, $f6, $f12 \n\t" \ - "paddh $f8, $f8, $f12 \n\t" \ + "paddh $f6, $f6, %[ff_pw_128] \n\t" \ + "paddh $f8, $f8, %[ff_pw_128] \n\t" \ TRANSFER_DO_PACK(OP) \ "addiu %[h], %[h], -0x01 \n\t" \ PTR_ADDIU "%[src], %[src], 0x18 \n\t" \ @@ -1744,8 +1741,8 @@ static void OPNAME ## vc1_hor_16b_shift2_mmi(uint8_t *dst, mips_reg stride, \ [h]"+r"(h), \ [src]"+r"(src), [dst]"+r"(dst) \ : [stride]"r"(stride), [rnd]"m"(rnd), \ - [ff_pw_9]"m"(ff_pw_9), [ff_pw_128]"m"(ff_pw_128) \ - : "$8", "$f0", "$f2", "$f4", "$f6", "$f8", "$f10", "$f12", "$f14", \ + [ff_pw_9]"f"(ff_pw_9.f), [ff_pw_128]"f"(ff_pw_128.f) \ + : "$8", "$f0", "$f2", "$f4", "$f6", "$f8", "$f14", \ "$f16", "memory" \ ); \ } @@ -1768,10 +1765,9 @@ static void OPNAME ## vc1_shift2_mmi(uint8_t *dst, const uint8_t *src, \ rnd = 8 - rnd; \ \ __asm__ volatile( \ - "xor $f0, $f0, $f0 \n\t" \ + "pxor $f0, $f0, $f0 \n\t" \ "li $10, 0x08 \n\t" \ LOAD_ROUNDER_MMI("%[rnd]") \ - "ldc1 $f12, %[ff_pw_9] \n\t" \ "1: \n\t" \ MMI_ULWC1($f6, %[src], 0x00) \ MMI_ULWC1($f8, %[src], 0x04) \ @@ -1788,8 +1784,8 @@ static void OPNAME ## vc1_shift2_mmi(uint8_t *dst, const uint8_t *src, \ PTR_ADDU "$9, %[src], %[offset_x2n] \n\t" \ MMI_ULWC1($f2, $9, 0x00) \ MMI_ULWC1($f4, $9, 0x04) \ - "pmullh $f6, $f6, $f12 \n\t" /* 0,9,9,0*/ \ - "pmullh $f8, $f8, $f12 \n\t" /* 0,9,9,0*/ \ + "pmullh $f6, $f6, %[ff_pw_9] \n\t" /* 0,9,9,0*/ \ + "pmullh $f8, $f8, %[ff_pw_9] \n\t" /* 0,9,9,0*/ \ "punpcklbh $f2, $f2, $f0 \n\t" \ "punpcklbh $f4, $f4, $f0 \n\t" \ "psubh $f6, $f6, $f2 \n\t" /*-1,9,9,0*/ \ @@ -1816,9 +1812,9 @@ static void OPNAME ## vc1_shift2_mmi(uint8_t *dst, const uint8_t *src, \ : [offset]"r"(offset), [offset_x2n]"r"(-2*offset), \ [stride]"r"(stride), [rnd]"m"(rnd), \ [stride1]"r"(stride-offset), \ - [ff_pw_9]"m"(ff_pw_9) \ + [ff_pw_9]"f"(ff_pw_9.f) \ : "$8", "$9", "$10", "$f0", "$f2", "$f4", "$f6", "$f8", "$f10", \ - "$f12", "$f14", "$f16", "memory" \ + "$f14", "$f16", "memory" \ ); \ } @@ -1849,8 +1845,8 @@ VC1_SHIFT2(OP_AVG, avg_) LOAD($f8, $9, M*4) \ UNPACK("$f6") \ UNPACK("$f8") \ - "pmullh $f6, $f6, $f12 \n\t" /* *18 */ \ - "pmullh $f8, $f8, $f12 \n\t" /* *18 */ \ + "pmullh $f6, $f6, %[ff_pw_18] \n\t" /* *18 */ \ + "pmullh $f8, $f8, %[ff_pw_18] \n\t" /* *18 */ \ "psubh $f6, $f6, $f2 \n\t" /* *18, -3 */ \ "psubh $f8, $f8, $f4 \n\t" /* *18, -3 */ \ PTR_ADDU "$9, %[src], "#A4" \n\t" \ @@ -1869,8 +1865,8 @@ VC1_SHIFT2(OP_AVG, avg_) LOAD($f4, $9, M*4) \ UNPACK("$f2") \ UNPACK("$f4") \ - "pmullh $f2, $f2, $f10 \n\t" /* *53 */ \ - "pmullh $f4, $f4, $f10 \n\t" /* *53 */ \ + "pmullh $f2, $f2, %[ff_pw_53] \n\t" /* *53 */ \ + "pmullh $f4, $f4, %[ff_pw_53] \n\t" /* *53 */ \ "paddh $f6, $f6, $f2 \n\t" /* 4,53,18,-3 */ \ "paddh $f8, $f8, $f4 \n\t" /* 4,53,18,-3 */ @@ -1889,16 +1885,16 @@ vc1_put_ver_16b_ ## NAME ## _mmi(int16_t *dst, const uint8_t *src, \ int rnd, int64_t shift) \ { \ int h = 8; \ + union mmi_intfloat64 shift_u; \ DECLARE_VAR_LOW32; \ DECLARE_VAR_ADDRT; \ + shift_u.i = shift; \ \ src -= src_stride; \ \ __asm__ volatile( \ - "xor $f0, $f0, $f0 \n\t" \ + "pxor $f0, $f0, $f0 \n\t" \ LOAD_ROUNDER_MMI("%[rnd]") \ - "ldc1 $f10, %[ff_pw_53] \n\t" \ - "ldc1 $f12, %[ff_pw_18] \n\t" \ ".p2align 3 \n\t" \ "1: \n\t" \ MSPEL_FILTER13_CORE(DO_UNPACK, MMI_ULWC1, 1, A1, A2, A3, A4) \ @@ -1914,12 +1910,12 @@ vc1_put_ver_16b_ ## NAME ## _mmi(int16_t *dst, const uint8_t *src, \ PTR_ADDU "$9, %[src], "#A2" \n\t" \ MMI_ULWC1($f6, $9, 0x08) \ DO_UNPACK("$f6") \ - "pmullh $f6, $f6, $f12 \n\t" /* *18 */ \ + "pmullh $f6, $f6, %[ff_pw_18] \n\t" /* *18 */ \ "psubh $f6, $f6, $f2 \n\t" /* *18,-3 */ \ PTR_ADDU "$9, %[src], "#A3" \n\t" \ MMI_ULWC1($f2, $9, 0x08) \ DO_UNPACK("$f2") \ - "pmullh $f2, $f2, $f10 \n\t" /* *53 */ \ + "pmullh $f2, $f2, %[ff_pw_53] \n\t" /* *53 */ \ "paddh $f6, $f6, $f2 \n\t" /* *53,18,-3 */ \ PTR_ADDU "$9, %[src], "#A4" \n\t" \ MMI_ULWC1($f2, $9, 0x08) \ @@ -1942,10 +1938,10 @@ vc1_put_ver_16b_ ## NAME ## _mmi(int16_t *dst, const uint8_t *src, \ [src]"+r"(src), [dst]"+r"(dst) \ : [stride_x1]"r"(src_stride), [stride_x2]"r"(2*src_stride), \ [stride_x3]"r"(3*src_stride), \ - [rnd]"m"(rnd), [shift]"f"(shift), \ - [ff_pw_53]"m"(ff_pw_53), [ff_pw_18]"m"(ff_pw_18), \ - [ff_pw_3]"f"(ff_pw_3) \ - : "$8", "$9", "$f0", "$f2", "$f4", "$f6", "$f8", "$f10", "$f12", \ + [rnd]"m"(rnd), [shift]"f"(shift_u.f), \ + [ff_pw_53]"f"(ff_pw_53.f), [ff_pw_18]"f"(ff_pw_18.f), \ + [ff_pw_3]"f"(ff_pw_3.f) \ + : "$8", "$9", "$f0", "$f2", "$f4", "$f6", "$f8", \ "$f14", "$f16", "memory" \ ); \ } @@ -1970,10 +1966,8 @@ OPNAME ## vc1_hor_16b_ ## NAME ## _mmi(uint8_t *dst, mips_reg stride, \ rnd -= (-4+58+13-3)*256; /* Add -256 bias */ \ \ __asm__ volatile( \ - "xor $f0, $f0, $f0 \n\t" \ + "pxor $f0, $f0, $f0 \n\t" \ LOAD_ROUNDER_MMI("%[rnd]") \ - "ldc1 $f10, %[ff_pw_53] \n\t" \ - "ldc1 $f12, %[ff_pw_18] \n\t" \ ".p2align 3 \n\t" \ "1: \n\t" \ MSPEL_FILTER13_CORE(DONT_UNPACK, MMI_ULDC1, 2, A1, A2, A3, A4) \ @@ -1992,9 +1986,9 @@ OPNAME ## vc1_hor_16b_ ## NAME ## _mmi(uint8_t *dst, mips_reg stride, \ [h]"+r"(h), \ [src]"+r"(src), [dst]"+r"(dst) \ : [stride]"r"(stride), [rnd]"m"(rnd), \ - [ff_pw_53]"m"(ff_pw_53), [ff_pw_18]"m"(ff_pw_18), \ - [ff_pw_3]"f"(ff_pw_3), [ff_pw_128]"f"(ff_pw_128) \ - : "$8", "$9", "$f0", "$f2", "$f4", "$f6", "$f8", "$f10", "$f12", \ + [ff_pw_53]"f"(ff_pw_53.f), [ff_pw_18]"f"(ff_pw_18.f), \ + [ff_pw_3]"f"(ff_pw_3.f), [ff_pw_128]"f"(ff_pw_128.f) \ + : "$8", "$9", "$f0", "$f2", "$f4", "$f6", "$f8", \ "$f14", "$f16", "memory" \ ); \ } @@ -2020,10 +2014,8 @@ OPNAME ## vc1_## NAME ## _mmi(uint8_t *dst, const uint8_t *src, \ rnd = 32-rnd; \ \ __asm__ volatile ( \ - "xor $f0, $f0, $f0 \n\t" \ + "pxor $f0, $f0, $f0 \n\t" \ LOAD_ROUNDER_MMI("%[rnd]") \ - "ldc1 $f10, %[ff_pw_53] \n\t" \ - "ldc1 $f12, %[ff_pw_18] \n\t" \ ".p2align 3 \n\t" \ "1: \n\t" \ MSPEL_FILTER13_CORE(DO_UNPACK, MMI_ULWC1, 1, A1, A2, A3, A4) \ @@ -2041,9 +2033,9 @@ OPNAME ## vc1_## NAME ## _mmi(uint8_t *dst, const uint8_t *src, \ : [offset_x1]"r"(offset), [offset_x2]"r"(2*offset), \ [offset_x3]"r"(3*offset), [stride]"r"(stride), \ [rnd]"m"(rnd), \ - [ff_pw_53]"m"(ff_pw_53), [ff_pw_18]"m"(ff_pw_18), \ - [ff_pw_3]"f"(ff_pw_3) \ - : "$8", "$9", "$f0", "$f2", "$f4", "$f6", "$f8", "$f10", "$f12", \ + [ff_pw_53]"f"(ff_pw_53.f), [ff_pw_18]"f"(ff_pw_18.f), \ + [ff_pw_3]"f"(ff_pw_3.f) \ + : "$8", "$9", "$f0", "$f2", "$f4", "$f6", "$f8", \ "$f14", "$f16", "memory" \ ); \ } @@ -2243,20 +2235,21 @@ void ff_put_no_rnd_vc1_chroma_mc8_mmi(uint8_t *dst /* align 8 */, uint8_t *src /* align 1 */, ptrdiff_t stride, int h, int x, int y) { - const int A = (8 - x) * (8 - y); - const int B = (x) * (8 - y); - const int C = (8 - x) * (y); - const int D = (x) * (y); + union mmi_intfloat64 A, B, C, D; double ftmp[10]; uint32_t tmp[1]; DECLARE_VAR_ALL64; DECLARE_VAR_ADDRT; + A.i = (8 - x) * (8 - y); + B.i = (x) * (8 - y); + C.i = (8 - x) * (y); + D.i = (x) * (y); av_assert2(x < 8 && y < 8 && x >= 0 && y >= 0); __asm__ volatile( "li %[tmp0], 0x06 \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp9] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[B], %[B], %[ftmp0] \n\t" @@ -2287,9 +2280,9 @@ void ff_put_no_rnd_vc1_chroma_mc8_mmi(uint8_t *dst /* align 8 */, [src]"+&r"(src), [dst]"+&r"(dst), [h]"+&r"(h) : [stride]"r"((mips_reg)stride), - [A]"f"(A), [B]"f"(B), - [C]"f"(C), [D]"f"(D), - [ff_pw_28]"f"(ff_pw_28) + [A]"f"(A.f), [B]"f"(B.f), + [C]"f"(C.f), [D]"f"(D.f), + [ff_pw_28]"f"(ff_pw_28.f) : "memory" ); } @@ -2298,20 +2291,21 @@ void ff_put_no_rnd_vc1_chroma_mc4_mmi(uint8_t *dst /* align 8 */, uint8_t *src /* align 1 */, ptrdiff_t stride, int h, int x, int y) { - const int A = (8 - x) * (8 - y); - const int B = (x) * (8 - y); - const int C = (8 - x) * (y); - const int D = (x) * (y); + union mmi_intfloat64 A, B, C, D; double ftmp[6]; uint32_t tmp[1]; DECLARE_VAR_LOW32; DECLARE_VAR_ADDRT; + A.i = (8 - x) * (8 - y); + B.i = (x) * (8 - y); + C.i = (8 - x) * (y); + D.i = (x) * (y); av_assert2(x < 8 && y < 8 && x >= 0 && y >= 0); __asm__ volatile( "li %[tmp0], 0x06 \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp5] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[B], %[B], %[ftmp0] \n\t" @@ -2340,9 +2334,9 @@ void ff_put_no_rnd_vc1_chroma_mc4_mmi(uint8_t *dst /* align 8 */, [src]"+&r"(src), [dst]"+&r"(dst), [h]"+&r"(h) : [stride]"r"((mips_reg)stride), - [A]"f"(A), [B]"f"(B), - [C]"f"(C), [D]"f"(D), - [ff_pw_28]"f"(ff_pw_28) + [A]"f"(A.f), [B]"f"(B.f), + [C]"f"(C.f), [D]"f"(D.f), + [ff_pw_28]"f"(ff_pw_28.f) : "memory" ); } @@ -2351,20 +2345,21 @@ void ff_avg_no_rnd_vc1_chroma_mc8_mmi(uint8_t *dst /* align 8 */, uint8_t *src /* align 1 */, ptrdiff_t stride, int h, int x, int y) { - const int A = (8 - x) * (8 - y); - const int B = (x) * (8 - y); - const int C = (8 - x) * (y); - const int D = (x) * (y); + union mmi_intfloat64 A, B, C, D; double ftmp[10]; uint32_t tmp[1]; DECLARE_VAR_ALL64; DECLARE_VAR_ADDRT; + A.i = (8 - x) * (8 - y); + B.i = (x) * (8 - y); + C.i = (8 - x) * (y); + D.i = (x) * (y); av_assert2(x < 8 && y < 8 && x >= 0 && y >= 0); __asm__ volatile( "li %[tmp0], 0x06 \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp9] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[B], %[B], %[ftmp0] \n\t" @@ -2398,9 +2393,9 @@ void ff_avg_no_rnd_vc1_chroma_mc8_mmi(uint8_t *dst /* align 8 */, [src]"+&r"(src), [dst]"+&r"(dst), [h]"+&r"(h) : [stride]"r"((mips_reg)stride), - [A]"f"(A), [B]"f"(B), - [C]"f"(C), [D]"f"(D), - [ff_pw_28]"f"(ff_pw_28) + [A]"f"(A.f), [B]"f"(B.f), + [C]"f"(C.f), [D]"f"(D.f), + [ff_pw_28]"f"(ff_pw_28.f) : "memory" ); } @@ -2409,20 +2404,21 @@ void ff_avg_no_rnd_vc1_chroma_mc4_mmi(uint8_t *dst /* align 8 */, uint8_t *src /* align 1 */, ptrdiff_t stride, int h, int x, int y) { - const int A = (8 - x) * (8 - y); - const int B = ( x) * (8 - y); - const int C = (8 - x) * ( y); - const int D = ( x) * ( y); + union mmi_intfloat64 A, B, C, D; double ftmp[6]; uint32_t tmp[1]; DECLARE_VAR_LOW32; DECLARE_VAR_ADDRT; + A.i = (8 - x) * (8 - y); + B.i = (x) * (8 - y); + C.i = (8 - x) * (y); + D.i = (x) * (y); av_assert2(x < 8 && y < 8 && x >= 0 && y >= 0); __asm__ volatile( "li %[tmp0], 0x06 \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "mtc1 %[tmp0], %[ftmp5] \n\t" "pshufh %[A], %[A], %[ftmp0] \n\t" "pshufh %[B], %[B], %[ftmp0] \n\t" @@ -2454,9 +2450,9 @@ void ff_avg_no_rnd_vc1_chroma_mc4_mmi(uint8_t *dst /* align 8 */, [src]"+&r"(src), [dst]"+&r"(dst), [h]"+&r"(h) : [stride]"r"((mips_reg)stride), - [A]"f"(A), [B]"f"(B), - [C]"f"(C), [D]"f"(D), - [ff_pw_28]"f"(ff_pw_28) + [A]"f"(A.f), [B]"f"(B.f), + [C]"f"(C.f), [D]"f"(D.f), + [ff_pw_28]"f"(ff_pw_28.f) : "memory" ); } diff --git a/libavcodec/mips/vp3dsp_idct_mmi.c b/libavcodec/mips/vp3dsp_idct_mmi.c index c5c4cf3127..0d4cba19ec 100644 --- a/libavcodec/mips/vp3dsp_idct_mmi.c +++ b/libavcodec/mips/vp3dsp_idct_mmi.c @@ -34,7 +34,7 @@ static void idct_row_mmi(int16_t *input) double ftmp[23]; uint64_t tmp[2]; __asm__ volatile ( - "xor %[ftmp10], %[ftmp10], %[ftmp10] \n\t" + "pxor %[ftmp10], %[ftmp10], %[ftmp10] \n\t" LOAD_CONST(%[csth_1], 1) "li %[tmp0], 0x02 \n\t" "1: \n\t" @@ -51,14 +51,14 @@ static void idct_row_mmi(int16_t *input) LOAD_CONST(%[ftmp9], 12785) "pmulhh %[A], %[ftmp9], %[ftmp7] \n\t" "pcmpgth %[C], %[ftmp10], %[ftmp1] \n\t" - "or %[mask], %[C], %[csth_1] \n\t" + "por %[mask], %[C], %[csth_1] \n\t" "pmullh %[B], %[ftmp1], %[mask] \n\t" "pmulhuh %[B], %[ftmp8], %[B] \n\t" "pmullh %[B], %[B], %[mask] \n\t" "paddh %[A], %[A], %[B] \n\t" "paddh %[A], %[A], %[C] \n\t" "pcmpgth %[D], %[ftmp10], %[ftmp7] \n\t" - "or %[mask], %[D], %[csth_1] \n\t" + "por %[mask], %[D], %[csth_1] \n\t" "pmullh %[ftmp7], %[ftmp7], %[mask] \n\t" "pmulhuh %[B], %[ftmp8], %[ftmp7] \n\t" "pmullh %[B], %[B], %[mask] \n\t" @@ -69,12 +69,12 @@ static void idct_row_mmi(int16_t *input) LOAD_CONST(%[ftmp8], 54491) LOAD_CONST(%[ftmp9], 36410) "pcmpgth %[Ad], %[ftmp10], %[ftmp5] \n\t" - "or %[mask], %[Ad], %[csth_1] \n\t" + "por %[mask], %[Ad], %[csth_1] \n\t" "pmullh %[ftmp1], %[ftmp5], %[mask] \n\t" "pmulhuh %[C], %[ftmp9], %[ftmp1] \n\t" "pmullh %[C], %[C], %[mask] \n\t" "pcmpgth %[Bd], %[ftmp10], %[ftmp3] \n\t" - "or %[mask], %[Bd], %[csth_1] \n\t" + "por %[mask], %[Bd], %[csth_1] \n\t" "pmullh %[D], %[ftmp3], %[mask] \n\t" "pmulhuh %[D], %[ftmp8], %[D] \n\t" "pmullh %[D], %[D], %[mask] \n\t" @@ -82,12 +82,12 @@ static void idct_row_mmi(int16_t *input) "paddh %[C], %[C], %[Ad] \n\t" "paddh %[C], %[C], %[Bd] \n\t" "pcmpgth %[Bd], %[ftmp10], %[ftmp3] \n\t" - "or %[mask], %[Bd], %[csth_1] \n\t" + "por %[mask], %[Bd], %[csth_1] \n\t" "pmullh %[ftmp1], %[ftmp3], %[mask] \n\t" "pmulhuh %[D], %[ftmp9], %[ftmp1] \n\t" "pmullh %[D], %[D], %[mask] \n\t" "pcmpgth %[Ed], %[ftmp10], %[ftmp5] \n\t" - "or %[mask], %[Ed], %[csth_1] \n\t" + "por %[mask], %[Ed], %[csth_1] \n\t" "pmullh %[Ad], %[ftmp5], %[mask] \n\t" "pmulhuh %[Ad], %[ftmp8], %[Ad] \n\t" "pmullh %[Ad], %[Ad], %[mask] \n\t" @@ -98,14 +98,14 @@ static void idct_row_mmi(int16_t *input) LOAD_CONST(%[ftmp8], 46341) "psubh %[Ad], %[A], %[C] \n\t" "pcmpgth %[Bd], %[ftmp10], %[Ad] \n\t" - "or %[mask], %[Bd], %[csth_1] \n\t" + "por %[mask], %[Bd], %[csth_1] \n\t" "pmullh %[Ad], %[Ad], %[mask] \n\t" "pmulhuh %[Ad], %[ftmp8], %[Ad] \n\t" "pmullh %[Ad], %[Ad], %[mask] \n\t" "paddh %[Ad], %[Ad], %[Bd] \n\t" "psubh %[Bd], %[B], %[D] \n\t" "pcmpgth %[Cd], %[ftmp10], %[Bd] \n\t" - "or %[mask], %[Cd], %[csth_1] \n\t" + "por %[mask], %[Cd], %[csth_1] \n\t" "pmullh %[Bd], %[Bd], %[mask] \n\t" "pmulhuh %[Bd], %[ftmp8], %[Bd] \n\t" "pmullh %[Bd], %[Bd], %[mask] \n\t" @@ -114,14 +114,14 @@ static void idct_row_mmi(int16_t *input) "paddh %[Dd], %[B], %[D] \n\t" "paddh %[A], %[ftmp0], %[ftmp4] \n\t" "pcmpgth %[B], %[ftmp10], %[A] \n\t" - "or %[mask], %[B], %[csth_1] \n\t" + "por %[mask], %[B], %[csth_1] \n\t" "pmullh %[A], %[A], %[mask] \n\t" "pmulhuh %[A], %[ftmp8], %[A] \n\t" "pmullh %[A], %[A], %[mask] \n\t" "paddh %[A], %[A], %[B] \n\t" "psubh %[B], %[ftmp0], %[ftmp4] \n\t" "pcmpgth %[C], %[ftmp10], %[B] \n\t" - "or %[mask], %[C], %[csth_1] \n\t" + "por %[mask], %[C], %[csth_1] \n\t" "pmullh %[B], %[B], %[mask] \n\t" "pmulhuh %[B], %[ftmp8], %[B] \n\t" "pmullh %[B], %[B], %[mask] \n\t" @@ -131,14 +131,14 @@ static void idct_row_mmi(int16_t *input) LOAD_CONST(%[ftmp9], 25080) "pmulhh %[C], %[ftmp9], %[ftmp6] \n\t" "pcmpgth %[D], %[ftmp10], %[ftmp2] \n\t" - "or %[mask], %[D], %[csth_1] \n\t" + "por %[mask], %[D], %[csth_1] \n\t" "pmullh %[Ed], %[ftmp2], %[mask] \n\t" "pmulhuh %[Ed], %[ftmp8], %[Ed] \n\t" "pmullh %[Ed], %[Ed], %[mask] \n\t" "paddh %[C], %[C], %[Ed] \n\t" "paddh %[C], %[C], %[D] \n\t" "pcmpgth %[Ed], %[ftmp10], %[ftmp6] \n\t" - "or %[mask], %[Ed], %[csth_1] \n\t" + "por %[mask], %[Ed], %[csth_1] \n\t" "pmullh %[ftmp6], %[ftmp6], %[mask] \n\t" "pmulhuh %[D], %[ftmp8], %[ftmp6] \n\t" "pmullh %[D], %[D], %[mask] \n\t" @@ -193,7 +193,7 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) for (int i = 0; i < 8; ++i) temp_value[i] = av_clip_uint8(128 + ((46341 * input[i << 3] + (8 << 16)) >> 20)); __asm__ volatile ( - "xor %[ftmp10], %[ftmp10], %[ftmp10] \n\t" + "pxor %[ftmp10], %[ftmp10], %[ftmp10] \n\t" "li %[tmp0], 0x02 \n\t" "1: \n\t" "ldc1 %[ftmp0], 0x00(%[input]) \n\t" @@ -213,14 +213,14 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[Gd], 1) "pmulhh %[A], %[ftmp9], %[ftmp7] \n\t" "pcmpgth %[C], %[ftmp10], %[ftmp1] \n\t" - "or %[mask], %[C], %[Gd] \n\t" + "por %[mask], %[C], %[Gd] \n\t" "pmullh %[B], %[ftmp1], %[mask] \n\t" "pmulhuh %[B], %[ftmp8], %[B] \n\t" "pmullh %[B], %[B], %[mask] \n\t" "paddh %[A], %[A], %[B] \n\t" "paddh %[A], %[A], %[C] \n\t" "pcmpgth %[D], %[ftmp10], %[ftmp7] \n\t" - "or %[mask], %[D], %[Gd] \n\t" + "por %[mask], %[D], %[Gd] \n\t" "pmullh %[Ad], %[ftmp7], %[mask] \n\t" "pmulhuh %[B], %[ftmp8], %[Ad] \n\t" "pmullh %[B], %[B], %[mask] \n\t" @@ -231,12 +231,12 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[ftmp8], 54491) LOAD_CONST(%[ftmp9], 36410) "pcmpgth %[Ad], %[ftmp10], %[ftmp5] \n\t" - "or %[mask], %[Ad], %[Gd] \n\t" + "por %[mask], %[Ad], %[Gd] \n\t" "pmullh %[Cd], %[ftmp5], %[mask] \n\t" "pmulhuh %[C], %[ftmp9], %[Cd] \n\t" "pmullh %[C], %[C], %[mask] \n\t" "pcmpgth %[Bd], %[ftmp10], %[ftmp3] \n\t" - "or %[mask], %[Bd], %[Gd] \n\t" + "por %[mask], %[Bd], %[Gd] \n\t" "pmullh %[D], %[ftmp3], %[mask] \n\t" "pmulhuh %[D], %[ftmp8], %[D] \n\t" "pmullh %[D], %[D], %[mask] \n\t" @@ -244,12 +244,12 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) "paddh %[C], %[C], %[Ad] \n\t" "paddh %[C], %[C], %[Bd] \n\t" "pcmpgth %[Bd], %[ftmp10], %[ftmp3] \n\t" - "or %[mask], %[Bd], %[Gd] \n\t" + "por %[mask], %[Bd], %[Gd] \n\t" "pmullh %[Cd], %[ftmp3], %[mask] \n\t" "pmulhuh %[D], %[ftmp9], %[Cd] \n\t" "pmullh %[D], %[D], %[mask] \n\t" "pcmpgth %[Ed], %[ftmp10], %[ftmp5] \n\t" - "or %[mask], %[Ed], %[Gd] \n\t" + "por %[mask], %[Ed], %[Gd] \n\t" "pmullh %[Ad], %[ftmp5], %[mask] \n\t" "pmulhuh %[Ad], %[ftmp8], %[Ad] \n\t" "pmullh %[Ad], %[Ad], %[mask] \n\t" @@ -260,14 +260,14 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[ftmp8], 46341) "psubh %[Ad], %[A], %[C] \n\t" "pcmpgth %[Bd], %[ftmp10], %[Ad] \n\t" - "or %[mask], %[Bd], %[Gd] \n\t" + "por %[mask], %[Bd], %[Gd] \n\t" "pmullh %[Ad], %[Ad], %[mask] \n\t" "pmulhuh %[Ad], %[ftmp8], %[Ad] \n\t" "pmullh %[Ad], %[Ad], %[mask] \n\t" "paddh %[Ad], %[Ad], %[Bd] \n\t" "psubh %[Bd], %[B], %[D] \n\t" "pcmpgth %[Cd], %[ftmp10], %[Bd] \n\t" - "or %[mask], %[Cd], %[Gd] \n\t" + "por %[mask], %[Cd], %[Gd] \n\t" "pmullh %[Bd], %[Bd], %[mask] \n\t" "pmulhuh %[Bd], %[ftmp8], %[Bd] \n\t" "pmullh %[Bd], %[Bd], %[mask] \n\t" @@ -278,7 +278,7 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[Ed], 2056) "paddh %[A], %[ftmp0], %[ftmp4] \n\t" "pcmpgth %[B], %[ftmp10], %[A] \n\t" - "or %[mask], %[B], %[Gd] \n\t" + "por %[mask], %[B], %[Gd] \n\t" "pmullh %[A], %[A], %[mask] \n\t" "pmulhuh %[A], %[ftmp8], %[A] \n\t" "pmullh %[A], %[A], %[mask] \n\t" @@ -286,7 +286,7 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) "paddh %[A], %[A], %[Ed] \n\t" "psubh %[B], %[ftmp0], %[ftmp4] \n\t" "pcmpgth %[C], %[ftmp10], %[B] \n\t" - "or %[mask], %[C], %[Gd] \n\t" + "por %[mask], %[C], %[Gd] \n\t" "pmullh %[B], %[B], %[mask] \n\t" "pmulhuh %[B], %[ftmp8], %[B] \n\t" "pmullh %[B], %[B], %[mask] \n\t" @@ -297,14 +297,14 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[ftmp9], 25080) "pmulhh %[C], %[ftmp9], %[ftmp6] \n\t" "pcmpgth %[D], %[ftmp10], %[ftmp2] \n\t" - "or %[mask], %[D], %[Gd] \n\t" + "por %[mask], %[D], %[Gd] \n\t" "pmullh %[Ed], %[ftmp2], %[mask] \n\t" "pmulhuh %[Ed], %[ftmp8], %[Ed] \n\t" "pmullh %[Ed], %[Ed], %[mask] \n\t" "paddh %[C], %[C], %[Ed] \n\t" "paddh %[C], %[C], %[D] \n\t" "pcmpgth %[Ed], %[ftmp10], %[ftmp6] \n\t" - "or %[mask], %[Ed], %[Gd] \n\t" + "por %[mask], %[Ed], %[Gd] \n\t" "pmullh %[D], %[ftmp6], %[mask] \n\t" "pmulhuh %[D], %[ftmp8], %[D] \n\t" "pmullh %[D], %[D], %[mask] \n\t" @@ -317,12 +317,12 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) "psubh %[C], %[B], %[Ad] \n\t" "psubh %[B], %[Bd], %[D] \n\t" "paddh %[D], %[Bd], %[D] \n\t" - "or %[mask], %[ftmp1], %[ftmp2] \n\t" - "or %[mask], %[mask], %[ftmp3] \n\t" - "or %[mask], %[mask], %[ftmp4] \n\t" - "or %[mask], %[mask], %[ftmp5] \n\t" - "or %[mask], %[mask], %[ftmp6] \n\t" - "or %[mask], %[mask], %[ftmp7] \n\t" + "por %[mask], %[ftmp1], %[ftmp2] \n\t" + "por %[mask], %[mask], %[ftmp3] \n\t" + "por %[mask], %[mask], %[ftmp4] \n\t" + "por %[mask], %[mask], %[ftmp5] \n\t" + "por %[mask], %[mask], %[ftmp6] \n\t" + "por %[mask], %[mask], %[ftmp7] \n\t" "pcmpeqh %[mask], %[mask], %[ftmp10] \n\t" "packushb %[mask], %[mask], %[ftmp10] \n\t" "li %[tmp1], 0x04 \n\t" @@ -361,7 +361,7 @@ static void idct_column_true_mmi(uint8_t *dst, int stride, int16_t *input) "packushb %[ftmp7], %[ftmp7], %[ftmp10] \n\t" "lwc1 %[Ed], 0x00(%[temp_value]) \n\t" - "and %[Ed], %[Ed], %[mask] \n\t" + "pand %[Ed], %[Ed], %[mask] \n\t" "paddb %[ftmp0], %[ftmp0], %[Ed] \n\t" "paddb %[ftmp1], %[ftmp1], %[Ed] \n\t" "paddb %[ftmp2], %[ftmp2], %[Ed] \n\t" @@ -412,7 +412,7 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) for (int i = 0; i < 8; ++i) temp_value[i] = (46341 * input[i << 3] + (8 << 16)) >> 20; __asm__ volatile ( - "xor %[ftmp10], %[ftmp10], %[ftmp10] \n\t" + "pxor %[ftmp10], %[ftmp10], %[ftmp10] \n\t" "li %[tmp0], 0x02 \n\t" "1: \n\t" "ldc1 %[ftmp0], 0x00(%[input]) \n\t" @@ -432,14 +432,14 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[Gd], 1) "pmulhh %[A], %[ftmp9], %[ftmp7] \n\t" "pcmpgth %[C], %[ftmp10], %[ftmp1] \n\t" - "or %[mask], %[C], %[Gd] \n\t" + "por %[mask], %[C], %[Gd] \n\t" "pmullh %[B], %[ftmp1], %[mask] \n\t" "pmulhuh %[B], %[ftmp8], %[B] \n\t" "pmullh %[B], %[B], %[mask] \n\t" "paddh %[A], %[A], %[B] \n\t" "paddh %[A], %[A], %[C] \n\t" "pcmpgth %[D], %[ftmp10], %[ftmp7] \n\t" - "or %[mask], %[D], %[Gd] \n\t" + "por %[mask], %[D], %[Gd] \n\t" "pmullh %[Ad], %[ftmp7], %[mask] \n\t" "pmulhuh %[B], %[ftmp8], %[Ad] \n\t" "pmullh %[B], %[B], %[mask] \n\t" @@ -450,12 +450,12 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[ftmp8], 54491) LOAD_CONST(%[ftmp9], 36410) "pcmpgth %[Ad], %[ftmp10], %[ftmp5] \n\t" - "or %[mask], %[Ad], %[Gd] \n\t" + "por %[mask], %[Ad], %[Gd] \n\t" "pmullh %[Cd], %[ftmp5], %[mask] \n\t" "pmulhuh %[C], %[ftmp9], %[Cd] \n\t" "pmullh %[C], %[C], %[mask] \n\t" "pcmpgth %[Bd], %[ftmp10], %[ftmp3] \n\t" - "or %[mask], %[Bd], %[Gd] \n\t" + "por %[mask], %[Bd], %[Gd] \n\t" "pmullh %[D], %[ftmp3], %[mask] \n\t" "pmulhuh %[D], %[ftmp8], %[D] \n\t" "pmullh %[D], %[D], %[mask] \n\t" @@ -463,12 +463,12 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) "paddh %[C], %[C], %[Ad] \n\t" "paddh %[C], %[C], %[Bd] \n\t" "pcmpgth %[Bd], %[ftmp10], %[ftmp3] \n\t" - "or %[mask], %[Bd], %[Gd] \n\t" + "por %[mask], %[Bd], %[Gd] \n\t" "pmullh %[Cd], %[ftmp3], %[mask] \n\t" "pmulhuh %[D], %[ftmp9], %[Cd] \n\t" "pmullh %[D], %[D], %[mask] \n\t" "pcmpgth %[Ed], %[ftmp10], %[ftmp5] \n\t" - "or %[mask], %[Ed], %[Gd] \n\t" + "por %[mask], %[Ed], %[Gd] \n\t" "pmullh %[Ad], %[ftmp5], %[mask] \n\t" "pmulhuh %[Ad], %[ftmp8], %[Ad] \n\t" "pmullh %[Ad], %[Ad], %[mask] \n\t" @@ -479,14 +479,14 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[ftmp8], 46341) "psubh %[Ad], %[A], %[C] \n\t" "pcmpgth %[Bd], %[ftmp10], %[Ad] \n\t" - "or %[mask], %[Bd], %[Gd] \n\t" + "por %[mask], %[Bd], %[Gd] \n\t" "pmullh %[Ad], %[Ad], %[mask] \n\t" "pmulhuh %[Ad], %[ftmp8], %[Ad] \n\t" "pmullh %[Ad], %[Ad], %[mask] \n\t" "paddh %[Ad], %[Ad], %[Bd] \n\t" "psubh %[Bd], %[B], %[D] \n\t" "pcmpgth %[Cd], %[ftmp10], %[Bd] \n\t" - "or %[mask], %[Cd], %[Gd] \n\t" + "por %[mask], %[Cd], %[Gd] \n\t" "pmullh %[Bd], %[Bd], %[mask] \n\t" "pmulhuh %[Bd], %[ftmp8], %[Bd] \n\t" "pmullh %[Bd], %[Bd], %[mask] \n\t" @@ -497,7 +497,7 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[Ed], 8) "paddh %[A], %[ftmp0], %[ftmp4] \n\t" "pcmpgth %[B], %[ftmp10], %[A] \n\t" - "or %[mask], %[B], %[Gd] \n\t" + "por %[mask], %[B], %[Gd] \n\t" "pmullh %[A], %[A], %[mask] \n\t" "pmulhuh %[A], %[ftmp8], %[A] \n\t" "pmullh %[A], %[A], %[mask] \n\t" @@ -505,7 +505,7 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) "paddh %[A], %[A], %[Ed] \n\t" "psubh %[B], %[ftmp0], %[ftmp4] \n\t" "pcmpgth %[C], %[ftmp10], %[B] \n\t" - "or %[mask], %[C], %[Gd] \n\t" + "por %[mask], %[C], %[Gd] \n\t" "pmullh %[B], %[B], %[mask] \n\t" "pmulhuh %[B], %[ftmp8], %[B] \n\t" "pmullh %[B], %[B], %[mask] \n\t" @@ -516,14 +516,14 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) LOAD_CONST(%[ftmp9], 25080) "pmulhh %[C], %[ftmp9], %[ftmp6] \n\t" "pcmpgth %[D], %[ftmp10], %[ftmp2] \n\t" - "or %[mask], %[D], %[Gd] \n\t" + "por %[mask], %[D], %[Gd] \n\t" "pmullh %[Ed], %[ftmp2], %[mask] \n\t" "pmulhuh %[Ed], %[ftmp8], %[Ed] \n\t" "pmullh %[Ed], %[Ed], %[mask] \n\t" "paddh %[C], %[C], %[Ed] \n\t" "paddh %[C], %[C], %[D] \n\t" "pcmpgth %[Ed], %[ftmp10], %[ftmp6] \n\t" - "or %[mask], %[Ed], %[Gd] \n\t" + "por %[mask], %[Ed], %[Gd] \n\t" "pmullh %[D], %[ftmp6], %[mask] \n\t" "pmulhuh %[D], %[ftmp8], %[D] \n\t" "pmullh %[D], %[D], %[mask] \n\t" @@ -536,12 +536,12 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) "psubh %[C], %[B], %[Ad] \n\t" "psubh %[B], %[Bd], %[D] \n\t" "paddh %[D], %[Bd], %[D] \n\t" - "or %[mask], %[ftmp1], %[ftmp2] \n\t" - "or %[mask], %[mask], %[ftmp3] \n\t" - "or %[mask], %[mask], %[ftmp4] \n\t" - "or %[mask], %[mask], %[ftmp5] \n\t" - "or %[mask], %[mask], %[ftmp6] \n\t" - "or %[mask], %[mask], %[ftmp7] \n\t" + "por %[mask], %[ftmp1], %[ftmp2] \n\t" + "por %[mask], %[mask], %[ftmp3] \n\t" + "por %[mask], %[mask], %[ftmp4] \n\t" + "por %[mask], %[mask], %[ftmp5] \n\t" + "por %[mask], %[mask], %[ftmp6] \n\t" + "por %[mask], %[mask], %[ftmp7] \n\t" "pcmpeqh %[mask], %[mask], %[ftmp10] \n\t" "li %[tmp1], 0x04 \n\t" "dmtc1 %[tmp1], %[ftmp8] \n\t" @@ -587,16 +587,16 @@ static void idct_column_false_mmi(uint8_t *dst, int stride, int16_t *input) "punpcklbh %[Cd], %[Cd], %[ftmp10] \n\t" "punpcklbh %[Dd], %[Dd], %[ftmp10] \n\t" "ldc1 %[Ed], 0x00(%[temp_value]) \n\t" - "and %[Ed], %[Ed], %[mask] \n\t" - "nor %[mask], %[mask], %[mask] \n\t" - "and %[ftmp0], %[ftmp0], %[mask] \n\t" - "and %[ftmp1], %[ftmp1], %[mask] \n\t" - "and %[ftmp2], %[ftmp2], %[mask] \n\t" - "and %[ftmp3], %[ftmp3], %[mask] \n\t" - "and %[ftmp4], %[ftmp4], %[mask] \n\t" - "and %[ftmp5], %[ftmp5], %[mask] \n\t" - "and %[ftmp6], %[ftmp6], %[mask] \n\t" - "and %[ftmp7], %[ftmp7], %[mask] \n\t" + "pand %[Ed], %[Ed], %[mask] \n\t" + "pnor %[mask], %[mask], %[mask] \n\t" + "pand %[ftmp0], %[ftmp0], %[mask] \n\t" + "pand %[ftmp1], %[ftmp1], %[mask] \n\t" + "pand %[ftmp2], %[ftmp2], %[mask] \n\t" + "pand %[ftmp3], %[ftmp3], %[mask] \n\t" + "pand %[ftmp4], %[ftmp4], %[mask] \n\t" + "pand %[ftmp5], %[ftmp5], %[mask] \n\t" + "pand %[ftmp6], %[ftmp6], %[mask] \n\t" + "pand %[ftmp7], %[ftmp7], %[mask] \n\t" "paddh %[ftmp0], %[ftmp0], %[A] \n\t" "paddh %[ftmp1], %[ftmp1], %[B] \n\t" "paddh %[ftmp2], %[ftmp2], %[C] \n\t" @@ -689,7 +689,7 @@ void ff_vp3_idct_dc_add_mmi(uint8_t *dest, ptrdiff_t line_size, int16_t *block) double ftmp[7]; uint64_t tmp; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "mtc1 %[dc], %[ftmp5] \n\t" "pshufh %[ftmp5], %[ftmp5], %[ftmp0] \n\t" "li %[tmp0], 0x08 \n\t" @@ -734,10 +734,10 @@ void ff_put_no_rnd_pixels_l2_mmi(uint8_t *dst, const uint8_t *src1, "gsldrc1 %[ftmp1], 0x00(%[src1]) \n\t" "gsldlc1 %[ftmp2], 0x07(%[src2]) \n\t" "gsldrc1 %[ftmp2], 0x00(%[src2]) \n\t" - "xor %[ftmp3], %[ftmp1], %[ftmp2] \n\t" - "and %[ftmp3], %[ftmp3], %[ftmp4] \n\t" + "pxor %[ftmp3], %[ftmp1], %[ftmp2] \n\t" + "pand %[ftmp3], %[ftmp3], %[ftmp4] \n\t" "psrlw %[ftmp3], %[ftmp3], %[ftmp5] \n\t" - "and %[ftmp6], %[ftmp1], %[ftmp2] \n\t" + "pand %[ftmp6], %[ftmp1], %[ftmp2] \n\t" "paddw %[ftmp3], %[ftmp3], %[ftmp6] \n\t" "sdc1 %[ftmp3], 0x00(%[dst]) \n\t" PTR_ADDU "%[src1], %[src1], %[stride] \n\t" diff --git a/libavcodec/mips/vp8dsp_mmi.c b/libavcodec/mips/vp8dsp_mmi.c index bd80aa1445..20632c68c7 100644 --- a/libavcodec/mips/vp8dsp_mmi.c +++ b/libavcodec/mips/vp8dsp_mmi.c @@ -36,10 +36,10 @@ "pcmpeqb %[db_1], "#src1", "#src2" \n\t" \ "pmaxub %[db_2], "#src1", "#src2" \n\t" \ "pcmpeqb %[db_2], %[db_2], "#src1" \n\t" \ - "xor "#dst", %[db_2], %[db_1] \n\t" + "pxor "#dst", %[db_2], %[db_1] \n\t" #define MMI_BTOH(dst_l, dst_r, src) \ - "xor %[db_1], %[db_1], %[db_1] \n\t" \ + "pxor %[db_1], %[db_1], %[db_1] \n\t" \ "pcmpgtb %[db_2], %[db_1], "#src" \n\t" \ "punpcklbh "#dst_r", "#src", %[db_2] \n\t" \ "punpckhbh "#dst_l", "#src", %[db_2] \n\t" @@ -82,17 +82,17 @@ "punpcklwd %[ftmp3], %[ftmp3], %[ftmp3] \n\t" \ MMI_PCMPGTUB(%[mask], %[mask], %[ftmp3]) \ "pcmpeqw %[ftmp3], %[ftmp3], %[ftmp3] \n\t" \ - "xor %[mask], %[mask], %[ftmp3] \n\t" \ + "pxor %[mask], %[mask], %[ftmp3] \n\t" \ /* VP8_MBFILTER */ \ "li %[tmp0], 0x80808080 \n\t" \ "dmtc1 %[tmp0], %[ftmp7] \n\t" \ "punpcklwd %[ftmp7], %[ftmp7], %[ftmp7] \n\t" \ - "xor %[p2], %[p2], %[ftmp7] \n\t" \ - "xor %[p1], %[p1], %[ftmp7] \n\t" \ - "xor %[p0], %[p0], %[ftmp7] \n\t" \ - "xor %[q0], %[q0], %[ftmp7] \n\t" \ - "xor %[q1], %[q1], %[ftmp7] \n\t" \ - "xor %[q2], %[q2], %[ftmp7] \n\t" \ + "pxor %[p2], %[p2], %[ftmp7] \n\t" \ + "pxor %[p1], %[p1], %[ftmp7] \n\t" \ + "pxor %[p0], %[p0], %[ftmp7] \n\t" \ + "pxor %[q0], %[q0], %[ftmp7] \n\t" \ + "pxor %[q1], %[q1], %[ftmp7] \n\t" \ + "pxor %[q2], %[q2], %[ftmp7] \n\t" \ "psubsb %[ftmp4], %[p1], %[q1] \n\t" \ "psubb %[ftmp5], %[q0], %[p0] \n\t" \ MMI_BTOH(%[ftmp1], %[ftmp0], %[ftmp5]) \ @@ -107,8 +107,8 @@ "paddh %[ftmp1], %[ftmp3], %[ftmp1] \n\t" \ /* Combine left and right part */ \ "packsshb %[ftmp1], %[ftmp0], %[ftmp1] \n\t" \ - "and %[ftmp1], %[ftmp1], %[mask] \n\t" \ - "and %[ftmp2], %[ftmp1], %[hev] \n\t" \ + "pand %[ftmp1], %[ftmp1], %[mask] \n\t" \ + "pand %[ftmp2], %[ftmp1], %[hev] \n\t" \ "li %[tmp0], 0x04040404 \n\t" \ "dmtc1 %[tmp0], %[ftmp0] \n\t" \ "punpcklwd %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ @@ -127,8 +127,8 @@ "paddsb %[p0], %[p0], %[ftmp4] \n\t" \ /* filt_val &= ~hev */ \ "pcmpeqw %[ftmp0], %[ftmp0], %[ftmp0] \n\t" \ - "xor %[hev], %[hev], %[ftmp0] \n\t" \ - "and %[ftmp1], %[ftmp1], %[hev] \n\t" \ + "pxor %[hev], %[hev], %[ftmp0] \n\t" \ + "pand %[ftmp1], %[ftmp1], %[hev] \n\t" \ MMI_BTOH(%[ftmp5], %[ftmp6], %[ftmp1]) \ "li %[tmp0], 0x07 \n\t" \ "dmtc1 %[tmp0], %[ftmp2] \n\t" \ @@ -149,9 +149,9 @@ /* Combine left and right part */ \ "packsshb %[ftmp4], %[ftmp3], %[ftmp4] \n\t" \ "psubsb %[q0], %[q0], %[ftmp4] \n\t" \ - "xor %[q0], %[q0], %[ftmp7] \n\t" \ + "pxor %[q0], %[q0], %[ftmp7] \n\t" \ "paddsb %[p0], %[p0], %[ftmp4] \n\t" \ - "xor %[p0], %[p0], %[ftmp7] \n\t" \ + "pxor %[p0], %[p0], %[ftmp7] \n\t" \ "li %[tmp0], 0x00120012 \n\t" \ "dmtc1 %[tmp0], %[ftmp1] \n\t" \ "punpcklwd %[ftmp1], %[ftmp1], %[ftmp1] \n\t" \ @@ -166,9 +166,9 @@ /* Combine left and right part */ \ "packsshb %[ftmp4], %[ftmp3], %[ftmp4] \n\t" \ "psubsb %[q1], %[q1], %[ftmp4] \n\t" \ - "xor %[q1], %[q1], %[ftmp7] \n\t" \ + "pxor %[q1], %[q1], %[ftmp7] \n\t" \ "paddsb %[p1], %[p1], %[ftmp4] \n\t" \ - "xor %[p1], %[p1], %[ftmp7] \n\t" \ + "pxor %[p1], %[p1], %[ftmp7] \n\t" \ "li %[tmp0], 0x03 \n\t" \ "dmtc1 %[tmp0], %[ftmp1] \n\t" \ /* Right part */ \ @@ -184,9 +184,9 @@ /* Combine left and right part */ \ "packsshb %[ftmp4], %[ftmp3], %[ftmp4] \n\t" \ "psubsb %[q2], %[q2], %[ftmp4] \n\t" \ - "xor %[q2], %[q2], %[ftmp7] \n\t" \ + "pxor %[q2], %[q2], %[ftmp7] \n\t" \ "paddsb %[p2], %[p2], %[ftmp4] \n\t" \ - "xor %[p2], %[p2], %[ftmp7] \n\t" + "pxor %[p2], %[p2], %[ftmp7] \n\t" #define PUT_VP8_EPEL4_H6_MMI(src, dst) \ MMI_ULWC1(%[ftmp1], src, 0x00) \ @@ -1019,7 +1019,7 @@ void ff_vp8_luma_dc_wht_mmi(int16_t block[4][4][16], int16_t dc[16]) block[3][3][0] = (dc[12] - dc[15] + 3 - dc[13] + dc[14]) >> 3; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" MMI_SDC1(%[ftmp0], %[dc], 0x00) MMI_SDC1(%[ftmp0], %[dc], 0x08) MMI_SDC1(%[ftmp0], %[dc], 0x10) @@ -1126,15 +1126,17 @@ void ff_vp8_luma_dc_wht_dc_mmi(int16_t block[4][4][16], int16_t dc[16]) void ff_vp8_idct_add_mmi(uint8_t *dst, int16_t block[16], ptrdiff_t stride) { #if 1 - DECLARE_ALIGNED(8, const uint64_t, ff_ph_4e7b) = {0x4e7b4e7b4e7b4e7bULL}; - DECLARE_ALIGNED(8, const uint64_t, ff_ph_22a3) = {0x22a322a322a322a3ULL}; double ftmp[12]; uint32_t tmp[1]; + union av_intfloat64 ff_ph_4e7b_u; + union av_intfloat64 ff_ph_22a3_u; DECLARE_VAR_LOW32; DECLARE_VAR_ALL64; + ff_ph_4e7b_u.i = 0x4e7b4e7b4e7b4e7bULL; + ff_ph_22a3_u.i = 0x22a322a322a322a3ULL; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" MMI_LDC1(%[ftmp1], %[block], 0x00) MMI_LDC1(%[ftmp2], %[block], 0x08) MMI_LDC1(%[ftmp3], %[block], 0x10) @@ -1251,8 +1253,8 @@ void ff_vp8_idct_add_mmi(uint8_t *dst, int16_t block[16], ptrdiff_t stride) [tmp0]"=&r"(tmp[0]) : [dst0]"r"(dst), [dst1]"r"(dst+stride), [dst2]"r"(dst+2*stride), [dst3]"r"(dst+3*stride), - [block]"r"(block), [ff_pw_4]"f"(ff_pw_4), - [ff_ph_4e7b]"f"(ff_ph_4e7b), [ff_ph_22a3]"f"(ff_ph_22a3) + [block]"r"(block), [ff_pw_4]"f"(ff_pw_4.f), + [ff_ph_4e7b]"f"(ff_ph_4e7b_u.f), [ff_ph_22a3]"f"(ff_ph_22a3_u.f) : "memory" ); #else @@ -1300,7 +1302,7 @@ void ff_vp8_idct_dc_add_mmi(uint8_t *dst, int16_t block[16], ptrdiff_t stride) block[0] = 0; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "mtc1 %[dc], %[ftmp5] \n\t" MMI_LWC1(%[ftmp1], %[dst0], 0x00) MMI_LWC1(%[ftmp2], %[dst1], 0x00) @@ -1593,8 +1595,16 @@ void ff_put_vp8_epel16_h4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, const uint64_t *filter = fourtap_subpel_filters[mx - 1]; double ftmp[9]; uint32_t tmp[1]; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; mips_reg src1, dst1; DECLARE_VAR_ALL64; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; /* dst[0] = cm[(filter[2] * src[0] - filter[1] * src[-1] + filter[3] * src[1] - filter[4] * src[2] + 64) >> 7]; @@ -1616,7 +1626,7 @@ void ff_put_vp8_epel16_h4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[15] = cm[(filter[2] * src[15] - filter[1] * src[14] + filter[3] * src[16] - filter[4] * src[17] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -1642,11 +1652,11 @@ void ff_put_vp8_epel16_h4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, [dst1]"=&r"(dst1), [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter1]"f"(filter[1]), [filter2]"f"(filter[2]), - [filter3]"f"(filter[3]), [filter4]"f"(filter[4]) + [filter1]"f"(filter1.f), [filter2]"f"(filter2.f), + [filter3]"f"(filter3.f), [filter4]"f"(filter4.f) : "memory" ); #else @@ -1670,7 +1680,16 @@ void ff_put_vp8_epel8_h4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, const uint64_t *filter = fourtap_subpel_filters[mx - 1]; double ftmp[9]; uint32_t tmp[1]; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; DECLARE_VAR_ALL64; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; + /* dst[0] = cm[(filter[2] * src[0] - filter[1] * src[-1] + filter[3] * src[1] - filter[4] * src[2] + 64) >> 7]; @@ -1683,7 +1702,7 @@ void ff_put_vp8_epel8_h4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[7] = cm[(filter[2] * src[7] - filter[1] * src[ 6] + filter[3] * src[8] - filter[4] * src[9] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -1703,11 +1722,11 @@ void ff_put_vp8_epel8_h4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, RESTRICT_ASM_ALL64 [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter1]"f"(filter[1]), [filter2]"f"(filter[2]), - [filter3]"f"(filter[3]), [filter4]"f"(filter[4]) + [filter1]"f"(filter1.f), [filter2]"f"(filter2.f), + [filter3]"f"(filter3.f), [filter4]"f"(filter4.f) : "memory" ); #else @@ -1731,7 +1750,15 @@ void ff_put_vp8_epel4_h4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, const uint64_t *filter = fourtap_subpel_filters[mx - 1]; double ftmp[6]; uint32_t tmp[1]; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; DECLARE_VAR_LOW32; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; /* dst[0] = cm[(filter[2] * src[0] - filter[1] * src[-1] + filter[3] * src[1] - filter[4] * src[2] + 64) >> 7]; @@ -1740,7 +1767,7 @@ void ff_put_vp8_epel4_h4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[3] = cm[(filter[2] * src[3] - filter[1] * src[ 2] + filter[3] * src[4] - filter[4] * src[5] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -1758,11 +1785,11 @@ void ff_put_vp8_epel4_h4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, RESTRICT_ASM_LOW32 [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter1]"f"(filter[1]), [filter2]"f"(filter[2]), - [filter3]"f"(filter[3]), [filter4]"f"(filter[4]) + [filter1]"f"(filter1.f), [filter2]"f"(filter2.f), + [filter3]"f"(filter3.f), [filter4]"f"(filter4.f) : "memory" ); #else @@ -1787,7 +1814,19 @@ void ff_put_vp8_epel16_h6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, double ftmp[9]; uint32_t tmp[1]; mips_reg src1, dst1; + union av_intfloat64 filter0; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; + union av_intfloat64 filter5; DECLARE_VAR_ALL64; + filter0.i = filter[0]; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; + filter5.i = filter[5]; /* dst[ 0] = cm[(filter[2]*src[ 0] - filter[1]*src[-1] + filter[0]*src[-2] + filter[3]*src[ 1] - filter[4]*src[ 2] + filter[5]*src[ 3] + 64) >> 7]; @@ -1809,7 +1848,7 @@ void ff_put_vp8_epel16_h6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[15] = cm[(filter[2]*src[15] - filter[1]*src[14] + filter[0]*src[13] + filter[3]*src[16] - filter[4]*src[17] + filter[5]*src[18] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -1835,12 +1874,12 @@ void ff_put_vp8_epel16_h6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, [dst1]"=&r"(dst1), [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter0]"f"(filter[0]), [filter1]"f"(filter[1]), - [filter2]"f"(filter[2]), [filter3]"f"(filter[3]), - [filter4]"f"(filter[4]), [filter5]"f"(filter[5]) + [filter0]"f"(filter0.f), [filter1]"f"(filter1.f), + [filter2]"f"(filter2.f), [filter3]"f"(filter3.f), + [filter4]"f"(filter4.f), [filter5]"f"(filter5.f) : "memory" ); #else @@ -1864,7 +1903,19 @@ void ff_put_vp8_epel8_h6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, const uint64_t *filter = fourtap_subpel_filters[mx - 1]; double ftmp[9]; uint32_t tmp[1]; + union av_intfloat64 filter0; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; + union av_intfloat64 filter5; DECLARE_VAR_ALL64; + filter0.i = filter[0]; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; + filter5.i = filter[5]; /* dst[0] = cm[(filter[2]*src[0] - filter[1]*src[-1] + filter[0]*src[-2] + filter[3]*src[1] - filter[4]*src[2] + filter[5]*src[ 3] + 64) >> 7]; @@ -1877,7 +1928,7 @@ void ff_put_vp8_epel8_h6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[7] = cm[(filter[2]*src[7] - filter[1]*src[ 6] + filter[0]*src[ 5] + filter[3]*src[8] - filter[4]*src[9] + filter[5]*src[10] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -1897,12 +1948,12 @@ void ff_put_vp8_epel8_h6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, RESTRICT_ASM_ALL64 [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter0]"f"(filter[0]), [filter1]"f"(filter[1]), - [filter2]"f"(filter[2]), [filter3]"f"(filter[3]), - [filter4]"f"(filter[4]), [filter5]"f"(filter[5]) + [filter0]"f"(filter0.f), [filter1]"f"(filter1.f), + [filter2]"f"(filter2.f), [filter3]"f"(filter3.f), + [filter4]"f"(filter4.f), [filter5]"f"(filter5.f) : "memory" ); #else @@ -1926,7 +1977,19 @@ void ff_put_vp8_epel4_h6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, const uint64_t *filter = fourtap_subpel_filters[mx - 1]; double ftmp[6]; uint32_t tmp[1]; + union av_intfloat64 filter0; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; + union av_intfloat64 filter5; DECLARE_VAR_LOW32; + filter0.i = filter[0]; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; + filter5.i = filter[5]; /* dst[0] = cm[(filter[2]*src[0] - filter[1]*src[-1] + filter[0]*src[-2] + filter[3]*src[1] - filter[4]*src[2] + filter[5]*src[ 3] + 64) >> 7]; @@ -1935,7 +1998,7 @@ void ff_put_vp8_epel4_h6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[3] = cm[(filter[2]*src[3] - filter[1]*src[ 2] + filter[0]*src[ 1] + filter[3]*src[4] - filter[4]*src[5] + filter[5]*src[ 6] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -1953,12 +2016,12 @@ void ff_put_vp8_epel4_h6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, RESTRICT_ASM_LOW32 [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter0]"f"(filter[0]), [filter1]"f"(filter[1]), - [filter2]"f"(filter[2]), [filter3]"f"(filter[3]), - [filter4]"f"(filter[4]), [filter5]"f"(filter[5]) + [filter0]"f"(filter0.f), [filter1]"f"(filter1.f), + [filter2]"f"(filter2.f), [filter3]"f"(filter3.f), + [filter4]"f"(filter4.f), [filter5]"f"(filter5.f) : "memory" ); #else @@ -1983,7 +2046,15 @@ void ff_put_vp8_epel16_v4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, double ftmp[9]; uint32_t tmp[1]; mips_reg src0, src1, dst0; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; DECLARE_VAR_ALL64; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; /* dst[0] = cm[(filter[2] * src[0] - filter[1] * src[ -srcstride] + filter[3] * src[ srcstride] - filter[4] * src[ 2*srcstride] + 64) >> 7]; @@ -2005,7 +2076,7 @@ void ff_put_vp8_epel16_v4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[15] = cm[(filter[2] * src[15] - filter[1] * src[15-srcstride] + filter[3] * src[15+srcstride] - filter[4] * src[15+2*srcstride] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -2032,11 +2103,11 @@ void ff_put_vp8_epel16_v4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter1]"f"(filter[1]), [filter2]"f"(filter[2]), - [filter3]"f"(filter[3]), [filter4]"f"(filter[4]) + [filter1]"f"(filter1.f), [filter2]"f"(filter2.f), + [filter3]"f"(filter3.f), [filter4]"f"(filter4.f) : "memory" ); #else @@ -2061,7 +2132,15 @@ void ff_put_vp8_epel8_v4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, double ftmp[9]; uint32_t tmp[1]; mips_reg src1; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; DECLARE_VAR_ALL64; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; /* dst[0] = cm[(filter[2] * src[0] - filter[1] * src[ -srcstride] + filter[3] * src[ srcstride] - filter[4] * src[ 2*srcstride] + 64) >> 7]; @@ -2074,7 +2153,7 @@ void ff_put_vp8_epel8_v4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[7] = cm[(filter[2] * src[7] - filter[1] * src[7-srcstride] + filter[3] * src[7+srcstride] - filter[4] * src[7+2*srcstride] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -2095,11 +2174,11 @@ void ff_put_vp8_epel8_v4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter1]"f"(filter[1]), [filter2]"f"(filter[2]), - [filter3]"f"(filter[3]), [filter4]"f"(filter[4]) + [filter1]"f"(filter1.f), [filter2]"f"(filter2.f), + [filter3]"f"(filter3.f), [filter4]"f"(filter4.f) : "memory" ); #else @@ -2124,7 +2203,15 @@ void ff_put_vp8_epel4_v4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, double ftmp[6]; uint32_t tmp[1]; mips_reg src1; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; DECLARE_VAR_LOW32; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; /* dst[0] = cm[(filter[2] * src[0] - filter[1] * src[ -srcstride] + filter[3] * src[ srcstride] - filter[4] * src[ 2*srcstride] + 64) >> 7]; @@ -2133,7 +2220,7 @@ void ff_put_vp8_epel4_v4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[3] = cm[(filter[2] * src[3] - filter[1] * src[3-srcstride] + filter[3] * src[3+srcstride] - filter[4] * src[3+2*srcstride] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -2152,11 +2239,11 @@ void ff_put_vp8_epel4_v4_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter1]"f"(filter[1]), [filter2]"f"(filter[2]), - [filter3]"f"(filter[3]), [filter4]"f"(filter[4]) + [filter1]"f"(filter1.f), [filter2]"f"(filter2.f), + [filter3]"f"(filter3.f), [filter4]"f"(filter4.f) : "memory" ); #else @@ -2181,7 +2268,19 @@ void ff_put_vp8_epel16_v6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, double ftmp[9]; uint32_t tmp[1]; mips_reg src0, src1, dst0; + union av_intfloat64 filter0; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; + union av_intfloat64 filter5; DECLARE_VAR_ALL64; + filter0.i = filter[0]; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; + filter5.i = filter[5]; /* dst[0] = cm[(filter[2]*src[0] - filter[1]*src[0-srcstride] + filter[0]*src[0-2*srcstride] + filter[3]*src[0+srcstride] - filter[4]*src[0+2*srcstride] + filter[5]*src[0+3*srcstride] + 64) >> 7]; @@ -2203,7 +2302,7 @@ void ff_put_vp8_epel16_v6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[15] = cm[(filter[2]*src[15] - filter[1]*src[15-srcstride] + filter[0]*src[15-2*srcstride] + filter[3]*src[15+srcstride] - filter[4]*src[15+2*srcstride] + filter[5]*src[15+3*srcstride] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -2230,12 +2329,12 @@ void ff_put_vp8_epel16_v6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter0]"f"(filter[0]), [filter1]"f"(filter[1]), - [filter2]"f"(filter[2]), [filter3]"f"(filter[3]), - [filter4]"f"(filter[4]), [filter5]"f"(filter[5]) + [filter0]"f"(filter0.f), [filter1]"f"(filter1.f), + [filter2]"f"(filter2.f), [filter3]"f"(filter3.f), + [filter4]"f"(filter4.f), [filter5]"f"(filter5.f) : "memory" ); #else @@ -2260,7 +2359,19 @@ void ff_put_vp8_epel8_v6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, double ftmp[9]; uint32_t tmp[1]; mips_reg src1; + union av_intfloat64 filter0; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; + union av_intfloat64 filter5; DECLARE_VAR_ALL64; + filter0.i = filter[0]; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; + filter5.i = filter[5]; /* dst[0] = cm[(filter[2]*src[0] - filter[1]*src[0-srcstride] + filter[0]*src[0-2*srcstride] + filter[3]*src[0+srcstride] - filter[4]*src[0+2*srcstride] + filter[5]*src[0+3*srcstride] + 64) >> 7]; @@ -2273,7 +2384,7 @@ void ff_put_vp8_epel8_v6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[7] = cm[(filter[2]*src[7] - filter[1]*src[7-srcstride] + filter[0]*src[7-2*srcstride] + filter[3]*src[7+srcstride] - filter[4]*src[7+2*srcstride] + filter[5]*src[7+3*srcstride] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -2294,12 +2405,12 @@ void ff_put_vp8_epel8_v6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter0]"f"(filter[0]), [filter1]"f"(filter[1]), - [filter2]"f"(filter[2]), [filter3]"f"(filter[3]), - [filter4]"f"(filter[4]), [filter5]"f"(filter[5]) + [filter0]"f"(filter0.f), [filter1]"f"(filter1.f), + [filter2]"f"(filter2.f), [filter3]"f"(filter3.f), + [filter4]"f"(filter4.f), [filter5]"f"(filter5.f) : "memory" ); #else @@ -2324,7 +2435,19 @@ void ff_put_vp8_epel4_v6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, double ftmp[6]; uint32_t tmp[1]; mips_reg src1; + union av_intfloat64 filter0; + union av_intfloat64 filter1; + union av_intfloat64 filter2; + union av_intfloat64 filter3; + union av_intfloat64 filter4; + union av_intfloat64 filter5; DECLARE_VAR_LOW32; + filter0.i = filter[0]; + filter1.i = filter[1]; + filter2.i = filter[2]; + filter3.i = filter[3]; + filter4.i = filter[4]; + filter5.i = filter[5]; /* dst[0] = cm[(filter[2]*src[0] - filter[1]*src[0-srcstride] + filter[0]*src[0-2*srcstride] + filter[3]*src[0+srcstride] - filter[4]*src[0+2*srcstride] + filter[5]*src[0+3*srcstride] + 64) >> 7]; @@ -2333,7 +2456,7 @@ void ff_put_vp8_epel4_v6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, dst[3] = cm[(filter[2]*src[3] - filter[1]*src[3-srcstride] + filter[0]*src[3-2*srcstride] + filter[3]*src[3+srcstride] - filter[4]*src[3+2*srcstride] + filter[5]*src[3+3*srcstride] + 64) >> 7]; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x07 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" @@ -2352,12 +2475,12 @@ void ff_put_vp8_epel4_v6_mmi(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src) - : [ff_pw_64]"f"(ff_pw_64), + : [ff_pw_64]"f"(ff_pw_64.f), [srcstride]"r"((mips_reg)srcstride), [dststride]"r"((mips_reg)dststride), - [filter0]"f"(filter[0]), [filter1]"f"(filter[1]), - [filter2]"f"(filter[2]), [filter3]"f"(filter[3]), - [filter4]"f"(filter[4]), [filter5]"f"(filter[5]) + [filter0]"f"(filter0.f), [filter1]"f"(filter1.f), + [filter2]"f"(filter2.f), [filter3]"f"(filter3.f), + [filter4]"f"(filter4.f), [filter5]"f"(filter5.f) : "memory" ); #else @@ -2845,11 +2968,13 @@ void ff_put_vp8_bilinear16_h_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, ptrdiff_t sstride, int h, int mx, int my) { #if 1 - int a = 8 - mx, b = mx; + union mmi_intfloat64 a, b; double ftmp[7]; uint32_t tmp[1]; mips_reg dst0, src0; DECLARE_VAR_ALL64; + a.i = 8 - mx; + b.i = mx; /* dst[0] = (a * src[0] + b * src[1] + 4) >> 3; @@ -2871,7 +2996,7 @@ void ff_put_vp8_bilinear16_h_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, dst[15] = (a * src[15] + b * src[16] + 4) >> 3; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x03 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" "pshufh %[a], %[a], %[ftmp0] \n\t" @@ -2898,10 +3023,10 @@ void ff_put_vp8_bilinear16_h_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, [dst0]"=&r"(dst0), [src0]"=&r"(src0), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src), - [a]"+&f"(a), [b]"+&f"(b) + [a]"+&f"(a.f), [b]"+&f"(b.f) : [sstride]"r"((mips_reg)sstride), [dstride]"r"((mips_reg)dstride), - [ff_pw_4]"f"(ff_pw_4) + [ff_pw_4]"f"(ff_pw_4.f) : "memory" ); #else @@ -2921,11 +3046,13 @@ void ff_put_vp8_bilinear16_v_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, ptrdiff_t sstride, int h, int mx, int my) { #if 1 - int c = 8 - my, d = my; + union mmi_intfloat64 c, d; double ftmp[7]; uint32_t tmp[1]; mips_reg src0, src1, dst0; DECLARE_VAR_ALL64; + c.i = 8 - my; + d.i = my; /* dst[0] = (c * src[0] + d * src[ sstride] + 4) >> 3; @@ -2938,7 +3065,7 @@ void ff_put_vp8_bilinear16_v_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, dst[7] = (c * src[7] + d * src[7 + sstride] + 4) >> 3; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x03 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" "pshufh %[c], %[c], %[ftmp0] \n\t" @@ -2966,10 +3093,10 @@ void ff_put_vp8_bilinear16_v_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src), - [c]"+&f"(c), [d]"+&f"(d) + [c]"+&f"(c.f), [d]"+&f"(d.f) : [sstride]"r"((mips_reg)sstride), [dstride]"r"((mips_reg)dstride), - [ff_pw_4]"f"(ff_pw_4) + [ff_pw_4]"f"(ff_pw_4.f) : "memory" ); #else @@ -3023,10 +3150,12 @@ void ff_put_vp8_bilinear8_h_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, ptrdiff_t sstride, int h, int mx, int my) { #if 1 - int a = 8 - mx, b = mx; + union mmi_intfloat64 a, b; double ftmp[7]; uint32_t tmp[1]; DECLARE_VAR_ALL64; + a.i = 8 - mx; + b.i = mx; /* dst[0] = (a * src[0] + b * src[1] + 4) >> 3; @@ -3039,7 +3168,7 @@ void ff_put_vp8_bilinear8_h_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, dst[7] = (a * src[7] + b * src[8] + 4) >> 3; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x03 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" "pshufh %[a], %[a], %[ftmp0] \n\t" @@ -3060,10 +3189,10 @@ void ff_put_vp8_bilinear8_h_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, RESTRICT_ASM_ALL64 [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src), - [a]"+&f"(a), [b]"+&f"(b) + [a]"+&f"(a.f), [b]"+&f"(b.f) : [sstride]"r"((mips_reg)sstride), [dstride]"r"((mips_reg)dstride), - [ff_pw_4]"f"(ff_pw_4) + [ff_pw_4]"f"(ff_pw_4.f) : "memory" ); #else @@ -3083,11 +3212,13 @@ void ff_put_vp8_bilinear8_v_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, ptrdiff_t sstride, int h, int mx, int my) { #if 1 - int c = 8 - my, d = my; + union mmi_intfloat64 c, d; double ftmp[7]; uint32_t tmp[1]; mips_reg src1; DECLARE_VAR_ALL64; + c.i = 8 - my; + d.i = my; /* dst[0] = (c * src[0] + d * src[ sstride] + 4) >> 3; @@ -3100,7 +3231,7 @@ void ff_put_vp8_bilinear8_v_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, dst[7] = (c * src[7] + d * src[7 + sstride] + 4) >> 3; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x03 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" "pshufh %[c], %[c], %[ftmp0] \n\t" @@ -3122,10 +3253,10 @@ void ff_put_vp8_bilinear8_v_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src), - [c]"+&f"(c), [d]"+&f"(d) + [c]"+&f"(c.f), [d]"+&f"(d.f) : [sstride]"r"((mips_reg)sstride), [dstride]"r"((mips_reg)dstride), - [ff_pw_4]"f"(ff_pw_4) + [ff_pw_4]"f"(ff_pw_4.f) : "memory" ); #else @@ -3179,11 +3310,13 @@ void ff_put_vp8_bilinear4_h_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, ptrdiff_t sstride, int h, int mx, int my) { #if 1 - int a = 8 - mx, b = mx; + union mmi_intfloat64 a, b; double ftmp[5]; uint32_t tmp[1]; DECLARE_VAR_LOW32; DECLARE_VAR_ALL64; + a.i = 8 - mx; + b.i = mx; /* dst[0] = (a * src[0] + b * src[1] + 4) >> 3; @@ -3192,7 +3325,7 @@ void ff_put_vp8_bilinear4_h_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, dst[3] = (a * src[3] + b * src[4] + 4) >> 3; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x03 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" "pshufh %[a], %[a], %[ftmp0] \n\t" @@ -3213,10 +3346,10 @@ void ff_put_vp8_bilinear4_h_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, RESTRICT_ASM_ALL64 [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src), - [a]"+&f"(a), [b]"+&f"(b) + [a]"+&f"(a.f), [b]"+&f"(b.f) : [sstride]"r"((mips_reg)sstride), [dstride]"r"((mips_reg)dstride), - [ff_pw_4]"f"(ff_pw_4) + [ff_pw_4]"f"(ff_pw_4.f) : "memory" ); #else @@ -3236,12 +3369,14 @@ void ff_put_vp8_bilinear4_v_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, ptrdiff_t sstride, int h, int mx, int my) { #if 1 - int c = 8 - my, d = my; + union mmi_intfloat64 c, d; double ftmp[7]; uint32_t tmp[1]; mips_reg src1; DECLARE_VAR_LOW32; DECLARE_VAR_ALL64; + c.i = 8 - my; + d.i = my; /* dst[0] = (c * src[0] + d * src[ sstride] + 4) >> 3; @@ -3250,7 +3385,7 @@ void ff_put_vp8_bilinear4_v_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, dst[3] = (c * src[3] + d * src[3 + sstride] + 4) >> 3; */ __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x03 \n\t" "mtc1 %[tmp0], %[ftmp4] \n\t" "pshufh %[c], %[c], %[ftmp0] \n\t" @@ -3272,10 +3407,10 @@ void ff_put_vp8_bilinear4_v_mmi(uint8_t *dst, ptrdiff_t dstride, uint8_t *src, [src1]"=&r"(src1), [h]"+&r"(h), [dst]"+&r"(dst), [src]"+&r"(src), - [c]"+&f"(c), [d]"+&f"(d) + [c]"+&f"(c.f), [d]"+&f"(d.f) : [sstride]"r"((mips_reg)sstride), [dstride]"r"((mips_reg)dstride), - [ff_pw_4]"f"(ff_pw_4) + [ff_pw_4]"f"(ff_pw_4.f) : "memory" ); #else diff --git a/libavcodec/mips/vp9_mc_mmi.c b/libavcodec/mips/vp9_mc_mmi.c index e7a83875b9..812f7a6994 100644 --- a/libavcodec/mips/vp9_mc_mmi.c +++ b/libavcodec/mips/vp9_mc_mmi.c @@ -82,10 +82,10 @@ static void convolve_horiz_mmi(const uint8_t *src, int32_t src_stride, dst_stride -= w; __asm__ volatile ( "move %[tmp1], %[width] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "gsldlc1 %[filter1], 0x03(%[filter]) \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "gsldlc1 %[filter1], 0x07(%[filter]) \n\t" "gsldrc1 %[filter1], 0x00(%[filter]) \n\t" - "gsldlc1 %[filter2], 0x0b(%[filter]) \n\t" + "gsldlc1 %[filter2], 0x0f(%[filter]) \n\t" "gsldrc1 %[filter2], 0x08(%[filter]) \n\t" "li %[tmp0], 0x07 \n\t" "dmtc1 %[tmp0], %[ftmp13] \n\t" @@ -157,10 +157,10 @@ static void convolve_vert_mmi(const uint8_t *src, int32_t src_stride, dst_stride -= w; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "gsldlc1 %[ftmp4], 0x03(%[filter]) \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "gsldlc1 %[ftmp4], 0x07(%[filter]) \n\t" "gsldrc1 %[ftmp4], 0x00(%[filter]) \n\t" - "gsldlc1 %[ftmp5], 0x0b(%[filter]) \n\t" + "gsldlc1 %[ftmp5], 0x0f(%[filter]) \n\t" "gsldrc1 %[ftmp5], 0x08(%[filter]) \n\t" "punpcklwd %[filter10], %[ftmp4], %[ftmp4] \n\t" "punpckhwd %[filter32], %[ftmp4], %[ftmp4] \n\t" @@ -253,10 +253,10 @@ static void convolve_avg_horiz_mmi(const uint8_t *src, int32_t src_stride, __asm__ volatile ( "move %[tmp1], %[width] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "gsldlc1 %[filter1], 0x03(%[filter]) \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "gsldlc1 %[filter1], 0x07(%[filter]) \n\t" "gsldrc1 %[filter1], 0x00(%[filter]) \n\t" - "gsldlc1 %[filter2], 0x0b(%[filter]) \n\t" + "gsldlc1 %[filter2], 0x0f(%[filter]) \n\t" "gsldrc1 %[filter2], 0x08(%[filter]) \n\t" "li %[tmp0], 0x07 \n\t" "dmtc1 %[tmp0], %[ftmp13] \n\t" @@ -339,10 +339,10 @@ static void convolve_avg_vert_mmi(const uint8_t *src, int32_t src_stride, dst_stride -= w; __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" - "gsldlc1 %[ftmp4], 0x03(%[filter]) \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "gsldlc1 %[ftmp4], 0x07(%[filter]) \n\t" "gsldrc1 %[ftmp4], 0x00(%[filter]) \n\t" - "gsldlc1 %[ftmp5], 0x0b(%[filter]) \n\t" + "gsldlc1 %[ftmp5], 0x0f(%[filter]) \n\t" "gsldrc1 %[ftmp5], 0x08(%[filter]) \n\t" "punpcklwd %[filter10], %[ftmp4], %[ftmp4] \n\t" "punpckhwd %[filter32], %[ftmp4], %[ftmp4] \n\t" @@ -444,7 +444,7 @@ static void convolve_avg_mmi(const uint8_t *src, int32_t src_stride, __asm__ volatile ( "move %[tmp1], %[width] \n\t" - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" "li %[tmp0], 0x10001 \n\t" "dmtc1 %[tmp0], %[ftmp3] \n\t" "punpcklhw %[ftmp3], %[ftmp3], %[ftmp3] \n\t" diff --git a/libavcodec/mips/wmv2dsp_mmi.c b/libavcodec/mips/wmv2dsp_mmi.c index 82e16f929b..1a6781ae77 100644 --- a/libavcodec/mips/wmv2dsp_mmi.c +++ b/libavcodec/mips/wmv2dsp_mmi.c @@ -106,7 +106,7 @@ void ff_wmv2_idct_add_mmi(uint8_t *dest, ptrdiff_t line_size, int16_t *block) wmv2_idct_col_mmi(block + i); __asm__ volatile ( - "xor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" + "pxor %[ftmp0], %[ftmp0], %[ftmp0] \n\t" // low 4 loop MMI_LDC1(%[ftmp1], %[block], 0x00) diff --git a/libavcodec/vc1dsp.c b/libavcodec/vc1dsp.c index c25a6f3adf..a543792e14 100644 --- a/libavcodec/vc1dsp.c +++ b/libavcodec/vc1dsp.c @@ -445,7 +445,6 @@ static void vc1_inv_trans_4x8_c(uint8_t *dest, ptrdiff_t stride, int16_t *block) dst[1] = (t2 - t4) >> 3; dst[2] = (t2 + t4) >> 3; dst[3] = (t1 - t3) >> 3; - src += 8; dst += 8; } @@ -1039,4 +1038,6 @@ av_cold void ff_vc1dsp_init(VC1DSPContext *dsp) ff_vc1dsp_init_x86(dsp); if (ARCH_MIPS) ff_vc1dsp_init_mips(dsp); + if (ARCH_LOONGARCH) + ff_vc1dsp_init_loongarch(dsp); } diff --git a/libavcodec/vc1dsp.h b/libavcodec/vc1dsp.h index 75db62b1b4..c6443acb20 100644 --- a/libavcodec/vc1dsp.h +++ b/libavcodec/vc1dsp.h @@ -88,5 +88,6 @@ void ff_vc1dsp_init_arm(VC1DSPContext* dsp); void ff_vc1dsp_init_ppc(VC1DSPContext *c); void ff_vc1dsp_init_x86(VC1DSPContext* dsp); void ff_vc1dsp_init_mips(VC1DSPContext* dsp); +void ff_vc1dsp_init_loongarch(VC1DSPContext* dsp); #endif /* AVCODEC_VC1DSP_H */ diff --git a/libavcodec/videodsp.c b/libavcodec/videodsp.c index ce9e9eb143..fe64b5e81a 100644 --- a/libavcodec/videodsp.c +++ b/libavcodec/videodsp.c @@ -54,4 +54,6 @@ av_cold void ff_videodsp_init(VideoDSPContext *ctx, int bpc) ff_videodsp_init_x86(ctx, bpc); if (ARCH_MIPS) ff_videodsp_init_mips(ctx, bpc); + if (ARCH_LOONGARCH) + ff_videodsp_init_loongarch(ctx, bpc); } diff --git a/libavcodec/videodsp.h b/libavcodec/videodsp.h index c0545f22b0..ac971dc57f 100644 --- a/libavcodec/videodsp.h +++ b/libavcodec/videodsp.h @@ -84,5 +84,6 @@ void ff_videodsp_init_arm(VideoDSPContext *ctx, int bpc); void ff_videodsp_init_ppc(VideoDSPContext *ctx, int bpc); void ff_videodsp_init_x86(VideoDSPContext *ctx, int bpc); void ff_videodsp_init_mips(VideoDSPContext *ctx, int bpc); +void ff_videodsp_init_loongarch(VideoDSPContext *ctx, int bpc); #endif /* AVCODEC_VIDEODSP_H */ diff --git a/libavcodec/vp8dsp.c b/libavcodec/vp8dsp.c index fed5c67a90..ca21e41cd0 100644 --- a/libavcodec/vp8dsp.c +++ b/libavcodec/vp8dsp.c @@ -739,5 +739,7 @@ av_cold void ff_vp8dsp_init(VP8DSPContext *dsp) ff_vp8dsp_init_x86(dsp); if (ARCH_MIPS) ff_vp8dsp_init_mips(dsp); + if (ARCH_LOONGARCH) + ff_vp8dsp_init_loongarch(dsp); } #endif /* CONFIG_VP8_DECODER */ diff --git a/libavcodec/vp8dsp.h b/libavcodec/vp8dsp.h index eaae4aed6d..c603d075f6 100644 --- a/libavcodec/vp8dsp.h +++ b/libavcodec/vp8dsp.h @@ -99,6 +99,7 @@ void ff_vp8dsp_init(VP8DSPContext *c); void ff_vp8dsp_init_arm(VP8DSPContext *c); void ff_vp8dsp_init_x86(VP8DSPContext *c); void ff_vp8dsp_init_mips(VP8DSPContext *c); +void ff_vp8dsp_init_loongarch(VP8DSPContext *c); #define IS_VP7 1 #define IS_VP8 0 diff --git a/libavcodec/vp9dsp.c b/libavcodec/vp9dsp.c index f6d73f73cd..5fe8888da6 100644 --- a/libavcodec/vp9dsp.c +++ b/libavcodec/vp9dsp.c @@ -96,4 +96,5 @@ av_cold void ff_vp9dsp_init(VP9DSPContext *dsp, int bpp, int bitexact) if (ARCH_ARM) ff_vp9dsp_init_arm(dsp, bpp); if (ARCH_X86) ff_vp9dsp_init_x86(dsp, bpp, bitexact); if (ARCH_MIPS) ff_vp9dsp_init_mips(dsp, bpp); + if (ARCH_LOONGARCH) ff_vp9dsp_init_loongarch(dsp, bpp); } diff --git a/libavcodec/vp9dsp.h b/libavcodec/vp9dsp.h index e2256316a8..700dd72de8 100644 --- a/libavcodec/vp9dsp.h +++ b/libavcodec/vp9dsp.h @@ -132,5 +132,6 @@ void ff_vp9dsp_init_aarch64(VP9DSPContext *dsp, int bpp); void ff_vp9dsp_init_arm(VP9DSPContext *dsp, int bpp); void ff_vp9dsp_init_x86(VP9DSPContext *dsp, int bpp, int bitexact); void ff_vp9dsp_init_mips(VP9DSPContext *dsp, int bpp); +void ff_vp9dsp_init_loongarch(VP9DSPContext *dsp, int bpp); #endif /* AVCODEC_VP9DSP_H */ diff --git a/libavutil/cpu.c b/libavutil/cpu.c index 52f6b9a3bf..efb48eb544 100644 --- a/libavutil/cpu.c +++ b/libavutil/cpu.c @@ -61,6 +61,8 @@ static int get_cpu_flags(void) return ff_get_cpu_flags_ppc(); if (ARCH_X86) return ff_get_cpu_flags_x86(); + if (ARCH_LOONGARCH) + return ff_get_cpu_flags_loongarch(); return 0; } @@ -174,6 +176,9 @@ int av_parse_cpu_flags(const char *s) #elif ARCH_MIPS { "mmi", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = AV_CPU_FLAG_MMI }, .unit = "flags" }, { "msa", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = AV_CPU_FLAG_MSA }, .unit = "flags" }, +#elif ARCH_LOONGARCH + { "lsx", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = AV_CPU_FLAG_LSX }, .unit = "flags" }, + { "lasx", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = AV_CPU_FLAG_LASX }, .unit = "flags" }, #endif { NULL }, }; @@ -258,6 +263,9 @@ int av_parse_cpu_caps(unsigned *flags, const char *s) #elif ARCH_MIPS { "mmi", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = AV_CPU_FLAG_MMI }, .unit = "flags" }, { "msa", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = AV_CPU_FLAG_MSA }, .unit = "flags" }, +#elif ARCH_LOONGARCH + { "lsx", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = AV_CPU_FLAG_LSX }, .unit = "flags" }, + { "lasx", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = AV_CPU_FLAG_LASX }, .unit = "flags" }, #endif { NULL }, }; @@ -326,6 +334,8 @@ size_t av_cpu_max_align(void) return ff_get_cpu_max_align_ppc(); if (ARCH_X86) return ff_get_cpu_max_align_x86(); + if (ARCH_LOONGARCH) + return ff_get_cpu_max_align_loongarch(); return 8; } diff --git a/libavutil/cpu.h b/libavutil/cpu.h index 83099dd969..c1dbdc0061 100644 --- a/libavutil/cpu.h +++ b/libavutil/cpu.h @@ -74,6 +74,10 @@ #define AV_CPU_FLAG_MMI (1 << 0) #define AV_CPU_FLAG_MSA (1 << 1) +//Loongarch SIMD extension. +#define AV_CPU_FLAG_LSX (1 << 0) +#define AV_CPU_FLAG_LASX (1 << 1) + /** * Return the flags which specify extensions supported by the CPU. * The returned value is affected by av_force_cpu_flags() if that was used diff --git a/libavutil/cpu_internal.h b/libavutil/cpu_internal.h index 889764320b..e207b2d480 100644 --- a/libavutil/cpu_internal.h +++ b/libavutil/cpu_internal.h @@ -46,11 +46,13 @@ int ff_get_cpu_flags_aarch64(void); int ff_get_cpu_flags_arm(void); int ff_get_cpu_flags_ppc(void); int ff_get_cpu_flags_x86(void); +int ff_get_cpu_flags_loongarch(void); size_t ff_get_cpu_max_align_mips(void); size_t ff_get_cpu_max_align_aarch64(void); size_t ff_get_cpu_max_align_arm(void); size_t ff_get_cpu_max_align_ppc(void); size_t ff_get_cpu_max_align_x86(void); +size_t ff_get_cpu_max_align_loongarch(void); #endif /* AVUTIL_CPU_INTERNAL_H */ diff --git a/libavutil/loongarch/Makefile b/libavutil/loongarch/Makefile new file mode 100644 index 0000000000..2addd9351c --- /dev/null +++ b/libavutil/loongarch/Makefile @@ -0,0 +1 @@ +OBJS += loongarch/cpu.o diff --git a/libavutil/loongarch/cpu.c b/libavutil/loongarch/cpu.c new file mode 100644 index 0000000000..e4b240bc44 --- /dev/null +++ b/libavutil/loongarch/cpu.c @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include "cpu.h" + +#define LOONGARCH_CFG2 0x2 +#define LOONGARCH_CFG2_LSX (1 << 6) +#define LOONGARCH_CFG2_LASX (1 << 7) + +static int cpu_flags_cpucfg(void) +{ + int flags = 0; + uint32_t cfg2 = 0; + + __asm__ volatile( + "cpucfg %0, %1 \n\t" + : "+&r"(cfg2) + : "r"(LOONGARCH_CFG2) + ); + + if (cfg2 & LOONGARCH_CFG2_LSX) + flags |= AV_CPU_FLAG_LSX; + + if (cfg2 & LOONGARCH_CFG2_LASX) + flags |= AV_CPU_FLAG_LASX; + + return flags; +} + +int ff_get_cpu_flags_loongarch(void) +{ +#if defined __linux__ + return cpu_flags_cpucfg(); +#else + /* Assume no SIMD ASE supported */ + return 0; +#endif +} + +size_t ff_get_cpu_max_align_loongarch(void) +{ + int flags = av_get_cpu_flags(); + + if (flags & AV_CPU_FLAG_LASX) + return 32; + if (flags & AV_CPU_FLAG_LSX) + return 16; + + return 8; +} diff --git a/libavutil/loongarch/cpu.h b/libavutil/loongarch/cpu.h new file mode 100644 index 0000000000..1a445c69bc --- /dev/null +++ b/libavutil/loongarch/cpu.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2020 Loongson Technology Corporation Limited + * Contributed by Shiyou Yin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_LOONGARCH_CPU_H +#define AVUTIL_LOONGARCH_CPU_H + +#include "libavutil/cpu.h" +#include "libavutil/cpu_internal.h" + +#define have_lsx(flags) CPUEXT(flags, LSX) +#define have_lasx(flags) CPUEXT(flags, LASX) + +#endif /* AVUTIL_LOONGARCH_CPU_H */ diff --git a/libavutil/loongarch/generic_macros_lasx.h b/libavutil/loongarch/generic_macros_lasx.h new file mode 100644 index 0000000000..aca12e6959 --- /dev/null +++ b/libavutil/loongarch/generic_macros_lasx.h @@ -0,0 +1,3561 @@ +/* + * Copyright (c) 2020 Loongson Technology Corporation Limited + * All rights reserved. + * Contributed by Shiyou Yin + * Xiwei Gu + * Jin Bo + * Hao Chen + * Lu Wang + * Peng Zhou + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifndef AVUTIL_LOONGARCH_GENERIC_MACROS_LASX_H +#define AVUTIL_LOONGARCH_GENERIC_MACROS_LASX_H + +/* + * Copyright (c) 2020 Loongson Technology Corporation Limited + * All rights reserved. + * Contributed by Shiyou Yin + * Xiwei Gu + * Jin Bo + * Hao Chen + * Lu Wang + * Peng Zhou + * + * This file is maintained in LSOM project, don't change it directly. + * You can get the latest version of this header from: *** + * + */ + +#ifndef GENERIC_MACROS_LASX_H +#define GENERIC_MACROS_LASX_H + +#include +#include + +/** + * MAJOR version: Macro usage changes. + * MINOR version: Add new macros, or bug fix. + * MICRO version: Comment changes or implementation changes。 + */ +#define LSOM_LASX_VERSION_MAJOR 3 +#define LSOM_LASX_VERSION_MINOR 0 +#define LSOM_LASX_VERSION_MICRO 0 + +/* Description : Load 256-bit vector data with stride + * Arguments : Inputs - psrc (source pointer to load from) + * - stride + * Outputs - out0, out1, ~ + * Details : Load 256-bit data in 'out0' from (psrc) + * Load 256-bit data in 'out1' from (psrc + stride) + */ +#define LASX_LD(psrc) *((__m256i *)(psrc)) + +#define LASX_LD_2(psrc, stride, out0, out1) \ +{ \ + out0 = LASX_LD(psrc); \ + out1 = LASX_LD((psrc) + stride); \ +} + +#define LASX_LD_4(psrc, stride, out0, out1, out2, out3) \ +{ \ + LASX_LD_2((psrc), stride, out0, out1); \ + LASX_LD_2((psrc) + 2 * stride , stride, out2, out3); \ +} + +#define LASX_LD_8(psrc, stride, out0, out1, out2, out3, out4, out5, \ + out6, out7) \ +{ \ + LASX_LD_4((psrc), stride, out0, out1, out2, out3); \ + LASX_LD_4((psrc) + 4 * stride, stride, out4, out5, out6, out7); \ +} + +/* Description : Store 256-bit vector data with stride + * Arguments : Inputs - in0, in1, ~ + * - pdst (destination pointer to store to) + * - stride + * Details : Store 256-bit data from 'in0' to (pdst) + * Store 256-bit data from 'in1' to (pdst + stride) + */ +#define LASX_ST(in, pdst) *((__m256i *)(pdst)) = (in) + +#define LASX_ST_2(in0, in1, pdst, stride) \ +{ \ + LASX_ST(in0, (pdst)); \ + LASX_ST(in1, (pdst) + stride); \ +} + +#define LASX_ST_4(in0, in1, in2, in3, pdst, stride) \ +{ \ + LASX_ST_2(in0, in1, (pdst), stride); \ + LASX_ST_2(in2, in3, (pdst) + 2 * stride, stride); \ +} + +#define LASX_ST_8(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ +{ \ + LASX_ST_4(in0, in1, in2, in3, (pdst), stride); \ + LASX_ST_4(in4, in5, in6, in7, (pdst) + 4 * stride, stride); \ +} + +/* Description : Store half word elements of vector with stride + * Arguments : Inputs - in source vector + * - idx, idx0, idx1, ~ + * - pdst (destination pointer to store to) + * - stride + * Details : Store half word 'idx0' from 'in' to (pdst) + * Store half word 'idx1' from 'in' to (pdst + stride) + * Similar for other elements + * Example : LASX_ST_H(in, idx, pdst) + * in : 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + * idx0 : 0x01 + * out0 : 2 + */ +#define LASX_ST_H(in, idx, pdst) \ +{ \ + __lasx_xvstelm_h(in, pdst, 0, idx); \ +} + +#define LASX_ST_H_2(in, idx0, idx1, pdst, stride) \ +{ \ + LASX_ST_H(in, idx0, (pdst)); \ + LASX_ST_H(in, idx1, (pdst) + stride); \ +} + +#define LASX_ST_H_4(in, idx0, idx1, idx2, idx3, pdst, stride) \ +{ \ + LASX_ST_H_2(in, idx0, idx1, (pdst), stride); \ + LASX_ST_H_2(in, idx2, idx3, (pdst) + 2 * stride, stride); \ +} + + +/* Description : Store word elements of vector with stride + * Arguments : Inputs - in source vector + * - idx, idx0, idx1, ~ + * - pdst (destination pointer to store to) + * - stride + * Details : Store word 'idx0' from 'in' to (pdst) + * Store word 'idx1' from 'in' to (pdst + stride) + * Similar for other elements + * Example : LASX_ST_W(in, idx, pdst) + * in : 1, 2, 3, 4, 5, 6, 7, 8 + * idx0 : 0x01 + * out0 : 2 + */ +#define LASX_ST_W(in, idx, pdst) \ +{ \ + __lasx_xvstelm_w(in, pdst, 0, idx); \ +} + +#define LASX_ST_W_2(in, idx0, idx1, pdst, stride) \ +{ \ + LASX_ST_W(in, idx0, (pdst)); \ + LASX_ST_W(in, idx1, (pdst) + stride); \ +} + +#define LASX_ST_W_4(in, idx0, idx1, idx2, idx3, pdst, stride) \ +{ \ + LASX_ST_W_2(in, idx0, idx1, (pdst), stride); \ + LASX_ST_W_2(in, idx2, idx3, (pdst) + 2 * stride, stride); \ +} + +#define LASX_ST_W_8(in, idx0, idx1, idx2, idx3, idx4, idx5, idx6, idx7, \ + pdst, stride) \ +{ \ + LASX_ST_W_4(in, idx0, idx1, idx2, idx3, (pdst), stride); \ + LASX_ST_W_4(in, idx4, idx5, idx6, idx7, (pdst) + 4 * stride, stride); \ +} + +/* Description : Store double word elements of vector with stride + * Arguments : Inputs - in source vector + * - idx, idx0, idx1, ~ + * - pdst (destination pointer to store to) + * - stride + * Details : Store double word 'idx0' from 'in' to (pdst) + * Store double word 'idx1' from 'in' to (pdst + stride) + * Similar for other elements + * Example : See LASX_ST_W(in, idx, pdst) + */ +#define LASX_ST_D(in, idx, pdst) \ +{ \ + __lasx_xvstelm_d(in, pdst, 0, idx); \ +} + +#define LASX_ST_D_2(in, idx0, idx1, pdst, stride) \ +{ \ + LASX_ST_D(in, idx0, (pdst)); \ + LASX_ST_D(in, idx1, (pdst) + stride); \ +} + +#define LASX_ST_D_4(in, idx0, idx1, idx2, idx3, pdst, stride) \ +{ \ + LASX_ST_D_2(in, idx0, idx1, (pdst), stride); \ + LASX_ST_D_2(in, idx2, idx3, (pdst) + 2 * stride, stride); \ +} + +/* Description : Store quad word elements of vector with stride + * Arguments : Inputs - in source vector + * - idx, idx0, idx1, ~ + * - pdst (destination pointer to store to) + * - stride + * Details : Store quad word 'idx0' from 'in' to (pdst) + * Store quad word 'idx1' from 'in' to (pdst + stride) + * Similar for other elements + * Example : See LASX_ST_W(in, idx, pdst) + */ +#define LASX_ST_Q(in, idx, pdst) \ +{ \ + LASX_ST_D(in, (idx << 1), pdst); \ + LASX_ST_D(in, (( idx << 1) + 1), (char*)(pdst) + 8); \ +} + +#define LASX_ST_Q_2(in, idx0, idx1, pdst, stride) \ +{ \ + LASX_ST_Q(in, idx0, (pdst)); \ + LASX_ST_Q(in, idx1, (pdst) + stride); \ +} + +#define LASX_ST_Q_4(in, idx0, idx1, idx2, idx3, pdst, stride) \ +{ \ + LASX_ST_Q_2(in, idx0, idx1, (pdst), stride); \ + LASX_ST_Q_2(in, idx2, idx3, (pdst) + 2 * stride, stride); \ +} + +/* Description : Dot product of byte vector elements + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - unsigned halfword + * Details : Unsigned byte elements from in0 are iniplied with + * unsigned byte elements from in0 producing a result + * twice the size of input i.e. unsigned halfword. + * Then this iniplication results of adjacent odd-even elements + * are added together and stored to the out vector + * (2 unsigned halfword results) + * Example : see LASX_DP2_W_H + */ +#define LASX_DP2_H_BU(in0, in1, out0) \ +{ \ + __m256i _tmp0_m ; \ + \ + _tmp0_m = __lasx_xvmulwev_h_bu( in0, in1 ); \ + out0 = __lasx_xvmaddwod_h_bu( _tmp0_m, in0, in1 ); \ +} +#define LASX_DP2_H_BU_2(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_DP2_H_BU(in0, in1, out0); \ + LASX_DP2_H_BU(in2, in3, out1); \ +} +#define LASX_DP2_H_BU_4(in0, in1, in2, in3, \ + in4, in5, in6, in7, \ + out0, out1, out2, out3) \ +{ \ + LASX_DP2_H_BU_2(in0, in1, in0, in1, out0, out1); \ + LASX_DP2_H_BU_2(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : Dot product of byte vector elements + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - signed halfword + * Details : Signed byte elements from in0 are iniplied with + * signed byte elements from in0 producing a result + * twice the size of input i.e. signed halfword. + * Then this iniplication results of adjacent odd-even elements + * are added together and stored to the out vector + * (2 signed halfword results) + * Example : see LASX_DP2_W_H + */ +#define LASX_DP2_H_B(in0, in1, out0) \ +{ \ + __m256i _tmp0_m ; \ + \ + _tmp0_m = __lasx_xvmulwev_h_b( in0, in1 ); \ + out0 = __lasx_xvmaddwod_h_b( _tmp0_m, in0, in1 ); \ +} +#define LASX_DP2_H_B_2(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_DP2_H_B(in0, in1, out0); \ + LASX_DP2_H_B(in2, in3, out1); \ +} +#define LASX_DP2_H_B_4(in0, in1, in2, in3, \ + in4, in5, in6, in7, \ + out0, out1, out2, out3) \ +{ \ + LASX_DP2_H_B_2(in0, in1, in2, in3, out0, out1); \ + LASX_DP2_H_B_2(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : Dot product of half word vector elements + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Return Type - signed word + * Details : Signed half word elements from in* are iniplied with + * signed half word elements from in* producing a result + * twice the size of input i.e. signed word. + * Then this iniplication results of adjacent odd-even elements + * are added together and stored to the out vector. + * Example : LASX_DP2_W_H(in0, in1, out0) + * in0: 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8, + * in0: 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1, + * out0: 22,38,38,22, 22,38,38,22 + */ +#define LASX_DP2_W_H(in0, in1, out0) \ +{ \ + __m256i _tmp0_m ; \ + \ + _tmp0_m = __lasx_xvmulwev_w_h( in0, in1 ); \ + out0 = __lasx_xvmaddwod_w_h( _tmp0_m, in0, in1 ); \ +} +#define LASX_DP2_W_H_2(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_DP2_W_H(in0, in1, out0); \ + LASX_DP2_W_H(in2, in3, out1); \ +} +#define LASX_DP2_W_H_4(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_DP2_W_H_2(in0, in1, in2, in3, out0, out1); \ + LASX_DP2_W_H_2(in4, in5, in6, in7, out2, out3); \ +} +#define LASX_DP2_W_H_8(in0, in1, in2, in3, in4, in5, in6, in7, \ + in8, in9, in10, in11, in12, in13, in14, in15, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_DP2_W_H_4(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3); \ + LASX_DP2_W_H_4(in8, in9, in10, in11, in12, in13, in14, in15, \ + out4, out5, out6, out7); \ +} + +/* Description : Dot product of word vector elements + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Retrun Type - signed double + * Details : Signed word elements from in* are iniplied with + * signed word elements from in* producing a result + * twice the size of input i.e. signed double word. + * Then this iniplication results of adjacent odd-even elements + * are added together and stored to the out vector. + * Example : see LASX_DP2_W_H + */ +#define LASX_DP2_D_W(in0, in1, out0) \ +{ \ + __m256i _tmp0_m ; \ + \ + _tmp0_m = __lasx_xvmulwev_d_w( in0, in1 ); \ + out0 = __lasx_xvmaddwod_d_w( _tmp0_m, in0, in1 ); \ +} +#define LASX_DP2_D_W_2(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_DP2_D_W(in0, in1, out0); \ + LASX_DP2_D_W(in2, in3, out1); \ +} +#define LASX_DP2_D_W_4(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_DP2_D_W_2(in0, in1, in2, in3, out0, out1); \ + LASX_DP2_D_W_2(in4, in5, in6, in7, out2, out3); \ +} +#define LASX_DP2_D_W_8(in0, in1, in2, in3, in4, in5, in6, in7, \ + in8, in9, in10, in11, in12, in13, in14, in15, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_DP2_D_W_4(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3); \ + LASX_DP2_D_W_4(in8, in9, in10, in11, in12, in13, in14, in15, \ + out4, out5, out6, out7); \ +} + +/* Description : Dot product of halfword vector elements + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Unsigned halfword elements from 'in0' are iniplied with + * halfword elements from 'in0' producing a result + * twice the size of input i.e. unsigned word. + * Multiplication result of adjacent odd-even elements + * are added together and written to the 'out0' vector + */ +#define LASX_DP2_W_HU_H(in0, in1, out0) \ +{ \ + __m256i _tmp0_m; \ + \ + _tmp0_m = __lasx_xvmulwev_w_hu_h( in0, in1 ); \ + out0 = __lasx_xvmaddwod_w_hu_h( _tmp0_m, in0, in1 ); \ +} + +#define LASX_DP2_W_HU_H_2(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_DP2_W_HU_H(in0, in1, out0); \ + LASX_DP2_W_HU_H(in2, in3, out1); \ +} + +#define LASX_DP2_W_HU_H_4(in0, in1, in2, in3, \ + in4, in5, in6, in7, \ + out0, out1, out2, out3) \ +{ \ + LASX_DP2_W_HU_H_2(in0, in1, in2, in3, out0, out1); \ + LASX_DP2_W_HU_H_2(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Retrun Type - halfword + * Details : Signed byte elements from in0 are iniplied with + * signed byte elements from in0 producing a result + * twice the size of input i.e. signed halfword. + * Then this iniplication results of adjacent odd-even elements + * are added to the out vector + * (2 signed halfword results) + * Example : LASX_DP2ADD_H_B(in0, in1, in2, out0) + * in0: 1,2,3,4, 1,2,3,4, 1,2,3,4, 1,2,3,4, + * in1: 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8, + * 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in2: 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1, + * 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out0: 23,40,41,26, 23,40,41,26, 23,40,41,26, 23,40,41,26 + */ +#define LASX_DP2ADD_H_B(in0, in1, in2, out0) \ +{ \ + __m256i _tmp0_m; \ + \ + _tmp0_m = __lasx_xvmaddwev_h_b( in0, in1, in2 ); \ + out0 = __lasx_xvmaddwod_h_b( _tmp0_m, in1, in2 ); \ +} +#define LASX_DP2ADD_H_B_2(in0, in1, in2, in3, in4, in5, out0, out1) \ +{ \ + LASX_DP2ADD_H_B(in0, in1, in2, out0); \ + LASX_DP2ADD_H_B(in3, in4, in5, out1); \ +} +#define LASX_DP2ADD_H_B_4(in0, in1, in2, in3, in4, in5, \ + in6, in7, in8, in9, in10, in11, \ + out0, out1, out2, out3) \ +{ \ + LASX_DP2ADD_H_B_2(in0, in1, in2, in3, in4, in5, out0, out1); \ + LASX_DP2ADD_H_B_2(in6, in7, in8, in9, in10, in11, out2, out3); \ +} + +/* Description : Dot product of halfword vector elements + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Signed halfword elements from 'in0' are iniplied with + * signed halfword elements from 'in0' producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added together and written to the 'out0' vector + */ +#define LASX_DP2ADD_W_H(in0, in1, in2, out0) \ +{ \ + __m256i _tmp0_m; \ + \ + _tmp0_m = __lasx_xvmaddwev_w_h( in0, in1, in2 ); \ + out0 = __lasx_xvmaddwod_w_h( _tmp0_m, in1, in2 ); \ +} +#define LASX_DP2ADD_W_H_2(in0, in1, in2, in3, in4, in5, out0, out1 ) \ +{ \ + LASX_DP2ADD_W_H(in0, in1, in2, out0); \ + LASX_DP2ADD_W_H(in3, in4, in5, out1); \ +} +#define LASX_DP2ADD_W_H_4(in0, in1, in2, in3, in4, in5, \ + in6, in7, in8, in9, in10, in11, \ + out0, out1, out2, out3) \ +{ \ + LASX_DP2ADD_W_H_2(in0, in1, in2, in3, in4, in5, out0, out1); \ + LASX_DP2ADD_W_H_2(in6, in7, in8, in9, in10, in11, out2, out3); \ +} + +/* Description : Dot product of halfword vector elements + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Unsigned halfword elements from 'in0' are iniplied with + * unsigned halfword elements from 'in0' producing a result + * twice the size of input i.e. unsigned word. + * Multiplication result of adjacent odd-even elements + * are added together and written to the 'out0' vector + */ +#define LASX_DP2ADD_W_HU(in0, in1, in2, out0) \ +{ \ + __m256i _tmp0_m; \ + \ + _tmp0_m = __lasx_xvmaddwev_w_hu( in0, in1, in2 ); \ + out0 = __lasx_xvmaddwod_w_hu( _tmp0_m, in1, in2 ); \ +} +#define LASX_DP2ADD_W_HU_2(in0, in1, in2, in3, in4, in5, out0, out1) \ +{ \ + LASX_DP2ADD_W_HU(in0, in1, in2, out0); \ + LASX_DP2ADD_W_HU(in3, in4, in5, out1); \ +} +#define LASX_DP2ADD_W_HU_4(in0, in1, in2, in3, in4, in5, \ + in6, in7, in8, in9, in10, in11, \ + out0, out1, out2, out3) \ +{ \ + LASX_DP2ADD_W_HU_2(in0, in1, in2, in3, in4, in5, out0, out1); \ + LASX_DP2ADD_W_HU_2(in6, in7, in8, in9, in10, in11, out2, out3); \ +} + +/* Description : Dot product of halfword vector elements + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Unsigned halfword elements from 'in0' are iniplied with + * halfword elements from 'in0' producing a result + * twice the size of input i.e. unsigned word. + * Multiplication result of adjacent odd-even elements + * are added together and written to the 'out0' vector + */ +#define LASX_DP2ADD_W_HU_H(in0, in1, in2, out0) \ +{ \ + __m256i _tmp0_m; \ + \ + _tmp0_m = __lasx_xvmaddwev_w_hu_h( in0, in1, in2 ); \ + out0 = __lasx_xvmaddwod_w_hu_h( _tmp0_m, in1, in2 ); \ +} + +#define LASX_DP2ADD_W_HU_H_2(in0, in1, in2, in3, in4, in5, out0, out1) \ +{ \ + LASX_DP2ADD_W_HU_H(in0, in1, in2, out0); \ + LASX_DP2ADD_W_HU_H(in3, in4, in5, out1); \ +} + +#define LASX_DP2ADD_W_HU_H_4(in0, in1, in2, in3, in4, in5, \ + in6, in7, in8, in9, in10, in11, \ + out0, out1, out2, out3) \ +{ \ + LASX_DP2ADD_W_HU_H_2(in0, in1, in2, in3, in4, in5, out0, out1); \ + LASX_DP2ADD_W_HU_H_2(in6, in7, in8, in9, in10, in11, out2, out3); \ +} + +/* Description : Vector Unsigned Dot Product and Subtract. + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Unsigned byte elements from 'in0' are iniplied with + * unsigned byte elements from 'in0' producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added together and subtract from double width elements, + * then written to the 'out0' vector. + */ +#define LASX_DP2SUB_H_BU(in0, in1, in2, out0) \ +{ \ + __m256i _tmp0_m; \ + \ + _tmp0_m = __lasx_xvmulwev_h_bu( in1, in2 ); \ + _tmp0_m = __lasx_xvmaddwod_h_bu( _tmp0_m, in1, in2 ); \ + out0 = __lasx_xvsub_h( in0, _tmp0_m ); \ +} + +#define LASX_DP2SUB_H_BU_2(in0, in1, in2, in3, in4, in5, out0, out1) \ +{ \ + LASX_DP2SUB_H_BU(in0, in1, in2, out0); \ + LASX_DP2SUB_H_BU(in0, in1, in2, out0); \ +} + +#define LASX_DP2SUB_H_BU_4(in0, in1, in2, in3, in4, in5, \ + in6, in7, in8, in9, in10, in11, \ + out0, out1, out2, out3) \ +{ \ + LASX_DP2SUB_H_BU_2(in0, in1, in2, in3, in4, in5, out0, out1); \ + LASX_DP2SUB_H_BU_2(in6, in7, in8, in9, in10, in11, out2, out3); \ +} + +/* Description : Vector Signed Dot Product and Subtract. + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Signed halfword elements from 'in0' are iniplied with + * signed halfword elements from 'in0' producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added together and subtract from double width elements, + * then written to the 'out0' vector. + */ +#define LASX_DP2SUB_W_H(in0, in1, in2, out0) \ +{ \ + __m256i _tmp0_m; \ + \ + _tmp0_m = __lasx_xvmulwev_w_h( in1, in2 ); \ + _tmp0_m = __lasx_xvmaddwod_w_h( _tmp0_m, in1, in2 ); \ + out0 = __lasx_xvsub_w( in0, _tmp0_m ); \ +} + +#define LASX_DP2SUB_W_H_2(in0, in1, in2, in3, in4, in5, out0, out1) \ +{ \ + LASX_DP2SUB_W_H(in0, in1, in2, out0); \ + LASX_DP2SUB_W_H(in3, in4, in5, out1); \ +} + +#define LASX_DP2SUB_W_H_4(in0, in1, in2, in3, in4, in5, \ + in6, in7, in8, in9, in10, in11, \ + out0, out1, out2, out3) \ +{ \ + LASX_DP2SUB_W_H_2(in0, in1, in2, in3, in4, in5, out0, out1); \ + LASX_DP2SUB_W_H_2(in6, in7, in8, in9, in10, in11, out2, out3); \ +} + +/* Description : Dot product of half word vector elements + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Return Type - signed word + * Details : Signed half word elements from in* are iniplied with + * signed half word elements from in* producing a result + * twice the size of input i.e. signed word. + * Then this iniplication results of four adjacent elements + * are added together and stored to the out vector. + * Example : LASX_DP2_W_H(in0, in0, out0) + * in0: 3,1,3,0, 0,0,0,1, 0,0,1,-1, 0,0,0,1, + * in0: -2,1,1,0, 1,0,0,0, 0,0,1,0, 1,0,0,1, + * out0: -2,0,1,1, + */ +#define LASX_DP4_D_H(in0, in1, out0) \ +{ \ + __m256i _tmp0_m ; \ + \ + _tmp0_m = __lasx_xvmulwev_w_h( in0, in1 ); \ + _tmp0_m = __lasx_xvmaddwod_w_h( _tmp0_m, in0, in1 ); \ + out0 = __lasx_xvhaddw_d_w( _tmp0_m, _tmp0_m ); \ +} +#define LASX_DP4_D_H_2(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_DP4_D_H(in0, in1, out0); \ + LASX_DP4_D_H(in2, in3, out1); \ +} +#define LASX_DP4_D_H_4(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_DP4_D_H_2(in0, in1, in2, in3, out0, out1); \ + LASX_DP4_D_H_2(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : The high half of the vector elements are expanded and + * added after being doubled + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in0 vector and the in1 vector are added after the + * higher half of the two-fold sign extension ( signed byte + * to signed half word ) and stored to the out vector. + * Example : see LASX_ADDWL_W_H_128SV + */ +#define LASX_ADDWH_H_B_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp0_m, _tmp1_m; \ + \ + _tmp0_m = __lasx_xvilvh_b( in0, in0 ); \ + _tmp1_m = __lasx_xvilvh_b( in1, in1 ); \ + out0 = __lasx_xvaddwev_h_b( _tmp0_m, _tmp1_m ); \ +} +#define LASX_ADDWH_H_B_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_ADDWH_H_B_128SV(in0, in1, out0); \ + LASX_ADDWH_H_B_128SV(in2, in3, out1); \ +} +#define LASX_ADDWH_H_B_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_ADDWH_H_B_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_ADDWH_H_B_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : The high half of the vector elements are expanded and + * added after being doubled + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in0 vector and the in1 vector are added after the + * higher half of the two-fold sign extension ( signed half word + * to signed word ) and stored to the out vector. + * Example : see LASX_ADDWL_W_H_128SV + */ +#define LASX_ADDWH_W_H_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp0_m, _tmp1_m; \ + \ + _tmp0_m = __lasx_xvilvh_h( in0, in0 ); \ + _tmp1_m = __lasx_xvilvh_h( in1, in1 ); \ + out0 = __lasx_xvaddwev_w_h( _tmp0_m, _tmp1_m ); \ +} +#define LASX_ADDWH_W_H_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_ADDWH_W_H_128SV(in0, in1, out0); \ + LASX_ADDWH_W_H_128SV(in2, in3, out1); \ +} +#define LASX_ADDWH_W_H_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_ADDWH_W_H_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_ADDWH_W_H_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : The low half of the vector elements are expanded and + * added after being doubled + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in0 vector and the in1 vector are added after the + * lower half of the two-fold sign extension ( signed byte + * to signed half word ) and stored to the out vector. + * Example : see LASX_ADDWL_W_H_128SV + */ +#define LASX_ADDWL_H_B_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp0_m, _tmp1_m; \ + \ + _tmp0_m = __lasx_xvsllwil_h_b( in0, 0 ); \ + _tmp1_m = __lasx_xvsllwil_h_b( in1, 0 ); \ + out0 = __lasx_xvadd_h( _tmp0_m, _tmp1_m ); \ +} +#define LASX_ADDWL_H_B_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_ADDWL_H_B_128SV(in0, in1, out0); \ + LASX_ADDWL_H_B_128SV(in2, in3, out1); \ +} +#define LASX_ADDWL_H_B_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_ADDWL_H_B_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_ADDWL_H_B_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : The low half of the vector elements are expanded and + * added after being doubled + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in0 vector and the in1 vector are added after the + * lower half of the two-fold sign extension ( signed half word + * to signed word ) and stored to the out vector. + * Example : LASX_ADDWL_W_H_128SV(in0, in1, out0) + * in0 3,0,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1, + * in1 2,-1,1,2, 1,0,0,0, 1,0,1,0, 1,0,0,1, + * out0 5,-1,4,2, 1,0,2,-1, + */ +#define LASX_ADDWL_W_H_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp0_m; \ + \ + _tmp0_m = __lasx_xvilvl_h(in1, in0); \ + out0 = __lasx_xvhaddw_w_h( _tmp0_m, _tmp0_m ); \ +} +#define LASX_ADDWL_W_H_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_ADDWL_W_H_128SV(in0, in1, out0); \ + LASX_ADDWL_W_H_128SV(in2, in3, out1); \ +} +#define LASX_ADDWL_W_H_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_ADDWL_W_H_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_ADDWL_W_H_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : The low half of the vector elements are expanded and + * added after being doubled + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in0 vector and the in1 vector are added after the + * lower half of the two-fold zero extension ( unsigned byte + * to unsigned half word ) and stored to the out vector. + */ +#define LASX_ADDWL_H_BU_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp0_m; \ + \ + _tmp0_m = __lasx_xvilvl_b(in1, in0); \ + out0 = __lasx_xvhaddw_hu_bu( _tmp0_m, _tmp0_m ); \ +} +#define LASX_ADDWL_H_BU_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_ADDWL_H_BU_128SV(in0, in1, out0); \ + LASX_ADDWL_H_BU_128SV(in2, in3, out1); \ +} +#define LASX_ADDWL_H_BU_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_ADDWL_H_BU_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_ADDWL_H_BU_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : The low half of the vector elements are expanded and + * added after being doubled + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : In1 vector plus in0 vector after double zero extension + * ( unsigned byte to half word ),add and stored to the out vector. + * Example : reference to LASX_ADDW_W_W_H_128SV(in0, in1, out0) + */ +#define LASX_ADDW_H_H_BU_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp1_m; \ + \ + _tmp1_m = __lasx_xvsllwil_hu_bu( in1, 0 ); \ + out0 = __lasx_xvadd_h( in0, _tmp1_m ); \ +} +#define LASX_ADDW_H_H_BU_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_ADDW_H_H_BU_128SV(in0, in1, out0); \ + LASX_ADDW_H_H_BU_128SV(in2, in3, out1); \ +} +#define LASX_ADDW_H_H_BU_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_ADDW_H_H_BU_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_ADDW_H_H_BU_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : The low half of the vector elements are expanded and + * added after being doubled + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : In1 vector plus in0 vector after double sign extension + * ( signed half word to word ),add and stored to the out vector. + * Example : LASX_ADDW_W_W_H_128SV(in0, in1, out0) + * in0 0,1,0,0, -1,0,0,1, + * in1 2,-1,1,2, 1,0,0,0, 0,0,1,0, 1,0,0,1, + * out0 2,0,1,2, -1,0,1,1, + */ +#define LASX_ADDW_W_W_H_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp1_m; \ + \ + _tmp1_m = __lasx_xvsllwil_w_h( in1, 0 ); \ + out0 = __lasx_xvadd_w( in0, _tmp1_m ); \ +} +#define LASX_ADDW_W_W_H_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_ADDW_W_W_H_128SV(in0, in1, out0); \ + LASX_ADDW_W_W_H_128SV(in2, in3, out1); \ +} +#define LASX_ADDW_W_W_H_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_ADDW_W_W_H_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_ADDW_W_W_H_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : Multiplication and addition calculation after expansion + * of the lower half of the vector + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in1 vector and the in0 vector are multiplied after + * the lower half of the two-fold sign extension ( signed + * half word to signed word ) , and the result is added to + * the vector in0, the stored to the out vector. + * Example : LASX_MADDWL_W_H_128SV(in0, in1, in2, out0) + * in0 1,2,3,4, 5,6,7 8 + * in1 1,2,3,4, 1,2,3,4, 5,6,7,8, 5,6,7,8 + * in2 200,300,400,500, 2000,3000,4000,5000, + * -200,-300,-400,-500, -2000,-3000,-4000,-5000 + * out0 5,-1,4,2, 1,0,2,-1, + */ +#define LASX_MADDWL_W_H_128SV(in0, in1, in2, out0) \ +{ \ + __m256i _tmp0_m, _tmp1_m; \ + \ + _tmp0_m = __lasx_xvsllwil_w_h( in1, 0 ); \ + _tmp1_m = __lasx_xvsllwil_w_h( in2, 0 ); \ + _tmp0_m = __lasx_xvmul_w( _tmp0_m, _tmp1_m ); \ + out0 = __lasx_xvadd_w( _tmp0_m, in0 ); \ +} +#define LASX_MADDWL_W_H_2_128SV(in0, in1, in2, in3, in4, in5, out0, out1) \ +{ \ + LASX_MADDWL_W_H_128SV(in0, in1, in2, out0); \ + LASX_MADDWL_W_H_128SV(in3, in4, in5, out1); \ +} +#define LASX_MADDWL_W_H_4_128SV(in0, in1, in2, in3, in4, in5, \ + in6, in7, in8, in9, in10, in11, \ + out0, out1, out2, out3) \ +{ \ + LASX_MADDWL_W_H_2_128SV(in0, in1, in2, in3, in4, in5, out0, out1); \ + LASX_MADDWL_W_H_2_128SV(in6, in7, in8, in9, in10, in11, out2, out3); \ +} + +/* Description : Multiplication and addition calculation after expansion + * of the higher half of the vector + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in1 vector and the in0 vector are multiplied after + * the higher half of the two-fold sign extension ( signed + * half word to signed word ) , and the result is added to + * the vector in0, the stored to the out vector. + * Example : see LASX_MADDWL_W_H_128SV + */ +#define LASX_MADDWH_W_H_128SV(in0, in1, in2, out0) \ +{ \ + __m256i _tmp0_m, _tmp1_m; \ + \ + _tmp0_m = __lasx_xvilvh_h( in1, in1 ); \ + _tmp1_m = __lasx_xvilvh_h( in2, in2 ); \ + _tmp0_m = __lasx_xvmulwev_w_h( _tmp0_m, _tmp1_m ); \ + out0 = __lasx_xvadd_w( _tmp0_m, in0 ); \ +} +#define LASX_MADDWH_W_H_2_128SV(in0, in1, in2, in3, in4, in5, out0, out1) \ +{ \ + LASX_MADDWH_W_H_128SV(in0, in1, in2, out0); \ + LASX_MADDWH_W_H_128SV(in3, in4, in5, out1); \ +} +#define LASX_MADDWH_W_H_4_128SV(in0, in1, in2, in3, in4, in5, \ + in6, in7, in8, in9, in10, in11, \ + out0, out1, out2, out3) \ +{ \ + LASX_MADDWH_W_H_2_128SV(in0, in1, in2, in3, in4, in5, out0, out1); \ + LASX_MADDWH_W_H_2_128SV(in6, in7, in8, in9, in10, in11, out2, out3); \ +} + +/* Description : Multiplication calculation after expansion + * of the lower half of the vector + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in1 vector and the in0 vector are multiplied after + * the lower half of the two-fold sign extension ( signed + * half word to signed word ) , the stored to the out vector. + * Example : LASX_MULWL_W_H_128SV(in0, in1, out0) + * in0 3,-1,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1, + * in1 2,-1,1,2, 1,0,0,0, 0,0,1,0, 1,0,0,1, + * out0 6,1,3,0, 0,0,1,0, + */ +#define LASX_MULWL_W_H_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp0_m, _tmp1_m; \ + \ + _tmp0_m = __lasx_xvsllwil_w_h( in0, 0 ); \ + _tmp1_m = __lasx_xvsllwil_w_h( in1, 0 ); \ + out0 = __lasx_xvmul_w( _tmp0_m, _tmp1_m ); \ +} +#define LASX_MULWL_W_H_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_MULWL_W_H_128SV(in0, in1, out0); \ + LASX_MULWL_W_H_128SV(in2, in3, out1); \ +} +#define LASX_MULWL_W_H_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, \ + out0, out1, out2, out3) \ +{ \ + LASX_MULWL_W_H_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_MULWL_W_H_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : Multiplication calculation after expansion + * of the lower half of the vector + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in1 vector and the in0 vector are multiplied after + * the lower half of the two-fold sign extension ( signed + * half word to signed word ) , the stored to the out vector. + * Example : see LASX_MULWL_W_H_128SV + */ +#define LASX_MULWH_W_H_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp0_m, _tmp1_m; \ + \ + _tmp0_m = __lasx_xvilvh_h( in0, in0 ); \ + _tmp1_m = __lasx_xvilvh_h( in1, in1 ); \ + out0 = __lasx_xvmulwev_w_h( _tmp0_m, _tmp1_m ); \ +} +#define LASX_MULWH_W_H_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_MULWH_W_H_128SV(in0, in1, out0); \ + LASX_MULWH_W_H_128SV(in2, in3, out1); \ +} +#define LASX_MULWH_W_H_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, \ + out0, out1, out2, out3) \ +{ \ + LASX_MULWH_W_H_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_MULWH_W_H_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : The low half of the vector elements are expanded and + * added after being doubled + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The in1 vector add the in0 vector after the + * lower half of the two-fold zero extension ( unsigned byte + * to unsigned half word ) and stored to the out vector. + */ +#define LASX_SADDW_HU_HU_BU_128SV(in0, in1, out0) \ +{ \ + __m256i _tmp1_m; \ + __m256i _zero_m = { 0 }; \ + \ + _tmp1_m = __lasx_xvilvl_b( _zero_m, in1 ); \ + out0 = __lasx_xvsadd_hu( in0, _tmp1_m ); \ +} +#define LASX_SADDW_HU_HU_BU_2_128SV(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_SADDW_HU_HU_BU_128SV(in0, in1, out0); \ + LASX_SADDW_HU_HU_BU_128SV(in2, in3, out1); \ +} +#define LASX_SADDW_HU_HU_BU_4_128SV(in0, in1, in2, in3, \ + in4, in5, in6, in7, \ + out0, out1, out2, out3) \ +{ \ + LASX_SADDW_HU_HU_BU_2_128SV(in0, in1, in2, in3, out0, out1); \ + LASX_SADDW_HU_HU_BU_2_128SV(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : Low 8-bit vector elements unsigned extension to halfword + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : Low 8-bit elements from in0 unsigned extension to halfword, + * written to output vector out0. Similar for in1. + * Example : See LASX_UNPCK_L_W_H(in0, out0) + */ +#define LASX_UNPCK_L_HU_BU(in0, out0) \ +{ \ + out0 = __lasx_vext2xv_hu_bu(in0); \ +} + +#define LASX_UNPCK_L_HU_BU_2(in0, in1, out0, out1) \ +{ \ + LASX_UNPCK_L_HU_BU(in0, out0); \ + LASX_UNPCK_L_HU_BU(in1, out1); \ +} + +#define LASX_UNPCK_L_HU_BU_4(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + LASX_UNPCK_L_HU_BU_2(in0, in1, out0, out1); \ + LASX_UNPCK_L_HU_BU_2(in2, in3, out2, out3); \ +} + +#define LASX_UNPCK_L_HU_BU_8(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_UNPCK_L_HU_BU_4(in0, in1, in2, in3, out0, out1, out2, out3); \ + LASX_UNPCK_L_HU_BU_4(in4, in5, in6, in7, out4, out5, out6, out7); \ +} + +/* Description : Low 8-bit vector elements unsigned extension to word + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : Low 8-bit elements from in0 unsigned extension to word, + * written to output vector out0. Similar for in1. + * Example : See LASX_UNPCK_L_W_H(in0, out0) + */ +#define LASX_UNPCK_L_WU_BU(in0, out0) \ +{ \ + out0 = __lasx_vext2xv_wu_bu(in0); \ +} + +#define LASX_UNPCK_L_WU_BU_2(in0, in1, out0, out1) \ +{ \ + LASX_UNPCK_L_WU_BU(in0, out0); \ + LASX_UNPCK_L_WU_BU(in1, out1); \ +} + +#define LASX_UNPCK_L_WU_BU_4(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + LASX_UNPCK_L_WU_BU_2(in0, in1, out0, out1); \ + LASX_UNPCK_L_WU_BU_2(in2, in3, out2, out3); \ +} + +#define LASX_UNPCK_L_WU_BU_8(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_UNPCK_L_WU_BU_4(in0, in1, in2, in3, out0, out1, out2, out3); \ + LASX_UNPCK_L_WU_BU_4(in4, in5, in6, in7, out4, out5, out6, out7); \ +} + +/* Description : Low 8-bit vector elements signed extension to halfword + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : Low 8-bit elements from in0 signed extension to halfword, + * written to output vector out0. Similar for in1. + * Example : See LASX_UNPCK_L_W_H(in0, out0) + */ +#define LASX_UNPCK_L_H_B(in0, out0) \ +{ \ + out0 = __lasx_vext2xv_h_b(in0); \ +} + +#define LASX_UNPCK_L_H_B_2(in0, in1, out0, out1) \ +{ \ + LASX_UNPCK_L_H_B(in0, out0); \ + LASX_UNPCK_L_H_B(in1, out1); \ +} + +#define LASX_UNPCK_L_H_B_4(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + LASX_UNPCK_L_H_B_2(in0, in1, out0, out1); \ + LASX_UNPCK_L_H_B_2(in2, in3, out2, out3); \ +} + +/* Description : Low halfword vector elements signed extension to word + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : Low halfword elements from in0 signed extension to + * word, written to output vector out0. Similar for in1. + * Similar for other pairs. + * Example : LASX_UNPCK_L_W_H(in0, out0) + * in0 : 3, 0, 3, 0, 0, 0, 0, -1, 0, 0, 1, 1, 0, 0, 0, 1 + * out0 : 3, 0, 3, 0, 0, 0, 0, -1 + */ +#define LASX_UNPCK_L_W_H(in0, out0) \ +{ \ + out0 = __lasx_vext2xv_w_h(in0); \ +} + +#define LASX_UNPCK_L_W_H_2(in0, in1, out0, out1) \ +{ \ + LASX_UNPCK_L_W_H(in0, out0); \ + LASX_UNPCK_L_W_H(in1, out1); \ +} + +#define LASX_UNPCK_L_W_H_4(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + LASX_UNPCK_L_W_H_2(in0, in1, out0, out1); \ + LASX_UNPCK_L_W_H_2(in2, in3, out2, out3); \ +} + +#define LASX_UNPCK_L_W_H_8(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_UNPCK_L_W_H_4(in0, in1, in2, in3, out0, out1, out2, out3); \ + LASX_UNPCK_L_W_H_4(in4, in5, in6, in7, out4, out5, out6, out7); \ +} + +/* Description : Interleave odd byte elements from vectors. + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out, out0, ~ + * Details : Odd byte elements of in_h and odd byte + * elements of in_l are interleaved and copied to out. + * Example : See LASX_ILVOD_W(in_h, in_l, out) + */ +#define LASX_ILVOD_B(in_h, in_l, out) \ +{ \ + out = __lasx_xvpackod_b((in_h, in1_l); \ +} + +#define LASX_ILVOD_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVOD_B(in0_h, in0_l, out0); \ + LASX_ILVOD_B(in1_h, in1_l, out1); \ +} + +#define LASX_ILVOD_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3) \ +{ \ + LASX_ILVOD_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVOD_B_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +/* Description : Interleave odd half word elements from vectors. + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out, out0, ~ + * Details : Odd half word elements of in_h and odd half word + * elements of in_l are interleaved and copied to out. + * Example : See LASX_ILVOD_W(in_h, in_l, out) + */ +#define LASX_ILVOD_H(in_h, in_l, out) \ +{ \ + out = __lasx_xvpackod_h(in_h, in_l); \ +} + +#define LASX_ILVOD_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVOD_H(in0_h, in0_l, out0); \ + LASX_ILVOD_H(in1_h, in1_l, out1); \ +} + +#define LASX_ILVOD_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3) \ +{ \ + LASX_ILVOD_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVOD_H_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +/* Description : Interleave odd word elements from vectors. + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out, out0, ~ + * Details : Odd word elements of in_h and odd word + * elements of in_l are interleaved and copied to out. + * Example : See LASX_ILVOD_W(in_h, in_l, out) + * in_h : 1, 2, 3, 4, 5, 6, 7, 8 + * in_l : 1, 0, 3, 1, 1, 2, 3, 4 + * out : 0, 2, 1, 4, 2, 6, 4, 8 + */ +#define LASX_ILVOD_W(in_h, in_l, out) \ +{ \ + out = __lasx_xvpackod_w(in_h, in_l); \ +} + +#define LASX_ILVOD_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVOD_W(in0_h, in0_l, out0); \ + LASX_ILVOD_W(in1_h, in1_l, out1); \ +} + +#define LASX_ILVOD_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3) \ +{ \ + LASX_ILVOD_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVOD_W_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +/* Description : Interleave odd double word elements from vectors. + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out, out0, ~ + * Details : Odd double word elements of in_h and odd double word + * elements of in_l are interleaved and copied to out. + * Example : LASX_ILVOD_W(in_h, in_l, out) + */ +#define LASX_ILVOD_D(in_h, in_l, out) \ +{ \ + out = __lasx_xvpackod_d(in_h, in_l); \ +} + +#define LASX_ILVOD_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVOD_D(in0_h, in0_l, out0); \ + LASX_ILVOD_D(in1_h, in1_l, out1); \ +} + +#define LASX_ILVOD_D_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3) \ +{ \ + LASX_ILVOD_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVOD_D_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +/* Description : Interleave right half of byte elements from vectors + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : Low half of byte elements of in_l and high half of byte + * elements of in_h are interleaved and copied to out0. + * Similar for other pairs + * Example : See LASX_ILVL_W(in_h, in_l, out0) + */ +#define LASX_ILVL_B(in_h, in_l, out0) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_b(in_h, in_l); \ + tmp1 = __lasx_xvilvh_b(in_h, in_l); \ + out0 = __lasx_xvpermi_q(tmp0, tmp1, 0x02); \ +} + +#define LASX_ILVL_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVL_B(in0_h, in0_l, out0) \ + LASX_ILVL_B(in1_h, in1_l, out1) \ +} + +#define LASX_ILVL_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVL_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_ILVL_B_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +#define LASX_ILVL_B_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVL_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVL_B_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave low half of byte elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : Low half of byte elements of in_l and low half of byte + * elements of in_h are interleaved and copied to out0. + * Similar for other pairs + * Example : See LASX_ILVL_W_128SV(in_h, in_l, out0) + */ +#define LASX_ILVL_B_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvilvl_b(in_h, in_l); \ +} + +#define LASX_ILVL_B_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVL_B_128SV(in0_h, in0_l, out0); \ + LASX_ILVL_B_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_ILVL_B_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVL_B_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVL_B_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_ILVL_B_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVL_B_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVL_B_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave low half of half word elements from vectors + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : Low half of half word elements of in_l and right half of + * half word elements of in_h are interleaved and copied to + * out0. Similar for other pairs. + * Example : See LASX_ILVL_W(in_h, in_l, out0) + */ +#define LASX_ILVL_H(in_h, in_l, out0) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_h(in_h, in_l); \ + tmp1 = __lasx_xvilvh_h(in_h, in_l); \ + out0 = __lasx_xvpermi_q(tmp0, tmp1, 0x02); \ +} + +#define LASX_ILVL_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVL_H(in0_h, in0_l, out0) \ + LASX_ILVL_H(in1_h, in1_l, out1) \ +} + +#define LASX_ILVL_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVL_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_ILVL_H_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +#define LASX_ILVL_H_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVL_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVL_H_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave low half of half word elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : Low half of half word elements of in_l and low half of half + * word elements of in_h are interleaved and copied to + * out0. Similar for other pairs. + * Example : See LASX_ILVL_W_128SV(in_h, in_l, out0) + */ +#define LASX_ILVL_H_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvilvl_h(in_h, in_l); \ +} + +#define LASX_ILVL_H_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVL_H_128SV(in0_h, in0_l, out0); \ + LASX_ILVL_H_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_ILVL_H_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVL_H_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVL_H_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_ILVL_H_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVL_H_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVL_H_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave low half of word elements from vectors + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : Low half of halfword elements of in_l and low half of word + * elements of in_h are interleaved and copied to out0. + * Similar for other pairs + * Example : LASX_ILVL_W(in_h, in_l, out0) + * in_h : 0, 1, 0, 1, 0, 1, 0, 1 + * in_l : 1, 2, 3, 4, 5, 6, 7, 8 + * out0 : 1, 0, 2, 1, 3, 0, 4, 1 + */ +#define LASX_ILVL_W(in_h, in_l, out0) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_w(in_h, in_l); \ + tmp1 = __lasx_xvilvh_w(in_h, in_l); \ + out0 = __lasx_xvpermi_q(tmp0, tmp1, 0x02); \ +} + +#define LASX_ILVL_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVL_W(in0_h, in0_l, out0) \ + LASX_ILVL_W(in1_h, in1_l, out1) \ +} + +#define LASX_ILVL_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVL_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_ILVL_W_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +#define LASX_ILVL_W_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVL_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVL_W_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave low half of word elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : Low half of halfword elements of in_l and low half of word + * elements of in_h are interleaved and copied to out0. + * Similar for other pairs + * Example : LASX_ILVL_W_128SV(in_h, in_l, out0) + * in_h : 0, 1, 0, 1, 0, 1, 0, 1 + * in_l : 1, 2, 3, 4, 5, 6, 7, 8 + * out0 : 1, 0, 2, 1, 5, 0, 6, 1 + */ +#define LASX_ILVL_W_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvilvl_w(in_h, in_l); \ +} + +#define LASX_ILVL_W_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVL_W_128SV(in0_h, in0_l, out0); \ + LASX_ILVL_W_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_ILVL_W_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVL_W_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVL_W_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_ILVL_W_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVL_W_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVL_W_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave low half of double word elements from vectors + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : Low half of double word elements of in_l and low half of + * double word elements of in_h are interleaved and copied to out0. + * Similar for other pairs + * Example : See LASX_ILVL_W(in_h, in_l, out0) + */ +#define LASX_ILVL_D(in_h, in_l, out0) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_d(in_h, in_l); \ + tmp1 = __lasx_xvilvh_d(in_h, in_l); \ + out0 = __lasx_xvpermi_q(tmp0, tmp1, 0x02); \ +} + +#define LASX_ILVL_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVL_D(in0_h, in0_l, out0) \ + LASX_ILVL_D(in1_h, in1_l, out1) \ +} + +#define LASX_ILVL_D_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVL_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_ILVL_D_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +#define LASX_ILVL_D_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVL_D_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVL_D_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave right half of double word elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : Right half of double word elements of in_l and right half of + * double word elements of in_h are interleaved and copied to out0. + * Similar for other pairs. + * Example : See LASX_ILVL_W_128SV(in_h, in_l, out0) + */ +#define LASX_ILVL_D_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvilvl_d(in_h, in_l); \ +} + +#define LASX_ILVL_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVL_D_128SV(in0_h, in0_l, out0); \ + LASX_ILVL_D_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_ILVL_D_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVL_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVL_D_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_ILVL_D_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVL_D_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVL_D_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave high half of byte elements from vectors + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : High half of byte elements of in_l and high half of + * byte + * elements of in_h are interleaved and copied to out0. + * Similar for other pairs. + * Example : see LASX_ILVH_W(in_h, in_l, out0) + */ +#define LASX_ILVH_B(in_h, in_l, out0) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_b(in_h, in_l); \ + tmp1 = __lasx_xvilvh_b(in_h, in_l); \ + out0 = __lasx_xvpermi_q(tmp0, tmp1, 0x13); \ +} + +#define LASX_ILVH_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVH_B(in0_h, in0_l, out0) \ + LASX_ILVH_B(in1_h, in1_l, out1) \ +} + +#define LASX_ILVH_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVH_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_ILVH_B_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +#define LASX_ILVH_B_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVH_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVH_B_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave high half of byte elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : High half of byte elements of in_l and high half + * of byte elements of in_h are interleaved and copied + * to out0. + * Similar for other pairs. + * Example : see LASX_ILVH_W_128SV(in_h, in_l, out0) + */ +#define LASX_ILVH_B_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvilvh_b(in_h, in_l); \ +} + +#define LASX_ILVH_B_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVH_B_128SV(in0_h, in0_l, out0); \ + LASX_ILVH_B_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_ILVH_B_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVH_B_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVH_B_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_ILVH_B_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVH_B_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVH_B_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave high half of half word elements from vectors + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : High half of half word elements of in_l and high half of + * half word + * elements of in_h are interleaved and copied to out0. + * Similar for other pairs. + * Example : see LASX_ILVH_W(in_h, in_l, out0) + */ +#define LASX_ILVH_H(in_h, in_l, out0) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_h(in_h, in_l); \ + tmp1 = __lasx_xvilvh_h(in_h, in_l); \ + out0 = __lasx_xvpermi_q(tmp0, tmp1, 0x13); \ +} + +#define LASX_ILVH_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVH_H(in0_h, in0_l, out0) \ + LASX_ILVH_H(in1_h, in1_l, out1) \ +} + +#define LASX_ILVH_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVH_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_ILVH_H_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +#define LASX_ILVH_H_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVH_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVH_H_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave high half of half word elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : High half of half word elements of in_l and high half + * of half word elements of in_h are interleaved and copied + * to out0. + * Similar for other pairs. + * Example : see LASX_ILVH_W_128SV(in_h, in_l, out0) + */ +#define LASX_ILVH_H_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvilvh_h(in_h, in_l); \ +} + +#define LASX_ILVH_H_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVH_H_128SV(in0_h, in0_l, out0); \ + LASX_ILVH_H_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_ILVH_H_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVH_H_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVH_H_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_ILVH_H_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVH_H_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVH_H_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave high half of word elements from vectors + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : High half of word elements of in_l and high half of + * word elements of in_h are interleaved and copied to + * out0. + * Similar for other pairs. + * Example : LASX_ILVH_W(in_h, in_l, out0) + * in_h:-1, -2, -3, -4, -5, -6, -7, -8 + * in_l: 1, 2, 3, 4, 5, 6, 7, 8 + * out0: 5, -5, 6, -6, 7, -7, 8, -8 + */ +#define LASX_ILVH_W(in_h, in_l, out0) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_w(in_h, in_l); \ + tmp1 = __lasx_xvilvh_w(in_h, in_l); \ + out0 = __lasx_xvpermi_q(tmp0, tmp1, 0x13); \ +} + +#define LASX_ILVH_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVH_W(in0_h, in0_l, out0) \ + LASX_ILVH_W(in1_h, in1_l, out1) \ +} + +#define LASX_ILVH_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVH_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_ILVH_W_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +#define LASX_ILVH_W_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVH_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVH_W_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave high half of word elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0, out1, ~ + * Details : High half of word elements of every 128-bit of in_l + * and high half of word elements of every 128-bit of + * in_h are interleaved and copied to out0. + * Similar for other pairs. + * Example : LASX_ILVH_W_128SV(in_h, in_l, out0) + * in_h:-1, -2, -3, -4, -5, -6, -7, -8 + * in_l: 1, 2, 3, 4, 5, 6, 7, 8 + * out0: 3, -3, 4, -4, 7, -7, 8, -8* + */ +#define LASX_ILVH_W_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvilvh_w(in_h, in_l); \ +} + +#define LASX_ILVH_W_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVH_W_128SV(in0_h, in0_l, out0); \ + LASX_ILVH_W_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_ILVH_W_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVH_W_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVH_W_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_ILVH_W_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVH_W_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVH_W_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave high half of double word elements from vectors + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + :* Details : High half of double word elements of in_l and high half of + * double word elements of in_h are interleaved and copied to + * out0. + * Similar for other pairs. + * Example : see LASX_ILVH_W(in_h, in_l, out0) + */ +#define LASX_ILVH_D(in_h, in_l, out0) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_d(in_h, in_l); \ + tmp1 = __lasx_xvilvh_d(in_h, in_l); \ + out0 = __lasx_xvpermi_q(tmp0, tmp1, 0x13); \ +} + +#define LASX_ILVH_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVH_D(in0_h, in0_l, out0) \ + LASX_ILVH_D(in1_h, in1_l, out1) \ +} + +#define LASX_ILVH_D_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVH_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_ILVH_D_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +#define LASX_ILVH_D_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVH_D_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVH_D_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave high half of double word elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : High half of double word elements of every 128-bit in_l and + * high half of double word elements of every 128-bit in_h are + * interleaved and copied to out0. + * Similar for other pairs. + * Example : see LASX_ILVH_W_128SV(in_h, in_l, out0) + */ +#define LASX_ILVH_D_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvilvh_d(in_h, in_l); \ +} + +#define LASX_ILVH_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_ILVH_D_128SV(in0_h, in0_l, out0); \ + LASX_ILVH_D_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_ILVH_D_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_ILVH_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_ILVH_D_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_ILVH_D_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ILVH_D_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0, out1, out2, out3); \ + LASX_ILVH_D_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4, out5, out6, out7); \ +} + +/* Description : Interleave byte elements from vectors + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out_h, out_l, ~ + * Details : Low half of byte elements of in_l and low half of byte + * elements of in_h are interleaved and copied to out_l. + * High half of byte elements of in_l and high half of byte + * elements of in_h are interleaved and copied to out_h. + * Similar for other pairs. + * Example : see LASX_ILVLH_W(in_h, in_l, out_l, out_h) + */ +#define LASX_ILVLH_B(in_h, in_l, out_h, out_l) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_b(in_h, in_l); \ + tmp1 = __lasx_xvilvh_b(in_h, in_l); \ + out_l = __lasx_xvpermi_q(tmp0, tmp1, 0x02); \ + out_h = __lasx_xvpermi_q(tmp0, tmp1, 0x13); \ +} + +#define LASX_ILVLH_B_2(in0_h, in0_l, in1_h, in1_l, \ + out0_h, out0_l, out1_h, out1_l) \ +{ \ + LASX_ILVLH_B(in0_h, in0_l, out0_h, out0_l); \ + LASX_ILVLH_B(in1_h, in1_l, out1_h, out1_l); \ +} + +#define LASX_ILVLH_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l) \ +{ \ + LASX_ILVLH_B_2(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l); \ + LASX_ILVLH_B_2(in2_h, in2_l, in3_h, in3_l, out2_h, out2_l, out3_h, out3_l); \ +} + +#define LASX_ILVLH_B_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l) \ +{ \ + LASX_ILVLH_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l); \ + LASX_ILVLH_B_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l); \ +} + +/* Description : Interleave byte elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out_h, out_l, ~ + * Details : Low half of byte elements of in_l and low half of byte elements + * of in_h are interleaved and copied to out_l. High half of byte + * elements of in_h and high half of byte elements of in_l are + * interleaved and copied to out_h. + * Similar for other pairs. + * Example : see LASX_ILVLH_W_128SV(in_h, in_l, out_h, out_l) + */ +#define LASX_ILVLH_B_128SV(in_h, in_l, out_h, out_l) \ +{ \ + LASX_ILVL_B_128SV(in_h, in_l, out_l); \ + LASX_ILVH_B_128SV(in_h, in_l, out_h); \ +} + +#define LASX_ILVLH_B_2_128SV(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l) \ +{ \ + LASX_ILVLH_B_128SV(in0_h, in0_l, out0_h, out0_l); \ + LASX_ILVLH_B_128SV(in1_h, in1_l, out1_h, out1_l); \ +} + +#define LASX_ILVLH_B_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l) \ +{ \ + LASX_ILVLH_B_2_128SV(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l); \ + LASX_ILVLH_B_2_128SV(in2_h, in2_l, in3_h, in3_l, out2_h, out2_l, out3_h, out3_l); \ +} + +#define LASX_ILVLH_B_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l) \ +{ \ + LASX_ILVLH_B_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l); \ + LASX_ILVLH_B_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l); \ +} + +/* Description : Interleave half word elements from vectors + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out_h, out_l, ~ + * Details : Low half of half word elements of in_l and low half of half + * word elements of in_h are interleaved and copied to out_l. + * High half of half word elements of in_l and high half of half + * word elements of in_h are interleaved and copied to out_h. + * Similar for other pairs. + * Example : see LASX_ILVLH_W(in_h, in_l, out_h, out_l) + */ +#define LASX_ILVLH_H(in_h, in_l, out_h, out_l) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_h(in_h, in_l); \ + tmp1 = __lasx_xvilvh_h(in_h, in_l); \ + out_l = __lasx_xvpermi_q(tmp0, tmp1, 0x02); \ + out_h = __lasx_xvpermi_q(tmp0, tmp1, 0x13); \ +} + +#define LASX_ILVLH_H_2(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, \ + out1_h, out1_l) \ +{ \ + LASX_ILVLH_H(in0_h, in0_l, out0_h, out0_l); \ + LASX_ILVLH_H(in1_h, in1_l, out1_h, out1_l); \ +} + +#define LASX_ILVLH_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l) \ +{ \ + LASX_ILVLH_H_2(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l); \ + LASX_ILVLH_H_2(in2_h, in2_l, in3_h, in3_l, out2_h, out2_l, out3_h, out3_l); \ +} + +#define LASX_ILVLH_H_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l) \ +{ \ + LASX_ILVLH_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l); \ + LASX_ILVLH_H_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l); \ +} + +/* Description : Interleave half word elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0_h, out0_l, ~ + * Details : Low half of half word elements of every 128-bit of in_l and + * low half of half word elements of every 128-bit of in_h are + * interleaved and copied to out_l. + * High half of half word elements of every 128-bit of in_l and + * high half of half word elements of every 128-bit of in_h are + * interleaved and copied to out_h. + * Similar for other pairs. + * Example : see LASX_ILVLH_W_128SV(in_h, in_l, out_h, out_l) + */ +#define LASX_ILVLH_H_128SV(in_h, in_l, out_h, out_l) \ +{ \ + LASX_ILVL_H_128SV(in_h, in_l, out_l); \ + LASX_ILVH_H_128SV(in_h, in_l, out_h); \ +} + +#define LASX_ILVLH_H_2_128SV(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l) \ +{ \ + LASX_ILVLH_H_128SV(in0_h, in0_l, out0_h, out0_l); \ + LASX_ILVLH_H_128SV(in1_h, in1_l, out1_h, out1_l); \ +} + +#define LASX_ILVLH_H_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l) \ +{ \ + LASX_ILVLH_H_2_128SV(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l); \ + LASX_ILVLH_H_2_128SV(in2_h, in2_l, in3_h, in3_l, out2_h, out2_l, out3_h, out3_l); \ +} + +#define LASX_ILVLH_H_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l) \ +{ \ + LASX_ILVLH_H_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l); \ + LASX_ILVLH_H_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l); \ +} + +/* Description : Interleave word elements from vectors + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out_h, out_l, ~ + * Details : Low half of word elements of in_l and low half of word + * elements of in_h are interleaved and copied to out_l. + * High half of word elements of in_l and high half of word + * elements of in_h are interleaved and copied to out_h. + * Similar for other pairs. + * Example : LASX_ILVLH_W(in_h, in_l, out_h, out_l) + * in_h:-1, -2, -3, -4, -5, -6, -7, -8 + * in_l: 1, 2, 3, 4, 5, 6, 7, 8 + * out_h: 5, -5, 6, -6, 7, -7, 8, -8 + * out_l: 1, -1, 2, -2, 3, -3, 4, -4 + */ +#define LASX_ILVLH_W(in_h, in_l, out_h, out_l) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_w(in_h, in_l); \ + tmp1 = __lasx_xvilvh_w(in_h, in_l); \ + out_l = __lasx_xvpermi_q(tmp0, tmp1, 0x02); \ + out_h = __lasx_xvpermi_q(tmp0, tmp1, 0x13); \ +} + +#define LASX_ILVLH_W_2(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, \ + out1_h, out1_l) \ +{ \ + LASX_ILVLH_W(in0_h, in0_l, out0_h, out0_l); \ + LASX_ILVLH_W(in1_h, in1_l, out1_h, out1_l); \ +} + +#define LASX_ILVLH_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l) \ +{ \ + LASX_ILVLH_W_2(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l); \ + LASX_ILVLH_W_2(in2_h, in2_l, in3_h, in3_l, out2_h, out2_l, out3_h, out3_l); \ +} + +#define LASX_ILVLH_W_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l) \ +{ \ + LASX_ILVLH_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l); \ + LASX_ILVLH_W_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l); \ +} + +/* Description : Interleave word elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in0_h, in0_l, ~ + * Outputs - out0_h, out0_l, ~ + * Details : Low half of word elements of every 128-bit of in_l and + * low half of word elements of every 128-bit of in_h are + * interleaved and copied to out_l. + * High half of word elements of every 128-bit of in_l and + * high half of word elements of every 128-bit of in_h are + * interleaved and copied to out_h. + * Similar for other pairs. + * Example : LASX_ILVLH_W_128SV(in_h, in_l, out_h, out_l) + * in_h:-1, -2, -3, -4, -5, -6, -7, -8 + * in_l: 1, 2, 3, 4, 5, 6, 7, 8 + * out_h: 3, -3, 4, -4, 7, -7, 8, -8 + * out_l: 1, -1, 2, -2, 5, -5, 6, -6 + */ +#define LASX_ILVLH_W_128SV(in_h, in_l, out_h, out_l) \ +{ \ + LASX_ILVL_W_128SV(in_h, in_l, out_l); \ + LASX_ILVH_W_128SV(in_h, in_l, out_h); \ +} + +#define LASX_ILVLH_W_2_128SV(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l) \ +{ \ + LASX_ILVLH_W_128SV(in0_h, in0_l, out0_h, out0_l); \ + LASX_ILVLH_W_128SV(in1_h, in1_l, out1_h, out1_l); \ +} + +#define LASX_ILVLH_W_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l) \ +{ \ + LASX_ILVLH_W_2_128SV(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l); \ + LASX_ILVLH_W_2_128SV(in2_h, in2_l, in3_h, in3_l, out2_h, out2_l, out3_h, out3_l); \ +} + +#define LASX_ILVLH_W_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l) \ +{ \ + LASX_ILVLH_W_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l); \ + LASX_ILVLH_W_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l); \ +} + +/* Description : Interleave double word elements from vectors + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out_h, out_l, ~ + * Details : Low half of double word elements of in_l and low half of + * double word elements of in_h are interleaved and copied to + * out_l. High half of double word elements of in_l and high + * half of double word elements of in_h are interleaved and + * copied to out_h. + * Similar for other pairs. + * Example : see LASX_ILVLH_W(in_h, in_l, out_h, out_l) + */ +#define LASX_ILVLH_D(in_h, in_l, out_h, out_l) \ +{ \ + __m256i tmp0, tmp1; \ + tmp0 = __lasx_xvilvl_d(in_h, in_l); \ + tmp1 = __lasx_xvilvh_d(in_h, in_l); \ + out_l = __lasx_xvpermi_q(tmp0, tmp1, 0x02); \ + out_h = __lasx_xvpermi_q(tmp0, tmp1, 0x13); \ +} + +#define LASX_ILVLH_D_2(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, \ + out1_h, out1_l) \ +{ \ + LASX_ILVLH_D(in0_h, in0_l, out0_h, out0_l); \ + LASX_ILVLH_D(in1_h, in1_l, out1_h, out1_l); \ +} + +#define LASX_ILVLH_D_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l) \ +{ \ + LASX_ILVLH_D_2(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l); \ + LASX_ILVLH_D_2(in2_h, in2_l, in3_h, in3_l, out2_h, out2_l, out3_h, out3_l); \ +} + +#define LASX_ILVLH_D_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l) \ +{ \ + LASX_ILVLH_D_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l); \ + LASX_ILVLH_D_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l); \ +} + +/* Description : Interleave double word elements from vectors + * (128-bit symmetry version) + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out_h, out_l, ~ + * Details : Low half of double word elements of every 128-bit of in_l and + * low half of double word elements of every 128-bit of in_h are + * interleaved and copied to out_l. + * High half of double word elements of every 128-bit of in_l and + * high half of double word elements of every 128-bit of in_h are + * interleaved and copied to out_h. + * Similar for other pairs. + * Example : see LASX_ILVLH_W_128SV(in_h, in_l, out_h, out_l) + */ +#define LASX_ILVLH_D_128SV(in_h, in_l, out_h, out_l) \ +{ \ + LASX_ILVL_D_128SV(in_h, in_l, out_l); \ + LASX_ILVH_D_128SV(in_h, in_l, out_h); \ +} + +#define LASX_ILVLH_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l) \ +{ \ + LASX_ILVLH_D_128SV(in0_h, in0_l, out0_h, out0_l); \ + LASX_ILVLH_D_128SV(in1_h, in1_l, out1_h, out1_l); \ +} + +#define LASX_ILVLH_D_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l) \ +{ \ + LASX_ILVLH_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0_h, out0_l, out1_h, out1_l); \ + LASX_ILVLH_D_2_128SV(in2_h, in2_l, in3_h, in3_l, out2_h, out2_l, out3_h, out3_l); \ +} + +#define LASX_ILVLH_D_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l) \ +{ \ + LASX_ILVLH_D_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + out0_h, out0_l, out1_h, out1_l, out2_h, out2_l, out3_h, out3_l); \ + LASX_ILVLH_D_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out4_h, out4_l, out5_h, out5_l, out6_h, out6_l, out7_h, out7_l); \ +} + +/* Description : Immediate number of columns to slide with zero + * Arguments : Inputs - in0, in1, slide_val, ~ + * Outputs - out0, out1, ~ + * Details : Byte elements from every 128-bit of in0 vector + * are slide into out0 by number of elements + * specified by slide_val. + * Example : LASX_SLDI_B_0_128SV(in0, out0, slide_val) + * in0: 1, 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18, + * 19,20,21,22,23,24,25,26,27,28,29,30,31,32 + * out0: 4, 5,6,7,8,9,10,11,12,13,14,15,16,0,0,0,20,21, + * 22,23,24,25,26,27,28,29,30,31,32,0,0,0 + * slide_val: 3 + */ +#define LASX_SLDI_B_0_128SV(in0, out0, slide_val) \ +{ \ + out0 = __lasx_xvbsrl_v(in0, slide_val); \ +} + +#define LASX_SLDI_B_2_0_128SV(in0, in1, out0, out1, slide_val) \ +{ \ + LASX_SLDI_B_0_128SV(in0, out0, slide_val); \ + LASX_SLDI_B_0_128SV(in1, out1, slide_val); \ +} + +#define LASX_SLDI_B_4_0_128SV(in0, in1, in2, in3, \ + out0, out1, out2, out3, slide_val) \ +{ \ + LASX_SLDI_B_2_0_128SV(in0, in1, out0, out1, slide_val); \ + LASX_SLDI_B_2_0_128SV(in2, in3, out2, out3, slide_val); \ +} + +/* Description : Pack even byte elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Even byte elements of in_l are copied to the low half of + * out0. Even byte elements of in_h are copied to the high + * half of out0. + * Similar for other pairs. + * Example : see LASX_PCKEV_W(in_h, in_l, out0) + */ +#define LASX_PCKEV_B(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickev_b(in_h, in_l); \ + out0 = __lasx_xvpermi_d(out0, 0xd8); \ +} + +#define LASX_PCKEV_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKEV_B(in0_h, in0_l, out0); \ + LASX_PCKEV_B(in1_h, in1_l, out1); \ +} + +#define LASX_PCKEV_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKEV_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKEV_B_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKEV_B_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKEV_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKEV_B_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack even byte elements of vector pairs + * (128-bit symmetry version) + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Even byte elements of in_l are copied to the low half of + * out0. Even byte elements of in_h are copied to the high + * half of out0. + * Similar for other pairs. + * Example : see LASX_PCKEV_W_128SV(in_h, in_l, out0) + */ +#define LASX_PCKEV_B_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickev_b(in_h, in_l); \ +} + +#define LASX_PCKEV_B_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKEV_B_128SV(in0_h, in0_l, out0); \ + LASX_PCKEV_B_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_PCKEV_B_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKEV_B_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKEV_B_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKEV_B_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, in4_h, in4_l, in5_h, in5_l, \ + in6_h, in6_l, in7_h, in7_l, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKEV_B_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKEV_B_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack even half word elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Even half word elements of in_l are copied to the low + * half of out0. Even half word elements of in_h are + * copied to the high half of out0. + * Example : see LASX_PCKEV_W(in_h, in_l, out0) + */ +#define LASX_PCKEV_H(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickev_h(in_h, in_l); \ + out0 = __lasx_xvpermi_d(out0, 0xd8); \ +} + +#define LASX_PCKEV_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKEV_H(in0_h, in0_l, out0); \ + LASX_PCKEV_H(in1_h, in1_l, out1); \ +} + +#define LASX_PCKEV_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKEV_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKEV_H_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKEV_H_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKEV_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKEV_H_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack even half word elements of vector pairs + * (128-bit symmetry version) + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Even half word elements of in_l are copied to the low + * half of out0. Even half word elements of in_h are + * copied to the high half of out0. + * Example : see LASX_PCKEV_W_128SV(in_h, in_l, out0) + */ +#define LASX_PCKEV_H_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickev_h(in_h, in_l); \ +} + +#define LASX_PCKEV_H_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKEV_H_128SV(in0_h, in0_l, out0); \ + LASX_PCKEV_H_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_PCKEV_H_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKEV_H_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKEV_H_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKEV_H_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, in4_h, in4_l, in5_h, in5_l, \ + in6_h, in6_l, in7_h, in7_l, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKEV_H_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKEV_H_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack even word elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Even word elements of in_l are copied to + * the low half of out0. Even word elements + * of in_h are copied to the high half of out0. + * Example : LASX_PCKEV_W(in_h, in_l, out0) + * in_h: -1, -2, -3, -4, -5, -6, -7, -8 + * in_l: 1, 2, 3, 4, 5, 6, 7, 8 + * out0: 1, 3, 5, 7, -1, -3, -5, -7 + */ +#define LASX_PCKEV_W(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickev_w(in_h, in_l); \ + out0 = __lasx_xvpermi_d(out0, 0xd8); \ +} + +#define LASX_PCKEV_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKEV_W(in0_h, in0_l, out0); \ + LASX_PCKEV_W(in1_h, in1_l, out1); \ +} + +#define LASX_PCKEV_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKEV_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKEV_W_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKEV_W_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKEV_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKEV_W_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack even word elements of vector pairs + * (128-bit symmetry version) + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Even word elements of in_l are copied to + * the low half of out0. Even word elements + * of in_h are copied to the high half of out0. + * Example : LASX_PCKEV_W_128SV(in_h, in_l, out0) + * in_h: -1, -2, -3, -4, -5, -6, -7, -8 + * in_l: 1, 2, 3, 4, 5, 6, 7, 8 + * out0: 1, 3, -1, -3, 5, 7, -5, -7 + */ +#define LASX_PCKEV_W_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickev_w(in_h, in_l); \ +} + +#define LASX_PCKEV_W_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKEV_W_128SV(in0_h, in0_l, out0); \ + LASX_PCKEV_W_128SV(in1_h, in1_l, out1); \ +} + +#define LASX_PCKEV_W_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKEV_W_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKEV_W_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKEV_W_8_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, in4_h, in4_l, in5_h, in5_l, \ + in6_h, in6_l, in7_h, in7_l, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKEV_W_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKEV_W_4_128SV(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack even half word elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Even half word elements of in_l are copied to the low + * half of out0. Even half word elements of in_h are + * copied to the high half of out0. + * Example : See LASX_PCKEV_W(in_h, in_l, out0) + */ +#define LASX_PCKEV_D(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickev_d(in_h, in_l); \ + out0 = __lasx_xvpermi_d(out0, 0xd8); \ +} + +#define LASX_PCKEV_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKEV_D(in0_h, in0_l, out0) \ + LASX_PCKEV_D(in1_h, in1_l, out1) \ +} + +#define LASX_PCKEV_D_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKEV_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_PCKEV_D_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +/* Description : Pack even half word elements of vector pairs + * (128-bit symmetry version) + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Even half word elements of in_l are copied to the low + * half of out0. Even half word elements of in_h are + * copied to the high half of out0. + * Example : LASX_PCKEV_D_128SV(in_h, in_l, out0) + * in_h : 1, 2, 3, 4 + * in_l : 5, 6, 7, 8 + * out0 : 5, 1, 7, 3 + */ +#define LASX_PCKEV_D_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickev_d(in_h, in_l); \ +} + +#define LASX_PCKEV_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKEV_D_128SV(in0_h, in0_l, out0) \ + LASX_PCKEV_D_128SV(in1_h, in1_l, out1) \ +} + +#define LASX_PCKEV_D_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKEV_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_PCKEV_D_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +/* Description : Pack even quad word elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Even quad elements of in_l are copied to the low + * half of out0. Even quad elements of in_h are + * copied to the high half of out0. + * Similar for other pairs. + * Example : see LASX_PCKEV_W(in_h, in_l, out0) + */ +#define LASX_PCKEV_Q(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpermi_q(in_h, in_l, 0x20); \ +} + +#define LASX_PCKEV_Q_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKEV_Q(in0_h, in0_l, out0); \ + LASX_PCKEV_Q(in1_h, in1_l, out1); \ +} + +#define LASX_PCKEV_Q_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKEV_Q_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKEV_Q_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKEV_Q_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKEV_Q_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKEV_Q_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack odd byte elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Odd byte elements of in_l are copied to the low half of + * out0. Odd byte elements of in_h are copied to the high + * half of out0. + * Similar for other pairs. + * Example : see LASX_PCKOD_W(in_h, in_l, out0) + */ +#define LASX_PCKOD_B(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickod_b(in_h, in_l); \ + out0 = __lasx_xvpermi_d(out0, 0xd8); \ +} + +#define LASX_PCKOD_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKOD_B(in0_h, in0_l, out0); \ + LASX_PCKOD_B(in1_h, in1_l, out1); \ +} + +#define LASX_PCKOD_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKOD_B_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKOD_B_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKOD_B_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKOD_B_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKOD_B_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack odd half word elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Odd half word elements of in_l are copied to the low + * half of out0. Odd half word elements of in_h are copied + * to the high half of out0. + * Example : see LASX_PCKOD_W(in_h, in_l, out0) + */ +#define LASX_PCKOD_H(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickod_h(in_h, in_l); \ + out0 = __lasx_xvpermi_d(out0, 0xd8); \ +} + +#define LASX_PCKOD_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKOD_H(in0_h, in0_l, out0); \ + LASX_PCKOD_H(in1_h, in1_l, out1); \ +} + +#define LASX_PCKOD_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKOD_H_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKOD_H_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKOD_H_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKOD_H_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKOD_H_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack odd word elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Odd word elements of in_l are copied to the low half of out0. + * Odd word elements of in_h are copied to the high half of out0. + * Example : LASX_PCKOD_W(in_h, in_l, out0) + * in_h: -1, -2, -3, -4, -5, -6, -7, -8 + * in_l: 1, 2, 3, 4, 5, 6, 7, 8 + * out0: 2, 4, 6, 8, -2, -4, -6, -8 + */ +#define LASX_PCKOD_W(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickod_w(in_h, in_l); \ + out0 = __lasx_xvpermi_d(out0, 0xd8); \ +} + +#define LASX_PCKOD_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKOD_W(in0_h, in0_l, out0); \ + LASX_PCKOD_W(in1_h, in1_l, out1); \ +} + +#define LASX_PCKOD_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKOD_W_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKOD_W_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKOD_W_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKOD_W_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKOD_W_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack odd half word elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Odd half word elements of in_l are copied to the low + * half of out0. Odd half word elements of in_h are + * copied to the high half of out0. + * Example : See LASX_PCKOD_W(in_h, in_l, out0) + */ +#define LASX_PCKOD_D(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickod_d(in_h, in_l); \ + out0 = __lasx_xvpermi_d(out0, 0xd8); \ +} + +#define LASX_PCKOD_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKOD_D(in0_h, in0_l, out0) \ + LASX_PCKOD_D(in1_h, in1_l, out1) \ +} + +#define LASX_PCKOD_D_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKOD_D_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_PCKOD_D_2(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + +/* Description : Pack odd quad word elements of vector pairs + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Odd quad elements of in0_h are copied to the high half of + * out0 & odd quad elements of in0_l are copied to the low + * half of out0. + * Odd quad elements of in1_h are copied to the high half of + * out1 & odd quad elements of in1_l are copied to the low + * half of out1. + * LASX_PCKOD_Q(in_h, in_l, out0) + * in_h: 0,0,0,0, 0,0,0,0, 19,10,11,12, 13,14,15,16 + * in_l: 0,0,0,0, 0,0,0,0, 1,2,3,4, 5,6,7,8 + * out0: 1,2,3,4, 5,6,7,8, 19,10,11,12, 13,14,15,16 + */ +#define LASX_PCKOD_Q(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpermi_q(in_h, in_l, 0x31); \ +} + +#define LASX_PCKOD_Q_2(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKOD_Q(in0_h, in0_l, out0); \ + LASX_PCKOD_Q(in1_h, in1_l, out1); \ +} + +#define LASX_PCKOD_Q_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKOD_Q_2(in0_h, in0_l, in1_h, in1_l, out0, out1); \ + LASX_PCKOD_Q_2(in2_h, in2_l, in3_h, in3_l, out2, out3); \ +} + +#define LASX_PCKOD_Q_8(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, in3_h, in3_l, \ + in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, in7_h, in7_l, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_PCKOD_Q_4(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3); \ + LASX_PCKOD_Q_4(in4_h, in4_l, in5_h, in5_l, in6_h, in6_l, \ + in7_h, in7_l, out4, out5, out6, out7); \ +} + +/* Description : Pack odd half word elements of vector pairsi + * (128-bit symmetry version) + * Arguments : Inputs - in_h, in_l, ~ + * Outputs - out0, out1, ~ + * Details : Odd half word elements of in_l are copied to the low + * half of out0 of . Odd half word elements of in_h are + * copied to the high half of out0. + * Example : LASX_PCKOD_D_128SV(in_h, in_l, out0) + * in_h : 1, 2, 3, 4 + * in_l : 5, 6, 7, 8 + * out0 : 6, 2, 8, 4 + */ +#define LASX_PCKOD_D_128SV(in_h, in_l, out0) \ +{ \ + out0 = __lasx_xvpickod_d(in_h, in_l); \ +} + +#define LASX_PCKOD_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ +{ \ + LASX_PCKOD_D_128SV(in0_h, in0_l, out0) \ + LASX_PCKOD_D_128SV(in1_h, in1_l, out1) \ +} + +#define LASX_PCKOD_D_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, out0, out1, out2, out3) \ +{ \ + LASX_PCKOD_D_2_128SV(in0_h, in0_l, in1_h, in1_l, out0, out1) \ + LASX_PCKOD_D_2_128SV(in2_h, in2_l, in3_h, in3_l, out2, out3) \ +} + + +/* Description : Transposes 8x8 block with half word elements in vectors. + * Arguments : Inputs - in0, in1, ~ + * Outputs - out0, out1, ~ + * Details : The rows of the matrix become columns, and the columns become rows. + * Example : LASX_TRANSPOSE8x8_H_128SV + * in0 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in1 : 8,2,3,4, 5,6,7,8, 8,2,3,4, 5,6,7,8 + * in2 : 8,2,3,4, 5,6,7,8, 8,2,3,4, 5,6,7,8 + * in3 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in4 : 9,2,3,4, 5,6,7,8, 9,2,3,4, 5,6,7,8 + * in5 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in6 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in7 : 9,2,3,4, 5,6,7,8, 9,2,3,4, 5,6,7,8 + * + * out0 : 1,8,8,1, 9,1,1,9, 1,8,8,1, 9,1,1,9 + * out1 : 2,2,2,2, 2,2,2,2, 2,2,2,2, 2,2,2,2 + * out2 : 3,3,3,3, 3,3,3,3, 3,3,3,3, 3,3,3,3 + * out3 : 4,4,4,4, 4,4,4,4, 4,4,4,4, 4,4,4,4 + * out4 : 5,5,5,5, 5,5,5,5, 5,5,5,5, 5,5,5,5 + * out5 : 6,6,6,6, 6,6,6,6, 6,6,6,6, 6,6,6,6 + * out6 : 7,7,7,7, 7,7,7,7, 7,7,7,7, 7,7,7,7 + * out7 : 8,8,8,8, 8,8,8,8, 8,8,8,8, 8,8,8,8 + */ +#define LASX_TRANSPOSE8x8_H_128SV(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + __m256i s0_m, s1_m; \ + __m256i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + __m256i tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ + \ + LASX_ILVL_H_2_128SV(in6, in4, in7, in5, s0_m, s1_m); \ + LASX_ILVLH_H_128SV(s1_m, s0_m, tmp1_m, tmp0_m); \ + LASX_ILVH_H_2_128SV(in6, in4, in7, in5, s0_m, s1_m); \ + LASX_ILVLH_H_128SV(s1_m, s0_m, tmp3_m, tmp2_m); \ + \ + LASX_ILVL_H_2_128SV(in2, in0, in3, in1, s0_m, s1_m); \ + LASX_ILVLH_H_128SV(s1_m, s0_m, tmp5_m, tmp4_m); \ + LASX_ILVH_H_2_128SV(in2, in0, in3, in1, s0_m, s1_m); \ + LASX_ILVLH_H_128SV(s1_m, s0_m, tmp7_m, tmp6_m); \ + \ + LASX_PCKEV_D_4_128SV(tmp0_m, tmp4_m, tmp1_m, tmp5_m, tmp2_m, tmp6_m, \ + tmp3_m, tmp7_m, out0, out2, out4, out6); \ + LASX_PCKOD_D_4_128SV(tmp0_m, tmp4_m, tmp1_m, tmp5_m, tmp2_m, tmp6_m, \ + tmp3_m, tmp7_m, out1, out3, out5, out7); \ +} + +/* Description : Transposes 8x8 block with word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * Details : + */ +#define LASX_TRANSPOSE8x8_W(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + __m256i s0_m, s1_m; \ + __m256i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + __m256i tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ + \ + LASX_ILVL_W_2_128SV(in2, in0, in3, in1, s0_m, s1_m); \ + LASX_ILVLH_W_128SV(s1_m, s0_m, tmp1_m, tmp0_m); \ + LASX_ILVH_W_2_128SV(in2, in0, in3, in1, s0_m, s1_m); \ + LASX_ILVLH_W_128SV(s1_m, s0_m, tmp3_m, tmp2_m); \ + \ + LASX_ILVL_W_2_128SV(in6, in4, in7, in5, s0_m, s1_m); \ + LASX_ILVLH_W_128SV(s1_m, s0_m, tmp5_m, tmp4_m); \ + LASX_ILVH_W_2_128SV(in6, in4, in7, in5, s0_m, s1_m); \ + LASX_ILVLH_W_128SV(s1_m, s0_m, tmp7_m, tmp6_m); \ + LASX_PCKEV_Q_4(tmp4_m, tmp0_m, tmp5_m, tmp1_m, tmp6_m, tmp2_m, \ + tmp7_m, tmp3_m, out0, out1, out2, out3); \ + LASX_PCKOD_Q_4(tmp4_m, tmp0_m, tmp5_m, tmp1_m, tmp6_m, tmp2_m, \ + tmp7_m, tmp3_m, out4, out5, out6, out7); \ +} + +/* Description : Transposes 2x2 block with quad word elements in vectors + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Details : + */ +#define LASX_TRANSPOSE2x2_Q(in0, in1, out0, out1) \ +{ \ + __m256i tmp0; \ + tmp0 = __lasx_xvpermi_q(in1, in0, 0x02); \ + out1 = __lasx_xvpermi_q(in1, in0, 0x13); \ + out0 = tmp0; \ +} + +/* Description : Transposes 4x4 block with double word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : + */ +#define LASX_TRANSPOSE4x4_D(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + __m256i tmp0, tmp1, tmp2, tmp3; \ + LASX_ILVLH_D_2_128SV(in1, in0, in3, in2, tmp0, tmp1, tmp2, tmp3); \ + out0 = __lasx_xvpermi_q(tmp2, tmp0, 0x20); \ + out2 = __lasx_xvpermi_q(tmp2, tmp0, 0x31); \ + out1 = __lasx_xvpermi_q(tmp3, tmp1, 0x20); \ + out3 = __lasx_xvpermi_q(tmp3, tmp1, 0x31); \ +} + +/* Description : Transpose 4x4 block with half word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Return Type - signed halfword + */ +#define LASX_TRANSPOSE4x4_H_128SV(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + __m256i s0_m, s1_m; \ + \ + LASX_ILVL_H_2_128SV(in1, in0, in3, in2, s0_m, s1_m); \ + LASX_ILVLH_W_128SV(s1_m, s0_m, out2, out0); \ + out1 = __lasx_xvilvh_d(out0, out0); \ + out3 = __lasx_xvilvh_d(out2, out2); \ +} + +/* Description : Transposes input 8x8 byte block + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + * (input 8x8 byte block) + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * (output 8x8 byte block) + * Details : + */ +#define LASX_TRANSPOSE8x8_B(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + __m256i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + __m256i tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ + LASX_ILVL_B_4_128SV(in2, in0, in3, in1, in6, in4, in7, in5, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ + LASX_ILVLH_B_128SV(tmp1_m, tmp0_m, tmp5_m, tmp4_m); \ + LASX_ILVLH_B_128SV(tmp3_m, tmp2_m, tmp7_m, tmp6_m); \ + LASX_ILVLH_W_128SV(tmp6_m, tmp4_m, out2, out0); \ + LASX_ILVLH_W_128SV(tmp7_m, tmp5_m, out6, out4); \ + LASX_SLDI_B_2_0_128SV(out0, out2, out1, out3, 8); \ + LASX_SLDI_B_2_0_128SV(out4, out6, out5, out7, 8); \ +} + +/* Description : Transposes input 16x8 byte block + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, + * in8, in9, in10, in11, in12, in13, in14, in15 + * (input 16x8 byte block) + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * (output 8x16 byte block) + * Details : + */ +#define LASX_TRANSPOSE16x8_B(in0, in1, in2, in3, in4, in5, in6, in7, \ + in8, in9, in10, in11, in12, in13, in14, in15, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + __m256i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + __m256i tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ + __m256i t0, t1, t2, t3, t4, t5, t6, t7; \ + LASX_ILVL_B_8_128SV(in2, in0, in3, in1, in6, in4, in7, in5, \ + in10, in8, in11, in9, in14, in12, in15, in13, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m, \ + tmp4_m, tmp5_m, tmp6_m, tmp7_m); \ + LASX_ILVLH_B_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, t1, t0, t3, t2); \ + LASX_ILVLH_B_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, t5, t4, t7, t6); \ + LASX_ILVLH_W_2_128SV(t2, t0, t3, t1, tmp2_m, tmp0_m, tmp6_m, tmp4_m); \ + LASX_ILVLH_W_2_128SV(t6, t4, t7, t5, tmp3_m, tmp1_m, tmp7_m, tmp5_m); \ + LASX_ILVLH_D_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, out1, out0, out3, out2); \ + LASX_ILVLH_D_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, out5, out4, out7, out6); \ +} + +/* Description : Transposes input 16x8 byte block + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, + * in8, in9, in10, in11, in12, in13, in14, in15 + * (input 16x8 byte block) + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * (output 8x16 byte block) + * Details : The rows of the matrix become columns, and the columns become rows. + * Example : LASX_TRANSPOSE16x8_H + * in0 : 1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in1 : 2,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in2 : 3,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in3 : 4,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in4 : 5,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in5 : 6,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in6 : 7,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in7 : 8,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in8 : 9,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in9 : 1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in10 : 0,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in11 : 2,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in12 : 3,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in13 : 7,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in14 : 5,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * in15 : 6,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * + * out0 : 1,2,3,4,5,6,7,8,9,1,0,2,3,7,5,6 + * out1 : 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2 + * out2 : 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3 + * out3 : 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4 + * out4 : 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 + * out5 : 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6 + * out6 : 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 + * out7 : 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 + */ +#define LASX_TRANSPOSE16x8_H(in0, in1, in2, in3, in4, in5, in6, in7, \ + in8, in9, in10, in11, in12, in13, in14, in15, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + __m256i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + __m256i tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ + __m256i t0, t1, t2, t3, t4, t5, t6, t7; \ + LASX_ILVL_H_8_128SV(in2, in0, in3, in1, in6, in4, in7, in5, \ + in10, in8, in11, in9, in14, in12, in15, in13, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m, \ + tmp4_m, tmp5_m, tmp6_m, tmp7_m); \ + LASX_ILVLH_H_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, t1, t0, t3, t2); \ + LASX_ILVLH_H_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, t5, t4, t7, t6); \ + LASX_ILVLH_D_2_128SV(t2, t0, t3, t1, tmp2_m, tmp0_m, tmp6_m, tmp4_m); \ + LASX_ILVLH_D_2_128SV(t6, t4, t7, t5, tmp3_m, tmp1_m, tmp7_m, tmp5_m); \ + LASX_PCKEV_Q_2(tmp1_m, tmp0_m, tmp3_m, tmp2_m, out0, out1); \ + LASX_PCKEV_Q_2(tmp5_m, tmp4_m, tmp7_m, tmp6_m, out2, out3); \ + \ + LASX_ILVH_H_8_128SV(in2, in0, in3, in1, in6, in4, in7, in5, \ + in10, in8, in11, in9, in14, in12, in15, in13, \ + tmp0_m, tmp1_m, tmp2_m, tmp3_m, \ + tmp4_m, tmp5_m, tmp6_m, tmp7_m); \ + LASX_ILVLH_H_2_128SV(tmp1_m, tmp0_m, tmp3_m, tmp2_m, t1, t0, t3, t2); \ + LASX_ILVLH_H_2_128SV(tmp5_m, tmp4_m, tmp7_m, tmp6_m, t5, t4, t7, t6); \ + LASX_ILVLH_D_2_128SV(t2, t0, t3, t1, tmp2_m, tmp0_m, tmp6_m, tmp4_m); \ + LASX_ILVLH_D_2_128SV(t6, t4, t7, t5, tmp3_m, tmp1_m, tmp7_m, tmp5_m); \ + LASX_PCKEV_Q_2(tmp1_m, tmp0_m, tmp3_m, tmp2_m, out4, out5); \ + LASX_PCKEV_Q_2(tmp5_m, tmp4_m, tmp7_m, tmp6_m, out6, out7); \ +} + +/* Description : Clips all signed word elements of input vector + * between 0 & 255 + * Arguments : Inputs - in (input vector) + * Outputs - out_m (output vector with clipped elements) + * Return Type - signed word + */ +#define LASX_CLIP_W_0_255(in, out_m) \ +{ \ + out_m = __lasx_xvmaxi_w(in, 0); \ + out_m = __lasx_xvsat_wu(out_m, 7); \ +} + +#define LASX_CLIP_W_0_255_2(in0, in1, out0, out1) \ +{ \ + LASX_CLIP_W_0_255(in0, out0); \ + LASX_CLIP_W_0_255(in1, out1); \ +} + +#define LASX_CLIP_W_0_255_4(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + LASX_CLIP_W_0_255_2(in0, in1, out0, out1); \ + LASX_CLIP_W_0_255_2(in2, in3, out2, out3); \ +} + +/* Description : Clips all signed halfword elements of input vector + * between 0 & 255 + * Arguments : Inputs - in (input vector) + * Outputs - out_m (output vector with clipped elements) + * Return Type - signed halfword + */ +#define LASX_CLIP_H_0_255(in, out_m) \ +{ \ + out_m = __lasx_xvmaxi_h(in, 0); \ + out_m = __lasx_xvsat_hu(out_m, 7); \ +} + +#define LASX_CLIP_H_0_255_2(in0, in1, out0, out1) \ +{ \ + LASX_CLIP_H_0_255(in0, out0); \ + LASX_CLIP_H_0_255(in1, out1); \ +} + +#define LASX_CLIP_H_0_255_4(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + LASX_CLIP_H_0_255_2(in0, in1, out0, out1); \ + LASX_CLIP_H_0_255_2(in2, in3, out2, out3); \ +} + +/* Description : Clips all halfword elements of input vector between min & max + * out = ((in) < (min)) ? (min) : (((in) > (max)) ? (max) : (in)) + * Arguments : Inputs - in (input vector) + * - min (min threshold) + * - max (max threshold) + * Outputs - in (output vector with clipped elements) + * Return Type - signed halfword + */ +#define LASX_CLIP_H(in, min, max) \ +{ \ + in = __lasx_xvmax_h(min, in); \ + in = __lasx_xvmin_h(max, in); \ +} + +/* Description : Dot product and addition of 3 signed byte input vectors + * Arguments : Inputs - in0, in1, in2, coeff0, coeff1, coeff2 + * Outputs - out0_m + * Return Type - signed halfword + * Details : Dot product of 'in0' with 'coeff0' + * Dot product of 'in1' with 'coeff1' + * Dot product of 'in2' with 'coeff2' + * Addition of all the 3 vector results + * out0_m = (in0 * coeff0) + (in1 * coeff1) + (in2 * coeff2) + */ +#define LASX_DP2ADD_H_B_3(in0, in1, in2, out0_m, coeff0, coeff1, coeff2) \ +{ \ + LASX_DP2_H_B(in0, coeff0, out0_m); \ + LASX_DP2ADD_H_B(out0_m, in1, coeff1, out0_m); \ + LASX_DP2ADD_H_B(out0_m, in2, coeff2, out0_m); \ +} + +/* Description : Each byte element is logically xor'ed with immediate 128 + * Arguments : Inputs - in0, in1 + * Outputs - in0, in1 (in-place) + * Details : Each unsigned byte element from input vector 'in0' is + * logically xor'ed with 128 and result is in-place stored in + * 'in0' vector + * Each unsigned byte element from input vector 'in1' is + * logically xor'ed with 128 and result is in-place stored in + * 'in1' vector + * Similar for other pairs + * Example : LASX_XORI_B_128(in0) + * in0: 9,10,11,12, 13,14,15,16, 121,122,123,124, 125,126,127,128, 17,18,19,20, 21,22,23,24, + * 248,249,250,251, 252,253,254,255, + * in0: 137,138,139,140, 141,142,143,144, 249,250,251,252, 253,254,255,0, 145,146,147,148, + * 149,150,151,152, 120,121,122,123, 124,125,126,127 + */ +#define LASX_XORI_B_128(in0) \ +{ \ + in0 = __lasx_xvxori_b(in0, 128); \ +} +#define LASX_XORI_B_2_128(in0, in1) \ +{ \ + LASX_XORI_B_128(in0); \ + LASX_XORI_B_128(in1); \ +} +#define LASX_XORI_B_4_128(in0, in1, in2, in3) \ +{ \ + LASX_XORI_B_2_128(in0, in1); \ + LASX_XORI_B_2_128(in2, in3); \ +} +#define LASX_XORI_B_8_128(in0, in1, in2, in3, in4, in5, in6, in7) \ +{ \ + LASX_XORI_B_4_128(in0, in1, in2, in3); \ + LASX_XORI_B_4_128(in4, in5, in6, in7); \ +} + +/* Description : Indexed halfword element values are replicated to all + * elements in output vector. If 'indx0 < 8' use SPLATI_R_*, + * if 'indx0 >= 8' use SPLATI_L_* + * Arguments : Inputs - in, idx0, idx1 + * Outputs - out0, out1 + * Details : 'idx0' element value from 'in' vector is replicated to all + * elements in 'out0' vector + * Valid index range for halfword operation is 0-7 + */ +#define LASX_SPLATI_L_H(in, idx0, out0) \ +{ \ + in = __lasx_xvpermi_q(in, in, 0x02); \ + out0 = __lasx_xvrepl128vei_h(in, idx0); \ +} +#define LASX_SPLATI_H_H(in, idx0, out0) \ +{ \ + in = __lasx_xvpermi_q(in, in, 0X13); \ + out0 = __lasx_xvrepl128vei_h(in, idx0 - 8); \ +} +#define LASX_SPLATI_L_H_2(in, idx0, idx1, out0, out1) \ +{ \ + LASX_SPLATI_L_H(in, idx0, out0); \ + out1 = __lasx_xvrepl128vei_h(in, idx1); \ +} +#define LASX_SPLATI_H_H_2(in, idx0, idx1, out0, out1) \ +{ \ + LASX_SPLATI_H_H(in, idx0, out0); \ + out1 = __lasx_xvrepl128vei_h(in, idx1 - 8); \ +} +#define LASX_SPLATI_L_H_4(in, idx0, idx1, idx2, idx3, \ + out0, out1, out2, out3) \ +{ \ + LASX_SPLATI_L_H_2(in, idx0, idx1, out0, out1); \ + out2 = __lasx_xvrepl128vei_h(in, idx2); \ + out3 = __lasx_xvrepl128vei_h(in, idx3); \ +} +#define SPLATI_H_H_4(in, idx0, idx1, idx2, idx3, \ + out0, out1, out2, out3) \ +{ \ + LASX_SPLATI_H_H_2(in, idx0, idx1, out0, out1); \ + out2 = __lasx_xvrepl128vei_h(in, idx2 - 8); \ + out3 = __lasx_xvrepl128vei_h(in, idx3 - 8); \ +} + +/* Description : Pack even elements of input vectors & xor with 128 + * Arguments : Inputs - in0, in1 + * Outputs - out_m + * Details : Signed byte even elements from 'in0' and 'in1' are packed + * together in one vector and the resulted vector is xor'ed with + * 128 to shift the range from signed to unsigned byte + */ +#define LASX_PICKEV_XORI128_B(in0, in1, out_m) \ +{ \ + out_m = __lasx_xvpickev_b(in1, in0); \ + out_m = __lasx_xvxori_b(out_m, 128); \ +} + +/* Description : Shift right logical all byte elements of vector. + * Arguments : Inputs - in, shift + * Outputs - in (in place) + * Details : Each element of vector in is shifted right logical by + * number of bits respective element holds in vector shift and + * result is in place written to in. + * Here, shift is a vector passed in. + * Example : See LASX_SRL_W(in, shift) + */ +#define LASX_SRL_B(in, shift) \ +{ \ + in = __lasx_xvsrl_b(in, shift); \ +} + +#define LASX_SRL_B_2(in0, in1, shift) \ +{ \ + LASX_SRL_B(in0, shift); \ + LASX_SRL_B(in1, shift); \ +} + +#define LASX_SRL_B_4(in0, in1, in2, in3, shift) \ +{ \ + LASX_SRL_B_2(in0, in1, shift); \ + LASX_SRL_B_2(in2, in3, shift); \ +} + +/* Description : Shift right logical all halfword elements of vector. + * Arguments : Inputs - in, shift + * Outputs - in (in place) + * Details : Each element of vector in is shifted right logical by + * number of bits respective element holds in vector shift and + * result is in place written to in. + * Here, shift is a vector passed in. + * Example : See LASX_SRL_W(in, shift) + */ +#define LASX_SRL_H(in, shift) \ +{ \ + in = __lasx_xvsrl_h(in, shift); \ +} + +#define LASX_SRL_H_2(in0, in1, shift) \ +{ \ + LASX_SRL_H(in0, shift); \ + LASX_SRL_H(in1, shift); \ +} + +#define LASX_SRL_H_4(in0, in1, in2, in3, shift) \ +{ \ + LASX_SRL_H_2(in0, in1, shift); \ + LASX_SRL_H_2(in2, in3, shift); \ +} + +/* Description : Shift right logical all word elements of vector. + * Arguments : Inputs - in, shift + * Outputs - in (in place) + * Details : Each element of vector in is shifted right logical by + * number of bits respective element holds in vector shift and + * result is in place written to in. + * Here, shift is a vector passed in. + * Example : LASX_SRL_W(in, shift) + * in : 1, 3, 2, -4, 0, -2, 25, 0 + * shift : 1, 1, 1, 1, 2, 2, 2, 2 + * in(output) : 0, 1, 1, 32766, 0, 16383, 6, 0 + */ +#define LASX_SRL_W(in, shift) \ +{ \ + in = __lasx_xvsrl_w(in, shift); \ +} + +#define LASX_SRL_W_2(in0, in1, shift) \ +{ \ + LASX_SRL_W(in0, shift); \ + LASX_SRL_W(in1, shift); \ +} + +#define LASX_SRL_W_4(in0, in1, in2, in3, shift) \ +{ \ + LASX_SRL_W_2(in0, in1, shift); \ + LASX_SRL_W_2(in2, in3, shift); \ +} + +/* Description : Shift right logical all double word elements of vector. + * Arguments : Inputs - in, shift + * Outputs - in (in place) + * Details : Each element of vector in is shifted right logical by + * number of bits respective element holds in vector shift and + * result is in place written to in. + * Here, shift is a vector passed in. + * Example : See LASX_SRL_W(in, shift) + */ +#define LASX_SRL_D(in, shift) \ +{ \ + in = __lasx_xvsrl_d(in, shift); \ +} + +#define LASX_SRL_D_2(in0, in1, shift) \ +{ \ + LASX_SRL_D(in0, shift); \ + LASX_SRL_D(in1, shift); \ +} + +#define LASX_SRL_D_4(in0, in1, in2, in3, shift) \ +{ \ + LASX_SRL_D_2(in0, in1, shift); \ + LASX_SRL_D_2(in2, in3, shift); \ +} + + +/* Description : Shift right arithmetic rounded (immediate) + * Arguments : Inputs - in0, in1, shift + * Outputs - in0, in1, (in place) + * Details : Each element of vector 'in0' is shifted right arithmetic by + * value in 'shift'. + * The last discarded bit is added to shifted value for rounding + * and the result is in place written to 'in0' + * Similar for other pairs + * Example : LASX_SRARI_H(in0, out0, shift) + * in0: 1,2,3,4, -5,-6,-7,-8, 19,10,11,12, 13,14,15,16 + * shift: 2 + * out0: 0,1,1,1, -1,-1,-2,-2, 5,3,3,3, 3,4,4,4 + */ +#define LASX_SRARI_H(in0, out0, shift) \ +{ \ + out0 = __lasx_xvsrari_h(in0, shift); \ +} +#define LASX_SRARI_H_2(in0, in1, out0, out1, shift) \ +{ \ + LASX_SRARI_H(in0, out0, shift); \ + LASX_SRARI_H(in1, out1, shift); \ +} +#define LASX_SRARI_H_4(in0, in1, in2, in3, out0, out1, out2, out3, shift) \ +{ \ + LASX_SRARI_H_2(in0, in1, out0, out1, shift); \ + LASX_SRARI_H_2(in2, in3, out2, out3, shift); \ +} + +/* Description : Shift right arithmetic (immediate) + * Arguments : Inputs - in0, in1, shift + * Outputs - in0, in1, (in place) + * Details : Each element of vector 'in0' is shifted right arithmetic by + * value in 'shift'. + * Similar for other pairs + * Example : see LASX_SRARI_H(in0, out0, shift) + */ +#define LASX_SRAI_W(in0, out0, shift) \ +{ \ + out0 = __lasx_xvsrai_w(in0, shift); \ +} +#define LASX_SRAI_W_2(in0, in1, out0, out1, shift) \ +{ \ + LASX_SRAI_W(in0, out0, shift); \ + LASX_SRAI_W(in1, out1, shift); \ +} +#define LASX_SRAI_W_4(in0, in1, in2, in3, out0, out1, out2, out3, shift) \ +{ \ + LASX_SRAI_W_2(in0, in1, out0, out1, shift); \ + LASX_SRAI_W_2(in2, in3, out2, out3, shift); \ +} +#define LASX_SRAI_W_8(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7, shift) \ +{ \ + LASX_SRAI_W_4(in0, in1, in2, in3, out0, out1, out2, out3, shift); \ + LASX_SRAI_W_4(in4, in5, in6, in7, out4, out5, out6, out7, shift); \ +} + +/* Description : Saturate the halfword element values to the max + * unsigned value of (sat_val+1 bits) + * The element data width remains unchanged + * Arguments : Inputs - in0, in1, in2, in3, sat_val + * Outputs - in0, in1, in2, in3 (in place) + * Return Type - unsigned halfword + * Details : Each unsigned halfword element from 'in0' is saturated to the + * value generated with (sat_val+1) bit range + * Results are in placed to original vectors + * Example : LASX_SAT_H(in0, out0, sat_val) + * in0: 1,2,3,4, 5,6,7,8, 19,10,11,12, 13,14,15,16 + * sat_val:3 + * out0: 1,2,3,4, 5,6,7,7, 7,7,7,7, 7,7,7,7 + */ +#define LASX_SAT_H(in0, out0, sat_val) \ +{ \ + out0 = __lasx_xvsat_h(in0, sat_val); \ +} //some error in xvsat_h built-in function +#define LASX_SAT_H_2(in0, in1, out0, out1, sat_val) \ +{ \ + LASX_SAT_H(in0, out0, sat_val); \ + LASX_SAT_H(in1, out1, sat_val); \ +} +#define LASX_SAT_H_4(in0, in1, in2, in3, out0, out1, out2, out3, sat_val) \ +{ \ + LASX_SAT_H_2(in0, in1, out0, out1, sat_val); \ + LASX_SAT_H_2(in2, in3, out2, out3, sat_val); \ +} + +/* Description : Addition of 2 pairs of vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1 + * Details : Each halfwords element from 2 pairs vectors is added + * and 2 results are produced + * Example : LASX_ADD_H(in0, in1, out) + * in0: 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in1: 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out: 9,9,9,9, 9,9,9,9, 9,9,9,9, 9,9,9,9 + */ +#define LASX_ADD_H(in0, in1, out) \ +{ \ + out = __lasx_xvadd_h(in0, in1); \ +} +#define LASX_ADD_H_2(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_ADD_H(in0, in1, out0); \ + LASX_ADD_H(in2, in3, out1); \ +} +#define LASX_ADD_H_4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ +{ \ + LASX_ADD_H_2(in0, in1, in2, in3, out0, out1); \ + LASX_ADD_H_2(in4, in5, in6, in7, out2, out3); \ +} +#define LASX_ADD_H_8(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, in12, \ + in13, in14, in15, out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + LASX_ADD_H_4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3); \ + LASX_ADD_H_4(in8, in9, in10, in11, in12, in13, in14, in15, out4, out5, out6, out7); \ +} + +/* Description : Horizontal subtraction of unsigned byte vector elements + * Arguments : Inputs - in0, in1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Each unsigned odd byte element from 'in0' is subtracted from + * even unsigned byte element from 'in0' (pairwise) and the + * halfword result is written to 'out0' + */ +#define LASX_HSUB_UB_2(in0, in1, out0, out1) \ +{ \ + out0 = __lasx_xvhsubw_hu_bu(in0, in0); \ + out1 = __lasx_xvhsubw_hu_bu(in1, in1); \ +} + +#define LASX_HSUB_UB_4(in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + LASX_HSUB_UB_2(in0, in1, out0, out1); \ + LASX_HSUB_UB_2(in2, in3, out2, out3); \ +} + +/* Description : Shuffle byte vector elements as per mask vector + * Arguments : Inputs - in0, in1, in2, in3, mask0, mask1 + * Outputs - out0, out1 + * Return Type - as per RTYPE + * Details : Selective byte elements from in0 & in1 are copied to out0 as + * per control vector mask0 + * Selective byte elements from in2 & in3 are copied to out1 as + * per control vector mask1 + * Example : LASX_SHUF_B_128SV(in0, in1, mask0, out0) + * in_h : 9,10,11,12, 13,14,15,16, 0,0,0,0, 0,0,0,0, + * 17,18,19,20, 21,22,23,24, 0,0,0,0, 0,0,0,0 + * in_l : 1, 2, 3, 4, 5, 6, 7, 8, 0,0,0,0, 0,0,0,0, + * 25,26,27,28, 29,30,31,32, 0,0,0,0, 0,0,0,0 + * mask0: 0, 1, 2, 3, 4, 5, 6, 7, 16,17,18,19, 20,21,22,23, + * 16,17,18,19, 20,21,22,23, 0, 1, 2, 3, 4, 5, 6, 7 + * out0 : 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12, 13,14,15,16, + * 17,18,19,20, 21,22,23,24, 25,26,27,28, 29,30,31,32 + */ + +#define LASX_SHUF_B_128SV(in_h, in_l, mask0, out0) \ +{ \ + out0 = __lasx_xvshuf_b(in_h, in_l, mask0); \ +} +#define LASX_SHUF_B_2_128SV(in0_h, in0_l, in1_h, in1_l, mask0, mask1, \ + out0, out1) \ +{ \ + LASX_SHUF_B_128SV(in0_h, in0_l, mask0, out0); \ + LASX_SHUF_B_128SV(in1_h, in1_l, mask1, out1); \ +} +#define LASX_SHUF_B_4_128SV(in0_h, in0_l, in1_h, in1_l, in2_h, in2_l, \ + in3_h, in3_l, mask0, mask1, mask2, mask3, \ + out0, out1, out2, out3) \ +{ \ + LASX_SHUF_B_2_128SV(in0_h, in0_l, in1_h, in1_l, mask0, mask1, out0, out1); \ + LASX_SHUF_B_2_128SV(in2_h, in2_l, in3_h, in3_l, mask2, mask3, out2, out3); \ +} + +/* Description : Addition of signed halfword elements and signed saturation + * Arguments : Inputs - in0, in1, in2, in3 ~ + * Outputs - out0, out1 ~ + * Details : Signed halfword elements from 'in0' are added to signed + * halfword elements of 'in1'. The result is then signed saturated + * between -32768 to +32767 (as per halfword data type) + * Similar for other pairs + * Example : LASX_SADD_H(in0, in1, out0) + * in0: 1,2,32766,4, 5,6,7,8, 1,2,3,4, 5,6,7,8, + * in1: 8,7,30586,5, 4,3,2,1, 8,7,6,5, 4,3,2,1, + * out0: 9,9,32767,9, 9,9,9,9, 9,9,9,9, 9,9,9,9, + */ +#define LASX_SADD_H(in0, in1, out0) \ +{ \ + out0 = __lasx_xvsadd_h(in0, in1); \ +} +#define LASX_SADD_H_2(in0, in1, in2, in3, out0, out1) \ +{ \ + LASX_SADD_H(in0, in1, out0); \ + LASX_SADD_H(in2, in3, out1); \ +} +#define LASX_SADD_H_4(in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3) \ +{ \ + LASX_SADD_H_2(in0, in1, in2, in3, out0, out1); \ + LASX_SADD_H_2(in4, in5, in6, in7, out2, out3); \ +} + +/* Description : Average with rounding (in0 + in1 + 1) / 2. + * Arguments : Inputs - in0, in1, in2, in3, + * Outputs - out0, out1 + * Details : Each unsigned byte element from 'in0' vector is added with + * each unsigned byte element from 'in1' vector. + * Average with rounding is calculated and written to 'out0' + */ +#define LASX_AVER_BU( in0, in1, out0 ) \ +{ \ + out0 = __lasx_xvavgr_bu( in0, in1 ); \ +} + +#define LASX_AVER_BU_2( in0, in1, in2, in3, out0, out1 ) \ +{ \ + LASX_AVER_BU( in0, in1, out0 ); \ + LASX_AVER_BU( in2, in3, out1 ); \ +} + +#define LASX_AVER_BU_4( in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3 ) \ +{ \ + LASX_AVER_BU_2( in0, in1, in2, in3, out0, out1 ); \ + LASX_AVER_BU_2( in4, in5, in6, in7, out2, out3 ); \ +} + +/* Description : Butterfly of 4 input vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : Butterfly operationuu + */ +#define LASX_BUTTERFLY_4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) \ +{ \ + out0 = (__m256i)( (RTYPE)in0 + (RTYPE)in3 ); \ + out1 = (__m256i)( (RTYPE)in1 + (RTYPE)in2 ); \ + \ + out2 = (__m256i)( (RTYPE)in1 - (RTYPE)in2 ); \ + out3 = (__m256i)( (RTYPE)in0 - (RTYPE)in3 ); \ +} + +/* Description : Butterfly of 8 input vectors + * Arguments : Inputs - in0 in1 in2 ~ + * Outputs - out0 out1 out2 ~ + * Details : Butterfly operation + */ +#define LASX_BUTTERFLY_8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ + out0, out1, out2, out3, out4, out5, out6, out7) \ +{ \ + out0 = (__m256i)( (RTYPE)in0 + (RTYPE)in7 ); \ + out1 = (__m256i)( (RTYPE)in1 + (RTYPE)in6 ); \ + out2 = (__m256i)( (RTYPE)in2 + (RTYPE)in5 ); \ + out3 = (__m256i)( (RTYPE)in3 + (RTYPE)in4 ); \ + \ + out4 = (__m256i)( (RTYPE)in3 - (RTYPE)in4 ); \ + out5 = (__m256i)( (RTYPE)in2 - (RTYPE)in5 ); \ + out6 = (__m256i)( (RTYPE)in1 - (RTYPE)in6 ); \ + out7 = (__m256i)( (RTYPE)in0 - (RTYPE)in7 ); \ +} + + +#endif /* GENERIC_MACROS_LASX_H */ +#endif /* AVUTIL_LOONGARCH_GENERIC_MACROS_LASX_H */ diff --git a/libavutil/loongarch/generic_macros_lsx.h b/libavutil/loongarch/generic_macros_lsx.h new file mode 100644 index 0000000000..8edebb4be0 --- /dev/null +++ b/libavutil/loongarch/generic_macros_lsx.h @@ -0,0 +1,670 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * All rights reserved. + * Contributed by Shiyou Yin + * Xiwei Gu + * Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifndef AVUTIL_LOONGARCH_GENERIC_MACROS_LSX_H +#define AVUTIL_LOONGARCH_GENERIC_MACROS_LSX_H + +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * All rights reserved. + * Contributed by Shiyou Yin + * Xiwei Gu + * Lu Wang + * Hecai Yuan + * + * This file is maintained in LSOM project, don't change it directly. + * You can get the latest version of this header from: *** + * + */ + +#ifndef GENERIC_MACROS_LSX_H +#define GENERIC_MACROS_LSX_H + +#include + +/** + * MAJOR version: Macro usage changes. + * MINOR version: Add new functions, or bug fix. + * MICRO version: Comment changes or implementation changes. + */ +#define LSOM_LSX_VERSION_MAJOR 0 +#define LSOM_LSX_VERSION_MINOR 7 +#define LSOM_LSX_VERSION_MICRO 0 + +#define LSX_DUP2_ARG1(_LSX_INS, _IN0, _IN1, _OUT0, _OUT1) \ +{ \ + _OUT0 = _LSX_INS(_IN0); \ + _OUT1 = _LSX_INS(_IN1); \ +} + +#define LSX_DUP2_ARG2(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1) \ +{ \ + _OUT0 = _LSX_INS(_IN0, _IN1); \ + _OUT1 = _LSX_INS(_IN2, _IN3); \ +} + +#define LSX_DUP2_ARG3(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1) \ +{ \ + _OUT0 = _LSX_INS(_IN0, _IN1, _IN2); \ + _OUT1 = _LSX_INS(_IN3, _IN4, _IN5); \ +} + +#define LSX_DUP4_ARG1(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1, _OUT2, _OUT3) \ +{ \ + LSX_DUP2_ARG1(_LSX_INS, _IN0, _IN1, _OUT0, _OUT1); \ + LSX_DUP2_ARG1(_LSX_INS, _IN2, _IN3, _OUT2, _OUT3); \ +} + +#define LSX_DUP4_ARG2(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, \ + _OUT0, _OUT1, _OUT2, _OUT3) \ +{ \ + LSX_DUP2_ARG2(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1); \ + LSX_DUP2_ARG2(_LSX_INS, _IN4, _IN5, _IN6, _IN7, _OUT2, _OUT3); \ +} + +#define LSX_DUP4_ARG3(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, \ + _IN8, _IN9, _IN10, _IN11, _OUT0, _OUT1, _OUT2, _OUT3) \ +{ \ + LSX_DUP2_ARG3(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1); \ + LSX_DUP2_ARG3(_LSX_INS, _IN6, _IN7, _IN8, _IN9, _IN10, _IN11, _OUT2, _OUT3); \ +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Then the results plus to signed half word elements from in_c. + * Example : out = __lsx_dp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_dp2add_h_b(__m128i in_c, __m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmaddwev_h_b(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * unsigned byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * The results plus to signed half word elements from in_c. + * Example : out = __lsx_dp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_dp2add_h_bu(__m128i in_c, __m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmaddwev_h_bu(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of half word vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Retrun Type - __m128i + * Details : Signed half word elements from in_h are multiplied by + * signed half word elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Then the results plus to signed word elements from in_c. + * Example : out = __lsx_dp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_dp2add_w_h(__m128i in_c, __m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmaddwev_w_h(in_c, in_h, in_l); + out = __lsx_vmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Example : out = __lsx_dp2_h_b(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22, 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_dp2_h_b(__m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmulwev_h_b(in_h, in_l); + out = __lsx_vmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * unsigned byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Example : out = __lsx_dp2_h_bu(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22, 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_dp2_h_bu(__m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmulwev_h_bu(in_h, in_l); + out = __lsx_vmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Example : out = __lsx_dp2_h_bu_b(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,-1 + * out : 22,38,38,22, 22,38,38,6 + * ============================================================================= + */ +static inline __m128i __lsx_dp2_h_bu_b(__m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmulwev_h_bu_b(in_h, in_l); + out = __lsx_vmaddwod_h_bu_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Example : out = __lsx_dp2_w_h(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_dp2_w_h(__m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmulwev_w_h(in_h, in_l); + out = __lsx_vmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Clip all halfword elements of input vector between min & max + out = ((_in) < (min)) ? (min) : (((_in) > (max)) ? (max) : (_in)) + Arguments : Inputs - _in (input vector) + - min (min threshold) + - max (max threshold) + Outputs - out (output vector with clipped elements) + Return Type - signed halfword + * Example : out = __lsx_clip_h(_in) + * _in : -8,2,280,249, -8,255,280,249 + * min : 1,1,1,1, 1,1,1,1 + * max : 9,9,9,9, 9,9,9,9 + * out : 1,2,9,9, 1,9,9,9 + * ============================================================================= + */ +static inline __m128i __lsx_clip_h(__m128i _in, __m128i min, __m128i max) +{ + __m128i out; + + out = __lsx_vmax_h(min, _in); + out = __lsx_vmin_h(max, out); + return out; +} + +/* + * ============================================================================= + * Description : Set each element of vector between 0 and 255 + * Arguments : Inputs - _in + * Outputs - out + * Retrun Type - halfword + * Details : Signed byte elements from _in are clamped between 0 and 255. + * Example : out = __lsx_clamp255_h(_in) + * _in : -8,255,280,249, -8,255,280,249 + * out : 0,255,255,249, 0,255,255,249 + * ============================================================================= + */ +static inline __m128i __lsx_clamp255_h(__m128i _in) +{ + __m128i out; + + out = __lsx_vmaxi_h(_in, 0); + out = __lsx_vsat_hu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Set each element of vector between 0 and 255 + * Arguments : Inputs - _in + * Outputs - out + * Retrun Type - word + * Details : Signed byte elements from _in are clamped between 0 and 255. + * Example : out = __lsx_clamp255_w(_in) + * _in : -8,255,280,249 + * out : 0,255,255,249 + * ============================================================================= + */ +static inline __m128i __lsx_clamp255_w(__m128i _in) +{ + __m128i out; + + out = __lsx_vmaxi_w(_in, 0); + out = __lsx_vsat_wu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Swap two variables + Arguments : Inputs - _in0, _in1 + Outputs - _in0, _in1 (in-place) + Details : Swapping of two input variables using xor + * Example : SWAP(_in0, _in1) + * _in0 : 1,2,3,4 + * _in1 : 5,6,7,8 + * _in0(out) : 5,6,7,8 + * _in1(out) : 1,2,3,4 + * ============================================================================= + */ +#define SWAP(_in0, _in1) \ +{ \ + _in0 = __lsx_vxor_v(_in0, _in1); \ + _in1 = __lsx_vxor_v(_in0, _in1); \ + _in0 = __lsx_vxor_v(_in0, _in1); \ +} \ + +/* + * ============================================================================= + * Description : Transpose 4x4 block with word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : + * Example : + * 1, 2, 3, 4 1, 5, 9,13 + * 5, 6, 7, 8 to 2, 6,10,14 + * 9,10,11,12 =====> 3, 7,11,15 + * 13,14,15,16 4, 8,12,16 + * ============================================================================= + */ +#define TRANSPOSE4x4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + __m128i _t0, _t1, _t2, _t3; \ + \ + _t0 = __lsx_vilvl_w(_in1, _in0); \ + _t1 = __lsx_vilvh_w(_in1, _in0); \ + _t2 = __lsx_vilvl_w(_in3, _in2); \ + _t3 = __lsx_vilvh_w(_in3, _in2); \ + _out0 = __lsx_vilvl_d(_t2, _t0); \ + _out1 = __lsx_vilvh_d(_t2, _t0); \ + _out2 = __lsx_vilvl_d(_t3, _t1); \ + _out3 = __lsx_vilvh_d(_t3, _t1); \ +} + +/* + * ============================================================================= + * Description : Transpose 8x8 block with byte elements in vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, _out7 + * Details : The rows of the matrix become columns, and the columns become rows. + * Example : TRANSPOSE8x8_B + * _in0 : 00,01,02,03,04,05,06,07, 00,00,00,00,00,00,00,00 + * _in1 : 10,11,12,13,14,15,16,17, 00,00,00,00,00,00,00,00 + * _in2 : 20,21,22,23,24,25,26,27, 00,00,00,00,00,00,00,00 + * _in3 : 30,31,32,33,34,35,36,37, 00,00,00,00,00,00,00,00 + * _in4 : 40,41,42,43,44,45,46,47, 00,00,00,00,00,00,00,00 + * _in5 : 50,51,52,53,54,55,56,57, 00,00,00,00,00,00,00,00 + * _in6 : 60,61,62,63,64,65,66,67, 00,00,00,00,00,00,00,00 + * _in7 : 70,71,72,73,74,75,76,77, 00,00,00,00,00,00,00,00 + * + * _ out0 : 00,10,20,30,40,50,60,70, 00,00,00,00,00,00,00,00 + * _ out1 : 01,11,21,31,41,51,61,71, 00,00,00,00,00,00,00,00 + * _ out2 : 02,12,22,32,42,52,62,72, 00,00,00,00,00,00,00,00 + * _ out3 : 03,13,23,33,43,53,63,73, 00,00,00,00,00,00,00,00 + * _ out4 : 04,14,24,34,44,54,64,74, 00,00,00,00,00,00,00,00 + * _ out5 : 05,15,25,35,45,55,65,75, 00,00,00,00,00,00,00,00 + * _ out6 : 06,16,26,36,46,56,66,76, 00,00,00,00,00,00,00,00 + * _ out7 : 07,17,27,37,47,57,67,77, 00,00,00,00,00,00,00,00 + * ============================================================================= + */ +#define TRANSPOSE8x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, _out7) \ +{ \ + __m128i zero = {0}; \ + __m128i shuf8 = {0x0F0E0D0C0B0A0908, 0x1716151413121110}; \ + __m128i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + \ + _t0 = __lsx_vilvl_b(_in2, _in0); \ + _t1 = __lsx_vilvl_b(_in3, _in1); \ + _t2 = __lsx_vilvl_b(_in6, _in4); \ + _t3 = __lsx_vilvl_b(_in7, _in5); \ + _t4 = __lsx_vilvl_b(_t1, _t0); \ + _t5 = __lsx_vilvh_b(_t1, _t0); \ + _t6 = __lsx_vilvl_b(_t3, _t2); \ + _t7 = __lsx_vilvh_b(_t3, _t2); \ + _out0 = __lsx_vilvl_w(_t6, _t4); \ + _out2 = __lsx_vilvh_w(_t6, _t4); \ + _out4 = __lsx_vilvl_w(_t7, _t5); \ + _out6 = __lsx_vilvh_w(_t7, _t5); \ + _out1 = __lsx_vshuf_b(zero, _out0, shuf8); \ + _out3 = __lsx_vshuf_b(zero, _out2, shuf8); \ + _out5 = __lsx_vshuf_b(zero, _out4, shuf8); \ + _out7 = __lsx_vshuf_b(zero, _out6, shuf8); \ +} + +/* + * ============================================================================= + * Description : Transpose 8x8 block with half word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * Details : + * Example : + * 00,01,02,03,04,05,06,07 00,10,20,30,40,50,60,70 + * 10,11,12,13,14,15,16,17 01,11,21,31,41,51,61,71 + * 20,21,22,23,24,25,26,27 02,12,22,32,42,52,62,72 + * 30,31,32,33,34,35,36,37 to 03,13,23,33,43,53,63,73 + * 40,41,42,43,44,45,46,47 ======> 04,14,24,34,44,54,64,74 + * 50,51,52,53,54,55,56,57 05,15,25,35,45,55,65,75 + * 60,61,62,63,64,65,66,67 06,16,26,36,46,56,66,76 + * 70,71,72,73,74,75,76,77 07,17,27,37,47,57,67,77 + * ============================================================================= + */ +#define TRANSPOSE8x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, _out7) \ +{ \ + __m128i _s0, _s1, _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + \ + _s0 = __lsx_vilvl_h(_in6, _in4); \ + _s1 = __lsx_vilvl_h(_in7, _in5); \ + _t0 = __lsx_vilvl_h(_s1, _s0); \ + _t1 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvh_h(_in6, _in4); \ + _s1 = __lsx_vilvh_h(_in7, _in5); \ + _t2 = __lsx_vilvl_h(_s1, _s0); \ + _t3 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvl_h(_in2, _in0); \ + _s1 = __lsx_vilvl_h(_in3, _in1); \ + _t4 = __lsx_vilvl_h(_s1, _s0); \ + _t5 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvh_h(_in2, _in0); \ + _s1 = __lsx_vilvh_h(_in3, _in1); \ + _t6 = __lsx_vilvl_h(_s1, _s0); \ + _t7 = __lsx_vilvh_h(_s1, _s0); \ + \ + _out0 = __lsx_vpickev_d(_t0, _t4); \ + _out2 = __lsx_vpickev_d(_t1, _t5); \ + _out4 = __lsx_vpickev_d(_t2, _t6); \ + _out6 = __lsx_vpickev_d(_t3, _t7); \ + _out1 = __lsx_vpickod_d(_t0, _t4); \ + _out3 = __lsx_vpickod_d(_t1, _t5); \ + _out5 = __lsx_vpickod_d(_t2, _t6); \ + _out7 = __lsx_vpickod_d(_t3, _t7); \ +} + +/* + * ============================================================================= + * Description : Transpose input 8x4 byte block into 4x8 + Arguments : Inputs - _in0, _in1, _in2, _in3 (input 8x4 byte block) + Outputs - _out0, _out1, _out2, _out3 (output 4x8 byte block) + Return Type - as per RTYPE + * Details : The rows of the matrix become columns, and the columns become rows. + * Example : TRANSPOSE8x4_B + * _in0 : 00,01,02,03,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in1 : 10,11,12,13,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in2 : 20,21,22,23,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in3 : 30,31,32,33,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in4 : 40,41,42,43,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in5 : 50,51,52,53,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in6 : 60,61,62,63,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in7 : 70,71,72,73,00,00,00,00, 00,00,00,00,00,00,00,00 + * + * _out0 : 00,10,20,30,40,50,60,70, 00,00,00,00,00,00,00,00 + * _out1 : 01,11,21,31,41,51,61,71, 00,00,00,00,00,00,00,00 + * _out2 : 02,12,22,32,42,52,62,72, 00,00,00,00,00,00,00,00 + * _out3 : 03,13,23,33,43,53,63,73, 00,00,00,00,00,00,00,00 + * ============================================================================= + */ +#define TRANSPOSE8x4_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3) \ +{ \ + __m128i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + \ + _tmp0_m = __lsx_vpackev_w(_in4, _in0); \ + _tmp1_m = __lsx_vpackev_w(_in5, _in1); \ + _tmp2_m = __lsx_vilvl_b(_tmp1_m, _tmp0_m); \ + _tmp0_m = __lsx_vpackev_w(_in6, _in2); \ + _tmp1_m = __lsx_vpackev_w(_in7, _in3); \ + \ + _tmp3_m = __lsx_vilvl_b(_tmp1_m, _tmp0_m); \ + _tmp0_m = __lsx_vilvl_h(_tmp3_m, _tmp2_m); \ + _tmp1_m = __lsx_vilvh_h(_tmp3_m, _tmp2_m); \ + \ + _out0 = __lsx_vilvl_w(_tmp1_m, _tmp0_m); \ + _out2 = __lsx_vilvh_w(_tmp1_m, _tmp0_m); \ + _out1 = __lsx_vilvh_d(_out2, _out0); \ + _out3 = __lsx_vilvh_d(_out0, _out2); \ +} + +/* + * ============================================================================= + * Description : Transpose 16x8 block with byte elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, in8 + * in9, in10, in11, in12, in13, in14, in15 + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * Details : + * Example : + * 000,001,002,003,004,005,006,007 + * 008,009,010,011,012,013,014,015 + * 016,017,018,019,020,021,022,023 + * 024,025,026,027,028,029,030,031 + * 032,033,034,035,036,037,038,039 + * 040,041,042,043,044,045,046,047 000,008,...,112,120 + * 048,049,050,051,052,053,054,055 001,009,...,113,121 + * 056,057,058,059,060,061,062,063 to 002,010,...,114,122 + * 064,068,066,067,068,069,070,071 =====> 003,011,...,115,123 + * 072,073,074,075,076,077,078,079 004,012,...,116,124 + * 080,081,082,083,084,085,086,087 005,013,...,117,125 + * 088,089,090,091,092,093,094,095 006,014,...,118,126 + * 096,097,098,099,100,101,102,103 007,015,...,119,127 + * 104,105,106,107,108,109,110,111 + * 112,113,114,115,116,117,118,119 + * 120,121,122,123,124,125,126,127 + * ============================================================================= + */ +#define TRANSPOSE16x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, _in8, \ + _in9, _in10, _in11, _in12, _in13, _in14, _in15, _out0, \ + _out1, _out2, _out3, _out4, _out5, _out6, _out7) \ +{ \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + LSX_DUP4_ARG2(__lsx_vilvl_b, _in2, _in0, _in3, _in1, _in6, _in4, _in7, _in5, \ + _tmp0, _tmp1, _tmp2, _tmp3); \ + LSX_DUP4_ARG2(__lsx_vilvl_b, _in10, _in8, _in11, _in9, _in14, _in12, _in15, \ + _in13, _tmp4, _tmp5, _tmp6, _tmp7); \ + LSX_DUP2_ARG2(__lsx_vilvl_b, _tmp1, _tmp0, _tmp3, _tmp2, _t0, _t2); \ + LSX_DUP2_ARG2(__lsx_vilvh_b, _tmp1, _tmp0, _tmp3, _tmp2, _t1, _t3); \ + LSX_DUP2_ARG2(__lsx_vilvl_b, _tmp5, _tmp4, _tmp7, _tmp6, _t4, _t6); \ + LSX_DUP2_ARG2(__lsx_vilvh_b, _tmp5, _tmp4, _tmp7, _tmp6, _t5, _t7); \ + LSX_DUP2_ARG2(__lsx_vilvl_w, _t2, _t0, _t3, _t1, _tmp0, _tmp4); \ + LSX_DUP2_ARG2(__lsx_vilvh_w, _t2, _t0, _t3, _t1, _tmp2, _tmp6); \ + LSX_DUP2_ARG2(__lsx_vilvl_w, _t6, _t4, _t7, _t5, _tmp1, _tmp5); \ + LSX_DUP2_ARG2(__lsx_vilvh_w, _t6, _t4, _t7, _t5, _tmp3, _tmp7); \ + LSX_DUP2_ARG2(__lsx_vilvl_d, _tmp1, _tmp0, _tmp3, _tmp2, _out0, _out2); \ + LSX_DUP2_ARG2(__lsx_vilvh_d, _tmp1, _tmp0, _tmp3, _tmp2, _out1, _out3); \ + LSX_DUP2_ARG2(__lsx_vilvl_d, _tmp5, _tmp4, _tmp7, _tmp6, _out4, _out6); \ + LSX_DUP2_ARG2(__lsx_vilvh_d, _tmp5, _tmp4, _tmp7, _tmp6, _out5, _out7); \ +} + +/* + * ============================================================================= + * Description : Butterfly of 4 input vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : Butterfly operation + * Example : + * out0 = in0 + in3; + * out1 = in1 + in2; + * out2 = in1 - in2; + * out3 = in0 - in3; + * ============================================================================= + */ +#define BUTTERFLY_4_B(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + _out0 = __lsx_vadd_b(_in0, _in3); \ + _out1 = __lsx_vadd_b(_in1, _in2); \ + _out2 = __lsx_vsub_b(_in1, _in2); \ + _out3 = __lsx_vsub_b(_in0, _in3); \ +} +#define BUTTERFLY_4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + _out0 = __lsx_vadd_h(_in0, _in3); \ + _out1 = __lsx_vadd_h(_in1, _in2); \ + _out2 = __lsx_vsub_h(_in1, _in2); \ + _out3 = __lsx_vsub_h(_in0, _in3); \ +} +#define BUTTERFLY_4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + _out0 = __lsx_vadd_w(_in0, _in3); \ + _out1 = __lsx_vadd_w(_in1, _in2); \ + _out2 = __lsx_vsub_w(_in1, _in2); \ + _out3 = __lsx_vsub_w(_in0, _in3); \ +} +#define BUTTERFLY_4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + _out0 = __lsx_vadd_d(_in0, _in3); \ + _out1 = __lsx_vadd_d(_in1, _in2); \ + _out2 = __lsx_vsub_d(_in1, _in2); \ + _out3 = __lsx_vsub_d(_in0, _in3); \ +} + +/* + * ============================================================================= + * Description : Butterfly of 8 input vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, ~ + * Outputs - _out0, _out1, _out2, _out3, ~ + * Details : Butterfly operation + * Example : + * _out0 = _in0 + _in7; + * _out1 = _in1 + _in6; + * _out2 = _in2 + _in5; + * _out3 = _in3 + _in4; + * _out4 = _in3 - _in4; + * _out5 = _in2 - _in5; + * _out6 = _in1 - _in6; + * _out7 = _in0 - _in7; + * ============================================================================= + */ +#define BUTTERFLY_8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, _out7) \ +{ \ + _out0 = __lsx_vadd_h(_in0, _in7); \ + _out1 = __lsx_vadd_h(_in1, _in6); \ + _out2 = __lsx_vadd_h(_in2, _in5); \ + _out3 = __lsx_vadd_h(_in3, _in4); \ + _out4 = __lsx_vsub_h(_in3, _in4); \ + _out5 = __lsx_vsub_h(_in2, _in5); \ + _out6 = __lsx_vsub_h(_in1, _in6); \ + _out7 = __lsx_vsub_h(_in0, _in7); \ +} + +/* + * ============================================================================= + * Description : Print out elements in vector. + * Arguments : Inputs - RTYPE, _element_num, _in0, _enter + * Outputs - + * Details : Print out '_element_num' elements in 'RTYPE' vector '_in0', if + * '_enter' is TRUE, prefix "\nVP:" will be added first. + * Example : VECT_PRINT(v4i32,4,in0,1); // in0: 1,2,3,4 + * VP:1,2,3,4, + * ============================================================================= + */ +#define VECT_PRINT(RTYPE, element_num, in0, enter) \ +{ \ + RTYPE _tmp0 = (RTYPE)in0; \ + int _i = 0; \ + if (enter) \ + printf("\nVP:"); \ + for(_i = 0; _i < element_num; _i++) \ + printf("%d,",_tmp0[_i]); \ +} + +#endif /* GENERIC_MACROS_LSX_H */ +#endif /* AVUTIL_LOONGARCH_GENERIC_MACROS_LSX_H */ diff --git a/libavutil/loongarch/generic_macros_lsx.h.bak b/libavutil/loongarch/generic_macros_lsx.h.bak new file mode 100644 index 0000000000..5d3eef9cf1 --- /dev/null +++ b/libavutil/loongarch/generic_macros_lsx.h.bak @@ -0,0 +1,425 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * All rights reserved. + * Contributed by Shiyou Yin + * Xiwei Gu + * Lu Wang + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#ifndef AVUTIL_LOONGARCH_GENERIC_MACROS_LSX_H +#define AVUTIL_LOONGARCH_GENERIC_MACROS_LSX_H + +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * All rights reserved. + * Contributed by Shiyou Yin + * Xiwei Gu + * Lu Wang + * + * This file is maintained in LSOM project, don't change it directly. + * You can get the latest version of this header from: *** + * + */ + +#ifndef GENERIC_MACROS_LSX_H +#define GENERIC_MACROS_LSX_H + +#include + +/** + * MAJOR version: Macro usage changes. + * MINOR version: Add new functions, or bug fix. + * MICRO version: Comment changes or implementation changes. + */ +#define LSOM_LSX_VERSION_MAJOR 0 +#define LSOM_LSX_VERSION_MINOR 4 +#define LSOM_LSX_VERSION_MICRO 0 + +#define LSX_DUP2_ARG1(_LSX_INS, _IN0, _IN1, _OUT0, _OUT1) \ +{ \ + _OUT0 = _LSX_INS(_IN0); \ + _OUT1 = _LSX_INS(_IN1); \ +} + +#define LSX_DUP2_ARG2(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1) \ +{ \ + _OUT0 = _LSX_INS(_IN0, _IN1); \ + _OUT1 = _LSX_INS(_IN2, _IN3); \ +} + +#define LSX_DUP2_ARG3(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1) \ +{ \ + _OUT0 = _LSX_INS(_IN0, _IN1, _IN2); \ + _OUT1 = _LSX_INS(_IN3, _IN4, _IN5); \ +} + +#define LSX_DUP4_ARG1(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1, _OUT2, _OUT3) \ +{ \ + LSX_DUP2_ARG1(_LSX_INS, _IN0, _IN1, _OUT0, _OUT1); \ + LSX_DUP2_ARG1(_LSX_INS, _IN2, _IN3, _OUT2, _OUT3); \ +} + +#define LSX_DUP4_ARG2(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, \ + _OUT0, _OUT1, _OUT2, _OUT3) \ +{ \ + LSX_DUP2_ARG2(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1); \ + LSX_DUP2_ARG2(_LSX_INS, _IN4, _IN5, _IN6, _IN7, _OUT2, _OUT3); \ +} + +#define LSX_DUP4_ARG3(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, \ + _IN8, _IN9, _IN10, _IN11, _OUT0, _OUT1, _OUT2, _OUT3) \ +{ \ + LSX_DUP2_ARG3(_LSX_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1); \ + LSX_DUP2_ARG3(_LSX_INS, _IN6, _IN7, _IN8, _IN9, _IN10, _IN11, _OUT2, _OUT3); \ +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Then the results plus to signed half word elements from in_c. + * Example : out = __lsx_dp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_dp2add_h_b(__m128i in_c, __m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmaddwev_h_b(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * unsigned byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * The results plus to signed half word elements from in_c. + * Example : out = __lsx_dp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_dp2add_h_bu(__m128i in_c, __m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmaddwev_h_bu(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Example : out = __lsx_dp2_h_b(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22, 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_dp2_h_b(__m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmulwev_h_b(in_h, in_l); + out = __lsx_vmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Example : out = __lsx_dp2_h_bu_b(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,-1 + * out : 22,38,38,22, 22,38,38,6 + * ============================================================================= + */ +static inline __m128i __lsx_dp2_h_bu_b(__m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmulwev_h_bu_b(in_h, in_l); + out = __lsx_vmaddwod_h_bu_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Retrun Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Example : out = __lsx_dp2_w_h(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_dp2_w_h(__m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmulwev_w_h(in_h, in_l); + out = __lsx_vmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of half word vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Retrun Type - __m128i + * Details : Signed half word elements from in_h are multiplied by + * signed half word elements from in_l, and then added adjacent to + * each other to get results with the twice size of input. + * Then the results plus to signed word elements from in_c. + * Example : out = __lsx_dp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_dp2add_w_h(__m128i in_c, __m128i in_h, __m128i in_l) +{ + __m128i out; + + out = __lsx_vmaddwev_w_h(in_c, in_h, in_l); + out = __lsx_vmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Set each element of vector between 0 and 255 + * Arguments : Inputs - _in + * Outputs - out + * Retrun Type - halfword + * Details : Signed byte elements from _in are clamped between 0 and 255. + * Example : out = __lsx_clamp255_h(_in) + * _in : -8,255,280,249, -8,255,280,249 + * out : 0,255,255,249, 0,255,255,249 + * ============================================================================= + */ +static inline __m128i __lsx_clamp255_h(__m128i _in) +{ + __m128i out; + + out = __lsx_vmaxi_h(_in, 0); + out = __lsx_vsat_hu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Set each element of vector between 0 and 255 + * Arguments : Inputs - _in + * Outputs - out + * Retrun Type - word + * Details : Signed byte elements from _in are clamped between 0 and 255. + * Example : out = __lsx_clamp255_w(_in) + * _in : -8,255,280,249 + * out : 0,255,255,249 + * ============================================================================= + */ +static inline __m128i __lsx_clamp255_w(__m128i _in) +{ + __m128i out; + + out = __lsx_vmaxi_w(_in, 0); + out = __lsx_vsat_wu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Transposes 4x4 block with word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : + * Example : + * 1, 2, 3, 4 1, 5, 9,13 + * 5, 6, 7, 8 to 2, 6,10,14 + * 9,10,11,12 =====> 3, 7,11,15 + * 13,14,15,16 4, 8,12,16 + * ============================================================================= + */ +#define TRANSPOSE4x4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + __m128i _t0, _t1, _t2, _t3; \ + \ + _t0 = __lsx_vilvl_w(_in1, _in0); \ + _t1 = __lsx_vilvh_w(_in1, _in0); \ + _t2 = __lsx_vilvl_w(_in3, _in2); \ + _t3 = __lsx_vilvh_w(_in3, _in2); \ + _out0 = __lsx_vilvl_d(_t2, _t0); \ + _out1 = __lsx_vilvh_d(_t2, _t0); \ + _out2 = __lsx_vilvl_d(_t3, _t1); \ + _out3 = __lsx_vilvh_d(_t3, _t1); \ +} + +/* + * ============================================================================= + * Description : Transposes 8x8 block with half word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * Details : + * Example : + * 00,01,02,03,04,05,06,07 00,10,20,30,40,50,60,70 + * 10,11,12,13,14,15,16,17 01,11,21,31,41,51,61,71 + * 20,21,22,23,24,25,26,27 02,12,22,32,42,52,62,72 + * 30,31,32,33,34,35,36,37 to 03,13,23,33,43,53,63,73 + * 40,41,42,43,44,45,46,47 ======> 04,14,24,34,44,54,64,74 + * 50,51,52,53,54,55,56,57 05,15,25,35,45,55,65,75 + * 60,61,62,63,64,65,66,67 06,16,26,36,46,56,66,76 + * 70,71,72,73,74,75,76,77 07,17,27,37,47,57,67,77 + * ============================================================================= + */ +#define TRANSPOSE8x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, _out7) \ +{ \ + __m128i _s0, _s1, _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + \ + _s0 = __lsx_vilvl_h(_in6, _in4); \ + _s1 = __lsx_vilvl_h(_in7, _in5); \ + _t0 = __lsx_vilvl_h(_s1, _s0); \ + _t1 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvh_h(_in6, _in4); \ + _s1 = __lsx_vilvh_h(_in7, _in5); \ + _t2 = __lsx_vilvl_h(_s1, _s0); \ + _t3 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvl_h(_in2, _in0); \ + _s1 = __lsx_vilvl_h(_in3, _in1); \ + _t4 = __lsx_vilvl_h(_s1, _s0); \ + _t5 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvh_h(_in2, _in0); \ + _s1 = __lsx_vilvh_h(_in3, _in1); \ + _t6 = __lsx_vilvl_h(_s1, _s0); \ + _t7 = __lsx_vilvh_h(_s1, _s0); \ + \ + _out0 = __lsx_vpickev_d(_t0, _t4); \ + _out2 = __lsx_vpickev_d(_t1, _t5); \ + _out4 = __lsx_vpickev_d(_t2, _t6); \ + _out6 = __lsx_vpickev_d(_t3, _t7); \ + _out1 = __lsx_vpickod_d(_t0, _t4); \ + _out3 = __lsx_vpickod_d(_t1, _t5); \ + _out5 = __lsx_vpickod_d(_t2, _t6); \ + _out7 = __lsx_vpickod_d(_t3, _t7); \ +} + +/* + * ============================================================================= + * Description : Butterfly of 4 input vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : Butterfly operation + * Example : + * out0 = in0 + in3; + * out1 = in1 + in2; + * out2 = in1 - in2; + * out3 = in0 - in3; + * ============================================================================= + */ +#define BUTTERFLY_4_B(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + _out0 = __lsx_vadd_b(_in0, _in3); \ + _out1 = __lsx_vadd_b(_in1, _in2); \ + _out2 = __lsx_vsub_b(_in1, _in2); \ + _out3 = __lsx_vsub_b(_in0, _in3); \ +} +#define BUTTERFLY_4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + _out0 = __lsx_vadd_h(_in0, _in3); \ + _out1 = __lsx_vadd_h(_in1, _in2); \ + _out2 = __lsx_vsub_h(_in1, _in2); \ + _out3 = __lsx_vsub_h(_in0, _in3); \ +} +#define BUTTERFLY_4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + _out0 = __lsx_vadd_w(_in0, _in3); \ + _out1 = __lsx_vadd_w(_in1, _in2); \ + _out2 = __lsx_vsub_w(_in1, _in2); \ + _out3 = __lsx_vsub_w(_in0, _in3); \ +} +#define BUTTERFLY_4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ +{ \ + _out0 = __lsx_vadd_d(_in0, _in3); \ + _out1 = __lsx_vadd_d(_in1, _in2); \ + _out2 = __lsx_vsub_d(_in1, _in2); \ + _out3 = __lsx_vsub_d(_in0, _in3); \ +} + +/* + * ============================================================================= + * Description : Print out elements in vector. + * Arguments : Inputs - RTYPE, _element_num, _in0, _enter + * Outputs - + * Details : Print out '_element_num' elements in 'RTYPE' vector '_in0', if + * '_enter' is TRUE, prefix "\nVP:" will be added first. + * Example : VECT_PRINT(v4i32,4,in0,1); // in0: 1,2,3,4 + * VP:1,2,3,4, + * ============================================================================= + */ +#define VECT_PRINT(RTYPE, element_num, in0, enter) \ +{ \ + RTYPE _tmp0 = (RTYPE)in0; \ + int _i = 0; \ + if (enter) \ + printf("\nVP:"); \ + for(_i = 0; _i < element_num; _i++) \ + printf("%d,",_tmp0[_i]); \ +} + +#endif /* GENERIC_MACROS_LSX_H */ +#endif /* AVUTIL_LOONGARCH_GENERIC_MACROS_LSX_H */ diff --git a/libavutil/mips/asmdefs.h b/libavutil/mips/asmdefs.h index 76bb2b93fa..659342bab9 100644 --- a/libavutil/mips/asmdefs.h +++ b/libavutil/mips/asmdefs.h @@ -27,6 +27,8 @@ #ifndef AVUTIL_MIPS_ASMDEFS_H #define AVUTIL_MIPS_ASMDEFS_H +#include + #if defined(_ABI64) && _MIPS_SIM == _ABI64 # define mips_reg int64_t # define PTRSIZE " 8 " @@ -97,4 +99,10 @@ __asm__(".macro parse_r var r\n\t" ".endif\n\t" ".endm"); +/* General union structure for clang adaption */ +union mmi_intfloat64 { + int64_t i; + double f; +}; + #endif /* AVCODEC_MIPS_ASMDEFS_H */ diff --git a/libavutil/mips/generic_macros_msa.h b/libavutil/mips/generic_macros_msa.h index bb25e9fd74..1486f7296e 100644 --- a/libavutil/mips/generic_macros_msa.h +++ b/libavutil/mips/generic_macros_msa.h @@ -25,10 +25,6 @@ #include #include -#if HAVE_MSA2 -#include -#endif - #define ALIGNMENT 16 #define ALLOC_ALIGNED(align) __attribute__ ((aligned((align) << 1))) @@ -1119,15 +1115,6 @@ unsigned absolute diff values, even-odd pairs are added together to generate 8 halfword results. */ -#if HAVE_MSA2 -#define SAD_UB2_UH(in0, in1, ref0, ref1) \ -( { \ - v8u16 sad_m = { 0 }; \ - sad_m += __builtin_msa2_sad_adj2_u_w2x_b((v16u8) in0, (v16u8) ref0); \ - sad_m += __builtin_msa2_sad_adj2_u_w2x_b((v16u8) in1, (v16u8) ref1); \ - sad_m; \ -} ) -#else #define SAD_UB2_UH(in0, in1, ref0, ref1) \ ( { \ v16u8 diff0_m, diff1_m; \ @@ -1141,7 +1128,6 @@ \ sad_m; \ } ) -#endif // #if HAVE_MSA2 /* Description : Insert specified word elements from input vectors to 1 destination vector @@ -2183,12 +2169,6 @@ extracted and interleaved with same vector 'in0' to generate 4 word elements keeping sign intact */ -#if HAVE_MSA2 -#define UNPCK_R_SH_SW(in, out) \ -{ \ - out = (v4i32) __builtin_msa2_w2x_lo_s_h((v8i16) in); \ -} -#else #define UNPCK_R_SH_SW(in, out) \ { \ v8i16 sign_m; \ @@ -2196,7 +2176,6 @@ sign_m = __msa_clti_s_h((v8i16) in, 0); \ out = (v4i32) __msa_ilvr_h(sign_m, (v8i16) in); \ } -#endif // #if HAVE_MSA2 /* Description : Sign extend byte elements from input vector and return halfword results in pair of vectors @@ -2209,13 +2188,6 @@ Then interleaved left with same vector 'in0' to generate 8 signed halfword elements in 'out1' */ -#if HAVE_MSA2 -#define UNPCK_SB_SH(in, out0, out1) \ -{ \ - out0 = (v4i32) __builtin_msa2_w2x_lo_s_b((v16i8) in); \ - out1 = (v4i32) __builtin_msa2_w2x_hi_s_b((v16i8) in); \ -} -#else #define UNPCK_SB_SH(in, out0, out1) \ { \ v16i8 tmp_m; \ @@ -2223,7 +2195,6 @@ tmp_m = __msa_clti_s_b((v16i8) in, 0); \ ILVRL_B2_SH(tmp_m, in, out0, out1); \ } -#endif // #if HAVE_MSA2 /* Description : Zero extend unsigned byte elements to halfword elements Arguments : Inputs - in (1 input unsigned byte vector) @@ -2250,13 +2221,6 @@ Then interleaved left with same vector 'in0' to generate 4 signed word elements in 'out1' */ -#if HAVE_MSA2 -#define UNPCK_SH_SW(in, out0, out1) \ -{ \ - out0 = (v4i32) __builtin_msa2_w2x_lo_s_h((v8i16) in); \ - out1 = (v4i32) __builtin_msa2_w2x_hi_s_h((v8i16) in); \ -} -#else #define UNPCK_SH_SW(in, out0, out1) \ { \ v8i16 tmp_m; \ @@ -2264,7 +2228,6 @@ tmp_m = __msa_clti_s_h((v8i16) in, 0); \ ILVRL_H2_SW(tmp_m, in, out0, out1); \ } -#endif // #if HAVE_MSA2 /* Description : Swap two variables Arguments : Inputs - in0, in1 diff --git a/libavutil/tests/cpu.c b/libavutil/tests/cpu.c index c853371fb3..0a6c0cd32e 100644 --- a/libavutil/tests/cpu.c +++ b/libavutil/tests/cpu.c @@ -77,6 +77,9 @@ static const struct { { AV_CPU_FLAG_BMI2, "bmi2" }, { AV_CPU_FLAG_AESNI, "aesni" }, { AV_CPU_FLAG_AVX512, "avx512" }, +#elif ARCH_LOONGARCH + { AV_CPU_FLAG_LSX, "lsx" }, + { AV_CPU_FLAG_LASX, "lasx" }, #endif { 0 } }; diff --git a/libswscale/loongarch/Makefile b/libswscale/loongarch/Makefile new file mode 100644 index 0000000000..47dae42ff7 --- /dev/null +++ b/libswscale/loongarch/Makefile @@ -0,0 +1,6 @@ +OBJS-$(CONFIG_SWSCALE) += loongarch/swscale_init_loongarch.o +LASX-OBJS-$(CONFIG_SWSCALE) += loongarch/swscale_lasx.o \ + loongarch/output_lasx.o \ + loongarch/input_lasx.o \ + loongarch/yuv2rgb_lasx.o \ + loongarch/rgb2rgb_lasx.o diff --git a/libswscale/loongarch/input_lasx.c b/libswscale/loongarch/input_lasx.c new file mode 100644 index 0000000000..49b37cd0f6 --- /dev/null +++ b/libswscale/loongarch/input_lasx.c @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen(chenhao@loongson.cn) + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "swscale_loongarch.h" +#include "libavutil/loongarch/generic_macros_lasx.h" + +void planar_rgb_to_uv_lasx(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[4], + int width, int32_t *rgb2yuv) +{ + int i; + uint16_t *dstU = (uint16_t *)_dstU; + uint16_t *dstV = (uint16_t *)_dstV; + int set = 0x4001 << (RGB2YUV_SHIFT - 7); + int len = width - 15; + int32_t tem_ru = rgb2yuv[RU_IDX], tem_gu = rgb2yuv[GU_IDX], tem_bu = rgb2yuv[BU_IDX]; + int32_t tem_rv = rgb2yuv[RV_IDX], tem_gv = rgb2yuv[GV_IDX], tem_bv = rgb2yuv[BV_IDX]; + int shift = RGB2YUV_SHIFT - 6; + const uint8_t *src0 = src[0], *src1 = src[1], *src2 = src[2]; + __m256i ru, gu, bu, rv, gv, bv; + __m256i mask = {0x0D0C090805040100, 0x1D1C191815141110, 0x0D0C090805040100, 0x1D1C191815141110}; + __m256i temp = __lasx_xvreplgr2vr_w(set); + __m256i sra = __lasx_xvreplgr2vr_w(shift); + + ru = __lasx_xvreplgr2vr_w(tem_ru); + gu = __lasx_xvreplgr2vr_w(tem_gu); + bu = __lasx_xvreplgr2vr_w(tem_bu); + rv = __lasx_xvreplgr2vr_w(tem_rv); + gv = __lasx_xvreplgr2vr_w(tem_gv); + bv = __lasx_xvreplgr2vr_w(tem_bv); + for (i = 0; i < len; i += 16) { + __m256i _g, _b, _r; + __m256i g_l, g_h, b_l, b_h, r_l, r_h; + __m256i v_l, v_h, u_l, u_h, u_lh, v_lh; + + _g = LASX_LD((src0 + i)); + _b = LASX_LD((src1 + i)); + _r = LASX_LD((src2 + i)); + LASX_UNPCK_L_WU_BU(_g, g_l); + LASX_UNPCK_L_WU_BU_2(_b, _r, b_l, r_l); + _g = __lasx_xvpermi_d(_g, 0x01); + _b = __lasx_xvpermi_d(_b, 0x01); + _r = __lasx_xvpermi_d(_r, 0x01); + LASX_UNPCK_L_WU_BU(_g, g_h); + LASX_UNPCK_L_WU_BU_2(_b, _r, b_h, r_h); + u_l = __lasx_xvmadd_w(temp, ru, r_l); + u_h = __lasx_xvmadd_w(temp, ru, r_h); + v_l = __lasx_xvmadd_w(temp, rv, r_l); + v_h = __lasx_xvmadd_w(temp, rv, r_h); + u_l = __lasx_xvmadd_w(u_l, gu, g_l); + u_l = __lasx_xvmadd_w(u_l, bu, b_l); + u_h = __lasx_xvmadd_w(u_h, gu, g_h); + u_h = __lasx_xvmadd_w(u_h, bu, b_h); + v_l = __lasx_xvmadd_w(v_l, gv, g_l); + v_l = __lasx_xvmadd_w(v_l, bv, b_l); + v_h = __lasx_xvmadd_w(v_h, gv, g_h); + v_h = __lasx_xvmadd_w(v_h, bv, b_h); + u_l = __lasx_xvsra_w(u_l, sra); + u_h = __lasx_xvsra_w(u_h, sra); + v_l = __lasx_xvsra_w(v_l, sra); + v_h = __lasx_xvsra_w(v_h, sra); + LASX_SHUF_B_2_128SV(u_h, u_l, v_h, v_l, mask, mask, u_lh, v_lh); + u_lh = __lasx_xvpermi_d(u_lh, 0xD8); + v_lh = __lasx_xvpermi_d(v_lh, 0xD8); + __lasx_xvst(u_lh, (dstU + i), 0); + __lasx_xvst(v_lh, (dstV + i), 0); + } + if (width - i >= 8) { + __m256i _g, _b, _r; + __m256i g_l, b_l, r_l; + __m256i v_l, u_l, u, v; + + _g = __lasx_xvldrepl_d((src0 + i), 0); + _b = __lasx_xvldrepl_d((src1 + i), 0); + _r = __lasx_xvldrepl_d((src2 + i), 0); + LASX_UNPCK_L_WU_BU(_g, g_l); + LASX_UNPCK_L_WU_BU_2(_b, _r, b_l, r_l); + u_l = __lasx_xvmadd_w(temp, ru, r_l); + v_l = __lasx_xvmadd_w(temp, rv, r_l); + u_l = __lasx_xvmadd_w(u_l, gu, g_l); + u_l = __lasx_xvmadd_w(u_l, bu, b_l); + v_l = __lasx_xvmadd_w(v_l, gv, g_l); + v_l = __lasx_xvmadd_w(v_l, bv, b_l); + u_l = __lasx_xvsra_w(u_l, sra); + v_l = __lasx_xvsra_w(v_l, sra); + LASX_SHUF_B_2_128SV(u_l, u_l, v_l, v_l, mask, mask, u, v); + __lasx_xvstelm_d(u, (dstU + i), 0, 0); + __lasx_xvstelm_d(u, (dstU + i), 8, 2); + __lasx_xvstelm_d(v, (dstV + i), 0, 0); + __lasx_xvstelm_d(v, (dstV + i), 8, 2); + i += 8; + } + for (; i < width; i++) { + int g = src[0][i]; + int b = src[1][i]; + int r = src[2][i]; + + dstU[i] = (tem_ru * r + tem_gu * g + tem_bu * b + set) >> shift; + dstV[i] = (tem_rv * r + tem_gv * g + tem_bv * b + set) >> shift; + } +} + +void planar_rgb_to_y_lasx(uint8_t *_dst, const uint8_t *src[4], int width, + int32_t *rgb2yuv) +{ + int i; + int shift = (RGB2YUV_SHIFT - 6); + int set = 0x801 << (RGB2YUV_SHIFT - 7); + int len = width - 15; + uint16_t *dst = (uint16_t *)_dst; + int32_t tem_ry = rgb2yuv[RY_IDX], tem_gy = rgb2yuv[GY_IDX]; + int32_t tem_by = rgb2yuv[BY_IDX]; + const uint8_t *src0 = src[0], *src1 = src[1], *src2 = src[2]; + __m256i mask = {0x0D0C090805040100, 0x1D1C191815141110, 0x0D0C090805040100, 0x1D1C191815141110}; + __m256i temp = __lasx_xvreplgr2vr_w(set); + __m256i sra = __lasx_xvreplgr2vr_w(shift); + __m256i ry = __lasx_xvreplgr2vr_w(tem_ry); + __m256i gy = __lasx_xvreplgr2vr_w(tem_gy); + __m256i by = __lasx_xvreplgr2vr_w(tem_by); + + for (i = 0; i < len; i += 16) { + __m256i _g, _b, _r; + __m256i g_l, g_h, b_l, b_h, r_l, r_h; + __m256i y_l, y_h, y_lh; + + _g = LASX_LD((src0 + i)); + _b = LASX_LD((src1 + i)); + _r = LASX_LD((src2 + i)); + LASX_UNPCK_L_WU_BU(_g, g_l); + LASX_UNPCK_L_WU_BU_2(_b, _r, b_l, r_l); + _g = __lasx_xvpermi_d(_g, 0x01); + _b = __lasx_xvpermi_d(_b, 0x01); + _r = __lasx_xvpermi_d(_r, 0x01); + LASX_UNPCK_L_WU_BU(_g, g_h); + LASX_UNPCK_L_WU_BU_2(_b, _r, b_h, r_h); + y_l = __lasx_xvmadd_w(temp, ry, r_l); + y_h = __lasx_xvmadd_w(temp, ry, r_h); + y_l = __lasx_xvmadd_w(y_l, gy, g_l); + y_l = __lasx_xvmadd_w(y_l, by, b_l); + y_h = __lasx_xvmadd_w(y_h, gy, g_h); + y_h = __lasx_xvmadd_w(y_h, by, b_h); + y_l = __lasx_xvsra_w(y_l, sra); + y_h = __lasx_xvsra_w(y_h, sra); + LASX_SHUF_B_128SV(y_h, y_l, mask, y_lh); + y_lh = __lasx_xvpermi_d(y_lh, 0xD8); + __lasx_xvst(y_lh, (dst + i), 0); + } + if (width - i >= 8) { + __m256i _g, _b, _r; + __m256i g_l, b_l, r_l; + __m256i y_l, y; + + _g = __lasx_xvldrepl_d((src0 + i), 0); + _b = __lasx_xvldrepl_d((src1 + i), 0); + _r = __lasx_xvldrepl_d((src2 + i), 0); + LASX_UNPCK_L_WU_BU(_g, g_l); + LASX_UNPCK_L_WU_BU_2(_b, _r, b_l, r_l); + y_l = __lasx_xvmadd_w(temp, ry, r_l); + y_l = __lasx_xvmadd_w(y_l, gy, g_l); + y_l = __lasx_xvmadd_w(y_l, by, b_l); + y_l = __lasx_xvsra_w(y_l, sra); + LASX_SHUF_B_128SV(y_l, y_l, mask, y); + __lasx_xvstelm_d(y, (dst + i), 0, 0); + __lasx_xvstelm_d(y, (dst + i), 8, 2); + i += 8; + } + for (; i < width; i++) { + int g = src[0][i]; + int b = src[1][i]; + int r = src[2][i]; + + dst[i] = (tem_ry * r + tem_gy * g + tem_by * b + set) >> shift; + } +} diff --git a/libswscale/loongarch/output_lasx.c b/libswscale/loongarch/output_lasx.c new file mode 100644 index 0000000000..b3fb1f5ce0 --- /dev/null +++ b/libswscale/loongarch/output_lasx.c @@ -0,0 +1,2436 @@ +/* + * Copyright (C) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen(chenhao@loongson.cn) + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "swscale_loongarch.h" +#include "libavutil/loongarch/generic_macros_lasx.h" + +void ff_yuv2planeX_8_lasx(const int16_t *filter, int filterSize, + const int16_t **src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset) +{ + int i; + int len = dstW - 15; + __m256i mask = {0x1C1814100C080400, 0x0101010101010101, 0x1C1814100C080400, 0x0101010101010101}; + __m256i val1, val2, val3; + uint8_t dither0 = dither[offset & 7]; + uint8_t dither1 = dither[(offset + 1) & 7]; + uint8_t dither2 = dither[(offset + 2) & 7]; + uint8_t dither3 = dither[(offset + 3) & 7]; + uint8_t dither4 = dither[(offset + 4) & 7]; + uint8_t dither5 = dither[(offset + 5) & 7]; + uint8_t dither6 = dither[(offset + 6) & 7]; + uint8_t dither7 = dither[(offset + 7) & 7]; + int val_1[8] = {dither0, dither1, dither2, dither3, dither0, dither1, dither2, dither3}; + int val_2[8] = {dither4, dither5, dither6, dither7, dither4, dither5, dither6, dither7}; + int val_3[8] = {dither0, dither1, dither2, dither3, dither4, dither5, dither6, dither7}; + + val1 = LASX_LD(val_1); + val2 = LASX_LD(val_2); + val3 = LASX_LD(val_3); + + for (i = 0; i < len; i += 16) { + int j; + __m256i src0, filter0, val_lh; + __m256i val_l, val_h; + + val_l = __lasx_xvslli_w(val1, 12); + val_h = __lasx_xvslli_w(val2, 12); + + for (j = 0; j < filterSize; j++) { + src0 = LASX_LD(src[j]+ i); + filter0 = __lasx_xvldrepl_h((filter + j), 0); + LASX_MADDWL_W_H_128SV(val_l, src0, filter0, val_l); + LASX_MADDWH_W_H_128SV(val_h, src0, filter0, val_h); + } + val_l = __lasx_xvsrai_w(val_l, 19); + val_h = __lasx_xvsrai_w(val_h, 19); + LASX_CLIP_W_0_255(val_l, val_l); + LASX_CLIP_W_0_255(val_h, val_h); + LASX_SHUF_B_128SV(val_h, val_l, mask, val_lh); + __lasx_xvstelm_d(val_lh, (dest + i), 0, 0); + __lasx_xvstelm_d(val_lh, (dest + i), 8, 2); + } + if (dstW - i >= 8){ + int j; + __m256i src0, filter0, val_h; + __m256i val_l; + + val_l = __lasx_xvslli_w(val3, 12); + + for (j = 0; j < filterSize; j++) { + src0 = LASX_LD(src[j] + i); + src0 = __lasx_xvpermi_d(src0, 0xD8); + filter0 = __lasx_xvldrepl_h((filter + j), 0); + LASX_MADDWL_W_H_128SV(val_l, src0, filter0, val_l); + } + val_l = __lasx_xvsrai_w(val_l, 19); + LASX_CLIP_W_0_255(val_l, val_l); + val_h = __lasx_xvpermi_d(val_l, 0x4E); + LASX_SHUF_B_128SV(val_h, val_l, mask, val_l); + __lasx_xvstelm_d(val_l, (dest + i), 0, 0); + i += 8; + } + for (; i < dstW; i++) { + int val = dither[(i + offset) & 7] << 12; + int j; + for (j = 0; j< filterSize; j++) + val += src[j][i] * filter[j]; + + dest[i] = av_clip_uint8(val >> 19); + } +} + +/*Copy from libswscale/output.c*/ +static av_always_inline void +yuv2rgb_write(uint8_t *_dest, int i, int Y1, int Y2, + unsigned A1, unsigned A2, + const void *_r, const void *_g, const void *_b, int y, + enum AVPixelFormat target, int hasAlpha) +{ + if (target == AV_PIX_FMT_ARGB || target == AV_PIX_FMT_RGBA || + target == AV_PIX_FMT_ABGR || target == AV_PIX_FMT_BGRA) { + uint32_t *dest = (uint32_t *) _dest; + const uint32_t *r = (const uint32_t *) _r; + const uint32_t *g = (const uint32_t *) _g; + const uint32_t *b = (const uint32_t *) _b; + +#if CONFIG_SMALL + dest[i * 2 + 0] = r[Y1] + g[Y1] + b[Y1]; + dest[i * 2 + 1] = r[Y2] + g[Y2] + b[Y2]; +#else +#if defined(ASSERT_LEVEL) && ASSERT_LEVEL > 1 + int sh = (target == AV_PIX_FMT_RGB32_1 || + target == AV_PIX_FMT_BGR32_1) ? 0 : 24; + av_assert2((((r[Y1] + g[Y1] + b[Y1]) >> sh) & 0xFF) == 0xFF); +#endif + dest[i * 2 + 0] = r[Y1] + g[Y1] + b[Y1]; + dest[i * 2 + 1] = r[Y2] + g[Y2] + b[Y2]; +#endif + } else if (target == AV_PIX_FMT_RGB24 || target == AV_PIX_FMT_BGR24) { + uint8_t *dest = (uint8_t *) _dest; + const uint8_t *r = (const uint8_t *) _r; + const uint8_t *g = (const uint8_t *) _g; + const uint8_t *b = (const uint8_t *) _b; + +#define r_b ((target == AV_PIX_FMT_RGB24) ? r : b) +#define b_r ((target == AV_PIX_FMT_RGB24) ? b : r) + + dest[i * 6 + 0] = r_b[Y1]; + dest[i * 6 + 1] = g[Y1]; + dest[i * 6 + 2] = b_r[Y1]; + dest[i * 6 + 3] = r_b[Y2]; + dest[i * 6 + 4] = g[Y2]; + dest[i * 6 + 5] = b_r[Y2]; +#undef r_b +#undef b_r + } else if (target == AV_PIX_FMT_RGB565 || target == AV_PIX_FMT_BGR565 || + target == AV_PIX_FMT_RGB555 || target == AV_PIX_FMT_BGR555 || + target == AV_PIX_FMT_RGB444 || target == AV_PIX_FMT_BGR444) { + uint16_t *dest = (uint16_t *) _dest; + const uint16_t *r = (const uint16_t *) _r; + const uint16_t *g = (const uint16_t *) _g; + const uint16_t *b = (const uint16_t *) _b; + int dr1, dg1, db1, dr2, dg2, db2; + + if (target == AV_PIX_FMT_RGB565 || target == AV_PIX_FMT_BGR565) { + dr1 = ff_dither_2x2_8[ y & 1 ][0]; + dg1 = ff_dither_2x2_4[ y & 1 ][0]; + db1 = ff_dither_2x2_8[(y & 1) ^ 1][0]; + dr2 = ff_dither_2x2_8[ y & 1 ][1]; + dg2 = ff_dither_2x2_4[ y & 1 ][1]; + db2 = ff_dither_2x2_8[(y & 1) ^ 1][1]; + } else if (target == AV_PIX_FMT_RGB555 || target == AV_PIX_FMT_BGR555) { + dr1 = ff_dither_2x2_8[ y & 1 ][0]; + dg1 = ff_dither_2x2_8[ y & 1 ][1]; + db1 = ff_dither_2x2_8[(y & 1) ^ 1][0]; + dr2 = ff_dither_2x2_8[ y & 1 ][1]; + dg2 = ff_dither_2x2_8[ y & 1 ][0]; + db2 = ff_dither_2x2_8[(y & 1) ^ 1][1]; + } else { + dr1 = ff_dither_4x4_16[ y & 3 ][0]; + dg1 = ff_dither_4x4_16[ y & 3 ][1]; + db1 = ff_dither_4x4_16[(y & 3) ^ 3][0]; + dr2 = ff_dither_4x4_16[ y & 3 ][1]; + dg2 = ff_dither_4x4_16[ y & 3 ][0]; + db2 = ff_dither_4x4_16[(y & 3) ^ 3][1]; + } + + dest[i * 2 + 0] = r[Y1 + dr1] + g[Y1 + dg1] + b[Y1 + db1]; + dest[i * 2 + 1] = r[Y2 + dr2] + g[Y2 + dg2] + b[Y2 + db2]; + } else /* 8/4 bits */ { + uint8_t *dest = (uint8_t *) _dest; + const uint8_t *r = (const uint8_t *) _r; + const uint8_t *g = (const uint8_t *) _g; + const uint8_t *b = (const uint8_t *) _b; + int dr1, dg1, db1, dr2, dg2, db2; + + if (target == AV_PIX_FMT_RGB8 || target == AV_PIX_FMT_BGR8) { + const uint8_t * const d64 = ff_dither_8x8_73[y & 7]; + const uint8_t * const d32 = ff_dither_8x8_32[y & 7]; + dr1 = dg1 = d32[(i * 2 + 0) & 7]; + db1 = d64[(i * 2 + 0) & 7]; + dr2 = dg2 = d32[(i * 2 + 1) & 7]; + db2 = d64[(i * 2 + 1) & 7]; + } else { + const uint8_t * const d64 = ff_dither_8x8_73 [y & 7]; + const uint8_t * const d128 = ff_dither_8x8_220[y & 7]; + dr1 = db1 = d128[(i * 2 + 0) & 7]; + dg1 = d64[(i * 2 + 0) & 7]; + dr2 = db2 = d128[(i * 2 + 1) & 7]; + dg2 = d64[(i * 2 + 1) & 7]; + } + + if (target == AV_PIX_FMT_RGB4 || target == AV_PIX_FMT_BGR4) { + dest[i] = r[Y1 + dr1] + g[Y1 + dg1] + b[Y1 + db1] + + ((r[Y2 + dr2] + g[Y2 + dg2] + b[Y2 + db2]) << 4); + } else { + dest[i * 2 + 0] = r[Y1 + dr1] + g[Y1 + dg1] + b[Y1 + db1]; + dest[i * 2 + 1] = r[Y2 + dr2] + g[Y2 + dg2] + b[Y2 + db2]; + } + } +} + +#define WRITE_YUV2RGB_16(y_l, y_h, u, v, count, r, g, b, y, \ + target, Y1, Y2, U, V) \ +{ \ + Y1 = __lasx_xvpickve2gr_w(y_l, 0); \ + Y2 = __lasx_xvpickve2gr_w(y_l, 1); \ + U = __lasx_xvpickve2gr_w(u, 0); \ + V = __lasx_xvpickve2gr_w(v, 0); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_l, 2); \ + Y2 = __lasx_xvpickve2gr_w(y_l, 3); \ + U = __lasx_xvpickve2gr_w(u, 1); \ + V = __lasx_xvpickve2gr_w(v, 1); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_h, 0); \ + Y2 = __lasx_xvpickve2gr_w(y_h, 1); \ + U = __lasx_xvpickve2gr_w(u, 2); \ + V = __lasx_xvpickve2gr_w(v, 2); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_h, 2); \ + Y2 = __lasx_xvpickve2gr_w(y_h, 3); \ + U = __lasx_xvpickve2gr_w(u, 3); \ + V = __lasx_xvpickve2gr_w(v, 3); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_l, 4); \ + Y2 = __lasx_xvpickve2gr_w(y_l, 5); \ + U = __lasx_xvpickve2gr_w(u, 4); \ + V = __lasx_xvpickve2gr_w(v, 4); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_l, 6); \ + Y2 = __lasx_xvpickve2gr_w(y_l, 7); \ + U = __lasx_xvpickve2gr_w(u, 5); \ + V = __lasx_xvpickve2gr_w(v, 5); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_h, 4); \ + Y2 = __lasx_xvpickve2gr_w(y_h, 5); \ + U = __lasx_xvpickve2gr_w(u, 6); \ + V = __lasx_xvpickve2gr_w(v, 6); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_h, 6); \ + Y2 = __lasx_xvpickve2gr_w(y_h, 7); \ + U = __lasx_xvpickve2gr_w(u, 7); \ + V = __lasx_xvpickve2gr_w(v, 7); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ +} + +#define WRITE_YUV2RGB_16_L(y_ev, y_od, u_ev, u_od, v_ev, v_od, \ + count, r, g, b, y, target, Y1, Y2, U, V) \ +{ \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 0); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 0); \ + U = __lasx_xvpickve2gr_w(u_ev, 0); \ + V = __lasx_xvpickve2gr_w(v_ev, 0); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 1); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 1); \ + U = __lasx_xvpickve2gr_w(u_od, 0); \ + V = __lasx_xvpickve2gr_w(v_od, 0); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 2); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 2); \ + U = __lasx_xvpickve2gr_w(u_ev, 1); \ + V = __lasx_xvpickve2gr_w(v_ev, 1); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 3); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 3); \ + U = __lasx_xvpickve2gr_w(u_od, 1); \ + V = __lasx_xvpickve2gr_w(v_od, 1); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 4); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 4); \ + U = __lasx_xvpickve2gr_w(u_ev, 2); \ + V = __lasx_xvpickve2gr_w(v_ev, 2); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 5); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 5); \ + U = __lasx_xvpickve2gr_w(u_od, 2); \ + V = __lasx_xvpickve2gr_w(v_od, 2); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 6); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 6); \ + U = __lasx_xvpickve2gr_w(u_ev, 3); \ + V = __lasx_xvpickve2gr_w(v_ev, 3); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 7); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 7); \ + U = __lasx_xvpickve2gr_w(u_od, 3); \ + V = __lasx_xvpickve2gr_w(v_od, 3); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ +} + +#define WRITE_YUV2RGB_16_H(y_ev, y_od, u_ev, u_od, v_ev, v_od, \ + count, r, g, b, y, target, Y1, Y2, U, V) \ +{ \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 0); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 0); \ + U = __lasx_xvpickve2gr_w(u_ev, 4); \ + V = __lasx_xvpickve2gr_w(v_ev, 4); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 1); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 1); \ + U = __lasx_xvpickve2gr_w(u_od, 4); \ + V = __lasx_xvpickve2gr_w(v_od, 4); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 2); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 2); \ + U = __lasx_xvpickve2gr_w(u_ev, 5); \ + V = __lasx_xvpickve2gr_w(v_ev, 5); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 3); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 3); \ + U = __lasx_xvpickve2gr_w(u_od, 5); \ + V = __lasx_xvpickve2gr_w(v_od, 5); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 4); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 4); \ + U = __lasx_xvpickve2gr_w(u_ev, 6); \ + V = __lasx_xvpickve2gr_w(v_ev, 6); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 5); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 5); \ + U = __lasx_xvpickve2gr_w(u_od, 6); \ + V = __lasx_xvpickve2gr_w(v_od, 6); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 6); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 6); \ + U = __lasx_xvpickve2gr_w(u_ev, 7); \ + V = __lasx_xvpickve2gr_w(v_ev, 7); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 7); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 7); \ + U = __lasx_xvpickve2gr_w(u_od, 7); \ + V = __lasx_xvpickve2gr_w(v_od, 7); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ +} + +#define WRITE_YUV2RGB_16_N(y_ev, y_od, u, v, count, \ + r, g, b, y, target, Y1, Y2, U, V) \ +{ \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 0); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 0); \ + U = __lasx_xvpickve2gr_w(u, 0); \ + V = __lasx_xvpickve2gr_w(v, 0); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 1); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 1); \ + U = __lasx_xvpickve2gr_w(u, 1); \ + V = __lasx_xvpickve2gr_w(v, 1); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 2); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 2); \ + U = __lasx_xvpickve2gr_w(u, 2); \ + V = __lasx_xvpickve2gr_w(v, 2); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 3); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 3); \ + U = __lasx_xvpickve2gr_w(u, 3); \ + V = __lasx_xvpickve2gr_w(v, 3); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 4); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 4); \ + U = __lasx_xvpickve2gr_w(u, 4); \ + V = __lasx_xvpickve2gr_w(v, 4); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 5); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 5); \ + U = __lasx_xvpickve2gr_w(u, 5); \ + V = __lasx_xvpickve2gr_w(v, 5); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 6); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 6); \ + U = __lasx_xvpickve2gr_w(u, 6); \ + V = __lasx_xvpickve2gr_w(v, 6); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 7); \ + Y2 = __lasx_xvpickve2gr_w(y_od, 7); \ + U = __lasx_xvpickve2gr_w(u, 7); \ + V = __lasx_xvpickve2gr_w(v, 7); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ +} + +#define WRITE_YUV2RGB_8_N(y_ev, uv, count, r, g, b, \ + y, target, Y1, Y2, U, V) \ +{ \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 0); \ + Y2 = __lasx_xvpickve2gr_w(y_ev, 1); \ + U = __lasx_xvpickve2gr_w(uv, 0); \ + V = __lasx_xvpickve2gr_w(uv, 4); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 2); \ + Y2 = __lasx_xvpickve2gr_w(y_ev, 3); \ + U = __lasx_xvpickve2gr_w(uv, 1); \ + V = __lasx_xvpickve2gr_w(uv, 5); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 4); \ + Y2 = __lasx_xvpickve2gr_w(y_ev, 5); \ + U = __lasx_xvpickve2gr_w(uv, 2); \ + V = __lasx_xvpickve2gr_w(uv, 6); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_ev, 6); \ + Y2 = __lasx_xvpickve2gr_w(y_ev, 7); \ + U = __lasx_xvpickve2gr_w(uv, 3); \ + V = __lasx_xvpickve2gr_w(uv, 7); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ +} + +#define WRITE_YUV2RGBL_8(y_l, u, v, count, r, g, b, y, target, \ + Y1, Y2, U, V) \ +{ \ + Y1 = __lasx_xvpickve2gr_w(y_l, 0); \ + Y2 = __lasx_xvpickve2gr_w(y_l, 1); \ + U = __lasx_xvpickve2gr_w(u, 0); \ + V = __lasx_xvpickve2gr_w(v, 0); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_l, 2); \ + Y2 = __lasx_xvpickve2gr_w(y_l, 3); \ + U = __lasx_xvpickve2gr_w(u, 1); \ + V = __lasx_xvpickve2gr_w(v, 1); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_l, 4); \ + Y2 = __lasx_xvpickve2gr_w(y_l, 5); \ + U = __lasx_xvpickve2gr_w(u, 2); \ + V = __lasx_xvpickve2gr_w(v, 2); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_l, 6); \ + Y2 = __lasx_xvpickve2gr_w(y_l, 7); \ + U = __lasx_xvpickve2gr_w(u, 3); \ + V = __lasx_xvpickve2gr_w(v, 3); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ +} + +#define WRITE_YUV2RGBH_8(y_h, u, v, count, r, g, b, y, target, \ + Y1, Y2, U, V) \ +{ \ + Y1 = __lasx_xvpickve2gr_w(y_h, 0); \ + Y2 = __lasx_xvpickve2gr_w(y_h, 1); \ + U = __lasx_xvpickve2gr_w(u, 4); \ + V = __lasx_xvpickve2gr_w(v, 4); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_h, 2); \ + Y2 = __lasx_xvpickve2gr_w(y_h, 3); \ + U = __lasx_xvpickve2gr_w(u, 5); \ + V = __lasx_xvpickve2gr_w(v, 5); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_h, 4); \ + Y2 = __lasx_xvpickve2gr_w(y_h, 5); \ + U = __lasx_xvpickve2gr_w(u, 6); \ + V = __lasx_xvpickve2gr_w(v, 6); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ + \ + Y1 = __lasx_xvpickve2gr_w(y_h, 6); \ + Y2 = __lasx_xvpickve2gr_w(y_h, 7); \ + U = __lasx_xvpickve2gr_w(u, 7); \ + V = __lasx_xvpickve2gr_w(v, 7); \ + r = c->table_rV[V]; \ + g = (c->table_gU[U] + c->table_gV[V]); \ + b = c->table_bU[U]; \ + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, \ + r, g, b, y, target, 0); \ + count++; \ +} + + +static void +yuv2rgb_X_template_lasx(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y, enum AVPixelFormat target, int hasAlpha) +{ + int i, j; + int count = 0; + int t = 1 << 18; + int len = dstW >> 6; + int res = dstW & 63; + int len_count = (dstW + 1) >> 1; + const void *r, *g, *b; + int head = YUVRGB_TABLE_HEADROOM; + __m256i headroom = __lasx_xvldrepl_w(&head, 0); + + for (i = 0; i < len; i++) { + int Y1, Y2, U, V, count_lum = count << 1; + __m256i l_src1, l_src2, l_src3, l_src4, u_src1, u_src2, v_src1, v_src2; + __m256i yl1_ev, yl1_od, yh1_ev, yh1_od, yl2_ev, yl2_od, yh2_ev, yh2_od; + __m256i u1_ev, u1_od, v1_ev, v1_od, u2_ev, u2_od, v2_ev, v2_od, temp; + + yl1_ev = __lasx_xvldrepl_w(&t, 0); + yl1_od = yl1_ev; + yh1_ev = yl1_ev; + yh1_od = yl1_ev; + u1_ev = yl1_ev; + v1_ev = yl1_ev; + u1_od = yl1_ev; + v1_od = yl1_ev; + yl2_ev = yl1_ev; + yl2_od = yl1_ev; + yh2_ev = yl1_ev; + yh2_od = yl1_ev; + u2_ev = yl1_ev; + v2_ev = yl1_ev; + u2_od = yl1_ev; + v2_od = yl1_ev; + for (j = 0; j < lumFilterSize; j++) { + temp = __lasx_xvldrepl_h((lumFilter + j), 0); + LASX_LD_4((lumSrc[j] + count_lum), 16, l_src1, l_src2, l_src3, l_src4); + yl1_ev = __lasx_xvmaddwev_w_h(yl1_ev, temp, l_src1); + yl1_od = __lasx_xvmaddwod_w_h(yl1_od, temp, l_src1); + yh1_ev = __lasx_xvmaddwev_w_h(yh1_ev, temp, l_src2); + yh1_od = __lasx_xvmaddwod_w_h(yh1_od, temp, l_src2); + yl2_ev = __lasx_xvmaddwev_w_h(yl2_ev, temp, l_src3); + yl2_od = __lasx_xvmaddwod_w_h(yl2_od, temp, l_src3); + yh2_ev = __lasx_xvmaddwev_w_h(yh2_ev, temp, l_src4); + yh2_od = __lasx_xvmaddwod_w_h(yh2_od, temp, l_src4); + } + for (j = 0; j < chrFilterSize; j++) { + LASX_LD_2((chrUSrc[j] + count), 16, u_src1, u_src2); + LASX_LD_2((chrVSrc[j] + count), 16, v_src1, v_src2); + temp = __lasx_xvldrepl_h((chrFilter + j), 0); + u1_ev = __lasx_xvmaddwev_w_h(u1_ev, temp, u_src1); + u1_od = __lasx_xvmaddwod_w_h(u1_od, temp, u_src1); + v1_ev = __lasx_xvmaddwev_w_h(v1_ev, temp, v_src1); + v1_od = __lasx_xvmaddwod_w_h(v1_od, temp, v_src1); + u2_ev = __lasx_xvmaddwev_w_h(u2_ev, temp, u_src2); + u2_od = __lasx_xvmaddwod_w_h(u2_od, temp, u_src2); + v2_ev = __lasx_xvmaddwev_w_h(v2_ev, temp, v_src2); + v2_od = __lasx_xvmaddwod_w_h(v2_od, temp, v_src2); + } + yl1_ev = __lasx_xvsrai_w(yl1_ev, 19); + yh1_ev = __lasx_xvsrai_w(yh1_ev, 19); + yl1_od = __lasx_xvsrai_w(yl1_od, 19); + yh1_od = __lasx_xvsrai_w(yh1_od, 19); + u1_ev = __lasx_xvsrai_w(u1_ev, 19); + v1_ev = __lasx_xvsrai_w(v1_ev, 19); + u1_od = __lasx_xvsrai_w(u1_od, 19); + v1_od = __lasx_xvsrai_w(v1_od, 19); + yl2_ev = __lasx_xvsrai_w(yl2_ev, 19); + yh2_ev = __lasx_xvsrai_w(yh2_ev, 19); + yl2_od = __lasx_xvsrai_w(yl2_od, 19); + yh2_od = __lasx_xvsrai_w(yh2_od, 19); + u2_ev = __lasx_xvsrai_w(u2_ev, 19); + v2_ev = __lasx_xvsrai_w(v2_ev, 19); + u2_od = __lasx_xvsrai_w(u2_od, 19); + v2_od = __lasx_xvsrai_w(v2_od, 19); + u1_ev = __lasx_xvadd_w(u1_ev, headroom); + v1_ev = __lasx_xvadd_w(v1_ev, headroom); + u1_od = __lasx_xvadd_w(u1_od, headroom); + v1_od = __lasx_xvadd_w(v1_od, headroom); + u2_ev = __lasx_xvadd_w(u2_ev, headroom); + v2_ev = __lasx_xvadd_w(v2_ev, headroom); + u2_od = __lasx_xvadd_w(u2_od, headroom); + v2_od = __lasx_xvadd_w(v2_od, headroom); + WRITE_YUV2RGB_16_L(yl1_ev, yl1_od, u1_ev, u1_od, v1_ev, v1_od, count, + r, g, b, y, target, Y1, Y2, U, V); + WRITE_YUV2RGB_16_H(yh1_ev, yh1_od, u1_ev, u1_od, v1_ev, v1_od, count, + r, g, b, y, target, Y1, Y2, U, V); + WRITE_YUV2RGB_16_L(yl2_ev, yl2_od, u2_ev, u2_od, v2_ev, v2_od, count, + r, g, b, y, target, Y1, Y2, U, V); + WRITE_YUV2RGB_16_H(yh2_ev, yh2_od, u2_ev, u2_od, v2_ev, v2_od, count, + r, g, b, y, target, Y1, Y2, U, V); + } + if (res >= 32) { + int Y1, Y2, U, V, count_lum = count << 1; + __m256i l_src1, l_src2, u_src, v_src; + __m256i yl_ev, yl_od, yh_ev, yh_od; + __m256i u_ev, u_od, v_ev, v_od, temp; + + yl_ev = __lasx_xvldrepl_w(&t, 0); + yl_od = yl_ev; + yh_ev = yl_ev; + yh_od = yl_ev; + u_ev = yl_ev; + v_ev = yl_ev; + u_od = yl_ev; + v_od = yl_ev; + for (j = 0; j < lumFilterSize; j++) { + temp = __lasx_xvldrepl_h((lumFilter + j), 0); + l_src1 = LASX_LD((lumSrc[j] + count_lum)); + l_src2 = LASX_LD((lumSrc[j] + count_lum + 16)); + yl_ev = __lasx_xvmaddwev_w_h(yl_ev, temp, l_src1); + yl_od = __lasx_xvmaddwod_w_h(yl_od, temp, l_src1); + yh_ev = __lasx_xvmaddwev_w_h(yh_ev, temp, l_src2); + yh_od = __lasx_xvmaddwod_w_h(yh_od, temp, l_src2); + } + for (j = 0; j < chrFilterSize; j++) { + u_src = LASX_LD((chrUSrc[j] + count)); + v_src = LASX_LD((chrVSrc[j] + count)); + temp = __lasx_xvldrepl_h((chrFilter + j), 0); + u_ev = __lasx_xvmaddwev_w_h(u_ev, temp, u_src); + u_od = __lasx_xvmaddwod_w_h(u_od, temp, u_src); + v_ev = __lasx_xvmaddwev_w_h(v_ev, temp, v_src); + v_od = __lasx_xvmaddwod_w_h(v_od, temp, v_src); + } + yl_ev = __lasx_xvsrai_w(yl_ev, 19); + yh_ev = __lasx_xvsrai_w(yh_ev, 19); + yl_od = __lasx_xvsrai_w(yl_od, 19); + yh_od = __lasx_xvsrai_w(yh_od, 19); + u_ev = __lasx_xvsrai_w(u_ev, 19); + v_ev = __lasx_xvsrai_w(v_ev, 19); + u_od = __lasx_xvsrai_w(u_od, 19); + v_od = __lasx_xvsrai_w(v_od, 19); + u_ev = __lasx_xvadd_w(u_ev, headroom); + v_ev = __lasx_xvadd_w(v_ev, headroom); + u_od = __lasx_xvadd_w(u_od, headroom); + v_od = __lasx_xvadd_w(v_od, headroom); + WRITE_YUV2RGB_16_L(yl_ev, yl_od, u_ev, u_od, v_ev, v_od, count, + r, g, b, y, target, Y1, Y2, U, V); + WRITE_YUV2RGB_16_H(yh_ev, yh_od, u_ev, u_od, v_ev, v_od, count, + r, g, b, y, target, Y1, Y2, U, V); + res -= 32; + } + if (res >= 16) { + int Y1, Y2, U, V; + int count_lum = count << 1; + __m256i l_src, u_src, v_src; + __m256i y_ev, y_od, u, v, temp; + + y_ev = __lasx_xvldrepl_w(&t, 0); + y_od = y_ev; + u = y_ev; + v = y_ev; + for (j = 0; j < lumFilterSize; j++) { + temp = __lasx_xvldrepl_h((lumFilter + j), 0); + l_src = LASX_LD((lumSrc[j] + count_lum)); + y_ev = __lasx_xvmaddwev_w_h(y_ev, temp, l_src); + y_od = __lasx_xvmaddwod_w_h(y_od, temp, l_src); + } + for (j = 0; j < chrFilterSize; j++) { + u_src = LASX_LD((chrUSrc[j] + count)); + v_src = LASX_LD((chrVSrc[j] + count)); + temp = __lasx_xvldrepl_h((chrFilter + j), 0); + u_src = __lasx_vext2xv_w_h(u_src); + v_src = __lasx_vext2xv_w_h(v_src); + u = __lasx_xvmaddwev_w_h(u, temp, u_src); + v = __lasx_xvmaddwev_w_h(v, temp, v_src); + } + y_ev = __lasx_xvsrai_w(y_ev, 19); + y_od = __lasx_xvsrai_w(y_od, 19); + u = __lasx_xvsrai_w(u, 19); + v = __lasx_xvsrai_w(v, 19); + u = __lasx_xvadd_w(u, headroom); + v = __lasx_xvadd_w(v, headroom); + WRITE_YUV2RGB_16_N(y_ev, y_od, u, v, count, r, g, b, y, target, Y1, Y2, U, V); + res -= 16; + } + if (res >= 8) { + int Y1, Y2, U, V; + int count_lum = count << 1; + __m256i l_src, u_src, v_src; + __m256i y_ev, uv, temp; + + y_ev = __lasx_xvldrepl_w(&t, 0); + uv = y_ev; + for (j = 0; j < lumFilterSize; j++) { + temp = __lasx_xvldrepl_h((lumFilter + j), 0); + l_src = LASX_LD((lumSrc[j] + count_lum)); + l_src = __lasx_vext2xv_w_h(l_src); + y_ev = __lasx_xvmaddwev_w_h(y_ev, temp, l_src); + } + for (j = 0; j < chrFilterSize; j++) { + u_src = __lasx_xvldrepl_d((chrUSrc[j] + count), 0); + v_src = __lasx_xvldrepl_d((chrVSrc[j] + count), 0); + temp = __lasx_xvldrepl_h((chrFilter + j), 0); + u_src = __lasx_xvilvl_d(v_src, u_src); + u_src = __lasx_vext2xv_w_h(u_src); + uv = __lasx_xvmaddwev_w_h(uv, temp, u_src); + } + y_ev = __lasx_xvsrai_w(y_ev, 19); + uv = __lasx_xvsrai_w(uv, 19); + uv = __lasx_xvadd_w(uv, headroom); + WRITE_YUV2RGB_8_N(y_ev, uv, count, r, g, b, y, target, Y1, Y2, U, V); + } + for (; count < len_count; count++) { + int Y1 = 1 << 18; + int Y2 = Y1; + int U = Y1; + int V = Y1; + + for (j = 0; j < lumFilterSize; j++) { + Y1 += lumSrc[j][count * 2] * lumFilter[j]; + Y2 += lumSrc[j][count * 2 + 1] * lumFilter[j]; + } + for (j = 0; j < chrFilterSize; j++) { + U += chrUSrc[j][count] * chrFilter[j]; + V += chrVSrc[j][count] * chrFilter[j]; + } + Y1 >>= 19; + Y2 >>= 19; + U >>= 19; + V >>= 19; + r = c->table_rV[V + YUVRGB_TABLE_HEADROOM]; + g = (c->table_gU[U + YUVRGB_TABLE_HEADROOM] + + c->table_gV[V + YUVRGB_TABLE_HEADROOM]); + b = c->table_bU[U + YUVRGB_TABLE_HEADROOM]; + + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } +} + +static void +yuv2rgb_2_template_lasx(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y, + enum AVPixelFormat target, int hasAlpha) +{ + const int16_t *buf0 = buf[0], *buf1 = buf[1], + *ubuf0 = ubuf[0], *ubuf1 = ubuf[1], + *vbuf0 = vbuf[0], *vbuf1 = vbuf[1]; + int yalpha1 = 4096 - yalpha; + int uvalpha1 = 4096 - uvalpha; + int i, count = 0; + int len = dstW - 15; + int len_count = (dstW + 1) >> 1; + const void *r, *g, *b; + int head = YUVRGB_TABLE_HEADROOM; + __m256i v_yalpha1 = __lasx_xvreplgr2vr_w(yalpha1); + __m256i v_uvalpha1 = __lasx_xvreplgr2vr_w(uvalpha1); + __m256i v_yalpha = __lasx_xvreplgr2vr_w(yalpha); + __m256i v_uvalpha = __lasx_xvreplgr2vr_w(uvalpha); + __m256i headroom = __lasx_xvreplgr2vr_w(head); + + for (i = 0; i < len; i += 16) { + int Y1, Y2, U, V; + __m256i y0_h, y0_l, y0, u0, v0; + __m256i y1_h, y1_l, y1, u1, v1; + __m256i y_l, y_h, u, v; + + y0 = LASX_LD(buf0 + i); + u0 = LASX_LD(ubuf0 + count); + v0 = LASX_LD(vbuf0 + count); + y1 = LASX_LD(buf1 + i); + u1 = LASX_LD(ubuf1 + count); + v1 = LASX_LD(vbuf1 + count); + LASX_UNPCK_L_W_H_2(y0, y1, y0_l, y1_l); + y0 = __lasx_xvpermi_d(y0, 0x4E); + y1 = __lasx_xvpermi_d(y1, 0x4E); + LASX_UNPCK_L_W_H_2(y0, y1, y0_h, y1_h); + LASX_UNPCK_L_W_H_4(u0, u1, v0, v1, u0, u1, v0, v1); + y0_l = __lasx_xvmul_w(y0_l, v_yalpha1); + y0_h = __lasx_xvmul_w(y0_h, v_yalpha1); + u0 = __lasx_xvmul_w(u0, v_uvalpha1); + v0 = __lasx_xvmul_w(v0, v_uvalpha1); + y_l = __lasx_xvmadd_w(y0_l, v_yalpha, y1_l); + y_h = __lasx_xvmadd_w(y0_h, v_yalpha, y1_h); + u = __lasx_xvmadd_w(u0, v_uvalpha, u1); + v = __lasx_xvmadd_w(v0, v_uvalpha, v1); + y_l = __lasx_xvsrai_w(y_l, 19); + y_h = __lasx_xvsrai_w(y_h, 19); + u = __lasx_xvsrai_w(u, 19); + v = __lasx_xvsrai_w(v, 19); + u = __lasx_xvadd_w(u, headroom); + v = __lasx_xvadd_w(v, headroom); + WRITE_YUV2RGBL_8(y_l, u, v, count, r, g, b, y, target, Y1, Y2, U, V); + WRITE_YUV2RGBH_8(y_h, u, v, count, r, g, b, y, target, Y1, Y2, U, V); + } + if (dstW - i >= 8) { + int Y1, Y2, U, V; + __m256i y0_l, y0, u0, v0; + __m256i y1_l, y1, u1, v1; + __m256i y_l, u, v; + + y0 = LASX_LD(buf0 + i); + u0 = __lasx_xvldrepl_d((ubuf0 + count), 0); + v0 = __lasx_xvldrepl_d((vbuf0 + count), 0); + y1 = LASX_LD(buf1 + i); + u1 = __lasx_xvldrepl_d((ubuf1 + count), 0); + v1 = __lasx_xvldrepl_d((vbuf1 + count), 0); + LASX_UNPCK_L_W_H_2(y0, y1, y0_l, y1_l); + LASX_UNPCK_L_W_H_4(u0, u1, v0, v1, u0, u1, v0, v1); + y0_l = __lasx_xvmul_w(y0_l, v_yalpha1); + u0 = __lasx_xvmul_w(u0, v_uvalpha1); + v0 = __lasx_xvmul_w(v0, v_uvalpha1); + y_l = __lasx_xvmadd_w(y0_l, v_yalpha, y1_l); + u = __lasx_xvmadd_w(u0, v_uvalpha, u1); + v = __lasx_xvmadd_w(v0, v_uvalpha, v1); + y_l = __lasx_xvsrai_w(y_l, 19); + u = __lasx_xvsrai_w(u, 19); + v = __lasx_xvsrai_w(v, 19); + u = __lasx_xvadd_w(u, headroom); + v = __lasx_xvadd_w(v, headroom); + WRITE_YUV2RGBL_8(y_l, u, v, count, r, g, b, y, target, Y1, Y2, U, V); + i += 8; + } + for (; count < len_count; count++) { + int Y1 = (buf0[count * 2] * yalpha1 + + buf1[count * 2] * yalpha) >> 19; + int Y2 = (buf0[count * 2 + 1] * yalpha1 + + buf1[count * 2 + 1] * yalpha) >> 19; + int U = (ubuf0[count] * uvalpha1 + ubuf1[count] * uvalpha) >> 19; + int V = (vbuf0[count] * uvalpha1 + vbuf1[count] * uvalpha) >> 19; + + r = c->table_rV[V + YUVRGB_TABLE_HEADROOM], + g = (c->table_gU[U + YUVRGB_TABLE_HEADROOM] + + c->table_gV[V + YUVRGB_TABLE_HEADROOM]), + b = c->table_bU[U + YUVRGB_TABLE_HEADROOM]; + + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } +} + +static void +yuv2rgb_1_template_lasx(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y, enum AVPixelFormat target, + int hasAlpha) +{ + const int16_t *ubuf0 = ubuf[0], *vbuf0 = vbuf[0]; + int i; + int len = (dstW - 15); + int len_count = (dstW + 1) >> 1; + const void *r, *g, *b; + + if (uvalpha < 2048) { + int count = 0; + int16_t bias_int = 64; + int head = YUVRGB_TABLE_HEADROOM; + __m256i headroom = __lasx_xvreplgr2vr_w(head); + __m256i bias_64 = __lasx_xvreplgr2vr_h(bias_int); + + for (i = 0; i < len; i += 16) { + int Y1, Y2, U, V; + __m256i src_y, src_u, src_v; + __m256i y_h, y_l, u, v; + + src_y = LASX_LD(buf0 + i); + src_u = LASX_LD(ubuf0 + count); + src_v = LASX_LD(vbuf0 + count); + LASX_ADDWL_W_H_128SV(src_y, bias_64, y_l); + LASX_ADDWH_W_H_128SV(src_y, bias_64, y_h); + src_u = __lasx_xvpermi_d(src_u, 0xD8); + src_v = __lasx_xvpermi_d(src_v, 0xD8); + LASX_ADDWL_W_H_2_128SV(src_u, bias_64, src_v, bias_64, u, v); + y_l = __lasx_xvsrai_w(y_l, 7); + y_h = __lasx_xvsrai_w(y_h, 7); + u = __lasx_xvsrai_w(u, 7); + v = __lasx_xvsrai_w(v, 7); + u = __lasx_xvadd_w(u, headroom); + v = __lasx_xvadd_w(v, headroom); + WRITE_YUV2RGB_16(y_l, y_h, u, v, count, r, g, b, y, target, Y1, Y2, U, V); + } + if (dstW - i >= 8){ + int Y1, Y2, U, V; + __m256i src_y, src_u, src_v; + __m256i y_l, u, v; + + src_y = LASX_LD(buf0 + i); + src_u = __lasx_xvldrepl_d((ubuf0 + count), 0); + src_v = __lasx_xvldrepl_d((vbuf0 + count), 0); + src_y = __lasx_xvpermi_d(src_y, 0xD8); + LASX_ADDWL_W_H_2_128SV(src_y, bias_64, src_u, bias_64, y_l, u); + LASX_ADDWL_W_H_128SV(src_v, bias_64, v); + y_l = __lasx_xvsrai_w(y_l, 7); + u = __lasx_xvsrai_w(u, 7); + v = __lasx_xvsrai_w(v, 7); + u = __lasx_xvadd_w(u, headroom); + v = __lasx_xvadd_w(v, headroom); + WRITE_YUV2RGBL_8(y_l, u, v, count, r, g, b, y, target, Y1, Y2, U, V); + i += 8; + } + for (; count < len_count; count++) { + int Y1 = (buf0[count * 2 ] + 64) >> 7; + int Y2 = (buf0[count * 2 + 1] + 64) >> 7; + int U = (ubuf0[count] + 64) >> 7; + int V = (vbuf0[count] + 64) >> 7; + + r = c->table_rV[V + YUVRGB_TABLE_HEADROOM], + g = (c->table_gU[U + YUVRGB_TABLE_HEADROOM] + + c->table_gV[V + YUVRGB_TABLE_HEADROOM]), + b = c->table_bU[U + YUVRGB_TABLE_HEADROOM]; + + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } + } else { + const int16_t *ubuf1 = ubuf[1], *vbuf1 = vbuf[1]; + int count = 0; + int16_t bias_int_64 = 64; + int bias_int_128 = 128; + int HEADROOM = YUVRGB_TABLE_HEADROOM; + __m256i headroom = __lasx_xvreplgr2vr_w(HEADROOM); + __m256i bias_64 = __lasx_xvreplgr2vr_h(bias_int_64); + __m256i bias_128 = __lasx_xvreplgr2vr_w(bias_int_128); + + for (i = 0; i < len; i += 16) { + int Y1, Y2, U, V; + __m256i src_y, src_u0, src_v0, src_u1, src_v1; + __m256i y_h, y_l, u, v; + + src_y = LASX_LD(buf0 + i); + src_u0 = LASX_LD(ubuf0 + count); + src_v0 = LASX_LD(vbuf0 + count); + src_u1 = LASX_LD(ubuf1 + count); + src_v1 = LASX_LD(vbuf1 + count); + src_u0 = __lasx_xvpermi_d(src_u0, 0xD8); + src_v0 = __lasx_xvpermi_d(src_v0, 0xD8); + src_u1 = __lasx_xvpermi_d(src_u1, 0xD8); + src_v1 = __lasx_xvpermi_d(src_v1, 0xD8); + LASX_ADDWL_W_H_128SV(src_y, bias_64, y_l); + LASX_ADDWH_W_H_128SV(src_y, bias_64, y_h); + LASX_ADDWL_W_H_2_128SV(src_u0, src_u1, src_v0, src_v1, u, v); + u = __lasx_xvadd_w(u, bias_128); + v = __lasx_xvadd_w(v, bias_128); + y_l = __lasx_xvsrai_w(y_l, 7); + y_h = __lasx_xvsrai_w(y_h, 7); + u = __lasx_xvsrai_w(u, 8); + v = __lasx_xvsrai_w(v, 8); + u = __lasx_xvadd_w(u, headroom); + v = __lasx_xvadd_w(v, headroom); + WRITE_YUV2RGB_16(y_l, y_h, u, v, count, r, g, b, y, target, Y1, Y2, U, V); + } + if (dstW - i >= 8){ + int Y1, Y2, U, V; + __m256i src_y, src_u0, src_v0, src_u1, src_v1; + __m256i y_l, u, v; + + src_y = LASX_LD(buf0 + i); + src_u0 = __lasx_xvldrepl_d((ubuf0 + count), 0); + src_v0 = __lasx_xvldrepl_d((vbuf0 + count), 0); + src_u1 = __lasx_xvldrepl_d((ubuf1 + count), 0); + src_v1 = __lasx_xvldrepl_d((vbuf1 + count), 0); + + src_y = __lasx_xvpermi_d(src_y, 0xD8); + LASX_ADDWL_W_H_128SV(src_y, bias_64, y_l); + LASX_ADDWL_W_H_2_128SV(src_u0, src_u1, src_v0, src_v1, u, v); + u = __lasx_xvadd_w(u, bias_128); + v = __lasx_xvadd_w(v, bias_128); + y_l = __lasx_xvsrai_w(y_l, 7); + u = __lasx_xvsrai_w(u, 8); + v = __lasx_xvsrai_w(v, 8); + u = __lasx_xvadd_w(u, headroom); + v = __lasx_xvadd_w(v, headroom); + WRITE_YUV2RGBL_8(y_l, u, v, count, r, g, b, y, target, Y1, Y2, U, V); + i += 8; + } + for (; count < len_count; count++) { + int Y1 = (buf0[count * 2 ] + 64) >> 7; + int Y2 = (buf0[count * 2 + 1] + 64) >> 7; + int U = (ubuf0[count] + ubuf1[count] + 128) >> 8; + int V = (vbuf0[count] + vbuf1[count] + 128) >> 8; + + r = c->table_rV[V + YUVRGB_TABLE_HEADROOM], + g = (c->table_gU[U + YUVRGB_TABLE_HEADROOM] + + c->table_gV[V + YUVRGB_TABLE_HEADROOM]), + b = c->table_bU[U + YUVRGB_TABLE_HEADROOM]; + + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } + } +} + +#define YUV2RGBWRAPPERX(name, base, ext, fmt, hasAlpha) \ +static void name ## ext ## _X_lasx(SwsContext *c, const int16_t *lumFilter, \ + const int16_t **lumSrc, int lumFilterSize, \ + const int16_t *chrFilter, const int16_t **chrUSrc, \ + const int16_t **chrVSrc, int chrFilterSize, \ + const int16_t **alpSrc, uint8_t *dest, int dstW, \ + int y) \ +{ \ + name ## base ## _X_template_lasx(c, lumFilter, lumSrc, lumFilterSize, \ + chrFilter, chrUSrc, chrVSrc, chrFilterSize, \ + alpSrc, dest, dstW, y, fmt, hasAlpha); \ +} + +#define YUV2RGBWRAPPERX2(name, base, ext, fmt, hasAlpha) \ +YUV2RGBWRAPPERX(name, base, ext, fmt, hasAlpha) \ +static void name ## ext ## _2_lasx(SwsContext *c, const int16_t *buf[2], \ + const int16_t *ubuf[2], const int16_t *vbuf[2], \ + const int16_t *abuf[2], uint8_t *dest, int dstW, \ + int yalpha, int uvalpha, int y) \ +{ \ + name ## base ## _2_template_lasx(c, buf, ubuf, vbuf, abuf, dest, \ + dstW, yalpha, uvalpha, y, fmt, hasAlpha); \ +} + +#define YUV2RGBWRAPPER(name, base, ext, fmt, hasAlpha) \ +YUV2RGBWRAPPERX2(name, base, ext, fmt, hasAlpha) \ +static void name ## ext ## _1_lasx(SwsContext *c, const int16_t *buf0, \ + const int16_t *ubuf[2], const int16_t *vbuf[2], \ + const int16_t *abuf0, uint8_t *dest, int dstW, \ + int uvalpha, int y) \ +{ \ + name ## base ## _1_template_lasx(c, buf0, ubuf, vbuf, abuf0, dest, \ + dstW, uvalpha, y, fmt, hasAlpha); \ +} + + +#if CONFIG_SMALL +#else +#if CONFIG_SWSCALE_ALPHA +#endif +YUV2RGBWRAPPER(yuv2rgb,, x32_1, AV_PIX_FMT_RGB32_1, 0) +YUV2RGBWRAPPER(yuv2rgb,, x32, AV_PIX_FMT_RGB32, 0) +#endif +YUV2RGBWRAPPER(yuv2, rgb, rgb24, AV_PIX_FMT_RGB24, 0) +YUV2RGBWRAPPER(yuv2, rgb, bgr24, AV_PIX_FMT_BGR24, 0) +YUV2RGBWRAPPER(yuv2rgb,, 16, AV_PIX_FMT_RGB565, 0) +YUV2RGBWRAPPER(yuv2rgb,, 15, AV_PIX_FMT_RGB555, 0) +YUV2RGBWRAPPER(yuv2rgb,, 12, AV_PIX_FMT_RGB444, 0) +YUV2RGBWRAPPER(yuv2rgb,, 8, AV_PIX_FMT_RGB8, 0) +YUV2RGBWRAPPER(yuv2rgb,, 4, AV_PIX_FMT_RGB4, 0) +YUV2RGBWRAPPER(yuv2rgb,, 4b, AV_PIX_FMT_RGB4_BYTE, 0) + +// This function is copied from libswscale/output.c +static av_always_inline void yuv2rgb_write_full(SwsContext *c, + uint8_t *dest, int i, int R, int A, int G, int B, + int y, enum AVPixelFormat target, int hasAlpha, int err[4]) +{ + int isrgb8 = target == AV_PIX_FMT_BGR8 || target == AV_PIX_FMT_RGB8; + + if ((R | G | B) & 0xC0000000) { + R = av_clip_uintp2(R, 30); + G = av_clip_uintp2(G, 30); + B = av_clip_uintp2(B, 30); + } + + switch(target) { + case AV_PIX_FMT_ARGB: + dest[0] = hasAlpha ? A : 255; + dest[1] = R >> 22; + dest[2] = G >> 22; + dest[3] = B >> 22; + break; + case AV_PIX_FMT_RGB24: + dest[0] = R >> 22; + dest[1] = G >> 22; + dest[2] = B >> 22; + break; + case AV_PIX_FMT_RGBA: + dest[0] = R >> 22; + dest[1] = G >> 22; + dest[2] = B >> 22; + dest[3] = hasAlpha ? A : 255; + break; + case AV_PIX_FMT_ABGR: + dest[0] = hasAlpha ? A : 255; + dest[1] = B >> 22; + dest[2] = G >> 22; + dest[3] = R >> 22; + break; + case AV_PIX_FMT_BGR24: + dest[0] = B >> 22; + dest[1] = G >> 22; + dest[2] = R >> 22; + break; + case AV_PIX_FMT_BGRA: + dest[0] = B >> 22; + dest[1] = G >> 22; + dest[2] = R >> 22; + dest[3] = hasAlpha ? A : 255; + break; + case AV_PIX_FMT_BGR4_BYTE: + case AV_PIX_FMT_RGB4_BYTE: + case AV_PIX_FMT_BGR8: + case AV_PIX_FMT_RGB8: + { + int r,g,b; + + switch (c->dither) { + default: + case SWS_DITHER_AUTO: + case SWS_DITHER_ED: + R >>= 22; + G >>= 22; + B >>= 22; + R += (7*err[0] + 1*c->dither_error[0][i] + 5*c->dither_error[0][i+1] + 3*c->dither_error[0][i+2])>>4; + G += (7*err[1] + 1*c->dither_error[1][i] + 5*c->dither_error[1][i+1] + 3*c->dither_error[1][i+2])>>4; + B += (7*err[2] + 1*c->dither_error[2][i] + 5*c->dither_error[2][i+1] + 3*c->dither_error[2][i+2])>>4; + c->dither_error[0][i] = err[0]; + c->dither_error[1][i] = err[1]; + c->dither_error[2][i] = err[2]; + r = R >> (isrgb8 ? 5 : 7); + g = G >> (isrgb8 ? 5 : 6); + b = B >> (isrgb8 ? 6 : 7); + r = av_clip(r, 0, isrgb8 ? 7 : 1); + g = av_clip(g, 0, isrgb8 ? 7 : 3); + b = av_clip(b, 0, isrgb8 ? 3 : 1); + err[0] = R - r*(isrgb8 ? 36 : 255); + err[1] = G - g*(isrgb8 ? 36 : 85); + err[2] = B - b*(isrgb8 ? 85 : 255); + break; + case SWS_DITHER_A_DITHER: + if (isrgb8) { + /* see http://pippin.gimp.org/a_dither/ for details/origin */ +#define A_DITHER(u,v) (((((u)+((v)*236))*119)&0xff)) + r = (((R >> 19) + A_DITHER(i,y) -96)>>8); + g = (((G >> 19) + A_DITHER(i + 17,y) - 96)>>8); + b = (((B >> 20) + A_DITHER(i + 17*2,y) -96)>>8); + r = av_clip_uintp2(r, 3); + g = av_clip_uintp2(g, 3); + b = av_clip_uintp2(b, 2); + } else { + r = (((R >> 21) + A_DITHER(i,y)-256)>>8); + g = (((G >> 19) + A_DITHER(i + 17,y)-256)>>8); + b = (((B >> 21) + A_DITHER(i + 17*2,y)-256)>>8); + r = av_clip_uintp2(r, 1); + g = av_clip_uintp2(g, 2); + b = av_clip_uintp2(b, 1); + } + break; + case SWS_DITHER_X_DITHER: + if (isrgb8) { + /* see http://pippin.gimp.org/a_dither/ for details/origin */ +#define X_DITHER(u,v) (((((u)^((v)*237))*181)&0x1ff)/2) + r = (((R >> 19) + X_DITHER(i,y) - 96)>>8); + g = (((G >> 19) + X_DITHER(i + 17,y) - 96)>>8); + b = (((B >> 20) + X_DITHER(i + 17*2,y) - 96)>>8); + r = av_clip_uintp2(r, 3); + g = av_clip_uintp2(g, 3); + b = av_clip_uintp2(b, 2); + } else { + r = (((R >> 21) + X_DITHER(i,y)-256)>>8); + g = (((G >> 19) + X_DITHER(i + 17,y)-256)>>8); + b = (((B >> 21) + X_DITHER(i + 17*2,y)-256)>>8); + r = av_clip_uintp2(r, 1); + g = av_clip_uintp2(g, 2); + b = av_clip_uintp2(b, 1); + } + + break; + } + + if(target == AV_PIX_FMT_BGR4_BYTE) { + dest[0] = r + 2*g + 8*b; + } else if(target == AV_PIX_FMT_RGB4_BYTE) { + dest[0] = b + 2*g + 8*r; + } else if(target == AV_PIX_FMT_BGR8) { + dest[0] = r + 8*g + 64*b; + } else if(target == AV_PIX_FMT_RGB8) { + dest[0] = b + 4*g + 32*r; + } else + av_assert2(0); + break; } + } +} + +#define YUVTORGB_SETUP \ + int y_offset = c->yuv2rgb_y_offset; \ + int y_coeff = c->yuv2rgb_y_coeff; \ + int v2r_coe = c->yuv2rgb_v2r_coeff; \ + int v2g_coe = c->yuv2rgb_v2g_coeff; \ + int u2g_coe = c->yuv2rgb_u2g_coeff; \ + int u2b_coe = c->yuv2rgb_u2b_coeff; \ + __m256i offset = __lasx_xvreplgr2vr_w(y_offset); \ + __m256i coeff = __lasx_xvreplgr2vr_w(y_coeff); \ + __m256i v2r = __lasx_xvreplgr2vr_w(v2r_coe); \ + __m256i v2g = __lasx_xvreplgr2vr_w(v2g_coe); \ + __m256i u2g = __lasx_xvreplgr2vr_w(u2g_coe); \ + __m256i u2b = __lasx_xvreplgr2vr_w(u2b_coe); \ + + +#define YUVTORGB(y, u, v, R, G, B, offset, coeff, \ + y_temp, v2r, v2g, u2g, u2b) \ +{ \ + y = __lasx_xvsub_w(y, offset); \ + y = __lasx_xvmul_w(y, coeff); \ + y = __lasx_xvadd_w(y, y_temp); \ + R = __lasx_xvmadd_w(y, v, v2r); \ + v = __lasx_xvmadd_w(y, v, v2g); \ + G = __lasx_xvmadd_w(v, u, u2g); \ + B = __lasx_xvmadd_w(y, u, u2b); \ +} + +#define WRITE_FULL_L_A(r, g, b, a, c, dest, i, R, A, G, B, \ + y, target, hasAlpha, err) \ +{ \ + R = __lasx_xvpickve2gr_w(r, 0); \ + G = __lasx_xvpickve2gr_w(g, 0); \ + B = __lasx_xvpickve2gr_w(b, 0); \ + A = __lasx_xvpickve2gr_w(a, 0); \ + if (A & 0x100) \ + A = av_clip_uint8(A); \ + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 1); \ + G = __lasx_xvpickve2gr_w(g, 1); \ + B = __lasx_xvpickve2gr_w(b, 1); \ + A = __lasx_xvpickve2gr_w(a, 1); \ + if (A & 0x100) \ + A = av_clip_uint8(A); \ + yuv2rgb_write_full(c, dest, i + 1, R, A, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 2); \ + G = __lasx_xvpickve2gr_w(g, 2); \ + B = __lasx_xvpickve2gr_w(b, 2); \ + A = __lasx_xvpickve2gr_w(a, 2); \ + if (A & 0x100) \ + A = av_clip_uint8(A); \ + yuv2rgb_write_full(c, dest, i + 2, R, A, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 3); \ + G = __lasx_xvpickve2gr_w(g, 3); \ + B = __lasx_xvpickve2gr_w(b, 3); \ + A = __lasx_xvpickve2gr_w(a, 3); \ + if (A & 0x100) \ + A = av_clip_uint8(A); \ + yuv2rgb_write_full(c, dest, i + 3, R, A, G, B, y, target, hasAlpha, err); \ + dest += step; \ +} + +#define WRITE_FULL_H_A(r, g, b, a, c, dest, i, R, A, G, B, \ + y, target, hasAlpha, err) \ +{ \ + R = __lasx_xvpickve2gr_w(r, 4); \ + G = __lasx_xvpickve2gr_w(g, 4); \ + B = __lasx_xvpickve2gr_w(b, 4); \ + A = __lasx_xvpickve2gr_w(a, 4); \ + if (A & 0x100) \ + A = av_clip_uint8(A); \ + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 5); \ + G = __lasx_xvpickve2gr_w(g, 5); \ + B = __lasx_xvpickve2gr_w(b, 5); \ + A = __lasx_xvpickve2gr_w(a, 5); \ + if (A & 0x100) \ + A = av_clip_uint8(A); \ + yuv2rgb_write_full(c, dest, i + 1, R, A, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 6); \ + G = __lasx_xvpickve2gr_w(g, 6); \ + B = __lasx_xvpickve2gr_w(b, 6); \ + A = __lasx_xvpickve2gr_w(a, 6); \ + if (A & 0x100) \ + A = av_clip_uint8(A); \ + yuv2rgb_write_full(c, dest, i + 2, R, A, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 7); \ + G = __lasx_xvpickve2gr_w(g, 7); \ + B = __lasx_xvpickve2gr_w(b, 7); \ + A = __lasx_xvpickve2gr_w(a, 7); \ + if (A & 0x100) \ + A = av_clip_uint8(A); \ + yuv2rgb_write_full(c, dest, i + 3, R, A, G, B, y, target, hasAlpha, err); \ + dest += step; \ +} + + +#define WRITE_FULL_L(r, g, b, c, dest, i, R, G, B, \ + y, target, hasAlpha, err) \ +{ \ + R = __lasx_xvpickve2gr_w(r, 0); \ + G = __lasx_xvpickve2gr_w(g, 0); \ + B = __lasx_xvpickve2gr_w(b, 0); \ + yuv2rgb_write_full(c, dest, i, R, 0, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 1); \ + G = __lasx_xvpickve2gr_w(g, 1); \ + B = __lasx_xvpickve2gr_w(b, 1); \ + yuv2rgb_write_full(c, dest, i + 1, R, 0, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 2); \ + G = __lasx_xvpickve2gr_w(g, 2); \ + B = __lasx_xvpickve2gr_w(b, 2); \ + yuv2rgb_write_full(c, dest, i + 2, R, 0, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 3); \ + G = __lasx_xvpickve2gr_w(g, 3); \ + B = __lasx_xvpickve2gr_w(b, 3); \ + yuv2rgb_write_full(c, dest, i + 3, R, 0, G, B, y, target, hasAlpha, err); \ + dest += step; \ +} + +#define WRITE_FULL_H(r, g, b, c, dest, i, R, G, B, \ + y, target, hasAlpha, err) \ +{ \ + R = __lasx_xvpickve2gr_w(r, 4); \ + G = __lasx_xvpickve2gr_w(g, 4); \ + B = __lasx_xvpickve2gr_w(b, 4); \ + yuv2rgb_write_full(c, dest, i, R, 0, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 5); \ + G = __lasx_xvpickve2gr_w(g, 5); \ + B = __lasx_xvpickve2gr_w(b, 5); \ + yuv2rgb_write_full(c, dest, i + 1, R, 0, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 6); \ + G = __lasx_xvpickve2gr_w(g, 6); \ + B = __lasx_xvpickve2gr_w(b, 6); \ + yuv2rgb_write_full(c, dest, i + 2, R, 0, G, B, y, target, hasAlpha, err); \ + dest += step; \ + R = __lasx_xvpickve2gr_w(r, 7); \ + G = __lasx_xvpickve2gr_w(g, 7); \ + B = __lasx_xvpickve2gr_w(b, 7); \ + yuv2rgb_write_full(c, dest, i + 3, R, 0, G, B, y, target, hasAlpha, err); \ + dest += step; \ +} + +static void +yuv2rgb_full_X_template_lasx(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, + int dstW, int y, enum AVPixelFormat target, int hasAlpha) +{ + int i, j, B, G, R, A; + int step = (target == AV_PIX_FMT_RGB24 || target == AV_PIX_FMT_BGR24) ? 3 : 4; + int err[4] = {0}; + int a_temp = 1 << 18; + int templ = 1 << 9; + int tempc = templ - (128 << 19); + int ytemp = 1 << 21; + int len = dstW - 15; + __m256i y_temp = __lasx_xvreplgr2vr_w(ytemp); + YUVTORGB_SETUP + + if( target == AV_PIX_FMT_BGR4_BYTE || target == AV_PIX_FMT_RGB4_BYTE + || target == AV_PIX_FMT_BGR8 || target == AV_PIX_FMT_RGB8) + step = 1; + + for (i = 0; i < len; i += 16) { + __m256i l_src, u_src, v_src; + __m256i y_l, y_h, u_l, u_h, v_l, v_h, temp; + __m256i R_l, R_h, G_l, G_h, B_l, B_h; + + y_l = y_h = __lasx_xvreplgr2vr_w(templ); + u_l = u_h = v_l = v_h = __lasx_xvreplgr2vr_w(tempc); + for (j = 0; j < lumFilterSize; j++) { + temp = __lasx_xvldrepl_h((lumFilter + j), 0); + l_src = LASX_LD((lumSrc[j] + i)); + LASX_MADDWL_W_H_128SV(y_l, l_src, temp, y_l); + LASX_MADDWH_W_H_128SV(y_h, l_src, temp, y_h); + } + for (j = 0; j < chrFilterSize; j++) { + temp = __lasx_xvldrepl_h((chrFilter + j), 0); + u_src = LASX_LD((chrUSrc[j] + i)); + v_src = LASX_LD((chrVSrc[j] + i)); + LASX_MADDWL_W_H_2_128SV(u_l, u_src, temp, v_l, v_src, temp, u_l, v_l); + LASX_MADDWH_W_H_2_128SV(u_h, u_src, temp, v_h, v_src, temp, u_h, v_h); + } + y_l = __lasx_xvsrai_w(y_l, 10); + y_h = __lasx_xvsrai_w(y_h, 10); + u_l = __lasx_xvsrai_w(u_l, 10); + u_h = __lasx_xvsrai_w(u_h, 10); + v_l = __lasx_xvsrai_w(v_l, 10); + v_h = __lasx_xvsrai_w(v_h, 10); + YUVTORGB(y_l, u_l, v_l, R_l, G_l, B_l, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + YUVTORGB(y_h, u_h, v_h, R_h, G_h, B_h, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + + if (hasAlpha) { + __m256i a_src, a_l, a_h; + + a_l = a_h = __lasx_xvreplgr2vr_w(a_temp); + for (j = 0; j < lumFilterSize; j++) { + temp = __lasx_xvldrepl_h(lumFilter + j, 0); + a_src = LASX_LD((alpSrc[j] + i)); + LASX_MADDWL_W_H_128SV(a_l, a_src, temp, a_l); + LASX_MADDWH_W_H_128SV(a_h, a_src, temp, a_h); + } + a_h = __lasx_xvsrai_w(a_h, 19); + a_l = __lasx_xvsrai_w(a_l, 19); + WRITE_FULL_L_A(R_l, G_l, B_l, a_l, c, dest, i, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_L_A(R_h, G_h, B_h, a_h, c, dest, i + 4, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_l, G_l, B_l, a_l, c, dest, i + 8, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_h, G_h, B_h, a_h, c, dest, i + 12, R, A, G, B, + y, target, hasAlpha, err); + } else { + WRITE_FULL_L(R_l, G_l, B_l, c, dest, i, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_L(R_h, G_h, B_h, c, dest, i + 4, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_l, G_l, B_l, c, dest, i + 8, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_h, G_h, B_h, c, dest, i + 12, R, G, B, + y, target, hasAlpha, err); + } + } + if (dstW - i >= 8) { + __m256i l_src, u_src, v_src; + __m256i y_l, u_l, v_l, temp; + __m256i R_l, G_l, B_l; + + y_l = __lasx_xvreplgr2vr_w(templ); + u_l = v_l = __lasx_xvreplgr2vr_w(tempc); + for (j = 0; j < lumFilterSize; j++) { + temp = __lasx_xvldrepl_h((lumFilter + j), 0); + l_src = LASX_LD((lumSrc[j] + i)); + l_src = __lasx_xvpermi_d(l_src, 0xD8); + LASX_MADDWL_W_H_128SV(y_l, l_src, temp, y_l); + } + for (j = 0; j < chrFilterSize; j++) { + temp = __lasx_xvldrepl_h((chrFilter + j), 0); + u_src = LASX_LD((chrUSrc[j] + i)); + v_src = LASX_LD((chrVSrc[j] + i)); + u_src = __lasx_xvpermi_d(u_src, 0xD8); + v_src = __lasx_xvpermi_d(v_src, 0xD8); + LASX_MADDWL_W_H_2_128SV(u_l, u_src, temp, v_l, v_src, temp, u_l, v_l); + } + y_l = __lasx_xvsrai_w(y_l, 10); + u_l = __lasx_xvsrai_w(u_l, 10); + v_l = __lasx_xvsrai_w(v_l, 10); + YUVTORGB(y_l, u_l, v_l, R_l, G_l, B_l, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + + if (hasAlpha) { + __m256i a_src, a_l; + + a_l = __lasx_xvreplgr2vr_w(a_temp); + for (j = 0; j < lumFilterSize; j++) { + temp = __lasx_xvldrepl_h(lumFilter + j, 0); + a_src = LASX_LD((alpSrc[j] + i)); + a_src = __lasx_xvpermi_d(a_src, 0xD8); + LASX_MADDWL_W_H_128SV(a_l, a_src, temp, a_l); + } + a_l = __lasx_xvsrai_w(a_l, 19); + WRITE_FULL_L_A(R_l, G_l, B_l, a_l, c, dest, i, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_l, G_l, B_l, a_l, c, dest, i + 4, R, A, G, B, + y, target, hasAlpha, err); + } else { + WRITE_FULL_L(R_l, G_l, B_l, c, dest, i, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_l, G_l, B_l, c, dest, i + 4, R, G, B, + y, target, hasAlpha, err); + } + i += 8; + } + for (; i < dstW; i++) { + int Y = templ; + int V, U = V = tempc; + + A = 0; + for (j = 0; j < lumFilterSize; j++) { + Y += lumSrc[j][i] * lumFilter[j]; + } + for (j = 0; j < chrFilterSize; j++) { + U += chrUSrc[j][i] * chrFilter[j]; + V += chrVSrc[j][i] * chrFilter[j]; + + } + Y >>= 10; + U >>= 10; + V >>= 10; + if (hasAlpha) { + A = 1 << 18; + for (j = 0; j < lumFilterSize; j++) { + A += alpSrc[j][i] * lumFilter[j]; + } + A >>= 19; + if (A & 0x100) + A = av_clip_uint8(A); + } + Y -= y_offset; + Y *= y_coeff; + Y += ytemp; + R = (unsigned)Y + V * v2r_coe; + G = (unsigned)Y + V * v2g_coe + U * u2g_coe; + B = (unsigned)Y + U * u2b_coe; + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + c->dither_error[0][i] = err[0]; + c->dither_error[1][i] = err[1]; + c->dither_error[2][i] = err[2]; +} + +static void +yuv2rgb_full_2_template_lasx(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y, + enum AVPixelFormat target, int hasAlpha) +{ + const int16_t *buf0 = buf[0], *buf1 = buf[1], + *ubuf0 = ubuf[0], *ubuf1 = ubuf[1], + *vbuf0 = vbuf[0], *vbuf1 = vbuf[1], + *abuf0 = hasAlpha ? abuf[0] : NULL, + *abuf1 = hasAlpha ? abuf[1] : NULL; + int yalpha1 = 4096 - yalpha; + int uvalpha1 = 4096 - uvalpha; + int uvtemp = 128 << 19; + int atemp = 1 << 18; + int err[4] = {0}; + int ytemp = 1 << 21; + int len = dstW - 15; + int i, R, G, B, A; + int step = (target == AV_PIX_FMT_RGB24 || target == AV_PIX_FMT_BGR24) ? 3 : 4; + __m256i v_uvalpha1 = __lasx_xvreplgr2vr_w(uvalpha1); + __m256i v_yalpha1 = __lasx_xvreplgr2vr_w(yalpha1); + __m256i v_uvalpha = __lasx_xvreplgr2vr_w(uvalpha); + __m256i v_yalpha = __lasx_xvreplgr2vr_w(yalpha); + __m256i uv = __lasx_xvreplgr2vr_w(uvtemp); + __m256i a_bias = __lasx_xvreplgr2vr_w(atemp); + __m256i y_temp = __lasx_xvreplgr2vr_w(ytemp); + YUVTORGB_SETUP + + av_assert2(yalpha <= 4096U); + av_assert2(uvalpha <= 4096U); + + if( target == AV_PIX_FMT_BGR4_BYTE || target == AV_PIX_FMT_RGB4_BYTE + || target == AV_PIX_FMT_BGR8 || target == AV_PIX_FMT_RGB8) + step = 1; + + for (i = 0; i < len; i += 16) { + __m256i b0, b1, ub0, ub1, vb0, vb1; + __m256i y0_l, y0_h, y1_l, y1_h, u0_l, u0_h; + __m256i v0_l, v0_h, u1_l, u1_h, v1_l, v1_h; + __m256i y_l, y_h, v_l, v_h, u_l, u_h; + __m256i R_l, R_h, G_l, G_h, B_l, B_h; + + b0 = LASX_LD((buf0 + i)); + b1 = LASX_LD((buf1 + i)); + ub0 = LASX_LD((ubuf0 + i)); + ub1 = LASX_LD((ubuf1 + i)); + vb0 = LASX_LD((vbuf0 + i)); + vb1 = LASX_LD((vbuf1 + i)); + LASX_UNPCK_L_W_H_2(b0, b1, y0_l, y1_l); + LASX_UNPCK_L_W_H_4(ub0, ub1, vb0, vb1, u0_l, u1_l, v0_l, v1_l); + b0 = __lasx_xvpermi_d(b0, 0x4E); + b1 = __lasx_xvpermi_d(b1, 0x4E); + ub0 = __lasx_xvpermi_d(ub0, 0x4E); + ub1 = __lasx_xvpermi_d(ub1, 0x4E); + vb0 = __lasx_xvpermi_d(vb0, 0x4E); + vb1 = __lasx_xvpermi_d(vb1, 0x4E); + LASX_UNPCK_L_W_H_2(b0, b1, y0_h, y1_h); + LASX_UNPCK_L_W_H_4(ub0, ub1, vb0, vb1, u0_h, u1_h, v0_h, v1_h); + y0_l = __lasx_xvmul_w(y0_l, v_yalpha1); + y0_h = __lasx_xvmul_w(y0_h, v_yalpha1); + u0_l = __lasx_xvmul_w(u0_l, v_uvalpha1); + u0_h = __lasx_xvmul_w(u0_h, v_uvalpha1); + v0_l = __lasx_xvmul_w(v0_l, v_uvalpha1); + v0_h = __lasx_xvmul_w(v0_h, v_uvalpha1); + y_l = __lasx_xvmadd_w(y0_l, v_yalpha, y1_l); + y_h = __lasx_xvmadd_w(y0_h, v_yalpha, y1_h); + u_l = __lasx_xvmadd_w(u0_l, v_uvalpha, u1_l); + u_h = __lasx_xvmadd_w(u0_h, v_uvalpha, u1_h); + v_l = __lasx_xvmadd_w(v0_l, v_uvalpha, v1_l); + v_h = __lasx_xvmadd_w(v0_h, v_uvalpha, v1_h); + u_l = __lasx_xvsub_w(u_l, uv); + u_h = __lasx_xvsub_w(u_h, uv); + v_l = __lasx_xvsub_w(v_l, uv); + v_h = __lasx_xvsub_w(v_h, uv); + y_l = __lasx_xvsrai_w(y_l, 10); + y_h = __lasx_xvsrai_w(y_h, 10); + u_l = __lasx_xvsrai_w(u_l, 10); + u_h = __lasx_xvsrai_w(u_h, 10); + v_l = __lasx_xvsrai_w(v_l, 10); + v_h = __lasx_xvsrai_w(v_h, 10); + YUVTORGB(y_l, u_l, v_l, R_l, G_l, B_l, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + YUVTORGB(y_h, u_h, v_h, R_h, G_h, B_h, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + + if (hasAlpha) { + __m256i a0, a1, a0_l, a0_h; + __m256i a_l, a_h, a1_l, a1_h; + + a0 = LASX_LD((abuf0 + i)); + a1 = LASX_LD((abuf1 + i)); + LASX_UNPCK_L_W_H_2(a0, a1, a0_l, a1_l); + a0 = __lasx_xvpermi_d(a0, 0x4E); + a1 = __lasx_xvpermi_d(a1, 0x4E); + LASX_UNPCK_L_W_H_2(a0, a1, a0_h, a1_h); + a_l = __lasx_xvmadd_w(a_bias, a0_l, v_yalpha1); + a_h = __lasx_xvmadd_w(a_bias, a0_h, v_yalpha1); + a_l = __lasx_xvmadd_w(a_l, v_yalpha, a1_l); + a_h = __lasx_xvmadd_w(a_h, v_yalpha, a1_h); + a_l = __lasx_xvsrai_w(a_l, 19); + a_h = __lasx_xvsrai_w(a_h, 19); + WRITE_FULL_L_A(R_l, G_l, B_l, a_l, c, dest, i, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_l, G_l, B_l, a_l, c, dest, i + 4, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_L_A(R_h, G_h, B_h, a_h, c, dest, i + 8, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_h, G_h, B_h, a_h, c, dest, i + 12, R, A, G, B, + y, target, hasAlpha, err); + } else { + WRITE_FULL_L(R_l, G_l, B_l, c, dest, i, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_l, G_l, B_l, c, dest, i + 4, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_L(R_h, G_h, B_h, c, dest, i + 8, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_h, G_h, B_h, c, dest, i + 12, R, G, B, + y, target, hasAlpha, err); + } + } + if (dstW - i >= 8) { + __m256i b0, b1, ub0, ub1, vb0, vb1; + __m256i y0_l, y1_l, u0_l; + __m256i v0_l, u1_l, v1_l; + __m256i y_l, u_l, v_l; + __m256i R_l, G_l, B_l; + + b0 = LASX_LD((buf0 + i)); + b1 = LASX_LD((buf1 + i)); + ub0 = LASX_LD((ubuf0 + i)); + ub1 = LASX_LD((ubuf1 + i)); + vb0 = LASX_LD((vbuf0 + i)); + vb1 = LASX_LD((vbuf1 + i)); + LASX_UNPCK_L_W_H_2(b0, b1, y0_l, y1_l); + LASX_UNPCK_L_W_H_4(ub0, ub1, vb0, vb1, u0_l, u1_l, v0_l, v1_l); + y0_l = __lasx_xvmul_w(y0_l, v_yalpha1); + u0_l = __lasx_xvmul_w(u0_l, v_uvalpha1); + v0_l = __lasx_xvmul_w(v0_l, v_uvalpha1); + y_l = __lasx_xvmadd_w(y0_l, v_yalpha, y1_l); + u_l = __lasx_xvmadd_w(u0_l, v_uvalpha, u1_l); + v_l = __lasx_xvmadd_w(v0_l, v_uvalpha, v1_l); + u_l = __lasx_xvsub_w(u_l, uv); + v_l = __lasx_xvsub_w(v_l, uv); + y_l = __lasx_xvsrai_w(y_l, 10); + u_l = __lasx_xvsrai_w(u_l, 10); + v_l = __lasx_xvsrai_w(v_l, 10); + YUVTORGB(y_l, u_l, v_l, R_l, G_l, B_l, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + + if (hasAlpha) { + __m256i a0, a1, a0_l; + __m256i a_l, a1_l; + + a0 = LASX_LD((abuf0 + i)); + a1 = LASX_LD((abuf1 + i)); + LASX_UNPCK_L_W_H_2(a0, a1, a0_l, a1_l); + a_l = __lasx_xvmadd_w(a_bias, a0_l, v_yalpha1); + a_l = __lasx_xvmadd_w(a_l, v_yalpha, a1_l); + a_l = __lasx_xvsrai_w(a_l, 19); + WRITE_FULL_L_A(R_l, G_l, B_l, a_l, c, dest, i, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_l, G_l, B_l, a_l, c, dest, i + 4, R, A, G, B, + y, target, hasAlpha, err); + } else { + WRITE_FULL_L(R_l, G_l, B_l, c, dest, i, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_l, G_l, B_l, c, dest, i + 4, R, G, B, + y, target, hasAlpha, err); + } + i += 8; + } + for (; i < dstW; i++){ + int Y = ( buf0[i] * yalpha1 + buf1[i] * yalpha ) >> 10; //FIXME rounding + int U = (ubuf0[i] * uvalpha1 + ubuf1[i] * uvalpha- uvtemp) >> 10; + int V = (vbuf0[i] * uvalpha1 + vbuf1[i] * uvalpha- uvtemp) >> 10; + + A = 0; + if (hasAlpha){ + A = (abuf0[i] * yalpha1 + abuf1[i] * yalpha + atemp) >> 19; + if (A & 0x100) + A = av_clip_uint8(A); + } + + Y -= y_offset; + Y *= y_coeff; + Y += ytemp; + R = (unsigned)Y + V * v2r_coe; + G = (unsigned)Y + V * v2g_coe + U * u2g_coe; + B = (unsigned)Y + U * u2b_coe; + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + c->dither_error[0][i] = err[0]; + c->dither_error[1][i] = err[1]; + c->dither_error[2][i] = err[2]; +} + +static void +yuv2rgb_full_1_template_lasx(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y, enum AVPixelFormat target, + int hasAlpha) +{ + const int16_t *ubuf0 = ubuf[0], *vbuf0 = vbuf[0]; + int i, B, G, R, A; + int step = (target == AV_PIX_FMT_RGB24 || target == AV_PIX_FMT_BGR24) ? 3 : 4; + int err[4] = {0}; + int ytemp = 1 << 21; + int bias_int = 64; + int len = dstW - 15; + __m256i bias = __lasx_xvreplgr2vr_w(bias_int); + __m256i y_temp = __lasx_xvreplgr2vr_w(ytemp); + YUVTORGB_SETUP + + if( target == AV_PIX_FMT_BGR4_BYTE || target == AV_PIX_FMT_RGB4_BYTE + || target == AV_PIX_FMT_BGR8 || target == AV_PIX_FMT_RGB8) + step = 1; + if (uvalpha < 2048) { + int uvtemp = 128 << 7; + __m256i uv = __lasx_xvreplgr2vr_w(uvtemp); + + for (i = 0; i < len; i += 16) { + __m256i b, ub, vb, ub_l, ub_h, vb_l, vb_h; + __m256i y_l, y_h, u_l, u_h, v_l, v_h; + __m256i R_l, R_h, G_l, G_h, B_l, B_h; + + b = LASX_LD((buf0 + i)); + ub = LASX_LD((ubuf0 + i)); + vb = LASX_LD((vbuf0 + i)); + LASX_UNPCK_L_W_H(b, y_l); + LASX_UNPCK_L_W_H_2(ub, vb, ub_l, vb_l); + b = __lasx_xvpermi_d(b, 0x4E); + ub = __lasx_xvpermi_d(ub, 0x4E); + vb = __lasx_xvpermi_d(vb, 0x4E); + LASX_UNPCK_L_W_H(b, y_h); + LASX_UNPCK_L_W_H_2(ub, vb, ub_h, vb_h); + y_l = __lasx_xvslli_w(y_l, 2); + y_h = __lasx_xvslli_w(y_h, 2); + u_l = __lasx_xvsub_w(ub_l, uv); + u_h = __lasx_xvsub_w(ub_h, uv); + v_l = __lasx_xvsub_w(vb_l, uv); + v_h = __lasx_xvsub_w(vb_h, uv); + u_l = __lasx_xvslli_w(u_l, 2); + u_h = __lasx_xvslli_w(u_h, 2); + v_l = __lasx_xvslli_w(v_l, 2); + v_h = __lasx_xvslli_w(v_h, 2); + YUVTORGB(y_l, u_l, v_l, R_l, G_l, B_l, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + YUVTORGB(y_h, u_h, v_h, R_h, G_h, B_h, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + + if(hasAlpha) { + __m256i a_src; + __m256i a_l, a_h; + + a_src = LASX_LD((abuf0 + i)); + a_src = __lasx_xvpermi_d(a_src, 0xD8); + LASX_ADDW_W_W_H_128SV(bias, a_src, a_l); + a_src = __lasx_xvpermi_d(a_src, 0xB1); + LASX_ADDW_W_W_H_128SV(bias, a_src, a_h); + a_l = __lasx_xvsrai_w(a_l, 7); + a_h = __lasx_xvsrai_w(a_h, 7); + WRITE_FULL_L_A(R_l, G_l, B_l, a_l, c, dest, i, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_l, G_l, B_l, a_l, c, dest, i + 4, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_L_A(R_h, G_h, B_h, a_h, c, dest, i + 8, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_h, G_h, B_h, a_h, c, dest, i + 12, R, A, G, B, + y, target, hasAlpha, err); + } else { + WRITE_FULL_L(R_l, G_l, B_l, c, dest, i, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_l, G_l, B_l, c, dest, i + 4, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_L(R_h, G_h, B_h, c, dest, i + 8, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_h, G_h, B_h, c, dest, i + 12, R, G, B, + y, target, hasAlpha, err); + } + } + if (dstW - i >= 8) { + __m256i b, ub, vb, ub_l, vb_l; + __m256i y_l, u_l, v_l; + __m256i R_l, G_l, B_l; + + b = LASX_LD((buf0 + i)); + ub = LASX_LD((ubuf0 + i)); + vb = LASX_LD((vbuf0 + i)); + LASX_UNPCK_L_W_H(b, y_l); + LASX_UNPCK_L_W_H_2(ub, vb, ub_l, vb_l); + y_l = __lasx_xvslli_w(y_l, 2); + u_l = __lasx_xvsub_w(ub_l, uv); + v_l = __lasx_xvsub_w(vb_l, uv); + u_l = __lasx_xvslli_w(u_l, 2); + v_l = __lasx_xvslli_w(v_l, 2); + YUVTORGB(y_l, u_l, v_l, R_l, G_l, B_l, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + + if(hasAlpha) { + __m256i a_src, a_l; + + a_src = LASX_LD((abuf0 + i)); + a_src = __lasx_xvpermi_d(a_src, 0xD8); + LASX_ADDW_W_W_H_128SV(bias, a_src, a_l); + a_l = __lasx_xvsrai_w(a_l, 7); + WRITE_FULL_L_A(R_l, G_l, B_l, a_l, c, dest, i, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_l, G_l, B_l, a_l, c, dest, i + 4, R, A, G, B, + y, target, hasAlpha, err); + } else { + WRITE_FULL_L(R_l, G_l, B_l, c, dest, i, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_l, G_l, B_l, c, dest, i + 4, R, G, B, + y, target, hasAlpha, err); + } + i += 8; + } + for (; i < dstW; i++) { + int Y = buf0[i] << 2; + int U = (ubuf0[i] - uvtemp) << 2; + int V = (vbuf0[i] - uvtemp) << 2; + + A = 0; + if(hasAlpha) { + A = (abuf0[i] + 64) >> 7; + if (A & 0x100) + A = av_clip_uint8(A); + } + Y -= y_offset; + Y *= y_coeff; + Y += ytemp; + R = (unsigned)Y + V * v2r_coe; + G = (unsigned)Y + V * v2g_coe + U * u2g_coe; + B = (unsigned)Y + U * u2b_coe; + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + } else { + const int16_t *ubuf1 = ubuf[1], *vbuf1 = vbuf[1]; + int uvtemp = 128 << 8; + __m256i uv = __lasx_xvreplgr2vr_w(uvtemp); + + for (i = 0; i < len; i += 16) { + __m256i b, ub0, ub1, vb0, vb1; + __m256i y_l, y_h, u_l, u_h, v_l, v_h; + __m256i R_l, R_h, G_l, G_h, B_l, B_h; + + b = LASX_LD((buf0 + i)); + ub0 = LASX_LD((ubuf0 + i)); + vb0 = LASX_LD((vbuf0 + i)); + ub1 = LASX_LD((ubuf1 + i)); + vb1 = LASX_LD((vbuf1 + i)); + LASX_UNPCK_L_W_H(b, y_l); + b = __lasx_xvpermi_d(b, 0X4E); + LASX_UNPCK_L_W_H(b, y_h); + y_l = __lasx_xvslli_w(y_l, 2); + y_h = __lasx_xvslli_w(y_h, 2); + ub0 = __lasx_xvpermi_d(ub0, 0xD8); + vb0 = __lasx_xvpermi_d(vb0, 0xD8); + ub1 = __lasx_xvpermi_d(ub1, 0xD8); + vb1 = __lasx_xvpermi_d(vb1, 0xD8); + LASX_ADDWL_W_H_2_128SV(ub0, ub1, vb0, vb1, u_l, v_l); + LASX_ADDWH_W_H_2_128SV(ub0, ub1, vb0, vb1, u_h, v_h); + u_l = __lasx_xvsub_w(u_l, uv); + u_h = __lasx_xvsub_w(u_h, uv); + v_l = __lasx_xvsub_w(v_l, uv); + v_h = __lasx_xvsub_w(v_h, uv); + u_l = __lasx_xvslli_w(u_l, 1); + u_h = __lasx_xvslli_w(u_h, 1); + v_l = __lasx_xvslli_w(v_l, 1); + v_h = __lasx_xvslli_w(v_h, 1); + YUVTORGB(y_l, u_l, v_l, R_l, G_l, B_l, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + YUVTORGB(y_h, u_h, v_h, R_h, G_h, B_h, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + + if(hasAlpha) { + __m256i a_src; + __m256i a_l, a_h; + + a_src = LASX_LD((abuf0 + i)); + a_src = __lasx_xvpermi_d(a_src, 0xD8); + LASX_ADDW_W_W_H_128SV(bias, a_src, a_l); + a_src = __lasx_xvpermi_d(a_src, 0xB1); + LASX_ADDW_W_W_H_128SV(bias, a_src, a_h); + a_l = __lasx_xvsrai_w(a_l, 7); + a_h = __lasx_xvsrai_w(a_h, 7); + WRITE_FULL_L_A(R_l, G_l, B_l, a_l, c, dest, i, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_l, G_l, B_l, a_l, c, dest, i + 4, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_L_A(R_h, G_h, B_h, a_h, c, dest, i + 8, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_h, G_h, B_h, a_h, c, dest, i + 12, R, A, G, B, + y, target, hasAlpha, err); + } else { + WRITE_FULL_L(R_l, G_l, B_l, c, dest, i, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_l, G_l, B_l, c, dest, i + 4, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_L(R_h, G_h, B_h, c, dest, i + 8, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_h, G_h, B_h, c, dest, i + 12, R, G, B, + y, target, hasAlpha, err); + } + } + if (dstW - i >= 8) { + __m256i b, ub0, ub1, vb0, vb1; + __m256i y_l, u_l, v_l; + __m256i R_l, G_l, B_l; + + b = LASX_LD((buf0 + i)); + ub0 = LASX_LD((ubuf0 + i)); + vb0 = LASX_LD((vbuf0 + i)); + ub1 = LASX_LD((ubuf1 + i)); + vb1 = LASX_LD((vbuf1 + i)); + LASX_UNPCK_L_W_H(b, y_l); + y_l = __lasx_xvslli_w(y_l, 2); + ub0 = __lasx_xvpermi_d(ub0, 0xD8); + vb0 = __lasx_xvpermi_d(vb0, 0xD8); + ub1 = __lasx_xvpermi_d(ub1, 0xD8); + vb1 = __lasx_xvpermi_d(vb1, 0xD8); + LASX_ADDWL_W_H_2_128SV(ub0, ub1, vb0, vb1, u_l, v_l); + u_l = __lasx_xvsub_w(u_l, uv); + v_l = __lasx_xvsub_w(v_l, uv); + u_l = __lasx_xvslli_w(u_l, 1); + v_l = __lasx_xvslli_w(v_l, 1); + YUVTORGB(y_l, u_l, v_l, R_l, G_l, B_l, offset, coeff, + y_temp, v2r, v2g, u2g, u2b); + + if(hasAlpha) { + __m256i a_src; + __m256i a_l; + + a_src = LASX_LD((abuf0 + i)); + a_src = __lasx_xvpermi_d(a_src, 0xD8); + LASX_ADDW_W_W_H_128SV(bias, a_src, a_l); + a_l = __lasx_xvsrai_w(a_l, 7); + WRITE_FULL_L_A(R_l, G_l, B_l, a_l, c, dest, i, R, A, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H_A(R_l, G_l, B_l, a_l, c, dest, i + 4, R, A, G, B, + y, target, hasAlpha, err); + } else { + WRITE_FULL_L(R_l, G_l, B_l, c, dest, i, R, G, B, + y, target, hasAlpha, err); + WRITE_FULL_H(R_l, G_l, B_l, c, dest, i + 4, R, G, B, + y, target, hasAlpha, err); + } + i += 8; + } + for (; i < dstW; i++) { + int Y = buf0[i] << 2; + int U = (ubuf0[i] + ubuf1[i] - uvtemp) << 1; + int V = (vbuf0[i] + vbuf1[i] - uvtemp) << 1; + + A = 0; + if(hasAlpha) { + A = (abuf0[i] + 64) >> 7; + if (A & 0x100) + A = av_clip_uint8(A); + } + Y -= y_offset; + Y *= y_coeff; + Y += ytemp; + R = (unsigned)Y + V * v2r_coe; + G = (unsigned)Y + V * v2g_coe + U * u2g_coe; + B = (unsigned)Y + U * u2b_coe; + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + } + c->dither_error[0][i] = err[0]; + c->dither_error[1][i] = err[1]; + c->dither_error[2][i] = err[2]; +} +#if CONFIG_SMALL +YUV2RGBWRAPPER(yuv2, rgb_full, bgra32_full, AV_PIX_FMT_BGRA, CONFIG_SWSCALE_ALPHA && c->needAlpha) +YUV2RGBWRAPPER(yuv2, rgb_full, abgr32_full, AV_PIX_FMT_ABGR, CONFIG_SWSCALE_ALPHA && c->needAlpha) +YUV2RGBWRAPPER(yuv2, rgb_full, rgba32_full, AV_PIX_FMT_RGBA, CONFIG_SWSCALE_ALPHA && c->needAlpha) +YUV2RGBWRAPPER(yuv2, rgb_full, argb32_full, AV_PIX_FMT_ARGB, CONFIG_SWSCALE_ALPHA && c->needAlpha) +#else +#if CONFIG_SWSCALE_ALPHA +YUV2RGBWRAPPER(yuv2, rgb_full, bgra32_full, AV_PIX_FMT_BGRA, 1) +YUV2RGBWRAPPER(yuv2, rgb_full, abgr32_full, AV_PIX_FMT_ABGR, 1) +YUV2RGBWRAPPER(yuv2, rgb_full, rgba32_full, AV_PIX_FMT_RGBA, 1) +YUV2RGBWRAPPER(yuv2, rgb_full, argb32_full, AV_PIX_FMT_ARGB, 1) +#endif +YUV2RGBWRAPPER(yuv2, rgb_full, bgrx32_full, AV_PIX_FMT_BGRA, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, xbgr32_full, AV_PIX_FMT_ABGR, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, rgbx32_full, AV_PIX_FMT_RGBA, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, xrgb32_full, AV_PIX_FMT_ARGB, 0) +#endif +YUV2RGBWRAPPER(yuv2, rgb_full, bgr24_full, AV_PIX_FMT_BGR24, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, rgb24_full, AV_PIX_FMT_RGB24, 0) + +YUV2RGBWRAPPER(yuv2, rgb_full, bgr4_byte_full, AV_PIX_FMT_BGR4_BYTE, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, rgb4_byte_full, AV_PIX_FMT_RGB4_BYTE, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, bgr8_full, AV_PIX_FMT_BGR8, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, rgb8_full, AV_PIX_FMT_RGB8, 0) +#undef yuvTorgb +#undef yuvTorgb_setup + + +av_cold void ff_sws_init_output_loongarch(SwsContext *c) +{ + + if(c->flags & SWS_FULL_CHR_H_INT) { + switch (c->dstFormat) { + case AV_PIX_FMT_RGBA: +#if CONFIG_SMALL + c->yuv2packedX = yuv2rgba32_full_X_lasx; + c->yuv2packed2 = yuv2rgba32_full_2_lasx; + c->yuv2packed1 = yuv2rgba32_full_1_lasx; +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + c->yuv2packedX = yuv2rgba32_full_X_lasx; + c->yuv2packed2 = yuv2rgba32_full_2_lasx; + c->yuv2packed1 = yuv2rgba32_full_1_lasx; + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packedX = yuv2rgbx32_full_X_lasx; + c->yuv2packed2 = yuv2rgbx32_full_2_lasx; + c->yuv2packed1 = yuv2rgbx32_full_1_lasx; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_ARGB: +#if CONFIG_SMALL + c->yuv2packedX = yuv2argb32_full_X_lasx; + c->yuv2packed2 = yuv2argb32_full_2_lasx; + c->yuv2packed1 = yuv2argb32_full_1_lasx; +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + c->yuv2packedX = yuv2argb32_full_X_lasx; + c->yuv2packed2 = yuv2argb32_full_2_lasx; + c->yuv2packed1 = yuv2argb32_full_1_lasx; + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packedX = yuv2xrgb32_full_X_lasx; + c->yuv2packed2 = yuv2xrgb32_full_2_lasx; + c->yuv2packed1 = yuv2xrgb32_full_1_lasx; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_BGRA: +#if CONFIG_SMALL + c->yuv2packedX = yuv2bgra32_full_X_lasx; + c->yuv2packed2 = yuv2bgra32_full_2_lasx; + c->yuv2packed1 = yuv2bgra32_full_1_lasx; +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + c->yuv2packedX = yuv2bgra32_full_X_lasx; + c->yuv2packed2 = yuv2bgra32_full_2_lasx; + c->yuv2packed1 = yuv2bgra32_full_1_lasx; + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packedX = yuv2bgrx32_full_X_lasx; + c->yuv2packed2 = yuv2bgrx32_full_2_lasx; + c->yuv2packed1 = yuv2bgrx32_full_1_lasx; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_ABGR: +#if CONFIG_SMALL + c->yuv2packedX = yuv2abgr32_full_X_lasx; + c->yuv2packed2 = yuv2abgr32_full_2_lasx; + c->yuv2packed1 = yuv2abgr32_full_1_lasx; +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + c->yuv2packedX = yuv2abgr32_full_X_lasx; + c->yuv2packed2 = yuv2abgr32_full_2_lasx; + c->yuv2packed1 = yuv2abgr32_full_1_lasx; + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packedX = yuv2xbgr32_full_X_lasx; + c->yuv2packed2 = yuv2xbgr32_full_2_lasx; + c->yuv2packed1 = yuv2xbgr32_full_1_lasx; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_RGB24: + c->yuv2packedX = yuv2rgb24_full_X_lasx; + c->yuv2packed2 = yuv2rgb24_full_2_lasx; + c->yuv2packed1 = yuv2rgb24_full_1_lasx; + break; + case AV_PIX_FMT_BGR24: + c->yuv2packedX = yuv2bgr24_full_X_lasx; + c->yuv2packed2 = yuv2bgr24_full_2_lasx; + c->yuv2packed1 = yuv2bgr24_full_1_lasx; + break; + case AV_PIX_FMT_BGR4_BYTE: + c->yuv2packedX = yuv2bgr4_byte_full_X_lasx; + c->yuv2packed2 = yuv2bgr4_byte_full_2_lasx; + c->yuv2packed1 = yuv2bgr4_byte_full_1_lasx; + break; + case AV_PIX_FMT_RGB4_BYTE: + c->yuv2packedX = yuv2rgb4_byte_full_X_lasx; + c->yuv2packed2 = yuv2rgb4_byte_full_2_lasx; + c->yuv2packed1 = yuv2rgb4_byte_full_1_lasx; + break; + case AV_PIX_FMT_BGR8: + c->yuv2packedX = yuv2bgr8_full_X_lasx; + c->yuv2packed2 = yuv2bgr8_full_2_lasx; + c->yuv2packed1 = yuv2bgr8_full_1_lasx; + break; + case AV_PIX_FMT_RGB8: + c->yuv2packedX = yuv2rgb8_full_X_lasx; + c->yuv2packed2 = yuv2rgb8_full_2_lasx; + c->yuv2packed1 = yuv2rgb8_full_1_lasx; + break; + } + } else { + switch (c->dstFormat) { + case AV_PIX_FMT_RGB32: + case AV_PIX_FMT_BGR32: +#if CONFIG_SMALL +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packed1 = yuv2rgbx32_1_lasx; + c->yuv2packed2 = yuv2rgbx32_2_lasx; + c->yuv2packedX = yuv2rgbx32_X_lasx; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_RGB32_1: + case AV_PIX_FMT_BGR32_1: +#if CONFIG_SMALL +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packed1 = yuv2rgbx32_1_1_lasx; + c->yuv2packed2 = yuv2rgbx32_1_2_lasx; + c->yuv2packedX = yuv2rgbx32_1_X_lasx; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_RGB24: + c->yuv2packed1 = yuv2rgb24_1_lasx; + c->yuv2packed2 = yuv2rgb24_2_lasx; + c->yuv2packedX = yuv2rgb24_X_lasx; + break; + case AV_PIX_FMT_BGR24: + c->yuv2packed1 = yuv2bgr24_1_lasx; + c->yuv2packed2 = yuv2bgr24_2_lasx; + c->yuv2packedX = yuv2bgr24_X_lasx; + break; + case AV_PIX_FMT_RGB565LE: + case AV_PIX_FMT_RGB565BE: + case AV_PIX_FMT_BGR565LE: + case AV_PIX_FMT_BGR565BE: + c->yuv2packed1 = yuv2rgb16_1_lasx; + c->yuv2packed2 = yuv2rgb16_2_lasx; + c->yuv2packedX = yuv2rgb16_X_lasx; + break; + case AV_PIX_FMT_RGB555LE: + case AV_PIX_FMT_RGB555BE: + case AV_PIX_FMT_BGR555LE: + case AV_PIX_FMT_BGR555BE: + c->yuv2packed1 = yuv2rgb15_1_lasx; + c->yuv2packed2 = yuv2rgb15_2_lasx; + c->yuv2packedX = yuv2rgb15_X_lasx; + break; + case AV_PIX_FMT_RGB444LE: + case AV_PIX_FMT_RGB444BE: + case AV_PIX_FMT_BGR444LE: + case AV_PIX_FMT_BGR444BE: + c->yuv2packed1 = yuv2rgb12_1_lasx; + c->yuv2packed2 = yuv2rgb12_2_lasx; + c->yuv2packedX = yuv2rgb12_X_lasx; + break; + case AV_PIX_FMT_RGB8: + case AV_PIX_FMT_BGR8: + c->yuv2packed1 = yuv2rgb8_1_lasx; + c->yuv2packed2 = yuv2rgb8_2_lasx; + c->yuv2packedX = yuv2rgb8_X_lasx; + break; + case AV_PIX_FMT_RGB4: + case AV_PIX_FMT_BGR4: + c->yuv2packed1 = yuv2rgb4_1_lasx; + c->yuv2packed2 = yuv2rgb4_2_lasx; + c->yuv2packedX = yuv2rgb4_X_lasx; + break; + case AV_PIX_FMT_RGB4_BYTE: + case AV_PIX_FMT_BGR4_BYTE: + c->yuv2packed1 = yuv2rgb4b_1_lasx; + c->yuv2packed2 = yuv2rgb4b_2_lasx; + c->yuv2packedX = yuv2rgb4b_X_lasx; + break; + } + } +} diff --git a/libswscale/loongarch/rgb2rgb_lasx.c b/libswscale/loongarch/rgb2rgb_lasx.c new file mode 100644 index 0000000000..d04cecd57f --- /dev/null +++ b/libswscale/loongarch/rgb2rgb_lasx.c @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2021 Loongson Technology Co. Ltd. + * Contributed by Hao Chen(chenhao@loongson.cn) + * All rights reserved. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "swscale_loongarch.h" +#include "libavutil/loongarch/generic_macros_lasx.h" + +void ff_interleave_bytes_lasx(const uint8_t *src1, const uint8_t *src2, + uint8_t *dest, int width, int height, + int src1Stride, int src2Stride, int dstStride) +{ + int h; + int len = width & (0xFFFFFFF0); + + for (h = 0; h < height; h++) { + int w, index = 0; + __m256i src_1, src_2, dst; + + for (w = 0; w < len; w += 16) { + src_1 = LASX_LD(src1 + w); + src_2 = LASX_LD(src2 + w); + src_1 = __lasx_xvpermi_d(src_1, 0xD8); + src_2 = __lasx_xvpermi_d(src_2, 0xD8); + dst = __lasx_xvilvl_b(src_2, src_1); + LASX_ST(dst, dest + index); + index += 32; + } + for (w = 0; w < width; w++) { + dest[(w << 1) + 0] = src1[w]; + dest[(w << 1) + 1] = src2[w]; + } + dest += dstStride; + src1 += src1Stride; + src2 += src2Stride; + } +} diff --git a/libswscale/loongarch/swscale_init_loongarch.c b/libswscale/loongarch/swscale_init_loongarch.c new file mode 100644 index 0000000000..d4fd3c2842 --- /dev/null +++ b/libswscale/loongarch/swscale_init_loongarch.c @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen(chenhao@loongson.cn) + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "swscale_loongarch.h" +#include "libswscale/swscale_internal.h" +#include "libswscale/rgb2rgb.h" +#include "libavutil/loongarch/cpu.h" + +av_cold void ff_sws_init_swscale_loongarch(SwsContext *c) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lasx(cpu_flags)) { + ff_sws_init_output_loongarch(c); + if (c->srcBpc == 8) { + if (c->dstBpc <= 14) { + c->hyScale = c->hcScale = ff_hscale_8_to_15_lasx; + } else { + c->hyScale = c->hcScale = ff_hscale_8_to_19_lasx; + } + } else { + c->hyScale = c->hcScale = c->dstBpc > 14 ? ff_hscale_16_to_19_lasx + : ff_hscale_16_to_15_lasx; + } + switch (c->srcFormat) { + case AV_PIX_FMT_GBRAP: + case AV_PIX_FMT_GBRP: + { + c->readChrPlanar = planar_rgb_to_uv_lasx; + c->readLumPlanar = planar_rgb_to_y_lasx; + } + break; + } + if (c->dstBpc == 8) + c->yuv2planeX = ff_yuv2planeX_8_lasx; + } +} + +av_cold void rgb2rgb_init_loongarch(void) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lasx(cpu_flags)) + interleaveBytes = ff_interleave_bytes_lasx; +} + +av_cold SwsFunc ff_yuv2rgb_init_loongarch(SwsContext *c) +{ + int cpu_flags = av_get_cpu_flags(); + if (have_lasx(cpu_flags)) { + switch (c->dstFormat) { + case AV_PIX_FMT_RGB24: + return yuv420_rgb24_lasx; + case AV_PIX_FMT_BGR24: + return yuv420_bgr24_lasx; + case AV_PIX_FMT_RGBA: + if (CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat)) { + break; + } else + return yuv420_rgba32_lasx; + case AV_PIX_FMT_ARGB: + if (CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat)) { + break; + } else + return yuv420_argb32_lasx; + case AV_PIX_FMT_BGRA: + if (CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat)) { + break; + } else + return yuv420_bgra32_lasx; + case AV_PIX_FMT_ABGR: + if (CONFIG_SWSCALE_ALPHA && isALPHA(c->srcFormat)) { + break; + } else + return yuv420_abgr32_lasx; + } + } + return NULL; +} diff --git a/libswscale/loongarch/swscale_lasx.c b/libswscale/loongarch/swscale_lasx.c new file mode 100644 index 0000000000..b3dc96681e --- /dev/null +++ b/libswscale/loongarch/swscale_lasx.c @@ -0,0 +1,944 @@ +/* + * Copyright (C) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen(chenhao@loongson.cn) + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "swscale_loongarch.h" +#include "libavutil/loongarch/generic_macros_lasx.h" +#include "libavutil/intreadwrite.h" + +#define SCALE_8_16(_sh) \ +{ \ + src0 = __lasx_xvldrepl_d(src + filterPos[0], 0); \ + src1 = __lasx_xvldrepl_d(src + filterPos[1], 0); \ + src2 = __lasx_xvldrepl_d(src + filterPos[2], 0); \ + src3 = __lasx_xvldrepl_d(src + filterPos[3], 0); \ + src4 = __lasx_xvldrepl_d(src + filterPos[4], 0); \ + src5 = __lasx_xvldrepl_d(src + filterPos[5], 0); \ + src6 = __lasx_xvldrepl_d(src + filterPos[6], 0); \ + src7 = __lasx_xvldrepl_d(src + filterPos[7], 0); \ + src8 = __lasx_xvldrepl_d(src + filterPos[8], 0); \ + src9 = __lasx_xvldrepl_d(src + filterPos[9], 0); \ + src10 = __lasx_xvldrepl_d(src + filterPos[10], 0); \ + src11 = __lasx_xvldrepl_d(src + filterPos[11], 0); \ + src12 = __lasx_xvldrepl_d(src + filterPos[12], 0); \ + src13 = __lasx_xvldrepl_d(src + filterPos[13], 0); \ + src14 = __lasx_xvldrepl_d(src + filterPos[14], 0); \ + src15 = __lasx_xvldrepl_d(src + filterPos[15], 0); \ + LASX_LD_8(filter, 16, filter0, filter1, filter2, filter3, \ + filter4, filter5, filter6, filter7); \ + LASX_PCKEV_D_4_128SV(src1, src0, src3, src2, src5, src4, \ + src7, src6, src0, src2, src4, src6); \ + LASX_PCKEV_D_4_128SV(src9, src8, src11, src10, src13, src12, \ + src15, src14, src8, src10, src12, src14);\ + LASX_UNPCK_L_HU_BU_8(src0, src2, src4, src6, src8, src10, \ + src12, src14, src0, src2, src4, src6, \ + src8, src10, src12, src14); \ + LASX_DP2_W_H_8(filter0, src0, filter1, src2, filter2, src4, \ + filter3, src6, filter4, src8, filter5, src10, \ + filter6, src12, filter7, src14, src0, src1, \ + src2, src3, src4, src5, src6, src7); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + src1 = __lasx_xvhaddw_d_w(src1, src1); \ + src2 = __lasx_xvhaddw_d_w(src2, src2); \ + src3 = __lasx_xvhaddw_d_w(src3, src3); \ + src4 = __lasx_xvhaddw_d_w(src4, src4); \ + src5 = __lasx_xvhaddw_d_w(src5, src5); \ + src6 = __lasx_xvhaddw_d_w(src6, src6); \ + src7 = __lasx_xvhaddw_d_w(src7, src7); \ + LASX_PCKEV_W_4_128SV(src1, src0, src3, src2, src5, src4, \ + src7, src6, src0, src1, src2, src3); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + src1 = __lasx_xvhaddw_d_w(src1, src1); \ + src2 = __lasx_xvhaddw_d_w(src2, src2); \ + src3 = __lasx_xvhaddw_d_w(src3, src3); \ + LASX_PCKEV_W_2_128SV(src1, src0, src3, src2, src0, src1); \ + LASX_SRAI_W_2(src0, src1, src0, src1, _sh); \ + src0 = __lasx_xvmin_w(src0, vmax); \ + src1 = __lasx_xvmin_w(src1, vmax); \ + src0 = __lasx_xvperm_w(src0, shuf); \ + src1 = __lasx_xvperm_w(src1, shuf); \ + LASX_PCKEV_H(src1, src0, src0); \ + LASX_ST(src0, dst); \ + filterPos += 16; \ + filter += 128; \ + dst += 16; \ +} + +#define SCALE_8_8(_sh) \ +{ \ + src0 = __lasx_xvldrepl_d(src + filterPos[0], 0); \ + src1 = __lasx_xvldrepl_d(src + filterPos[1], 0); \ + src2 = __lasx_xvldrepl_d(src + filterPos[2], 0); \ + src3 = __lasx_xvldrepl_d(src + filterPos[3], 0); \ + src4 = __lasx_xvldrepl_d(src + filterPos[4], 0); \ + src5 = __lasx_xvldrepl_d(src + filterPos[5], 0); \ + src6 = __lasx_xvldrepl_d(src + filterPos[6], 0); \ + src7 = __lasx_xvldrepl_d(src + filterPos[7], 0); \ + LASX_LD_4(filter, 16, filter0, filter1, filter2, filter3); \ + LASX_PCKEV_D_4_128SV(src1, src0, src3, src2, src5, src4, \ + src7, src6, src0, src2, src4, src6); \ + LASX_UNPCK_L_HU_BU_4(src0, src2, src4, src6, \ + src0, src2, src4, src6); \ + LASX_DP2_W_H_4(filter0, src0, filter1, src2, filter2, src4, \ + filter3, src6, src0, src1, src2, src3); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + src1 = __lasx_xvhaddw_d_w(src1, src1); \ + src2 = __lasx_xvhaddw_d_w(src2, src2); \ + src3 = __lasx_xvhaddw_d_w(src3, src3); \ + LASX_PCKEV_W_2_128SV(src1, src0, src3, src2, src0, src1); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + src1 = __lasx_xvhaddw_d_w(src1, src1); \ + LASX_PCKEV_W_128SV(src1, src0, src0); \ + LASX_SRAI_W(src0, src0, _sh); \ + src0 = __lasx_xvmin_w(src0, vmax); \ + src0 = __lasx_xvperm_w(src0, shuf); \ +} + +#define SCALE_8_4(_sh) \ +{ \ + src0 = __lasx_xvldrepl_d(src + filterPos[0], 0); \ + src1 = __lasx_xvldrepl_d(src + filterPos[1], 0); \ + src2 = __lasx_xvldrepl_d(src + filterPos[2], 0); \ + src3 = __lasx_xvldrepl_d(src + filterPos[3], 0); \ + LASX_LD_2(filter, 16, filter0, filter1); \ + LASX_PCKEV_D_2_128SV(src1, src0, src3, src2, src0, src2); \ + LASX_UNPCK_L_HU_BU_2(src0, src2, src0, src2); \ + LASX_DP2_W_H_2(filter0, src0, filter1, src2, src0, src1); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + src1 = __lasx_xvhaddw_d_w(src1, src1); \ + LASX_PCKEV_W_128SV(src1, src0, src0); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + LASX_PCKEV_W_128SV(src0, src0, src0); \ + LASX_SRAI_W(src0, src0, _sh); \ + src0 = __lasx_xvmin_w(src0, vmax); \ + src0 = __lasx_xvperm_w(src0, shuf); \ +} + +#define SCALE_8_2(_sh) \ +{ \ + src0 = __lasx_xvldrepl_d(src + filterPos[0], 0); \ + src1 = __lasx_xvldrepl_d(src + filterPos[1], 0); \ + filter0 = LASX_LD(filter); \ + LASX_PCKEV_D_128SV(src1, src0, src0); \ + LASX_UNPCK_L_HU_BU(src0, src0); \ + LASX_DP2_W_H(filter0, src0, src0); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + src0 = __lasx_xvhaddw_q_d(src0, src0); \ + LASX_SRAI_W(src0, src0, _sh); \ + src0 = __lasx_xvmin_w(src0, vmax); \ + dst[0] = __lasx_xvpickve2gr_w(src0, 0); \ + dst[1] = __lasx_xvpickve2gr_w(src0, 4); \ + filterPos += 2; \ + filter += 16; \ + dst += 2; \ +} + +#define SCALE_4_16(_sh) \ +{ \ + src0 = __lasx_xvldrepl_w(src + filterPos[0], 0); \ + src1 = __lasx_xvldrepl_w(src + filterPos[1], 0); \ + src2 = __lasx_xvldrepl_w(src + filterPos[2], 0); \ + src3 = __lasx_xvldrepl_w(src + filterPos[3], 0); \ + src4 = __lasx_xvldrepl_w(src + filterPos[4], 0); \ + src5 = __lasx_xvldrepl_w(src + filterPos[5], 0); \ + src6 = __lasx_xvldrepl_w(src + filterPos[6], 0); \ + src7 = __lasx_xvldrepl_w(src + filterPos[7], 0); \ + src8 = __lasx_xvldrepl_w(src + filterPos[8], 0); \ + src9 = __lasx_xvldrepl_w(src + filterPos[9], 0); \ + src10 = __lasx_xvldrepl_w(src + filterPos[10], 0); \ + src11 = __lasx_xvldrepl_w(src + filterPos[11], 0); \ + src12 = __lasx_xvldrepl_w(src + filterPos[12], 0); \ + src13 = __lasx_xvldrepl_w(src + filterPos[13], 0); \ + src14 = __lasx_xvldrepl_w(src + filterPos[14], 0); \ + src15 = __lasx_xvldrepl_w(src + filterPos[15], 0); \ + LASX_LD_4(filter, 16, filter0, filter1, filter2, filter3); \ + LASX_ILVL_W_4_128SV(src1, src0, src3, src2, src5, src4, \ + src7, src6, src0, src2, src4, src6); \ + LASX_ILVL_W_4_128SV(src9, src8, src11, src10, src13, src12, \ + src15, src14, src8, src10, src12, src14); \ + LASX_ILVL_D_4_128SV(src2, src0, src6, src4, src10, src8, \ + src14, src12, src0, src1, src2, src3); \ + LASX_UNPCK_L_HU_BU_4(src0, src1, src2, src3, \ + src0, src1, src2, src3); \ + LASX_DP2_W_H_4(filter0, src0, filter1, src1, filter2, src2, \ + filter3, src3, src0, src1, src2, src3); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + src1 = __lasx_xvhaddw_d_w(src1, src1); \ + src2 = __lasx_xvhaddw_d_w(src2, src2); \ + src3 = __lasx_xvhaddw_d_w(src3, src3); \ + LASX_PCKEV_W_2_128SV(src1, src0, src3, src2, src0, src1); \ + LASX_SRAI_W_2(src0, src1, src0, src1, _sh); \ + src0 = __lasx_xvmin_w(src0, vmax); \ + src1 = __lasx_xvmin_w(src1, vmax); \ + LASX_PCKEV_H_128SV(src1, src0, src0); \ + src0 = __lasx_xvperm_w(src0, shuf); \ + LASX_ST(src0, dst); \ + filterPos += 16; \ + filter += 64; \ + dst += 16; \ +} + +#define SCALE_4_8(_sh) \ +{ \ + src0 = __lasx_xvldrepl_w(src + filterPos[0], 0); \ + src1 = __lasx_xvldrepl_w(src + filterPos[1], 0); \ + src2 = __lasx_xvldrepl_w(src + filterPos[2], 0); \ + src3 = __lasx_xvldrepl_w(src + filterPos[3], 0); \ + src4 = __lasx_xvldrepl_w(src + filterPos[4], 0); \ + src5 = __lasx_xvldrepl_w(src + filterPos[5], 0); \ + src6 = __lasx_xvldrepl_w(src + filterPos[6], 0); \ + src7 = __lasx_xvldrepl_w(src + filterPos[7], 0); \ + LASX_LD_2(filter, 16, filter0, filter1); \ + LASX_ILVL_W_4_128SV(src1, src0, src3, src2, src5, src4, \ + src7, src6, src0, src2, src4, src6); \ + LASX_ILVL_D_2_128SV(src2, src0, src6, src4, src0, src1); \ + LASX_UNPCK_L_HU_BU_2(src0, src1, src0, src1); \ + LASX_DP2_W_H_2(filter0, src0, filter1, src1, src0, src1); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + src1 = __lasx_xvhaddw_d_w(src1, src1); \ + LASX_PCKEV_W_128SV(src1, src0, src0); \ + LASX_SRAI_W(src0, src0, _sh); \ + src0 = __lasx_xvmin_w(src0, vmax); \ +} + +#define SCALE_4_4(_sh) \ +{ \ + src0 = __lasx_xvldrepl_w(src + filterPos[0], 0); \ + src1 = __lasx_xvldrepl_w(src + filterPos[1], 0); \ + src2 = __lasx_xvldrepl_w(src + filterPos[2], 0); \ + src3 = __lasx_xvldrepl_w(src + filterPos[3], 0); \ + filter0 = LASX_LD(filter); \ + LASX_ILVL_W_2_128SV(src1, src0, src3, src2, src0, src1); \ + LASX_ILVL_D_128SV(src1, src0, src0); \ + LASX_UNPCK_L_HU_BU(src0, src0); \ + LASX_DP2_W_H(filter0, src0, src0); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + LASX_SRAI_W(src0, src0, _sh); \ + src0 = __lasx_xvmin_w(src0, vmax); \ + LASX_PCKEV_W(src0, src0, src0); \ +} + +#define SCALE_4_2(_sh) \ +{ \ + src0 = __lasx_xvldrepl_w(src + filterPos[0], 0); \ + src1 = __lasx_xvldrepl_w(src + filterPos[1], 0); \ + filter0 = LASX_LD(filter); \ + LASX_ILVL_W_128SV(src1, src0, src0); \ + LASX_UNPCK_L_HU_BU(src0, src0); \ + LASX_DP2_W_H(filter0, src0, src0); \ + src0 = __lasx_xvhaddw_d_w(src0, src0); \ + LASX_SRAI_W(src0, src0, _sh); \ + src0 = __lasx_xvmin_w(src0, vmax); \ + dst[0] = __lasx_xvpickve2gr_w(src0, 0); \ + dst[1] = __lasx_xvpickve2gr_w(src0, 2); \ + filterPos += 2; \ + filter += 8; \ + dst += 2; \ +} + +#define SCALE_16 \ +{ \ + src0 = __lasx_xvldrepl_d((srcPos1 + j), 0); \ + src1 = __lasx_xvldrepl_d((srcPos2 + j), 0); \ + src2 = __lasx_xvldrepl_d((srcPos3 + j), 0); \ + src3 = __lasx_xvldrepl_d((srcPos4 + j), 0); \ + filter0 = LASX_LD(filterStart1 + j); \ + filter1 = LASX_LD(filterStart2 + j); \ + filter2 = LASX_LD(filterStart3 + j); \ + filter3 = LASX_LD(filterStart4 + j); \ + src0 = __lasx_xvpermi_q(src0, src1, 0x02); \ + src1 = __lasx_xvpermi_q(src2, src3, 0x02); \ + filter0 = __lasx_xvpermi_q(filter0, filter1, 0x02); \ + filter1 = __lasx_xvpermi_q(filter2, filter3, 0x02); \ + LASX_ILVL_B_2_128SV(zero, src0, zero, src1, src0, src1); \ + LASX_DP2_W_H(filter0, src0, out0); \ + LASX_DP2_W_H(filter1, src1, out1); \ + src0 = __lasx_xvhaddw_d_w(out0, out0); \ + src1 = __lasx_xvhaddw_d_w(out1, out1); \ + out0 = __lasx_xvpackev_d(src1, src0); \ + out1 = __lasx_xvpackod_d(src1, src0); \ + out0 = __lasx_xvadd_w(out0, out1); \ + out = __lasx_xvadd_w(out, out0); \ +} + +void ff_hscale_8_to_15_lasx(SwsContext *c, int16_t *dst, int dstW, + const uint8_t *src, const int16_t *filter, + const int32_t *filterPos, int filterSize) +{ + int i; + int max = (1 << 15) - 1; + + if (filterSize == 8) { + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i src8, src9, src10, src11, src12, src13, src14, src15; + __m256i filter0, filter1, filter2, filter3; + __m256i filter4, filter5, filter6, filter7; + __m256i vmax = __lasx_xvreplgr2vr_w(max); + __m256i shuf = {0x0000000400000000, 0x0000000500000001, 0x0000000600000002, 0x0000000700000003}; + int len = dstW >> 4; + int res = dstW & 15; + while (len--) { + SCALE_8_16(7); + } + if (res & 8) { + SCALE_8_8(7); + LASX_PCKEV_H_128SV(src0, src0, src0); + LASX_ST_D_2(src0, 0, 2, dst, 4); + filterPos += 8; + filter += 64; + dst += 8; + } + if (res & 4) { + SCALE_8_4(7); + LASX_PCKEV_H_128SV(src0, src0, src0); + LASX_ST_D(src0, 0, dst); + filterPos += 4; + filter += 32; + dst += 4; + } + if (res & 2) { + SCALE_8_2(7); + } + if (res & 1) { + int val = 0; + src0 = __lasx_xvldrepl_d(src + filterPos[0], 0); + filter0 = LASX_LD(filter); + LASX_UNPCK_L_HU_BU(src0, src0); + LASX_DP2_W_H(filter0, src0, src0); + src0 = __lasx_xvhaddw_d_w(src0, src0); + src0 = __lasx_xvhaddw_q_d(src0, src0); + val = __lasx_xvpickve2gr_w(src0, 0); + dst[0] = FFMIN(val >> 7, max); + } + } else if (filterSize == 4) { + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i src8, src9, src10, src11, src12, src13, src14, src15; + __m256i filter0, filter1, filter2, filter3; + __m256i vmax = __lasx_xvreplgr2vr_w(max); + __m256i shuf = {0x0000000400000000, 0x0000000500000001, 0x0000000600000002, 0x0000000700000003}; + int len = dstW >> 4; + int res = dstW & 15; + while (len--) { + SCALE_4_16(7); + } + if (res & 8) { + SCALE_4_8(7); + LASX_PCKEV_H_128SV(src1, src0, src0); + src0 = __lasx_xvperm_w(src0, shuf); + LASX_ST_D_2(src0, 0, 1, dst, 4); + filterPos += 8; + filter += 32; + dst += 8; + } + if (res & 4) { + SCALE_4_4(7); + LASX_PCKEV_H_128SV(src0, src0, src0); + LASX_ST_D(src0, 0, dst); + filterPos += 4; + filter += 16; + dst += 4; + } + if (res & 2) { + SCALE_4_2(7); + } + if (res & 1) { + int val = 0; + const uint8_t *srcPos = src + filterPos[0]; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[0] = FFMIN(val >> 7, max); + } + } else if (filterSize > 8) { + int filterlen = filterSize - 7; + int len = dstW >> 2; + int res = dstW & 3; + __m256i zero = __lasx_xvldi(0); + + while (len--) { + __m256i src0, src1, src2, src3; + __m256i filter0, filter1, filter2, filter3, out0, out1; + __m256i out = zero; + const uint8_t *srcPos1 = src + filterPos[0]; + const uint8_t *srcPos2 = src + filterPos[1]; + const uint8_t *srcPos3 = src + filterPos[2]; + const uint8_t *srcPos4 = src + filterPos[3]; + const int16_t *filterStart1 = filter; + const int16_t *filterStart2 = filterStart1 + filterSize; + const int16_t *filterStart3 = filterStart2 + filterSize; + const int16_t *filterStart4 = filterStart3 + filterSize; + int j, val1 = 0, val2 = 0, val3 = 0, val4 = 0; + + for (j = 0; j < filterlen; j += 8) { + SCALE_16 + } + val1 = __lasx_xvpickve2gr_w(out, 0); + val2 = __lasx_xvpickve2gr_w(out, 4); + val3 = __lasx_xvpickve2gr_w(out, 2); + val4 = __lasx_xvpickve2gr_w(out, 6); + for (; j < filterSize; j++) { + val1 += ((int)srcPos1[j]) * filterStart1[j]; + val2 += ((int)srcPos2[j]) * filterStart2[j]; + val3 += ((int)srcPos3[j]) * filterStart3[j]; + val4 += ((int)srcPos4[j]) * filterStart4[j]; + } + dst[0] = FFMIN(val1 >> 7, max); + dst[1] = FFMIN(val2 >> 7, max); + dst[2] = FFMIN(val3 >> 7, max); + dst[3] = FFMIN(val4 >> 7, max); + dst += 4; + filterPos += 4; + filter = filterStart4 + filterSize; + } + for(i = 0; i < res; i++) { + int j, val = 0; + const uint8_t *srcPos = src + filterPos[i]; + __m256i src1, filter0, out0; + + for (j = 0; j < filterlen; j += 8) { + src1 = __lasx_xvldrepl_d((srcPos + j), 0); + filter0 = LASX_LD(filter + j); + LASX_ILVL_B_128SV(zero, src1, src1); + LASX_DP2_W_H(filter0, src1, out0); + out0 = __lasx_xvhaddw_d_w(out0, out0); + out0 = __lasx_xvhaddw_q_d(out0, out0); + val += __lasx_xvpickve2gr_w(out0, 0); + } + for (; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> 7, max); + filter += filterSize; + } + } else { + for (i = 0; i < dstW; i++) { + int val = 0; + const uint8_t *srcPos = src + filterPos[i]; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> 7, max); + filter += filterSize; + } + } +} + +void ff_hscale_8_to_19_lasx(SwsContext *c, int16_t *_dst, int dstW, + const uint8_t *src, const int16_t *filter, + const int32_t *filterPos, int filterSize) +{ + int i; + int max = (1 << 19) - 1; + int32_t *dst = (int32_t *) _dst; + + if (filterSize == 8) { + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i filter0, filter1, filter2, filter3; + __m256i vmax = __lasx_xvreplgr2vr_w(max); + __m256i shuf = {0x0000000400000000, 0x0000000500000001, 0x0000000600000002, 0x0000000700000003}; + int len = dstW >> 3; + int res = dstW & 7; + while (len--) { + SCALE_8_8(3); + LASX_ST(src0, dst); + filterPos += 8; + filter += 64; + dst += 8; + } + if (res & 4) { + SCALE_8_4(3); + LASX_ST_D_2(src0, 0, 1, dst, 4); + filterPos += 4; + filter += 32; + dst += 4; + } + if (res & 2) { + SCALE_8_2(3); + } + if (res & 1) { + int val = 0; + __m256i src0, filter0, out0; + + src0 = __lasx_xvldrepl_d(src + filterPos[0], 0); + filter0 = LASX_LD(filter); + LASX_UNPCK_L_HU_BU(src0, src0); + LASX_DP2_W_H(filter0, src0, out0); + out0 = __lasx_xvhaddw_d_w(out0, out0); + out0 = __lasx_xvhaddw_q_d(out0, out0); + val = __lasx_xvpickve2gr_w(out0, 0); + dst[0] = FFMIN(val >> 3, max); + } + } else if (filterSize == 4) { + __m256i src0, src1, src2, src3, src4, src5, src6, src7; + __m256i filter0, filter1; + __m256i vmax = __lasx_xvreplgr2vr_w(max); + __m256i shuf = {0x0000000100000000, 0x0000000500000004, 0x0000000300000002, 0x0000000700000006}; + int len = dstW >> 3; + int res = dstW & 7; + while (len--) { + SCALE_4_8(3); + src0 = __lasx_xvperm_w(src0, shuf); + LASX_ST(src0, dst); + filterPos += 8; + filter += 32; + dst += 8; + } + if (res & 4) { + SCALE_4_4(3); + LASX_ST_D_2(src0, 0, 2, dst, 4); + filterPos += 4; + filter += 16; + dst += 4; + } + if (res & 2) { + SCALE_4_2(3); + } + if (res & 1) { + int val = 0; + const uint8_t *srcPos = src + filterPos[0]; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[0] = FFMIN(val >> 3, max); + } + } else if (filterSize > 8) { + int len = dstW >> 2; + int res = dstW & 3; + int filterlen = filterSize - 7; + __m256i zero = __lasx_xvldi(0); + + while (len--) { + __m256i src0, src1, src2, src3; + __m256i filter0, filter1, filter2, filter3, out0, out1; + __m256i out = zero; + const uint8_t *srcPos1 = src + filterPos[0]; + const uint8_t *srcPos2 = src + filterPos[1]; + const uint8_t *srcPos3 = src + filterPos[2]; + const uint8_t *srcPos4 = src + filterPos[3]; + const int16_t *filterStart1 = filter; + const int16_t *filterStart2 = filterStart1 + filterSize; + const int16_t *filterStart3 = filterStart2 + filterSize; + const int16_t *filterStart4 = filterStart3 + filterSize; + int j, val1 = 0, val2 = 0, val3 = 0, val4 = 0; + + for (j = 0; j < filterlen; j += 8) { + SCALE_16 + } + val1 = __lasx_xvpickve2gr_w(out, 0); + val2 = __lasx_xvpickve2gr_w(out, 4); + val3 = __lasx_xvpickve2gr_w(out, 2); + val4 = __lasx_xvpickve2gr_w(out, 6); + for (; j < filterSize; j++) { + val1 += ((int)srcPos1[j]) * filterStart1[j]; + val2 += ((int)srcPos2[j]) * filterStart2[j]; + val3 += ((int)srcPos3[j]) * filterStart3[j]; + val4 += ((int)srcPos4[j]) * filterStart4[j]; + } + dst[0] = FFMIN(val1 >> 3, max); + dst[1] = FFMIN(val2 >> 3, max); + dst[2] = FFMIN(val3 >> 3, max); + dst[3] = FFMIN(val4 >> 3, max); + dst += 4; + filterPos += 4; + filter = filterStart4 + filterSize; + } + for (i = 0; i < res; i++) { + int j, val = 0; + const uint8_t *srcPos = src + filterPos[i]; + __m256i src1, filter0, out0; + + for (j = 0; j < filterlen; j += 8) { + src1 = __lasx_xvldrepl_d((srcPos + j), 0); + filter0 = LASX_LD(filter + j); + LASX_ILVL_B_128SV(zero, src1, src1); + LASX_DP2_W_H(filter0, src1, out0); + out0 = __lasx_xvhaddw_d_w(out0, out0); + out0 = __lasx_xvhaddw_q_d(out0, out0); + val += __lasx_xvpickve2gr_w(out0, 0); + } + for (; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> 3, max); + filter += filterSize; + } + } else { + for (i = 0; i < dstW; i++) { + int val = 0; + const uint8_t *srcPos = src + filterPos[i]; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> 3, max); + filter += filterSize; + } + } +} + +#undef SCALE_16 + +#define SCALE_8 \ +{ \ + int val1, val2, val3, val4; \ + __m256i src0, src1, src2, src3, filter0, filter1, out0, out1; \ + src0 = LASX_LD(src + filterPos[0]); \ + src1 = LASX_LD(src + filterPos[1]); \ + src2 = LASX_LD(src + filterPos[2]); \ + src3 = LASX_LD(src + filterPos[3]); \ + filter0 = LASX_LD(filter); \ + filter1 = LASX_LD(filter + 16); \ + src0 = __lasx_xvpermi_q(src0, src1, 0x02); \ + src2 = __lasx_xvpermi_q(src2, src3, 0x02); \ + LASX_DP2_W_HU_H_2(src0, filter0, src2, filter1, out0, out1); \ + src0 = __lasx_xvhaddw_d_w(out0, out0); \ + src1 = __lasx_xvhaddw_d_w(out1, out1); \ + out0 = __lasx_xvpackev_d(src1, src0); \ + out1 = __lasx_xvpackod_d(src1, src0); \ + out0 = __lasx_xvadd_w(out0, out1); \ + out0 = __lasx_xvsra_w(out0, shift); \ + val1 = __lasx_xvpickve2gr_w(out0, 0); \ + val2 = __lasx_xvpickve2gr_w(out0, 4); \ + val3 = __lasx_xvpickve2gr_w(out0, 2); \ + val4 = __lasx_xvpickve2gr_w(out0, 6); \ + dst[0] = FFMIN(val1, max); \ + dst[1] = FFMIN(val2, max); \ + dst[2] = FFMIN(val3, max); \ + dst[3] = FFMIN(val4, max); \ + filterPos += 4; \ + filter += 32; \ + dst += 4; \ +} + +#define SCALE_16 \ +{ \ + src0 = LASX_LD(srcPos1 + j); \ + src1 = LASX_LD(srcPos2 + j); \ + src2 = LASX_LD(srcPos3 + j); \ + src3 = LASX_LD(srcPos4 + j); \ + filter0 = LASX_LD(filterStart1 + j); \ + filter1 = LASX_LD(filterStart2 + j); \ + filter2 = LASX_LD(filterStart3 + j); \ + filter3 = LASX_LD(filterStart4 + j); \ + src0 = __lasx_xvpermi_q(src0, src1, 0x02); \ + src1 = __lasx_xvpermi_q(src2, src3, 0x02); \ + filter0 = __lasx_xvpermi_q(filter0, filter1, 0x02); \ + filter1 = __lasx_xvpermi_q(filter2, filter3, 0x02); \ + LASX_DP2_W_HU_H_2(src0, filter0, src1, filter1, out0, out1); \ + src0 = __lasx_xvhaddw_d_w(out0, out0); \ + src1 = __lasx_xvhaddw_d_w(out1, out1); \ + out0 = __lasx_xvpackev_d(src1, src0); \ + out1 = __lasx_xvpackod_d(src1, src0); \ + out0 = __lasx_xvadd_w(out0, out1); \ + out = __lasx_xvadd_w(out, out0); \ +} + +void ff_hscale_16_to_15_lasx(SwsContext *c, int16_t *dst, int dstW, + const uint8_t *_src, const int16_t *filter, + const int32_t *filterPos, int filterSize) +{ + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(c->srcFormat); + int i; + const uint16_t *src = (const uint16_t *) _src; + int sh = desc->comp[0].depth - 1; + int max = (1 << 15) - 1; + int len = dstW >> 2; + int res = dstW & 3; + __m256i shift; + __m256i zero = __lasx_xvldi(0); + + if (sh < 15) { + sh = isAnyRGB(c->srcFormat) || c->srcFormat==AV_PIX_FMT_PAL8 ? 13 : + (desc->comp[0].depth - 1); + } else if (desc->flags && AV_PIX_FMT_FLAG_FLOAT) { + sh = 15; + } + shift = __lasx_xvreplgr2vr_w(sh); + + if (filterSize == 8) { + for (i = 0; i < len; i++) { + SCALE_8 + } + for (i = 0; i < res; i++) { + int val = 0; + __m256i src0, filter0, out0; + + src0 = LASX_LD(src + filterPos[i]); + filter0 = LASX_LD(filter); + LASX_DP2_W_HU_H(src0, filter0, out0); + out0 = __lasx_xvhaddw_d_w(out0, out0); + out0 = __lasx_xvhaddw_q_d(out0, out0); + val = __lasx_xvpickve2gr_w(out0, 0); + dst[i] = FFMIN(val >> sh, max); + filter += 8; + } + } else if (filterSize == 4) { + for (i = 0; i < len; i++) { + __m256i src1, src2, src3, src4, src0, filter0, out0; + + src1 = __lasx_xvldrepl_d(src + filterPos[0], 0); + src2 = __lasx_xvldrepl_d(src + filterPos[1], 0); + src3 = __lasx_xvldrepl_d(src + filterPos[2], 0); + src4 = __lasx_xvldrepl_d(src + filterPos[3], 0); + filter0 = LASX_LD(filter); + src1 = __lasx_xvextrins_d(src1, src2, 0x10); + src3 = __lasx_xvextrins_d(src3, src4, 0x10); + src0 = __lasx_xvpermi_q(src1, src3, 0x02); + LASX_DP2_W_HU_H(src0, filter0, out0); + out0 = __lasx_xvhaddw_d_w(out0, out0); + out0 = __lasx_xvsra_w(out0, shift); + dst[0] = FFMIN((__lasx_xvpickve2gr_w(out0, 0)), max); + dst[1] = FFMIN((__lasx_xvpickve2gr_w(out0, 2)), max); + dst[2] = FFMIN((__lasx_xvpickve2gr_w(out0, 4)), max); + dst[3] = FFMIN((__lasx_xvpickve2gr_w(out0, 6)), max); + dst += 4; + filterPos += 4; + filter += 16; + } + for (i = 0; i < res; i++) { + int val = 0; + const uint16_t *srcPos = src + filterPos[i]; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> sh, max); + filter += 4; + } + } else if (filterSize > 8) { + int filterlen = filterSize - 7; + + for (i = 0; i < len; i++) { + __m256i src0, src1, src2, src3; + __m256i filter0, filter1, filter2, filter3, out0, out1; + __m256i out = zero; + const uint16_t *srcPos1 = src + filterPos[0]; + const uint16_t *srcPos2 = src + filterPos[1]; + const uint16_t *srcPos3 = src + filterPos[2]; + const uint16_t *srcPos4 = src + filterPos[3]; + const int16_t *filterStart1 = filter; + const int16_t *filterStart2 = filterStart1 + filterSize; + const int16_t *filterStart3 = filterStart2 + filterSize; + const int16_t *filterStart4 = filterStart3 + filterSize; + int j, val1 = 0, val2 = 0, val3 = 0, val4 = 0; + + for (j = 0; j < filterlen; j += 8) { + SCALE_16 + } + val1 = __lasx_xvpickve2gr_w(out, 0); + val2 = __lasx_xvpickve2gr_w(out, 4); + val3 = __lasx_xvpickve2gr_w(out, 2); + val4 = __lasx_xvpickve2gr_w(out, 6); + for (; j < filterSize; j++) { + val1 += ((int)srcPos1[j]) * filterStart1[j]; + val2 += ((int)srcPos2[j]) * filterStart2[j]; + val3 += ((int)srcPos3[j]) * filterStart3[j]; + val4 += ((int)srcPos4[j]) * filterStart4[j]; + } + dst[0] = FFMIN(val1 >> sh, max); + dst[1] = FFMIN(val2 >> sh, max); + dst[2] = FFMIN(val3 >> sh, max); + dst[3] = FFMIN(val4 >> sh, max); + dst += 4; + filterPos += 4; + filter = filterStart4 + filterSize; + } + for (i = 0; i < res; i++) { + int j, val = 0; + const uint16_t *srcPos = src + filterPos[i]; + __m256i src0, filter0, out0; + + for (j = 0; j < filterlen; j += 8) { + src0 = LASX_LD(srcPos + j); + filter0 = LASX_LD(filter + j); + LASX_DP2_W_HU_H(src0, filter0, out0); + out0 = __lasx_xvhaddw_d_w(out0, out0); + out0 = __lasx_xvhaddw_q_d(out0, out0); + val += __lasx_xvpickve2gr_w(out0, 0); + } + for (; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> sh, max); + filter += filterSize; + } + } else { + for (i = 0; i < dstW; i++) { + int val = 0; + const uint16_t *srcPos = src + filterPos[i]; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> sh, max); + filter += filterSize; + } + } +} + +void ff_hscale_16_to_19_lasx(SwsContext *c, int16_t *_dst, int dstW, + const uint8_t *_src, const int16_t *filter, + const int32_t *filterPos, int filterSize) +{ + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(c->srcFormat); + int i; + int32_t *dst = (int32_t *) _dst; + const uint16_t *src = (const uint16_t *) _src; + int sh = desc->comp[0].depth - 5; + int max = (1 << 19) - 1; + int len = dstW >> 2; + int res = dstW & 3; + __m256i shift; + __m256i zero = __lasx_xvldi(0); + + if ((isAnyRGB(c->srcFormat) || c->srcFormat == AV_PIX_FMT_PAL8) + && desc->comp[0].depth<16) { + sh = 9; + } else if (desc->flags & AV_PIX_FMT_FLAG_FLOAT) { + sh = 11; + } + shift = __lasx_xvreplgr2vr_w(sh); + + if (filterSize == 8) { + for (i = 0; i < len; i++) { + SCALE_8 + } + for (i = 0; i < res; i++) { + int val = 0; + __m256i src0, filter0, out0; + + src0 = LASX_LD(src + filterPos[i]); + filter0 = LASX_LD(filter); + LASX_DP2_W_HU_H(src0, filter0, out0); + out0 = __lasx_xvhaddw_d_w(out0, out0); + out0 = __lasx_xvhaddw_q_d(out0, out0); + val = __lasx_xvpickve2gr_w(out0, 0); + dst[i] = FFMIN(val >> sh, max); + filter += 8; + } + } else if (filterSize == 4) { + for (i = 0; i < len; i++) { + __m256i src1, src2, src3, src4, src0, filter0, out0; + + src1 = __lasx_xvldrepl_d(src + filterPos[0], 0); + src2 = __lasx_xvldrepl_d(src + filterPos[1], 0); + src3 = __lasx_xvldrepl_d(src + filterPos[2], 0); + src4 = __lasx_xvldrepl_d(src + filterPos[3], 0); + filter0 = LASX_LD(filter); + src1 = __lasx_xvextrins_d(src1, src2, 0x10); + src3 = __lasx_xvextrins_d(src3, src4, 0x10); + src0 = __lasx_xvpermi_q(src1, src3, 0x02); + LASX_DP2_W_HU_H(src0, filter0, out0); + out0 = __lasx_xvhaddw_d_w(out0, out0); + out0 = __lasx_xvsra_w(out0, shift); + dst[0] = FFMIN((__lasx_xvpickve2gr_w(out0, 0)), max); + dst[1] = FFMIN((__lasx_xvpickve2gr_w(out0, 2)), max); + dst[2] = FFMIN((__lasx_xvpickve2gr_w(out0, 4)), max); + dst[3] = FFMIN((__lasx_xvpickve2gr_w(out0, 6)), max); + dst += 4; + filterPos += 4; + filter += 16; + } + for (i = 0; i < res; i++) { + int val = 0; + const uint16_t *srcPos = src + filterPos[i]; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> sh, max); + filter += 4; + } + } else if (filterSize > 8) { + int filterlen = filterSize - 7; + + for (i = 0; i < len; i ++) { + __m256i src0, src1, src2, src3; + __m256i filter0, filter1, filter2, filter3, out0, out1; + __m256i out = zero; + const uint16_t *srcPos1 = src + filterPos[0]; + const uint16_t *srcPos2 = src + filterPos[1]; + const uint16_t *srcPos3 = src + filterPos[2]; + const uint16_t *srcPos4 = src + filterPos[3]; + const int16_t *filterStart1 = filter; + const int16_t *filterStart2 = filterStart1 + filterSize; + const int16_t *filterStart3 = filterStart2 + filterSize; + const int16_t *filterStart4 = filterStart3 + filterSize; + int j, val1 = 0, val2 = 0, val3 = 0, val4 = 0; + + for (j = 0; j < filterlen; j += 8) { + SCALE_16 + } + val1 = __lasx_xvpickve2gr_w(out, 0); + val2 = __lasx_xvpickve2gr_w(out, 4); + val3 = __lasx_xvpickve2gr_w(out, 2); + val4 = __lasx_xvpickve2gr_w(out, 6); + for (; j < filterSize; j++) { + val1 += ((int)srcPos1[j]) * filterStart1[j]; + val2 += ((int)srcPos2[j]) * filterStart2[j]; + val3 += ((int)srcPos3[j]) * filterStart3[j]; + val4 += ((int)srcPos4[j]) * filterStart4[j]; + } + dst[0] = FFMIN(val1 >> sh, max); + dst[1] = FFMIN(val2 >> sh, max); + dst[2] = FFMIN(val3 >> sh, max); + dst[3] = FFMIN(val4 >> sh, max); + dst += 4; + filterPos += 4; + filter = filterStart4 + filterSize; + } + for (i = 0; i < res; i++) { + int j, val = 0; + const uint16_t *srcPos = src + filterPos[i]; + __m256i src0, filter0, out0; + + for (j = 0; j < filterlen; j += 8) { + src0 = LASX_LD(srcPos + j); + filter0 = LASX_LD(filter + j); + LASX_DP2_W_HU_H(src0, filter0, out0); + out0 = __lasx_xvhaddw_d_w(out0, out0); + out0 = __lasx_xvhaddw_q_d(out0, out0); + val += __lasx_xvpickve2gr_w(out0, 0); + } + for (; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> sh, max); + filter += filterSize; + } + } else { + for (i = 0; i < dstW; i++) { + int val = 0; + const uint16_t *srcPos = src + filterPos[i]; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filter[j]; + } + dst[i] = FFMIN(val >> sh, max); + filter += filterSize; + } + } +} + +#undef SCALE_8 +#undef SCALE_16 diff --git a/libswscale/loongarch/swscale_loongarch.h b/libswscale/loongarch/swscale_loongarch.h new file mode 100644 index 0000000000..cb5e0452bc --- /dev/null +++ b/libswscale/loongarch/swscale_loongarch.h @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2021 Loongson Technology Corporation Limited + * Contributed by Hao Chen(chenhao@loongson.cn) + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SWSCALE_LOONGARCH_SWSCALE_LOONGARCH_H +#define SWSCALE_LOONGARCH_SWSCALE_LOONGARCH_H + +#include "libswscale/swscale.h" +#include "libswscale/swscale_internal.h" + +void ff_hscale_8_to_15_lasx(SwsContext *c, int16_t *dst, int dstW, + const uint8_t *src, const int16_t *filter, + const int32_t *filterPos, int filterSize); + +void ff_hscale_8_to_19_lasx(SwsContext *c, int16_t *_dst, int dstW, + const uint8_t *src, const int16_t *filter, + const int32_t *filterPos, int filterSize); + +void ff_hscale_16_to_19_lasx(SwsContext *c, int16_t *_dst, int dstW, + const uint8_t *_src, const int16_t *filter, + const int32_t *filterPos, int filterSize); + +void ff_hscale_16_to_15_lasx(SwsContext *c, int16_t *dst, int dstW, + const uint8_t *_src, const int16_t *filter, + const int32_t *filterPos, int filterSize); + +void ff_yuv2planeX_8_lasx(const int16_t *filter, int filterSize, + const int16_t **src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +int yuv420_rgb24_lasx(SwsContext *c, const uint8_t *src[], int srcStride[], + int srcSliceY, int srcSliceH, uint8_t *dst[], int dstStride[]); + +int yuv420_bgr24_lasx(SwsContext *c, const uint8_t *src[], int srcStride[], + int srcSliceY, int srcSliceH, uint8_t *dst[], int dstStride[]); + +int yuv420_rgba32_lasx(SwsContext *c, const uint8_t *src[], int srcStride[], + int srcSliceY, int srcSliceH, uint8_t *dst[], int dstStride[]); + +int yuv420_bgra32_lasx(SwsContext *c, const uint8_t *src[], int srcStride[], + int srcSliceY, int srcSliceH, uint8_t *dst[], int dstStride[]); + +int yuv420_argb32_lasx(SwsContext *c, const uint8_t *src[], int srcStride[], + int srcSliceY, int srcSliceH, uint8_t *dst[], int dstStride[]); + +int yuv420_abgr32_lasx(SwsContext *c, const uint8_t *src[], int srcStride[], + int srcSliceY, int srcSliceH, uint8_t *dst[], int dstStride[]); + +void ff_interleave_bytes_lasx(const uint8_t *src1, const uint8_t *src2, + uint8_t *dest, int width, int height, + int src1Stride, int src2Stride, int dstStride); + +av_cold void ff_sws_init_output_loongarch(SwsContext *c); + +void planar_rgb_to_uv_lasx(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[4], + int width, int32_t *rgb2yuv); + +void planar_rgb_to_y_lasx(uint8_t *_dst, const uint8_t *src[4], int width, + int32_t *rgb2yuv); + +#endif /* SWSCALE_LOONGARCH_SWSCALE_LOONGARCH_H */ diff --git a/libswscale/loongarch/yuv2rgb_lasx.c b/libswscale/loongarch/yuv2rgb_lasx.c new file mode 100644 index 0000000000..be8aca4c66 --- /dev/null +++ b/libswscale/loongarch/yuv2rgb_lasx.c @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2021 Loongson Technology Co. Ltd. + * Contributed by Hao Chen(chenhao@loongson.cn) + * All rights reserved. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "swscale_loongarch.h" +#include "libavutil/loongarch/generic_macros_lasx.h" + +#define YUV2RGB_LOAD_COE \ + /* Load x_offset */ \ + __m256i y_offset = __lasx_xvreplgr2vr_d(c->yOffset); \ + __m256i u_offset = __lasx_xvreplgr2vr_d(c->uOffset); \ + __m256i v_offset = __lasx_xvreplgr2vr_d(c->vOffset); \ + /* Load x_coeff */ \ + __m256i ug_coeff = __lasx_xvreplgr2vr_d(c->ugCoeff); \ + __m256i vg_coeff = __lasx_xvreplgr2vr_d(c->vgCoeff); \ + __m256i y_coeff = __lasx_xvreplgr2vr_d(c->yCoeff); \ + __m256i ub_coeff = __lasx_xvreplgr2vr_d(c->ubCoeff); \ + __m256i vr_coeff = __lasx_xvreplgr2vr_d(c->vrCoeff); \ + +#define LOAD_YUV_16 \ + m_y = LASX_LD(py + (w << 4)); \ + m_u = __lasx_xvldrepl_d(pu + (w << 3), 0); \ + m_v = __lasx_xvldrepl_d(pv + (w << 3), 0); \ + LASX_UNPCK_L_HU_BU(m_y, m_y); \ + LASX_UNPCK_L_HU_BU_2(m_u, m_v, m_u, m_v); \ + +/* YUV2RGB method + * The conversion method is as follows: + * R = Y' * y_coeff + V' * vr_coeff + * G = Y' * y_coeff + V' * vg_coeff + U' * ug_coeff + * B = Y' * y_coeff + U' * ub_coeff + * + * where X' = X * 8 - x_offset + * + */ + +#define YUV2RGB \ + m_y = __lasx_xvslli_h(m_y, 3); \ + m_u = __lasx_xvslli_h(m_u, 3); \ + m_v = __lasx_xvslli_h(m_v, 3); \ + m_y = __lasx_xvsub_h(m_y, y_offset); \ + m_u = __lasx_xvsub_h(m_u, u_offset); \ + m_v = __lasx_xvsub_h(m_v, v_offset); \ + m_u = __lasx_xvshuf_h(shuf1, m_u, m_u); \ + m_v = __lasx_xvshuf_h(shuf1, m_v, m_v); \ + y_1 = __lasx_xvmuh_h(m_y, y_coeff); \ + u2g = __lasx_xvmuh_h(m_u, ug_coeff); \ + u2b = __lasx_xvmuh_h(m_u, ub_coeff); \ + v2r = __lasx_xvmuh_h(m_v, vr_coeff); \ + v2g = __lasx_xvmuh_h(m_v, vg_coeff); \ + r = __lasx_xvsadd_h(y_1, v2r); \ + v2g = __lasx_xvsadd_h(v2g, u2g); \ + g = __lasx_xvsadd_h(v2g, y_1); \ + b = __lasx_xvsadd_h(y_1, u2b); \ + LASX_CLIP_H_0_255_2(r, g, r, g); \ + LASX_CLIP_H_0_255(b, b); \ + +#define RGB_PACK_16(r, g, b, rgb_l, rgb_h) \ +{ \ + __m256i rg; \ + rg = __lasx_xvpackev_b(g, r); \ + LASX_SHUF_B_2_128SV(b, rg, b, rg, shuf2, shuf3, rgb_l, rgb_h); \ +} + +#define RGB_PACK_32(r, g, b, a, rgb_l, rgb_h) \ +{ \ + __m256i rg, ba; \ + rgb_l = __lasx_xvpackev_b(g, r); \ + rgb_h = __lasx_xvpackev_b(a, b); \ + LASX_ILVL_H_128SV(rgb_h, rgb_l, rg); \ + LASX_ILVH_H_128SV(rgb_h, rgb_l, ba); \ + rgb_l = __lasx_xvpermi_q(ba, rg, 0x20); \ + rgb_h = __lasx_xvpermi_q(ba, rg, 0x31); \ +} + +#define RGB_STORE_32(rgb_l, rgb_h, iamge, w) \ +{ \ + uint8_t *index = image + (w * 64); \ + LASX_ST_2(rgb_l, rgb_h, index, 32); \ +} + +#define RGB_STORE(rgb_l, rgb_h, image, w) \ +{ \ + uint8_t *index = image + (w * 48); \ + __lasx_xvstelm_d(rgb_l, (index), 0, 0); \ + __lasx_xvstelm_d(rgb_l, (index), 8, 1); \ + __lasx_xvstelm_d(rgb_h, (index), 16, 0); \ + __lasx_xvstelm_d(rgb_l, (index), 24, 2); \ + __lasx_xvstelm_d(rgb_l, (index), 32, 3); \ + __lasx_xvstelm_d(rgb_h, (index), 40, 2); \ +} + +#define YUV2RGBFUNC(func_name, dst_type, alpha) \ + int func_name(SwsContext *c, const uint8_t *src[], \ + int srcStride[], int srcSliceY, int srcSliceH, \ + uint8_t *dst[], int dstStride[]) \ +{ \ + int w, y, h_size, vshift; \ + __m256i m_y, m_u, m_v; \ + __m256i y_1, u2g, v2g, u2b, v2r, rgb_l, rgb_h; \ + __m256i r, g, b; \ + __m256i shuf2 = {0x0504120302100100, 0x0A18090816070614, \ + 0x0504120302100100, 0x0A18090816070614}; \ + __m256i shuf3 = {0x1E0F0E1C0D0C1A0B, 0x0101010101010101, \ + 0x1E0F0E1C0D0C1A0B, 0x0101010101010101}; \ + __m256i shuf1 = {0x0001000100000000, 0x0003000300020002, \ + 0x0005000500040004, 0x0007000700060006}; \ + YUV2RGB_LOAD_COE \ + \ + h_size = c->dstW >> 4; \ + vshift = c->srcFormat != AV_PIX_FMT_YUV422P; \ + for (y = 0; y < srcSliceH; y++) { \ + dst_type *image = dst[0] + (y + srcSliceY) * dstStride[0]; \ + const uint8_t *py = src[0] + y * srcStride[0]; \ + const uint8_t *pu = src[1] + (y >> vshift) * srcStride[1]; \ + const uint8_t *pv = src[2] + (y >> vshift) * srcStride[2]; \ + for (w = 0; w < h_size; w ++) { \ + + +#define END_FUNC() \ + } \ + } \ + return srcSliceH; \ +} + + +#define YUV2RGBFUNC_32(func_name, dst_type, alpha) \ + int func_name(SwsContext *c, const uint8_t *src[], \ + int srcStride[], int srcSliceY, int srcSliceH, \ + uint8_t *dst[], int dstStride[]) \ +{ \ + int w, y, h_size, vshift, a = -1; \ + __m256i m_y, m_u, m_v; \ + __m256i y_1, u2g, v2g, u2b, v2r, rgb_l, rgb_h; \ + __m256i r, g, b; \ + __m256i alp = __lasx_xvreplgr2vr_w(a); \ + __m256i shuf1 = {0x0001000100000000, 0x0003000300020002, \ + 0x0005000500040004, 0x0007000700060006}; \ + YUV2RGB_LOAD_COE \ + \ + h_size = c->dstW >> 4; \ + vshift = c->srcFormat != AV_PIX_FMT_YUV422P; \ + for (y = 0; y < srcSliceH; y++) { \ + dst_type *image = dst[0] + (y + srcSliceY) * dstStride[0]; \ + const uint8_t *py = src[0] + y * srcStride[0]; \ + const uint8_t *pu = src[1] + (y >> vshift) * srcStride[1]; \ + const uint8_t *pv = src[2] + (y >> vshift) * srcStride[2]; \ + for (w = 0; w < h_size; w ++) { \ + + +YUV2RGBFUNC(yuv420_rgb24_lasx, uint8_t, 0) + LOAD_YUV_16 + YUV2RGB + RGB_PACK_16(r, g, b, rgb_l, rgb_h); + RGB_STORE(rgb_l, rgb_h, image, w); + END_FUNC() + +YUV2RGBFUNC(yuv420_bgr24_lasx, uint8_t, 0) + LOAD_YUV_16 + YUV2RGB + RGB_PACK_16(b, g, r, rgb_l, rgb_h); + RGB_STORE(rgb_l, rgb_h, image, w); + END_FUNC() + +YUV2RGBFUNC_32(yuv420_rgba32_lasx, uint8_t, 0) + LOAD_YUV_16 + YUV2RGB + RGB_PACK_32(r, g, b, alp, rgb_l, rgb_h); + RGB_STORE_32(rgb_l, rgb_h, image, w); + END_FUNC() + +YUV2RGBFUNC_32(yuv420_bgra32_lasx, uint8_t, 0) + LOAD_YUV_16 + YUV2RGB + RGB_PACK_32(b, g, r, alp, rgb_l, rgb_h); + RGB_STORE_32(rgb_l, rgb_h, image, w); + END_FUNC() + +YUV2RGBFUNC_32(yuv420_argb32_lasx, uint8_t, 0) + LOAD_YUV_16 + YUV2RGB + RGB_PACK_32(alp, r, g, b, rgb_l, rgb_h); + RGB_STORE_32(rgb_l, rgb_h, image, w); + END_FUNC() + +YUV2RGBFUNC_32(yuv420_abgr32_lasx, uint8_t, 0) + LOAD_YUV_16 + YUV2RGB + RGB_PACK_32(alp, b, g, r, rgb_l, rgb_h); + RGB_STORE_32(rgb_l, rgb_h, image, w); + END_FUNC() diff --git a/libswscale/mips/Makefile b/libswscale/mips/Makefile new file mode 100644 index 0000000000..48c2425eff --- /dev/null +++ b/libswscale/mips/Makefile @@ -0,0 +1,4 @@ +OBJS += mips/swscale_init_mips.o +OBJS += mips/rgb2rgb_init_mips.o +MSA-OBJS += mips/swscale_msa.o +MSA-OBJS += mips/rgb2rgb_msa.o diff --git a/libswscale/mips/rgb2rgb_init_mips.c b/libswscale/mips/rgb2rgb_init_mips.c new file mode 100644 index 0000000000..f1bcd6f726 --- /dev/null +++ b/libswscale/mips/rgb2rgb_init_mips.c @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2020 Loongson Technology Co. Ltd. + * Contributed by Gu Xiwei(guxiwei-hf@loongson.cn) + * All rights reserved. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "rgb2rgb_mips.h" +#include "libavutil/mips/cpu.h" + +av_cold void rgb2rgb_init_mips(void) +{ + int cpu_flags = av_get_cpu_flags(); +#if HAVE_MSA + if (have_msa(cpu_flags)) + interleaveBytes = ff_interleave_bytes_msa; +#endif /* #if HAVE_MSA */ +} diff --git a/libswscale/mips/rgb2rgb_mips.h b/libswscale/mips/rgb2rgb_mips.h new file mode 100644 index 0000000000..5271d2f032 --- /dev/null +++ b/libswscale/mips/rgb2rgb_mips.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2020 Loongson Technology Co. Ltd. + * Contributed by Gu Xiwei(guxiwei-hf@loongson.cn) + * All rights reserved. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SWSCALE_MIPS_RGB2RGB_MIPS_H +#define SWSCALE_MIPS_RGB2RGB_MIPS_H + +#include "libswscale/rgb2rgb.h" + +void ff_interleave_bytes_msa(const uint8_t *src1, const uint8_t *src2, + uint8_t *dest, int width, int height, + int src1Stride, int src2Stride, int dstStride); + +#endif /* SWSCALE_MIPS_RGB2RGB_MIPS_H */ diff --git a/libswscale/mips/rgb2rgb_msa.c b/libswscale/mips/rgb2rgb_msa.c new file mode 100644 index 0000000000..594a34c651 --- /dev/null +++ b/libswscale/mips/rgb2rgb_msa.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2020 Loongson Technology Co. Ltd. + * Contributed by Gu Xiwei(guxiwei-hf@loongson.cn) + * All rights reserved. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "rgb2rgb_mips.h" +#include "libavutil/mips/generic_macros_msa.h" + +void ff_interleave_bytes_msa(const uint8_t *src1, const uint8_t *src2, + uint8_t *dest, int width, int height, + int src1Stride, int src2Stride, int dstStride) +{ + int h; + int len = width >> 4; + int part = len << 4; + for (h = 0; h < height; h++) { + int w; + v16u8 src_0, src_1; + v16u8 dst_0, dst_1; + for (w = 0; w < len; w++) { + src_0 = LD_UB(src1 + w * 16); + src_1 = LD_UB(src2 + w * 16); + ILVRL_B2_UB(src_1, src_0, dst_0, dst_1); + ST_UB2(dst_0, dst_1, dest + w * 32, 16); + } + for (w = part; w < width; w++) { + dest[2 * w + 0] = src1[w]; + dest[2 * w + 1] = src2[w]; + } + dest += dstStride; + src1 += src1Stride; + src2 += src2Stride; + } +} diff --git a/libswscale/mips/swscale_init_mips.c b/libswscale/mips/swscale_init_mips.c new file mode 100644 index 0000000000..8e26486f1e --- /dev/null +++ b/libswscale/mips/swscale_init_mips.c @@ -0,0 +1,211 @@ +/* + * Copyright (C) 2020 Loongson Technology Co. Ltd. + * Contributed by Gu Xiwei(guxiwei-hf@loongson.cn) + * All rights reserved. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "swscale_mips.h" +#include "libavutil/mips/cpu.h" + +av_cold void ff_sws_init_swscale_mips(SwsContext *c) +{ + int cpu_flags = av_get_cpu_flags(); +#if HAVE_MSA + if (have_msa(cpu_flags)) { + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(c->dstFormat); + + if (c->srcBpc == 8) { + if (c->dstBpc <= 14) { + c->hyScale = c->hcScale = ff_hscale_8_to_15_msa; + } else { + c->hyScale = c->hcScale = ff_hscale_8_to_19_msa; + } + } else { + c->hyScale = c->hcScale = c->dstBpc > 14 ? ff_hscale_16_to_19_msa + : ff_hscale_16_to_15_msa; + } + switch (c->srcFormat) { + case AV_PIX_FMT_GBRAP: + case AV_PIX_FMT_GBRP: + { + c->readChrPlanar = planar_rgb_to_uv_msa; + c->readLumPlanar = planar_rgb_to_y_msa; + } + break; + } + if (c->dstBpc == 8) + c->yuv2planeX = ff_yuv2planeX_8_msa; + if (c->flags & SWS_FULL_CHR_H_INT) { + switch (c->dstFormat) { + case AV_PIX_FMT_RGBA: +#if CONFIG_SMALL + c->yuv2packedX = yuv2rgba32_full_X_msa; + c->yuv2packed2 = yuv2rgba32_full_2_msa; + c->yuv2packed1 = yuv2rgba32_full_1_msa; +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + c->yuv2packedX = yuv2rgba32_full_X_msa; + c->yuv2packed2 = yuv2rgba32_full_2_msa; + c->yuv2packed1 = yuv2rgba32_full_1_msa; + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packedX = yuv2rgbx32_full_X_msa; + c->yuv2packed2 = yuv2rgbx32_full_2_msa; + c->yuv2packed1 = yuv2rgbx32_full_1_msa; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_ARGB: +#if CONFIG_SMALL + c->yuv2packedX = yuv2argb32_full_X_msa; + c->yuv2packed2 = yuv2argb32_full_2_msa; + c->yuv2packed1 = yuv2argb32_full_1_msa; +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + c->yuv2packedX = yuv2argb32_full_X_msa; + c->yuv2packed2 = yuv2argb32_full_2_msa; + c->yuv2packed1 = yuv2argb32_full_1_msa; + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packedX = yuv2xrgb32_full_X_msa; + c->yuv2packed2 = yuv2xrgb32_full_2_msa; + c->yuv2packed1 = yuv2xrgb32_full_1_msa; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_BGRA: +#if CONFIG_SMALL + c->yuv2packedX = yuv2bgra32_full_X_msa; + c->yuv2packed2 = yuv2bgra32_full_2_msa; + c->yuv2packed1 = yuv2bgra32_full_1_msa; +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + c->yuv2packedX = yuv2bgra32_full_X_msa; + c->yuv2packed2 = yuv2bgra32_full_2_msa; + c->yuv2packed1 = yuv2bgra32_full_1_msa; + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packedX = yuv2bgrx32_full_X_msa; + c->yuv2packed2 = yuv2bgrx32_full_2_msa; + c->yuv2packed1 = yuv2bgrx32_full_1_msa; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_ABGR: +#if CONFIG_SMALL + c->yuv2packedX = yuv2abgr32_full_X_msa; + c->yuv2packed2 = yuv2abgr32_full_2_msa; + c->yuv2packed1 = yuv2abgr32_full_1_msa; +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + c->yuv2packedX = yuv2abgr32_full_X_msa; + c->yuv2packed2 = yuv2abgr32_full_2_msa; + c->yuv2packed1 = yuv2abgr32_full_1_msa; + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packedX = yuv2xbgr32_full_X_msa; + c->yuv2packed2 = yuv2xbgr32_full_2_msa; + c->yuv2packed1 = yuv2xbgr32_full_1_msa; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_RGB24: + c->yuv2packedX = yuv2rgb24_full_X_msa; + c->yuv2packed2 = yuv2rgb24_full_2_msa; + c->yuv2packed1 = yuv2rgb24_full_1_msa; + break; + case AV_PIX_FMT_BGR24: + c->yuv2packedX = yuv2bgr24_full_X_msa; + c->yuv2packed2 = yuv2bgr24_full_2_msa; + c->yuv2packed1 = yuv2bgr24_full_1_msa; + break; + case AV_PIX_FMT_BGR4_BYTE: + c->yuv2packedX = yuv2bgr4_byte_full_X_msa; + c->yuv2packed2 = yuv2bgr4_byte_full_2_msa; + c->yuv2packed1 = yuv2bgr4_byte_full_1_msa; + break; + case AV_PIX_FMT_RGB4_BYTE: + c->yuv2packedX = yuv2rgb4_byte_full_X_msa; + c->yuv2packed2 = yuv2rgb4_byte_full_2_msa; + c->yuv2packed1 = yuv2rgb4_byte_full_1_msa; + break; + case AV_PIX_FMT_BGR8: + c->yuv2packedX = yuv2bgr8_full_X_msa; + c->yuv2packed2 = yuv2bgr8_full_2_msa; + c->yuv2packed1 = yuv2bgr8_full_1_msa; + break; + case AV_PIX_FMT_RGB8: + c->yuv2packedX = yuv2rgb8_full_X_msa; + c->yuv2packed2 = yuv2rgb8_full_2_msa; + c->yuv2packed1 = yuv2rgb8_full_1_msa; + break; + } + } else { + switch (c->dstFormat) { + case AV_PIX_FMT_RGB32: + case AV_PIX_FMT_BGR32: +#if CONFIG_SMALL +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packed1 = yuv2rgbx32_1_msa; + c->yuv2packed2 = yuv2rgbx32_2_msa; + c->yuv2packedX = yuv2rgbx32_X_msa; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_RGB32_1: + case AV_PIX_FMT_BGR32_1: +#if CONFIG_SMALL +#else +#if CONFIG_SWSCALE_ALPHA + if (c->needAlpha) { + } else +#endif /* CONFIG_SWSCALE_ALPHA */ + { + c->yuv2packed1 = yuv2rgbx32_1_1_msa; + c->yuv2packed2 = yuv2rgbx32_1_2_msa; + c->yuv2packedX = yuv2rgbx32_1_X_msa; + } +#endif /* !CONFIG_SMALL */ + break; + case AV_PIX_FMT_RGB565LE: + case AV_PIX_FMT_RGB565BE: + case AV_PIX_FMT_BGR565LE: + case AV_PIX_FMT_BGR565BE: + c->yuv2packed1 = yuv2rgb16_1_msa; + c->yuv2packed2 = yuv2rgb16_2_msa; + c->yuv2packedX = yuv2rgb16_X_msa; + break; + } + } + } +#endif /* #if HAVE_MSA */ +} diff --git a/libswscale/mips/swscale_mips.h b/libswscale/mips/swscale_mips.h new file mode 100644 index 0000000000..ff30593e0d --- /dev/null +++ b/libswscale/mips/swscale_mips.h @@ -0,0 +1,404 @@ +/* + * Copyright (C) 2020 Loongson Technology Co. Ltd. + * Contributed by Gu Xiwei(guxiwei-hf@loongson.cn) + * All rights reserved. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SWSCALE_MIPS_SWSCALE_MIPS_H +#define SWSCALE_MIPS_SWSCALE_MIPS_H + +#include "libswscale/swscale.h" +#include "libswscale/swscale_internal.h" + +void ff_hscale_8_to_15_msa(SwsContext *c, int16_t *dst, int dstW, + const uint8_t *src, const int16_t *filter, + const int32_t *filterPos, int filterSize); + +void ff_hscale_8_to_19_msa(SwsContext *c, int16_t *_dst, int dstW, + const uint8_t *src, const int16_t *filter, + const int32_t *filterPos, int filterSize); + +void ff_hscale_16_to_19_msa(SwsContext *c, int16_t *_dst, int dstW, + const uint8_t *_src, const int16_t *filter, + const int32_t *filterPos, int filterSize); + +void ff_hscale_16_to_15_msa(SwsContext *c, int16_t *dst, int dstW, + const uint8_t *_src, const int16_t *filter, + const int32_t *filterPos, int filterSize); + +void ff_yuv2planeX_8_msa(const int16_t *filter, int filterSize, + const int16_t **src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2rgbx32_1_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2rgbx32_1_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2rgbx32_1_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2rgbx32_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2rgbx32_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2rgbx32_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2rgb16_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2rgb16_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2rgb16_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2bgra32_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2bgra32_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2bgra32_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2abgr32_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2abgr32_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2abgr32_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2rgba32_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2rgba32_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2rgba32_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2argb32_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2argb32_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2argb32_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2bgrx32_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2bgrx32_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2bgrx32_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2xbgr32_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2xbgr32_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2xbgr32_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2rgbx32_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2rgbx32_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2rgbx32_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2xrgb32_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2xrgb32_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2xrgb32_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2bgr24_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2bgr24_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2bgr24_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2rgb24_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2rgb24_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2rgb24_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2bgr4_byte_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2bgr4_byte_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2bgr4_byte_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2rgb4_byte_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2rgb4_byte_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2rgb4_byte_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2bgr8_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2bgr8_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2bgr8_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2rgb8_full_2_msa(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y); + +void yuv2rgb8_full_1_msa(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y); + +void yuv2rgb8_full_X_msa(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y); + +void yuv2plane1_9BE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2plane1_9LE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2planeX_9BE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void yuv2planeX_9LE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void yuv2plane1_10BE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2plane1_10LE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2planeX_10BE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void yuv2planeX_10LE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void yuv2plane1_12BE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2plane1_12LE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2planeX_12BE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void yuv2planeX_12LE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void yuv2plane1_14BE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2plane1_14LE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2planeX_14BE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void yuv2planeX_14LE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void yuv2plane1_16BE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2plane1_16LE_msa(const int16_t *src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset); + +void yuv2planeX_16BE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void yuv2planeX_16LE_msa(const int16_t *filter, int filterSize, const int16_t **src, + uint8_t *dest, int dstW, const uint8_t *ditherm, int offset); + +void planar_rgb_to_uv_msa(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[4], + int width, int32_t *rgb2yuv); + +void planar_rgb_to_y_msa(uint8_t *_dst, const uint8_t *src[4], int width, + int32_t *rgb2yuv); + +#endif /* SWSCALE_MIPS_SWSCALE_MIPS_H */ diff --git a/libswscale/mips/swscale_msa.c b/libswscale/mips/swscale_msa.c new file mode 100644 index 0000000000..4f6422c4cc --- /dev/null +++ b/libswscale/mips/swscale_msa.c @@ -0,0 +1,1805 @@ +/* + * Copyright (C) 2020 Loongson Technology Co. Ltd. + * Contributed by Gu Xiwei(guxiwei-hf@loongson.cn) + * All rights reserved. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "swscale_mips.h" +#include "libavutil/mips/generic_macros_msa.h" +#include "libavutil/intreadwrite.h" + +void ff_hscale_8_to_15_msa(SwsContext *c, int16_t *dst, int dstW, + const uint8_t *src, const int16_t *filter, + const int32_t *filterPos, int filterSize) +{ + int i; + + if (filterSize == 8) { + for (i = 0; i < dstW; i++) { + int val = 0; + v8i16 src0, filter0, out0; + v16i8 zero = { 0 }; + + src0 = LD_V(v8i16, src + filterPos[i]); + filter0 = LD_V(v8i16, filter + (i << 3)); + src0 = (v8i16)__msa_ilvr_b(zero, (v16i8)src0); + out0 = (v8i16)__msa_dotp_s_w(src0, filter0); + out0 = (v8i16)__msa_hadd_s_d((v4i32)out0, (v4i32)out0); + val += (__msa_copy_s_w((v4i32)out0, 0) + + __msa_copy_s_w((v4i32)out0, 2)); + dst[i] = FFMIN(val >> 7, (1 << 15) - 1); + } + } else if (filterSize == 4) { + int len = dstW & (~1); + + for (i = 0; i < len; i += 2) { + v8i16 src1, src2, src3; + v8i16 filter0; + v4i32 out0; + int val1 = 0; + int val2 = 0; + v16i8 zero = {0}; + + src1 = LD_V(v8i16, src + filterPos[i]); + src2 = LD_V(v8i16, src + filterPos[i + 1]); + filter0 = LD_V(v8i16, filter + (i << 2)); + src1 = (v8i16)__msa_ilvr_b(zero, (v16i8)src1); + src2 = (v8i16)__msa_ilvr_b(zero, (v16i8)src2); + src3 = (v8i16)__msa_ilvr_d((v2i64)src2, (v2i64)src1); + out0 = (v4i32)__msa_dotp_s_w(src3, filter0); + val1 = __msa_copy_s_w(out0, 0) + __msa_copy_s_w(out0, 1); + val2 = __msa_copy_s_w(out0, 2) + __msa_copy_s_w(out0, 3); + dst[i] = FFMIN(val1 >> 7, (1 << 15) - 1); + dst[i + 1] = FFMIN(val2 >> 7, (1 << 15) - 1); + } + if (i < dstW) { + int val = 0; + uint8_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + + for (int j = 0; j < 4; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> 7, (1 << 15) - 1); + } + } else if (filterSize > 8) { + int len = filterSize >> 3; + int part = len << 3; + + for (i = 0; i < dstW; i++) { + v8i16 src0, filter0, out0; + v16i8 zero = { 0 }; + uint8_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + int j, val = 0; + + for (j = 0; j < len; j++) { + src0 = LD_V(v8i16, srcPos + (j << 3)); + filter0 = LD_V(v8i16, filterStart + (j << 3)); + src0 = (v8i16)__msa_ilvr_b(zero, (v16i8)src0); + out0 = (v8i16)__msa_dotp_s_w(src0, filter0); + out0 = (v8i16)__msa_hadd_s_d((v4i32)out0, (v4i32)out0); + val += (__msa_copy_s_w((v4i32)out0, 0) + + __msa_copy_s_w((v4i32)out0, 2)); + } + for (j = part; j < filterSize; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> 7, (1 << 15) - 1); + } + } else { + for (i = 0; i < dstW; i++) { + int val = 0; + uint8_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> 7, (1 << 15) - 1); + } + } +} + +void ff_hscale_8_to_19_msa(SwsContext *c, int16_t *_dst, int dstW, + const uint8_t *src, const int16_t *filter, + const int32_t *filterPos, int filterSize) +{ + int i; + int32_t *dst = (int32_t *) _dst; + + if (filterSize == 8) { + for (i = 0; i < dstW; i++) { + int val = 0; + v8i16 src0, filter0, out0; + v16i8 zero = { 0 }; + + src0 = LD_V(v8i16, src + filterPos[i]); + filter0 = LD_V(v8i16, filter + (i << 3)); + src0 = (v8i16)__msa_ilvr_b(zero, (v16i8)src0); + out0 = (v8i16)__msa_dotp_s_w(src0, filter0); + out0 = (v8i16)__msa_hadd_s_d((v4i32)out0, (v4i32)out0); + val += (__msa_copy_s_w((v4i32)out0, 0) + + __msa_copy_s_w((v4i32)out0, 2)); + dst[i] = FFMIN(val >> 3, (1 << 19) - 1); + } + } else if (filterSize == 4) { + int len = dstW & (~1); + + for (i = 0; i < len; i += 2) { + v8i16 src1, src2, src3; + v8i16 filter0; + v4i32 out0; + int val1 = 0; + int val2 = 0; + v16i8 zero = {0}; + + src1 = LD_V(v8i16, src + filterPos[i]); + src2 = LD_V(v8i16, src + filterPos[i + 1]); + filter0 = LD_V(v8i16, filter + (i << 2)); + src1 = (v8i16)__msa_ilvr_b(zero, (v16i8)src1); + src2 = (v8i16)__msa_ilvr_b(zero, (v16i8)src2); + src3 = (v8i16)__msa_ilvr_d((v2i64)src2, (v2i64)src1); + out0 = (v4i32)__msa_dotp_s_w(src3, filter0); + val1 = __msa_copy_s_w(out0, 0) + __msa_copy_s_w(out0, 1); + val2 = __msa_copy_s_w(out0, 2) + __msa_copy_s_w(out0, 3); + dst[i] = FFMIN(val1 >> 3, (1 << 19) - 1); + dst[i + 1] = FFMIN(val2 >> 3, (1 << 19) - 1); + } + if (i < dstW) { + int val = 0; + uint8_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + + for (int j = 0; j < 4; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> 3, (1 << 19) - 1); + } + } else if (filterSize > 8) { + int len = filterSize >> 3; + int part = len << 3; + + for (i = 0; i < dstW; i++) { + v8i16 src0, filter0, out0; + v16i8 zero = { 0 }; + uint8_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + int j, val = 0; + + for (j = 0; j < len; j++) { + src0 = LD_V(v8i16, srcPos + (j << 3)); + filter0 = LD_V(v8i16, filterStart + (j << 3)); + src0 = (v8i16)__msa_ilvr_b(zero, (v16i8)src0); + out0 = (v8i16)__msa_dotp_s_w(src0, filter0); + out0 = (v8i16)__msa_hadd_s_d((v4i32)out0, (v4i32)out0); + val += (__msa_copy_s_w((v4i32)out0, 0) + + __msa_copy_s_w((v4i32)out0, 2)); + } + for (j = part; j < filterSize; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> 3, (1 << 19) - 1); + } + } else { + for (i = 0; i < dstW; i++) { + int val = 0; + uint8_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> 3, (1 << 19) - 1); + } + } +} + +void ff_hscale_16_to_19_msa(SwsContext *c, int16_t *_dst, int dstW, + const uint8_t *_src, const int16_t *filter, + const int32_t *filterPos, int filterSize) +{ + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(c->srcFormat); + int i; + int32_t *dst = (int32_t *) _dst; + const uint16_t *src = (const uint16_t *) _src; + int bits = desc->comp[0].depth - 1; + int sh = bits - 4; + + if ((isAnyRGB(c->srcFormat) || c->srcFormat == AV_PIX_FMT_PAL8) + && desc->comp[0].depth<16) { + sh = 9; + } else if (desc->flags & AV_PIX_FMT_FLAG_FLOAT) { + sh = 11; + } + if (filterSize == 8) { + for (i = 0; i < dstW; i++) { + int val = 0; + v8i16 src0, filter0; + v4i32 src_l, src_r, filter_l, filter_r, out_l, out_r, out; + v8i16 zero = { 0 }; + + src0 = LD_V(v8i16, src + filterPos[i]); + filter0 = LD_V(v8i16, filter + (i << 3)); + src_r = (v4i32)__msa_ilvr_h(zero, (v8i16)src0); + src_l = (v4i32)__msa_ilvl_h(zero, (v8i16)src0); + UNPCK_SH_SW(filter0, filter_r, filter_l); + out_r = (v4i32)__msa_dotp_s_d(src_r, filter_r); + out_l = (v4i32)__msa_dotp_s_d(src_l, filter_l); + out = (v4i32)__msa_addv_w(out_r, out_l); + val += (__msa_copy_s_w(out, 0) + + __msa_copy_s_w(out, 2)); + dst[i] = FFMIN(val >> sh, (1 << 19) - 1); + } + } else if (filterSize == 4) { + int len = dstW & (~1); + + for (i = 0; i < len; i += 2) { + v8i16 src1, src2, filter0; + v4i32 src1_r, src2_r, filter_r, filter_l; + v4i32 out1, out2; + int val1 = 0; + int val2 = 0; + v8i16 zero = {0}; + + src1 = LD_V(v8i16, src + filterPos[i]); + src2 = LD_V(v8i16, src + filterPos[i + 1]); + filter0 = LD_V(v8i16, filter + (i << 2)); + src1_r = (v4i32)__msa_ilvr_h(zero, src1); + src2_r = (v4i32)__msa_ilvr_h(zero, src2); + UNPCK_SH_SW(filter0, filter_r, filter_l); + out1 = (v4i32)__msa_dotp_s_d(src1_r, filter_r); + out2 = (v4i32)__msa_dotp_s_d(src2_r, filter_l); + val1 = __msa_copy_s_w(out1, 0) + __msa_copy_s_w(out1, 2); + val2 = __msa_copy_s_w(out2, 0) + __msa_copy_s_w(out2, 2); + dst[i] = FFMIN(val1 >> sh, (1 << 19) - 1); + dst[i + 1] = FFMIN(val2 >> sh, (1 << 19) - 1); + } + if (i < dstW) { + int val = 0; + uint8_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + + for (int j = 0; j < 4; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> sh, (1 << 19) - 1); + } + } else if (filterSize > 8) { + int len = filterSize >> 3; + int part = len << 3; + + for (i = 0; i < dstW; i++) { + v8i16 src0, filter0; + v4i32 src_r, src_l, filter_r, filter_l, out_r, out_l, out; + v8i16 zero = { 0 }; + uint16_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + int j, val = 0; + + for (j = 0; j < len; j++) { + src0 = LD_V(v8i16, srcPos + (j << 3)); + filter0 = LD_V(v8i16, filterStart + (j << 3)); + src_r = (v4i32)__msa_ilvr_h(zero, (v8i16)src0); + src_l = (v4i32)__msa_ilvl_h(zero, (v8i16)src0); + UNPCK_SH_SW(filter0, filter_r, filter_l); + out_r = (v4i32)__msa_dotp_s_d(src_r, filter_r); + out_l = (v4i32)__msa_dotp_s_d(src_l, filter_l); + out = (v4i32)__msa_addv_w(out_r, out_l); + val += (__msa_copy_s_w(out, 0) + + __msa_copy_s_w(out, 2)); + } + for (j = part; j < filterSize; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> sh, (1 << 19) - 1); + } + } else { + for (i = 0; i < dstW; i++) { + int val = 0; + uint16_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> sh, (1 << 19) - 1); + } + } +} + +void ff_hscale_16_to_15_msa(SwsContext *c, int16_t *dst, int dstW, + const uint8_t *_src, const int16_t *filter, + const int32_t *filterPos, int filterSize) +{ + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(c->srcFormat); + int i; + const uint16_t *src = (const uint16_t *) _src; + int sh = desc->comp[0].depth - 1; + + if (sh < 15) { + sh = isAnyRGB(c->srcFormat) || c->srcFormat==AV_PIX_FMT_PAL8 ? 13 : + (desc->comp[0].depth - 1); + } else if (desc->flags && AV_PIX_FMT_FLAG_FLOAT) { + sh = 15; + } + if (filterSize == 8) { + for (i = 0; i < dstW; i++) { + int val = 0; + v8i16 src0, filter0; + v4i32 src_l, src_r, filter_l, filter_r, out_l, out_r, out; + v8i16 zero = { 0 }; + + src0 = LD_V(v8i16, src + filterPos[i]); + filter0 = LD_V(v8i16, filter + (i << 3)); + src_r = (v4i32)__msa_ilvr_h(zero, (v8i16)src0); + src_l = (v4i32)__msa_ilvl_h(zero, (v8i16)src0); + UNPCK_SH_SW(filter0, filter_r, filter_l); + out_r = (v4i32)__msa_dotp_s_d(src_r, filter_r); + out_l = (v4i32)__msa_dotp_s_d(src_l, filter_l); + out = (v4i32)__msa_addv_w(out_r, out_l); + val += (__msa_copy_s_w(out, 0) + + __msa_copy_s_w(out, 2)); + dst[i] = FFMIN(val >> sh, (1 << 15) - 1); + } + } else if (filterSize == 4) { + int len = dstW & (~1); + + for (i = 0; i < len; i += 2) { + v8i16 src1, src2, filter0; + v4i32 src1_r, src2_r, filter_r, filter_l; + v4i32 out1, out2; + int val1 = 0; + int val2 = 0; + v8i16 zero = {0}; + + src1 = LD_V(v8i16, src + filterPos[i]); + src2 = LD_V(v8i16, src + filterPos[i + 1]); + filter0 = LD_V(v8i16, filter + (i << 2)); + src1_r = (v4i32)__msa_ilvr_h(zero, src1); + src2_r = (v4i32)__msa_ilvr_h(zero, src2); + UNPCK_SH_SW(filter0, filter_r, filter_l); + out1 = (v4i32)__msa_dotp_s_d(src1_r, filter_r); + out2 = (v4i32)__msa_dotp_s_d(src2_r, filter_l); + val1 = __msa_copy_s_w(out1, 0) + __msa_copy_s_w(out1, 2); + val2 = __msa_copy_s_w(out2, 0) + __msa_copy_s_w(out2, 2); + dst[i] = FFMIN(val1 >> sh, (1 << 15) - 1); + dst[i + 1] = FFMIN(val2 >> sh, (1 << 15) - 1); + } + if (i < dstW) { + int val = 0; + uint8_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + + for (int j = 0; j < 4; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> sh, (1 << 15) - 1); + } + } else if (filterSize > 8) { + int len = filterSize >> 3; + int part = len << 3; + + for (i = 0; i < dstW; i++) { + v8i16 src0, filter0; + v4i32 src_r, src_l, filter_r, filter_l, out_r, out_l, out; + v8i16 zero = { 0 }; + uint16_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + int j, val = 0; + + for (j = 0; j < len; j++) { + src0 = LD_V(v8i16, srcPos + (j << 3)); + filter0 = LD_V(v8i16, filterStart + (j << 3)); + src_r = (v4i32)__msa_ilvr_h(zero, (v8i16)src0); + src_l = (v4i32)__msa_ilvl_h(zero, (v8i16)src0); + UNPCK_SH_SW(filter0, filter_r, filter_l); + out_r = (v4i32)__msa_dotp_s_d(src_r, filter_r); + out_l = (v4i32)__msa_dotp_s_d(src_l, filter_l); + out = (v4i32)__msa_addv_w(out_r, out_l); + val += (__msa_copy_s_w(out, 0) + + __msa_copy_s_w(out, 2)); + } + for (j = part; j < filterSize; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> sh, (1 << 15) - 1); + } + } else { + for (i = 0; i < dstW; i++) { + int val = 0; + uint16_t *srcPos = src + filterPos[i]; + int16_t *filterStart = filter + filterSize * i; + + for (int j = 0; j < filterSize; j++) { + val += ((int)srcPos[j]) * filterStart[j]; + } + dst[i] = FFMIN(val >> sh, (1 << 15) - 1); + } + } +} +void ff_yuv2planeX_8_msa(const int16_t *filter, int filterSize, + const int16_t **src, uint8_t *dest, int dstW, + const uint8_t *dither, int offset) +{ + int i; + int len = dstW >> 3; + int part = len << 3; + for (i = 0; i < len; i++) { + int j; + v8i16 src0, flags; + v4i32 src_l, src_r, filter0; + v4i32 val_r = { dither[(i * 8 + 0 + offset) & 7] << 12, + dither[(i * 8 + 1 + offset) & 7] << 12, + dither[(i * 8 + 2 + offset) & 7] << 12, + dither[(i * 8 + 3 + offset) & 7] << 12 }; + v4i32 val_l = { dither[(i * 8 + 4 + offset) & 7] << 12, + dither[(i * 8 + 5 + offset) & 7] << 12, + dither[(i * 8 + 6 + offset) & 7] << 12, + dither[(i * 8 + 7 + offset) & 7] << 12 }; + v8i16 zero = { 0 }; + + for (j = 0; j < filterSize; j++) { + src0 = LD_V(v8i16, &src[j][i * 8]); + filter0 = __msa_fill_w(filter[j]); + flags = __msa_clt_s_h(src0, zero); + ILVRL_H2_SW(flags, src0, src_r, src_l); + val_r += src_r * filter0; + val_l += src_l * filter0; + } + val_r >>= 19; + val_l >>= 19; + CLIP_SW2_0_255(val_r, val_l); + src0 = __msa_pckev_h((v8i16)val_l, (v8i16)val_r); + src0 = (v8i16)__msa_pckev_b((v16i8)src0, (v16i8)src0); + SD(__msa_copy_s_d((v2i64)src0, 0), dest + i * 8); + } + for (i = part; i < dstW; i++) { + int val = dither[(i + offset) & 7] << 12; + int j; + for (j = 0; j< filterSize; j++) + val += src[j][i] * filter[j]; + + dest[i] = av_clip_uint8(val >> 19); + } +} + +/*Copy from libswscale/output.c*/ +static av_always_inline void +yuv2rgb_write(uint8_t *_dest, int i, int Y1, int Y2, + unsigned A1, unsigned A2, + const void *_r, const void *_g, const void *_b, int y, + enum AVPixelFormat target, int hasAlpha) +{ + if (target == AV_PIX_FMT_ARGB || target == AV_PIX_FMT_RGBA || + target == AV_PIX_FMT_ABGR || target == AV_PIX_FMT_BGRA) { + uint32_t *dest = (uint32_t *) _dest; + const uint32_t *r = (const uint32_t *) _r; + const uint32_t *g = (const uint32_t *) _g; + const uint32_t *b = (const uint32_t *) _b; + +#if CONFIG_SMALL + int sh = hasAlpha ? ((target ==AV_PIX_FMT_RGB32_1 || target == AV_PIX_FMT_BGR32_1) ? 0 : 24) : 0; + dest[i * 2 + 0] = r[Y1] + g[Y1] + b[Y1]; + dest[i * 2 + 1] = r[Y2] + g[Y2] + b[Y2]; +#else +#if defined(ASSERT_LEVEL) && ASSERT_LEVEL > 1 + int sh = (target == AV_PIX_FMT_RGB32_1 || + target == AV_PIX_FMT_BGR32_1) ? 0 : 24; + av_assert2((((r[Y1] + g[Y1] + b[Y1]) >> sh) & 0xFF) == 0xFF); +#endif + dest[i * 2 + 0] = r[Y1] + g[Y1] + b[Y1]; + dest[i * 2 + 1] = r[Y2] + g[Y2] + b[Y2]; +#endif + + } else if (target == AV_PIX_FMT_RGB565 || target == AV_PIX_FMT_BGR565 || + target == AV_PIX_FMT_RGB555 || target == AV_PIX_FMT_BGR555 || + target == AV_PIX_FMT_RGB444 || target == AV_PIX_FMT_BGR444) { + uint16_t *dest = (uint16_t *) _dest; + const uint16_t *r = (const uint16_t *) _r; + const uint16_t *g = (const uint16_t *) _g; + const uint16_t *b = (const uint16_t *) _b; + int dr1, dg1, db1, dr2, dg2, db2; + + if (target == AV_PIX_FMT_RGB565 || target == AV_PIX_FMT_BGR565) { + dr1 = ff_dither_2x2_8[ y & 1 ][0]; + dg1 = ff_dither_2x2_4[ y & 1 ][0]; + db1 = ff_dither_2x2_8[(y & 1) ^ 1][0]; + dr2 = ff_dither_2x2_8[ y & 1 ][1]; + dg2 = ff_dither_2x2_4[ y & 1 ][1]; + db2 = ff_dither_2x2_8[(y & 1) ^ 1][1]; + } + + dest[i * 2 + 0] = r[Y1 + dr1] + g[Y1 + dg1] + b[Y1 + db1]; + dest[i * 2 + 1] = r[Y2 + dr2] + g[Y2 + dg2] + b[Y2 + db2]; + } +} + +static av_always_inline void +yuv2rgb_X_msa_template(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, int dstW, + int y, enum AVPixelFormat target, int hasAlpha) +{ + int i, j; + int count = 0; + int len = dstW & (~0x07); + int len_count = (dstW + 1) >> 1; + const void *r, *g, *b; + v4i32 headroom = (v4i32)__msa_fill_w(YUVRGB_TABLE_HEADROOM); + + for (i = 0; i < len; i += 8) { + int t = 1 << 18; + v8i16 l_src, u_src, v_src; + v4i32 lumsrc_r, lumsrc_l, usrc, vsrc, temp; + v4i32 y_r, y_l, u, v; + + y_r = __msa_fill_w(t); + y_l = y_r; + u = y_r; + v = y_r; + for (j = 0; j < lumFilterSize; j++) { + temp = __msa_fill_w(lumFilter[j]); + l_src = LD_V(v8i16, (lumSrc[j] + i)); + UNPCK_SH_SW(l_src, lumsrc_r, lumsrc_l); /*can use lsx optimization*/ + y_r = __msa_maddv_w(lumsrc_r, temp, y_r); + y_l = __msa_maddv_w(lumsrc_l, temp, y_l); + } + for (j = 0; j < chrFilterSize; j++) { + u_src = LD_V(v8i16, (chrUSrc[j] + count)); + v_src = LD_V(v8i16, (chrVSrc[j] + count)); + UNPCK_R_SH_SW(u_src, usrc); + UNPCK_R_SH_SW(v_src, vsrc); + temp = __msa_fill_w(chrFilter[j]); + u = __msa_maddv_w(usrc, temp, u); + v = __msa_maddv_w(vsrc, temp, v); + } + y_r = __msa_srai_w(y_r, 19); + y_l = __msa_srai_w(y_l, 19); + u = __msa_srai_w(u, 19); + v = __msa_srai_w(v, 19); + u = __msa_addv_w(u, headroom); + v = __msa_addv_w(v, headroom); + for (j = 0; j < 2; j++) { + int Y1, Y2, U, V; + int m = j * 2; + int n = j + 2; + + Y1 = y_r[m]; + Y2 = y_r[m + 1]; + U = u[j]; + V = v[j]; + r = c->table_rV[V]; + g = (c->table_gU[U] + c->table_gV[V]); + b = c->table_bU[U]; + + yuv2rgb_write(dest, count + j, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + Y1 = y_l[m]; + Y2 = y_l[m + 1]; + U = u[n]; + V = v[n]; + r = c->table_rV[V]; + g = (c->table_gU[U] + c->table_gV[V]); + b = c->table_bU[U]; + + yuv2rgb_write(dest, count + n, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } + count += 4; + } + for (count; count < len_count; count++) { + int Y1 = 1 << 18; + int Y2 = 1 << 18; + int U = 1 << 18; + int V = 1 << 18; + + for (j = 0; j < lumFilterSize; j++) { + Y1 += lumSrc[j][count * 2] * lumFilter[j]; + Y2 += lumSrc[j][count * 2 + 1] * lumFilter[j]; + } + for (j = 0; j < chrFilterSize; j++) { + U += chrUSrc[j][count] * chrFilter[j]; + V += chrVSrc[j][count] * chrFilter[j]; + } + Y1 >>= 19; + Y2 >>= 19; + U >>= 19; + V >>= 19; + r = c->table_rV[V + YUVRGB_TABLE_HEADROOM]; + g = (c->table_gU[U + YUVRGB_TABLE_HEADROOM] + + c->table_gV[V + YUVRGB_TABLE_HEADROOM]); + b = c->table_bU[U + YUVRGB_TABLE_HEADROOM]; + + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } +} + +static av_always_inline void +yuv2rgb_2_msa_template(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y, + enum AVPixelFormat target, int hasAlpha) +{ + const int16_t *buf0 = buf[0], *buf1 = buf[1], + *ubuf0 = ubuf[0], *ubuf1 = ubuf[1], + *vbuf0 = vbuf[0], *vbuf1 = vbuf[1], + *abuf0 = NULL, + *abuf1 = NULL; + int yalpha1 = 4096 - yalpha; + int uvalpha1 = 4096 - uvalpha; + int i, count = 0; + int len = dstW & (~0x07); + int len_count = (dstW + 1) >> 1; + const void *r, *g, *b; + v4i32 v_yalpha1 = (v4i32)__msa_fill_w(yalpha1); + v4i32 v_uvalpha1 = (v4i32)__msa_fill_w(uvalpha1); + v4i32 v_yalpha = (v4i32)__msa_fill_w(yalpha); + v4i32 v_uvalpha = (v4i32)__msa_fill_w(uvalpha); + v4i32 headroom = (v4i32)__msa_fill_w(YUVRGB_TABLE_HEADROOM); + + for (i = 0; i < len; i += 8) { + v8i16 src_y, src_u, src_v; + v4i32 y0_r, y0_l, u0, v0; + v4i32 y1_r, y1_l, u1, v1; + v4i32 y_r, y_l, u, v; + + src_y = LD_V(v8i16, buf0 + i); + src_u = LD_V(v8i16, ubuf0 + count); + src_v = LD_V(v8i16, vbuf0 + count); + UNPCK_SH_SW(src_y, y0_r, y0_l); + UNPCK_R_SH_SW(src_u, u0); + UNPCK_R_SH_SW(src_v, v0); + src_y = LD_V(v8i16, buf1 + i); + src_u = LD_V(v8i16, ubuf1 + count); + src_v = LD_V(v8i16, vbuf1 + count); + UNPCK_SH_SW(src_y, y1_r, y1_l); + UNPCK_R_SH_SW(src_u, u1); + UNPCK_R_SH_SW(src_v, v1); + y0_r = __msa_mulv_w(y0_r, v_yalpha1); + y0_l = __msa_mulv_w(y0_l, v_yalpha1); + u0 = __msa_mulv_w(u0, v_uvalpha1); + v0 = __msa_mulv_w(v0, v_uvalpha1); + y_r = __msa_maddv_w(y1_r, v_yalpha, y0_r); + y_l = __msa_maddv_w(y1_l, v_yalpha, y0_l); + u = __msa_maddv_w(u1, v_uvalpha, u0); + v = __msa_maddv_w(v1, v_uvalpha, v0); + y_r = __msa_srai_w(y_r, 19); + y_l = __msa_srai_w(y_l, 19); + u = __msa_srai_w(u, 19); + v = __msa_srai_w(v, 19); + u = __msa_addv_w(u, headroom); + v = __msa_addv_w(v, headroom); + for (int j = 0; j < 2; j++) { + int Y1, Y2, U, V; + int m = j * 2; + int n = j + 2; + + Y1 = y_r[m]; + Y2 = y_r[m + 1]; + U = u[j]; + V = v[j]; + r = c->table_rV[V]; + g = (c->table_gU[U] + c->table_gV[V]); + b = c->table_bU[U]; + + yuv2rgb_write(dest, count + j, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + Y1 = y_l[m]; + Y2 = y_l[m + 1]; + U = u[n]; + V = v[n]; + r = c->table_rV[V]; + g = (c->table_gU[U] + c->table_gV[V]); + b = c->table_bU[U]; + + yuv2rgb_write(dest, count + n, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } + count += 4; + } + + for (count; count < len_count; count++) { + int Y1 = (buf0[count * 2] * yalpha1 + + buf1[count * 2] * yalpha) >> 19; + int Y2 = (buf0[count * 2 + 1] * yalpha1 + + buf1[count * 2 + 1] * yalpha) >> 19; + int U = (ubuf0[count] * uvalpha1 + ubuf1[count] * uvalpha) >> 19; + int V = (vbuf0[count] * uvalpha1 + vbuf1[count] * uvalpha) >> 19; + + r = c->table_rV[V + YUVRGB_TABLE_HEADROOM], + g = (c->table_gU[U + YUVRGB_TABLE_HEADROOM] + + c->table_gV[V + YUVRGB_TABLE_HEADROOM]), + b = c->table_bU[U + YUVRGB_TABLE_HEADROOM]; + + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } +} + +static av_always_inline void +yuv2rgb_1_msa_template(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y, enum AVPixelFormat target, + int hasAlpha) +{ + const int16_t *ubuf0 = ubuf[0], *vbuf0 = vbuf[0]; + int i, j; + const void *r, *g, *b; + int len = dstW & (~0x07); + int len_count = (dstW + 1) >> 1; + + if (uvalpha < 2048) { + int count = 0; + v4i32 headroom = (v4i32)__msa_fill_w(YUVRGB_TABLE_HEADROOM); + v4i32 bias_64 = (v4i32)__msa_fill_w(64); + + for (i = 0; i < len; i += 8) { + v8i16 src_y, src_u, src_v; + v4i32 y_r, y_l, u, v; + + src_y = LD_V(v8i16, buf0 + i); + src_u = LD_V(v8i16, ubuf0 + count); + src_v = LD_V(v8i16, vbuf0 + count); + UNPCK_SH_SW(src_y, y_r, y_l); + UNPCK_R_SH_SW(src_u, u); + UNPCK_R_SH_SW(src_v, v); + y_r = __msa_addv_w(y_r, bias_64); + y_l = __msa_addv_w(y_l, bias_64); + u = __msa_addv_w(u, bias_64); + v = __msa_addv_w(v, bias_64); + y_r = __msa_srai_w(y_r, 7); + y_l = __msa_srai_w(y_l, 7); + u = __msa_srai_w(u, 7); + v = __msa_srai_w(v, 7); + u = __msa_addv_w(u, headroom); + v = __msa_addv_w(v, headroom); + for (j = 0; j < 2; j++) { + int Y1, Y2, U, V; + int m = j * 2; + int n = j + 2; + + Y1 = y_r[m]; + Y2 = y_r[m + 1]; + U = u[j]; + V = v[j]; + r = c->table_rV[V]; + g = (c->table_gU[U] + c->table_gV[V]); + b = c->table_bU[U]; + + yuv2rgb_write(dest, count + j, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + Y1 = y_l[m]; + Y2 = y_l[m + 1]; + U = u[n]; + V = v[n]; + r = c->table_rV[V]; + g = (c->table_gU[U] + c->table_gV[V]); + b = c->table_bU[U]; + + yuv2rgb_write(dest, count + n, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } + count += 4; + } + for (count; count < len_count; count++) { + int Y1 = (buf0[count * 2 ] + 64) >> 7; + int Y2 = (buf0[count * 2 + 1] + 64) >> 7; + int U = (ubuf0[count] + 64) >> 7; + int V = (vbuf0[count] + 64) >> 7; + + r = c->table_rV[V + YUVRGB_TABLE_HEADROOM], + g = (c->table_gU[U + YUVRGB_TABLE_HEADROOM] + + c->table_gV[V + YUVRGB_TABLE_HEADROOM]), + b = c->table_bU[U + YUVRGB_TABLE_HEADROOM]; + + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } + } else { + const int16_t *ubuf1 = ubuf[1], *vbuf1 = vbuf[1]; + int count = 0; + v4i32 headroom = (v4i32)__msa_fill_w(YUVRGB_TABLE_HEADROOM); + v4i32 bias_64 = (v4i32)__msa_fill_w(64); + v4i32 bias_128 = (v4i32)__msa_fill_w(128); + + for (i = 0; i < len; i += 8) { + v8i16 src_y, src_u, src_v; + v4i32 y_r, y_l, u0, v0, u1, v1, u, v; + + src_y = LD_V(v8i16, buf0 + i); + src_u = LD_V(v8i16, ubuf0 + count); + src_v = LD_V(v8i16, vbuf0 + count); + UNPCK_SH_SW(src_y, y_r, y_l); + UNPCK_R_SH_SW(src_u, u0); + UNPCK_R_SH_SW(src_v, v0); + src_u = LD_V(v8i16, ubuf1 + count); + src_v = LD_V(v8i16, vbuf1 + count); + UNPCK_R_SH_SW(src_u, u1); + UNPCK_R_SH_SW(src_v, v1); + + u = __msa_addv_w(u0, u1); + v = __msa_addv_w(v0, v1); + y_r = __msa_addv_w(y_r, bias_64); + y_l = __msa_addv_w(y_l, bias_64); + u = __msa_addv_w(u, bias_128); + v = __msa_addv_w(v, bias_128); + y_r = __msa_srai_w(y_r, 7); + y_l = __msa_srai_w(y_l, 7); + u = __msa_srai_w(u, 8); + v = __msa_srai_w(v, 8); + u = __msa_addv_w(u, headroom); + v = __msa_addv_w(v, headroom); + for (j = 0; j < 2; j++) { + int Y1, Y2, U, V; + int m = j * 2; + int n = j + 2; + + Y1 = y_r[m]; + Y2 = y_r[m + 1]; + U = u[j]; + V = v[j]; + r = c->table_rV[V]; + g = (c->table_gU[U] + c->table_gV[V]); + b = c->table_bU[U]; + + yuv2rgb_write(dest, count + j, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + Y1 = y_l[m]; + Y2 = y_l[m + 1]; + U = u[n]; + V = v[n]; + r = c->table_rV[V]; + g = (c->table_gU[U] + c->table_gV[V]); + b = c->table_bU[U]; + + yuv2rgb_write(dest, count + n, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } + count += 4; + } + for (count; count < len_count; count++) { + int Y1 = (buf0[count * 2 ] + 64) >> 7; + int Y2 = (buf0[count * 2 + 1] + 64) >> 7; + int U = (ubuf0[count] + ubuf1[count] + 128) >> 8; + int V = (vbuf0[count] + vbuf1[count] + 128) >> 8; + + r = c->table_rV[V + YUVRGB_TABLE_HEADROOM], + g = (c->table_gU[U + YUVRGB_TABLE_HEADROOM] + + c->table_gV[V + YUVRGB_TABLE_HEADROOM]), + b = c->table_bU[U + YUVRGB_TABLE_HEADROOM]; + + yuv2rgb_write(dest, count, Y1, Y2, 0, 0, + r, g, b, y, target, 0); + } + } +} + +#define YUV2RGBWRAPPERX(name, base, ext, fmt, hasAlpha) \ +void name ## ext ## _X_msa(SwsContext *c, const int16_t *lumFilter, \ + const int16_t **lumSrc, int lumFilterSize, \ + const int16_t *chrFilter, const int16_t **chrUSrc, \ + const int16_t **chrVSrc, int chrFilterSize, \ + const int16_t **alpSrc, uint8_t *dest, int dstW, \ + int y) \ +{ \ + name ## base ## _X_msa_template(c, lumFilter, lumSrc, lumFilterSize, \ + chrFilter, chrUSrc, chrVSrc, chrFilterSize,\ + alpSrc, dest, dstW, y, fmt, hasAlpha); \ +} + +#define YUV2RGBWRAPPERX2(name, base, ext, fmt, hasAlpha) \ +YUV2RGBWRAPPERX(name, base, ext, fmt, hasAlpha) \ +void name ## ext ## _2_msa(SwsContext *c, const int16_t *buf[2], \ + const int16_t *ubuf[2], const int16_t *vbuf[2], \ + const int16_t *abuf[2], uint8_t *dest, int dstW, \ + int yalpha, int uvalpha, int y) \ +{ \ + name ## base ## _2_msa_template(c, buf, ubuf, vbuf, abuf, dest, \ + dstW, yalpha, uvalpha, y, fmt, hasAlpha); \ +} + +#define YUV2RGBWRAPPER(name, base, ext, fmt, hasAlpha) \ +YUV2RGBWRAPPERX2(name, base, ext, fmt, hasAlpha) \ +void name ## ext ## _1_msa(SwsContext *c, const int16_t *buf0, \ + const int16_t *ubuf[2], const int16_t *vbuf[2], \ + const int16_t *abuf0, uint8_t *dest, int dstW, \ + int uvalpha, int y) \ +{ \ + name ## base ## _1_msa_template(c, buf0, ubuf, vbuf, abuf0, dest, \ + dstW, uvalpha, y, fmt, hasAlpha); \ +} + + +#if CONFIG_SMALL +#else +#if CONFIG_SWSCALE_ALPHA +#endif +YUV2RGBWRAPPER(yuv2rgb,, x32_1, AV_PIX_FMT_RGB32_1, 0) +YUV2RGBWRAPPER(yuv2rgb,, x32, AV_PIX_FMT_RGB32, 0) +#endif +YUV2RGBWRAPPER(yuv2rgb,, 16, AV_PIX_FMT_RGB565, 0) + +// This function is copied from libswscale/output.c +static av_always_inline void yuv2rgb_write_full(SwsContext *c, + uint8_t *dest, int i, int R, int A, int G, int B, + int y, enum AVPixelFormat target, int hasAlpha, int err[4]) +{ + int isrgb8 = target == AV_PIX_FMT_BGR8 || target == AV_PIX_FMT_RGB8; + + if ((R | G | B) & 0xC0000000) { + R = av_clip_uintp2(R, 30); + G = av_clip_uintp2(G, 30); + B = av_clip_uintp2(B, 30); + } + + switch(target) { + case AV_PIX_FMT_ARGB: + dest[0] = hasAlpha ? A : 255; + dest[1] = R >> 22; + dest[2] = G >> 22; + dest[3] = B >> 22; + break; + case AV_PIX_FMT_RGB24: + dest[0] = R >> 22; + dest[1] = G >> 22; + dest[2] = B >> 22; + break; + case AV_PIX_FMT_RGBA: + dest[0] = R >> 22; + dest[1] = G >> 22; + dest[2] = B >> 22; + dest[3] = hasAlpha ? A : 255; + break; + case AV_PIX_FMT_ABGR: + dest[0] = hasAlpha ? A : 255; + dest[1] = B >> 22; + dest[2] = G >> 22; + dest[3] = R >> 22; + break; + case AV_PIX_FMT_BGR24: + dest[0] = B >> 22; + dest[1] = G >> 22; + dest[2] = R >> 22; + break; + case AV_PIX_FMT_BGRA: + dest[0] = B >> 22; + dest[1] = G >> 22; + dest[2] = R >> 22; + dest[3] = hasAlpha ? A : 255; + break; + case AV_PIX_FMT_BGR4_BYTE: + case AV_PIX_FMT_RGB4_BYTE: + case AV_PIX_FMT_BGR8: + case AV_PIX_FMT_RGB8: + { + int r,g,b; + + switch (c->dither) { + default: + case SWS_DITHER_AUTO: + case SWS_DITHER_ED: + R >>= 22; + G >>= 22; + B >>= 22; + R += (7*err[0] + 1*c->dither_error[0][i] + 5*c->dither_error[0][i+1] + 3*c->dither_error[0][i+2])>>4; + G += (7*err[1] + 1*c->dither_error[1][i] + 5*c->dither_error[1][i+1] + 3*c->dither_error[1][i+2])>>4; + B += (7*err[2] + 1*c->dither_error[2][i] + 5*c->dither_error[2][i+1] + 3*c->dither_error[2][i+2])>>4; + c->dither_error[0][i] = err[0]; + c->dither_error[1][i] = err[1]; + c->dither_error[2][i] = err[2]; + r = R >> (isrgb8 ? 5 : 7); + g = G >> (isrgb8 ? 5 : 6); + b = B >> (isrgb8 ? 6 : 7); + r = av_clip(r, 0, isrgb8 ? 7 : 1); + g = av_clip(g, 0, isrgb8 ? 7 : 3); + b = av_clip(b, 0, isrgb8 ? 3 : 1); + err[0] = R - r*(isrgb8 ? 36 : 255); + err[1] = G - g*(isrgb8 ? 36 : 85); + err[2] = B - b*(isrgb8 ? 85 : 255); + break; + case SWS_DITHER_A_DITHER: + if (isrgb8) { + /* see http://pippin.gimp.org/a_dither/ for details/origin */ +#define A_DITHER(u,v) (((((u)+((v)*236))*119)&0xff)) + r = (((R >> 19) + A_DITHER(i,y) -96)>>8); + g = (((G >> 19) + A_DITHER(i + 17,y) - 96)>>8); + b = (((B >> 20) + A_DITHER(i + 17*2,y) -96)>>8); + r = av_clip_uintp2(r, 3); + g = av_clip_uintp2(g, 3); + b = av_clip_uintp2(b, 2); + } else { + r = (((R >> 21) + A_DITHER(i,y)-256)>>8); + g = (((G >> 19) + A_DITHER(i + 17,y)-256)>>8); + b = (((B >> 21) + A_DITHER(i + 17*2,y)-256)>>8); + r = av_clip_uintp2(r, 1); + g = av_clip_uintp2(g, 2); + b = av_clip_uintp2(b, 1); + } + break; + case SWS_DITHER_X_DITHER: + if (isrgb8) { + /* see http://pippin.gimp.org/a_dither/ for details/origin */ +#define X_DITHER(u,v) (((((u)^((v)*237))*181)&0x1ff)/2) + r = (((R >> 19) + X_DITHER(i,y) - 96)>>8); + g = (((G >> 19) + X_DITHER(i + 17,y) - 96)>>8); + b = (((B >> 20) + X_DITHER(i + 17*2,y) - 96)>>8); + r = av_clip_uintp2(r, 3); + g = av_clip_uintp2(g, 3); + b = av_clip_uintp2(b, 2); + } else { + r = (((R >> 21) + X_DITHER(i,y)-256)>>8); + g = (((G >> 19) + X_DITHER(i + 17,y)-256)>>8); + b = (((B >> 21) + X_DITHER(i + 17*2,y)-256)>>8); + r = av_clip_uintp2(r, 1); + g = av_clip_uintp2(g, 2); + b = av_clip_uintp2(b, 1); + } + + break; + } + + if(target == AV_PIX_FMT_BGR4_BYTE) { + dest[0] = r + 2*g + 8*b; + } else if(target == AV_PIX_FMT_RGB4_BYTE) { + dest[0] = b + 2*g + 8*r; + } else if(target == AV_PIX_FMT_BGR8) { + dest[0] = r + 8*g + 64*b; + } else if(target == AV_PIX_FMT_RGB8) { + dest[0] = b + 4*g + 32*r; + } else + av_assert2(0); + break; } + } +} + +#define yuvTorgb_setup \ + int y_offset = c->yuv2rgb_y_offset; \ + int y_coeff = c->yuv2rgb_y_coeff; \ + int v2r_coe = c->yuv2rgb_v2r_coeff; \ + int v2g_coe = c->yuv2rgb_v2g_coeff; \ + int u2g_coe = c->yuv2rgb_u2g_coeff; \ + int u2b_coe = c->yuv2rgb_u2b_coeff; \ + v4i32 offset = __msa_fill_w(y_offset); \ + v4i32 coeff = __msa_fill_w(y_coeff); \ + v4i32 v2r = __msa_fill_w(v2r_coe); \ + v4i32 v2g = __msa_fill_w(v2g_coe); \ + v4i32 u2g = __msa_fill_w(u2g_coe); \ + v4i32 u2b = __msa_fill_w(u2b_coe); \ + + +#define yuvTorgb \ + y_r -= offset; \ + y_l -= offset; \ + y_r *= coeff; \ + y_l *= coeff; \ + y_r += y_temp; \ + y_l += y_temp; \ + R_r = __msa_maddv_w(v_r, v2r, y_r); \ + R_l = __msa_maddv_w(v_l, v2r, y_l); \ + v_r = __msa_maddv_w(v_r, v2g, y_r); \ + v_l = __msa_maddv_w(v_l, v2g, y_l); \ + G_r = __msa_maddv_w(u_r, u2g, v_r); \ + G_l = __msa_maddv_w(u_l, u2g, v_l); \ + B_r = __msa_maddv_w(u_r, u2b, y_r); \ + B_l = __msa_maddv_w(u_l, u2b, y_l); \ + + +static av_always_inline void +yuv2rgb_full_X_msa_template(SwsContext *c, const int16_t *lumFilter, + const int16_t **lumSrc, int lumFilterSize, + const int16_t *chrFilter, const int16_t **chrUSrc, + const int16_t **chrVSrc, int chrFilterSize, + const int16_t **alpSrc, uint8_t *dest, + int dstW, int y, enum AVPixelFormat target, int hasAlpha) +{ + int i, j, B, G, R, A; + int step = (target == AV_PIX_FMT_RGB24 || target == AV_PIX_FMT_BGR24) ? 3 : 4; + int err[4] = {0}; + int a_temp = 1 << 18; //init to silence warning + int templ = 1 << 9; + int tempc = templ - (128 << 19); + int len = dstW & (~0x07); + int ytemp = 1 << 21; + v4i32 y_temp = __msa_fill_w(ytemp); + yuvTorgb_setup + + if( target == AV_PIX_FMT_BGR4_BYTE || target == AV_PIX_FMT_RGB4_BYTE + || target == AV_PIX_FMT_BGR8 || target == AV_PIX_FMT_RGB8) + step = 1; + + for (i = 0; i < len; i += 8) { + int m = i + 4; + v8i16 l_src, u_src, v_src; + v4i32 lum_r, lum_l, usrc_r, usrc_l, vsrc_r, vsrc_l; + v4i32 y_r, y_l, u_r, u_l, v_r, v_l, temp; + v4i32 R_r, R_l, G_r, G_l, B_r, B_l; + + y_r = y_l = (v4i32)__msa_fill_w(templ); + u_r = u_l = v_r = v_l = (v4i32)__msa_fill_w(tempc); + for (j = 0; j < lumFilterSize; j++) { + temp = __msa_fill_w(lumFilter[j]); + l_src = LD_V(v8i16, (lumSrc[j] + i)); + UNPCK_SH_SW(l_src, lum_r, lum_l); + y_l = __msa_maddv_w(lum_l, temp, y_l); + y_r = __msa_maddv_w(lum_r, temp, y_r); + } + for (j = 0; j < chrFilterSize; j++) { + temp = __msa_fill_w(chrFilter[j]); + u_src = LD_V(v8i16, (chrUSrc[j] + i)); + v_src = LD_V(v8i16, (chrVSrc[j] + i)); + UNPCK_SH_SW(u_src, usrc_r, usrc_l); + UNPCK_SH_SW(v_src, vsrc_r, vsrc_l); + u_l = __msa_maddv_w(usrc_l, temp, u_l); + u_r = __msa_maddv_w(usrc_r, temp, u_r); + v_l = __msa_maddv_w(vsrc_l, temp, v_l); + v_r = __msa_maddv_w(vsrc_r, temp, v_r); + } + y_r = __msa_srai_w(y_r, 10); + y_l = __msa_srai_w(y_l, 10); + u_r = __msa_srai_w(u_r, 10); + u_l = __msa_srai_w(u_l, 10); + v_r = __msa_srai_w(v_r, 10); + v_l = __msa_srai_w(v_l, 10); + yuvTorgb + + if (hasAlpha) { + v8i16 a_src; + v4i32 asrc_r, asrc_l, a_r, a_l; + + a_r = a_l = __msa_fill_w(a_temp); + for (j = 0; j < lumFilterSize; j++) { + temp = __msa_fill_w(lumFilter[j]); + a_src = LD_V(v8i16, (alpSrc[j] + i)); + UNPCK_SH_SW(a_src, asrc_r, asrc_l); + a_l = __msa_maddv_w(asrc_l, temp, a_l); + a_r = __msa_maddv_w(asrc_r, temp, a_r); + } + a_l = __msa_srai_w(a_l, 19); + a_r = __msa_srai_w(a_r, 19); + for (j = 0; j < 4; j++) { + R = R_r[j]; + G = G_r[j]; + B = B_r[j]; + A = a_r[j]; + if (A & 0x100) + A = av_clip_uint8(A); + yuv2rgb_write_full(c, dest, i + j, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + for (j = 0; j < 4; j++) { + R = R_l[j]; + G = G_l[j]; + B = B_l[j]; + A = a_l[j]; + if (A & 0x100) + A = av_clip_uint8(A); + yuv2rgb_write_full(c, dest, m + j, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + } else { + for (j = 0; j < 4; j++) { + R = R_r[j]; + G = G_r[j]; + B = B_r[j]; + yuv2rgb_write_full(c, dest, i + j, R, 0, G, B, y, target, hasAlpha, err); + dest += step; + } + for (j = 0; j < 4; j++) { + R = R_l[j]; + G = G_l[j]; + B = B_l[j]; + yuv2rgb_write_full(c, dest, m + j, R, 0, G, B, y, target, hasAlpha, err); + dest += step; + } + } + } + for (i; i < dstW; i++) { + int Y = templ; + int V, U = V = tempc; + + A = 0; + for (j = 0; j < lumFilterSize; j++) { + Y += lumSrc[j][i] * lumFilter[j]; + } + for (j = 0; j < chrFilterSize; j++) { + U += chrUSrc[j][i] * chrFilter[j]; + V += chrVSrc[j][i] * chrFilter[j]; + + } + Y >>= 10; + U >>= 10; + V >>= 10; + if (hasAlpha) { + A = 1 << 18; + for (j = 0; j < lumFilterSize; j++) { + A += alpSrc[j][i] * lumFilter[j]; + } + A >>= 19; + if (A & 0x100) + A = av_clip_uint8(A); + } + Y -= y_offset; + Y *= y_coeff; + Y += ytemp; + R = (unsigned)Y + V * v2r_coe; + G = (unsigned)Y + V * v2g_coe + U * u2g_coe; + B = (unsigned)Y + U * u2b_coe; + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + c->dither_error[0][i] = err[0]; + c->dither_error[1][i] = err[1]; + c->dither_error[2][i] = err[2]; +} + +static av_always_inline void +yuv2rgb_full_2_msa_template(SwsContext *c, const int16_t *buf[2], + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf[2], uint8_t *dest, int dstW, + int yalpha, int uvalpha, int y, + enum AVPixelFormat target, int hasAlpha) +{ + const int16_t *buf0 = buf[0], *buf1 = buf[1], + *ubuf0 = ubuf[0], *ubuf1 = ubuf[1], + *vbuf0 = vbuf[0], *vbuf1 = vbuf[1], + *abuf0 = hasAlpha ? abuf[0] : NULL, + *abuf1 = hasAlpha ? abuf[1] : NULL; + int yalpha1 = 4096 - yalpha; + int uvalpha1 = 4096 - uvalpha; + int uvtemp = 128 << 19; + int atemp = 1 << 18; + int i, j, R, G, B, A; + int step = (target == AV_PIX_FMT_RGB24 || target == AV_PIX_FMT_BGR24) ? 3 : 4; + int err[4] = {0}; + int len = dstW & (~0x07); + int ytemp = 1 << 21; + v4i32 uvalp1 = (v4i32)__msa_fill_w(uvalpha1); + v4i32 yalp1 = (v4i32)__msa_fill_w(yalpha1); + v4i32 uvalp = (v4i32)__msa_fill_w(uvalpha); + v4i32 yalp = (v4i32)__msa_fill_w(yalpha); + v4i32 uv = (v4i32)__msa_fill_w(uvtemp); + v4i32 a_bias = (v4i32)__msa_fill_w(atemp); + v4i32 y_temp = __msa_fill_w(ytemp); + yuvTorgb_setup + + av_assert2(yalpha <= 4096U); + av_assert2(uvalpha <= 4096U); + + if( target == AV_PIX_FMT_BGR4_BYTE || target == AV_PIX_FMT_RGB4_BYTE + || target == AV_PIX_FMT_BGR8 || target == AV_PIX_FMT_RGB8) + step = 1; + + for (i = 0; i < len; i += 8) { + int m = i + 4; + v8i16 b0, b1, ub0, ub1, vb0, vb1; + v4i32 b0_r, b0_l, b1_r, b1_l, ub0_r, ub0_l, ub1_r, ub1_l, vb0_r, vb0_l, vb1_r, vb1_l; + v4i32 y_r, y_l, u_r, u_l, v_r, v_l; + v4i32 R_r, R_l, G_r, G_l, B_r, B_l; + + b0 = LD_V(v8i16, (buf0 + i)); + b1 = LD_V(v8i16, (buf1 + i)); + ub0 = LD_V(v8i16, (ubuf0 + i)); + ub1 = LD_V(v8i16, (ubuf1 + i)); + vb0 = LD_V(v8i16, (vbuf0 + i)); + vb1 = LD_V(v8i16, (vbuf1 + i)); + UNPCK_SH_SW(b0, b0_r, b0_l); + UNPCK_SH_SW(b1, b1_r, b1_l); + UNPCK_SH_SW(ub0, ub0_r, ub0_l); + UNPCK_SH_SW(ub1, ub1_r, ub1_l); + UNPCK_SH_SW(vb0, vb0_r, vb0_l); + UNPCK_SH_SW(vb1, vb1_r, vb1_l); + y_r = b0_r * yalp1; + y_l = b0_l * yalp1; + y_r = __msa_maddv_w(b1_r, yalp, y_r); + y_l = __msa_maddv_w(b1_l, yalp, y_l); + u_r = ub0_r * uvalp1; + u_l = ub0_l * uvalp1; + u_r = __msa_maddv_w(ub1_r, uvalp, u_r); + u_l = __msa_maddv_w(ub1_l, uvalp, u_l); + v_r = vb0_r * uvalp1; + v_l = vb0_l * uvalp1; + v_r = __msa_maddv_w(vb1_r, uvalp, v_r); + v_l = __msa_maddv_w(vb1_l, uvalp, v_l); + u_r -= uv; + u_l -= uv; + v_r -= uv; + v_l -= uv; + y_r = __msa_srai_w(y_r, 10); + y_l = __msa_srai_w(y_l, 10); + u_r = __msa_srai_w(u_r, 10); + u_l = __msa_srai_w(u_l, 10); + v_r = __msa_srai_w(v_r, 10); + v_l = __msa_srai_w(v_l, 10); + yuvTorgb + + if (hasAlpha) { + v8i16 a0, a1; + v4i32 a_r, a_l, a1_r, a1_l; + + a0 = LD_V(v8i16, (abuf0 + i)); + a1 = LD_V(v8i16, (abuf1 + i)); + UNPCK_SH_SW(a0, a_r, a_l); + UNPCK_SH_SW(a1, a1_r, a1_l); + a_r *= yalp1; + a_l *= yalp1; + a_r = __msa_maddv_w(a1_r, yalp, a_r); + a_l = __msa_maddv_w(a1_l, yalp, a_l); + a_r += a_bias; + a_l += a_bias; + a_r = __msa_srai_w(a_r, 19); + a_l = __msa_srai_w(a_l, 19); + for (j = 0; j < 4; j++) { + R = R_r[j]; + G = G_r[j]; + B = B_r[j]; + A = a_r[j]; + if (A & 0x100) + A = av_clip_uint8(A); + yuv2rgb_write_full(c, dest, i + j, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + for (j = 0; j < 4; j++) { + R = R_l[j]; + G = G_l[j]; + B = B_l[j]; + A = a_l[j]; + if (A & 0x100) + A = av_clip_uint8(A); + yuv2rgb_write_full(c, dest, m + j, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + } else { + for (j = 0; j < 4; j++) { + R = R_r[j]; + G = G_r[j]; + B = B_r[j]; + yuv2rgb_write_full(c, dest, i + j, R, 0, G, B, y, target, hasAlpha, err); + dest += step; + } + for (j = 0; j < 4; j++) { + R = R_l[j]; + G = G_l[j]; + B = B_l[j]; + yuv2rgb_write_full(c, dest, m + j, R, 0, G, B, y, target, hasAlpha, err); + dest += step; + } + } + } + for (i; i < dstW; i++){ + int Y = ( buf0[i] * yalpha1 + buf1[i] * yalpha ) >> 10; //FIXME rounding + int U = (ubuf0[i] * uvalpha1 + ubuf1[i] * uvalpha- uvtemp) >> 10; + int V = (vbuf0[i] * uvalpha1 + vbuf1[i] * uvalpha- uvtemp) >> 10; + + A = 0; + if (hasAlpha){ + A = (abuf0[i] * yalpha1 + abuf1[i] * yalpha + atemp) >> 19; + if (A & 0x100) + A = av_clip_uint8(A); + } + + Y -= y_offset; + Y *= y_coeff; + Y += ytemp; + R = (unsigned)Y + V * v2r_coe; + G = (unsigned)Y + V * v2g_coe + U * u2g_coe; + B = (unsigned)Y + U * u2b_coe; + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + c->dither_error[0][i] = err[0]; + c->dither_error[1][i] = err[1]; + c->dither_error[2][i] = err[2]; +} + +static av_always_inline void +yuv2rgb_full_1_msa_template(SwsContext *c, const int16_t *buf0, + const int16_t *ubuf[2], const int16_t *vbuf[2], + const int16_t *abuf0, uint8_t *dest, int dstW, + int uvalpha, int y, enum AVPixelFormat target, + int hasAlpha) +{ + const int16_t *ubuf0 = ubuf[0], *vbuf0 = vbuf[0]; + int i, j, B, G, R, A; + int step = (target == AV_PIX_FMT_RGB24 || target == AV_PIX_FMT_BGR24) ? 3 : 4; + int err[4] = {0}; + int len = dstW & (~0x07); + int ytemp = 1 << 21; + v4i32 bias = __msa_fill_w(64); + v4i32 y_temp = __msa_fill_w(ytemp); + yuvTorgb_setup + + if( target == AV_PIX_FMT_BGR4_BYTE || target == AV_PIX_FMT_RGB4_BYTE + || target == AV_PIX_FMT_BGR8 || target == AV_PIX_FMT_RGB8) + step = 1; + if (uvalpha < 2048) { + int uvtemp = 128 << 7; + v4i32 uv = __msa_fill_w(uvtemp); + + for (i = 0; i < len; i += 8) { + int m = i + 4; + v8i16 b, ub, vb; + v4i32 y_r, y_l, u_r, u_l, v_r, v_l; + v4i32 R_r, R_l, G_r, G_l, B_r, B_l; + + b = LD_V(v8i16, (buf0 + i)); + ub = LD_V(v8i16, (ubuf0 + i)); + vb = LD_V(v8i16, (vbuf0 + i)); + UNPCK_SH_SW(b, y_r, y_l); + UNPCK_SH_SW(ub, u_r, u_l); + UNPCK_SH_SW(vb, v_r, v_l); + y_r = __msa_slli_w(y_r, 2); + y_l = __msa_slli_w(y_l, 2); + u_r -= uv; + u_l -= uv; + v_r -= uv; + v_l -= uv; + u_r = __msa_slli_w(u_r, 2); + u_l = __msa_slli_w(u_l, 2); + v_r = __msa_slli_w(v_r, 2); + v_l = __msa_slli_w(v_l, 2); + yuvTorgb + + if(hasAlpha) { + v8i16 a_src; + v4i32 a_r, a_l; + + a_src = LD_V(v8i16, (abuf0 + i)); + UNPCK_SH_SW(a_src, a_r, a_l); + a_r += bias; + a_l += bias; + a_r = __msa_srai_w(a_r, 7); + a_l = __msa_srai_w(a_l, 7); + for (j = 0; j < 4; j++) { + R = R_r[j]; + G = G_r[j]; + B = B_r[j]; + A = a_r[j]; + if (A & 0x100) + A = av_clip_uint8(A); + yuv2rgb_write_full(c, dest, i + j, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + for (j = 0; j < 4; j++) { + R = R_l[j]; + G = G_l[j]; + B = B_l[j]; + A = a_l[j]; + if (A & 0x100) + A = av_clip_uint8(A); + yuv2rgb_write_full(c, dest, m + j, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + } else { + for (j = 0; j < 4; j++) { + R = R_r[j]; + G = G_r[j]; + B = B_r[j]; + yuv2rgb_write_full(c, dest, i + j, R, 0, G, B, y, target, hasAlpha, err); + dest += step; + } + for (j = 0; j < 4; j++) { + R = R_l[j]; + G = G_l[j]; + B = B_l[j]; + yuv2rgb_write_full(c, dest, m + j, R, 0, G, B, y, target, hasAlpha, err); + dest += step; + } + } + } + for (i; i < dstW; i++) { + int Y = buf0[i] << 2; + int U = (ubuf0[i] - uvtemp) << 2; + int V = (vbuf0[i] - uvtemp) << 2; + + A = 0; + if(hasAlpha) { + A = (abuf0[i] + 64) >> 7; + if (A & 0x100) + A = av_clip_uint8(A); + } + Y -= y_offset; + Y *= y_coeff; + Y += ytemp; + R = (unsigned)Y + V * v2r_coe; + G = (unsigned)Y + V * v2g_coe + U * u2g_coe; + B = (unsigned)Y + U * u2b_coe; + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + } else { + const int16_t *ubuf1 = ubuf[1], *vbuf1 = vbuf[1]; + int uvtemp = 128 << 8; + v4i32 uv = __msa_fill_w(uvtemp); + + for (i = 0; i < len; i += 8) { + v8i16 b, ub, vb; + v4i32 y_r, y_l, u_r, u_l, v_r, v_l; + v4i32 u1_r, u1_l, v1_r, v1_l; + v4i32 R_r, R_l, G_r, G_l, B_r, B_l; + int m = i + 4; + + b = LD_V(v8i16, (buf0 + i)); + ub = LD_V(v8i16, (ubuf0 + i)); + vb = LD_V(v8i16, (vbuf0 + i)); + UNPCK_SH_SW(b, y_r, y_l); + UNPCK_SH_SW(ub, u_r, u_l); + UNPCK_SH_SW(vb, v_r, v_l); + y_r = __msa_slli_w(y_r, 2); + y_l = __msa_slli_w(y_l, 2); + u_r -= uv; + u_l -= uv; + v_r -= uv; + v_l -= uv; + ub = LD_V(v8i16, (ubuf1 + i)); + vb = LD_V(v8i16, (vbuf1 + i)); + UNPCK_SH_SW(ub, u1_r, u1_l); + UNPCK_SH_SW(vb, v1_r, v1_l); + u_r += u1_r; + u_l += u1_l; + v_r += v1_r; + v_l += v1_l; + u_r = __msa_slli_w(u_r, 1); + u_l = __msa_slli_w(u_l, 1); + yuvTorgb + + if(hasAlpha) { + v8i16 a_src; + v4i32 a_r, a_l; + + a_src = LD_V(v8i16, (abuf0 + i)); + UNPCK_SH_SW(a_src, a_r, a_l); + a_r += bias; + a_l += bias; + a_r = __msa_srai_w(a_r, 7); + a_l = __msa_srai_w(a_l, 7); + for (j = 0; j < 4; j++) { + R = R_r[j]; + G = G_r[j]; + B = B_r[j]; + A = a_r[j]; + if (A & 0x100) + A = av_clip_uint8(A); + yuv2rgb_write_full(c, dest, i + j, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + for (j = 0; j < 4; j++) { + R = R_l[j]; + G = G_l[j]; + B = B_l[j]; + A = a_l[j]; + if (A & 0x100) + A = av_clip_uint8(A); + yuv2rgb_write_full(c, dest, m + j, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + } else { + for (j = 0; j < 4; j++) { + R = R_r[j]; + G = G_r[j]; + B = B_r[j]; + yuv2rgb_write_full(c, dest, i + j, R, 0, G, B, y, target, hasAlpha, err); + dest += step; + } + for (j = 0; j < 4; j++) { + R = R_l[j]; + G = G_l[j]; + B = B_l[j]; + yuv2rgb_write_full(c, dest, m + j, R, 0, G, B, y, target, hasAlpha, err); + dest += step; + } + } + } + for (i; i < dstW; i++) { + int Y = buf0[i] << 2; + int U = (ubuf0[i] + ubuf1[i] - uvtemp) << 1; + int V = (vbuf0[i] + vbuf1[i] - uvtemp) << 1; + + A = 0; + if(hasAlpha) { + A = (abuf0[i] + 64) >> 7; + if (A & 0x100) + A = av_clip_uint8(A); + } + Y -= y_offset; + Y *= y_coeff; + Y += ytemp; + R = (unsigned)Y + V * v2r_coe; + G = (unsigned)Y + V * v2g_coe + U * u2g_coe; + B = (unsigned)Y + U * u2b_coe; + yuv2rgb_write_full(c, dest, i, R, A, G, B, y, target, hasAlpha, err); + dest += step; + } + } + c->dither_error[0][i] = err[0]; + c->dither_error[1][i] = err[1]; + c->dither_error[2][i] = err[2]; +} +#if CONFIG_SMALL +YUV2RGBWRAPPER(yuv2, rgb_full, bgra32_full, AV_PIX_FMT_BGRA, CONFIG_SWSCALE_ALPHA && c->needAlpha) +YUV2RGBWRAPPER(yuv2, rgb_full, abgr32_full, AV_PIX_FMT_ABGR, CONFIG_SWSCALE_ALPHA && c->needAlpha) +YUV2RGBWRAPPER(yuv2, rgb_full, rgba32_full, AV_PIX_FMT_RGBA, CONFIG_SWSCALE_ALPHA && c->needAlpha) +YUV2RGBWRAPPER(yuv2, rgb_full, argb32_full, AV_PIX_FMT_ARGB, CONFIG_SWSCALE_ALPHA && c->needAlpha) +#else +#if CONFIG_SWSCALE_ALPHA +YUV2RGBWRAPPER(yuv2, rgb_full, bgra32_full, AV_PIX_FMT_BGRA, 1) +YUV2RGBWRAPPER(yuv2, rgb_full, abgr32_full, AV_PIX_FMT_ABGR, 1) +YUV2RGBWRAPPER(yuv2, rgb_full, rgba32_full, AV_PIX_FMT_RGBA, 1) +YUV2RGBWRAPPER(yuv2, rgb_full, argb32_full, AV_PIX_FMT_ARGB, 1) +#endif +YUV2RGBWRAPPER(yuv2, rgb_full, bgrx32_full, AV_PIX_FMT_BGRA, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, xbgr32_full, AV_PIX_FMT_ABGR, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, rgbx32_full, AV_PIX_FMT_RGBA, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, xrgb32_full, AV_PIX_FMT_ARGB, 0) +#endif +YUV2RGBWRAPPER(yuv2, rgb_full, bgr24_full, AV_PIX_FMT_BGR24, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, rgb24_full, AV_PIX_FMT_RGB24, 0) + +YUV2RGBWRAPPER(yuv2, rgb_full, bgr4_byte_full, AV_PIX_FMT_BGR4_BYTE, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, rgb4_byte_full, AV_PIX_FMT_RGB4_BYTE, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, bgr8_full, AV_PIX_FMT_BGR8, 0) +YUV2RGBWRAPPER(yuv2, rgb_full, rgb8_full, AV_PIX_FMT_RGB8, 0) +#undef yuvTorgb +#undef yuvTorgb_setup + +void planar_rgb_to_uv_msa(uint8_t *_dstU, uint8_t *_dstV, const uint8_t *src[4], + int width, int32_t *rgb2yuv) +{ + uint16_t *dstU = (uint16_t *)_dstU; + uint16_t *dstV = (uint16_t *)_dstV; + int i; + int len = width & (~0x07); + int set = 0x4001<<(RGB2YUV_SHIFT - 7); + int32_t tem_ru = rgb2yuv[RU_IDX], tem_gu = rgb2yuv[GU_IDX]; + int32_t tem_bu = rgb2yuv[BU_IDX]; + int32_t tem_rv = rgb2yuv[RV_IDX], tem_gv = rgb2yuv[GV_IDX]; + int32_t tem_bv = rgb2yuv[BV_IDX]; + int shift = RGB2YUV_SHIFT - 6; + v4i32 ru, gu, bu, rv, gv, bv; + v4i32 temp = __msa_fill_w(set); + v4i32 sra = __msa_fill_w(shift); + v16i8 zero = {0}; + + ru = __msa_fill_w(tem_ru); + gu = __msa_fill_w(tem_gu); + bu = __msa_fill_w(tem_bu); + rv = __msa_fill_w(tem_rv); + gv = __msa_fill_w(tem_gv); + bv = __msa_fill_w(tem_bv); + for (i = 0; i < len; i += 8) { + v16i8 _g, _b, _r; + v8i16 t_g, t_b, t_r; + v4i32 g_r, g_l, b_r, b_l, r_r, r_l; + v4i32 v_r, v_l, u_r, u_l; + + _g = LD_V(v16i8, (src[0] + i)); + _b = LD_V(v16i8, (src[1] + i)); + _r = LD_V(v16i8, (src[2] + i)); + t_g = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)_g); + t_b = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)_b); + t_r = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)_r); + g_r = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)t_g); + g_l = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)t_g); + b_r = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)t_b); + b_l = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)t_b); + r_r = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)t_r); + r_l = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)t_r); + v_r = (v4i32)__msa_mulv_w(r_r, rv); + v_l = (v4i32)__msa_mulv_w(r_l, rv); + v_r = (v4i32)__msa_maddv_w(g_r, gv, v_r); + v_l = (v4i32)__msa_maddv_w(g_l, gv, v_l); + v_r = (v4i32)__msa_maddv_w(b_r, bv, v_r); + v_l = (v4i32)__msa_maddv_w(b_l, bv, v_l); + u_r = (v4i32)__msa_mulv_w(r_r, ru); + u_l = (v4i32)__msa_mulv_w(r_l, ru); + u_r = (v4i32)__msa_maddv_w(g_r, gu, u_r); + u_l = (v4i32)__msa_maddv_w(g_l, gu, u_l); + u_r = (v4i32)__msa_maddv_w(b_r, bu, u_r); + u_l = (v4i32)__msa_maddv_w(b_l, bu, u_l); + v_r = (v4i32)__msa_addv_w(v_r, temp); + v_l = (v4i32)__msa_addv_w(v_l, temp); + u_r = (v4i32)__msa_addv_w(u_r, temp); + u_l = (v4i32)__msa_addv_w(u_l, temp); + v_r = (v4i32)__msa_sra_w(v_r, sra); + v_l = (v4i32)__msa_sra_w(v_l, sra); + u_r = (v4i32)__msa_sra_w(u_r, sra); + u_l = (v4i32)__msa_sra_w(u_l, sra); + for (int j = 0; j < 4; j++) { + int m = i + j; + + dstU[m] = u_r[j]; + dstV[m] = v_r[j]; + dstU[m + 4] = u_l[j]; + dstV[m + 4] = v_l[j]; + } + } + for (i; i < width; i++) { + int g = src[0][i]; + int b = src[1][i]; + int r = src[2][i]; + + dstU[i] = (tem_ru * r + tem_gu * g + tem_bu * b + set) >> shift; + dstV[i] = (tem_rv * r + tem_gv * g + tem_bv * b + set) >> shift; + } +} + +void planar_rgb_to_y_msa(uint8_t *_dst, const uint8_t *src[4], int width, + int32_t *rgb2yuv) +{ + uint16_t *dst = (uint16_t *)_dst; + int32_t tem_ry = rgb2yuv[RY_IDX], tem_gy = rgb2yuv[GY_IDX]; + int32_t tem_by = rgb2yuv[BY_IDX]; + int len = width & (~0x07); + int shift = (RGB2YUV_SHIFT-6); + int set = 0x801 << (RGB2YUV_SHIFT - 7); + int i; + v4i32 temp = (v4i32)__msa_fill_w(set); + v4i32 sra = (v4i32)__msa_fill_w(shift); + v4i32 ry = (v4i32)__msa_fill_w(tem_ry); + v4i32 gy = (v4i32)__msa_fill_w(tem_gy); + v4i32 by = (v4i32)__msa_fill_w(tem_by); + v16i8 zero = {0}; + + for (i = 0; i < len; i += 8) { + v16i8 _g, _b, _r; + v8i16 t_g, t_b, t_r; + v4i32 g_r, g_l, b_r, b_l, r_r, r_l; + v4i32 out_r, out_l; + + _g = LD_V(v16i8, src[0] + i); + _b = LD_V(v16i8, src[1] + i); + _r = LD_V(v16i8, src[2] + i); + t_g = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)_g); + t_b = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)_b); + t_r = (v8i16)__msa_ilvr_b((v16i8)zero, (v16i8)_r); + g_r = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)t_g); + g_l = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)t_g); + b_r = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)t_b); + b_l = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)t_b); + r_r = (v4i32)__msa_ilvr_h((v8i16)zero, (v8i16)t_r); + r_l = (v4i32)__msa_ilvl_h((v8i16)zero, (v8i16)t_r); + out_r = (v4i32)__msa_mulv_w(r_r, ry); + out_l = (v4i32)__msa_mulv_w(r_l, ry); + out_r = (v4i32)__msa_maddv_w(g_r, gy, out_r); + out_l = (v4i32)__msa_maddv_w(g_l, gy, out_l); + out_r = (v4i32)__msa_maddv_w(b_r, by, out_r); + out_l = (v4i32)__msa_maddv_w(b_l, by, out_l); + out_r = (v4i32)__msa_addv_w(out_r, temp); + out_l = (v4i32)__msa_addv_w(out_l, temp); + out_r = (v4i32)__msa_sra_w(out_r, sra); + out_l = (v4i32)__msa_sra_w(out_l, sra); + for (int j = 0; j < 4; j++) { + int m = i + j; + dst[m] = out_r[j]; + dst[m + 4] = out_l[j]; + } + } + for (i; i < width; i++) { + int g = src[0][i]; + int b = src[1][i]; + int r = src[2][i]; + + dst[i] = (tem_ry * r + tem_gy * g + tem_by * b + set) >> shift; + } +} diff --git a/libswscale/rgb2rgb.c b/libswscale/rgb2rgb.c index eab8e6aebb..b9cc922ec7 100644 --- a/libswscale/rgb2rgb.c +++ b/libswscale/rgb2rgb.c @@ -139,6 +139,10 @@ av_cold void ff_sws_rgb2rgb_init(void) rgb2rgb_init_c(); if (ARCH_X86) rgb2rgb_init_x86(); + if (ARCH_MIPS) + rgb2rgb_init_mips(); + if (ARCH_LOONGARCH) + rgb2rgb_init_loongarch(); } void rgb32to24(const uint8_t *src, uint8_t *dst, int src_size) diff --git a/libswscale/rgb2rgb.h b/libswscale/rgb2rgb.h index 3569254df9..c092f52a68 100644 --- a/libswscale/rgb2rgb.h +++ b/libswscale/rgb2rgb.h @@ -170,5 +170,7 @@ extern void (*yuyvtoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const u void ff_sws_rgb2rgb_init(void); void rgb2rgb_init_x86(void); +void rgb2rgb_init_mips(void); +void rgb2rgb_init_loongarch(void); #endif /* SWSCALE_RGB2RGB_H */ diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 36f7aa9a03..b5d77e7bfc 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -607,6 +607,10 @@ SwsFunc ff_getSwsFunc(SwsContext *c) ff_sws_init_swscale_aarch64(c); if (ARCH_ARM) ff_sws_init_swscale_arm(c); + if (ARCH_MIPS) + ff_sws_init_swscale_mips(c); + if (ARCH_LOONGARCH) + ff_sws_init_swscale_loongarch(c); return swscale; } diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index 4fa59386a6..1d2932dc43 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -642,6 +642,7 @@ av_cold void ff_sws_init_range_convert(SwsContext *c); SwsFunc ff_yuv2rgb_init_x86(SwsContext *c); SwsFunc ff_yuv2rgb_init_ppc(SwsContext *c); +SwsFunc ff_yuv2rgb_init_loongarch(SwsContext *c); static av_always_inline int is16BPS(enum AVPixelFormat pix_fmt) { @@ -871,6 +872,8 @@ void ff_sws_init_swscale_ppc(SwsContext *c); void ff_sws_init_swscale_x86(SwsContext *c); void ff_sws_init_swscale_aarch64(SwsContext *c); void ff_sws_init_swscale_arm(SwsContext *c); +void ff_sws_init_swscale_mips(SwsContext *c); +void ff_sws_init_swscale_loongarch(SwsContext *c); void ff_hyscale_fast_c(SwsContext *c, int16_t *dst, int dstWidth, const uint8_t *src, int srcW, int xInc); diff --git a/libswscale/utils.c b/libswscale/utils.c index 2484cdf8d7..2bb32a1661 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -53,6 +53,8 @@ #include "libavutil/ppc/cpu.h" #include "libavutil/x86/asm.h" #include "libavutil/x86/cpu.h" +#include "libavutil/mips/cpu.h" +#include "libavutil/loongarch/cpu.h" // We have to implement deprecated functions until they are removed, this is the // simplest way to prevent warnings @@ -589,6 +591,24 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos, filterAlign = 1; } + if (have_msa(cpu_flags)) { + int reNum = minFilterSize & (0x07); + + if (minFilterSize < 5) + filterAlign = 4; + if (reNum < 3) + filterAlign = 1; + } + + if (have_lasx(cpu_flags)) { + int reNum = minFilterSize & (0x07); + + if (minFilterSize < 5) + filterAlign = 4; + if (reNum < 3) + filterAlign = 1; + } + if (HAVE_MMX && cpu_flags & AV_CPU_FLAG_MMX) { // special case for unscaled vertical filtering if (minFilterSize == 1 && filterAlign == 2) @@ -1665,7 +1685,9 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter, { const int filterAlign = X86_MMX(cpu_flags) ? 4 : PPC_ALTIVEC(cpu_flags) ? 8 : - have_neon(cpu_flags) ? 8 : 1; + have_neon(cpu_flags) ? 8 : + have_lasx(cpu_flags) ? 8 : + have_msa(cpu_flags) ? 8 : 1; if ((ret = initFilter(&c->hLumFilter, &c->hLumFilterPos, &c->hLumFilterSize, c->lumXInc, diff --git a/libswscale/yuv2rgb.c b/libswscale/yuv2rgb.c index 9604019464..f7edfd3d0c 100644 --- a/libswscale/yuv2rgb.c +++ b/libswscale/yuv2rgb.c @@ -684,6 +684,8 @@ SwsFunc ff_yuv2rgb_get_func_ptr(SwsContext *c) t = ff_yuv2rgb_init_ppc(c); if (ARCH_X86) t = ff_yuv2rgb_init_x86(c); + if (ARCH_LOONGARCH) + t = ff_yuv2rgb_init_loongarch(c); if (t) return t; diff --git a/tests/checkasm/checkasm.c b/tests/checkasm/checkasm.c index ec00723b9b..b68c6b4aa0 100644 --- a/tests/checkasm/checkasm.c +++ b/tests/checkasm/checkasm.c @@ -218,6 +218,9 @@ static const struct { { "FMA4", "fma4", AV_CPU_FLAG_FMA4 }, { "AVX2", "avx2", AV_CPU_FLAG_AVX2 }, { "AVX-512", "avx512", AV_CPU_FLAG_AVX512 }, +#elif ARCH_LOONGARCH + { "LSX", "lsx", AV_CPU_FLAG_LSX }, + { "LASX", "lasx", AV_CPU_FLAG_LASX }, #endif { NULL } }; -- 2.20.1