Go to the documentation of this file. 21 #ifndef AVUTIL_MIPS_GENERIC_MACROS_MSA_H 22 #define AVUTIL_MIPS_GENERIC_MACROS_MSA_H 33 #define ALLOC_ALIGNED(align) __attribute__ ((aligned((align) << 1))) 35 #define LD_V(RTYPE, psrc) *((RTYPE *)(psrc)) 36 #define LD_UB(...) LD_V(v16u8, __VA_ARGS__) 37 #define LD_SB(...) LD_V(v16i8, __VA_ARGS__) 38 #define LD_UH(...) LD_V(v8u16, __VA_ARGS__) 39 #define LD_SH(...) LD_V(v8i16, __VA_ARGS__) 40 #define LD_UW(...) LD_V(v4u32, __VA_ARGS__) 41 #define LD_SW(...) LD_V(v4i32, __VA_ARGS__) 43 #define ST_V(RTYPE, in, pdst) *((RTYPE *)(pdst)) = (in) 44 #define ST_UB(...) ST_V(v16u8, __VA_ARGS__) 45 #define ST_SB(...) ST_V(v16i8, __VA_ARGS__) 46 #define ST_UH(...) ST_V(v8u16, __VA_ARGS__) 47 #define ST_SH(...) ST_V(v8i16, __VA_ARGS__) 48 #define ST_UW(...) ST_V(v4u32, __VA_ARGS__) 49 #define ST_SW(...) ST_V(v4i32, __VA_ARGS__) 51 #if (__mips_isa_rev >= 6) 54 uint16_t val_lh_m = *(uint16_t *)(psrc); \ 60 uint32_t val_lw_m = *(uint32_t *)(psrc); \ 67 uint64_t val_ld_m = *(uint64_t *)(psrc); \ 70 #else // !(__mips == 64) 73 uint8_t *psrc_ld_m = (uint8_t *) (psrc); \ 74 uint32_t val0_ld_m, val1_ld_m; \ 75 uint64_t val_ld_m = 0; \ 77 val0_ld_m = LW(psrc_ld_m); \ 78 val1_ld_m = LW(psrc_ld_m + 4); \ 80 val_ld_m = (uint64_t) (val1_ld_m); \ 81 val_ld_m = (uint64_t) ((val_ld_m << 32) & 0xFFFFFFFF00000000); \ 82 val_ld_m = (uint64_t) (val_ld_m | (uint64_t) val0_ld_m); \ 86 #endif // (__mips == 64) 88 #define SH(val, pdst) *(uint16_t *)(pdst) = (val); 89 #define SW(val, pdst) *(uint32_t *)(pdst) = (val); 90 #define SD(val, pdst) *(uint64_t *)(pdst) = (val); 92 #else // !(__mips_isa_rev >= 6) 95 uint8_t *psrc_lh_m = (uint8_t *) (psrc); \ 99 "ulh %[val_lh_m], %[psrc_lh_m] \n\t" \ 101 : [val_lh_m] "=r" (val_lh_m) \ 102 : [psrc_lh_m] "m" (*psrc_lh_m) \ 110 uint8_t *psrc_lw_m = (uint8_t *) (psrc); \ 114 "ulw %[val_lw_m], %[psrc_lw_m] \n\t" \ 116 : [val_lw_m] "=r" (val_lw_m) \ 117 : [psrc_lw_m] "m" (*psrc_lw_m) \ 126 uint8_t *psrc_ld_m = (uint8_t *) (psrc); \ 127 uint64_t val_ld_m = 0; \ 130 "uld %[val_ld_m], %[psrc_ld_m] \n\t" \ 132 : [val_ld_m] "=r" (val_ld_m) \ 133 : [psrc_ld_m] "m" (*psrc_ld_m) \ 138 #else // !(__mips == 64) 141 uint8_t *psrc_ld_m = (uint8_t *) (psrc); \ 142 uint32_t val0_ld_m, val1_ld_m; \ 143 uint64_t val_ld_m = 0; \ 145 val0_ld_m = LW(psrc_ld_m); \ 146 val1_ld_m = LW(psrc_ld_m + 4); \ 148 val_ld_m = (uint64_t) (val1_ld_m); \ 149 val_ld_m = (uint64_t) ((val_ld_m << 32) & 0xFFFFFFFF00000000); \ 150 val_ld_m = (uint64_t) (val_ld_m | (uint64_t) val0_ld_m); \ 154 #endif // (__mips == 64) 156 #define SH(val, pdst) \ 158 uint8_t *pdst_sh_m = (uint8_t *) (pdst); \ 159 uint16_t val_sh_m = (val); \ 162 "ush %[val_sh_m], %[pdst_sh_m] \n\t" \ 164 : [pdst_sh_m] "=m" (*pdst_sh_m) \ 165 : [val_sh_m] "r" (val_sh_m) \ 169 #define SW(val, pdst) \ 171 uint8_t *pdst_sw_m = (uint8_t *) (pdst); \ 172 uint32_t val_sw_m = (val); \ 175 "usw %[val_sw_m], %[pdst_sw_m] \n\t" \ 177 : [pdst_sw_m] "=m" (*pdst_sw_m) \ 178 : [val_sw_m] "r" (val_sw_m) \ 182 #define SD(val, pdst) \ 184 uint8_t *pdst_sd_m = (uint8_t *) (pdst); \ 185 uint32_t val0_sd_m, val1_sd_m; \ 187 val0_sd_m = (uint32_t) ((val) & 0x00000000FFFFFFFF); \ 188 val1_sd_m = (uint32_t) (((val) >> 32) & 0x00000000FFFFFFFF); \ 190 SW(val0_sd_m, pdst_sd_m); \ 191 SW(val1_sd_m, pdst_sd_m + 4); \ 193 #endif // (__mips_isa_rev >= 6) 204 #define LW4(psrc, stride, out0, out1, out2, out3) \ 207 out1 = LW((psrc) + stride); \ 208 out2 = LW((psrc) + 2 * stride); \ 209 out3 = LW((psrc) + 3 * stride); \ 212 #define LW2(psrc, stride, out0, out1) \ 215 out1 = LW((psrc) + stride); \ 225 #define LD2(psrc, stride, out0, out1) \ 228 out1 = LD((psrc) + stride); \ 230 #define LD4(psrc, stride, out0, out1, out2, out3) \ 232 LD2((psrc), stride, out0, out1); \ 233 LD2((psrc) + 2 * stride, stride, out2, out3); \ 243 #define SW4(in0, in1, in2, in3, pdst, stride) \ 246 SW(in1, (pdst) + stride); \ 247 SW(in2, (pdst) + 2 * stride); \ 248 SW(in3, (pdst) + 3 * stride); \ 258 #define SD4(in0, in1, in2, in3, pdst, stride) \ 261 SD(in1, (pdst) + stride); \ 262 SD(in2, (pdst) + 2 * stride); \ 263 SD(in3, (pdst) + 3 * stride); \ 274 #define LD_V2(RTYPE, psrc, stride, out0, out1) \ 276 out0 = LD_V(RTYPE, (psrc)); \ 277 out1 = LD_V(RTYPE, (psrc) + stride); \ 279 #define LD_UB2(...) LD_V2(v16u8, __VA_ARGS__) 280 #define LD_SB2(...) LD_V2(v16i8, __VA_ARGS__) 281 #define LD_UH2(...) LD_V2(v8u16, __VA_ARGS__) 282 #define LD_SH2(...) LD_V2(v8i16, __VA_ARGS__) 283 #define LD_SW2(...) LD_V2(v4i32, __VA_ARGS__) 285 #define LD_V3(RTYPE, psrc, stride, out0, out1, out2) \ 287 LD_V2(RTYPE, (psrc), stride, out0, out1); \ 288 out2 = LD_V(RTYPE, (psrc) + 2 * stride); \ 290 #define LD_UB3(...) LD_V3(v16u8, __VA_ARGS__) 291 #define LD_SB3(...) LD_V3(v16i8, __VA_ARGS__) 293 #define LD_V4(RTYPE, psrc, stride, out0, out1, out2, out3) \ 295 LD_V2(RTYPE, (psrc), stride, out0, out1); \ 296 LD_V2(RTYPE, (psrc) + 2 * stride , stride, out2, out3); \ 298 #define LD_UB4(...) LD_V4(v16u8, __VA_ARGS__) 299 #define LD_SB4(...) LD_V4(v16i8, __VA_ARGS__) 300 #define LD_UH4(...) LD_V4(v8u16, __VA_ARGS__) 301 #define LD_SH4(...) LD_V4(v8i16, __VA_ARGS__) 303 #define LD_V5(RTYPE, psrc, stride, out0, out1, out2, out3, out4) \ 305 LD_V4(RTYPE, (psrc), stride, out0, out1, out2, out3); \ 306 out4 = LD_V(RTYPE, (psrc) + 4 * stride); \ 308 #define LD_UB5(...) LD_V5(v16u8, __VA_ARGS__) 309 #define LD_SB5(...) LD_V5(v16i8, __VA_ARGS__) 311 #define LD_V6(RTYPE, psrc, stride, out0, out1, out2, out3, out4, out5) \ 313 LD_V4(RTYPE, (psrc), stride, out0, out1, out2, out3); \ 314 LD_V2(RTYPE, (psrc) + 4 * stride, stride, out4, out5); \ 316 #define LD_UB6(...) LD_V6(v16u8, __VA_ARGS__) 317 #define LD_SB6(...) LD_V6(v16i8, __VA_ARGS__) 318 #define LD_UH6(...) LD_V6(v8u16, __VA_ARGS__) 319 #define LD_SH6(...) LD_V6(v8i16, __VA_ARGS__) 321 #define LD_V7(RTYPE, psrc, stride, \ 322 out0, out1, out2, out3, out4, out5, out6) \ 324 LD_V5(RTYPE, (psrc), stride, out0, out1, out2, out3, out4); \ 325 LD_V2(RTYPE, (psrc) + 5 * stride, stride, out5, out6); \ 327 #define LD_UB7(...) LD_V7(v16u8, __VA_ARGS__) 328 #define LD_SB7(...) LD_V7(v16i8, __VA_ARGS__) 330 #define LD_V8(RTYPE, psrc, stride, \ 331 out0, out1, out2, out3, out4, out5, out6, out7) \ 333 LD_V4(RTYPE, (psrc), stride, out0, out1, out2, out3); \ 334 LD_V4(RTYPE, (psrc) + 4 * stride, stride, out4, out5, out6, out7); \ 336 #define LD_UB8(...) LD_V8(v16u8, __VA_ARGS__) 337 #define LD_SB8(...) LD_V8(v16i8, __VA_ARGS__) 338 #define LD_UH8(...) LD_V8(v8u16, __VA_ARGS__) 339 #define LD_SH8(...) LD_V8(v8i16, __VA_ARGS__) 341 #define LD_V16(RTYPE, psrc, stride, \ 342 out0, out1, out2, out3, out4, out5, out6, out7, \ 343 out8, out9, out10, out11, out12, out13, out14, out15) \ 345 LD_V8(RTYPE, (psrc), stride, \ 346 out0, out1, out2, out3, out4, out5, out6, out7); \ 347 LD_V8(RTYPE, (psrc) + 8 * stride, stride, \ 348 out8, out9, out10, out11, out12, out13, out14, out15); \ 350 #define LD_SH16(...) LD_V16(v8i16, __VA_ARGS__) 358 #define ST_V2(RTYPE, in0, in1, pdst, stride) \ 360 ST_V(RTYPE, in0, (pdst)); \ 361 ST_V(RTYPE, in1, (pdst) + stride); \ 363 #define ST_UB2(...) ST_V2(v16u8, __VA_ARGS__) 364 #define ST_SB2(...) ST_V2(v16i8, __VA_ARGS__) 365 #define ST_UH2(...) ST_V2(v8u16, __VA_ARGS__) 366 #define ST_SH2(...) ST_V2(v8i16, __VA_ARGS__) 367 #define ST_SW2(...) ST_V2(v4i32, __VA_ARGS__) 369 #define ST_V4(RTYPE, in0, in1, in2, in3, pdst, stride) \ 371 ST_V2(RTYPE, in0, in1, (pdst), stride); \ 372 ST_V2(RTYPE, in2, in3, (pdst) + 2 * stride, stride); \ 374 #define ST_UB4(...) ST_V4(v16u8, __VA_ARGS__) 375 #define ST_SB4(...) ST_V4(v16i8, __VA_ARGS__) 376 #define ST_SH4(...) ST_V4(v8i16, __VA_ARGS__) 377 #define ST_SW4(...) ST_V4(v4i32, __VA_ARGS__) 379 #define ST_V6(RTYPE, in0, in1, in2, in3, in4, in5, pdst, stride) \ 381 ST_V4(RTYPE, in0, in1, in2, in3, (pdst), stride); \ 382 ST_V2(RTYPE, in4, in5, (pdst) + 4 * stride, stride); \ 384 #define ST_SH6(...) ST_V6(v8i16, __VA_ARGS__) 386 #define ST_V8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 388 ST_V4(RTYPE, in0, in1, in2, in3, (pdst), stride); \ 389 ST_V4(RTYPE, in4, in5, in6, in7, (pdst) + 4 * stride, stride); \ 391 #define ST_UB8(...) ST_V8(v16u8, __VA_ARGS__) 392 #define ST_SH8(...) ST_V8(v8i16, __VA_ARGS__) 393 #define ST_SW8(...) ST_V8(v4i32, __VA_ARGS__) 403 #define ST_H1(in, idx, pdst) \ 406 out0_m = __msa_copy_u_h((v8i16) in, idx); \ 407 SH(out0_m, (pdst)); \ 409 #define ST_H2(in, idx0, idx1, pdst, stride) \ 411 uint16_t out0_m, out1_m; \ 412 out0_m = __msa_copy_u_h((v8i16) in, idx0); \ 413 out1_m = __msa_copy_u_h((v8i16) in, idx1); \ 414 SH(out0_m, (pdst)); \ 415 SH(out1_m, (pdst) + stride); \ 417 #define ST_H4(in, idx0, idx1, idx2, idx3, pdst, stride) \ 419 uint16_t out0_m, out1_m, out2_m, out3_m; \ 420 out0_m = __msa_copy_u_h((v8i16) in, idx0); \ 421 out1_m = __msa_copy_u_h((v8i16) in, idx1); \ 422 out2_m = __msa_copy_u_h((v8i16) in, idx2); \ 423 out3_m = __msa_copy_u_h((v8i16) in, idx3); \ 424 SH(out0_m, (pdst)); \ 425 SH(out1_m, (pdst) + stride); \ 426 SH(out2_m, (pdst) + 2 * stride); \ 427 SH(out3_m, (pdst) + 3 * stride); \ 429 #define ST_H8(in, idx0, idx1, idx2, idx3, idx4, idx5, \ 430 idx6, idx7, pdst, stride) \ 432 ST_H4(in, idx0, idx1, idx2, idx3, pdst, stride) \ 433 ST_H4(in, idx4, idx5, idx6, idx7, (pdst) + 4*stride, stride) \ 444 #define ST_W1(in, idx, pdst) \ 447 out0_m = __msa_copy_u_w((v4i32) in, idx); \ 448 SW(out0_m, (pdst)); \ 450 #define ST_W2(in, idx0, idx1, pdst, stride) \ 452 uint32_t out0_m, out1_m; \ 453 out0_m = __msa_copy_u_w((v4i32) in, idx0); \ 454 out1_m = __msa_copy_u_w((v4i32) in, idx1); \ 455 SW(out0_m, (pdst)); \ 456 SW(out1_m, (pdst) + stride); \ 458 #define ST_W4(in, idx0, idx1, idx2, idx3, pdst, stride) \ 460 uint32_t out0_m, out1_m, out2_m, out3_m; \ 461 out0_m = __msa_copy_u_w((v4i32) in, idx0); \ 462 out1_m = __msa_copy_u_w((v4i32) in, idx1); \ 463 out2_m = __msa_copy_u_w((v4i32) in, idx2); \ 464 out3_m = __msa_copy_u_w((v4i32) in, idx3); \ 465 SW(out0_m, (pdst)); \ 466 SW(out1_m, (pdst) + stride); \ 467 SW(out2_m, (pdst) + 2*stride); \ 468 SW(out3_m, (pdst) + 3*stride); \ 470 #define ST_W8(in0, in1, idx0, idx1, idx2, idx3, \ 471 idx4, idx5, idx6, idx7, pdst, stride) \ 473 ST_W4(in0, idx0, idx1, idx2, idx3, pdst, stride) \ 474 ST_W4(in1, idx4, idx5, idx6, idx7, pdst + 4*stride, stride) \ 485 #define ST_D1(in, idx, pdst) \ 488 out0_m = __msa_copy_u_d((v2i64) in, idx); \ 489 SD(out0_m, (pdst)); \ 491 #define ST_D2(in, idx0, idx1, pdst, stride) \ 493 uint64_t out0_m, out1_m; \ 494 out0_m = __msa_copy_u_d((v2i64) in, idx0); \ 495 out1_m = __msa_copy_u_d((v2i64) in, idx1); \ 496 SD(out0_m, (pdst)); \ 497 SD(out1_m, (pdst) + stride); \ 499 #define ST_D4(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) \ 501 uint64_t out0_m, out1_m, out2_m, out3_m; \ 502 out0_m = __msa_copy_u_d((v2i64) in0, idx0); \ 503 out1_m = __msa_copy_u_d((v2i64) in0, idx1); \ 504 out2_m = __msa_copy_u_d((v2i64) in1, idx2); \ 505 out3_m = __msa_copy_u_d((v2i64) in1, idx3); \ 506 SD(out0_m, (pdst)); \ 507 SD(out1_m, (pdst) + stride); \ 508 SD(out2_m, (pdst) + 2 * stride); \ 509 SD(out3_m, (pdst) + 3 * stride); \ 511 #define ST_D8(in0, in1, in2, in3, idx0, idx1, idx2, idx3, \ 512 idx4, idx5, idx6, idx7, pdst, stride) \ 514 ST_D4(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) \ 515 ST_D4(in2, in3, idx4, idx5, idx6, idx7, pdst + 4 * stride, stride) \ 527 #define ST12x8_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 529 uint64_t out0_m, out1_m, out2_m, out3_m; \ 530 uint64_t out4_m, out5_m, out6_m, out7_m; \ 531 uint32_t out8_m, out9_m, out10_m, out11_m; \ 532 uint32_t out12_m, out13_m, out14_m, out15_m; \ 533 uint8_t *pblk_12x8_m = (uint8_t *) (pdst); \ 535 out0_m = __msa_copy_u_d((v2i64) in0, 0); \ 536 out1_m = __msa_copy_u_d((v2i64) in1, 0); \ 537 out2_m = __msa_copy_u_d((v2i64) in2, 0); \ 538 out3_m = __msa_copy_u_d((v2i64) in3, 0); \ 539 out4_m = __msa_copy_u_d((v2i64) in4, 0); \ 540 out5_m = __msa_copy_u_d((v2i64) in5, 0); \ 541 out6_m = __msa_copy_u_d((v2i64) in6, 0); \ 542 out7_m = __msa_copy_u_d((v2i64) in7, 0); \ 544 out8_m = __msa_copy_u_w((v4i32) in0, 2); \ 545 out9_m = __msa_copy_u_w((v4i32) in1, 2); \ 546 out10_m = __msa_copy_u_w((v4i32) in2, 2); \ 547 out11_m = __msa_copy_u_w((v4i32) in3, 2); \ 548 out12_m = __msa_copy_u_w((v4i32) in4, 2); \ 549 out13_m = __msa_copy_u_w((v4i32) in5, 2); \ 550 out14_m = __msa_copy_u_w((v4i32) in6, 2); \ 551 out15_m = __msa_copy_u_w((v4i32) in7, 2); \ 553 SD(out0_m, pblk_12x8_m); \ 554 SW(out8_m, pblk_12x8_m + 8); \ 555 pblk_12x8_m += stride; \ 556 SD(out1_m, pblk_12x8_m); \ 557 SW(out9_m, pblk_12x8_m + 8); \ 558 pblk_12x8_m += stride; \ 559 SD(out2_m, pblk_12x8_m); \ 560 SW(out10_m, pblk_12x8_m + 8); \ 561 pblk_12x8_m += stride; \ 562 SD(out3_m, pblk_12x8_m); \ 563 SW(out11_m, pblk_12x8_m + 8); \ 564 pblk_12x8_m += stride; \ 565 SD(out4_m, pblk_12x8_m); \ 566 SW(out12_m, pblk_12x8_m + 8); \ 567 pblk_12x8_m += stride; \ 568 SD(out5_m, pblk_12x8_m); \ 569 SW(out13_m, pblk_12x8_m + 8); \ 570 pblk_12x8_m += stride; \ 571 SD(out6_m, pblk_12x8_m); \ 572 SW(out14_m, pblk_12x8_m + 8); \ 573 pblk_12x8_m += stride; \ 574 SD(out7_m, pblk_12x8_m); \ 575 SW(out15_m, pblk_12x8_m + 8); \ 590 #define AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) \ 592 out0 = (RTYPE) __msa_aver_u_b((v16u8) in0, (v16u8) in1); \ 593 out1 = (RTYPE) __msa_aver_u_b((v16u8) in2, (v16u8) in3); \ 595 #define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__) 597 #define AVER_UB4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 598 out0, out1, out2, out3) \ 600 AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) \ 601 AVER_UB2(RTYPE, in4, in5, in6, in7, out2, out3) \ 603 #define AVER_UB4_UB(...) AVER_UB4(v16u8, __VA_ARGS__) 612 #define SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val) \ 614 v16i8 zero_m = { 0 }; \ 615 out0 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in0, slide_val); \ 616 out1 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in1, slide_val); \ 618 #define SLDI_B2_0_UB(...) SLDI_B2_0(v16u8, __VA_ARGS__) 619 #define SLDI_B2_0_SB(...) SLDI_B2_0(v16i8, __VA_ARGS__) 620 #define SLDI_B2_0_SW(...) SLDI_B2_0(v4i32, __VA_ARGS__) 622 #define SLDI_B3_0(RTYPE, in0, in1, in2, out0, out1, out2, slide_val) \ 624 v16i8 zero_m = { 0 }; \ 625 SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val); \ 626 out2 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in2, slide_val); \ 628 #define SLDI_B3_0_UB(...) SLDI_B3_0(v16u8, __VA_ARGS__) 629 #define SLDI_B3_0_SB(...) SLDI_B3_0(v16i8, __VA_ARGS__) 631 #define SLDI_B4_0(RTYPE, in0, in1, in2, in3, \ 632 out0, out1, out2, out3, slide_val) \ 634 SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val); \ 635 SLDI_B2_0(RTYPE, in2, in3, out2, out3, slide_val); \ 637 #define SLDI_B4_0_UB(...) SLDI_B4_0(v16u8, __VA_ARGS__) 638 #define SLDI_B4_0_SB(...) SLDI_B4_0(v16i8, __VA_ARGS__) 639 #define SLDI_B4_0_SH(...) SLDI_B4_0(v8i16, __VA_ARGS__) 648 #define SLDI_B2(RTYPE, in0_0, in0_1, in1_0, in1_1, out0, out1, slide_val) \ 650 out0 = (RTYPE) __msa_sldi_b((v16i8) in0_0, (v16i8) in1_0, slide_val); \ 651 out1 = (RTYPE) __msa_sldi_b((v16i8) in0_1, (v16i8) in1_1, slide_val); \ 653 #define SLDI_B2_UB(...) SLDI_B2(v16u8, __VA_ARGS__) 654 #define SLDI_B2_SB(...) SLDI_B2(v16i8, __VA_ARGS__) 655 #define SLDI_B2_SH(...) SLDI_B2(v8i16, __VA_ARGS__) 657 #define SLDI_B3(RTYPE, in0_0, in0_1, in0_2, in1_0, in1_1, in1_2, \ 658 out0, out1, out2, slide_val) \ 660 SLDI_B2(RTYPE, in0_0, in0_1, in1_0, in1_1, out0, out1, slide_val) \ 661 out2 = (RTYPE) __msa_sldi_b((v16i8) in0_2, (v16i8) in1_2, slide_val); \ 663 #define SLDI_B3_SB(...) SLDI_B3(v16i8, __VA_ARGS__) 664 #define SLDI_B3_UH(...) SLDI_B3(v8u16, __VA_ARGS__) 675 #define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) \ 677 out0 = (RTYPE) __msa_vshf_b((v16i8) mask0, (v16i8) in1, (v16i8) in0); \ 678 out1 = (RTYPE) __msa_vshf_b((v16i8) mask1, (v16i8) in3, (v16i8) in2); \ 680 #define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__) 681 #define VSHF_B2_SB(...) VSHF_B2(v16i8, __VA_ARGS__) 682 #define VSHF_B2_UH(...) VSHF_B2(v8u16, __VA_ARGS__) 683 #define VSHF_B2_SH(...) VSHF_B2(v8i16, __VA_ARGS__) 685 #define VSHF_B3(RTYPE, in0, in1, in2, in3, in4, in5, mask0, mask1, mask2, \ 688 VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1); \ 689 out2 = (RTYPE) __msa_vshf_b((v16i8) mask2, (v16i8) in5, (v16i8) in4); \ 691 #define VSHF_B3_SB(...) VSHF_B3(v16i8, __VA_ARGS__) 693 #define VSHF_B4(RTYPE, in0, in1, mask0, mask1, mask2, mask3, \ 694 out0, out1, out2, out3) \ 696 VSHF_B2(RTYPE, in0, in1, in0, in1, mask0, mask1, out0, out1); \ 697 VSHF_B2(RTYPE, in0, in1, in0, in1, mask2, mask3, out2, out3); \ 699 #define VSHF_B4_SB(...) VSHF_B4(v16i8, __VA_ARGS__) 700 #define VSHF_B4_SH(...) VSHF_B4(v8i16, __VA_ARGS__) 711 #define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) \ 713 out0 = (RTYPE) __msa_vshf_h((v8i16) mask0, (v8i16) in1, (v8i16) in0); \ 714 out1 = (RTYPE) __msa_vshf_h((v8i16) mask1, (v8i16) in3, (v8i16) in2); \ 716 #define VSHF_H2_SH(...) VSHF_H2(v8i16, __VA_ARGS__) 718 #define VSHF_H3(RTYPE, in0, in1, in2, in3, in4, in5, mask0, mask1, mask2, \ 721 VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1); \ 722 out2 = (RTYPE) __msa_vshf_h((v8i16) mask2, (v8i16) in5, (v8i16) in4); \ 724 #define VSHF_H3_SH(...) VSHF_H3(v8i16, __VA_ARGS__) 735 #define VSHF_W2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) \ 737 out0 = (RTYPE) __msa_vshf_w((v4i32) mask0, (v4i32) in1, (v4i32) in0); \ 738 out1 = (RTYPE) __msa_vshf_w((v4i32) mask1, (v4i32) in3, (v4i32) in2); \ 740 #define VSHF_W2_SB(...) VSHF_W2(v16i8, __VA_ARGS__) 754 #define DOTP_UB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 756 out0 = (RTYPE) __msa_dotp_u_h((v16u8) mult0, (v16u8) cnst0); \ 757 out1 = (RTYPE) __msa_dotp_u_h((v16u8) mult1, (v16u8) cnst1); \ 759 #define DOTP_UB2_UH(...) DOTP_UB2(v8u16, __VA_ARGS__) 761 #define DOTP_UB4(RTYPE, mult0, mult1, mult2, mult3, \ 762 cnst0, cnst1, cnst2, cnst3, \ 763 out0, out1, out2, out3) \ 765 DOTP_UB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 766 DOTP_UB2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 768 #define DOTP_UB4_UH(...) DOTP_UB4(v8u16, __VA_ARGS__) 782 #define DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 784 out0 = (RTYPE) __msa_dotp_s_h((v16i8) mult0, (v16i8) cnst0); \ 785 out1 = (RTYPE) __msa_dotp_s_h((v16i8) mult1, (v16i8) cnst1); \ 787 #define DOTP_SB2_SH(...) DOTP_SB2(v8i16, __VA_ARGS__) 789 #define DOTP_SB3(RTYPE, mult0, mult1, mult2, cnst0, cnst1, cnst2, \ 792 DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 793 out2 = (RTYPE) __msa_dotp_s_h((v16i8) mult2, (v16i8) cnst2); \ 795 #define DOTP_SB3_SH(...) DOTP_SB3(v8i16, __VA_ARGS__) 797 #define DOTP_SB4(RTYPE, mult0, mult1, mult2, mult3, \ 798 cnst0, cnst1, cnst2, cnst3, out0, out1, out2, out3) \ 800 DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 801 DOTP_SB2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 803 #define DOTP_SB4_SH(...) DOTP_SB4(v8i16, __VA_ARGS__) 817 #define DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 819 out0 = (RTYPE) __msa_dotp_s_w((v8i16) mult0, (v8i16) cnst0); \ 820 out1 = (RTYPE) __msa_dotp_s_w((v8i16) mult1, (v8i16) cnst1); \ 822 #define DOTP_SH2_SW(...) DOTP_SH2(v4i32, __VA_ARGS__) 824 #define DOTP_SH4(RTYPE, mult0, mult1, mult2, mult3, \ 825 cnst0, cnst1, cnst2, cnst3, \ 826 out0, out1, out2, out3) \ 828 DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 829 DOTP_SH2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 831 #define DOTP_SH4_SW(...) DOTP_SH4(v4i32, __VA_ARGS__) 845 #define DPADD_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 847 out0 = (RTYPE) __msa_dpadd_s_h((v8i16) out0, \ 848 (v16i8) mult0, (v16i8) cnst0); \ 849 out1 = (RTYPE) __msa_dpadd_s_h((v8i16) out1, \ 850 (v16i8) mult1, (v16i8) cnst1); \ 852 #define DPADD_SB2_SH(...) DPADD_SB2(v8i16, __VA_ARGS__) 854 #define DPADD_SB4(RTYPE, mult0, mult1, mult2, mult3, \ 855 cnst0, cnst1, cnst2, cnst3, out0, out1, out2, out3) \ 857 DPADD_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 858 DPADD_SB2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 860 #define DPADD_SB4_SH(...) DPADD_SB4(v8i16, __VA_ARGS__) 874 #define DPADD_UB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 876 out0 = (RTYPE) __msa_dpadd_u_h((v8u16) out0, \ 877 (v16u8) mult0, (v16u8) cnst0); \ 878 out1 = (RTYPE) __msa_dpadd_u_h((v8u16) out1, \ 879 (v16u8) mult1, (v16u8) cnst1); \ 881 #define DPADD_UB2_UH(...) DPADD_UB2(v8u16, __VA_ARGS__) 895 #define DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 897 out0 = (RTYPE) __msa_dpadd_s_w((v4i32) out0, \ 898 (v8i16) mult0, (v8i16) cnst0); \ 899 out1 = (RTYPE) __msa_dpadd_s_w((v4i32) out1, \ 900 (v8i16) mult1, (v8i16) cnst1); \ 902 #define DPADD_SH2_SW(...) DPADD_SH2(v4i32, __VA_ARGS__) 904 #define DPADD_SH4(RTYPE, mult0, mult1, mult2, mult3, \ 905 cnst0, cnst1, cnst2, cnst3, out0, out1, out2, out3) \ 907 DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 908 DPADD_SH2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 910 #define DPADD_SH4_SW(...) DPADD_SH4(v4i32, __VA_ARGS__) 920 #define MIN_UH2(RTYPE, in0, in1, min_vec) \ 922 in0 = (RTYPE) __msa_min_u_h((v8u16) in0, min_vec); \ 923 in1 = (RTYPE) __msa_min_u_h((v8u16) in1, min_vec); \ 925 #define MIN_UH2_UH(...) MIN_UH2(v8u16, __VA_ARGS__) 927 #define MIN_UH4(RTYPE, in0, in1, in2, in3, min_vec) \ 929 MIN_UH2(RTYPE, in0, in1, min_vec); \ 930 MIN_UH2(RTYPE, in2, in3, min_vec); \ 932 #define MIN_UH4_UH(...) MIN_UH4(v8u16, __VA_ARGS__) 942 #define CLIP_SH(in, min, max) \ 946 out_m = __msa_max_s_h((v8i16) min, (v8i16) in); \ 947 out_m = __msa_min_s_h((v8i16) max, (v8i16) out_m); \ 957 #define CLIP_SH_0_255(in) \ 959 v8i16 max_m = __msa_ldi_h(255); \ 962 out_m = __msa_maxi_s_h((v8i16) in, 0); \ 963 out_m = __msa_min_s_h((v8i16) max_m, (v8i16) out_m); \ 966 #define CLIP_SH2_0_255(in0, in1) \ 968 in0 = CLIP_SH_0_255(in0); \ 969 in1 = CLIP_SH_0_255(in1); \ 971 #define CLIP_SH4_0_255(in0, in1, in2, in3) \ 973 CLIP_SH2_0_255(in0, in1); \ 974 CLIP_SH2_0_255(in2, in3); \ 977 #define CLIP_SH_0_255_MAX_SATU(in) \ 981 out_m = __msa_maxi_s_h((v8i16) in, 0); \ 982 out_m = (v8i16) __msa_sat_u_h((v8u16) out_m, 7); \ 985 #define CLIP_SH2_0_255_MAX_SATU(in0, in1) \ 987 in0 = CLIP_SH_0_255_MAX_SATU(in0); \ 988 in1 = CLIP_SH_0_255_MAX_SATU(in1); \ 990 #define CLIP_SH4_0_255_MAX_SATU(in0, in1, in2, in3) \ 992 CLIP_SH2_0_255_MAX_SATU(in0, in1); \ 993 CLIP_SH2_0_255_MAX_SATU(in2, in3); \ 1002 #define CLIP_SW_0_255(in) \ 1004 v4i32 max_m = __msa_ldi_w(255); \ 1007 out_m = __msa_maxi_s_w((v4i32) in, 0); \ 1008 out_m = __msa_min_s_w((v4i32) max_m, (v4i32) out_m); \ 1012 #define CLIP_SW_0_255_MAX_SATU(in) \ 1016 out_m = __msa_maxi_s_w((v4i32) in, 0); \ 1017 out_m = (v4i32) __msa_sat_u_w((v4u32) out_m, 7); \ 1020 #define CLIP_SW2_0_255_MAX_SATU(in0, in1) \ 1022 in0 = CLIP_SW_0_255_MAX_SATU(in0); \ 1023 in1 = CLIP_SW_0_255_MAX_SATU(in1); \ 1025 #define CLIP_SW4_0_255_MAX_SATU(in0, in1, in2, in3) \ 1027 CLIP_SW2_0_255_MAX_SATU(in0, in1); \ 1028 CLIP_SW2_0_255_MAX_SATU(in2, in3); \ 1038 #define HADD_SW_S32(in) \ 1040 v2i64 res0_m, res1_m; \ 1043 res0_m = __msa_hadd_s_d((v4i32) in, (v4i32) in); \ 1044 res1_m = __msa_splati_d(res0_m, 1); \ 1046 sum_m = __msa_copy_s_w((v4i32) res0_m, 0); \ 1057 #define HADD_UH_U32(in) \ 1060 v2u64 res0_m, res1_m; \ 1063 res_m = __msa_hadd_u_w((v8u16) in, (v8u16) in); \ 1064 res0_m = __msa_hadd_u_d(res_m, res_m); \ 1065 res1_m = (v2u64) __msa_splati_d((v2i64) res0_m, 1); \ 1067 sum_m = __msa_copy_u_w((v4i32) res0_m, 0); \ 1079 #define HADD_SB2(RTYPE, in0, in1, out0, out1) \ 1081 out0 = (RTYPE) __msa_hadd_s_h((v16i8) in0, (v16i8) in0); \ 1082 out1 = (RTYPE) __msa_hadd_s_h((v16i8) in1, (v16i8) in1); \ 1084 #define HADD_SB2_SH(...) HADD_SB2(v8i16, __VA_ARGS__) 1086 #define HADD_SB4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) \ 1088 HADD_SB2(RTYPE, in0, in1, out0, out1); \ 1089 HADD_SB2(RTYPE, in2, in3, out2, out3); \ 1091 #define HADD_SB4_UH(...) HADD_SB4(v8u16, __VA_ARGS__) 1092 #define HADD_SB4_SH(...) HADD_SB4(v8i16, __VA_ARGS__) 1102 #define HADD_UB2(RTYPE, in0, in1, out0, out1) \ 1104 out0 = (RTYPE) __msa_hadd_u_h((v16u8) in0, (v16u8) in0); \ 1105 out1 = (RTYPE) __msa_hadd_u_h((v16u8) in1, (v16u8) in1); \ 1107 #define HADD_UB2_UH(...) HADD_UB2(v8u16, __VA_ARGS__) 1109 #define HADD_UB3(RTYPE, in0, in1, in2, out0, out1, out2) \ 1111 HADD_UB2(RTYPE, in0, in1, out0, out1); \ 1112 out2 = (RTYPE) __msa_hadd_u_h((v16u8) in2, (v16u8) in2); \ 1114 #define HADD_UB3_UH(...) HADD_UB3(v8u16, __VA_ARGS__) 1116 #define HADD_UB4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) \ 1118 HADD_UB2(RTYPE, in0, in1, out0, out1); \ 1119 HADD_UB2(RTYPE, in2, in3, out2, out3); \ 1121 #define HADD_UB4_UB(...) HADD_UB4(v16u8, __VA_ARGS__) 1122 #define HADD_UB4_UH(...) HADD_UB4(v8u16, __VA_ARGS__) 1123 #define HADD_UB4_SH(...) HADD_UB4(v8i16, __VA_ARGS__) 1133 #define HSUB_UB2(RTYPE, in0, in1, out0, out1) \ 1135 out0 = (RTYPE) __msa_hsub_u_h((v16u8) in0, (v16u8) in0); \ 1136 out1 = (RTYPE) __msa_hsub_u_h((v16u8) in1, (v16u8) in1); \ 1138 #define HSUB_UB2_UH(...) HSUB_UB2(v8u16, __VA_ARGS__) 1139 #define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__) 1141 #define HSUB_UB4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) \ 1143 HSUB_UB2(RTYPE, in0, in1, out0, out1); \ 1144 HSUB_UB2(RTYPE, in2, in3, out2, out3); \ 1146 #define HSUB_UB4_UH(...) HSUB_UB4(v8u16, __VA_ARGS__) 1147 #define HSUB_UB4_SH(...) HSUB_UB4(v8i16, __VA_ARGS__) 1159 #define SAD_UB2_UH(in0, in1, ref0, ref1) \ 1161 v8u16 sad_m = { 0 }; \ 1162 sad_m += __builtin_msa2_sad_adj2_u_w2x_b((v16u8) in0, (v16u8) ref0); \ 1163 sad_m += __builtin_msa2_sad_adj2_u_w2x_b((v16u8) in1, (v16u8) ref1); \ 1167 #define SAD_UB2_UH(in0, in1, ref0, ref1) \ 1169 v16u8 diff0_m, diff1_m; \ 1170 v8u16 sad_m = { 0 }; \ 1172 diff0_m = __msa_asub_u_b((v16u8) in0, (v16u8) ref0); \ 1173 diff1_m = __msa_asub_u_b((v16u8) in1, (v16u8) ref1); \ 1175 sad_m += __msa_hadd_u_h((v16u8) diff0_m, (v16u8) diff0_m); \ 1176 sad_m += __msa_hadd_u_h((v16u8) diff1_m, (v16u8) diff1_m); \ 1180 #endif // #if HAVE_MSA2 1188 #define INSERT_W2(RTYPE, in0, in1, out) \ 1190 out = (RTYPE) __msa_insert_w((v4i32) out, 0, in0); \ 1191 out = (RTYPE) __msa_insert_w((v4i32) out, 1, in1); \ 1193 #define INSERT_W2_UB(...) INSERT_W2(v16u8, __VA_ARGS__) 1194 #define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__) 1196 #define INSERT_W4(RTYPE, in0, in1, in2, in3, out) \ 1198 out = (RTYPE) __msa_insert_w((v4i32) out, 0, in0); \ 1199 out = (RTYPE) __msa_insert_w((v4i32) out, 1, in1); \ 1200 out = (RTYPE) __msa_insert_w((v4i32) out, 2, in2); \ 1201 out = (RTYPE) __msa_insert_w((v4i32) out, 3, in3); \ 1203 #define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__) 1204 #define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__) 1205 #define INSERT_W4_SH(...) INSERT_W4(v8i16, __VA_ARGS__) 1206 #define INSERT_W4_SW(...) INSERT_W4(v4i32, __VA_ARGS__) 1214 #define INSERT_D2(RTYPE, in0, in1, out) \ 1216 out = (RTYPE) __msa_insert_d((v2i64) out, 0, in0); \ 1217 out = (RTYPE) __msa_insert_d((v2i64) out, 1, in1); \ 1219 #define INSERT_D2_UB(...) INSERT_D2(v16u8, __VA_ARGS__) 1220 #define INSERT_D2_SB(...) INSERT_D2(v16i8, __VA_ARGS__) 1221 #define INSERT_D2_SH(...) INSERT_D2(v8i16, __VA_ARGS__) 1222 #define INSERT_D2_SD(...) INSERT_D2(v2i64, __VA_ARGS__) 1233 #define ILVEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1235 out0 = (RTYPE) __msa_ilvev_b((v16i8) in1, (v16i8) in0); \ 1236 out1 = (RTYPE) __msa_ilvev_b((v16i8) in3, (v16i8) in2); \ 1238 #define ILVEV_B2_UB(...) ILVEV_B2(v16u8, __VA_ARGS__) 1239 #define ILVEV_B2_SB(...) ILVEV_B2(v16i8, __VA_ARGS__) 1240 #define ILVEV_B2_SH(...) ILVEV_B2(v8i16, __VA_ARGS__) 1241 #define ILVEV_B2_SD(...) ILVEV_B2(v2i64, __VA_ARGS__) 1252 #define ILVEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1254 out0 = (RTYPE) __msa_ilvev_h((v8i16) in1, (v8i16) in0); \ 1255 out1 = (RTYPE) __msa_ilvev_h((v8i16) in3, (v8i16) in2); \ 1257 #define ILVEV_H2_UB(...) ILVEV_H2(v16u8, __VA_ARGS__) 1258 #define ILVEV_H2_SH(...) ILVEV_H2(v8i16, __VA_ARGS__) 1259 #define ILVEV_H2_SW(...) ILVEV_H2(v4i32, __VA_ARGS__) 1270 #define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1272 out0 = (RTYPE) __msa_ilvev_w((v4i32) in1, (v4i32) in0); \ 1273 out1 = (RTYPE) __msa_ilvev_w((v4i32) in3, (v4i32) in2); \ 1275 #define ILVEV_W2_UB(...) ILVEV_W2(v16u8, __VA_ARGS__) 1276 #define ILVEV_W2_SB(...) ILVEV_W2(v16i8, __VA_ARGS__) 1277 #define ILVEV_W2_UH(...) ILVEV_W2(v8u16, __VA_ARGS__) 1278 #define ILVEV_W2_SD(...) ILVEV_W2(v2i64, __VA_ARGS__) 1289 #define ILVEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1291 out0 = (RTYPE) __msa_ilvev_d((v2i64) in1, (v2i64) in0); \ 1292 out1 = (RTYPE) __msa_ilvev_d((v2i64) in3, (v2i64) in2); \ 1294 #define ILVEV_D2_UB(...) ILVEV_D2(v16u8, __VA_ARGS__) 1295 #define ILVEV_D2_SB(...) ILVEV_D2(v16i8, __VA_ARGS__) 1296 #define ILVEV_D2_SW(...) ILVEV_D2(v4i32, __VA_ARGS__) 1307 #define ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1309 out0 = (RTYPE) __msa_ilvl_b((v16i8) in0, (v16i8) in1); \ 1310 out1 = (RTYPE) __msa_ilvl_b((v16i8) in2, (v16i8) in3); \ 1312 #define ILVL_B2_UB(...) ILVL_B2(v16u8, __VA_ARGS__) 1313 #define ILVL_B2_SB(...) ILVL_B2(v16i8, __VA_ARGS__) 1314 #define ILVL_B2_UH(...) ILVL_B2(v8u16, __VA_ARGS__) 1315 #define ILVL_B2_SH(...) ILVL_B2(v8i16, __VA_ARGS__) 1317 #define ILVL_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1318 out0, out1, out2, out3) \ 1320 ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1321 ILVL_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1323 #define ILVL_B4_UB(...) ILVL_B4(v16u8, __VA_ARGS__) 1324 #define ILVL_B4_SB(...) ILVL_B4(v16i8, __VA_ARGS__) 1325 #define ILVL_B4_UH(...) ILVL_B4(v8u16, __VA_ARGS__) 1326 #define ILVL_B4_SH(...) ILVL_B4(v8i16, __VA_ARGS__) 1337 #define ILVL_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1339 out0 = (RTYPE) __msa_ilvl_h((v8i16) in0, (v8i16) in1); \ 1340 out1 = (RTYPE) __msa_ilvl_h((v8i16) in2, (v8i16) in3); \ 1342 #define ILVL_H2_SH(...) ILVL_H2(v8i16, __VA_ARGS__) 1343 #define ILVL_H2_SW(...) ILVL_H2(v4i32, __VA_ARGS__) 1345 #define ILVL_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1346 out0, out1, out2, out3) \ 1348 ILVL_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1349 ILVL_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1351 #define ILVL_H4_SH(...) ILVL_H4(v8i16, __VA_ARGS__) 1352 #define ILVL_H4_SW(...) ILVL_H4(v4i32, __VA_ARGS__) 1363 #define ILVL_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1365 out0 = (RTYPE) __msa_ilvl_w((v4i32) in0, (v4i32) in1); \ 1366 out1 = (RTYPE) __msa_ilvl_w((v4i32) in2, (v4i32) in3); \ 1368 #define ILVL_W2_UB(...) ILVL_W2(v16u8, __VA_ARGS__) 1369 #define ILVL_W2_SB(...) ILVL_W2(v16i8, __VA_ARGS__) 1370 #define ILVL_W2_SH(...) ILVL_W2(v8i16, __VA_ARGS__) 1382 #define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1384 out0 = (RTYPE) __msa_ilvr_b((v16i8) in0, (v16i8) in1); \ 1385 out1 = (RTYPE) __msa_ilvr_b((v16i8) in2, (v16i8) in3); \ 1387 #define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__) 1388 #define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__) 1389 #define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__) 1390 #define ILVR_B2_SH(...) ILVR_B2(v8i16, __VA_ARGS__) 1391 #define ILVR_B2_SW(...) ILVR_B2(v4i32, __VA_ARGS__) 1393 #define ILVR_B3(RTYPE, in0, in1, in2, in3, in4, in5, out0, out1, out2) \ 1395 ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1396 out2 = (RTYPE) __msa_ilvr_b((v16i8) in4, (v16i8) in5); \ 1398 #define ILVR_B3_UB(...) ILVR_B3(v16u8, __VA_ARGS__) 1399 #define ILVR_B3_SB(...) ILVR_B3(v16i8, __VA_ARGS__) 1400 #define ILVR_B3_UH(...) ILVR_B3(v8u16, __VA_ARGS__) 1401 #define ILVR_B3_SH(...) ILVR_B3(v8i16, __VA_ARGS__) 1403 #define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1404 out0, out1, out2, out3) \ 1406 ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1407 ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1409 #define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__) 1410 #define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__) 1411 #define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__) 1412 #define ILVR_B4_SH(...) ILVR_B4(v8i16, __VA_ARGS__) 1413 #define ILVR_B4_SW(...) ILVR_B4(v4i32, __VA_ARGS__) 1415 #define ILVR_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1416 in8, in9, in10, in11, in12, in13, in14, in15, \ 1417 out0, out1, out2, out3, out4, out5, out6, out7) \ 1419 ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1420 out0, out1, out2, out3); \ 1421 ILVR_B4(RTYPE, in8, in9, in10, in11, in12, in13, in14, in15, \ 1422 out4, out5, out6, out7); \ 1424 #define ILVR_B8_UH(...) ILVR_B8(v8u16, __VA_ARGS__) 1436 #define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1438 out0 = (RTYPE) __msa_ilvr_h((v8i16) in0, (v8i16) in1); \ 1439 out1 = (RTYPE) __msa_ilvr_h((v8i16) in2, (v8i16) in3); \ 1441 #define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__) 1442 #define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__) 1444 #define ILVR_H3(RTYPE, in0, in1, in2, in3, in4, in5, out0, out1, out2) \ 1446 ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1447 out2 = (RTYPE) __msa_ilvr_h((v8i16) in4, (v8i16) in5); \ 1449 #define ILVR_H3_SH(...) ILVR_H3(v8i16, __VA_ARGS__) 1451 #define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1452 out0, out1, out2, out3) \ 1454 ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1455 ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1457 #define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__) 1458 #define ILVR_H4_SW(...) ILVR_H4(v4i32, __VA_ARGS__) 1460 #define ILVR_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1462 out0 = (RTYPE) __msa_ilvr_w((v4i32) in0, (v4i32) in1); \ 1463 out1 = (RTYPE) __msa_ilvr_w((v4i32) in2, (v4i32) in3); \ 1465 #define ILVR_W2_UB(...) ILVR_W2(v16u8, __VA_ARGS__) 1466 #define ILVR_W2_SB(...) ILVR_W2(v16i8, __VA_ARGS__) 1467 #define ILVR_W2_SH(...) ILVR_W2(v8i16, __VA_ARGS__) 1469 #define ILVR_W4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1470 out0, out1, out2, out3) \ 1472 ILVR_W2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1473 ILVR_W2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1475 #define ILVR_W4_SB(...) ILVR_W4(v16i8, __VA_ARGS__) 1476 #define ILVR_W4_UB(...) ILVR_W4(v16u8, __VA_ARGS__) 1487 #define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1489 out0 = (RTYPE) __msa_ilvr_d((v2i64) in0, (v2i64) in1); \ 1490 out1 = (RTYPE) __msa_ilvr_d((v2i64) in2, (v2i64) in3); \ 1492 #define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__) 1493 #define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__) 1494 #define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__) 1496 #define ILVR_D3(RTYPE, in0, in1, in2, in3, in4, in5, out0, out1, out2) \ 1498 ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1499 out2 = (RTYPE) __msa_ilvr_d((v2i64) in4, (v2i64) in5); \ 1501 #define ILVR_D3_SB(...) ILVR_D3(v16i8, __VA_ARGS__) 1503 #define ILVR_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1504 out0, out1, out2, out3) \ 1506 ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1507 ILVR_D2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1509 #define ILVR_D4_SB(...) ILVR_D4(v16i8, __VA_ARGS__) 1510 #define ILVR_D4_UB(...) ILVR_D4(v16u8, __VA_ARGS__) 1521 #define ILVL_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1523 out0 = (RTYPE) __msa_ilvl_d((v2i64) in0, (v2i64) in1); \ 1524 out1 = (RTYPE) __msa_ilvl_d((v2i64) in2, (v2i64) in3); \ 1526 #define ILVL_D2_UB(...) ILVL_D2(v16u8, __VA_ARGS__) 1527 #define ILVL_D2_SB(...) ILVL_D2(v16i8, __VA_ARGS__) 1528 #define ILVL_D2_SH(...) ILVL_D2(v8i16, __VA_ARGS__) 1539 #define ILVRL_B2(RTYPE, in0, in1, out0, out1) \ 1541 out0 = (RTYPE) __msa_ilvr_b((v16i8) in0, (v16i8) in1); \ 1542 out1 = (RTYPE) __msa_ilvl_b((v16i8) in0, (v16i8) in1); \ 1544 #define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__) 1545 #define ILVRL_B2_SB(...) ILVRL_B2(v16i8, __VA_ARGS__) 1546 #define ILVRL_B2_UH(...) ILVRL_B2(v8u16, __VA_ARGS__) 1547 #define ILVRL_B2_SH(...) ILVRL_B2(v8i16, __VA_ARGS__) 1548 #define ILVRL_B2_SW(...) ILVRL_B2(v4i32, __VA_ARGS__) 1550 #define ILVRL_H2(RTYPE, in0, in1, out0, out1) \ 1552 out0 = (RTYPE) __msa_ilvr_h((v8i16) in0, (v8i16) in1); \ 1553 out1 = (RTYPE) __msa_ilvl_h((v8i16) in0, (v8i16) in1); \ 1555 #define ILVRL_H2_UB(...) ILVRL_H2(v16u8, __VA_ARGS__) 1556 #define ILVRL_H2_SB(...) ILVRL_H2(v16i8, __VA_ARGS__) 1557 #define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__) 1558 #define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__) 1560 #define ILVRL_W2(RTYPE, in0, in1, out0, out1) \ 1562 out0 = (RTYPE) __msa_ilvr_w((v4i32) in0, (v4i32) in1); \ 1563 out1 = (RTYPE) __msa_ilvl_w((v4i32) in0, (v4i32) in1); \ 1565 #define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__) 1566 #define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__) 1567 #define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__) 1577 #define MAXI_SH2(RTYPE, in0, in1, max_val) \ 1579 in0 = (RTYPE) __msa_maxi_s_h((v8i16) in0, max_val); \ 1580 in1 = (RTYPE) __msa_maxi_s_h((v8i16) in1, max_val); \ 1582 #define MAXI_SH2_UH(...) MAXI_SH2(v8u16, __VA_ARGS__) 1583 #define MAXI_SH2_SH(...) MAXI_SH2(v8i16, __VA_ARGS__) 1585 #define MAXI_SH4(RTYPE, in0, in1, in2, in3, max_val) \ 1587 MAXI_SH2(RTYPE, in0, in1, max_val); \ 1588 MAXI_SH2(RTYPE, in2, in3, max_val); \ 1590 #define MAXI_SH4_UH(...) MAXI_SH4(v8u16, __VA_ARGS__) 1591 #define MAXI_SH4_SH(...) MAXI_SH4(v8i16, __VA_ARGS__) 1593 #define MAXI_SH8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, max_val) \ 1595 MAXI_SH4(RTYPE, in0, in1, in2, in3, max_val); \ 1596 MAXI_SH4(RTYPE, in4, in5, in6, in7, max_val); \ 1598 #define MAXI_SH8_UH(...) MAXI_SH8(v8u16, __VA_ARGS__) 1599 #define MAXI_SH8_SH(...) MAXI_SH8(v8i16, __VA_ARGS__) 1611 #define SAT_UH2(RTYPE, in0, in1, sat_val) \ 1613 in0 = (RTYPE) __msa_sat_u_h((v8u16) in0, sat_val); \ 1614 in1 = (RTYPE) __msa_sat_u_h((v8u16) in1, sat_val); \ 1616 #define SAT_UH2_UH(...) SAT_UH2(v8u16, __VA_ARGS__) 1617 #define SAT_UH2_SH(...) SAT_UH2(v8i16, __VA_ARGS__) 1619 #define SAT_UH4(RTYPE, in0, in1, in2, in3, sat_val) \ 1621 SAT_UH2(RTYPE, in0, in1, sat_val); \ 1622 SAT_UH2(RTYPE, in2, in3, sat_val); \ 1624 #define SAT_UH4_UH(...) SAT_UH4(v8u16, __VA_ARGS__) 1625 #define SAT_UH4_SH(...) SAT_UH4(v8i16, __VA_ARGS__) 1627 #define SAT_UH8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, sat_val) \ 1629 SAT_UH4(RTYPE, in0, in1, in2, in3, sat_val); \ 1630 SAT_UH4(RTYPE, in4, in5, in6, in7, sat_val); \ 1632 #define SAT_UH8_UH(...) SAT_UH8(v8u16, __VA_ARGS__) 1633 #define SAT_UH8_SH(...) SAT_UH8(v8i16, __VA_ARGS__) 1645 #define SAT_SH2(RTYPE, in0, in1, sat_val) \ 1647 in0 = (RTYPE) __msa_sat_s_h((v8i16) in0, sat_val); \ 1648 in1 = (RTYPE) __msa_sat_s_h((v8i16) in1, sat_val); \ 1650 #define SAT_SH2_SH(...) SAT_SH2(v8i16, __VA_ARGS__) 1652 #define SAT_SH3(RTYPE, in0, in1, in2, sat_val) \ 1654 SAT_SH2(RTYPE, in0, in1, sat_val); \ 1655 in2 = (RTYPE) __msa_sat_s_h((v8i16) in2, sat_val); \ 1657 #define SAT_SH3_SH(...) SAT_SH3(v8i16, __VA_ARGS__) 1659 #define SAT_SH4(RTYPE, in0, in1, in2, in3, sat_val) \ 1661 SAT_SH2(RTYPE, in0, in1, sat_val); \ 1662 SAT_SH2(RTYPE, in2, in3, sat_val); \ 1664 #define SAT_SH4_SH(...) SAT_SH4(v8i16, __VA_ARGS__) 1676 #define SAT_SW2(RTYPE, in0, in1, sat_val) \ 1678 in0 = (RTYPE) __msa_sat_s_w((v4i32) in0, sat_val); \ 1679 in1 = (RTYPE) __msa_sat_s_w((v4i32) in1, sat_val); \ 1681 #define SAT_SW2_SW(...) SAT_SW2(v4i32, __VA_ARGS__) 1683 #define SAT_SW4(RTYPE, in0, in1, in2, in3, sat_val) \ 1685 SAT_SW2(RTYPE, in0, in1, sat_val); \ 1686 SAT_SW2(RTYPE, in2, in3, sat_val); \ 1688 #define SAT_SW4_SW(...) SAT_SW4(v4i32, __VA_ARGS__) 1699 #define SPLATI_H2(RTYPE, in, idx0, idx1, out0, out1) \ 1701 out0 = (RTYPE) __msa_splati_h((v8i16) in, idx0); \ 1702 out1 = (RTYPE) __msa_splati_h((v8i16) in, idx1); \ 1704 #define SPLATI_H2_SB(...) SPLATI_H2(v16i8, __VA_ARGS__) 1705 #define SPLATI_H2_SH(...) SPLATI_H2(v8i16, __VA_ARGS__) 1707 #define SPLATI_H3(RTYPE, in, idx0, idx1, idx2, \ 1710 SPLATI_H2(RTYPE, in, idx0, idx1, out0, out1); \ 1711 out2 = (RTYPE) __msa_splati_h((v8i16) in, idx2); \ 1713 #define SPLATI_H3_SB(...) SPLATI_H3(v16i8, __VA_ARGS__) 1714 #define SPLATI_H3_SH(...) SPLATI_H3(v8i16, __VA_ARGS__) 1716 #define SPLATI_H4(RTYPE, in, idx0, idx1, idx2, idx3, \ 1717 out0, out1, out2, out3) \ 1719 SPLATI_H2(RTYPE, in, idx0, idx1, out0, out1); \ 1720 SPLATI_H2(RTYPE, in, idx2, idx3, out2, out3); \ 1722 #define SPLATI_H4_SB(...) SPLATI_H4(v16i8, __VA_ARGS__) 1723 #define SPLATI_H4_SH(...) SPLATI_H4(v8i16, __VA_ARGS__) 1736 #define SPLATI_W2(RTYPE, in, stidx, out0, out1) \ 1738 out0 = (RTYPE) __msa_splati_w((v4i32) in, stidx); \ 1739 out1 = (RTYPE) __msa_splati_w((v4i32) in, (stidx+1)); \ 1741 #define SPLATI_W2_SH(...) SPLATI_W2(v8i16, __VA_ARGS__) 1742 #define SPLATI_W2_SW(...) SPLATI_W2(v4i32, __VA_ARGS__) 1744 #define SPLATI_W4(RTYPE, in, out0, out1, out2, out3) \ 1746 SPLATI_W2(RTYPE, in, 0, out0, out1); \ 1747 SPLATI_W2(RTYPE, in, 2, out2, out3); \ 1749 #define SPLATI_W4_SH(...) SPLATI_W4(v8i16, __VA_ARGS__) 1750 #define SPLATI_W4_SW(...) SPLATI_W4(v4i32, __VA_ARGS__) 1763 #define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1765 out0 = (RTYPE) __msa_pckev_b((v16i8) in0, (v16i8) in1); \ 1766 out1 = (RTYPE) __msa_pckev_b((v16i8) in2, (v16i8) in3); \ 1768 #define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__) 1769 #define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__) 1770 #define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__) 1771 #define PCKEV_B2_SW(...) PCKEV_B2(v4i32, __VA_ARGS__) 1773 #define PCKEV_B3(RTYPE, in0, in1, in2, in3, in4, in5, out0, out1, out2) \ 1775 PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1776 out2 = (RTYPE) __msa_pckev_b((v16i8) in4, (v16i8) in5); \ 1778 #define PCKEV_B3_UB(...) PCKEV_B3(v16u8, __VA_ARGS__) 1779 #define PCKEV_B3_SB(...) PCKEV_B3(v16i8, __VA_ARGS__) 1781 #define PCKEV_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1782 out0, out1, out2, out3) \ 1784 PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1785 PCKEV_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1787 #define PCKEV_B4_SB(...) PCKEV_B4(v16i8, __VA_ARGS__) 1788 #define PCKEV_B4_UB(...) PCKEV_B4(v16u8, __VA_ARGS__) 1789 #define PCKEV_B4_SH(...) PCKEV_B4(v8i16, __VA_ARGS__) 1790 #define PCKEV_B4_SW(...) PCKEV_B4(v4i32, __VA_ARGS__) 1803 #define PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1805 out0 = (RTYPE) __msa_pckev_h((v8i16) in0, (v8i16) in1); \ 1806 out1 = (RTYPE) __msa_pckev_h((v8i16) in2, (v8i16) in3); \ 1808 #define PCKEV_H2_SH(...) PCKEV_H2(v8i16, __VA_ARGS__) 1809 #define PCKEV_H2_SW(...) PCKEV_H2(v4i32, __VA_ARGS__) 1811 #define PCKEV_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1812 out0, out1, out2, out3) \ 1814 PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1815 PCKEV_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1817 #define PCKEV_H4_SH(...) PCKEV_H4(v8i16, __VA_ARGS__) 1818 #define PCKEV_H4_SW(...) PCKEV_H4(v4i32, __VA_ARGS__) 1831 #define PCKEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1833 out0 = (RTYPE) __msa_pckev_d((v2i64) in0, (v2i64) in1); \ 1834 out1 = (RTYPE) __msa_pckev_d((v2i64) in2, (v2i64) in3); \ 1836 #define PCKEV_D2_UB(...) PCKEV_D2(v16u8, __VA_ARGS__) 1837 #define PCKEV_D2_SB(...) PCKEV_D2(v16i8, __VA_ARGS__) 1838 #define PCKEV_D2_SH(...) PCKEV_D2(v8i16, __VA_ARGS__) 1840 #define PCKEV_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1841 out0, out1, out2, out3) \ 1843 PCKEV_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1844 PCKEV_D2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1846 #define PCKEV_D4_UB(...) PCKEV_D4(v16u8, __VA_ARGS__) 1857 #define PCKOD_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1859 out0 = (RTYPE) __msa_pckod_d((v2i64) in0, (v2i64) in1); \ 1860 out1 = (RTYPE) __msa_pckod_d((v2i64) in2, (v2i64) in3); \ 1862 #define PCKOD_D2_UB(...) PCKOD_D2(v16u8, __VA_ARGS__) 1863 #define PCKOD_D2_SH(...) PCKOD_D2(v8i16, __VA_ARGS__) 1864 #define PCKOD_D2_SD(...) PCKOD_D2(v2i64, __VA_ARGS__) 1878 #define XORI_B2_128(RTYPE, in0, in1) \ 1880 in0 = (RTYPE) __msa_xori_b((v16u8) in0, 128); \ 1881 in1 = (RTYPE) __msa_xori_b((v16u8) in1, 128); \ 1883 #define XORI_B2_128_UB(...) XORI_B2_128(v16u8, __VA_ARGS__) 1884 #define XORI_B2_128_SB(...) XORI_B2_128(v16i8, __VA_ARGS__) 1885 #define XORI_B2_128_SH(...) XORI_B2_128(v8i16, __VA_ARGS__) 1887 #define XORI_B3_128(RTYPE, in0, in1, in2) \ 1889 XORI_B2_128(RTYPE, in0, in1); \ 1890 in2 = (RTYPE) __msa_xori_b((v16u8) in2, 128); \ 1892 #define XORI_B3_128_SB(...) XORI_B3_128(v16i8, __VA_ARGS__) 1894 #define XORI_B4_128(RTYPE, in0, in1, in2, in3) \ 1896 XORI_B2_128(RTYPE, in0, in1); \ 1897 XORI_B2_128(RTYPE, in2, in3); \ 1899 #define XORI_B4_128_UB(...) XORI_B4_128(v16u8, __VA_ARGS__) 1900 #define XORI_B4_128_SB(...) XORI_B4_128(v16i8, __VA_ARGS__) 1901 #define XORI_B4_128_SH(...) XORI_B4_128(v8i16, __VA_ARGS__) 1903 #define XORI_B5_128(RTYPE, in0, in1, in2, in3, in4) \ 1905 XORI_B3_128(RTYPE, in0, in1, in2); \ 1906 XORI_B2_128(RTYPE, in3, in4); \ 1908 #define XORI_B5_128_SB(...) XORI_B5_128(v16i8, __VA_ARGS__) 1910 #define XORI_B6_128(RTYPE, in0, in1, in2, in3, in4, in5) \ 1912 XORI_B4_128(RTYPE, in0, in1, in2, in3); \ 1913 XORI_B2_128(RTYPE, in4, in5); \ 1915 #define XORI_B6_128_SB(...) XORI_B6_128(v16i8, __VA_ARGS__) 1917 #define XORI_B7_128(RTYPE, in0, in1, in2, in3, in4, in5, in6) \ 1919 XORI_B4_128(RTYPE, in0, in1, in2, in3); \ 1920 XORI_B3_128(RTYPE, in4, in5, in6); \ 1922 #define XORI_B7_128_SB(...) XORI_B7_128(v16i8, __VA_ARGS__) 1924 #define XORI_B8_128(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7) \ 1926 XORI_B4_128(RTYPE, in0, in1, in2, in3); \ 1927 XORI_B4_128(RTYPE, in4, in5, in6, in7); \ 1929 #define XORI_B8_128_SB(...) XORI_B8_128(v16i8, __VA_ARGS__) 1930 #define XORI_B8_128_UB(...) XORI_B8_128(v16u8, __VA_ARGS__) 1941 #define ADDS_SH2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1943 out0 = (RTYPE) __msa_adds_s_h((v8i16) in0, (v8i16) in1); \ 1944 out1 = (RTYPE) __msa_adds_s_h((v8i16) in2, (v8i16) in3); \ 1946 #define ADDS_SH2_SH(...) ADDS_SH2(v8i16, __VA_ARGS__) 1948 #define ADDS_SH4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1949 out0, out1, out2, out3) \ 1951 ADDS_SH2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1952 ADDS_SH2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1954 #define ADDS_SH4_UH(...) ADDS_SH4(v8u16, __VA_ARGS__) 1955 #define ADDS_SH4_SH(...) ADDS_SH4(v8i16, __VA_ARGS__) 1965 #define SLLI_2V(in0, in1, shift) \ 1967 in0 = in0 << shift; \ 1968 in1 = in1 << shift; \ 1970 #define SLLI_4V(in0, in1, in2, in3, shift) \ 1972 in0 = in0 << shift; \ 1973 in1 = in1 << shift; \ 1974 in2 = in2 << shift; \ 1975 in3 = in3 << shift; \ 1988 #define SRA_4V(in0, in1, in2, in3, shift) \ 1990 in0 = in0 >> shift; \ 1991 in1 = in1 >> shift; \ 1992 in2 = in2 >> shift; \ 1993 in3 = in3 >> shift; \ 2006 #define SRL_H4(RTYPE, in0, in1, in2, in3, shift) \ 2008 in0 = (RTYPE) __msa_srl_h((v8i16) in0, (v8i16) shift); \ 2009 in1 = (RTYPE) __msa_srl_h((v8i16) in1, (v8i16) shift); \ 2010 in2 = (RTYPE) __msa_srl_h((v8i16) in2, (v8i16) shift); \ 2011 in3 = (RTYPE) __msa_srl_h((v8i16) in3, (v8i16) shift); \ 2013 #define SRL_H4_UH(...) SRL_H4(v8u16, __VA_ARGS__) 2015 #define SRLR_H4(RTYPE, in0, in1, in2, in3, shift) \ 2017 in0 = (RTYPE) __msa_srlr_h((v8i16) in0, (v8i16) shift); \ 2018 in1 = (RTYPE) __msa_srlr_h((v8i16) in1, (v8i16) shift); \ 2019 in2 = (RTYPE) __msa_srlr_h((v8i16) in2, (v8i16) shift); \ 2020 in3 = (RTYPE) __msa_srlr_h((v8i16) in3, (v8i16) shift); \ 2022 #define SRLR_H4_UH(...) SRLR_H4(v8u16, __VA_ARGS__) 2023 #define SRLR_H4_SH(...) SRLR_H4(v8i16, __VA_ARGS__) 2025 #define SRLR_H8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, shift) \ 2027 SRLR_H4(RTYPE, in0, in1, in2, in3, shift); \ 2028 SRLR_H4(RTYPE, in4, in5, in6, in7, shift); \ 2030 #define SRLR_H8_UH(...) SRLR_H8(v8u16, __VA_ARGS__) 2031 #define SRLR_H8_SH(...) SRLR_H8(v8i16, __VA_ARGS__) 2044 #define SRAR_H2(RTYPE, in0, in1, shift) \ 2046 in0 = (RTYPE) __msa_srar_h((v8i16) in0, (v8i16) shift); \ 2047 in1 = (RTYPE) __msa_srar_h((v8i16) in1, (v8i16) shift); \ 2049 #define SRAR_H2_UH(...) SRAR_H2(v8u16, __VA_ARGS__) 2050 #define SRAR_H2_SH(...) SRAR_H2(v8i16, __VA_ARGS__) 2052 #define SRAR_H3(RTYPE, in0, in1, in2, shift) \ 2054 SRAR_H2(RTYPE, in0, in1, shift) \ 2055 in2 = (RTYPE) __msa_srar_h((v8i16) in2, (v8i16) shift); \ 2057 #define SRAR_H3_SH(...) SRAR_H3(v8i16, __VA_ARGS__) 2059 #define SRAR_H4(RTYPE, in0, in1, in2, in3, shift) \ 2061 SRAR_H2(RTYPE, in0, in1, shift) \ 2062 SRAR_H2(RTYPE, in2, in3, shift) \ 2064 #define SRAR_H4_UH(...) SRAR_H4(v8u16, __VA_ARGS__) 2065 #define SRAR_H4_SH(...) SRAR_H4(v8i16, __VA_ARGS__) 2078 #define SRAR_W2(RTYPE, in0, in1, shift) \ 2080 in0 = (RTYPE) __msa_srar_w((v4i32) in0, (v4i32) shift); \ 2081 in1 = (RTYPE) __msa_srar_w((v4i32) in1, (v4i32) shift); \ 2083 #define SRAR_W2_SW(...) SRAR_W2(v4i32, __VA_ARGS__) 2085 #define SRAR_W4(RTYPE, in0, in1, in2, in3, shift) \ 2087 SRAR_W2(RTYPE, in0, in1, shift) \ 2088 SRAR_W2(RTYPE, in2, in3, shift) \ 2090 #define SRAR_W4_SW(...) SRAR_W4(v4i32, __VA_ARGS__) 2102 #define SRARI_H2(RTYPE, in0, in1, shift) \ 2104 in0 = (RTYPE) __msa_srari_h((v8i16) in0, shift); \ 2105 in1 = (RTYPE) __msa_srari_h((v8i16) in1, shift); \ 2107 #define SRARI_H2_UH(...) SRARI_H2(v8u16, __VA_ARGS__) 2108 #define SRARI_H2_SH(...) SRARI_H2(v8i16, __VA_ARGS__) 2110 #define SRARI_H4(RTYPE, in0, in1, in2, in3, shift) \ 2112 SRARI_H2(RTYPE, in0, in1, shift); \ 2113 SRARI_H2(RTYPE, in2, in3, shift); \ 2115 #define SRARI_H4_UH(...) SRARI_H4(v8u16, __VA_ARGS__) 2116 #define SRARI_H4_SH(...) SRARI_H4(v8i16, __VA_ARGS__) 2128 #define SRARI_W2(RTYPE, in0, in1, shift) \ 2130 in0 = (RTYPE) __msa_srari_w((v4i32) in0, shift); \ 2131 in1 = (RTYPE) __msa_srari_w((v4i32) in1, shift); \ 2133 #define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__) 2135 #define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) \ 2137 SRARI_W2(RTYPE, in0, in1, shift); \ 2138 SRARI_W2(RTYPE, in2, in3, shift); \ 2140 #define SRARI_W4_SH(...) SRARI_W4(v8i16, __VA_ARGS__) 2141 #define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__) 2150 #define MUL2(in0, in1, in2, in3, out0, out1) \ 2155 #define MUL4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ 2157 MUL2(in0, in1, in2, in3, out0, out1); \ 2158 MUL2(in4, in5, in6, in7, out2, out3); \ 2167 #define ADD2(in0, in1, in2, in3, out0, out1) \ 2172 #define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ 2174 ADD2(in0, in1, in2, in3, out0, out1); \ 2175 ADD2(in4, in5, in6, in7, out2, out3); \ 2184 #define SUB2(in0, in1, in2, in3, out0, out1) \ 2189 #define SUB4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ 2205 #define UNPCK_R_SB_SH(in, out) \ 2209 sign_m = __msa_clti_s_b((v16i8) in, 0); \ 2210 out = (v8i16) __msa_ilvr_b(sign_m, (v16i8) in); \ 2222 #define UNPCK_R_SH_SW(in, out) \ 2224 out = (v4i32) __builtin_msa2_w2x_lo_s_h((v8i16) in); \ 2227 #define UNPCK_R_SH_SW(in, out) \ 2231 sign_m = __msa_clti_s_h((v8i16) in, 0); \ 2232 out = (v4i32) __msa_ilvr_h(sign_m, (v8i16) in); \ 2234 #endif // #if HAVE_MSA2 2248 #define UNPCK_SB_SH(in, out0, out1) \ 2250 out0 = (v4i32) __builtin_msa2_w2x_lo_s_b((v16i8) in); \ 2251 out1 = (v4i32) __builtin_msa2_w2x_hi_s_b((v16i8) in); \ 2254 #define UNPCK_SB_SH(in, out0, out1) \ 2258 tmp_m = __msa_clti_s_b((v16i8) in, 0); \ 2259 ILVRL_B2_SH(tmp_m, in, out0, out1); \ 2261 #endif // #if HAVE_MSA2 2270 #define UNPCK_UB_SH(in, out0, out1) \ 2272 v16i8 zero_m = { 0 }; \ 2274 ILVRL_B2_SH(zero_m, in, out0, out1); \ 2289 #define UNPCK_SH_SW(in, out0, out1) \ 2291 out0 = (v4i32) __builtin_msa2_w2x_lo_s_h((v8i16) in); \ 2292 out1 = (v4i32) __builtin_msa2_w2x_hi_s_h((v8i16) in); \ 2295 #define UNPCK_SH_SW(in, out0, out1) \ 2299 tmp_m = __msa_clti_s_h((v8i16) in, 0); \ 2300 ILVRL_H2_SW(tmp_m, in, out0, out1); \ 2302 #endif // #if HAVE_MSA2 2309 #define SWAP(in0, in1) \ 2321 #define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) \ 2335 #define BUTTERFLY_8(in0, in1, in2, in3, in4, in5, in6, in7, \ 2336 out0, out1, out2, out3, out4, out5, out6, out7) \ 2354 #define BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, \ 2355 in8, in9, in10, in11, in12, in13, in14, in15, \ 2356 out0, out1, out2, out3, out4, out5, out6, out7, \ 2357 out8, out9, out10, out11, out12, out13, out14, out15) \ 2359 out0 = in0 + in15; \ 2360 out1 = in1 + in14; \ 2361 out2 = in2 + in13; \ 2362 out3 = in3 + in12; \ 2363 out4 = in4 + in11; \ 2364 out5 = in5 + in10; \ 2370 out10 = in5 - in10; \ 2371 out11 = in4 - in11; \ 2372 out12 = in3 - in12; \ 2373 out13 = in2 - in13; \ 2374 out14 = in1 - in14; \ 2375 out15 = in0 - in15; \ 2384 #define TRANSPOSE4x4_UB_UB(in0, in1, in2, in3, out0, out1, out2, out3) \ 2386 v16i8 zero_m = { 0 }; \ 2387 v16i8 s0_m, s1_m, s2_m, s3_m; \ 2389 ILVR_D2_SB(in1, in0, in3, in2, s0_m, s1_m); \ 2390 ILVRL_B2_SB(s1_m, s0_m, s2_m, s3_m); \ 2392 out0 = (v16u8) __msa_ilvr_b(s3_m, s2_m); \ 2393 out1 = (v16u8) __msa_sldi_b(zero_m, (v16i8) out0, 4); \ 2394 out2 = (v16u8) __msa_sldi_b(zero_m, (v16i8) out1, 4); \ 2395 out3 = (v16u8) __msa_sldi_b(zero_m, (v16i8) out2, 4); \ 2404 #define TRANSPOSE8x4_UB(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 2405 out0, out1, out2, out3) \ 2407 v16i8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2409 ILVEV_W2_SB(in0, in4, in1, in5, tmp0_m, tmp1_m); \ 2410 tmp2_m = __msa_ilvr_b(tmp1_m, tmp0_m); \ 2411 ILVEV_W2_SB(in2, in6, in3, in7, tmp0_m, tmp1_m); \ 2413 tmp3_m = __msa_ilvr_b(tmp1_m, tmp0_m); \ 2414 ILVRL_H2_SB(tmp3_m, tmp2_m, tmp0_m, tmp1_m); \ 2416 ILVRL_W2(RTYPE, tmp1_m, tmp0_m, out0, out2); \ 2417 out1 = (RTYPE) __msa_ilvl_d((v2i64) out2, (v2i64) out0); \ 2418 out3 = (RTYPE) __msa_ilvl_d((v2i64) out0, (v2i64) out2); \ 2420 #define TRANSPOSE8x4_UB_UB(...) TRANSPOSE8x4_UB(v16u8, __VA_ARGS__) 2421 #define TRANSPOSE8x4_UB_UH(...) TRANSPOSE8x4_UB(v8u16, __VA_ARGS__) 2431 #define TRANSPOSE8x8_UB(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 2432 out0, out1, out2, out3, out4, out5, out6, out7) \ 2434 v16i8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2435 v16i8 tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ 2437 ILVR_B4_SB(in2, in0, in3, in1, in6, in4, in7, in5, \ 2438 tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ 2439 ILVRL_B2_SB(tmp1_m, tmp0_m, tmp4_m, tmp5_m); \ 2440 ILVRL_B2_SB(tmp3_m, tmp2_m, tmp6_m, tmp7_m); \ 2441 ILVRL_W2(RTYPE, tmp6_m, tmp4_m, out0, out2); \ 2442 ILVRL_W2(RTYPE, tmp7_m, tmp5_m, out4, out6); \ 2443 SLDI_B2_0(RTYPE, out0, out2, out1, out3, 8); \ 2444 SLDI_B2_0(RTYPE, out4, out6, out5, out7, 8); \ 2446 #define TRANSPOSE8x8_UB_UB(...) TRANSPOSE8x8_UB(v16u8, __VA_ARGS__) 2447 #define TRANSPOSE8x8_UB_UH(...) TRANSPOSE8x8_UB(v8u16, __VA_ARGS__) 2456 #define TRANSPOSE16x4_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2457 in8, in9, in10, in11, in12, in13, in14, in15, \ 2458 out0, out1, out2, out3) \ 2460 v2i64 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2462 ILVEV_W2_SD(in0, in4, in8, in12, tmp0_m, tmp1_m); \ 2463 out1 = (v16u8) __msa_ilvev_d(tmp1_m, tmp0_m); \ 2465 ILVEV_W2_SD(in1, in5, in9, in13, tmp0_m, tmp1_m); \ 2466 out3 = (v16u8) __msa_ilvev_d(tmp1_m, tmp0_m); \ 2468 ILVEV_W2_SD(in2, in6, in10, in14, tmp0_m, tmp1_m); \ 2470 tmp2_m = __msa_ilvev_d(tmp1_m, tmp0_m); \ 2471 ILVEV_W2_SD(in3, in7, in11, in15, tmp0_m, tmp1_m); \ 2473 tmp3_m = __msa_ilvev_d(tmp1_m, tmp0_m); \ 2474 ILVEV_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \ 2475 out0 = (v16u8) __msa_ilvev_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2476 out2 = (v16u8) __msa_ilvod_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2478 tmp0_m = (v2i64) __msa_ilvod_b((v16i8) out3, (v16i8) out1); \ 2479 tmp1_m = (v2i64) __msa_ilvod_b((v16i8) tmp3_m, (v16i8) tmp2_m); \ 2480 out1 = (v16u8) __msa_ilvev_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2481 out3 = (v16u8) __msa_ilvod_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2491 #define TRANSPOSE16x8_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2492 in8, in9, in10, in11, in12, in13, in14, in15, \ 2493 out0, out1, out2, out3, out4, out5, out6, out7) \ 2495 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2496 v16u8 tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ 2498 ILVEV_D2_UB(in0, in8, in1, in9, out7, out6); \ 2499 ILVEV_D2_UB(in2, in10, in3, in11, out5, out4); \ 2500 ILVEV_D2_UB(in4, in12, in5, in13, out3, out2); \ 2501 ILVEV_D2_UB(in6, in14, in7, in15, out1, out0); \ 2503 tmp0_m = (v16u8) __msa_ilvev_b((v16i8) out6, (v16i8) out7); \ 2504 tmp4_m = (v16u8) __msa_ilvod_b((v16i8) out6, (v16i8) out7); \ 2505 tmp1_m = (v16u8) __msa_ilvev_b((v16i8) out4, (v16i8) out5); \ 2506 tmp5_m = (v16u8) __msa_ilvod_b((v16i8) out4, (v16i8) out5); \ 2507 out5 = (v16u8) __msa_ilvev_b((v16i8) out2, (v16i8) out3); \ 2508 tmp6_m = (v16u8) __msa_ilvod_b((v16i8) out2, (v16i8) out3); \ 2509 out7 = (v16u8) __msa_ilvev_b((v16i8) out0, (v16i8) out1); \ 2510 tmp7_m = (v16u8) __msa_ilvod_b((v16i8) out0, (v16i8) out1); \ 2512 ILVEV_H2_UB(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \ 2513 out0 = (v16u8) __msa_ilvev_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2514 out4 = (v16u8) __msa_ilvod_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2516 tmp2_m = (v16u8) __msa_ilvod_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2517 tmp3_m = (v16u8) __msa_ilvod_h((v8i16) out7, (v8i16) out5); \ 2518 out2 = (v16u8) __msa_ilvev_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2519 out6 = (v16u8) __msa_ilvod_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2521 ILVEV_H2_UB(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \ 2522 out1 = (v16u8) __msa_ilvev_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2523 out5 = (v16u8) __msa_ilvod_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2525 tmp2_m = (v16u8) __msa_ilvod_h((v8i16) tmp5_m, (v8i16) tmp4_m); \ 2526 tmp2_m = (v16u8) __msa_ilvod_h((v8i16) tmp5_m, (v8i16) tmp4_m); \ 2527 tmp3_m = (v16u8) __msa_ilvod_h((v8i16) tmp7_m, (v8i16) tmp6_m); \ 2528 tmp3_m = (v16u8) __msa_ilvod_h((v8i16) tmp7_m, (v8i16) tmp6_m); \ 2529 out3 = (v16u8) __msa_ilvev_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2530 out7 = (v16u8) __msa_ilvod_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2539 #define TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, out0, out1, out2, out3) \ 2543 ILVR_H2_SH(in1, in0, in3, in2, s0_m, s1_m); \ 2544 ILVRL_W2_SH(s1_m, s0_m, out0, out2); \ 2545 out1 = (v8i16) __msa_ilvl_d((v2i64) out0, (v2i64) out0); \ 2546 out3 = (v8i16) __msa_ilvl_d((v2i64) out0, (v2i64) out2); \ 2555 #define TRANSPOSE8x8_H(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 2556 out0, out1, out2, out3, out4, out5, out6, out7) \ 2559 v8i16 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2560 v8i16 tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ 2562 ILVR_H2_SH(in6, in4, in7, in5, s0_m, s1_m); \ 2563 ILVRL_H2_SH(s1_m, s0_m, tmp0_m, tmp1_m); \ 2564 ILVL_H2_SH(in6, in4, in7, in5, s0_m, s1_m); \ 2565 ILVRL_H2_SH(s1_m, s0_m, tmp2_m, tmp3_m); \ 2566 ILVR_H2_SH(in2, in0, in3, in1, s0_m, s1_m); \ 2567 ILVRL_H2_SH(s1_m, s0_m, tmp4_m, tmp5_m); \ 2568 ILVL_H2_SH(in2, in0, in3, in1, s0_m, s1_m); \ 2569 ILVRL_H2_SH(s1_m, s0_m, tmp6_m, tmp7_m); \ 2570 PCKEV_D4(RTYPE, tmp0_m, tmp4_m, tmp1_m, tmp5_m, tmp2_m, tmp6_m, \ 2571 tmp3_m, tmp7_m, out0, out2, out4, out6); \ 2572 out1 = (RTYPE) __msa_pckod_d((v2i64) tmp0_m, (v2i64) tmp4_m); \ 2573 out3 = (RTYPE) __msa_pckod_d((v2i64) tmp1_m, (v2i64) tmp5_m); \ 2574 out5 = (RTYPE) __msa_pckod_d((v2i64) tmp2_m, (v2i64) tmp6_m); \ 2575 out7 = (RTYPE) __msa_pckod_d((v2i64) tmp3_m, (v2i64) tmp7_m); \ 2577 #define TRANSPOSE8x8_UH_UH(...) TRANSPOSE8x8_H(v8u16, __VA_ARGS__) 2578 #define TRANSPOSE8x8_SH_SH(...) TRANSPOSE8x8_H(v8i16, __VA_ARGS__) 2586 #define TRANSPOSE4x4_SW_SW(in0, in1, in2, in3, out0, out1, out2, out3) \ 2588 v4i32 s0_m, s1_m, s2_m, s3_m; \ 2590 ILVRL_W2_SW(in1, in0, s0_m, s1_m); \ 2591 ILVRL_W2_SW(in3, in2, s2_m, s3_m); \ 2593 out0 = (v4i32) __msa_ilvr_d((v2i64) s2_m, (v2i64) s0_m); \ 2594 out1 = (v4i32) __msa_ilvl_d((v2i64) s2_m, (v2i64) s0_m); \ 2595 out2 = (v4i32) __msa_ilvr_d((v2i64) s3_m, (v2i64) s1_m); \ 2596 out3 = (v4i32) __msa_ilvl_d((v2i64) s3_m, (v2i64) s1_m); \ 2613 #define AVE_ST8x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 2615 uint64_t out0_m, out1_m, out2_m, out3_m; \ 2616 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2618 tmp0_m = __msa_ave_u_b((v16u8) in0, (v16u8) in1); \ 2619 tmp1_m = __msa_ave_u_b((v16u8) in2, (v16u8) in3); \ 2620 tmp2_m = __msa_ave_u_b((v16u8) in4, (v16u8) in5); \ 2621 tmp3_m = __msa_ave_u_b((v16u8) in6, (v16u8) in7); \ 2623 out0_m = __msa_copy_u_d((v2i64) tmp0_m, 0); \ 2624 out1_m = __msa_copy_u_d((v2i64) tmp1_m, 0); \ 2625 out2_m = __msa_copy_u_d((v2i64) tmp2_m, 0); \ 2626 out3_m = __msa_copy_u_d((v2i64) tmp3_m, 0); \ 2627 SD4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \ 2644 #define AVE_ST16x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 2646 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2648 tmp0_m = __msa_ave_u_b((v16u8) in0, (v16u8) in1); \ 2649 tmp1_m = __msa_ave_u_b((v16u8) in2, (v16u8) in3); \ 2650 tmp2_m = __msa_ave_u_b((v16u8) in4, (v16u8) in5); \ 2651 tmp3_m = __msa_ave_u_b((v16u8) in6, (v16u8) in7); \ 2653 ST_UB4(tmp0_m, tmp1_m, tmp2_m, tmp3_m, pdst, stride); \ 2670 #define AVER_ST8x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 2672 uint64_t out0_m, out1_m, out2_m, out3_m; \ 2673 v16u8 tp0_m, tp1_m, tp2_m, tp3_m; \ 2675 AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2676 tp0_m, tp1_m, tp2_m, tp3_m); \ 2678 out0_m = __msa_copy_u_d((v2i64) tp0_m, 0); \ 2679 out1_m = __msa_copy_u_d((v2i64) tp1_m, 0); \ 2680 out2_m = __msa_copy_u_d((v2i64) tp2_m, 0); \ 2681 out3_m = __msa_copy_u_d((v2i64) tp3_m, 0); \ 2682 SD4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \ 2699 #define AVER_ST16x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 2701 v16u8 t0_m, t1_m, t2_m, t3_m; \ 2703 AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2704 t0_m, t1_m, t2_m, t3_m); \ 2705 ST_UB4(t0_m, t1_m, t2_m, t3_m, pdst, stride); \ 2723 #define AVER_DST_ST8x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2726 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2727 v16u8 dst0_m, dst1_m, dst2_m, dst3_m; \ 2729 LD_UB4(pdst, stride, dst0_m, dst1_m, dst2_m, dst3_m); \ 2730 AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2731 tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ 2732 AVER_ST8x4_UB(dst0_m, tmp0_m, dst1_m, tmp1_m, \ 2733 dst2_m, tmp2_m, dst3_m, tmp3_m, pdst, stride); \ 2751 #define AVER_DST_ST16x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2754 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2755 v16u8 dst0_m, dst1_m, dst2_m, dst3_m; \ 2757 LD_UB4(pdst, stride, dst0_m, dst1_m, dst2_m, dst3_m); \ 2758 AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2759 tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ 2760 AVER_ST16x4_UB(dst0_m, tmp0_m, dst1_m, tmp1_m, \ 2761 dst2_m, tmp2_m, dst3_m, tmp3_m, pdst, stride); \ 2769 #define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) \ 2771 uint32_t src0_m, src1_m, src2_m, src3_m; \ 2772 uint32_t out0_m, out1_m, out2_m, out3_m; \ 2773 v8i16 inp0_m, inp1_m, res0_m, res1_m; \ 2774 v16i8 dst0_m = { 0 }; \ 2775 v16i8 dst1_m = { 0 }; \ 2776 v16i8 zero_m = { 0 }; \ 2778 ILVR_D2_SH(in1, in0, in3, in2, inp0_m, inp1_m) \ 2779 LW4(pdst, stride, src0_m, src1_m, src2_m, src3_m); \ 2780 INSERT_W2_SB(src0_m, src1_m, dst0_m); \ 2781 INSERT_W2_SB(src2_m, src3_m, dst1_m); \ 2782 ILVR_B2_SH(zero_m, dst0_m, zero_m, dst1_m, res0_m, res1_m); \ 2783 ADD2(res0_m, inp0_m, res1_m, inp1_m, res0_m, res1_m); \ 2784 CLIP_SH2_0_255(res0_m, res1_m); \ 2785 PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m); \ 2787 out0_m = __msa_copy_u_w((v4i32) dst0_m, 0); \ 2788 out1_m = __msa_copy_u_w((v4i32) dst0_m, 1); \ 2789 out2_m = __msa_copy_u_w((v4i32) dst1_m, 0); \ 2790 out3_m = __msa_copy_u_w((v4i32) dst1_m, 1); \ 2791 SW4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \ 2805 #define DPADD_SH3_SH(in0, in1, in2, coeff0, coeff1, coeff2) \ 2809 out0_m = __msa_dotp_s_h((v16i8) in0, (v16i8) coeff0); \ 2810 out0_m = __msa_dpadd_s_h(out0_m, (v16i8) in1, (v16i8) coeff1); \ 2811 out0_m = __msa_dpadd_s_h(out0_m, (v16i8) in2, (v16i8) coeff2); \ 2824 #define PCKEV_XORI128_UB(in0, in1) \ 2827 out_m = (v16u8) __msa_pckev_b((v16i8) in1, (v16i8) in0); \ 2828 out_m = (v16u8) __msa_xori_b((v16u8) out_m, 128); \ 2836 #define CONVERT_UB_AVG_ST8x4_UB(in0, in1, in2, in3, \ 2837 dst0, dst1, pdst, stride) \ 2839 v16u8 tmp0_m, tmp1_m; \ 2840 uint8_t *pdst_m = (uint8_t *) (pdst); \ 2842 tmp0_m = PCKEV_XORI128_UB(in0, in1); \ 2843 tmp1_m = PCKEV_XORI128_UB(in2, in3); \ 2844 AVER_UB2_UB(tmp0_m, dst0, tmp1_m, dst1, tmp0_m, tmp1_m); \ 2845 ST_D4(tmp0_m, tmp1_m, 0, 1, 0, 1, pdst_m, stride); \ 2853 #define PCKEV_ST4x4_UB(in0, in1, in2, in3, pdst, stride) \ 2855 uint32_t out0_m, out1_m, out2_m, out3_m; \ 2856 v16i8 tmp0_m, tmp1_m; \ 2858 PCKEV_B2_SB(in1, in0, in3, in2, tmp0_m, tmp1_m); \ 2860 out0_m = __msa_copy_u_w((v4i32) tmp0_m, 0); \ 2861 out1_m = __msa_copy_u_w((v4i32) tmp0_m, 2); \ 2862 out2_m = __msa_copy_u_w((v4i32) tmp1_m, 0); \ 2863 out3_m = __msa_copy_u_w((v4i32) tmp1_m, 2); \ 2865 SW4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \ 2872 #define PCKEV_ST_SB(in0, in1, pdst) \ 2875 tmp_m = __msa_pckev_b((v16i8) in1, (v16i8) in0); \ 2876 ST_SB(tmp_m, (pdst)); \ 2882 #define HORIZ_2TAP_FILT_UH(in0, in1, mask, coeff, shift) \ 2887 tmp0_m = __msa_vshf_b((v16i8) mask, (v16i8) in1, (v16i8) in0); \ 2888 tmp1_m = __msa_dotp_u_h((v16u8) tmp0_m, (v16u8) coeff); \ 2889 tmp1_m = (v8u16) __msa_srari_h((v8i16) tmp1_m, shift); \ 2890 tmp1_m = __msa_sat_u_h(tmp1_m, shift); \