dilim/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2026-08-09 09:32:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

swscale/x86/ops: add AVX512 vpermb packed shuffle path

Browse Source 
We can quite easily generalize the existing AVX512 pushfb implementation to
support vpermb when the shuffle mask is cross-lane.

  rgb24 1920x1080 -> rgba 1920x1080, speedup=1.372x faster

This does require some finesse w.r.t. the MOVSIZE macro, to avoid reading or
writing too many bits. In particular, for gray->rgba64le, the pattern even
devolves into 8/64 which would trigger an illegal operand error when combining
movq with zmm registers.

Other than that, we can use {k1}{z} zero masking to cleanly recreate the
semantics of pushfb's implicit zero masking, except for non-zero clear
values where we continue doing what we already do.

Lastly, while EVEX-encoded vpermb can take the source operand directly, this
somewhat defeats the purpose of the READ size optimization; in practice it
was better to keep the read and vpermb separate.

Signed-off-by: Niklas Haas <git@haasn.dev>
This commit is contained in:
Niklas Haas
2026-07-14 18:13:05 +02:00
committed by Niklas Haas
parent 8fdafc8198
commit a785ff4ca2
2 changed files with 71 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
libswscale/x86/ops.c
View File

@@ -489,9 +489,10 @@ SWS_DECL_FUNC(ff_sws_process3_x86);
SWS_DECL_FUNC(ff_sws_process4_x86);
/* Declare packed shuffle functions */
SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _sse4, NULL, NULL)
SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx2, NULL, NULL)
SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx512, NULL, NULL)
SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _sse4, NULL, NULL)
SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx2, NULL, NULL)
SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx512, NULL, NULL)
SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx512icl, NULL, NULL)
static int get_mmsize(void)
{
@@ -508,19 +509,22 @@ static int get_mmsize(void)
static int movsize(const int bytes, const int mmsize)
{
return bytes <= 4 ? 4 : /* movd */
bytes <= 8 ? 8 : /* movq */
mmsize; /* movu */
return bytes <= 4 ? 4 : /* movd */
bytes <= 8 ? 8 : /* movq */
bytes <= 16 ? 16 : /* xmm movu */
bytes <= 32 ? 32 : /* ymm movu */
mmsize; /* zmm movu */
}
static int translate_shuffle(const SwsUOp *uop, int mmsize, SwsCompiledOp *out)
{
/* We can't shuffle across lanes, so restrict the vector size to XMM
* whenever the read/write size would be a subset of the full vector */
* whenever the read/write size would be a subset of the full vector,
* unless we have access to AVX-512 ICL vpermb */
const SwsShuffleUOp *par = &uop->par.shuffle;
const int lane_aligned = par->read_size == par->write_size &&
16 % par->read_size == 0;
if (!lane_aligned)
if (!lane_aligned && !EXTERNAL_AVX512ICL(av_get_cpu_flags()))
mmsize = 16;
/* Generate the shuffle mask */
@@ -547,22 +551,27 @@ static int translate_shuffle(const SwsUOp *uop, int mmsize, SwsCompiledOp *out)
.block_size = groups * uop->data.shuffle.pixels * num_lanes,
.over_read = { movsize(in_total, mmsize) - in_total },
.over_write = { movsize(out_total, mmsize) - out_total },
.cpu_flags = mmsize > 32 ? AV_CPU_FLAG_AVX512 :
mmsize > 16 ? AV_CPU_FLAG_AVX2 :
AV_CPU_FLAG_SSE4,
};
#define ASSIGN_SHUFFLE_FUNC(EXT, NAME, ...) \
#define ASSIGN_SHUFFLE_FUNC(CPU, EXT, NAME, ...) \
do { \
const SwsUOpEntry *entry = &uop_##NAME##EXT; \
if (!memcmp(&uop->par, &entry->par, sizeof(uop->par))) \
if (!memcmp(&uop->par, &entry->par, sizeof(uop->par))) { \
out->func = (SwsOpFunc) entry->func; \
out->cpu_flags = AV_CPU_FLAG_##CPU; \
} \
} while (0);
switch (mmsize) {
case 16: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, _sse4); break;
case 32: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, _avx2); break;
case 64: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, _avx512); break;
case 16: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, SSE4, _sse4); break;
case 32: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, AVX2, _avx2); break;
case 64:
if (lane_aligned) {
SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, AVX512, _avx512);
} else { /* vpermb variant */
SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, AVX512ICL, _avx512icl);
}
break;
}
if (!out->func) {

58
libswscale/x86/ops_common.asm
View File

@@ -127,13 +127,31 @@ process_fn 4
; in which case we can also use `pshufb` on mmsize == 32/64. This is detected
; by the `LANE_ALIGNED` condition.
%macro MOVSIZE 3 ; size, dst, src
%if %1 <= 4
movd %2, %3
%elif %1 <= 8
movq %2, %3
%macro READ 2 ; dst, src
%if READ_SIZE <= 4
movd xmm%1, %2
%elif READ_SIZE <= 8
movq xmm%1, %2
%elif READ_SIZE <= 16
movu xmm%1, %2
%elif READ_SIZE <= 32
movu ymm%1, %2
%else
movu %2, %3
movu zmm%1, %2
%endif
%endmacro
%macro WRITE 2 ; dst, src
%if WRITE_SIZE <= 4
movd %1, xmm%2
%elif WRITE_SIZE <= 8
movq %1, xmm%2
%elif WRITE_SIZE <= 16
movu %1, xmm%2
%elif WRITE_SIZE <= 32
movu %1, ymm%2
%else
movu %1, zmm%2
%endif
%endmacro
@@ -152,7 +170,7 @@ process_fn 4
%assign WRITE_SIZE (WRITE_SIZE * GROUPS)
cglobal NAME, 6, 10, 3, exec, shuffle, bx, y, bxend, yend, src, dst, src_stride, dst_stride
%if mmsize > 16 && !LANE_ALIGNED
%if mmsize > 16 && !LANE_ALIGNED && !cpuflag(avx512icl)
ud2 ; runtime checks should prevent this variant from being called
%else
mov srcq, [execq + SwsOpExec.in0]
@@ -161,9 +179,16 @@ cglobal NAME, 6, 10, 3, exec, shuffle, bx, y, bxend, yend, src, dst, src_stride,
mov dst_strideq, [execq + SwsOpExec.out_stride0]
; setup shuffle mask
%if LANE_ALIGNED
VBROADCASTI128 m0, [shuffleq]
%if cpuflag(avx512) && CLEAR_VALUE != 0
vpmovb2m k1, m0 ; needed for vpblendmb
%else
mova m0, [shuffleq]
%endif
%if cpuflag(avx512)
vpmovb2m k1, m0 ; needed for vpblendmb / vpermb
%if CLEAR_VALUE == 0
knotq k1, k1
%endif
%endif
; setup clear value register if needed
@@ -173,7 +198,7 @@ cglobal NAME, 6, 10, 3, exec, shuffle, bx, y, bxend, yend, src, dst, src_stride,
%else
pcmpeqb m2, m2
%endif
%elif CLEAR_VALUE != 0 ; clear-to-0 is implicitly handled by pshufb
%elif CLEAR_VALUE != 0 ; clear-to-0 is implicitly handled by pshufb / vpermb
mov shuffled, CLEAR_VALUE * 0x1010101
movd xm2, shuffled
VPBROADCASTD m2, xm2
@@ -195,8 +220,14 @@ cglobal NAME, 6, 10, 3, exec, shuffle, bx, y, bxend, yend, src, dst, src_stride,
sub dstq, dstidxq
.loop:
MOVSIZE READ_SIZE, m1, [srcq + srcidxq]
READ m1, [srcq + srcidxq]
%if LANE_ALIGNED || mmsize == 16
pshufb m1, m0
%elif CLEAR_VALUE == 0
vpermb m1{k1}{z}, m0, m1
%else
vpermb m1, m0, m1
%endif
%if CLEAR_VALUE != 0
%if cpuflag(avx512)
@@ -208,7 +239,7 @@ cglobal NAME, 6, 10, 3, exec, shuffle, bx, y, bxend, yend, src, dst, src_stride,
%endif
%endif
MOVSIZE WRITE_SIZE, [dstq + dstidxq], m1
WRITE [dstq + dstidxq], m1
add srcidxq, READ_SIZE
%if READ_SIZE != WRITE_SIZE
add dstidxq, WRITE_SIZE
@@ -234,3 +265,6 @@ DECL_U8_RW_SHUFFLE (RW_SHUFFLE)
INIT_ZMM avx512
DECL_U8_RW_SHUFFLE (RW_SHUFFLE)
INIT_ZMM avx512icl
DECL_U8_RW_SHUFFLE (RW_SHUFFLE)