From 2a407092ab0a18628a4a789f4e06e2408a17c4cd Mon Sep 17 00:00:00 2001 From: Pascal Massimino Date: Wed, 18 Feb 2015 09:40:30 +0100 Subject: [PATCH] 4-5% faster encoding using SSE2 for GetResidualCost new file: cost_sse2.c Change-Id: I4896c07f5ff2443ef743f4435fe2758d95a672ed --- Android.mk | 1 + Makefile.vc | 1 + makefile.unix | 1 + src/dsp/Makefile.am | 1 + src/dsp/cost.c | 8 +-- src/dsp/cost_sse2.c | 121 ++++++++++++++++++++++++++++++++++++++++++++ src/dsp/enc_sse2.c | 28 ---------- 7 files changed, 127 insertions(+), 34 deletions(-) create mode 100644 src/dsp/cost_sse2.c diff --git a/Android.mk b/Android.mk index 93656190..aa9b159c 100644 --- a/Android.mk +++ b/Android.mk @@ -41,6 +41,7 @@ LOCAL_SRC_FILES := \ src/dsp/cost.c \ src/dsp/cost_mips32.c \ src/dsp/cost_mips_dsp_r2.c \ + src/dsp/cost_sse2.c \ src/dsp/cpu.c \ src/dsp/dec.c \ src/dsp/dec_clip_tables.c \ diff --git a/Makefile.vc b/Makefile.vc index 47abe665..74ccd64f 100644 --- a/Makefile.vc +++ b/Makefile.vc @@ -222,6 +222,7 @@ DSP_ENC_OBJS = \ $(DIROBJ)\dsp\cost.obj \ $(DIROBJ)\dsp\cost_mips32.obj \ $(DIROBJ)\dsp\cost_mips_dsp_r2.obj \ + $(DIROBJ)\dsp\cost_sse2.obj \ $(DIROBJ)\dsp\enc.obj \ $(DIROBJ)\dsp\enc_avx2.obj \ $(DIROBJ)\dsp\enc_mips32.obj \ diff --git a/makefile.unix b/makefile.unix index fc6e283d..277e5e4f 100644 --- a/makefile.unix +++ b/makefile.unix @@ -146,6 +146,7 @@ DSP_ENC_OBJS = \ src/dsp/cost.o \ src/dsp/cost_mips32.o \ src/dsp/cost_mips_dsp_r2.o \ + src/dsp/cost_sse2.o \ src/dsp/enc.o \ src/dsp/enc_avx2.o \ src/dsp/enc_mips32.o \ diff --git a/src/dsp/Makefile.am b/src/dsp/Makefile.am index 307c4409..30e4be86 100644 --- a/src/dsp/Makefile.am +++ b/src/dsp/Makefile.am @@ -67,6 +67,7 @@ libwebpdspdecode_sse2_la_CFLAGS = $(libwebpdsp_sse2_la_CFLAGS) libwebpdsp_sse2_la_SOURCES = libwebpdsp_sse2_la_SOURCES += argb_sse2.c +libwebpdsp_sse2_la_SOURCES += cost_sse2.c libwebpdsp_sse2_la_SOURCES += enc_sse2.c libwebpdsp_sse2_la_CPPFLAGS = $(libwebpdsp_la_CPPFLAGS) libwebpdsp_sse2_la_CFLAGS = $(AM_CFLAGS) $(SSE2_FLAGS) diff --git a/src/dsp/cost.c b/src/dsp/cost.c index 724e499f..ebd75013 100644 --- a/src/dsp/cost.c +++ b/src/dsp/cost.c @@ -376,11 +376,7 @@ VP8SetResidualCoeffsFunc VP8SetResidualCoeffs; extern void VP8EncDspCostInitMIPS32(void); extern void VP8EncDspCostInitMIPSdspR2(void); - -#if defined(WEBP_USE_SSE2) -extern void VP8SetResidualCoeffsSSE2(const int16_t* const coeffs, - VP8Residual* const res); -#endif // WEBP_USE_SSE2 +extern void VP8EncDspCostInitSSE2(void); static volatile VP8CPUInfo cost_last_cpuinfo_used = (VP8CPUInfo)&cost_last_cpuinfo_used; @@ -405,7 +401,7 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspCostInit(void) { #endif #if defined(WEBP_USE_SSE2) if (VP8GetCPUInfo(kSSE2)) { - VP8SetResidualCoeffs = VP8SetResidualCoeffsSSE2; + VP8EncDspCostInitSSE2(); } #endif } diff --git a/src/dsp/cost_sse2.c b/src/dsp/cost_sse2.c new file mode 100644 index 00000000..540d6820 --- /dev/null +++ b/src/dsp/cost_sse2.c @@ -0,0 +1,121 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE2 version of cost functions +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "./dsp.h" + +#if defined(WEBP_USE_SSE2) +#include + +#include "../enc/cost.h" +#include "../enc/vp8enci.h" +#include "../utils/utils.h" + +//------------------------------------------------------------------------------ + +static void SetResidualCoeffsSSE2(const int16_t* const coeffs, + VP8Residual* const res) { + const __m128i c0 = _mm_loadu_si128((const __m128i*)coeffs); + const __m128i c1 = _mm_loadu_si128((const __m128i*)(coeffs + 8)); + // Use SSE to compare 8 values with a single instruction. + const __m128i zero = _mm_setzero_si128(); + const __m128i m0 = _mm_cmpeq_epi16(c0, zero); + const __m128i m1 = _mm_cmpeq_epi16(c1, zero); + // Get the comparison results as a bitmask, consisting of two times 16 bits: + // two identical bits for each result. Concatenate both bitmasks to get a + // single 32 bit value. Negate the mask to get the position of entries that + // are not equal to zero. We don't need to mask out least significant bits + // according to res->first, since coeffs[0] is 0 if res->first > 0 + const uint32_t mask = + ~(((uint32_t)_mm_movemask_epi8(m1) << 16) | _mm_movemask_epi8(m0)); + // The position of the most significant non-zero bit indicates the position of + // the last non-zero value. Divide the result by two because __movemask_epi8 + // operates on 8 bit values instead of 16 bit values. + assert(res->first == 0 || coeffs[0] == 0); + res->last = mask ? (BitsLog2Floor(mask) >> 1) : -1; + res->coeffs = coeffs; +} + +static int GetResidualCostSSE2(int ctx0, const VP8Residual* const res) { + uint8_t levels[16], ctxs[16]; + uint16_t abs_levels[16]; + int n = res->first; + // should be prob[VP8EncBands[n]], but it's equivalent for n=0 or 1 + const int p0 = res->prob[n][ctx0][0]; + const uint16_t* t = res->cost[n][ctx0]; + // bit_cost(1, p0) is already incorporated in t[] tables, but only if ctx != 0 + // (as required by the syntax). For ctx0 == 0, we need to add it here or it'll + // be missing during the loop. + int cost = (ctx0 == 0) ? VP8BitCost(1, p0) : 0; + + if (res->last < 0) { + return VP8BitCost(0, p0); + } + + { // precompute clamped levels and contexts, packed to 8b. + const __m128i zero = _mm_setzero_si128(); + const __m128i kCst2 = _mm_set1_epi8(2); + const __m128i kCst67 = _mm_set1_epi8(MAX_VARIABLE_LEVEL); + const __m128i c0 = _mm_loadu_si128((const __m128i*)&res->coeffs[0]); + const __m128i c1 = _mm_loadu_si128((const __m128i*)&res->coeffs[8]); + const __m128i D0_m = _mm_min_epi16(c0, zero); + const __m128i D0_p = _mm_max_epi16(c0, zero); + const __m128i D1_m = _mm_min_epi16(c1, zero); + const __m128i D1_p = _mm_max_epi16(c1, zero); + const __m128i E0 = _mm_sub_epi16(D0_p, D0_m); // abs(v), 16b + const __m128i E1 = _mm_sub_epi16(D1_p, D1_m); + const __m128i F = _mm_packs_epi16(E0, E1); + const __m128i G = _mm_min_epu8(F, kCst2); // context = 0,1,2 + const __m128i H = _mm_min_epu8(F, kCst67); // clamp_level in [0..67] + + _mm_storeu_si128((__m128i*)&ctxs[0], G); + _mm_storeu_si128((__m128i*)&levels[0], H); + + _mm_storeu_si128((__m128i*)&abs_levels[0], E0); + _mm_storeu_si128((__m128i*)&abs_levels[8], E1); + } + for (; n < res->last; ++n) { + const int ctx = ctxs[n]; + const int level = levels[n]; + const int flevel = abs_levels[n]; // full level + const int b = VP8EncBands[n + 1]; + cost += VP8LevelFixedCosts[flevel] + t[level]; // simplified VP8LevelCost() + t = res->cost[b][ctx]; + } + // Last coefficient is always non-zero + { + const int level = levels[n]; + const int flevel = abs_levels[n]; + assert(flevel != 0); + cost += VP8LevelFixedCosts[flevel] + t[level]; + if (n < 15) { + const int b = VP8EncBands[n + 1]; + const int ctx = ctxs[n]; + const int last_p0 = res->prob[b][ctx][0]; + cost += VP8BitCost(0, last_p0); + } + } + return cost; +} +#endif // WEBP_USE_SSE2 + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8EncDspCostInitSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspCostInitSSE2(void) { +#if defined(WEBP_USE_SSE2) + VP8SetResidualCoeffs = SetResidualCoeffsSSE2; + VP8GetResidualCost = GetResidualCostSSE2; +#endif // WEBP_USE_SSE2 +} diff --git a/src/dsp/enc_sse2.c b/src/dsp/enc_sse2.c index fe02e701..b1b50f48 100644 --- a/src/dsp/enc_sse2.c +++ b/src/dsp/enc_sse2.c @@ -914,33 +914,6 @@ static int Quantize2Blocks(int16_t in[32], int16_t out[32], return nz; } -// Forward declaration. -void VP8SetResidualCoeffsSSE2(const int16_t* const coeffs, - VP8Residual* const res); - -void VP8SetResidualCoeffsSSE2(const int16_t* const coeffs, - VP8Residual* const res) { - const __m128i c0 = _mm_loadu_si128((const __m128i*)coeffs); - const __m128i c1 = _mm_loadu_si128((const __m128i*)(coeffs + 8)); - // Use SSE to compare 8 values with a single instruction. - const __m128i zero = _mm_setzero_si128(); - const __m128i m0 = _mm_cmpeq_epi16(c0, zero); - const __m128i m1 = _mm_cmpeq_epi16(c1, zero); - // Get the comparison results as a bitmask, consisting of two times 16 bits: - // two identical bits for each result. Concatenate both bitmasks to get a - // single 32 bit value. Negate the mask to get the position of entries that - // are not equal to zero. We don't need to mask out least significant bits - // according to res->first, since coeffs[0] is 0 if res->first > 0 - const uint32_t mask = - ~(((uint32_t)_mm_movemask_epi8(m1) << 16) | _mm_movemask_epi8(m0)); - // The position of the most significant non-zero bit indicates the position of - // the last non-zero value. Divide the result by two because __movemask_epi8 - // operates on 8 bit values instead of 16 bit values. - assert(res->first == 0 || coeffs[0] == 0); - res->last = mask ? (BitsLog2Floor(mask) >> 1) : -1; - res->coeffs = coeffs; -} - #endif // WEBP_USE_SSE2 //------------------------------------------------------------------------------ @@ -965,4 +938,3 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitSSE2(void) { VP8TDisto16x16 = Disto16x16; #endif // WEBP_USE_SSE2 } -