From e291fae0fca84faba48f75818e88b859a3a14d53 Mon Sep 17 00:00:00 2001 From: Pascal Massimino Date: Thu, 7 Jul 2011 16:50:04 -0700 Subject: [PATCH] SSE2 functions for the fancy upsampler. ~5-10% faster. Heavy 8bit arithmetic trickery! Patch by Somnath Banerjee (somnath at google dot com) Change-Id: I9fd2c511d9f631e9cf4b008c46127b49fb527b47 --- Makefile.vc | 1 + makefile.unix | 2 +- src/dec/Makefile.am | 2 +- src/dec/io.c | 52 ++++++----- src/dec/io_sse2.c | 205 ++++++++++++++++++++++++++++++++++++++++++++ src/dec/webpi.h | 16 ++++ src/enc/dsp_sse2.c | 148 +++++++++++++++++--------------- 7 files changed, 332 insertions(+), 94 deletions(-) create mode 100644 src/dec/io_sse2.c diff --git a/Makefile.vc b/Makefile.vc index ad5f7d0c..0afa4cf8 100644 --- a/Makefile.vc +++ b/Makefile.vc @@ -123,6 +123,7 @@ X_OBJS= \ $(DIROBJ)\dec\vp8.obj \ $(DIROBJ)\dec\webp.obj \ $(DIROBJ)\dec\io.obj \ + $(DIROBJ)\dec\io_sse2.obj \ $(DIROBJ)\dec\buffer.obj \ $(DIROBJ)\dec\yuv.obj \ $(DIROBJ)\dec\idec.obj \ diff --git a/makefile.unix b/makefile.unix index d4a8261b..a8149377 100644 --- a/makefile.unix +++ b/makefile.unix @@ -58,7 +58,7 @@ OBJS = src/enc/webpenc.o src/enc/bit_writer.o src/enc/syntax.o \ src/dec/bits.o src/dec/dsp.o src/dec/dsp_sse2.o src/dec/frame.o \ src/dec/webp.o src/dec/quant.o src/dec/tree.o src/dec/vp8.o \ src/dec/yuv.o src/dec/idec.o src/dec/alpha.o src/dec/layer.o \ - src/dec/io.o src/dec/buffer.o + src/dec/io.o src/dec/io_sse2.o src/dec/buffer.o HDRS = src/webp/encode.h src/enc/vp8enci.h src/enc/bit_writer.h \ src/enc/cost.h src/dec/bits.h src/dec/vp8i.h src/dec/yuv.h OUTPUT = examples/cwebp examples/dwebp src/libwebp.a diff --git a/src/dec/Makefile.am b/src/dec/Makefile.am index e859090c..382d9409 100644 --- a/src/dec/Makefile.am +++ b/src/dec/Makefile.am @@ -2,7 +2,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src libwebpdecode_la_SOURCES = bits.h vp8i.h yuv.h bits.c dsp.c dsp_sse2.c frame.c \ quant.c tree.c vp8.c webp.c yuv.c idec.c alpha.c \ - layer.c io.c buffer.c + layer.c io.c io_sse2.c buffer.c libwebpdecode_la_LDFLAGS = -version-info 0:0:0 libwebpdecode_la_CPPFLAGS = $(USE_EXPERIMENTAL_CODE) libwebpdecodeinclude_HEADERS = ../webp/decode.h ../webp/decode_vp8.h ../webp/types.h diff --git a/src/dec/io.c b/src/dec/io.c index 80233ae6..e8ee6ffb 100644 --- a/src/dec/io.c +++ b/src/dec/io.c @@ -105,29 +105,34 @@ UPSAMPLE_FUNC(UpsampleBgraLinePair, VP8YuvToBgra, 4) UPSAMPLE_FUNC(UpsampleRgbKeepAlphaLinePair, VP8YuvToRgb, 4) UPSAMPLE_FUNC(UpsampleBgrKeepAlphaLinePair, VP8YuvToBgr, 4) -typedef void (*UpsampleLinePairFunc)( - const uint8_t* top_y, const uint8_t* bottom_y, - const uint8_t* top_u, const uint8_t* top_v, - const uint8_t* cur_u, const uint8_t* cur_v, - uint8_t* top_dst, uint8_t* bottom_dst, int len); - -static const UpsampleLinePairFunc - kUpsamplers[MODE_BGRA + 1] = { - UpsampleRgbLinePair, // MODE_RGB - UpsampleRgbaLinePair, // MODE_RGBA - UpsampleBgrLinePair, // MODE_BGR - UpsampleBgraLinePair // MODE_BGRA - }, - kUpsamplersKeepAlpha[MODE_BGRA + 1] = { - UpsampleRgbLinePair, // MODE_RGB - UpsampleRgbKeepAlphaLinePair, // MODE_RGBA - UpsampleBgrLinePair, // MODE_BGR - UpsampleBgrKeepAlphaLinePair // MODE_BGRA - }; - #undef LOAD_UV #undef UPSAMPLE_FUNC +// Fancy upsampling functions to convert YUV to RGB +WebPUpsampleLinePairFunc WebPUpsamplers[MODE_BGRA + 1]; +WebPUpsampleLinePairFunc WebPUpsamplersKeepAlpha[MODE_BGRA + 1]; + +static void InitUpsamplers(void) { + WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair; + WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair; + WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair; + WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePair; + + WebPUpsamplersKeepAlpha[MODE_RGB] = UpsampleRgbLinePair; + WebPUpsamplersKeepAlpha[MODE_RGBA] = UpsampleRgbKeepAlphaLinePair; + WebPUpsamplersKeepAlpha[MODE_BGR] = UpsampleBgrLinePair; + WebPUpsamplersKeepAlpha[MODE_BGRA] = UpsampleBgrKeepAlphaLinePair; + + // If defined, use CPUInfo() to overwrite some pointers with faster versions. + if (VP8DecGetCPUInfo) { + if (VP8DecGetCPUInfo(kSSE2)) { +#if defined(__SSE2__) || defined(_MSC_VER) + WebPInitUpsamplersSSE2(); +#endif + } + } +} + #endif // FANCY_UPSAMPLING //------------------------------------------------------------------------------ @@ -287,9 +292,9 @@ static int EmitFancyRGB(const VP8Io* const io, WebPDecParams* const p) { int num_lines_out = io->mb_h; // a priori guess const WebPRGBABuffer* const buf = &p->output->u.RGBA; uint8_t* dst = buf->rgba + io->mb_y * buf->stride; - const UpsampleLinePairFunc upsample = - io->a ? kUpsamplersKeepAlpha[p->output->colorspace] - : kUpsamplers[p->output->colorspace]; + const WebPUpsampleLinePairFunc upsample = + io->a ? WebPUpsamplersKeepAlpha[p->output->colorspace] + : WebPUpsamplers[p->output->colorspace]; const uint8_t* cur_y = io->y; const uint8_t* cur_u = io->u; const uint8_t* cur_v = io->v; @@ -781,6 +786,7 @@ static int CustomSetup(VP8Io* io) { p->tmp_u = p->tmp_y + io->mb_w; p->tmp_v = p->tmp_u + uv_width; p->emit = EmitFancyRGB; + InitUpsamplers(); } #endif } else { diff --git a/src/dec/io_sse2.c b/src/dec/io_sse2.c new file mode 100644 index 00000000..0f42350b --- /dev/null +++ b/src/dec/io_sse2.c @@ -0,0 +1,205 @@ +// Copyright 2011 Google Inc. +// +// This code is licensed under the same terms as WebM: +// Software License Agreement: http://www.webmproject.org/license/software/ +// Additional IP Rights Grant: http://www.webmproject.org/license/additional/ +// ----------------------------------------------------------------------------- +// +// SSE2 version of YUV to RGB upsampling functions. +// +// Author: somnath@google.com (Somnath Banerjee) + +#if defined(__SSE2__) || defined(_MSC_VER) + +#include +#include +#include +#include "webpi.h" +#include "yuv.h" + +#if defined(__cplusplus) || defined(c_plusplus) +extern "C" { +#endif + +// We compute (9*a + 3*b + 3*c + d + 8) / 16 as follows +// u = (9*a + 3*b + 3*c + d + 8) / 16 +// = (a + (a + 3*b + 3*c + d) / 8 + 1) / 2 +// = (a + m + 1) / 2 +// where m = (a + 3*b + 3*c + d) / 8 +// = ((a + b + c + d) / 2 + b + c) / 4 +// +// Let's say k = (a + b + c + d) / 4. +// We can compute k as +// k = (s + t + 1) / 2 - ((a^d) | (b^c) | (s^t)) & 1 +// where s = (a + d + 1) / 2 and t = (b + c + 1) / 2 +// +// Then m can be written as +// m = (k + t + 1) / 2 - (((b^c) & (s^t)) | (k^t)) & 1 + +// Computes out = (k + in + 1) / 2 - ((ij & (s^t)) | (k^in)) & 1 +#define GET_M(ij, in, out) do { \ + const __m128i tmp0 = _mm_avg_epu8(k, (in)); /* (k + in + 1) / 2 */ \ + const __m128i tmp1 = _mm_and_si128((ij), st); /* (ij) & (s^t) */ \ + const __m128i tmp2 = _mm_xor_si128(k, (in)); /* (k^in) */ \ + const __m128i tmp3 = _mm_or_si128(tmp1, tmp2); /* ((ij) & (s^t)) | (k^in) */\ + const __m128i tmp4 = _mm_and_si128(tmp3, one); /* & 1 -> lsb_correction */ \ + (out) = _mm_sub_epi8(tmp0, tmp4); /* (k + in + 1) / 2 - lsb_correction */ \ +} while (0) + +// pack and store two alterning pixel rows +#define PACK_AND_STORE(a, b, da, db, out) do { \ + const __m128i ta = _mm_avg_epu8(a, da); /* (9a + 3b + 3c + d + 8) / 16 */ \ + const __m128i tb = _mm_avg_epu8(b, db); /* (3a + 9b + c + 3d + 8) / 16 */ \ + const __m128i t1 = _mm_unpacklo_epi8(ta, tb); \ + const __m128i t2 = _mm_unpackhi_epi8(ta, tb); \ + _mm_store_si128(((__m128i*)(out)) + 0, t1); \ + _mm_store_si128(((__m128i*)(out)) + 1, t2); \ +} while (0) + +// Loads 17 pixels each from rows r1 and r2 and generates 32 pixels. +#define UPSAMPLE_32PIXELS(r1, r2, out) { \ + const __m128i one = _mm_set1_epi8(1); \ + const __m128i a = _mm_loadu_si128((__m128i*)&(r1)[0]); \ + const __m128i b = _mm_loadu_si128((__m128i*)&(r1)[1]); \ + const __m128i c = _mm_loadu_si128((__m128i*)&(r2)[0]); \ + const __m128i d = _mm_loadu_si128((__m128i*)&(r2)[1]); \ + \ + const __m128i s = _mm_avg_epu8(a, d); /* s = (a + d + 1) / 2 */ \ + const __m128i t = _mm_avg_epu8(b, c); /* t = (b + c + 1) / 2 */ \ + const __m128i st = _mm_xor_si128(s, t); /* st = s^t */ \ + \ + const __m128i ad = _mm_xor_si128(a, d); /* ad = a^d */ \ + const __m128i bc = _mm_xor_si128(b, c); /* bc = b^c */ \ + \ + const __m128i t1 = _mm_or_si128(ad, bc); /* (a^d) | (b^c) */ \ + const __m128i t2 = _mm_or_si128(t1, st); /* (a^d) | (b^c) | (s^t) */ \ + const __m128i t3 = _mm_and_si128(t2, one); /* (a^d) | (b^c) | (s^t) & 1 */ \ + const __m128i t4 = _mm_avg_epu8(s, t); \ + const __m128i k = _mm_sub_epi8(t4, t3); /* k = (a + b + c + d) / 4 */ \ + __m128i diag1, diag2; \ + \ + GET_M(bc, t, diag1); /* diag1 = (a + 3b + 3c + d) / 8 */ \ + GET_M(ad, s, diag2); /* diag2 = (3a + b + c + 3d) / 8 */ \ + \ + /* pack the alternate pixels */ \ + PACK_AND_STORE(a, b, diag1, diag2, &(out)[0 * 32]); \ + PACK_AND_STORE(c, d, diag2, diag1, &(out)[2 * 32]); \ +} + +// Turn the macro into a function for reducing code-size when non-critical +static void Upsample32Pixels(const uint8_t r1[], const uint8_t r2[], + uint8_t* const out) { + UPSAMPLE_32PIXELS(r1, r2, out); +} + +#define UPSAMPLE_LAST_BLOCK(tb, bb, num_pixels, out) { \ + uint8_t r1[17], r2[17]; \ + memcpy(r1, (tb), (num_pixels)); \ + memcpy(r2, (bb), (num_pixels)); \ + /* replicate last byte */ \ + memset(r1 + (num_pixels), r1[(num_pixels) - 1], 17 - (num_pixels)); \ + memset(r2 + (num_pixels), r2[(num_pixels) - 1], 17 - (num_pixels)); \ + /* using the shared function instead of the macro saves ~3k code size */ \ + Upsample32Pixels(r1, r2, out); \ +} + +#define CONVERT2RGB(FUNC, XSTEP, top_y, bottom_y, uv, \ + top_dst, bottom_dst, cur_x, num_pixels) { \ + int n; \ + if (top_y) { \ + for (n = 0; n < (num_pixels); ++n) { \ + FUNC(top_y[(cur_x) + n], (uv)[n], (uv)[32 + n], \ + top_dst + ((cur_x) + n) * XSTEP); \ + } \ + } \ + if (bottom_y) { \ + for (n = 0; n < (num_pixels); ++n) { \ + FUNC(bottom_y[(cur_x) + n], (uv)[64 + n], (uv)[64 + 32 + n], \ + bottom_dst + ((cur_x) + n) * XSTEP); \ + } \ + } \ +} + +#define SSE2_UPSAMPLE_FUNC(FUNC_NAME, FUNC, XSTEP) \ +static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y, \ + const uint8_t* top_u, const uint8_t* top_v, \ + const uint8_t* cur_u, const uint8_t* cur_v, \ + uint8_t* top_dst, uint8_t* bottom_dst, int len) { \ + int b; \ + /* 16 byte aligned array to cache reconstructed u and v */ \ + uint8_t uv_buf[4 * 32 + 15]; \ + uint8_t* const r_uv = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15); \ + const int uv_len = (len + 1) >> 1; \ + /* 17 pixels must be read-able for each block */ \ + const int num_blocks = (uv_len - 1) >> 4; \ + const int leftover = uv_len - num_blocks * 16; \ + const int last_pos = 1 + 32 * num_blocks; \ + \ + const int u_diag = ((top_u[0] + cur_u[0]) >> 1) + 1; \ + const int v_diag = ((top_v[0] + cur_v[0]) >> 1) + 1; \ + \ + assert(len > 0); \ + /* Treat the first pixel in regular way */ \ + if (top_y) { \ + const int u0 = (top_u[0] + u_diag) >> 1; \ + const int v0 = (top_v[0] + v_diag) >> 1; \ + FUNC(top_y[0], u0, v0, top_dst); \ + } \ + if (bottom_y) { \ + const int u0 = (cur_u[0] + u_diag) >> 1; \ + const int v0 = (cur_v[0] + v_diag) >> 1; \ + FUNC(bottom_y[0], u0, v0, bottom_dst); \ + } \ + \ + for (b = 0; b < num_blocks; ++b) { \ + UPSAMPLE_32PIXELS(top_u, cur_u, r_uv + 0 * 32); \ + UPSAMPLE_32PIXELS(top_v, cur_v, r_uv + 1 * 32); \ + CONVERT2RGB(FUNC, XSTEP, top_y, bottom_y, r_uv, top_dst, bottom_dst, \ + 32 * b + 1, 32) \ + top_u += 16; \ + cur_u += 16; \ + top_v += 16; \ + cur_v += 16; \ + } \ + \ + UPSAMPLE_LAST_BLOCK(top_u, cur_u, leftover, r_uv + 0 * 32); \ + UPSAMPLE_LAST_BLOCK(top_v, cur_v, leftover, r_uv + 1 * 32); \ + CONVERT2RGB(FUNC, XSTEP, top_y, bottom_y, r_uv, top_dst, bottom_dst, \ + last_pos, len - last_pos); \ +} + +// SSE2 variants of the fancy upsampler. +SSE2_UPSAMPLE_FUNC(UpsampleRgbLinePairSSE2, VP8YuvToRgb, 3) +SSE2_UPSAMPLE_FUNC(UpsampleBgrLinePairSSE2, VP8YuvToBgr, 3) +SSE2_UPSAMPLE_FUNC(UpsampleRgbaLinePairSSE2, VP8YuvToRgba, 4) +SSE2_UPSAMPLE_FUNC(UpsampleBgraLinePairSSE2, VP8YuvToBgra, 4) +// These two don't erase the alpha value +SSE2_UPSAMPLE_FUNC(UpsampleRgbKeepAlphaLinePairSSE2, VP8YuvToRgb, 4) +SSE2_UPSAMPLE_FUNC(UpsampleBgrKeepAlphaLinePairSSE2, VP8YuvToBgr, 4) + +#undef GET_M +#undef PACK_AND_STORE +#undef UPSAMPLE_32PIXELS +#undef UPSAMPLE_LAST_BLOCK +#undef CONVERT2RGB +#undef SSE2_UPSAMPLE_FUNC + +//----------------------------------------------------------------------------- + +void WebPInitUpsamplersSSE2(void) { + WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePairSSE2; + WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePairSSE2; + WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePairSSE2; + WebPUpsamplers[MODE_BGRA] = UpsampleBgraLinePairSSE2; + + WebPUpsamplersKeepAlpha[MODE_RGB] = UpsampleRgbLinePairSSE2; + WebPUpsamplersKeepAlpha[MODE_RGBA] = UpsampleRgbKeepAlphaLinePairSSE2; + WebPUpsamplersKeepAlpha[MODE_BGR] = UpsampleBgrLinePairSSE2; + WebPUpsamplersKeepAlpha[MODE_BGRA] = UpsampleBgrKeepAlphaLinePairSSE2; +} + +#if defined(__cplusplus) || defined(c_plusplus) +} // extern "C" +#endif + +#endif //__SSE2__ || _MSC_VER diff --git a/src/dec/webpi.h b/src/dec/webpi.h index 662441da..dd40112c 100644 --- a/src/dec/webpi.h +++ b/src/dec/webpi.h @@ -57,6 +57,22 @@ struct WebPDecParams { // Should be called first, before any use of the WebPDecParams object. void WebPResetDecParams(WebPDecParams* const params); +//------------------------------------------------------------------------------ +// Upsampler function to overwrite fancy upsampler. + +typedef void (*WebPUpsampleLinePairFunc)( + const uint8_t* top_y, const uint8_t* bottom_y, + const uint8_t* top_u, const uint8_t* top_v, + const uint8_t* cur_u, const uint8_t* cur_v, + uint8_t* top_dst, uint8_t* bottom_dst, int len); + +// Upsampler functions to be used to convert YUV to RGB(A) modes +extern WebPUpsampleLinePairFunc WebPUpsamplers[MODE_BGRA + 1]; +extern WebPUpsampleLinePairFunc WebPUpsamplersKeepAlpha[MODE_BGRA + 1]; + +// Initializes SSE2 version of the fancy upsamplers. +void WebPInitUpsamplersSSE2(void); + //------------------------------------------------------------------------------ // Misc utils diff --git a/src/enc/dsp_sse2.c b/src/enc/dsp_sse2.c index 3b9a2c7d..db20e648 100644 --- a/src/enc/dsp_sse2.c +++ b/src/enc/dsp_sse2.c @@ -34,8 +34,8 @@ static int CollectHistogramSSE2(const uint8_t* ref, const uint8_t* pred, // Convert coefficients to bin (within out[]). { // Load. - const __m128i out0 = _mm_loadu_si128((__m128i *)&out[0]); - const __m128i out1 = _mm_loadu_si128((__m128i *)&out[8]); + const __m128i out0 = _mm_loadu_si128((__m128i*)&out[0]); + const __m128i out1 = _mm_loadu_si128((__m128i*)&out[8]); // sign(out) = out >> 15 (0x0000 if positive, 0xffff if negative) const __m128i sign0 = _mm_srai_epi16(out0, 15); const __m128i sign1 = _mm_srai_epi16(out1, 15); @@ -51,8 +51,8 @@ static int CollectHistogramSSE2(const uint8_t* ref, const uint8_t* pred, const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh); const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh); // Store. - _mm_storeu_si128((__m128i *)&out[0], bin0); - _mm_storeu_si128((__m128i *)&out[8], bin1); + _mm_storeu_si128((__m128i*)&out[0], bin0); + _mm_storeu_si128((__m128i*)&out[8], bin1); } // Use bin to update histogram. @@ -96,19 +96,19 @@ static void ITransformSSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, // use nor store. __m128i in0, in1, in2, in3; { - in0 = _mm_loadl_epi64((__m128i *)&in[0]); - in1 = _mm_loadl_epi64((__m128i *)&in[4]); - in2 = _mm_loadl_epi64((__m128i *)&in[8]); - in3 = _mm_loadl_epi64((__m128i *)&in[12]); + in0 = _mm_loadl_epi64((__m128i*)&in[0]); + in1 = _mm_loadl_epi64((__m128i*)&in[4]); + in2 = _mm_loadl_epi64((__m128i*)&in[8]); + in3 = _mm_loadl_epi64((__m128i*)&in[12]); // a00 a10 a20 a30 x x x x // a01 a11 a21 a31 x x x x // a02 a12 a22 a32 x x x x // a03 a13 a23 a33 x x x x if (do_two) { - const __m128i inB0 = _mm_loadl_epi64((__m128i *)&in[16]); - const __m128i inB1 = _mm_loadl_epi64((__m128i *)&in[20]); - const __m128i inB2 = _mm_loadl_epi64((__m128i *)&in[24]); - const __m128i inB3 = _mm_loadl_epi64((__m128i *)&in[28]); + const __m128i inB0 = _mm_loadl_epi64((__m128i*)&in[16]); + const __m128i inB1 = _mm_loadl_epi64((__m128i*)&in[20]); + const __m128i inB2 = _mm_loadl_epi64((__m128i*)&in[24]); + const __m128i inB3 = _mm_loadl_epi64((__m128i*)&in[28]); in0 = _mm_unpacklo_epi64(in0, inB0); in1 = _mm_unpacklo_epi64(in1, inB1); in2 = _mm_unpacklo_epi64(in2, inB2); @@ -242,10 +242,20 @@ static void ITransformSSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, { const __m128i zero = _mm_set1_epi16(0); // Load the reference(s). - __m128i ref0 = _mm_loadl_epi64((__m128i *)&ref[0 * BPS]); - __m128i ref1 = _mm_loadl_epi64((__m128i *)&ref[1 * BPS]); - __m128i ref2 = _mm_loadl_epi64((__m128i *)&ref[2 * BPS]); - __m128i ref3 = _mm_loadl_epi64((__m128i *)&ref[3 * BPS]); + __m128i ref0, ref1, ref2, ref3; + if (do_two) { + // Load eight bytes/pixels per line. + ref0 = _mm_loadl_epi64((__m128i*)&ref[0 * BPS]); + ref1 = _mm_loadl_epi64((__m128i*)&ref[1 * BPS]); + ref2 = _mm_loadl_epi64((__m128i*)&ref[2 * BPS]); + ref3 = _mm_loadl_epi64((__m128i*)&ref[3 * BPS]); + } else { + // Load four bytes/pixels per line. + ref0 = _mm_cvtsi32_si128(*(int*)&ref[0 * BPS]); + ref1 = _mm_cvtsi32_si128(*(int*)&ref[1 * BPS]); + ref2 = _mm_cvtsi32_si128(*(int*)&ref[2 * BPS]); + ref3 = _mm_cvtsi32_si128(*(int*)&ref[3 * BPS]); + } // Convert to 16b. ref0 = _mm_unpacklo_epi8(ref0, zero); ref1 = _mm_unpacklo_epi8(ref1, zero); @@ -264,10 +274,10 @@ static void ITransformSSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, // Store the results. if (do_two) { // Store eight bytes/pixels per line. - _mm_storel_epi64((__m128i *)&dst[0 * BPS], ref0); - _mm_storel_epi64((__m128i *)&dst[1 * BPS], ref1); - _mm_storel_epi64((__m128i *)&dst[2 * BPS], ref2); - _mm_storel_epi64((__m128i *)&dst[3 * BPS], ref3); + _mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0); + _mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1); + _mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2); + _mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3); } else { // Store four bytes/pixels per line. *((int32_t *)&dst[0 * BPS]) = _mm_cvtsi128_si32(ref0); @@ -296,19 +306,19 @@ static void FTransformSSE2(const uint8_t* src, const uint8_t* ref, // Difference between src and ref and initial transpose. { // Load src and convert to 16b. - const __m128i src0 = _mm_loadl_epi64((__m128i *)&src[0 * BPS]); - const __m128i src1 = _mm_loadl_epi64((__m128i *)&src[1 * BPS]); - const __m128i src2 = _mm_loadl_epi64((__m128i *)&src[2 * BPS]); - const __m128i src3 = _mm_loadl_epi64((__m128i *)&src[3 * BPS]); + const __m128i src0 = _mm_loadl_epi64((__m128i*)&src[0 * BPS]); + const __m128i src1 = _mm_loadl_epi64((__m128i*)&src[1 * BPS]); + const __m128i src2 = _mm_loadl_epi64((__m128i*)&src[2 * BPS]); + const __m128i src3 = _mm_loadl_epi64((__m128i*)&src[3 * BPS]); const __m128i src_0 = _mm_unpacklo_epi8(src0, zero); const __m128i src_1 = _mm_unpacklo_epi8(src1, zero); const __m128i src_2 = _mm_unpacklo_epi8(src2, zero); const __m128i src_3 = _mm_unpacklo_epi8(src3, zero); // Load ref and convert to 16b. - const __m128i ref0 = _mm_loadl_epi64((__m128i *)&ref[0 * BPS]); - const __m128i ref1 = _mm_loadl_epi64((__m128i *)&ref[1 * BPS]); - const __m128i ref2 = _mm_loadl_epi64((__m128i *)&ref[2 * BPS]); - const __m128i ref3 = _mm_loadl_epi64((__m128i *)&ref[3 * BPS]); + const __m128i ref0 = _mm_loadl_epi64((__m128i*)&ref[0 * BPS]); + const __m128i ref1 = _mm_loadl_epi64((__m128i*)&ref[1 * BPS]); + const __m128i ref2 = _mm_loadl_epi64((__m128i*)&ref[2 * BPS]); + const __m128i ref3 = _mm_loadl_epi64((__m128i*)&ref[3 * BPS]); const __m128i ref_0 = _mm_unpacklo_epi8(ref0, zero); const __m128i ref_1 = _mm_unpacklo_epi8(ref1, zero); const __m128i ref_2 = _mm_unpacklo_epi8(ref2, zero); @@ -419,10 +429,10 @@ static void FTransformSSE2(const uint8_t* src, const uint8_t* ref, // desired (0, 1), we add one earlier through k12000_plus_one. const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero)); - _mm_storel_epi64((__m128i *)&out[ 0], d0); - _mm_storel_epi64((__m128i *)&out[ 4], g1); - _mm_storel_epi64((__m128i *)&out[ 8], d2); - _mm_storel_epi64((__m128i *)&out[12], f3); + _mm_storel_epi64((__m128i*)&out[ 0], d0); + _mm_storel_epi64((__m128i*)&out[ 4], g1); + _mm_storel_epi64((__m128i*)&out[ 8], d2); + _mm_storel_epi64((__m128i*)&out[12], f3); } } @@ -433,14 +443,14 @@ static int SSE4x4SSE2(const uint8_t* a, const uint8_t* b) { const __m128i zero = _mm_set1_epi16(0); // Load values. - const __m128i a0 = _mm_loadl_epi64((__m128i *)&a[BPS * 0]); - const __m128i a1 = _mm_loadl_epi64((__m128i *)&a[BPS * 1]); - const __m128i a2 = _mm_loadl_epi64((__m128i *)&a[BPS * 2]); - const __m128i a3 = _mm_loadl_epi64((__m128i *)&a[BPS * 3]); - const __m128i b0 = _mm_loadl_epi64((__m128i *)&b[BPS * 0]); - const __m128i b1 = _mm_loadl_epi64((__m128i *)&b[BPS * 1]); - const __m128i b2 = _mm_loadl_epi64((__m128i *)&b[BPS * 2]); - const __m128i b3 = _mm_loadl_epi64((__m128i *)&b[BPS * 3]); + const __m128i a0 = _mm_loadl_epi64((__m128i*)&a[BPS * 0]); + const __m128i a1 = _mm_loadl_epi64((__m128i*)&a[BPS * 1]); + const __m128i a2 = _mm_loadl_epi64((__m128i*)&a[BPS * 2]); + const __m128i a3 = _mm_loadl_epi64((__m128i*)&a[BPS * 3]); + const __m128i b0 = _mm_loadl_epi64((__m128i*)&b[BPS * 0]); + const __m128i b1 = _mm_loadl_epi64((__m128i*)&b[BPS * 1]); + const __m128i b2 = _mm_loadl_epi64((__m128i*)&b[BPS * 2]); + const __m128i b3 = _mm_loadl_epi64((__m128i*)&b[BPS * 3]); // Combine pair of lines and convert to 16b. const __m128i a01 = _mm_unpacklo_epi32(a0, a1); @@ -471,7 +481,7 @@ static int SSE4x4SSE2(const uint8_t* a, const uint8_t* b) { const __m128i sum1 = _mm_add_epi32(madd2, madd3); const __m128i sum2 = _mm_add_epi32(sum0, sum1); int32_t tmp[4]; - _mm_storeu_si128((__m128i *)tmp, sum2); + _mm_storeu_si128((__m128i*)tmp, sum2); return (tmp[3] + tmp[2] + tmp[1] + tmp[0]); } @@ -494,14 +504,14 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB, // Load, combine and tranpose inputs. { - const __m128i inA_0 = _mm_loadl_epi64((__m128i *)&inA[BPS * 0]); - const __m128i inA_1 = _mm_loadl_epi64((__m128i *)&inA[BPS * 1]); - const __m128i inA_2 = _mm_loadl_epi64((__m128i *)&inA[BPS * 2]); - const __m128i inA_3 = _mm_loadl_epi64((__m128i *)&inA[BPS * 3]); - const __m128i inB_0 = _mm_loadl_epi64((__m128i *)&inB[BPS * 0]); - const __m128i inB_1 = _mm_loadl_epi64((__m128i *)&inB[BPS * 1]); - const __m128i inB_2 = _mm_loadl_epi64((__m128i *)&inB[BPS * 2]); - const __m128i inB_3 = _mm_loadl_epi64((__m128i *)&inB[BPS * 3]); + const __m128i inA_0 = _mm_loadl_epi64((__m128i*)&inA[BPS * 0]); + const __m128i inA_1 = _mm_loadl_epi64((__m128i*)&inA[BPS * 1]); + const __m128i inA_2 = _mm_loadl_epi64((__m128i*)&inA[BPS * 2]); + const __m128i inA_3 = _mm_loadl_epi64((__m128i*)&inA[BPS * 3]); + const __m128i inB_0 = _mm_loadl_epi64((__m128i*)&inB[BPS * 0]); + const __m128i inB_1 = _mm_loadl_epi64((__m128i*)&inB[BPS * 1]); + const __m128i inB_2 = _mm_loadl_epi64((__m128i*)&inB[BPS * 2]); + const __m128i inB_3 = _mm_loadl_epi64((__m128i*)&inB[BPS * 3]); // Combine inA and inB (we'll do two transforms in parallel). const __m128i inAB_0 = _mm_unpacklo_epi8(inA_0, inB_0); @@ -585,8 +595,8 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB, // Load all inputs. // TODO(cduvivier): Make variable declarations and allocations aligned so // we can use _mm_load_si128 instead of _mm_loadu_si128. - const __m128i w_0 = _mm_loadu_si128((__m128i *)&w[0]); - const __m128i w_8 = _mm_loadu_si128((__m128i *)&w[8]); + const __m128i w_0 = _mm_loadu_si128((__m128i*)&w[0]); + const __m128i w_8 = _mm_loadu_si128((__m128i*)&w[8]); // Calculate a and b (two 4x4 at once). const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); @@ -645,7 +655,7 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB, // difference of weighted sums A_b0 = _mm_sub_epi32(A_b0, B_b0); - _mm_storeu_si128((__m128i *)&sum[0], A_b0); + _mm_storeu_si128((__m128i*)&sum[0], A_b0); } return sum[0] + sum[1] + sum[2] + sum[3]; } @@ -686,18 +696,18 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], // Load all inputs. // TODO(cduvivier): Make variable declarations and allocations aligned so that // we can use _mm_load_si128 instead of _mm_loadu_si128. - __m128i in0 = _mm_loadu_si128((__m128i *)&in[0]); - __m128i in8 = _mm_loadu_si128((__m128i *)&in[8]); - const __m128i sharpen0 = _mm_loadu_si128((__m128i *)&mtx->sharpen_[0]); - const __m128i sharpen8 = _mm_loadu_si128((__m128i *)&mtx->sharpen_[8]); - const __m128i iq0 = _mm_loadu_si128((__m128i *)&mtx->iq_[0]); - const __m128i iq8 = _mm_loadu_si128((__m128i *)&mtx->iq_[8]); - const __m128i bias0 = _mm_loadu_si128((__m128i *)&mtx->bias_[0]); - const __m128i bias8 = _mm_loadu_si128((__m128i *)&mtx->bias_[8]); - const __m128i q0 = _mm_loadu_si128((__m128i *)&mtx->q_[0]); - const __m128i q8 = _mm_loadu_si128((__m128i *)&mtx->q_[8]); - const __m128i zthresh0 = _mm_loadu_si128((__m128i *)&mtx->zthresh_[0]); - const __m128i zthresh8 = _mm_loadu_si128((__m128i *)&mtx->zthresh_[8]); + __m128i in0 = _mm_loadu_si128((__m128i*)&in[0]); + __m128i in8 = _mm_loadu_si128((__m128i*)&in[8]); + const __m128i sharpen0 = _mm_loadu_si128((__m128i*)&mtx->sharpen_[0]); + const __m128i sharpen8 = _mm_loadu_si128((__m128i*)&mtx->sharpen_[8]); + const __m128i iq0 = _mm_loadu_si128((__m128i*)&mtx->iq_[0]); + const __m128i iq8 = _mm_loadu_si128((__m128i*)&mtx->iq_[8]); + const __m128i bias0 = _mm_loadu_si128((__m128i*)&mtx->bias_[0]); + const __m128i bias8 = _mm_loadu_si128((__m128i*)&mtx->bias_[8]); + const __m128i q0 = _mm_loadu_si128((__m128i*)&mtx->q_[0]); + const __m128i q8 = _mm_loadu_si128((__m128i*)&mtx->q_[8]); + const __m128i zthresh0 = _mm_loadu_si128((__m128i*)&mtx->zthresh_[0]); + const __m128i zthresh8 = _mm_loadu_si128((__m128i*)&mtx->zthresh_[8]); // sign(in) = in >> 15 (0x0000 if positive, 0xffff if negative) sign0 = _mm_srai_epi16(in0, 15); @@ -765,8 +775,8 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], __m128i cmp8 = _mm_cmpgt_epi16(coeff8, zthresh8); in0 = _mm_and_si128(in0, cmp0); in8 = _mm_and_si128(in8, cmp8); - _mm_storeu_si128((__m128i *)&in[0], in0); - _mm_storeu_si128((__m128i *)&in[8], in8); + _mm_storeu_si128((__m128i*)&in[0], in0); + _mm_storeu_si128((__m128i*)&in[8], in8); out0 = _mm_and_si128(out0, cmp0); out8 = _mm_and_si128(out8, cmp8); } @@ -784,8 +794,8 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], outZ8 = _mm_shufflelo_epi16(out8, _MM_SHUFFLE(3, 0, 2, 1)); outZ8 = _mm_shuffle_epi32 (outZ8, _MM_SHUFFLE(3, 1, 2, 0)); outZ8 = _mm_shufflelo_epi16(outZ8, _MM_SHUFFLE(1, 3, 2, 0)); - _mm_storeu_si128((__m128i *)&out[0], outZ0); - _mm_storeu_si128((__m128i *)&out[8], outZ8); + _mm_storeu_si128((__m128i*)&out[0], outZ0); + _mm_storeu_si128((__m128i*)&out[8], outZ8); packed_out = _mm_packs_epi16(outZ0, outZ8); } { @@ -798,7 +808,7 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], // detect if all 'out' values are zeroes or not { int32_t tmp[4]; - _mm_storeu_si128((__m128i *)tmp, packed_out); + _mm_storeu_si128((__m128i*)tmp, packed_out); if (n) { tmp[0] &= ~0xff; }