SSE2 functions for the fancy upsampler.

~5-10% faster.
Heavy 8bit arithmetic trickery!
Patch by Somnath Banerjee (somnath at google dot com)

Change-Id: I9fd2c511d9f631e9cf4b008c46127b49fb527b47

Makefile.vc

@@ -123,6 +123,7 @@ X_OBJS= \
 	$(DIROBJ)\dec\vp8.obj \
 	$(DIROBJ)\dec\webp.obj \
 	$(DIROBJ)\dec\io.obj \
+	$(DIROBJ)\dec\io_sse2.obj \
 	$(DIROBJ)\dec\buffer.obj \
 	$(DIROBJ)\dec\yuv.obj \
 	$(DIROBJ)\dec\idec.obj \

makefile.unix

@@ -58,7 +58,7 @@ OBJS = src/enc/webpenc.o src/enc/bit_writer.o src/enc/syntax.o \
        src/dec/bits.o src/dec/dsp.o src/dec/dsp_sse2.o src/dec/frame.o \
        src/dec/webp.o src/dec/quant.o src/dec/tree.o src/dec/vp8.o \
        src/dec/yuv.o src/dec/idec.o src/dec/alpha.o src/dec/layer.o \
-       src/dec/io.o src/dec/buffer.o
+       src/dec/io.o src/dec/io_sse2.o src/dec/buffer.o
 HDRS = src/webp/encode.h src/enc/vp8enci.h src/enc/bit_writer.h \
        src/enc/cost.h src/dec/bits.h  src/dec/vp8i.h src/dec/yuv.h
 OUTPUT = examples/cwebp examples/dwebp src/libwebp.a

src/dec/Makefile.am

@@ -2,7 +2,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src
 
 libwebpdecode_la_SOURCES = bits.h vp8i.h yuv.h bits.c dsp.c dsp_sse2.c frame.c \
                           quant.c tree.c vp8.c webp.c yuv.c idec.c alpha.c \
-                          layer.c io.c buffer.c
+                          layer.c io.c io_sse2.c buffer.c
 libwebpdecode_la_LDFLAGS = -version-info 0:0:0
 libwebpdecode_la_CPPFLAGS = $(USE_EXPERIMENTAL_CODE)
 libwebpdecodeinclude_HEADERS = ../webp/decode.h ../webp/decode_vp8.h ../webp/types.h

src/dec/io.c

@@ -105,29 +105,34 @@ UPSAMPLE_FUNC(UpsampleBgraLinePair, VP8YuvToBgra, 4)
 UPSAMPLE_FUNC(UpsampleRgbKeepAlphaLinePair, VP8YuvToRgb, 4)
 UPSAMPLE_FUNC(UpsampleBgrKeepAlphaLinePair, VP8YuvToBgr, 4)
 
-typedef void (*UpsampleLinePairFunc)(
-  const uint8_t* top_y, const uint8_t* bottom_y,
-  const uint8_t* top_u, const uint8_t* top_v,
-  const uint8_t* cur_u, const uint8_t* cur_v,
-  uint8_t* top_dst, uint8_t* bottom_dst, int len);
-
-static const UpsampleLinePairFunc
-  kUpsamplers[MODE_BGRA + 1] = {
-    UpsampleRgbLinePair,    // MODE_RGB
-    UpsampleRgbaLinePair,   // MODE_RGBA
-    UpsampleBgrLinePair,    // MODE_BGR
-    UpsampleBgraLinePair    // MODE_BGRA
-  },
-  kUpsamplersKeepAlpha[MODE_BGRA + 1] = {
-    UpsampleRgbLinePair,            // MODE_RGB
-    UpsampleRgbKeepAlphaLinePair,   // MODE_RGBA
-    UpsampleBgrLinePair,            // MODE_BGR
-    UpsampleBgrKeepAlphaLinePair    // MODE_BGRA
-  };
-
 #undef LOAD_UV
 #undef UPSAMPLE_FUNC
 
+// Fancy upsampling functions to convert YUV to RGB
+WebPUpsampleLinePairFunc WebPUpsamplers[MODE_BGRA + 1];
+WebPUpsampleLinePairFunc WebPUpsamplersKeepAlpha[MODE_BGRA + 1];
+
+static void InitUpsamplers(void) {
+  WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePair;
+  WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePair;
+  WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePair;
+  WebPUpsamplers[MODE_BGRA] =  UpsampleBgraLinePair;
+
+  WebPUpsamplersKeepAlpha[MODE_RGB] = UpsampleRgbLinePair;
+  WebPUpsamplersKeepAlpha[MODE_RGBA] = UpsampleRgbKeepAlphaLinePair;
+  WebPUpsamplersKeepAlpha[MODE_BGR] =  UpsampleBgrLinePair;
+  WebPUpsamplersKeepAlpha[MODE_BGRA] =  UpsampleBgrKeepAlphaLinePair;
+
+  // If defined, use CPUInfo() to overwrite some pointers with faster versions.
+  if (VP8DecGetCPUInfo) {
+    if (VP8DecGetCPUInfo(kSSE2)) {
+#if defined(__SSE2__) || defined(_MSC_VER)
+      WebPInitUpsamplersSSE2();
+#endif
+    }
+  }
+}
+
 #endif  // FANCY_UPSAMPLING
 
 //------------------------------------------------------------------------------
@@ -287,9 +292,9 @@ static int EmitFancyRGB(const VP8Io* const io, WebPDecParams* const p) {
   int num_lines_out = io->mb_h;   // a priori guess
   const WebPRGBABuffer* const buf = &p->output->u.RGBA;
   uint8_t* dst = buf->rgba + io->mb_y * buf->stride;
-  const UpsampleLinePairFunc upsample =
+  const WebPUpsampleLinePairFunc upsample =
-      io->a ? kUpsamplersKeepAlpha[p->output->colorspace]
+      io->a ? WebPUpsamplersKeepAlpha[p->output->colorspace]
-            : kUpsamplers[p->output->colorspace];
+            : WebPUpsamplers[p->output->colorspace];
   const uint8_t* cur_y = io->y;
   const uint8_t* cur_u = io->u;
   const uint8_t* cur_v = io->v;
@@ -781,6 +786,7 @@ static int CustomSetup(VP8Io* io) {
         p->tmp_u = p->tmp_y + io->mb_w;
         p->tmp_v = p->tmp_u + uv_width;
         p->emit = EmitFancyRGB;
+        InitUpsamplers();
       }
 #endif
     } else {

src/dec/io_sse2.c

@@ -0,0 +1,205 @@
+// Copyright 2011 Google Inc.
+//
+// This code is licensed under the same terms as WebM:
+//  Software License Agreement:  http://www.webmproject.org/license/software/
+//  Additional IP Rights Grant:  http://www.webmproject.org/license/additional/
+// -----------------------------------------------------------------------------
+//
+// SSE2 version of YUV to RGB upsampling functions.
+//
+// Author: somnath@google.com (Somnath Banerjee)
+
+#if defined(__SSE2__) || defined(_MSC_VER)
+
+#include <assert.h>
+#include <emmintrin.h>
+#include <string.h>
+#include "webpi.h"
+#include "yuv.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+// We compute (9*a + 3*b + 3*c + d + 8) / 16 as follows
+// u = (9*a + 3*b + 3*c + d + 8) / 16
+//   = (a + (a + 3*b + 3*c + d) / 8 + 1) / 2
+//   = (a + m + 1) / 2
+// where m = (a + 3*b + 3*c + d) / 8
+//         = ((a + b + c + d) / 2 + b + c) / 4
+//
+// Let's say  k = (a + b + c + d) / 4.
+// We can compute k as
+// k = (s + t + 1) / 2 - ((a^d) | (b^c) | (s^t)) & 1
+// where s = (a + d + 1) / 2 and t = (b + c + 1) / 2
+//
+// Then m can be written as
+// m = (k + t + 1) / 2 - (((b^c) & (s^t)) | (k^t)) & 1
+
+// Computes out = (k + in + 1) / 2 - ((ij & (s^t)) | (k^in)) & 1
+#define GET_M(ij, in, out) do {                                                \
+  const __m128i tmp0 = _mm_avg_epu8(k, (in));     /* (k + in + 1) / 2 */       \
+  const __m128i tmp1 = _mm_and_si128((ij), st);   /* (ij) & (s^t) */           \
+  const __m128i tmp2 = _mm_xor_si128(k, (in));    /* (k^in) */                 \
+  const __m128i tmp3 = _mm_or_si128(tmp1, tmp2);  /* ((ij) & (s^t)) | (k^in) */\
+  const __m128i tmp4 = _mm_and_si128(tmp3, one);  /* & 1 -> lsb_correction */  \
+  (out) = _mm_sub_epi8(tmp0, tmp4);    /* (k + in + 1) / 2 - lsb_correction */ \
+} while (0)
+
+// pack and store two alterning pixel rows
+#define PACK_AND_STORE(a, b, da, db, out) do {                                 \
+  const __m128i ta = _mm_avg_epu8(a, da);  /* (9a + 3b + 3c +  d + 8) / 16 */  \
+  const __m128i tb = _mm_avg_epu8(b, db);  /* (3a + 9b +  c + 3d + 8) / 16 */  \
+  const __m128i t1 = _mm_unpacklo_epi8(ta, tb);                                \
+  const __m128i t2 = _mm_unpackhi_epi8(ta, tb);                                \
+  _mm_store_si128(((__m128i*)(out)) + 0, t1);                                  \
+  _mm_store_si128(((__m128i*)(out)) + 1, t2);                                  \
+} while (0)
+
+// Loads 17 pixels each from rows r1 and r2 and generates 32 pixels.
+#define UPSAMPLE_32PIXELS(r1, r2, out) {                                       \
+  const __m128i one = _mm_set1_epi8(1);                                        \
+  const __m128i a = _mm_loadu_si128((__m128i*)&(r1)[0]);                       \
+  const __m128i b = _mm_loadu_si128((__m128i*)&(r1)[1]);                       \
+  const __m128i c = _mm_loadu_si128((__m128i*)&(r2)[0]);                       \
+  const __m128i d = _mm_loadu_si128((__m128i*)&(r2)[1]);                       \
+                                                                               \
+  const __m128i s = _mm_avg_epu8(a, d);        /* s = (a + d + 1) / 2 */       \
+  const __m128i t = _mm_avg_epu8(b, c);        /* t = (b + c + 1) / 2 */       \
+  const __m128i st = _mm_xor_si128(s, t);      /* st = s^t */                  \
+                                                                               \
+  const __m128i ad = _mm_xor_si128(a, d);      /* ad = a^d */                  \
+  const __m128i bc = _mm_xor_si128(b, c);      /* bc = b^c */                  \
+                                                                               \
+  const __m128i t1 = _mm_or_si128(ad, bc);     /* (a^d) | (b^c) */             \
+  const __m128i t2 = _mm_or_si128(t1, st);     /* (a^d) | (b^c) | (s^t) */     \
+  const __m128i t3 = _mm_and_si128(t2, one);   /* (a^d) | (b^c) | (s^t) & 1 */ \
+  const __m128i t4 = _mm_avg_epu8(s, t);                                       \
+  const __m128i k = _mm_sub_epi8(t4, t3);      /* k = (a + b + c + d) / 4 */   \
+  __m128i diag1, diag2;                                                        \
+                                                                               \
+  GET_M(bc, t, diag1);                  /* diag1 = (a + 3b + 3c + d) / 8 */    \
+  GET_M(ad, s, diag2);                  /* diag2 = (3a + b + c + 3d) / 8 */    \
+                                                                               \
+  /* pack the alternate pixels */                                              \
+  PACK_AND_STORE(a, b, diag1, diag2, &(out)[0 * 32]);                          \
+  PACK_AND_STORE(c, d, diag2, diag1, &(out)[2 * 32]);                          \
+}
+
+// Turn the macro into a function for reducing code-size when non-critical
+static void Upsample32Pixels(const uint8_t r1[], const uint8_t r2[],
+                             uint8_t* const out) {
+  UPSAMPLE_32PIXELS(r1, r2, out);
+}
+
+#define UPSAMPLE_LAST_BLOCK(tb, bb, num_pixels, out) {                         \
+  uint8_t r1[17], r2[17];                                                      \
+  memcpy(r1, (tb), (num_pixels));                                              \
+  memcpy(r2, (bb), (num_pixels));                                              \
+  /* replicate last byte */                                                    \
+  memset(r1 + (num_pixels), r1[(num_pixels) - 1], 17 - (num_pixels));          \
+  memset(r2 + (num_pixels), r2[(num_pixels) - 1], 17 - (num_pixels));          \
+  /* using the shared function instead of the macro saves ~3k code size */     \
+  Upsample32Pixels(r1, r2, out);                                               \
+}
+
+#define CONVERT2RGB(FUNC, XSTEP, top_y, bottom_y, uv,                          \
+                    top_dst, bottom_dst, cur_x, num_pixels) {                  \
+  int n;                                                                       \
+  if (top_y) {                                                                 \
+    for (n = 0; n < (num_pixels); ++n) {                                       \
+      FUNC(top_y[(cur_x) + n], (uv)[n], (uv)[32 + n],                          \
+           top_dst + ((cur_x) + n) * XSTEP);                                   \
+    }                                                                          \
+  }                                                                            \
+  if (bottom_y) {                                                              \
+    for (n = 0; n < (num_pixels); ++n) {                                       \
+      FUNC(bottom_y[(cur_x) + n], (uv)[64 + n], (uv)[64 + 32 + n],             \
+           bottom_dst + ((cur_x) + n) * XSTEP);                                \
+    }                                                                          \
+  }                                                                            \
+}
+
+#define SSE2_UPSAMPLE_FUNC(FUNC_NAME, FUNC, XSTEP)                             \
+static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y,           \
+                      const uint8_t* top_u, const uint8_t* top_v,              \
+                      const uint8_t* cur_u, const uint8_t* cur_v,              \
+                      uint8_t* top_dst, uint8_t* bottom_dst, int len) {        \
+  int b;                                                                       \
+  /* 16 byte aligned array to cache reconstructed u and v */                   \
+  uint8_t uv_buf[4 * 32 + 15];                                                 \
+  uint8_t* const r_uv = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15);            \
+  const int uv_len = (len + 1) >> 1;                                           \
+  /* 17 pixels must be read-able for each block */                             \
+  const int num_blocks = (uv_len - 1) >> 4;                                    \
+  const int leftover = uv_len - num_blocks * 16;                               \
+  const int last_pos = 1 + 32 * num_blocks;                                    \
+                                                                               \
+  const int u_diag = ((top_u[0] + cur_u[0]) >> 1) + 1;                         \
+  const int v_diag = ((top_v[0] + cur_v[0]) >> 1) + 1;                         \
+                                                                               \
+  assert(len > 0);                                                             \
+  /* Treat the first pixel in regular way */                                   \
+  if (top_y) {                                                                 \
+    const int u0 = (top_u[0] + u_diag) >> 1;                                   \
+    const int v0 = (top_v[0] + v_diag) >> 1;                                   \
+    FUNC(top_y[0], u0, v0, top_dst);                                           \
+  }                                                                            \
+  if (bottom_y) {                                                              \
+    const int u0 = (cur_u[0] + u_diag) >> 1;                                   \
+    const int v0 = (cur_v[0] + v_diag) >> 1;                                   \
+    FUNC(bottom_y[0], u0, v0, bottom_dst);                                     \
+  }                                                                            \
+                                                                               \
+  for (b = 0; b < num_blocks; ++b) {                                           \
+    UPSAMPLE_32PIXELS(top_u, cur_u, r_uv + 0 * 32);                            \
+    UPSAMPLE_32PIXELS(top_v, cur_v, r_uv + 1 * 32);                            \
+    CONVERT2RGB(FUNC, XSTEP, top_y, bottom_y, r_uv, top_dst, bottom_dst,       \
+                32 * b + 1, 32)                                                \
+    top_u += 16;                                                               \
+    cur_u += 16;                                                               \
+    top_v += 16;                                                               \
+    cur_v += 16;                                                               \
+  }                                                                            \
+                                                                               \
+  UPSAMPLE_LAST_BLOCK(top_u, cur_u, leftover, r_uv + 0 * 32);                  \
+  UPSAMPLE_LAST_BLOCK(top_v, cur_v, leftover, r_uv + 1 * 32);                  \
+  CONVERT2RGB(FUNC, XSTEP, top_y, bottom_y, r_uv, top_dst, bottom_dst,         \
+              last_pos, len - last_pos);                                       \
+}
+
+// SSE2 variants of the fancy upsampler.
+SSE2_UPSAMPLE_FUNC(UpsampleRgbLinePairSSE2,  VP8YuvToRgb,  3)
+SSE2_UPSAMPLE_FUNC(UpsampleBgrLinePairSSE2,  VP8YuvToBgr,  3)
+SSE2_UPSAMPLE_FUNC(UpsampleRgbaLinePairSSE2, VP8YuvToRgba, 4)
+SSE2_UPSAMPLE_FUNC(UpsampleBgraLinePairSSE2, VP8YuvToBgra, 4)
+// These two don't erase the alpha value
+SSE2_UPSAMPLE_FUNC(UpsampleRgbKeepAlphaLinePairSSE2, VP8YuvToRgb, 4)
+SSE2_UPSAMPLE_FUNC(UpsampleBgrKeepAlphaLinePairSSE2, VP8YuvToBgr, 4)
+
+#undef GET_M
+#undef PACK_AND_STORE
+#undef UPSAMPLE_32PIXELS
+#undef UPSAMPLE_LAST_BLOCK
+#undef CONVERT2RGB
+#undef SSE2_UPSAMPLE_FUNC
+
+//-----------------------------------------------------------------------------
+
+void WebPInitUpsamplersSSE2(void) {
+  WebPUpsamplers[MODE_RGB] = UpsampleRgbLinePairSSE2;
+  WebPUpsamplers[MODE_RGBA] = UpsampleRgbaLinePairSSE2;
+  WebPUpsamplers[MODE_BGR] = UpsampleBgrLinePairSSE2;
+  WebPUpsamplers[MODE_BGRA] =  UpsampleBgraLinePairSSE2;
+
+  WebPUpsamplersKeepAlpha[MODE_RGB] = UpsampleRgbLinePairSSE2;
+  WebPUpsamplersKeepAlpha[MODE_RGBA] = UpsampleRgbKeepAlphaLinePairSSE2;
+  WebPUpsamplersKeepAlpha[MODE_BGR] =  UpsampleBgrLinePairSSE2;
+  WebPUpsamplersKeepAlpha[MODE_BGRA] =  UpsampleBgrKeepAlphaLinePairSSE2;
+}
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}    // extern "C"
+#endif
+
+#endif   //__SSE2__ || _MSC_VER

src/dec/webpi.h

@@ -57,6 +57,22 @@ struct WebPDecParams {
 // Should be called first, before any use of the WebPDecParams object.
 void WebPResetDecParams(WebPDecParams* const params);
 
+//------------------------------------------------------------------------------
+// Upsampler function to overwrite fancy upsampler.
+
+typedef void (*WebPUpsampleLinePairFunc)(
+  const uint8_t* top_y, const uint8_t* bottom_y,
+  const uint8_t* top_u, const uint8_t* top_v,
+  const uint8_t* cur_u, const uint8_t* cur_v,
+  uint8_t* top_dst, uint8_t* bottom_dst, int len);
+
+// Upsampler functions to be used to convert YUV to RGB(A) modes
+extern WebPUpsampleLinePairFunc WebPUpsamplers[MODE_BGRA + 1];
+extern WebPUpsampleLinePairFunc WebPUpsamplersKeepAlpha[MODE_BGRA + 1];
+
+// Initializes SSE2 version of the fancy upsamplers.
+void WebPInitUpsamplersSSE2(void);
+
 //------------------------------------------------------------------------------
 // Misc utils
 

src/enc/dsp_sse2.c

@@ -34,8 +34,8 @@ static int CollectHistogramSSE2(const uint8_t* ref, const uint8_t* pred,
     // Convert coefficients to bin (within out[]).
     {
       // Load.
-      const __m128i out0 = _mm_loadu_si128((__m128i *)&out[0]);
+      const __m128i out0 = _mm_loadu_si128((__m128i*)&out[0]);
-      const __m128i out1 = _mm_loadu_si128((__m128i *)&out[8]);
+      const __m128i out1 = _mm_loadu_si128((__m128i*)&out[8]);
       // sign(out) = out >> 15  (0x0000 if positive, 0xffff if negative)
       const __m128i sign0 = _mm_srai_epi16(out0, 15);
       const __m128i sign1 = _mm_srai_epi16(out1, 15);
@@ -51,8 +51,8 @@ static int CollectHistogramSSE2(const uint8_t* ref, const uint8_t* pred,
       const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh);
       const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh);
       // Store.
-      _mm_storeu_si128((__m128i *)&out[0], bin0);
+      _mm_storeu_si128((__m128i*)&out[0], bin0);
-      _mm_storeu_si128((__m128i *)&out[8], bin1);
+      _mm_storeu_si128((__m128i*)&out[8], bin1);
     }
 
     // Use bin to update histogram.
@@ -96,19 +96,19 @@ static void ITransformSSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst,
   // use nor store.
   __m128i in0, in1, in2, in3;
   {
-    in0 = _mm_loadl_epi64((__m128i *)&in[0]);
+    in0 = _mm_loadl_epi64((__m128i*)&in[0]);
-    in1 = _mm_loadl_epi64((__m128i *)&in[4]);
+    in1 = _mm_loadl_epi64((__m128i*)&in[4]);
-    in2 = _mm_loadl_epi64((__m128i *)&in[8]);
+    in2 = _mm_loadl_epi64((__m128i*)&in[8]);
-    in3 = _mm_loadl_epi64((__m128i *)&in[12]);
+    in3 = _mm_loadl_epi64((__m128i*)&in[12]);
     // a00 a10 a20 a30   x x x x
     // a01 a11 a21 a31   x x x x
     // a02 a12 a22 a32   x x x x
     // a03 a13 a23 a33   x x x x
     if (do_two) {
-      const __m128i inB0 = _mm_loadl_epi64((__m128i *)&in[16]);
+      const __m128i inB0 = _mm_loadl_epi64((__m128i*)&in[16]);
-      const __m128i inB1 = _mm_loadl_epi64((__m128i *)&in[20]);
+      const __m128i inB1 = _mm_loadl_epi64((__m128i*)&in[20]);
-      const __m128i inB2 = _mm_loadl_epi64((__m128i *)&in[24]);
+      const __m128i inB2 = _mm_loadl_epi64((__m128i*)&in[24]);
-      const __m128i inB3 = _mm_loadl_epi64((__m128i *)&in[28]);
+      const __m128i inB3 = _mm_loadl_epi64((__m128i*)&in[28]);
       in0 = _mm_unpacklo_epi64(in0, inB0);
       in1 = _mm_unpacklo_epi64(in1, inB1);
       in2 = _mm_unpacklo_epi64(in2, inB2);
@@ -242,10 +242,20 @@ static void ITransformSSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst,
   {
     const __m128i zero = _mm_set1_epi16(0);
     // Load the reference(s).
-    __m128i ref0 = _mm_loadl_epi64((__m128i *)&ref[0 * BPS]);
+    __m128i ref0, ref1, ref2, ref3;
-    __m128i ref1 = _mm_loadl_epi64((__m128i *)&ref[1 * BPS]);
+    if (do_two) {
-    __m128i ref2 = _mm_loadl_epi64((__m128i *)&ref[2 * BPS]);
+      // Load eight bytes/pixels per line.
-    __m128i ref3 = _mm_loadl_epi64((__m128i *)&ref[3 * BPS]);
+      ref0 = _mm_loadl_epi64((__m128i*)&ref[0 * BPS]);
+      ref1 = _mm_loadl_epi64((__m128i*)&ref[1 * BPS]);
+      ref2 = _mm_loadl_epi64((__m128i*)&ref[2 * BPS]);
+      ref3 = _mm_loadl_epi64((__m128i*)&ref[3 * BPS]);
+    } else {
+      // Load four bytes/pixels per line.
+      ref0 = _mm_cvtsi32_si128(*(int*)&ref[0 * BPS]);
+      ref1 = _mm_cvtsi32_si128(*(int*)&ref[1 * BPS]);
+      ref2 = _mm_cvtsi32_si128(*(int*)&ref[2 * BPS]);
+      ref3 = _mm_cvtsi32_si128(*(int*)&ref[3 * BPS]);
+    }
     // Convert to 16b.
     ref0 = _mm_unpacklo_epi8(ref0, zero);
     ref1 = _mm_unpacklo_epi8(ref1, zero);
@@ -264,10 +274,10 @@ static void ITransformSSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst,
     // Store the results.
     if (do_two) {
       // Store eight bytes/pixels per line.
-      _mm_storel_epi64((__m128i *)&dst[0 * BPS], ref0);
+      _mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0);
-      _mm_storel_epi64((__m128i *)&dst[1 * BPS], ref1);
+      _mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1);
-      _mm_storel_epi64((__m128i *)&dst[2 * BPS], ref2);
+      _mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2);
-      _mm_storel_epi64((__m128i *)&dst[3 * BPS], ref3);
+      _mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3);
     } else {
       // Store four bytes/pixels per line.
       *((int32_t *)&dst[0 * BPS]) = _mm_cvtsi128_si32(ref0);
@@ -296,19 +306,19 @@ static void FTransformSSE2(const uint8_t* src, const uint8_t* ref,
   // Difference between src and ref and initial transpose.
   {
     // Load src and convert to 16b.
-    const __m128i src0 = _mm_loadl_epi64((__m128i *)&src[0 * BPS]);
+    const __m128i src0 = _mm_loadl_epi64((__m128i*)&src[0 * BPS]);
-    const __m128i src1 = _mm_loadl_epi64((__m128i *)&src[1 * BPS]);
+    const __m128i src1 = _mm_loadl_epi64((__m128i*)&src[1 * BPS]);
-    const __m128i src2 = _mm_loadl_epi64((__m128i *)&src[2 * BPS]);
+    const __m128i src2 = _mm_loadl_epi64((__m128i*)&src[2 * BPS]);
-    const __m128i src3 = _mm_loadl_epi64((__m128i *)&src[3 * BPS]);
+    const __m128i src3 = _mm_loadl_epi64((__m128i*)&src[3 * BPS]);
     const __m128i src_0 = _mm_unpacklo_epi8(src0, zero);
     const __m128i src_1 = _mm_unpacklo_epi8(src1, zero);
     const __m128i src_2 = _mm_unpacklo_epi8(src2, zero);
     const __m128i src_3 = _mm_unpacklo_epi8(src3, zero);
     // Load ref and convert to 16b.
-    const __m128i ref0 = _mm_loadl_epi64((__m128i *)&ref[0 * BPS]);
+    const __m128i ref0 = _mm_loadl_epi64((__m128i*)&ref[0 * BPS]);
-    const __m128i ref1 = _mm_loadl_epi64((__m128i *)&ref[1 * BPS]);
+    const __m128i ref1 = _mm_loadl_epi64((__m128i*)&ref[1 * BPS]);
-    const __m128i ref2 = _mm_loadl_epi64((__m128i *)&ref[2 * BPS]);
+    const __m128i ref2 = _mm_loadl_epi64((__m128i*)&ref[2 * BPS]);
-    const __m128i ref3 = _mm_loadl_epi64((__m128i *)&ref[3 * BPS]);
+    const __m128i ref3 = _mm_loadl_epi64((__m128i*)&ref[3 * BPS]);
     const __m128i ref_0 = _mm_unpacklo_epi8(ref0, zero);
     const __m128i ref_1 = _mm_unpacklo_epi8(ref1, zero);
     const __m128i ref_2 = _mm_unpacklo_epi8(ref2, zero);
@@ -419,10 +429,10 @@ static void FTransformSSE2(const uint8_t* src, const uint8_t* ref,
     // desired (0, 1), we add one earlier through k12000_plus_one.
     const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero));
 
-    _mm_storel_epi64((__m128i *)&out[ 0], d0);
+    _mm_storel_epi64((__m128i*)&out[ 0], d0);
-    _mm_storel_epi64((__m128i *)&out[ 4], g1);
+    _mm_storel_epi64((__m128i*)&out[ 4], g1);
-    _mm_storel_epi64((__m128i *)&out[ 8], d2);
+    _mm_storel_epi64((__m128i*)&out[ 8], d2);
-    _mm_storel_epi64((__m128i *)&out[12], f3);
+    _mm_storel_epi64((__m128i*)&out[12], f3);
   }
 }
 
@@ -433,14 +443,14 @@ static int SSE4x4SSE2(const uint8_t* a, const uint8_t* b) {
   const __m128i zero = _mm_set1_epi16(0);
 
   // Load values.
-  const __m128i a0 = _mm_loadl_epi64((__m128i *)&a[BPS * 0]);
+  const __m128i a0 = _mm_loadl_epi64((__m128i*)&a[BPS * 0]);
-  const __m128i a1 = _mm_loadl_epi64((__m128i *)&a[BPS * 1]);
+  const __m128i a1 = _mm_loadl_epi64((__m128i*)&a[BPS * 1]);
-  const __m128i a2 = _mm_loadl_epi64((__m128i *)&a[BPS * 2]);
+  const __m128i a2 = _mm_loadl_epi64((__m128i*)&a[BPS * 2]);
-  const __m128i a3 = _mm_loadl_epi64((__m128i *)&a[BPS * 3]);
+  const __m128i a3 = _mm_loadl_epi64((__m128i*)&a[BPS * 3]);
-  const __m128i b0 = _mm_loadl_epi64((__m128i *)&b[BPS * 0]);
+  const __m128i b0 = _mm_loadl_epi64((__m128i*)&b[BPS * 0]);
-  const __m128i b1 = _mm_loadl_epi64((__m128i *)&b[BPS * 1]);
+  const __m128i b1 = _mm_loadl_epi64((__m128i*)&b[BPS * 1]);
-  const __m128i b2 = _mm_loadl_epi64((__m128i *)&b[BPS * 2]);
+  const __m128i b2 = _mm_loadl_epi64((__m128i*)&b[BPS * 2]);
-  const __m128i b3 = _mm_loadl_epi64((__m128i *)&b[BPS * 3]);
+  const __m128i b3 = _mm_loadl_epi64((__m128i*)&b[BPS * 3]);
 
   // Combine pair of lines and convert to 16b.
   const __m128i a01 = _mm_unpacklo_epi32(a0, a1);
@@ -471,7 +481,7 @@ static int SSE4x4SSE2(const uint8_t* a, const uint8_t* b) {
   const __m128i sum1 = _mm_add_epi32(madd2, madd3);
   const __m128i sum2 = _mm_add_epi32(sum0, sum1);
   int32_t tmp[4];
-  _mm_storeu_si128((__m128i *)tmp, sum2);
+  _mm_storeu_si128((__m128i*)tmp, sum2);
   return (tmp[3] + tmp[2] + tmp[1] + tmp[0]);
 }
 
@@ -494,14 +504,14 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB,
 
   // Load, combine and tranpose inputs.
   {
-    const __m128i inA_0 = _mm_loadl_epi64((__m128i *)&inA[BPS * 0]);
+    const __m128i inA_0 = _mm_loadl_epi64((__m128i*)&inA[BPS * 0]);
-    const __m128i inA_1 = _mm_loadl_epi64((__m128i *)&inA[BPS * 1]);
+    const __m128i inA_1 = _mm_loadl_epi64((__m128i*)&inA[BPS * 1]);
-    const __m128i inA_2 = _mm_loadl_epi64((__m128i *)&inA[BPS * 2]);
+    const __m128i inA_2 = _mm_loadl_epi64((__m128i*)&inA[BPS * 2]);
-    const __m128i inA_3 = _mm_loadl_epi64((__m128i *)&inA[BPS * 3]);
+    const __m128i inA_3 = _mm_loadl_epi64((__m128i*)&inA[BPS * 3]);
-    const __m128i inB_0 = _mm_loadl_epi64((__m128i *)&inB[BPS * 0]);
+    const __m128i inB_0 = _mm_loadl_epi64((__m128i*)&inB[BPS * 0]);
-    const __m128i inB_1 = _mm_loadl_epi64((__m128i *)&inB[BPS * 1]);
+    const __m128i inB_1 = _mm_loadl_epi64((__m128i*)&inB[BPS * 1]);
-    const __m128i inB_2 = _mm_loadl_epi64((__m128i *)&inB[BPS * 2]);
+    const __m128i inB_2 = _mm_loadl_epi64((__m128i*)&inB[BPS * 2]);
-    const __m128i inB_3 = _mm_loadl_epi64((__m128i *)&inB[BPS * 3]);
+    const __m128i inB_3 = _mm_loadl_epi64((__m128i*)&inB[BPS * 3]);
 
     // Combine inA and inB (we'll do two transforms in parallel).
     const __m128i inAB_0 = _mm_unpacklo_epi8(inA_0, inB_0);
@@ -585,8 +595,8 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB,
     // Load all inputs.
     // TODO(cduvivier): Make variable declarations and allocations aligned so
     //                  we can use _mm_load_si128 instead of _mm_loadu_si128.
-    const __m128i w_0 = _mm_loadu_si128((__m128i *)&w[0]);
+    const __m128i w_0 = _mm_loadu_si128((__m128i*)&w[0]);
-    const __m128i w_8 = _mm_loadu_si128((__m128i *)&w[8]);
+    const __m128i w_8 = _mm_loadu_si128((__m128i*)&w[8]);
 
     // Calculate a and b (two 4x4 at once).
     const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2);
@@ -645,7 +655,7 @@ static int TTransformSSE2(const uint8_t* inA, const uint8_t* inB,
 
     // difference of weighted sums
     A_b0 = _mm_sub_epi32(A_b0, B_b0);
-    _mm_storeu_si128((__m128i *)&sum[0], A_b0);
+    _mm_storeu_si128((__m128i*)&sum[0], A_b0);
   }
   return sum[0] + sum[1] + sum[2] + sum[3];
 }
@@ -686,18 +696,18 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16],
   // Load all inputs.
   // TODO(cduvivier): Make variable declarations and allocations aligned so that
   //                  we can use _mm_load_si128 instead of _mm_loadu_si128.
-  __m128i in0 = _mm_loadu_si128((__m128i *)&in[0]);
+  __m128i in0 = _mm_loadu_si128((__m128i*)&in[0]);
-  __m128i in8 = _mm_loadu_si128((__m128i *)&in[8]);
+  __m128i in8 = _mm_loadu_si128((__m128i*)&in[8]);
-  const __m128i sharpen0 = _mm_loadu_si128((__m128i *)&mtx->sharpen_[0]);
+  const __m128i sharpen0 = _mm_loadu_si128((__m128i*)&mtx->sharpen_[0]);
-  const __m128i sharpen8 = _mm_loadu_si128((__m128i *)&mtx->sharpen_[8]);
+  const __m128i sharpen8 = _mm_loadu_si128((__m128i*)&mtx->sharpen_[8]);
-  const __m128i iq0 = _mm_loadu_si128((__m128i *)&mtx->iq_[0]);
+  const __m128i iq0 = _mm_loadu_si128((__m128i*)&mtx->iq_[0]);
-  const __m128i iq8 = _mm_loadu_si128((__m128i *)&mtx->iq_[8]);
+  const __m128i iq8 = _mm_loadu_si128((__m128i*)&mtx->iq_[8]);
-  const __m128i bias0 = _mm_loadu_si128((__m128i *)&mtx->bias_[0]);
+  const __m128i bias0 = _mm_loadu_si128((__m128i*)&mtx->bias_[0]);
-  const __m128i bias8 = _mm_loadu_si128((__m128i *)&mtx->bias_[8]);
+  const __m128i bias8 = _mm_loadu_si128((__m128i*)&mtx->bias_[8]);
-  const __m128i q0 = _mm_loadu_si128((__m128i *)&mtx->q_[0]);
+  const __m128i q0 = _mm_loadu_si128((__m128i*)&mtx->q_[0]);
-  const __m128i q8 = _mm_loadu_si128((__m128i *)&mtx->q_[8]);
+  const __m128i q8 = _mm_loadu_si128((__m128i*)&mtx->q_[8]);
-  const __m128i zthresh0 = _mm_loadu_si128((__m128i *)&mtx->zthresh_[0]);
+  const __m128i zthresh0 = _mm_loadu_si128((__m128i*)&mtx->zthresh_[0]);
-  const __m128i zthresh8 = _mm_loadu_si128((__m128i *)&mtx->zthresh_[8]);
+  const __m128i zthresh8 = _mm_loadu_si128((__m128i*)&mtx->zthresh_[8]);
 
   // sign(in) = in >> 15  (0x0000 if positive, 0xffff if negative)
   sign0 = _mm_srai_epi16(in0, 15);
@@ -765,8 +775,8 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16],
     __m128i cmp8 = _mm_cmpgt_epi16(coeff8, zthresh8);
     in0 = _mm_and_si128(in0, cmp0);
     in8 = _mm_and_si128(in8, cmp8);
-    _mm_storeu_si128((__m128i *)&in[0], in0);
+    _mm_storeu_si128((__m128i*)&in[0], in0);
-    _mm_storeu_si128((__m128i *)&in[8], in8);
+    _mm_storeu_si128((__m128i*)&in[8], in8);
     out0 = _mm_and_si128(out0, cmp0);
     out8 = _mm_and_si128(out8, cmp8);
   }
@@ -784,8 +794,8 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16],
     outZ8 = _mm_shufflelo_epi16(out8,  _MM_SHUFFLE(3, 0, 2, 1));
     outZ8 = _mm_shuffle_epi32  (outZ8, _MM_SHUFFLE(3, 1, 2, 0));
     outZ8 = _mm_shufflelo_epi16(outZ8, _MM_SHUFFLE(1, 3, 2, 0));
-    _mm_storeu_si128((__m128i *)&out[0], outZ0);
+    _mm_storeu_si128((__m128i*)&out[0], outZ0);
-    _mm_storeu_si128((__m128i *)&out[8], outZ8);
+    _mm_storeu_si128((__m128i*)&out[8], outZ8);
     packed_out = _mm_packs_epi16(outZ0, outZ8);
   }
   {
@@ -798,7 +808,7 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16],
   // detect if all 'out' values are zeroes or not
   {
     int32_t tmp[4];
-    _mm_storeu_si128((__m128i *)tmp, packed_out);
+    _mm_storeu_si128((__m128i*)tmp, packed_out);
     if (n) {
       tmp[0] &= ~0xff;
     }