Add SSE2 function for Inverse Cross-color Transform

Lossless decoding is now ~3% faster.

Change-Id: Idafb5c73e5cfb272cc3661d841f79971f9da0743

src/dsp/lossless_sse2.c

@@ -13,12 +13,14 @@
 
 #include "./dsp.h"
 
+#include <assert.h>
+
 #if defined(WEBP_USE_SSE2)
 #include <emmintrin.h>
 #include "./lossless.h"
 
 //------------------------------------------------------------------------------
-// Predictors
+// Predictor Transform
 
 static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1,
                                                    uint32_t c2) {
@@ -118,7 +120,7 @@ static uint32_t Predictor13(uint32_t left, const uint32_t* const top) {
 }
 
 //------------------------------------------------------------------------------
-// Colorspace conversion functions
+// Subtract-Green Transform
 
 static void SubtractGreenFromBlueAndRed(uint32_t* argb_data, int num_pixels) {
   const __m128i mask = _mm_set1_epi32(0x0000ff00);
@@ -152,6 +154,65 @@ static void AddGreenToBlueAndRed(uint32_t* argb_data, int num_pixels) {
   VP8LAddGreenToBlueAndRed_C(argb_data + i, num_pixels - i);
 }
 
+//------------------------------------------------------------------------------
+// Color Transform
+
+static WEBP_INLINE __m128i ColorTransformDelta(__m128i color_pred,
+                                               __m128i color) {
+  // We simulate signed 8-bit multiplication as:
+  // * Left shift the two (8-bit) numbers by 8 bits,
+  // * Perform a 16-bit signed multiplication and retain the higher 16-bits.
+  const __m128i color_pred_shifted = _mm_slli_epi32(color_pred, 8);
+  const __m128i color_shifted = _mm_slli_epi32(color, 8);
+  // Note: This performs multiplication on 8 packed 16-bit numbers, 4 of which
+  // happen to be zeroes.
+  const __m128i signed_mult =
+      _mm_mulhi_epi16(color_pred_shifted, color_shifted);
+  return _mm_srli_epi32(signed_mult, 5);
+}
+
+static WEBP_INLINE void TransformColorInverse(const VP8LMultipliers* const m,
+                                              uint32_t* argb_data,
+                                              int num_pixels) {
+  const __m128i g_to_r = _mm_set1_epi32(m->green_to_red_);       // multipliers
+  const __m128i g_to_b = _mm_set1_epi32(m->green_to_blue_);
+  const __m128i r_to_b = _mm_set1_epi32(m->red_to_blue_);
+
+  int i;
+
+  for (i = 0; i + 4 <= num_pixels; i += 4) {
+    const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]);
+    const __m128i alpha_green_mask = _mm_set1_epi32(0xff00ff00);  // masks
+    const __m128i red_mask = _mm_set1_epi32(0x00ff0000);
+    const __m128i green_mask = _mm_set1_epi32(0x0000ff00);
+    const __m128i lower_8bit_mask  = _mm_set1_epi32(0x000000ff);
+    const __m128i ag = _mm_and_si128(in, alpha_green_mask);      // alpha, green
+    const __m128i r = _mm_srli_epi32(_mm_and_si128(in, red_mask), 16);
+    const __m128i g = _mm_srli_epi32(_mm_and_si128(in, green_mask), 8);
+    const __m128i b = in;
+
+    const __m128i r_delta = ColorTransformDelta(g_to_r, g);      // red
+    const __m128i r_new =
+        _mm_and_si128(_mm_add_epi32(r, r_delta), lower_8bit_mask);
+    const __m128i r_new_shifted = _mm_slli_epi32(r_new, 16);
+
+    const __m128i b_delta_1 = ColorTransformDelta(g_to_b, g);    // blue
+    const __m128i b_delta_2 = ColorTransformDelta(r_to_b, r_new);
+    const __m128i b_delta = _mm_add_epi32(b_delta_1, b_delta_2);
+    const __m128i b_new =
+        _mm_and_si128(_mm_add_epi32(b, b_delta), lower_8bit_mask);
+
+    const __m128i out = _mm_or_si128(_mm_or_si128(ag, r_new_shifted), b_new);
+    _mm_storeu_si128((__m128i*)&argb_data[i], out);
+  }
+
+  // Fall-back to C-version for left-overs.
+  VP8LTransformColorInverse_C(m, argb_data + i, num_pixels - i);
+}
+
+//------------------------------------------------------------------------------
+// Color-space conversion functions
+
 static void ConvertBGRAToRGBA(const uint32_t* src,
                               int num_pixels, uint8_t* dst) {
   const __m128i* in = (const __m128i*)src;
@@ -298,6 +359,8 @@ void VP8LDspInitSSE2(void) {
   VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed;
   VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed;
 
+  VP8LTransformColorInverse = TransformColorInverse;
+
   VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA;
   VP8LConvertBGRAToRGBA4444 = ConvertBGRAToRGBA4444;
   VP8LConvertBGRAToRGB565 = ConvertBGRAToRGB565;