rescaler: add some SSE2 code

The rounding and arithmetic is not the same as previously, to prevent overflow cases for large upscale factors.

We still rely on 32b x 32b -> 64b multiplies. Raised the fixed-point precision to 32b
so that we have some nice shifts from epi64 to epi32.
Changed rescaler_t type to 'uint32_t' in order to squeeze in all the precision required.

The MIPS code has been disabled because it's now out-of-sync. Will be fixed in
a subsequent CL when the dust settles.
~30-35% faster

Change-Id: I32e4ddc00933f1b1aa3463403086199fd5dad07b

Android.mk

@@ -62,6 +62,7 @@ dsp_dec_srcs := \
     src/dsp/rescaler.c \
     src/dsp/rescaler_mips32.c \
     src/dsp/rescaler_mips_dsp_r2.c \
+    src/dsp/rescaler_sse2.c \
     src/dsp/upsampling.c \
     src/dsp/upsampling_mips_dsp_r2.c \
     src/dsp/upsampling_neon.$(NEON) \

Makefile.vc

@@ -207,6 +207,7 @@ DSP_DEC_OBJS = \
     $(DIROBJ)\dsp\rescaler.obj \
     $(DIROBJ)\dsp\rescaler_mips32.obj \
     $(DIROBJ)\dsp\rescaler_mips_dsp_r2.obj \
+    $(DIROBJ)\dsp\rescaler_sse2.obj \
     $(DIROBJ)\dsp\upsampling.obj \
     $(DIROBJ)\dsp\upsampling_mips_dsp_r2.obj \
     $(DIROBJ)\dsp\upsampling_neon.obj \

makefile.unix

@@ -149,6 +149,7 @@ DSP_DEC_OBJS = \
     src/dsp/rescaler.o \
     src/dsp/rescaler_mips32.o \
     src/dsp/rescaler_mips_dsp_r2.o \
+    src/dsp/rescaler_sse2.o \
     src/dsp/upsampling.o \
     src/dsp/upsampling_mips_dsp_r2.o \
     src/dsp/upsampling_neon.o \

src/dsp/Makefile.am

@@ -69,6 +69,7 @@ libwebpdspdecode_sse2_la_SOURCES += alpha_processing_sse2.c
 libwebpdspdecode_sse2_la_SOURCES += dec_sse2.c
 libwebpdspdecode_sse2_la_SOURCES += filters_sse2.c
 libwebpdspdecode_sse2_la_SOURCES += lossless_sse2.c
+libwebpdspdecode_sse2_la_SOURCES += rescaler_sse2.c
 libwebpdspdecode_sse2_la_SOURCES += upsampling_sse2.c
 libwebpdspdecode_sse2_la_SOURCES += yuv_sse2.c
 libwebpdspdecode_sse2_la_SOURCES += yuv_tables_sse2.h

src/dsp/rescaler.c

@@ -100,18 +100,24 @@ void WebPRescalerExportRowExpandC(WebPRescaler* const wrk) {
   assert(!WebPRescalerOutputDone(wrk));
   assert(wrk->y_accum <= 0);
   assert(wrk->y_expand);
+  assert(wrk->y_sub != 0);
   if (wrk->y_accum == 0) {
     for (x_out = 0; x_out < x_out_max; ++x_out) {
-      const int v = (int)MULT_FIX(frow[x_out], wrk->fy_scale);
+      const uint32_t J = frow[x_out];
-      dst[x_out] = (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255;
+      const int v = (int)MULT_FIX(J, wrk->fy_scale);
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
     }
   } else {
-    const int64_t A = wrk->y_sub + wrk->y_accum;
+    const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
-    const int64_t B = -wrk->y_accum;
+    const uint32_t A = WEBP_RESCALER_ONE - B;
     for (x_out = 0; x_out < x_out_max; ++x_out) {
-      const int64_t I = A * frow[x_out] + B * irow[x_out];
+      const uint64_t I = (uint64_t)A * frow[x_out]
-      const int v = (int)MULT_FIX(I, wrk->fxy_scale);
+                       + (uint64_t)B * irow[x_out];
-      dst[x_out] = (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255;
+      const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
+      const int v = (int)MULT_FIX(J, wrk->fy_scale);
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
     }
   }
 }
@@ -122,21 +128,28 @@ void WebPRescalerExportRowShrinkC(WebPRescaler* const wrk) {
   rescaler_t* const irow = wrk->irow;
   const int x_out_max = wrk->dst_width * wrk->num_channels;
   const rescaler_t* const frow = wrk->frow;
-  const int yscale = wrk->fy_scale * (-wrk->y_accum);
+  const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum);
   assert(!WebPRescalerOutputDone(wrk));
   assert(wrk->y_accum <= 0);
   assert(!wrk->y_expand);
   if (yscale) {
     for (x_out = 0; x_out < x_out_max; ++x_out) {
-      const int frac = (int)MULT_FIX(frow[x_out], yscale);
+      const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale);
       const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale);
-      dst[x_out] = (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255;
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
       irow[x_out] = frac;   // new fractional start
     }
-  } else {
+  } else if (wrk->fxy_scale) {
     for (x_out = 0; x_out < x_out_max; ++x_out) {
       const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale);
-      dst[x_out] = (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255;
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
+      irow[x_out] = 0;
+    }
+  } else {  // very special case for src = dst = 1x1
+    for (x_out = 0; x_out < x_out_max; ++x_out) {
+      dst[x_out] = irow[x_out];
       irow[x_out] = 0;
     }
   }
@@ -179,6 +192,7 @@ WebPRescalerImportRowFunc WebPRescalerImportRowShrink;
 WebPRescalerExportRowFunc WebPRescalerExportRowExpand;
 WebPRescalerExportRowFunc WebPRescalerExportRowShrink;
 
+extern void WebPRescalerDspInitSSE2(void);
 extern void WebPRescalerDspInitMIPS32(void);
 extern void WebPRescalerDspInitMIPSdspR2(void);
 
@@ -194,6 +208,11 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInit(void) {
   WebPRescalerExportRowShrink = WebPRescalerExportRowShrinkC;
 
   if (VP8GetCPUInfo != NULL) {
+#if defined(WEBP_USE_SSE2)
+    if (VP8GetCPUInfo(kSSE2)) {
+      WebPRescalerDspInitSSE2();
+    }
+#endif
 #if defined(WEBP_USE_MIPS32)
     if (VP8GetCPUInfo(kMIPS32)) {
       WebPRescalerDspInitMIPS32();

src/dsp/rescaler_mips32.c

@@ -31,7 +31,7 @@ static void ImportRowShrink(WebPRescaler* const wrk, const uint8_t* src) {
 
   for (channel = 0; channel < x_stride; ++channel) {
     const uint8_t* src1 = src + channel;
-    int* frow = wrk->frow + channel;
+    rescaler_t* frow = wrk->frow + channel;
     int temp1, temp2, temp3;
     int base, frac, sum;
     int accum, accum1;
@@ -90,7 +90,7 @@ static void ImportRowExpand(WebPRescaler* const wrk, const uint8_t* src) {
 
   for (channel = 0; channel < x_stride; ++channel) {
     const uint8_t* src1 = src + channel;
-    int* frow = wrk->frow + channel;
+    rescaler_t* frow = wrk->frow + channel;
     int temp1, temp2, temp3, temp4;
     int frac;
     int accum;
@@ -138,18 +138,15 @@ static void ImportRowExpand(WebPRescaler* const wrk, const uint8_t* src) {
 }
 
 static void ExportRowShrink(WebPRescaler* const wrk) {
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
+  uint8_t* dst = wrk->dst;
+  rescaler_t* irow = wrk->irow;
   assert(!WebPRescalerOutputDone(wrk));
   assert(wrk->y_accum <= 0);
   assert(!wrk->y_expand);
-  // if wrk->fxy_scale can fit into 32 bits use optimized code,
+  if (wrk->fxy_scale != 0) {
-  // otherwise use C code
-  if ((wrk->fxy_scale >> 32) == 0) {
-    uint8_t* dst = wrk->dst;
-    rescaler_t* irow = wrk->irow;
     const rescaler_t* frow = wrk->frow;
     const int yscale = wrk->fy_scale * (-wrk->y_accum);
-    const int x_out_max = wrk->dst_width * wrk->num_channels;
-
     int temp0, temp1, temp3, temp4, temp5, temp6, temp7, loop_end;
     const int temp2 = (int)(wrk->fxy_scale);
     const int temp8 = x_out_max << 2;
@@ -192,8 +189,12 @@ static void ExportRowShrink(WebPRescaler* const wrk) {
       : [temp2]"r"(temp2), [yscale]"r"(yscale), [temp8]"r"(temp8)
       : "memory", "hi", "lo"
     );
-  } else {
+  } else {  // very special case for src = dst = 1x1
-    WebPRescalerExportRowShrinkC(wrk);
+    int x_out;
+    for (x_out = 0; x_out < x_out_max; ++x_out) {
+      dst[x_out] = irow[x_out];
+      irow[x_out] = 0;
+    }
   }
 }
 
@@ -205,9 +206,17 @@ static void ExportRowShrink(WebPRescaler* const wrk) {
 extern void WebPRescalerDspInitMIPS32(void);
 
 WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMIPS32(void) {
+#if 0
+  // The assembly code is currently out-of-sync wrt the C-implementation.
+  // Disabled for now.
   WebPRescalerImportRowExpand = ImportRowExpand;
   WebPRescalerImportRowShrink = ImportRowShrink;
   WebPRescalerExportRowShrink = ExportRowShrink;
+#else
+  (void)ImportRowExpand;
+  (void)ImportRowShrink;
+  (void)ExportRowShrink;
+#endif
 }
 
 #else  // !WEBP_USE_MIPS32

src/dsp/rescaler_mips_dsp_r2.c

@@ -30,7 +30,7 @@ static void ImportRowShrink(WebPRescaler* const wrk, const uint8_t* src) {
   assert(!WebPRescalerInputDone(wrk));
 
   for (channel = 0; channel < x_stride; ++channel) {
-    int* frow = wrk->frow + channel;
+    rescaler_t* frow = wrk->frow + channel;
     const uint8_t* src1 = src + channel;
     int temp3;
     int base, frac, sum;
@@ -84,7 +84,7 @@ static void ImportRowExpand(WebPRescaler* const wrk, const uint8_t* src) {
   assert(!WebPRescalerInputDone(wrk));
 
   for (channel = 0; channel < x_stride; ++channel) {
-    int* frow = wrk->frow + channel;
+    rescaler_t* frow = wrk->frow + channel;
     const uint8_t* src1 = src + channel;
     int temp1, temp2, temp3, temp4;
     int frac;
@@ -133,17 +133,15 @@ static void ImportRowExpand(WebPRescaler* const wrk, const uint8_t* src) {
 }
 
 static void ExportRowShrink(WebPRescaler* const wrk) {
+  uint8_t* dst = wrk->dst;
+  rescaler_t* irow = wrk->irow;
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
   assert(!WebPRescalerOutputDone(wrk));
   assert(wrk->y_accum <= 0);
   assert(!wrk->y_expand);
-  // if wrk->fxy_scale can fit into 32 bits use optimized code,
+  if (wrk->fxy_scale) {
-  // otherwise use C code
-  if ((wrk->fxy_scale >> 32) == 0) {
-    uint8_t* dst = wrk->dst;
-    rescaler_t* irow = wrk->irow;
     const rescaler_t* frow = wrk->frow;
     const int yscale = wrk->fy_scale * (-wrk->y_accum);
-    const int x_out_max = wrk->dst_width * wrk->num_channels;
 
     int temp0, temp1, temp3, temp4, temp5, temp6, temp7;
     const int temp2 = (int)wrk->fxy_scale;
@@ -212,8 +210,12 @@ static void ExportRowShrink(WebPRescaler* const wrk) {
         [rest]"r"(rest)
       : "memory", "hi", "lo"
     );
-  } else {
+  } else {  // very special case for src = dst = 1x1
-    WebPRescalerExportRowShrinkC(wrk);
+    int x_out;
+    for (x_out = 0; x_out < x_out_max; ++x_out) {
+      dst[x_out] = irow[x_out];
+      irow[x_out] = 0;
+    }
   }
 }
 
@@ -225,9 +227,17 @@ static void ExportRowShrink(WebPRescaler* const wrk) {
 extern void WebPRescalerDspInitMIPSdspR2(void);
 
 WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitMIPSdspR2(void) {
+#if 0
+  // The assembly code is currently out-of-sync wrt the C-implementation.
+  // Disabled for now.
   WebPRescalerImportRowExpand = ImportRowExpand;
   WebPRescalerImportRowShrink = ImportRowShrink;
   WebPRescalerExportRowShrink = ExportRowShrink;
+#else
+  (void)ImportRowExpand;
+  (void)ImportRowShrink;
+  (void)ExportRowShrink;
+#endif
 }
 
 #else  // !WEBP_USE_MIPS_DSP_R2

src/dsp/rescaler_sse2.c

@@ -0,0 +1,232 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// SSE2 Rescaling functions
+//
+// Author: Skal (pascal.massimino@gmail.com)
+
+#include "./dsp.h"
+
+#if defined(WEBP_USE_SSE2)
+#include <emmintrin.h>
+
+#include <assert.h>
+#include "../utils/rescaler.h"
+
+//------------------------------------------------------------------------------
+// Implementations of critical functions ImportRow / ExportRow
+
+#define ROUNDER (WEBP_RESCALER_ONE >> 1)
+#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX)
+
+//------------------------------------------------------------------------------
+// Row export
+
+// load *src as epi64, multiply by mult and store result in [out0 ... out3]
+static WEBP_INLINE void LoadDispatchAndMult(const rescaler_t* const src,
+                                            const __m128i* const mult,
+                                            __m128i* const out0,
+                                            __m128i* const out1,
+                                            __m128i* const out2,
+                                            __m128i* const out3) {
+  const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + 0));
+  const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + 4));
+  const __m128i A2 = _mm_srli_epi64(A0, 32);
+  const __m128i A3 = _mm_srli_epi64(A1, 32);
+  if (mult != NULL) {
+    *out0 = _mm_mul_epu32(A0, *mult);
+    *out1 = _mm_mul_epu32(A1, *mult);
+    *out2 = _mm_mul_epu32(A2, *mult);
+    *out3 = _mm_mul_epu32(A3, *mult);
+  } else {
+    *out0 = A0;
+    *out1 = A1;
+    *out2 = A2;
+    *out3 = A3;
+  }
+}
+
+static WEBP_INLINE void ProcessRow(const __m128i* const A0,
+                                   const __m128i* const A1,
+                                   const __m128i* const A2,
+                                   const __m128i* const A3,
+                                   const __m128i* const mult,
+                                   uint8_t* const dst) {
+  const __m128i rounder = _mm_set_epi64x(ROUNDER, ROUNDER);
+  const __m128i mask = _mm_set_epi64x(0xffffffff00000000ull,
+                                      0xffffffff00000000ull);
+  const __m128i B0 = _mm_mul_epu32(*A0, *mult);
+  const __m128i B1 = _mm_mul_epu32(*A1, *mult);
+  const __m128i B2 = _mm_mul_epu32(*A2, *mult);
+  const __m128i B3 = _mm_mul_epu32(*A3, *mult);
+  const __m128i C0 = _mm_add_epi64(B0, rounder);
+  const __m128i C1 = _mm_add_epi64(B1, rounder);
+  const __m128i C2 = _mm_add_epi64(B2, rounder);
+  const __m128i C3 = _mm_add_epi64(B3, rounder);
+  const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX);
+  const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX);
+  const __m128i D2 = _mm_and_si128(C2, mask);
+  const __m128i D3 = _mm_and_si128(C3, mask);
+  const __m128i E0 = _mm_or_si128(D0, D2);
+  const __m128i E1 = _mm_or_si128(D1, D3);
+  const __m128i F = _mm_packs_epi32(E0, E1);
+  const __m128i G = _mm_packus_epi16(F, F);
+  _mm_storel_epi64((__m128i*)dst, G);
+}
+
+static void RescalerExportRowExpandSSE2(WebPRescaler* const wrk) {
+  int x_out;
+  uint8_t* const dst = wrk->dst;
+  rescaler_t* const irow = wrk->irow;
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
+  const rescaler_t* const frow = wrk->frow;
+  const __m128i mult = _mm_set_epi64x(wrk->fy_scale, wrk->fy_scale);
+
+  assert(!WebPRescalerOutputDone(wrk));
+  assert(wrk->y_accum <= 0 && wrk->y_sub + wrk->y_accum >= 0);
+  assert(wrk->y_expand);
+  if (wrk->y_accum == 0) {
+    for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) {
+      __m128i A0, A1, A2, A3;
+      LoadDispatchAndMult(frow + x_out, NULL, &A0, &A1, &A2, &A3);
+      ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out);
+    }
+    for (; x_out < x_out_max; ++x_out) {
+      const uint32_t J = frow[x_out];
+      const int v = (int)MULT_FIX(J, wrk->fy_scale);
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
+    }
+  } else {
+    const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
+    const uint32_t A = WEBP_RESCALER_ONE - B;
+    const __m128i mA = _mm_set_epi64x(A, A);
+    const __m128i mB = _mm_set_epi64x(B, B);
+    const __m128i rounder = _mm_set_epi64x(ROUNDER, ROUNDER);
+    for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) {
+      __m128i A0, A1, A2, A3, B0, B1, B2, B3;
+      LoadDispatchAndMult(frow + x_out, &mA, &A0, &A1, &A2, &A3);
+      LoadDispatchAndMult(irow + x_out, &mB, &B0, &B1, &B2, &B3);
+      {
+        const __m128i C0 = _mm_add_epi64(A0, B0);
+        const __m128i C1 = _mm_add_epi64(A1, B1);
+        const __m128i C2 = _mm_add_epi64(A2, B2);
+        const __m128i C3 = _mm_add_epi64(A3, B3);
+        const __m128i D0 = _mm_add_epi64(C0, rounder);
+        const __m128i D1 = _mm_add_epi64(C1, rounder);
+        const __m128i D2 = _mm_add_epi64(C2, rounder);
+        const __m128i D3 = _mm_add_epi64(C3, rounder);
+        const __m128i E0 = _mm_srli_epi64(D0, WEBP_RESCALER_RFIX);
+        const __m128i E1 = _mm_srli_epi64(D1, WEBP_RESCALER_RFIX);
+        const __m128i E2 = _mm_srli_epi64(D2, WEBP_RESCALER_RFIX);
+        const __m128i E3 = _mm_srli_epi64(D3, WEBP_RESCALER_RFIX);
+        ProcessRow(&E0, &E1, &E2, &E3, &mult, dst + x_out);
+      }
+    }
+    for (; x_out < x_out_max; ++x_out) {
+      const uint64_t I = (uint64_t)A * frow[x_out]
+                       + (uint64_t)B * irow[x_out];
+      const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
+      const int v = (int)MULT_FIX(J, wrk->fy_scale);
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
+    }
+  }
+}
+
+static void RescalerExportRowShrinkSSE2(WebPRescaler* const wrk) {
+  int x_out;
+  uint8_t* const dst = wrk->dst;
+  rescaler_t* const irow = wrk->irow;
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
+  const rescaler_t* const frow = wrk->frow;
+  const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum);
+  assert(!WebPRescalerOutputDone(wrk));
+  assert(wrk->y_accum <= 0);
+  assert(!wrk->y_expand);
+  if (yscale) {
+    const int scale_xy = wrk->fxy_scale;
+    const __m128i mult_xy = _mm_set_epi64x(scale_xy, scale_xy);
+    const __m128i mult_y = _mm_set_epi64x(yscale, yscale);
+    const __m128i rounder = _mm_set_epi64x(ROUNDER, ROUNDER);
+    for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) {
+      __m128i A0, A1, A2, A3, B0, B1, B2, B3;
+      LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3);
+      LoadDispatchAndMult(frow + x_out, &mult_y, &B0, &B1, &B2, &B3);
+      {
+        const __m128i C0 = _mm_add_epi64(B0, rounder);
+        const __m128i C1 = _mm_add_epi64(B1, rounder);
+        const __m128i C2 = _mm_add_epi64(B2, rounder);
+        const __m128i C3 = _mm_add_epi64(B3, rounder);
+        const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX);   // = frac
+        const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX);
+        const __m128i D2 = _mm_srli_epi64(C2, WEBP_RESCALER_RFIX);
+        const __m128i D3 = _mm_srli_epi64(C3, WEBP_RESCALER_RFIX);
+        const __m128i E0 = _mm_sub_epi64(A0, D0);   // irow[x] - frac
+        const __m128i E1 = _mm_sub_epi64(A1, D1);
+        const __m128i E2 = _mm_sub_epi64(A2, D2);
+        const __m128i E3 = _mm_sub_epi64(A3, D3);
+        const __m128i F2 = _mm_slli_epi64(D2, 32);
+        const __m128i F3 = _mm_slli_epi64(D3, 32);
+        const __m128i G0 = _mm_or_si128(D0, F2);
+        const __m128i G1 = _mm_or_si128(D1, F3);
+        _mm_storeu_si128((__m128i*)(irow + x_out + 0), G0);
+        _mm_storeu_si128((__m128i*)(irow + x_out + 4), G1);
+        ProcessRow(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out);
+      }
+    }
+    for (; x_out < x_out_max; ++x_out) {
+      const uint32_t frac = (int)MULT_FIX(frow[x_out], yscale);
+      const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale);
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
+      irow[x_out] = frac;   // new fractional start
+    }
+  } else if (wrk->fxy_scale) {
+    const uint32_t scale = wrk->fxy_scale;
+    const __m128i mult = _mm_set_epi64x(scale, scale);
+    const __m128i zero = _mm_setzero_si128();
+    for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) {
+      __m128i A0, A1, A2, A3;
+      LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3);
+      _mm_storeu_si128((__m128i*)(irow + x_out + 0), zero);
+      _mm_storeu_si128((__m128i*)(irow + x_out + 4), zero);
+      ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out);
+    }
+    for (; x_out < x_out_max; ++x_out) {
+      const int v = (int)MULT_FIX(irow[x_out], scale);
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
+      irow[x_out] = 0;
+    }
+  } else {  // very special case for src = 1x1
+    for (x_out = 0; x_out < x_out_max; ++x_out) {
+      dst[x_out] = irow[x_out];
+      irow[x_out] = 0;
+    }
+  }
+}
+
+#undef MULT_FIX
+#undef ROUNDER
+
+//------------------------------------------------------------------------------
+
+extern void WebPRescalerDspInitSSE2(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitSSE2(void) {
+  WebPRescalerExportRowExpand = RescalerExportRowExpandSSE2;
+  WebPRescalerExportRowShrink = RescalerExportRowShrinkSSE2;
+}
+
+#else  // !WEBP_USE_SSE2
+
+WEBP_DSP_INIT_STUB(WebPRescalerDspInitSSE2)
+
+#endif  // WEBP_USE_SSE2

src/utils/rescaler.c

@@ -41,19 +41,20 @@ void WebPRescalerInit(WebPRescaler* const wrk, int src_width, int src_height,
   wrk->x_add = wrk->x_expand ? (x_sub - 1) : x_add;
   wrk->x_sub = wrk->x_expand ? (x_add - 1) : x_sub;
   if (!wrk->x_expand) {  // fx_scale is not used otherwise
-    wrk->fx_scale = WEBP_RESCALER_ONE / wrk->x_sub;
+    wrk->fx_scale = WEBP_RESCALER_FRAC(1, wrk->x_sub);
   }
   // vertical scaling parameters
   wrk->y_add = wrk->y_expand ? y_add - 1 : y_add;
   wrk->y_sub = wrk->y_expand ? y_sub - 1 : y_sub;
   wrk->y_accum = wrk->y_expand ? wrk->y_sub : wrk->y_add;
   if (!wrk->y_expand) {
-    wrk->fy_scale = WEBP_RESCALER_ONE / wrk->y_sub;
+    // note the very special case where x_add = y_add = 1 cannot be represented.
-    wrk->fxy_scale = ((uint64_t)dst_height << WEBP_RESCALER_RFIX)
+    // We special-case fxy_scale = 0 in this case, in ExportRowShrink
-                   / (wrk->x_add * wrk->y_add);
+    wrk->fxy_scale = WEBP_RESCALER_FRAC(dst_height, wrk->x_add * wrk->y_add);
+    wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->y_sub);
   } else {
-    wrk->fy_scale = WEBP_RESCALER_ONE / wrk->x_add;
+    wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->x_add);
-    wrk->fxy_scale = WEBP_RESCALER_ONE / (wrk->x_add * wrk->y_sub);
+    // wrk->fxy_scale is unused here.
   }
   wrk->irow = work;
   wrk->frow = work + num_channels * dst_width;

src/utils/rescaler.h

@@ -20,11 +20,12 @@ extern "C" {
 
 #include "../webp/types.h"
 
-#define WEBP_RESCALER_RFIX 30   // fixed-point precision for multiplies
+#define WEBP_RESCALER_RFIX 32   // fixed-point precision for multiplies
-#define WEBP_RESCALER_ONE (1u << WEBP_RESCALER_RFIX)
+#define WEBP_RESCALER_ONE (1ull << WEBP_RESCALER_RFIX)
+#define WEBP_RESCALER_FRAC(x, y) (((uint64_t)(x) << WEBP_RESCALER_RFIX) / (y))
 
 // Structure used for on-the-fly rescaling
-typedef int32_t rescaler_t;   // type for side-buffer
+typedef uint32_t rescaler_t;   // type for side-buffer
 typedef struct WebPRescaler WebPRescaler;
 struct WebPRescaler {
   int x_expand;               // true if we're expanding in the x direction
@@ -32,7 +33,7 @@ struct WebPRescaler {
   int num_channels;           // bytes to jump between pixels
   uint32_t fx_scale;          // fixed-point scaling factors
   uint32_t fy_scale;          // ''
-  uint64_t fxy_scale;         // ''
+  uint32_t fxy_scale;         // ''
   int y_accum;                // vertical accumulator
   int y_add, y_sub;           // vertical increments
   int x_add, x_sub;           // horizontal increments