backport rescaler fix

backported from: 7df9389, 5ff0079

Change-Id: I11b4d97c3c483431528be9ccbd9895baac8c6a63

src/dec/io.c

@@ -322,37 +322,31 @@ static int InitYUVRescaler(const VP8Io* const io, WebPDecParams* const p) {
   const size_t work_size = 2 * out_width;   // scratch memory for luma rescaler
   const size_t uv_work_size = 2 * uv_out_width;  // and for each u/v ones
   size_t tmp_size;
-  int32_t* work;
+  rescaler_t* work;
 
   tmp_size = (work_size + 2 * uv_work_size) * sizeof(*work);
   if (has_alpha) {
     tmp_size += work_size * sizeof(*work);
   }
-  p->memory = WebPSafeCalloc(1ULL, tmp_size);
+  p->memory = WebPSafeMalloc(1ULL, tmp_size);
   if (p->memory == NULL) {
     return 0;   // memory error
   }
-  work = (int32_t*)p->memory;
+  work = (rescaler_t*)p->memory;
   WebPRescalerInit(&p->scaler_y, io->mb_w, io->mb_h,
                    buf->y, out_width, out_height, buf->y_stride, 1,
-                   io->mb_w, out_width, io->mb_h, out_height,
                    work);
   WebPRescalerInit(&p->scaler_u, uv_in_width, uv_in_height,
                    buf->u, uv_out_width, uv_out_height, buf->u_stride, 1,
-                   uv_in_width, uv_out_width,
-                   uv_in_height, uv_out_height,
                    work + work_size);
   WebPRescalerInit(&p->scaler_v, uv_in_width, uv_in_height,
                    buf->v, uv_out_width, uv_out_height, buf->v_stride, 1,
-                   uv_in_width, uv_out_width,
-                   uv_in_height, uv_out_height,
                    work + work_size + uv_work_size);
   p->emit = EmitRescaledYUV;
 
   if (has_alpha) {
     WebPRescalerInit(&p->scaler_a, io->mb_w, io->mb_h,
                      buf->a, out_width, out_height, buf->a_stride, 1,
-                     io->mb_w, out_width, io->mb_h, out_height,
                      work + work_size + 2 * uv_work_size);
     p->emit_alpha = EmitRescaledAlphaYUV;
     WebPInitAlphaProcessing();
@@ -375,9 +369,9 @@ static int ExportRGB(WebPDecParams* const p, int y_pos) {
          WebPRescalerHasPendingOutput(&p->scaler_u)) {
     assert(p->last_y + y_pos + num_lines_out < p->output->height);
     assert(p->scaler_u.y_accum == p->scaler_v.y_accum);
-    WebPRescalerExportRow(&p->scaler_y, 0);
+    WebPRescalerExportRow(&p->scaler_y);
-    WebPRescalerExportRow(&p->scaler_u, 0);
+    WebPRescalerExportRow(&p->scaler_u);
-    WebPRescalerExportRow(&p->scaler_v, 0);
+    WebPRescalerExportRow(&p->scaler_v);
     convert(p->scaler_y.dst, p->scaler_u.dst, p->scaler_v.dst,
             dst, p->scaler_y.dst_width);
     dst += buf->stride;
@@ -425,7 +419,7 @@ static int ExportAlpha(WebPDecParams* const p, int y_pos) {
   while (WebPRescalerHasPendingOutput(&p->scaler_a)) {
     int i;
     assert(p->last_y + y_pos + num_lines_out < p->output->height);
-    WebPRescalerExportRow(&p->scaler_a, 0);
+    WebPRescalerExportRow(&p->scaler_a);
     for (i = 0; i < width; ++i) {
       const uint32_t alpha_value = p->scaler_a.dst[i];
       dst[4 * i] = alpha_value;
@@ -458,7 +452,7 @@ static int ExportAlphaRGBA4444(WebPDecParams* const p, int y_pos) {
   while (WebPRescalerHasPendingOutput(&p->scaler_a)) {
     int i;
     assert(p->last_y + y_pos + num_lines_out < p->output->height);
-    WebPRescalerExportRow(&p->scaler_a, 0);
+    WebPRescalerExportRow(&p->scaler_a);
     for (i = 0; i < width; ++i) {
       // Fill in the alpha value (converted to 4 bits).
       const uint32_t alpha_value = p->scaler_a.dst[i] >> 4;
@@ -495,7 +489,7 @@ static int InitRGBRescaler(const VP8Io* const io, WebPDecParams* const p) {
   const int uv_in_width  = (io->mb_w + 1) >> 1;
   const int uv_in_height = (io->mb_h + 1) >> 1;
   const size_t work_size = 2 * out_width;   // scratch memory for one rescaler
-  int32_t* work;  // rescalers work area
+  rescaler_t* work;  // rescalers work area
   uint8_t* tmp;   // tmp storage for scaled YUV444 samples before RGB conversion
   size_t tmp_size1, tmp_size2, total_size;
 
@@ -506,30 +500,26 @@ static int InitRGBRescaler(const VP8Io* const io, WebPDecParams* const p) {
     tmp_size2 += out_width;
   }
   total_size = tmp_size1 * sizeof(*work) + tmp_size2 * sizeof(*tmp);
-  p->memory = WebPSafeCalloc(1ULL, total_size);
+  p->memory = WebPSafeMalloc(1ULL, total_size);
   if (p->memory == NULL) {
     return 0;   // memory error
   }
-  work = (int32_t*)p->memory;
+  work = (rescaler_t*)p->memory;
   tmp = (uint8_t*)(work + tmp_size1);
   WebPRescalerInit(&p->scaler_y, io->mb_w, io->mb_h,
                    tmp + 0 * out_width, out_width, out_height, 0, 1,
-                   io->mb_w, out_width, io->mb_h, out_height,
                    work + 0 * work_size);
   WebPRescalerInit(&p->scaler_u, uv_in_width, uv_in_height,
                    tmp + 1 * out_width, out_width, out_height, 0, 1,
-                   io->mb_w, 2 * out_width, io->mb_h, 2 * out_height,
                    work + 1 * work_size);
   WebPRescalerInit(&p->scaler_v, uv_in_width, uv_in_height,
                    tmp + 2 * out_width, out_width, out_height, 0, 1,
-                   io->mb_w, 2 * out_width, io->mb_h, 2 * out_height,
                    work + 2 * work_size);
   p->emit = EmitRescaledRGB;
 
   if (has_alpha) {
     WebPRescalerInit(&p->scaler_a, io->mb_w, io->mb_h,
                      tmp + 3 * out_width, out_width, out_height, 0, 1,
-                     io->mb_w, out_width, io->mb_h, out_height,
                      work + 3 * work_size);
     p->emit_alpha = EmitRescaledAlphaRGB;
     if (p->output->colorspace == MODE_RGBA_4444 ||

src/dec/vp8l.c

@@ -390,13 +390,13 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) {
   const int in_height = io->mb_h;
   const int out_height = io->scaled_height;
   const uint64_t work_size = 2 * num_channels * (uint64_t)out_width;
-  int32_t* work;        // Rescaler work area.
+  rescaler_t* work;        // Rescaler work area.
-  const uint64_t scaled_data_size = num_channels * (uint64_t)out_width;
+  const uint64_t scaled_data_size = (uint64_t)out_width;
   uint32_t* scaled_data;  // Temporary storage for scaled BGRA data.
   const uint64_t memory_size = sizeof(*dec->rescaler) +
                                work_size * sizeof(*work) +
                                scaled_data_size * sizeof(*scaled_data);
-  uint8_t* memory = (uint8_t*)WebPSafeCalloc(memory_size, sizeof(*memory));
+  uint8_t* memory = (uint8_t*)WebPSafeMalloc(memory_size, sizeof(*memory));
   if (memory == NULL) {
     dec->status_ = VP8_STATUS_OUT_OF_MEMORY;
     return 0;
@@ -406,13 +406,12 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) {
 
   dec->rescaler = (WebPRescaler*)memory;
   memory += sizeof(*dec->rescaler);
-  work = (int32_t*)memory;
+  work = (rescaler_t*)memory;
   memory += work_size * sizeof(*work);
   scaled_data = (uint32_t*)memory;
 
   WebPRescalerInit(dec->rescaler, in_width, in_height, (uint8_t*)scaled_data,
-                   out_width, out_height, 0, num_channels,
+                   out_width, out_height, 0, num_channels, work);
-                   in_width, out_width, in_height, out_height, work);
   return 1;
 }
 
@@ -427,7 +426,7 @@ static int Export(WebPRescaler* const rescaler, WEBP_CSP_MODE colorspace,
   int num_lines_out = 0;
   while (WebPRescalerHasPendingOutput(rescaler)) {
     uint8_t* const dst = rgba + num_lines_out * rgba_stride;
-    WebPRescalerExportRow(rescaler, 0);
+    WebPRescalerExportRow(rescaler);
     WebPMultARGBRow(src, dst_width, 1);
     VP8LConvertFromBGRA(src, dst_width, colorspace, dst);
     ++num_lines_out;
@@ -545,7 +544,7 @@ static int ExportYUVA(const VP8LDecoder* const dec, int y_pos) {
   const int dst_width = rescaler->dst_width;
   int num_lines_out = 0;
   while (WebPRescalerHasPendingOutput(rescaler)) {
-    WebPRescalerExportRow(rescaler, 0);
+    WebPRescalerExportRow(rescaler);
     WebPMultARGBRow(src, dst_width, 1);
     ConvertToYUVA(src, dst_width, y_pos, dec->output_);
     ++y_pos;

src/enc/picture_rescale.c

@@ -175,17 +175,13 @@ static void RescalePlane(const uint8_t* src,
                          int src_width, int src_height, int src_stride,
                          uint8_t* dst,
                          int dst_width, int dst_height, int dst_stride,
-                         int32_t* const work,
+                         rescaler_t* const work,
                          int num_channels) {
   WebPRescaler rescaler;
   int y = 0;
   WebPRescalerInit(&rescaler, src_width, src_height,
                    dst, dst_width, dst_height, dst_stride,
-                   num_channels,
+                   num_channels, work);
-                   src_width, dst_width,
-                   src_height, dst_height,
-                   work);
-  memset(work, 0, 2 * dst_width * num_channels * sizeof(*work));
   while (y < src_height) {
     y += WebPRescalerImport(&rescaler, src_height - y,
                             src + y * src_stride, src_stride);
@@ -209,7 +205,7 @@ static void AlphaMultiplyY(WebPPicture* const pic, int inverse) {
 int WebPPictureRescale(WebPPicture* pic, int width, int height) {
   WebPPicture tmp;
   int prev_width, prev_height;
-  int32_t* work;
+  rescaler_t* work;
 
   if (pic == NULL) return 0;
   prev_width = pic->width;
@@ -231,7 +227,7 @@ int WebPPictureRescale(WebPPicture* pic, int width, int height) {
   if (!WebPPictureAlloc(&tmp)) return 0;
 
   if (!pic->use_argb) {
-    work = (int32_t*)WebPSafeMalloc(2ULL * width, sizeof(*work));
+    work = (rescaler_t*)WebPSafeMalloc(2ULL * width, sizeof(*work));
     if (work == NULL) {
       WebPPictureFree(&tmp);
       return 0;
@@ -259,7 +255,7 @@ int WebPPictureRescale(WebPPicture* pic, int width, int height) {
                  tmp.v,
                  HALVE(width), HALVE(height), tmp.uv_stride, work, 1);
   } else {
-    work = (int32_t*)WebPSafeMalloc(2ULL * width * 4, sizeof(*work));
+    work = (rescaler_t*)WebPSafeMalloc(2ULL * width * 4, sizeof(*work));
     if (work == NULL) {
       WebPPictureFree(&tmp);
       return 0;

src/utils/rescaler.c

@@ -13,77 +13,192 @@
 
 #include <assert.h>
 #include <stdlib.h>
+#include <string.h>
 #include "./rescaler.h"
 #include "../dsp/dsp.h"
 
 //------------------------------------------------------------------------------
 // Implementations of critical functions ImportRow / ExportRow
 
-void (*WebPRescalerImportRow)(WebPRescaler* const wrk,
+// Import a row of data and save its contribution in the rescaler.
-                              const uint8_t* const src, int channel) = NULL;
+// 'channel' denotes the channel number to be imported. 'Expand' corresponds to
-void (*WebPRescalerExportRow)(WebPRescaler* const wrk, int x_out) = NULL;
+// the wrk->x_expand case. Otherwise, 'Shrink' is to be used.
+typedef void (*WebPRescalerImportRowFunc)(WebPRescaler* const wrk,
+                                          const uint8_t* src);
+static WebPRescalerImportRowFunc WebPRescalerImportRowExpand;
+static WebPRescalerImportRowFunc WebPRescalerImportRowShrink;
 
-#define RFIX 30
+// Export one row (starting at x_out position) from rescaler.
-#define MULT_FIX(x, y) (((int64_t)(x) * (y) + (1 << (RFIX - 1))) >> RFIX)
+// 'Expand' corresponds to the wrk->y_expand case.
+// Otherwise 'Shrink' is to be used
+typedef void (*WebPRescalerExportRowFunc)(WebPRescaler* const wrk);
+static WebPRescalerExportRowFunc WebPRescalerExportRowExpand;
+static WebPRescalerExportRowFunc WebPRescalerExportRowShrink;
 
-static void ImportRowC(WebPRescaler* const wrk,
+#define WEBP_RESCALER_RFIX 32   // fixed-point precision for multiplies
-                       const uint8_t* const src, int channel) {
+#define WEBP_RESCALER_ONE (1ull << WEBP_RESCALER_RFIX)
+#define WEBP_RESCALER_FRAC(x, y) \
+    ((uint32_t)(((uint64_t)(x) << WEBP_RESCALER_RFIX) / (y)))
+#define ROUNDER (WEBP_RESCALER_ONE >> 1)
+#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX)
+
+static void ImportRowExpandC(WebPRescaler* const wrk, const uint8_t* src) {
   const int x_stride = wrk->num_channels;
   const int x_out_max = wrk->dst_width * wrk->num_channels;
-  int x_in = channel;
+  int channel;
-  int x_out;
+  assert(!WebPRescalerInputDone(wrk));
-  int accum = 0;
+  assert(wrk->x_expand);
-  if (!wrk->x_expand) {
+  for (channel = 0; channel < x_stride; ++channel) {
-    int sum = 0;
+    int x_in = channel;
-    for (x_out = channel; x_out < x_out_max; x_out += x_stride) {
+    int x_out = channel;
-      accum += wrk->x_add;
+    // simple bilinear interpolation
-      for (; accum > 0; accum -= wrk->x_sub) {
+    int accum = wrk->x_add;
-        sum += src[x_in];
+    int left = src[x_in];
-        x_in += x_stride;
+    int right = (wrk->src_width > 1) ? src[x_in + x_stride] : left;
-      }
+    x_in += x_stride;
-      {        // Emit next horizontal pixel.
+    while (1) {
-        const int32_t base = src[x_in];
+      wrk->frow[x_out] = right * wrk->x_add + (left - right) * accum;
-        const int32_t frac = base * (-accum);
+      x_out += x_stride;
-        x_in += x_stride;
+      if (x_out >= x_out_max) break;
-        wrk->frow[x_out] = (sum + base) * wrk->x_sub - frac;
+      accum -= wrk->x_sub;
-        // fresh fractional start for next pixel
-        sum = (int)MULT_FIX(frac, wrk->fx_scale);
-      }
-    }
-  } else {        // simple bilinear interpolation
-    int left = src[channel], right = src[channel];
-    for (x_out = channel; x_out < x_out_max; x_out += x_stride) {
       if (accum < 0) {
         left = right;
         x_in += x_stride;
+        assert(x_in < wrk->src_width * x_stride);
         right = src[x_in];
         accum += wrk->x_add;
       }
-      wrk->frow[x_out] = right * wrk->x_add + (left - right) * accum;
-      accum -= wrk->x_sub;
     }
-  }
+    assert(wrk->x_sub == 0 /* <- special case for src_width=1 */ || accum == 0);
-  // Accumulate the contribution of the new row.
-  for (x_out = channel; x_out < x_out_max; x_out += x_stride) {
-    wrk->irow[x_out] += wrk->frow[x_out];
   }
 }
 
-static void ExportRowC(WebPRescaler* const wrk, int x_out) {
+static void ImportRowShrinkC(WebPRescaler* const wrk, const uint8_t* src) {
-  if (wrk->y_accum <= 0) {
+  const int x_stride = wrk->num_channels;
-    uint8_t* const dst = wrk->dst;
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
-    int32_t* const irow = wrk->irow;
+  int channel;
-    const int32_t* const frow = wrk->frow;
+  assert(!WebPRescalerInputDone(wrk));
-    const int yscale = wrk->fy_scale * (-wrk->y_accum);
+  assert(!wrk->x_expand);
-    const int x_out_max = wrk->dst_width * wrk->num_channels;
+  for (channel = 0; channel < x_stride; ++channel) {
-    for (; x_out < x_out_max; ++x_out) {
+    int x_in = channel;
-      const int frac = (int)MULT_FIX(frow[x_out], yscale);
+    int x_out = channel;
+    uint32_t sum = 0;
+    int accum = 0;
+    while (x_out < x_out_max) {
+      uint32_t base = 0;
+      accum += wrk->x_add;
+      while (accum > 0) {
+        accum -= wrk->x_sub;
+        assert(x_in < wrk->src_width * x_stride);
+        base = src[x_in];
+        sum += base;
+        x_in += x_stride;
+      }
+      {        // Emit next horizontal pixel.
+        const rescaler_t frac = base * (-accum);
+        wrk->frow[x_out] = sum * wrk->x_sub - frac;
+        // fresh fractional start for next pixel
+        sum = (int)MULT_FIX(frac, wrk->fx_scale);
+      }
+      x_out += x_stride;
+    }
+    assert(accum == 0);
+  }
+}
+
+//------------------------------------------------------------------------------
+// Row export
+
+static void ExportRowExpandC(WebPRescaler* const wrk) {
+  int x_out;
+  uint8_t* const dst = wrk->dst;
+  rescaler_t* const irow = wrk->irow;
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
+  const rescaler_t* const frow = wrk->frow;
+  assert(!WebPRescalerOutputDone(wrk));
+  assert(wrk->y_accum <= 0);
+  assert(wrk->y_expand);
+  assert(wrk->y_sub != 0);
+  if (wrk->y_accum == 0) {
+    for (x_out = 0; x_out < x_out_max; ++x_out) {
+      const uint32_t J = frow[x_out];
+      const int v = (int)MULT_FIX(J, wrk->fy_scale);
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
+    }
+  } else {
+    const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
+    const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B);
+    for (x_out = 0; x_out < x_out_max; ++x_out) {
+      const uint64_t I = (uint64_t)A * frow[x_out]
+                       + (uint64_t)B * irow[x_out];
+      const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
+      const int v = (int)MULT_FIX(J, wrk->fy_scale);
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
+    }
+  }
+}
+
+static void ExportRowShrinkC(WebPRescaler* const wrk) {
+  int x_out;
+  uint8_t* const dst = wrk->dst;
+  rescaler_t* const irow = wrk->irow;
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
+  const rescaler_t* const frow = wrk->frow;
+  const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum);
+  assert(!WebPRescalerOutputDone(wrk));
+  assert(wrk->y_accum <= 0);
+  assert(!wrk->y_expand);
+  if (yscale) {
+    for (x_out = 0; x_out < x_out_max; ++x_out) {
+      const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale);
       const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale);
-      dst[x_out] = (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255;
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
       irow[x_out] = frac;   // new fractional start
     }
+  } else {
+    for (x_out = 0; x_out < x_out_max; ++x_out) {
+      const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale);
+      assert(v >= 0 && v <= 255);
+      dst[x_out] = v;
+      irow[x_out] = 0;
+    }
+  }
+}
+
+//------------------------------------------------------------------------------
+// Main entry calls
+
+void WebPRescalerImportRow(WebPRescaler* const wrk, const uint8_t* src) {
+  assert(!WebPRescalerInputDone(wrk));
+  if (!wrk->x_expand) {
+    WebPRescalerImportRowShrink(wrk, src);
+  } else {
+    WebPRescalerImportRowExpand(wrk, src);
+  }
+}
+
+void WebPRescalerExportRow(WebPRescaler* const wrk) {
+  if (wrk->y_accum <= 0) {
+    assert(!WebPRescalerOutputDone(wrk));
+    if (wrk->y_expand) {
+      WebPRescalerExportRowExpand(wrk);
+    } else if (wrk->fxy_scale) {
+      WebPRescalerExportRowShrink(wrk);
+    } else {  // very special case for src = dst = 1x1
+      int i;
+      assert(wrk->src_width == 1 && wrk->dst_width <= 2);
+      assert(wrk->src_height == 1 && wrk->dst_height == 1);
+      for (i = 0; i < wrk->num_channels * wrk->dst_width; ++i) {
+        wrk->dst[i] = wrk->irow[i];
+        wrk->irow[i] = 0;
+      }
+    }
     wrk->y_accum += wrk->y_add;
     wrk->dst += wrk->dst_stride;
+    ++wrk->dst_y;
   }
 }
 
@@ -92,23 +207,25 @@ static void ExportRowC(WebPRescaler* const wrk, int x_out) {
 
 #if defined(WEBP_USE_MIPS32)
 
-static void ImportRowMIPS(WebPRescaler* const wrk,
+static void ImportRowShrinkMIPS(WebPRescaler* const wrk, const uint8_t* src) {
-                          const uint8_t* const src, int channel) {
   const int x_stride = wrk->num_channels;
   const int x_out_max = wrk->dst_width * wrk->num_channels;
   const int fx_scale = wrk->fx_scale;
   const int x_add = wrk->x_add;
   const int x_sub = wrk->x_sub;
-  int* frow = wrk->frow + channel;
-  int* irow = wrk->irow + channel;
-  const uint8_t* src1 = src + channel;
-  int temp1, temp2, temp3;
-  int base, frac, sum;
-  int accum, accum1;
   const int x_stride1 = x_stride << 2;
-  int loop_c = x_out_max - channel;
+  int channel;
+  assert(!wrk->x_expand);
+  assert(!WebPRescalerInputDone(wrk));
+
+  for (channel = 0; channel < x_stride; ++channel) {
+    const uint8_t* src1 = src + channel;
+    rescaler_t* frow = wrk->frow + channel;
+    int temp1, temp2, temp3;
+    int base, frac, sum;
+    int accum, accum1;
+    int loop_c = x_out_max - channel;
 
-  if (!wrk->x_expand) {
     __asm__ volatile (
       "li     %[temp1],   0x8000                    \n\t"
       "li     %[temp2],   0x10000                   \n\t"
@@ -116,179 +233,295 @@ static void ImportRowMIPS(WebPRescaler* const wrk,
       "li     %[accum],   0                         \n\t"
     "1:                                             \n\t"
       "addu   %[accum],   %[accum],   %[x_add]      \n\t"
+      "li     %[base],    0                         \n\t"
       "blez   %[accum],   3f                        \n\t"
     "2:                                             \n\t"
-      "lbu    %[temp3],   0(%[src1])                \n\t"
+      "lbu    %[base],    0(%[src1])                \n\t"
       "subu   %[accum],   %[accum],   %[x_sub]      \n\t"
       "addu   %[src1],    %[src1],    %[x_stride]   \n\t"
-      "addu   %[sum],     %[sum],     %[temp3]      \n\t"
+      "addu   %[sum],     %[sum],     %[base]       \n\t"
       "bgtz   %[accum],   2b                        \n\t"
     "3:                                             \n\t"
-      "lbu    %[base],    0(%[src1])                \n\t"
-      "addu   %[src1],    %[src1],    %[x_stride]   \n\t"
       "negu   %[accum1],  %[accum]                  \n\t"
       "mul    %[frac],    %[base],    %[accum1]     \n\t"
-      "addu   %[temp3],   %[sum],     %[base]       \n\t"
+      "mul    %[temp3],   %[sum],     %[x_sub]      \n\t"
-      "mul    %[temp3],   %[temp3],   %[x_sub]      \n\t"
-      "lw     %[base],    0(%[irow])                \n\t"
       "subu   %[loop_c],  %[loop_c],  %[x_stride]   \n\t"
-      "sll    %[accum1],  %[frac],    2             \n\t"
       "mult   %[temp1],   %[temp2]                  \n\t"
-      "madd   %[accum1],  %[fx_scale]               \n\t"
+      "maddu  %[frac],    %[fx_scale]               \n\t"
       "mfhi   %[sum]                                \n\t"
       "subu   %[temp3],   %[temp3],   %[frac]       \n\t"
       "sw     %[temp3],   0(%[frow])                \n\t"
-      "add    %[base],    %[base],    %[temp3]      \n\t"
-      "sw     %[base],    0(%[irow])                \n\t"
-      "addu   %[irow],    %[irow],    %[x_stride1]  \n\t"
       "addu   %[frow],    %[frow],    %[x_stride1]  \n\t"
       "bgtz   %[loop_c],  1b                        \n\t"
+      : [accum]"=&r"(accum), [src1]"+r"(src1), [temp3]"=&r"(temp3),
+        [sum]"=&r"(sum), [base]"=&r"(base), [frac]"=&r"(frac),
+        [frow]"+r"(frow), [accum1]"=&r"(accum1),
+        [temp2]"=&r"(temp2), [temp1]"=&r"(temp1)
+      : [x_stride]"r"(x_stride), [fx_scale]"r"(fx_scale),
+        [x_sub]"r"(x_sub), [x_add]"r"(x_add),
+        [loop_c]"r"(loop_c), [x_stride1]"r"(x_stride1)
+      : "memory", "hi", "lo"
+    );
+    assert(accum == 0);
+  }
+}
 
-      : [accum] "=&r" (accum), [src1] "+r" (src1), [temp3] "=&r" (temp3),
+static void ImportRowExpandMIPS(WebPRescaler* const wrk, const uint8_t* src) {
-        [sum] "=&r" (sum), [base] "=&r" (base), [frac] "=&r" (frac),
+  const int x_stride = wrk->num_channels;
-        [frow] "+r" (frow), [irow] "+r" (irow), [accum1] "=&r" (accum1),
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
-        [temp2] "=&r" (temp2), [temp1] "=&r" (temp1)
+  const int x_add = wrk->x_add;
-      : [x_stride] "r" (x_stride), [fx_scale] "r" (fx_scale),
+  const int x_sub = wrk->x_sub;
-        [x_sub] "r" (x_sub), [x_add] "r" (x_add),
+  const int src_width = wrk->src_width;
-        [loop_c] "r" (loop_c), [x_stride1] "r" (x_stride1)
+  const int x_stride1 = x_stride << 2;
+  int channel;
+  assert(wrk->x_expand);
+  assert(!WebPRescalerInputDone(wrk));
+
+  for (channel = 0; channel < x_stride; ++channel) {
+    const uint8_t* src1 = src + channel;
+    rescaler_t* frow = wrk->frow + channel;
+    int temp1, temp2, temp3, temp4;
+    int frac;
+    int accum;
+    int x_out = channel;
+
+    __asm__ volatile (
+      "addiu  %[temp3],   %[src_width], -1            \n\t"
+      "lbu    %[temp2],   0(%[src1])                  \n\t"
+      "addu   %[src1],    %[src1],      %[x_stride]   \n\t"
+      "bgtz   %[temp3],   0f                          \n\t"
+      "addiu  %[temp1],   %[temp2],     0             \n\t"
+      "b      3f                                      \n\t"
+    "0:                                               \n\t"
+      "lbu    %[temp1],   0(%[src1])                  \n\t"
+    "3:                                               \n\t"
+      "addiu  %[accum],   %[x_add],     0             \n\t"
+    "1:                                               \n\t"
+      "subu   %[temp3],   %[temp2],     %[temp1]      \n\t"
+      "mul    %[temp3],   %[temp3],     %[accum]      \n\t"
+      "mul    %[temp4],   %[temp1],     %[x_add]      \n\t"
+      "addu   %[temp3],   %[temp4],     %[temp3]      \n\t"
+      "sw     %[temp3],   0(%[frow])                  \n\t"
+      "addu   %[frow],    %[frow],      %[x_stride1]  \n\t"
+      "addu   %[x_out],   %[x_out],     %[x_stride]   \n\t"
+      "subu   %[temp3],   %[x_out],     %[x_out_max]  \n\t"
+      "bgez   %[temp3],   2f                          \n\t"
+      "subu   %[accum],   %[accum],     %[x_sub]      \n\t"
+      "bgez   %[accum],   4f                          \n\t"
+      "addiu  %[temp2],   %[temp1],     0             \n\t"
+      "addu   %[src1],    %[src1],      %[x_stride]   \n\t"
+      "lbu    %[temp1],   0(%[src1])                  \n\t"
+      "addu   %[accum],   %[accum],     %[x_add]      \n\t"
+    "4:                                               \n\t"
+      "b      1b                                      \n\t"
+    "2:                                               \n\t"
+      : [src1]"+r"(src1), [accum]"=&r"(accum), [temp1]"=&r"(temp1),
+        [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4),
+        [x_out]"+r"(x_out), [frac]"=&r"(frac), [frow]"+r"(frow)
+      : [x_stride]"r"(x_stride), [x_add]"r"(x_add), [x_sub]"r"(x_sub),
+        [x_stride1]"r"(x_stride1), [src_width]"r"(src_width),
+        [x_out_max]"r"(x_out_max)
+      : "memory", "hi", "lo"
+    );
+    assert(wrk->x_sub == 0 /* <- special case for src_width=1 */ || accum == 0);
+  }
+}
+
+//------------------------------------------------------------------------------
+// Row export
+
+static void ExportRowExpandMIPS(WebPRescaler* const wrk) {
+  uint8_t* dst = wrk->dst;
+  rescaler_t* irow = wrk->irow;
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
+  const rescaler_t* frow = wrk->frow;
+  int temp0, temp1, temp3, temp4, temp5, loop_end;
+  const int temp2 = (int)wrk->fy_scale;
+  const int temp6 = x_out_max << 2;
+  assert(!WebPRescalerOutputDone(wrk));
+  assert(wrk->y_accum <= 0);
+  assert(wrk->y_expand);
+  assert(wrk->y_sub != 0);
+  if (wrk->y_accum == 0) {
+    __asm__ volatile (
+      "li       %[temp3],    0x10000                    \n\t"
+      "li       %[temp4],    0x8000                     \n\t"
+      "addu     %[loop_end], %[frow],     %[temp6]      \n\t"
+    "1:                                                 \n\t"
+      "lw       %[temp0],    0(%[frow])                 \n\t"
+      "addiu    %[dst],      %[dst],      1             \n\t"
+      "addiu    %[frow],     %[frow],     4             \n\t"
+      "mult     %[temp3],    %[temp4]                   \n\t"
+      "maddu    %[temp0],    %[temp2]                   \n\t"
+      "mfhi     %[temp5]                                \n\t"
+      "sb       %[temp5],    -1(%[dst])                 \n\t"
+      "bne      %[frow],     %[loop_end], 1b            \n\t"
+      : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3),
+        [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [frow]"+r"(frow),
+        [dst]"+r"(dst), [loop_end]"=&r"(loop_end)
+      : [temp2]"r"(temp2), [temp6]"r"(temp6)
+      : "memory", "hi", "lo"
+    );
+  } else {
+    const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
+    const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B);
+    __asm__ volatile (
+      "li       %[temp3],    0x10000                    \n\t"
+      "li       %[temp4],    0x8000                     \n\t"
+      "addu     %[loop_end], %[frow],     %[temp6]      \n\t"
+    "1:                                                 \n\t"
+      "lw       %[temp0],    0(%[frow])                 \n\t"
+      "lw       %[temp1],    0(%[irow])                 \n\t"
+      "addiu    %[dst],      %[dst],      1             \n\t"
+      "mult     %[temp3],    %[temp4]                   \n\t"
+      "maddu    %[A],        %[temp0]                   \n\t"
+      "maddu    %[B],        %[temp1]                   \n\t"
+      "addiu    %[frow],     %[frow],     4             \n\t"
+      "addiu    %[irow],     %[irow],     4             \n\t"
+      "mfhi     %[temp5]                                \n\t"
+      "mult     %[temp3],    %[temp4]                   \n\t"
+      "maddu    %[temp5],    %[temp2]                   \n\t"
+      "mfhi     %[temp5]                                \n\t"
+      "sb       %[temp5],    -1(%[dst])                 \n\t"
+      "bne      %[frow],     %[loop_end], 1b            \n\t"
+      : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3),
+        [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [frow]"+r"(frow),
+        [irow]"+r"(irow), [dst]"+r"(dst), [loop_end]"=&r"(loop_end)
+      : [temp2]"r"(temp2), [temp6]"r"(temp6), [A]"r"(A), [B]"r"(B)
+      : "memory", "hi", "lo"
+    );
+  }
+}
+
+static void ExportRowShrinkMIPS(WebPRescaler* const wrk) {
+  const int x_out_max = wrk->dst_width * wrk->num_channels;
+  uint8_t* dst = wrk->dst;
+  rescaler_t* irow = wrk->irow;
+  const rescaler_t* frow = wrk->frow;
+  const int yscale = wrk->fy_scale * (-wrk->y_accum);
+  int temp0, temp1, temp3, temp4, temp5, loop_end;
+  const int temp2 = (int)wrk->fxy_scale;
+  const int temp6 = x_out_max << 2;
+
+  assert(!WebPRescalerOutputDone(wrk));
+  assert(wrk->y_accum <= 0);
+  assert(!wrk->y_expand);
+  assert(wrk->fxy_scale != 0);
+  if (yscale) {
+    __asm__ volatile (
+      "li       %[temp3],    0x10000                    \n\t"
+      "li       %[temp4],    0x8000                     \n\t"
+      "addu     %[loop_end], %[frow],     %[temp6]      \n\t"
+    "1:                                                 \n\t"
+      "lw       %[temp0],    0(%[frow])                 \n\t"
+      "mult     %[temp3],    %[temp4]                   \n\t"
+      "addiu    %[frow],     %[frow],     4             \n\t"
+      "maddu    %[temp0],    %[yscale]                  \n\t"
+      "mfhi     %[temp1]                                \n\t"
+      "lw       %[temp0],    0(%[irow])                 \n\t"
+      "addiu    %[dst],      %[dst],      1             \n\t"
+      "addiu    %[irow],     %[irow],     4             \n\t"
+      "subu     %[temp0],    %[temp0],    %[temp1]      \n\t"
+      "mult     %[temp3],    %[temp4]                   \n\t"
+      "maddu    %[temp0],    %[temp2]                   \n\t"
+      "mfhi     %[temp5]                                \n\t"
+      "sw       %[temp1],    -4(%[irow])                \n\t"
+      "sb       %[temp5],    -1(%[dst])                 \n\t"
+      "bne      %[frow],     %[loop_end], 1b            \n\t"
+      : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3),
+        [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [frow]"+r"(frow),
+        [irow]"+r"(irow), [dst]"+r"(dst), [loop_end]"=&r"(loop_end)
+      : [temp2]"r"(temp2), [yscale]"r"(yscale), [temp6]"r"(temp6)
       : "memory", "hi", "lo"
     );
   } else {
     __asm__ volatile (
-      "lbu    %[temp1],   0(%[src1])                \n\t"
+      "li       %[temp3],    0x10000                    \n\t"
-      "move   %[temp2],   %[temp1]                  \n\t"
+      "li       %[temp4],    0x8000                     \n\t"
-      "li     %[accum],   0                         \n\t"
+      "addu     %[loop_end], %[irow],     %[temp6]      \n\t"
-    "1:                                             \n\t"
+    "1:                                                 \n\t"
-      "bgez   %[accum],   2f                        \n\t"
+      "lw       %[temp0],    0(%[irow])                 \n\t"
-      "move   %[temp2],   %[temp1]                  \n\t"
+      "addiu    %[dst],      %[dst],      1             \n\t"
-      "addu   %[src1],    %[x_stride]               \n\t"
+      "addiu    %[irow],     %[irow],     4             \n\t"
-      "lbu    %[temp1],   0(%[src1])                \n\t"
+      "mult     %[temp3],    %[temp4]                   \n\t"
-      "addu   %[accum],   %[x_add]                  \n\t"
+      "maddu    %[temp0],    %[temp2]                   \n\t"
-    "2:                                             \n\t"
+      "mfhi     %[temp5]                                \n\t"
-      "subu   %[temp3],   %[temp2],   %[temp1]      \n\t"
+      "sw       $zero,       -4(%[irow])                \n\t"
-      "mul    %[temp3],   %[temp3],   %[accum]      \n\t"
+      "sb       %[temp5],    -1(%[dst])                 \n\t"
-      "mul    %[base],    %[temp1],   %[x_add]      \n\t"
+      "bne      %[irow],     %[loop_end], 1b            \n\t"
-      "subu   %[accum],   %[accum],   %[x_sub]      \n\t"
+      : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3),
-      "lw     %[frac],    0(%[irow])                \n\t"
+        [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [irow]"+r"(irow),
-      "subu   %[loop_c],  %[loop_c],  %[x_stride]   \n\t"
+        [dst]"+r"(dst), [loop_end]"=&r"(loop_end)
-      "addu   %[temp3],   %[base],    %[temp3]      \n\t"
+      : [temp2]"r"(temp2), [temp6]"r"(temp6)
-      "sw     %[temp3],   0(%[frow])                \n\t"
-      "addu   %[frow],    %[x_stride1]              \n\t"
-      "addu   %[frac],    %[temp3]                  \n\t"
-      "sw     %[frac],    0(%[irow])                \n\t"
-      "addu   %[irow],    %[x_stride1]              \n\t"
-      "bgtz   %[loop_c],  1b                        \n\t"
-
-      : [src1] "+r" (src1), [accum] "=&r" (accum), [temp1] "=&r" (temp1),
-        [temp2] "=&r" (temp2), [temp3] "=&r" (temp3), [base] "=&r" (base),
-        [frac] "=&r" (frac), [frow] "+r" (frow), [irow] "+r" (irow)
-      : [x_stride] "r" (x_stride), [x_add] "r" (x_add), [x_sub] "r" (x_sub),
-        [x_stride1] "r" (x_stride1), [loop_c] "r" (loop_c)
       : "memory", "hi", "lo"
     );
   }
 }
 
-static void ExportRowMIPS(WebPRescaler* const wrk, int x_out) {
-  if (wrk->y_accum <= 0) {
-    uint8_t* const dst = wrk->dst;
-    int32_t* const irow = wrk->irow;
-    const int32_t* const frow = wrk->frow;
-    const int yscale = wrk->fy_scale * (-wrk->y_accum);
-    const int x_out_max = wrk->dst_width * wrk->num_channels;
-    // if wrk->fxy_scale can fit into 32 bits use optimized code,
-    // otherwise use C code
-    if ((wrk->fxy_scale >> 32) == 0) {
-      int temp0, temp1, temp3, temp4, temp5, temp6, temp7, loop_end;
-      const int temp2 = (int)(wrk->fxy_scale);
-      const int temp8 = x_out_max << 2;
-      uint8_t* dst_t = (uint8_t*)dst;
-      int32_t* irow_t = (int32_t*)irow;
-      const int32_t* frow_t = (const int32_t*)frow;
-
-      __asm__ volatile(
-        "addiu    %[temp6],    $zero,       -256          \n\t"
-        "addiu    %[temp7],    $zero,       255           \n\t"
-        "li       %[temp3],    0x10000                    \n\t"
-        "li       %[temp4],    0x8000                     \n\t"
-        "addu     %[loop_end], %[frow_t],   %[temp8]      \n\t"
-      "1:                                                 \n\t"
-        "lw       %[temp0],    0(%[frow_t])               \n\t"
-        "mult     %[temp3],    %[temp4]                   \n\t"
-        "addiu    %[frow_t],   %[frow_t],   4             \n\t"
-        "sll      %[temp0],    %[temp0],    2             \n\t"
-        "madd     %[temp0],    %[yscale]                  \n\t"
-        "mfhi     %[temp1]                                \n\t"
-        "lw       %[temp0],    0(%[irow_t])               \n\t"
-        "addiu    %[dst_t],    %[dst_t],    1             \n\t"
-        "addiu    %[irow_t],   %[irow_t],   4             \n\t"
-        "subu     %[temp0],    %[temp0],    %[temp1]      \n\t"
-        "mult     %[temp3],    %[temp4]                   \n\t"
-        "sll      %[temp0],    %[temp0],    2             \n\t"
-        "madd     %[temp0],    %[temp2]                   \n\t"
-        "mfhi     %[temp5]                                \n\t"
-        "sw       %[temp1],    -4(%[irow_t])              \n\t"
-        "and      %[temp0],    %[temp5],    %[temp6]      \n\t"
-        "slti     %[temp1],    %[temp5],    0             \n\t"
-        "beqz     %[temp0],    2f                         \n\t"
-        "xor      %[temp5],    %[temp5],    %[temp5]      \n\t"
-        "movz     %[temp5],    %[temp7],    %[temp1]      \n\t"
-      "2:                                                 \n\t"
-        "sb       %[temp5],    -1(%[dst_t])               \n\t"
-        "bne      %[frow_t],   %[loop_end], 1b            \n\t"
-
-        : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp3]"=&r"(temp3),
-          [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp6]"=&r"(temp6),
-          [temp7]"=&r"(temp7), [frow_t]"+r"(frow_t), [irow_t]"+r"(irow_t),
-          [dst_t]"+r"(dst_t), [loop_end]"=&r"(loop_end)
-        : [temp2]"r"(temp2), [yscale]"r"(yscale), [temp8]"r"(temp8)
-        : "memory", "hi", "lo"
-      );
-      wrk->y_accum += wrk->y_add;
-      wrk->dst += wrk->dst_stride;
-    } else {
-      ExportRowC(wrk, x_out);
-    }
-  }
-}
 #endif   // WEBP_USE_MIPS32
 
 //------------------------------------------------------------------------------
 
 void WebPRescalerInit(WebPRescaler* const wrk, int src_width, int src_height,
-                      uint8_t* const dst, int dst_width, int dst_height,
+                      uint8_t* const dst,
-                      int dst_stride, int num_channels, int x_add, int x_sub,
+                      int dst_width, int dst_height, int dst_stride,
-                      int y_add, int y_sub, int32_t* const work) {
+                      int num_channels, rescaler_t* const work) {
+  const int x_add = src_width, x_sub = dst_width;
+  const int y_add = src_height, y_sub = dst_height;
   wrk->x_expand = (src_width < dst_width);
+  wrk->y_expand = (src_height < dst_height);
   wrk->src_width = src_width;
   wrk->src_height = src_height;
   wrk->dst_width = dst_width;
   wrk->dst_height = dst_height;
+  wrk->src_y = 0;
+  wrk->dst_y = 0;
   wrk->dst = dst;
   wrk->dst_stride = dst_stride;
   wrk->num_channels = num_channels;
+
   // for 'x_expand', we use bilinear interpolation
-  wrk->x_add = wrk->x_expand ? (x_sub - 1) : x_add - x_sub;
+  wrk->x_add = wrk->x_expand ? (x_sub - 1) : x_add;
   wrk->x_sub = wrk->x_expand ? (x_add - 1) : x_sub;
-  wrk->y_accum = y_add;
+  if (!wrk->x_expand) {  // fx_scale is not used otherwise
-  wrk->y_add = y_add;
+    wrk->fx_scale = WEBP_RESCALER_FRAC(1, wrk->x_sub);
-  wrk->y_sub = y_sub;
+  }
-  wrk->fx_scale = (1 << RFIX) / x_sub;
+  // vertical scaling parameters
-  wrk->fy_scale = (1 << RFIX) / y_sub;
+  wrk->y_add = wrk->y_expand ? y_add - 1 : y_add;
-  wrk->fxy_scale = wrk->x_expand ?
+  wrk->y_sub = wrk->y_expand ? y_sub - 1 : y_sub;
-      ((int64_t)dst_height << RFIX) / (x_sub * src_height) :
+  wrk->y_accum = wrk->y_expand ? wrk->y_sub : wrk->y_add;
-      ((int64_t)dst_height << RFIX) / (x_add * src_height);
+  if (!wrk->y_expand) {
+    // this is WEBP_RESCALER_FRAC(dst_height, x_add * y_add) without the cast.
+    const uint64_t ratio =
+        (uint64_t)dst_height * WEBP_RESCALER_ONE / (wrk->x_add * wrk->y_add);
+    if (ratio != (uint32_t)ratio) {
+      // We can't represent the ratio with the current fixed-point precision.
+      // => We special-case fxy_scale = 0, in WebPRescalerExportRow().
+      wrk->fxy_scale = 0;
+    } else {
+      wrk->fxy_scale = (uint32_t)ratio;
+    }
+    wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->y_sub);
+  } else {
+    wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->x_add);
+    // wrk->fxy_scale is unused here.
+  }
   wrk->irow = work;
   wrk->frow = work + num_channels * dst_width;
+  memset(work, 0, 2 * dst_width * num_channels * sizeof(*work));
 
-  if (WebPRescalerImportRow == NULL) {
+  if (WebPRescalerImportRowExpand == NULL) {
-    WebPRescalerImportRow = ImportRowC;
+    WebPRescalerImportRowExpand = ImportRowExpandC;
-    WebPRescalerExportRow = ExportRowC;
+    WebPRescalerImportRowShrink = ImportRowShrinkC;
+    WebPRescalerExportRowExpand = ExportRowExpandC;
+    WebPRescalerExportRowShrink = ExportRowShrinkC;
     if (VP8GetCPUInfo != NULL) {
 #if defined(WEBP_USE_MIPS32)
       if (VP8GetCPUInfo(kMIPS32)) {
-        WebPRescalerImportRow = ImportRowMIPS;
+        WebPRescalerImportRowExpand = ImportRowExpandMIPS;
-        WebPRescalerExportRow = ExportRowMIPS;
+        WebPRescalerImportRowShrink = ImportRowShrinkMIPS;
+        WebPRescalerExportRowExpand = ExportRowExpandMIPS;
+        WebPRescalerExportRowShrink = ExportRowShrinkMIPS;
       }
 #endif
     }
@@ -296,7 +529,10 @@ void WebPRescalerInit(WebPRescaler* const wrk, int src_width, int src_height,
 }
 
 #undef MULT_FIX
-#undef RFIX
+#undef WEBP_RESCALER_RFIX
+#undef WEBP_RESCALER_ONE
+#undef WEBP_RESCALER_FRAC
+#undef ROUNDER
 
 //------------------------------------------------------------------------------
 // all-in-one calls
@@ -309,11 +545,20 @@ int WebPRescaleNeededLines(const WebPRescaler* const wrk, int max_num_lines) {
 int WebPRescalerImport(WebPRescaler* const wrk, int num_lines,
                        const uint8_t* src, int src_stride) {
   int total_imported = 0;
-  while (total_imported < num_lines && wrk->y_accum > 0) {
+  while (total_imported < num_lines && !WebPRescalerHasPendingOutput(wrk)) {
-    int channel;
+    if (wrk->y_expand) {
-    for (channel = 0; channel < wrk->num_channels; ++channel) {
+      rescaler_t* const tmp = wrk->irow;
-      WebPRescalerImportRow(wrk, src, channel);
+      wrk->irow = wrk->frow;
+      wrk->frow = tmp;
     }
+    WebPRescalerImportRow(wrk, src);
+    if (!wrk->y_expand) {     // Accumulate the contribution of the new row.
+      int x;
+      for (x = 0; x < wrk->num_channels * wrk->dst_width; ++x) {
+        wrk->irow[x] += wrk->frow[x];
+      }
+    }
+    ++wrk->src_y;
     src += src_stride;
     ++total_imported;
     wrk->y_accum -= wrk->y_sub;
@@ -324,7 +569,7 @@ int WebPRescalerImport(WebPRescaler* const wrk, int num_lines,
 int WebPRescalerExport(WebPRescaler* const rescaler) {
   int total_exported = 0;
   while (WebPRescalerHasPendingOutput(rescaler)) {
-    WebPRescalerExportRow(rescaler, 0);
+    WebPRescalerExportRow(rescaler);
     ++total_exported;
   }
   return total_exported;

src/utils/rescaler.h

@@ -21,20 +21,23 @@ extern "C" {
 #include "../webp/types.h"
 
 // Structure used for on-the-fly rescaling
+typedef uint32_t rescaler_t;   // type for side-buffer
 typedef struct {
   int x_expand;               // true if we're expanding in the x direction
+  int y_expand;               // true if we're expanding in the y direction
   int num_channels;           // bytes to jump between pixels
-  int fy_scale, fx_scale;     // fixed-point scaling factor
+  uint32_t fx_scale;          // fixed-point scaling factors
-  int64_t fxy_scale;          // ''
+  uint32_t fy_scale;          // ''
-  // we need hpel-precise add/sub increments, for the downsampled U/V planes.
+  uint32_t fxy_scale;         // ''
   int y_accum;                // vertical accumulator
-  int y_add, y_sub;           // vertical increments (add ~= src, sub ~= dst)
+  int y_add, y_sub;           // vertical increments
-  int x_add, x_sub;           // horizontal increments (add ~= src, sub ~= dst)
+  int x_add, x_sub;           // horizontal increments
   int src_width, src_height;  // source dimensions
   int dst_width, dst_height;  // destination dimensions
+  int src_y, dst_y;           // row counters for input and output
   uint8_t* dst;
   int dst_stride;
-  int32_t* irow, *frow;       // work buffer
+  rescaler_t* irow, *frow;    // work buffer
 } WebPRescaler;
 
 // Initialize a rescaler given scratch area 'work' and dimensions of src & dst.
@@ -43,9 +46,7 @@ void WebPRescalerInit(WebPRescaler* const rescaler,
                       uint8_t* const dst,
                       int dst_width, int dst_height, int dst_stride,
                       int num_channels,
-                      int x_add, int x_sub,
+                      rescaler_t* const work);
-                      int y_add, int y_sub,
-                      int32_t* const work);
 
 // Returns the number of input lines needed next to produce one output line,
 // considering that the maximum available input lines are 'max_num_lines'.
@@ -57,21 +58,29 @@ int WebPRescaleNeededLines(const WebPRescaler* const rescaler,
 int WebPRescalerImport(WebPRescaler* const rescaler, int num_rows,
                        const uint8_t* src, int src_stride);
 
-// Import a row of data and save its contribution in the rescaler.
-// 'channel' denotes the channel number to be imported.
-extern void (*WebPRescalerImportRow)(WebPRescaler* const wrk,
-                                     const uint8_t* const src, int channel);
-// Export one row (starting at x_out position) from rescaler.
-extern void (*WebPRescalerExportRow)(WebPRescaler* const wrk, int x_out);
-
-// Return true if there is pending output rows ready.
-static WEBP_INLINE
-int WebPRescalerHasPendingOutput(const WebPRescaler* const rescaler) {
-  return (rescaler->y_accum <= 0);
-}
-
 // Export as many rows as possible. Return the numbers of rows written.
 int WebPRescalerExport(WebPRescaler* const rescaler);
+void WebPRescalerImportRow(WebPRescaler* const wrk,
+                           const uint8_t* src);
+// Export one row (starting at x_out position) from rescaler.
+void WebPRescalerExportRow(WebPRescaler* const wrk);
+
+// Return true if input is finished
+static WEBP_INLINE
+int WebPRescalerInputDone(const WebPRescaler* const rescaler) {
+  return (rescaler->src_y >= rescaler->src_height);
+}
+// Return true if output is finished
+static WEBP_INLINE
+int WebPRescalerOutputDone(const WebPRescaler* const rescaler) {
+  return (rescaler->dst_y >= rescaler->dst_height);
+}
+
+// Return true if there are pending output rows ready.
+static WEBP_INLINE
+int WebPRescalerHasPendingOutput(const WebPRescaler* const rescaler) {
+  return !WebPRescalerOutputDone(rescaler) && (rescaler->y_accum <= 0);
+}
 
 //------------------------------------------------------------------------------