Merge "update the Unfilter API in dsp to process one row independently"

src/dec/alpha.c

@@ -111,17 +111,29 @@ static int ALPHDecode(VP8Decoder* const dec, int row, int num_rows) {
   ALPHDecoder* const alph_dec = dec->alph_dec_;
   const int width = alph_dec->width_;
   const int height = alph_dec->io_.crop_bottom;
-  uint8_t* const output = dec->alpha_plane_;
   if (alph_dec->method_ == ALPHA_NO_COMPRESSION) {
-    const size_t offset = row * width;
+    int y;
-    const size_t num_pixels = num_rows * width;
+    const uint8_t* prev_line = dec->alpha_prev_line_;
-    assert(dec->alpha_data_size_ >= ALPHA_HEADER_LEN + offset + num_pixels);
+    const uint8_t* deltas = dec->alpha_data_ + ALPHA_HEADER_LEN + row * width;
-    memcpy(dec->alpha_plane_ + offset,
+    uint8_t* dst = dec->alpha_plane_ + row * width;
-           dec->alpha_data_ + ALPHA_HEADER_LEN + offset, num_pixels);
+    assert(deltas <= &dec->alpha_data_[dec->alpha_data_size_]);
-    if (WebPUnfilters[alph_dec->filter_] != NULL) {
+    if (alph_dec->filter_ != WEBP_FILTER_NONE) {
-      WebPUnfilters[alph_dec->filter_](width, height, width,
+      assert(WebPUnfilters[alph_dec->filter_] != NULL);
-                                       row, num_rows, output);
+      for (y = 0; y < num_rows; ++y) {
+        WebPUnfilters[alph_dec->filter_](prev_line, deltas, dst, width);
+        prev_line = dst;
+        dst += width;
+        deltas += width;
+      }
+    } else {
+      for (y = 0; y < num_rows; ++y) {
+        memcpy(dst, deltas, width * sizeof(*dst));
+        prev_line = dst;
+        dst += width;
+        deltas += width;
+      }
     }
+    dec->alpha_prev_line_ = prev_line;
   } else {  // alph_dec->method_ == ALPHA_LOSSLESS_COMPRESSION
     assert(alph_dec->vp8l_dec_ != NULL);
     if (!VP8LDecodeAlphaImageStream(alph_dec, row + num_rows)) {
@@ -146,6 +158,7 @@ static int AllocateAlphaPlane(VP8Decoder* const dec, const VP8Io* const io) {
     return 0;
   }
   dec->alpha_plane_ = dec->alpha_plane_mem_;
+  dec->alpha_prev_line_ = NULL;
   return 1;
 }
 

src/dec/alphai.h

@@ -36,6 +36,7 @@ struct ALPHDecoder {
                        // pixel, sometimes VP8LDecoder may need to allocate
                        // 4 bytes per pixel internally during decode.
   uint8_t* output_;
+  const uint8_t* prev_line_;   // last output row (or NULL)
 };
 
 //------------------------------------------------------------------------------

src/dec/vp8i.h

@@ -261,6 +261,7 @@ struct VP8Decoder {
   int is_alpha_decoded_;      // true if alpha_data_ is decoded in alpha_plane_
   uint8_t* alpha_plane_mem_;  // memory allocated for alpha_plane_
   uint8_t* alpha_plane_;      // output. Persistent, contains the whole data.
+  const uint8_t* alpha_prev_line_;  // last decoded alpha row (or NULL)
   int alpha_dithering_;       // derived from decoding options (0=off, 100=full)
 };
 

src/dec/vp8l.c

@@ -771,35 +771,46 @@ static int Is8bOptimizable(const VP8LMetadata* const hdr) {
   return 1;
 }
 
-static void ExtractPalettedAlphaRows(VP8LDecoder* const dec, int row) {
+static void AlphaApplyFilter(ALPHDecoder* const alph_dec,
+                             int first_row, int last_row,
+                             uint8_t* out, int stride) {
+  if (alph_dec->filter_ != WEBP_FILTER_NONE) {
+    int y;
+    const uint8_t* prev_line = alph_dec->prev_line_;
+    assert(WebPUnfilters[alph_dec->filter_] != NULL);
+    for (y = first_row; y < last_row; ++y) {
+      WebPUnfilters[alph_dec->filter_](prev_line, out, out, stride);
+      prev_line = out;
+      out += stride;
+    }
+    alph_dec->prev_line_ = prev_line;
+  }
+}
+
+static void ExtractPalettedAlphaRows(VP8LDecoder* const dec, int last_row) {
   // For vertical and gradient filtering, we need to decode the part above the
   // crop_top row, in order to have the correct spatial predictors.
-  const ALPHDecoder* const alph_dec = (const ALPHDecoder*)dec->io_->opaque;
+  ALPHDecoder* const alph_dec = (ALPHDecoder*)dec->io_->opaque;
   const int top_row =
       (alph_dec->filter_ == WEBP_FILTER_NONE ||
        alph_dec->filter_ == WEBP_FILTER_HORIZONTAL) ? dec->io_->crop_top
                                                     : dec->last_row_;
   const int first_row = (dec->last_row_ < top_row) ? top_row : dec->last_row_;
-  assert(row <= dec->io_->crop_bottom);
+  assert(last_row <= dec->io_->crop_bottom);
-  if (row > first_row) {
+  if (last_row > first_row) {
     // Special method for paletted alpha data. We only process the cropped area.
     const int width = dec->io_->width;
-    uint8_t* const out = alph_dec->output_ + width * first_row;
+    uint8_t* out = alph_dec->output_ + width * first_row;
     const uint8_t* const in =
       (uint8_t*)dec->pixels_ + dec->width_ * first_row;
     VP8LTransform* const transform = &dec->transforms_[0];
     assert(dec->next_transform_ == 1);
     assert(transform->type_ == COLOR_INDEXING_TRANSFORM);
-    VP8LColorIndexInverseTransformAlpha(transform, first_row, row,
+    VP8LColorIndexInverseTransformAlpha(transform, first_row, last_row,
                                         in, out);
-    if (alph_dec->filter_ != WEBP_FILTER_NONE) {
+    AlphaApplyFilter(alph_dec, first_row, last_row, out, width);
-      assert(WebPUnfilters[alph_dec->filter_] != NULL);
-      WebPUnfilters[alph_dec->filter_](width, dec->io_->height, width,
-                                       first_row, row - first_row,
-                                       alph_dec->output_);
-    }
   }
-  dec->last_row_ = dec->last_out_row_ = row;
+  dec->last_row_ = dec->last_out_row_ = last_row;
 }
 
 //------------------------------------------------------------------------------
@@ -1452,29 +1463,25 @@ static int AllocateInternalBuffers8b(VP8LDecoder* const dec) {
 //------------------------------------------------------------------------------
 
 // Special row-processing that only stores the alpha data.
-static void ExtractAlphaRows(VP8LDecoder* const dec, int row) {
+static void ExtractAlphaRows(VP8LDecoder* const dec, int last_row) {
-  const int num_rows = row - dec->last_row_;
+  const int num_rows = last_row - dec->last_row_;
   const uint32_t* const in = dec->pixels_ + dec->width_ * dec->last_row_;
 
-  assert(row <= dec->io_->crop_bottom);
+  assert(last_row <= dec->io_->crop_bottom);
   if (num_rows > 0) {
     // Extract alpha (which is stored in the green plane).
-    const ALPHDecoder* const alph_dec = (const ALPHDecoder*)dec->io_->opaque;
+    ALPHDecoder* const alph_dec = (ALPHDecoder*)dec->io_->opaque;
     uint8_t* const output = alph_dec->output_;
     const int width = dec->io_->width;      // the final width (!= dec->width_)
     const int cache_pixs = width * num_rows;
-    uint8_t* const dst = output + width * dec->last_row_;
+    uint8_t* dst = output + width * dec->last_row_;
     const uint32_t* const src = dec->argb_cache_;
     int i;
     ApplyInverseTransforms(dec, num_rows, in);
     for (i = 0; i < cache_pixs; ++i) dst[i] = (src[i] >> 8) & 0xff;
-    if (alph_dec->filter_ != WEBP_FILTER_NONE) {
+    AlphaApplyFilter(alph_dec, dec->last_row_, last_row, dst, width);
-      assert(WebPUnfilters[alph_dec->filter_] != NULL);
-      WebPUnfilters[alph_dec->filter_](width, dec->io_->height, width,
-                                       dec->last_row_, num_rows, output);
-    }
   }
-  dec->last_row_ = dec->last_out_row_ = row;
+  dec->last_row_ = dec->last_out_row_ = last_row;
 }
 
 int VP8LDecodeAlphaHeader(ALPHDecoder* const alph_dec,

src/dsp/dsp.h

@@ -503,8 +503,10 @@ typedef enum {     // Filter types.
 
 typedef void (*WebPFilterFunc)(const uint8_t* in, int width, int height,
                                int stride, uint8_t* out);
-typedef void (*WebPUnfilterFunc)(int width, int height, int stride,
+// In-place un-filtering.
-                                 int row, int num_rows, uint8_t* data);
+// Warning! 'prev_line' pointer can be equal to 'cur_line' or 'preds'.
+typedef void (*WebPUnfilterFunc)(const uint8_t* prev_line, const uint8_t* preds,
+                                 uint8_t* cur_line, int width);
 
 // Filter the given data using the given predictor.
 // 'in' corresponds to a 2-dimensional pixel array of size (stride * height)

src/dsp/filters.c

@@ -184,19 +184,40 @@ static void GradientFilter(const uint8_t* data, int width, int height,
 
 //------------------------------------------------------------------------------
 
-static void VerticalUnfilter(int width, int height, int stride, int row,
+static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in,
-                             int num_rows, uint8_t* data) {
+                               uint8_t* out, int width) {
-  DoVerticalFilter(data, width, height, stride, row, num_rows, 1, data);
+  uint8_t pred = (prev == NULL) ? 0 : prev[0];
+  int i;
+  for (i = 0; i < width; ++i) {
+    out[i] = pred + in[i];
+    pred = out[i];
+  }
 }
 
-static void HorizontalUnfilter(int width, int height, int stride, int row,
+static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in,
-                               int num_rows, uint8_t* data) {
+                             uint8_t* out, int width) {
-  DoHorizontalFilter(data, width, height, stride, row, num_rows, 1, data);
+  if (prev == NULL) {
+    HorizontalUnfilter(NULL, in, out, width);
+  } else {
+    int i;
+    for (i = 0; i < width; ++i) out[i] = prev[i] + in[i];
+  }
 }
 
-static void GradientUnfilter(int width, int height, int stride, int row,
+static void GradientUnfilter(const uint8_t* prev, const uint8_t* in,
-                             int num_rows, uint8_t* data) {
+                             uint8_t* out, int width) {
-  DoGradientFilter(data, width, height, stride, row, num_rows, 1, data);
+  if (prev == NULL) {
+    HorizontalUnfilter(NULL, in, out, width);
+  } else {
+    uint8_t top = prev[0], top_left = top, left = top;
+    int i;
+    for (i = 0; i < width; ++i) {
+      top = prev[i];  // need to read this first, in case prev==out
+      left = in[i] + GradientPredictor(left, top, top_left);
+      top_left = top;
+      out[i] = left;
+    }
+  }
 }
 
 //------------------------------------------------------------------------------

src/dsp/filters_mips_dsp_r2.c

@@ -33,12 +33,9 @@
   assert(row >= 0 && num_rows > 0 && row + num_rows <= height);                \
   (void)height;  // Silence unused warning.
 
-// if INVERSE
-//   preds == &dst[-1] == &src[-1]
-// else
 //   preds == &src[-1] != &dst[-1]
-#define DO_PREDICT_LINE(SRC, DST, LENGTH, INVERSE) do {                        \
+#define DO_PREDICT_LINE(SRC, DST, LENGTH) do {                                 \
-    const uint8_t* psrc = (uint8_t*)(SRC);                                     \
+    const uint8_t* psrc = (const uint8_t*)(SRC);                               \
     uint8_t* pdst = (uint8_t*)(DST);                                           \
     const int ilength = (int)(LENGTH);                                         \
     int temp0, temp1, temp2, temp3, temp4, temp5, temp6;                       \
@@ -48,25 +45,6 @@
       "srl       %[temp0],    %[length],    0x2         \n\t"                  \
       "beqz      %[temp0],    4f                        \n\t"                  \
       " andi     %[temp6],    %[length],    0x3         \n\t"                  \
-    ".if " #INVERSE "                                   \n\t"                  \
-      "lbu       %[temp1],    -1(%[src])                \n\t"                  \
-    "1:                                                 \n\t"                  \
-      "lbu       %[temp2],    0(%[src])                 \n\t"                  \
-      "lbu       %[temp3],    1(%[src])                 \n\t"                  \
-      "lbu       %[temp4],    2(%[src])                 \n\t"                  \
-      "lbu       %[temp5],    3(%[src])                 \n\t"                  \
-      "addiu     %[src],      %[src],       4           \n\t"                  \
-      "addiu     %[temp0],    %[temp0],     -1          \n\t"                  \
-      "addu      %[temp2],    %[temp2],     %[temp1]    \n\t"                  \
-      "addu      %[temp3],    %[temp3],     %[temp2]    \n\t"                  \
-      "addu      %[temp4],    %[temp4],     %[temp3]    \n\t"                  \
-      "addu      %[temp1],    %[temp5],     %[temp4]    \n\t"                  \
-      "sb        %[temp2],    -4(%[src])                \n\t"                  \
-      "sb        %[temp3],    -3(%[src])                \n\t"                  \
-      "sb        %[temp4],    -2(%[src])                \n\t"                  \
-      "bnez      %[temp0],    1b                        \n\t"                  \
-      " sb       %[temp1],    -1(%[src])                \n\t"                  \
-    ".else                                              \n\t"                  \
     "1:                                                 \n\t"                  \
       "ulw       %[temp1],    -1(%[src])                \n\t"                  \
       "ulw       %[temp2],    0(%[src])                 \n\t"                  \
@@ -76,7 +54,6 @@
       "usw       %[temp3],    0(%[dst])                 \n\t"                  \
       "bnez      %[temp0],    1b                        \n\t"                  \
       " addiu    %[dst],      %[dst],       4           \n\t"                  \
-    ".endif                                             \n\t"                  \
     "4:                                                 \n\t"                  \
       "beqz      %[temp6],    3f                        \n\t"                  \
       " nop                                             \n\t"                  \
@@ -84,13 +61,8 @@
       "lbu       %[temp1],    -1(%[src])                \n\t"                  \
       "lbu       %[temp2],    0(%[src])                 \n\t"                  \
       "addiu     %[src],      %[src],       1           \n\t"                  \
-    ".if " #INVERSE "                                   \n\t"                  \
-      "addu      %[temp3],    %[temp1],     %[temp2]    \n\t"                  \
-      "sb        %[temp3],    -1(%[src])                \n\t"                  \
-    ".else                                              \n\t"                  \
       "subu      %[temp3],    %[temp1],     %[temp2]    \n\t"                  \
       "sb        %[temp3],    0(%[dst])                 \n\t"                  \
-    ".endif                                             \n\t"                  \
       "addiu     %[temp6],    %[temp6],     -1          \n\t"                  \
       "bnez      %[temp6],    2b                        \n\t"                  \
       " addiu    %[dst],      %[dst],       1           \n\t"                  \
@@ -105,12 +77,8 @@
   } while (0)
 
 static WEBP_INLINE void PredictLine(const uint8_t* src, uint8_t* dst,
-                                    int length, int inverse) {
+                                    int length) {
-  if (inverse) {
+  DO_PREDICT_LINE(src, dst, length);
-    DO_PREDICT_LINE(src, dst, length, 1);
-  } else {
-    DO_PREDICT_LINE(src, dst, length, 0);
-  }
 }
 
 #define DO_PREDICT_LINE_VERTICAL(SRC, PRED, DST, LENGTH, INVERSE) do {         \
@@ -172,16 +140,12 @@ static WEBP_INLINE void PredictLine(const uint8_t* src, uint8_t* dst,
     );                                                                         \
   } while (0)
 
-#define PREDICT_LINE_ONE_PASS(SRC, PRED, DST, INVERSE) do {                    \
+#define PREDICT_LINE_ONE_PASS(SRC, PRED, DST) do {                             \
     int temp1, temp2, temp3;                                                   \
     __asm__ volatile (                                                         \
       "lbu       %[temp1],   0(%[src])               \n\t"                     \
       "lbu       %[temp2],   0(%[pred])              \n\t"                     \
-    ".if " #INVERSE "                                \n\t"                     \
-      "addu      %[temp3],   %[temp1],   %[temp2]    \n\t"                     \
-    ".else                                           \n\t"                     \
       "subu      %[temp3],   %[temp1],   %[temp2]    \n\t"                     \
-    ".endif                                          \n\t"                     \
       "sb        %[temp3],   0(%[dst])               \n\t"                     \
       : [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), [temp3]"=&r"(temp3)          \
       : [pred]"r"((PRED)), [dst]"r"((DST)), [src]"r"((SRC))                    \
@@ -192,10 +156,10 @@ static WEBP_INLINE void PredictLine(const uint8_t* src, uint8_t* dst,
 //------------------------------------------------------------------------------
 // Horizontal filter.
 
-#define FILTER_LINE_BY_LINE(INVERSE) do {                                      \
+#define FILTER_LINE_BY_LINE do {                                               \
     while (row < last_row) {                                                   \
-      PREDICT_LINE_ONE_PASS(in, preds - stride, out, INVERSE);                 \
+      PREDICT_LINE_ONE_PASS(in, preds - stride, out);                          \
-      DO_PREDICT_LINE(in + 1, out + 1, width - 1, INVERSE);                    \
+      DO_PREDICT_LINE(in + 1, out + 1, width - 1);                             \
       ++row;                                                                   \
       preds += stride;                                                         \
       in += stride;                                                            \
@@ -206,19 +170,19 @@ static WEBP_INLINE void PredictLine(const uint8_t* src, uint8_t* dst,
 static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in,
                                            int width, int height, int stride,
                                            int row, int num_rows,
-                                           int inverse, uint8_t* out) {
+                                           uint8_t* out) {
   const uint8_t* preds;
   const size_t start_offset = row * stride;
   const int last_row = row + num_rows;
   SANITY_CHECK(in, out);
   in += start_offset;
   out += start_offset;
-  preds = inverse ? out : in;
+  preds = in;
 
   if (row == 0) {
     // Leftmost pixel is the same as input for topmost scanline.
     out[0] = in[0];
-    PredictLine(in + 1, out + 1, width - 1, inverse);
+    PredictLine(in + 1, out + 1, width - 1);
     row = 1;
     preds += stride;
     in += stride;
@@ -226,31 +190,21 @@ static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in,
   }
 
   // Filter line-by-line.
-  if (inverse) {
+  FILTER_LINE_BY_LINE;
-    FILTER_LINE_BY_LINE(1);
-  } else {
-    FILTER_LINE_BY_LINE(0);
-  }
 }
-
 #undef FILTER_LINE_BY_LINE
 
 static void HorizontalFilter(const uint8_t* data, int width, int height,
                              int stride, uint8_t* filtered_data) {
-  DoHorizontalFilter(data, width, height, stride, 0, height, 0, filtered_data);
+  DoHorizontalFilter(data, width, height, stride, 0, height, filtered_data);
-}
-
-static void HorizontalUnfilter(int width, int height, int stride, int row,
-                               int num_rows, uint8_t* data) {
-  DoHorizontalFilter(data, width, height, stride, row, num_rows, 1, data);
 }
 
 //------------------------------------------------------------------------------
 // Vertical filter.
 
-#define FILTER_LINE_BY_LINE(INVERSE) do {                                      \
+#define FILTER_LINE_BY_LINE do {                                               \
     while (row < last_row) {                                                   \
-      DO_PREDICT_LINE_VERTICAL(in, preds, out, width, INVERSE);                \
+      DO_PREDICT_LINE_VERTICAL(in, preds, out, width, 0);                      \
       ++row;                                                                   \
       preds += stride;                                                         \
       in += stride;                                                            \
@@ -260,21 +214,20 @@ static void HorizontalUnfilter(int width, int height, int stride, int row,
 
 static WEBP_INLINE void DoVerticalFilter(const uint8_t* in,
                                          int width, int height, int stride,
-                                         int row, int num_rows,
+                                         int row, int num_rows, uint8_t* out) {
-                                         int inverse, uint8_t* out) {
   const uint8_t* preds;
   const size_t start_offset = row * stride;
   const int last_row = row + num_rows;
   SANITY_CHECK(in, out);
   in += start_offset;
   out += start_offset;
-  preds = inverse ? out : in;
+  preds = in;
 
   if (row == 0) {
     // Very first top-left pixel is copied.
     out[0] = in[0];
     // Rest of top scan-line is left-predicted.
-    PredictLine(in + 1, out + 1, width - 1, inverse);
+    PredictLine(in + 1, out + 1, width - 1);
     row = 1;
     in += stride;
     out += stride;
@@ -284,24 +237,13 @@ static WEBP_INLINE void DoVerticalFilter(const uint8_t* in,
   }
 
   // Filter line-by-line.
-  if (inverse) {
+  FILTER_LINE_BY_LINE;
-    FILTER_LINE_BY_LINE(1);
-  } else {
-    FILTER_LINE_BY_LINE(0);
-  }
 }
-
 #undef FILTER_LINE_BY_LINE
-#undef DO_PREDICT_LINE_VERTICAL
 
 static void VerticalFilter(const uint8_t* data, int width, int height,
                            int stride, uint8_t* filtered_data) {
-  DoVerticalFilter(data, width, height, stride, 0, height, 0, filtered_data);
+  DoVerticalFilter(data, width, height, stride, 0, height, filtered_data);
-}
-
-static void VerticalUnfilter(int width, int height, int stride, int row,
-                             int num_rows, uint8_t* data) {
-  DoVerticalFilter(data, width, height, stride, row, num_rows, 1, data);
 }
 
 //------------------------------------------------------------------------------
@@ -321,10 +263,10 @@ static WEBP_INLINE int GradientPredictor(uint8_t a, uint8_t b, uint8_t c) {
   return temp0;
 }
 
-#define FILTER_LINE_BY_LINE(INVERSE, PREDS, OPERATION) do {                    \
+#define FILTER_LINE_BY_LINE(PREDS, OPERATION) do {                             \
     while (row < last_row) {                                                   \
       int w;                                                                   \
-      PREDICT_LINE_ONE_PASS(in, PREDS - stride, out, INVERSE);                 \
+      PREDICT_LINE_ONE_PASS(in, PREDS - stride, out);                          \
       for (w = 1; w < width; ++w) {                                            \
         const int pred = GradientPredictor(PREDS[w - 1],                       \
                                            PREDS[w - stride],                  \
@@ -339,20 +281,19 @@ static WEBP_INLINE int GradientPredictor(uint8_t a, uint8_t b, uint8_t c) {
 
 static WEBP_INLINE void DoGradientFilter(const uint8_t* in,
                                          int width, int height, int stride,
-                                         int row, int num_rows,
+                                         int row, int num_rows, uint8_t* out) {
-                                         int inverse, uint8_t* out) {
   const uint8_t* preds;
   const size_t start_offset = row * stride;
   const int last_row = row + num_rows;
   SANITY_CHECK(in, out);
   in += start_offset;
   out += start_offset;
-  preds = inverse ? out : in;
+  preds = in;
 
   // left prediction for top scan-line
   if (row == 0) {
     out[0] = in[0];
-    PredictLine(in + 1, out + 1, width - 1, inverse);
+    PredictLine(in + 1, out + 1, width - 1);
     row = 1;
     preds += stride;
     in += stride;
@@ -360,25 +301,50 @@ static WEBP_INLINE void DoGradientFilter(const uint8_t* in,
   }
 
   // Filter line-by-line.
-  if (inverse) {
+  FILTER_LINE_BY_LINE(in, -);
-    FILTER_LINE_BY_LINE(1, out, +);
-  } else {
-    FILTER_LINE_BY_LINE(0, in, -);
-  }
 }
-
 #undef FILTER_LINE_BY_LINE
 
 static void GradientFilter(const uint8_t* data, int width, int height,
                            int stride, uint8_t* filtered_data) {
-  DoGradientFilter(data, width, height, stride, 0, height, 0, filtered_data);
+  DoGradientFilter(data, width, height, stride, 0, height, filtered_data);
 }
 
-static void GradientUnfilter(int width, int height, int stride, int row,
+//------------------------------------------------------------------------------
-                             int num_rows, uint8_t* data) {
+
-  DoGradientFilter(data, width, height, stride, row, num_rows, 1, data);
+static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in,
+                               uint8_t* out, int width) {
+  int i;
+  out[0] = in[0] + (prev == NULL ? 0 : prev[0]);
+  for (i = 1; i < width; ++i) out[i] = in[i] + out[i - 1];
 }
 
+static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in,
+                             uint8_t* out, int width) {
+  if (prev == NULL) {
+    HorizontalUnfilter(NULL, in, out, width);
+  } else {
+    DO_PREDICT_LINE_VERTICAL(in, prev, out, width, 1);
+  }
+}
+
+static void GradientUnfilter(const uint8_t* prev, const uint8_t* in,
+                             uint8_t* out, int width) {
+  if (prev == NULL) {
+    HorizontalUnfilter(NULL, in, out, width);
+  } else {
+    uint8_t top = prev[0], top_left = top, left = top;
+    int i;
+    for (i = 0; i < width; ++i) {
+      top = prev[i];  // need to read this first, in case prev==dst
+      left = in[i] + GradientPredictor(left, top, top_left);
+      top_left = top;
+      out[i] = left;
+    }
+  }
+}
+
+#undef DO_PREDICT_LINE_VERTICAL
 #undef PREDICT_LINE_ONE_PASS
 #undef DO_PREDICT_LINE
 #undef SANITY_CHECK
@@ -389,13 +355,13 @@ static void GradientUnfilter(int width, int height, int stride, int row,
 extern void VP8FiltersInitMIPSdspR2(void);
 
 WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitMIPSdspR2(void) {
-  WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter;
-  WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter;
-  WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter;
-
   WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter;
   WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter;
   WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter;
+
+  WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter;
+  WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter;
+  WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter;
 }
 
 #else  // !WEBP_USE_MIPS_DSP_R2

src/dsp/filters_sse2.c

@@ -33,82 +33,39 @@
   (void)height;  // Silence unused warning.
 
 static void PredictLineTop(const uint8_t* src, const uint8_t* pred,
-                           uint8_t* dst, int length, int inverse) {
+                           uint8_t* dst, int length) {
   int i;
   const int max_pos = length & ~31;
   assert(length >= 0);
-  if (inverse) {
+  for (i = 0; i < max_pos; i += 32) {
-    for (i = 0; i < max_pos; i += 32) {
+    const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i +  0]);
-      const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i +  0]);
+    const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]);
-      const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]);
+    const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i +  0]);
-      const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i +  0]);
+    const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]);
-      const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]);
+    const __m128i C0 = _mm_sub_epi8(A0, B0);
-      const __m128i C0 = _mm_add_epi8(A0, B0);
+    const __m128i C1 = _mm_sub_epi8(A1, B1);
-      const __m128i C1 = _mm_add_epi8(A1, B1);
+    _mm_storeu_si128((__m128i*)&dst[i +  0], C0);
-      _mm_storeu_si128((__m128i*)&dst[i +  0], C0);
+    _mm_storeu_si128((__m128i*)&dst[i + 16], C1);
-      _mm_storeu_si128((__m128i*)&dst[i + 16], C1);
-    }
-    for (; i < length; ++i) dst[i] = src[i] + pred[i];
-  } else {
-    for (i = 0; i < max_pos; i += 32) {
-      const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i +  0]);
-      const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]);
-      const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i +  0]);
-      const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]);
-      const __m128i C0 = _mm_sub_epi8(A0, B0);
-      const __m128i C1 = _mm_sub_epi8(A1, B1);
-      _mm_storeu_si128((__m128i*)&dst[i +  0], C0);
-      _mm_storeu_si128((__m128i*)&dst[i + 16], C1);
-    }
-    for (; i < length; ++i) dst[i] = src[i] - pred[i];
   }
+  for (; i < length; ++i) dst[i] = src[i] - pred[i];
 }
 
 // Special case for left-based prediction (when preds==dst-1 or preds==src-1).
-static void PredictLineLeft(const uint8_t* src, uint8_t* dst, int length,
+static void PredictLineLeft(const uint8_t* src, uint8_t* dst, int length) {
-                            int inverse) {
   int i;
-  if (length <= 0) return;
+  const int max_pos = length & ~31;
-  if (inverse) {
+  assert(length >= 0);
-    const int max_pos = length & ~7;
+  for (i = 0; i < max_pos; i += 32) {
-    __m128i last = _mm_set_epi32(0, 0, 0, dst[-1]);
+    const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i +  0    ));
-    for (i = 0; i < max_pos; i += 8) {
+    const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i +  0 - 1));
-      const __m128i A0 = _mm_loadl_epi64((const __m128i*)(src + i));
+    const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16    ));
-      const __m128i A1 = _mm_add_epi8(A0, last);
+    const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1));
-      const __m128i A2 = _mm_slli_si128(A1, 1);
+    const __m128i C0 = _mm_sub_epi8(A0, B0);
-      const __m128i A3 = _mm_add_epi8(A1, A2);
+    const __m128i C1 = _mm_sub_epi8(A1, B1);
-      const __m128i A4 = _mm_slli_si128(A3, 2);
+    _mm_storeu_si128((__m128i*)(dst + i +  0), C0);
-      const __m128i A5 = _mm_add_epi8(A3, A4);
+    _mm_storeu_si128((__m128i*)(dst + i + 16), C1);
-      const __m128i A6 = _mm_slli_si128(A5, 4);
-      const __m128i A7 = _mm_add_epi8(A5, A6);
-      _mm_storel_epi64((__m128i*)(dst + i), A7);
-      last = _mm_srli_epi64(A7, 56);
-    }
-    for (; i < length; ++i) dst[i] = src[i] + dst[i - 1];
-  } else {
-    const int max_pos = length & ~31;
-    for (i = 0; i < max_pos; i += 32) {
-      const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i +  0    ));
-      const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i +  0 - 1));
-      const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16    ));
-      const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1));
-      const __m128i C0 = _mm_sub_epi8(A0, B0);
-      const __m128i C1 = _mm_sub_epi8(A1, B1);
-      _mm_storeu_si128((__m128i*)(dst + i +  0), C0);
-      _mm_storeu_si128((__m128i*)(dst + i + 16), C1);
-    }
-    for (; i < length; ++i) dst[i] = src[i] - src[i - 1];
-  }
-}
-
-static void PredictLineC(const uint8_t* src, const uint8_t* pred,
-                         uint8_t* dst, int length, int inverse) {
-  int i;
-  if (inverse) {
-    for (i = 0; i < length; ++i) dst[i] = src[i] + pred[i];
-  } else {
-    for (i = 0; i < length; ++i) dst[i] = src[i] - pred[i];
   }
+  for (; i < length; ++i) dst[i] = src[i] - src[i - 1];
 }
 
 //------------------------------------------------------------------------------
@@ -117,21 +74,18 @@ static void PredictLineC(const uint8_t* src, const uint8_t* pred,
 static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in,
                                            int width, int height, int stride,
                                            int row, int num_rows,
-                                           int inverse, uint8_t* out) {
+                                           uint8_t* out) {
-  const uint8_t* preds;
   const size_t start_offset = row * stride;
   const int last_row = row + num_rows;
   SANITY_CHECK(in, out);
   in += start_offset;
   out += start_offset;
-  preds = inverse ? out : in;
 
   if (row == 0) {
     // Leftmost pixel is the same as input for topmost scanline.
     out[0] = in[0];
-    PredictLineLeft(in + 1, out + 1, width - 1, inverse);
+    PredictLineLeft(in + 1, out + 1, width - 1);
     row = 1;
-    preds += stride;
     in += stride;
     out += stride;
   }
@@ -139,10 +93,9 @@ static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in,
   // Filter line-by-line.
   while (row < last_row) {
     // Leftmost pixel is predicted from above.
-    PredictLineC(in, preds - stride, out, 1, inverse);
+    out[0] = in[0] - in[-stride];
-    PredictLineLeft(in + 1, out + 1, width - 1, inverse);
+    PredictLineLeft(in + 1, out + 1, width - 1);
     ++row;
-    preds += stride;
     in += stride;
     out += stride;
   }
@@ -153,34 +106,27 @@ static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in,
 
 static WEBP_INLINE void DoVerticalFilter(const uint8_t* in,
                                          int width, int height, int stride,
-                                         int row, int num_rows,
+                                         int row, int num_rows, uint8_t* out) {
-                                         int inverse, uint8_t* out) {
-  const uint8_t* preds;
   const size_t start_offset = row * stride;
   const int last_row = row + num_rows;
   SANITY_CHECK(in, out);
   in += start_offset;
   out += start_offset;
-  preds = inverse ? out : in;
 
   if (row == 0) {
     // Very first top-left pixel is copied.
     out[0] = in[0];
     // Rest of top scan-line is left-predicted.
-    PredictLineLeft(in + 1, out + 1, width - 1, inverse);
+    PredictLineLeft(in + 1, out + 1, width - 1);
     row = 1;
     in += stride;
     out += stride;
-  } else {
-    // We are starting from in-between. Make sure 'preds' points to prev row.
-    preds -= stride;
   }
 
   // Filter line-by-line.
   while (row < last_row) {
-    PredictLineTop(in, preds, out, width, inverse);
+    PredictLineTop(in, in - stride, out, width);
     ++row;
-    preds += stride;
     in += stride;
     out += stride;
   }
@@ -219,6 +165,101 @@ static void GradientPredictDirect(const uint8_t* const row,
   }
 }
 
+static WEBP_INLINE void DoGradientFilter(const uint8_t* in,
+                                         int width, int height, int stride,
+                                         int row, int num_rows,
+                                         uint8_t* out) {
+  const size_t start_offset = row * stride;
+  const int last_row = row + num_rows;
+  SANITY_CHECK(in, out);
+  in += start_offset;
+  out += start_offset;
+
+  // left prediction for top scan-line
+  if (row == 0) {
+    out[0] = in[0];
+    PredictLineLeft(in + 1, out + 1, width - 1);
+    row = 1;
+    in += stride;
+    out += stride;
+  }
+
+  // Filter line-by-line.
+  while (row < last_row) {
+    out[0] = in[0] - in[-stride];
+    GradientPredictDirect(in + 1, in + 1 - stride, out + 1, width - 1);
+    ++row;
+    in += stride;
+    out += stride;
+  }
+}
+
+#undef SANITY_CHECK
+
+//------------------------------------------------------------------------------
+
+static void HorizontalFilter(const uint8_t* data, int width, int height,
+                             int stride, uint8_t* filtered_data) {
+  DoHorizontalFilter(data, width, height, stride, 0, height, filtered_data);
+}
+
+static void VerticalFilter(const uint8_t* data, int width, int height,
+                           int stride, uint8_t* filtered_data) {
+  DoVerticalFilter(data, width, height, stride, 0, height, filtered_data);
+}
+
+static void GradientFilter(const uint8_t* data, int width, int height,
+                           int stride, uint8_t* filtered_data) {
+  DoGradientFilter(data, width, height, stride, 0, height, filtered_data);
+}
+
+//------------------------------------------------------------------------------
+// Inverse transforms
+
+static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in,
+                               uint8_t* out, int width) {
+  int i;
+  __m128i last;
+  out[0] = in[0] + (prev == NULL ? 0 : prev[0]);
+  if (width <= 1) return;
+  last = _mm_set_epi32(0, 0, 0, out[0]);
+  for (i = 1; i + 8 <= width; i += 8) {
+    const __m128i A0 = _mm_loadl_epi64((const __m128i*)(in + i));
+    const __m128i A1 = _mm_add_epi8(A0, last);
+    const __m128i A2 = _mm_slli_si128(A1, 1);
+    const __m128i A3 = _mm_add_epi8(A1, A2);
+    const __m128i A4 = _mm_slli_si128(A3, 2);
+    const __m128i A5 = _mm_add_epi8(A3, A4);
+    const __m128i A6 = _mm_slli_si128(A5, 4);
+    const __m128i A7 = _mm_add_epi8(A5, A6);
+    _mm_storel_epi64((__m128i*)(out + i), A7);
+    last = _mm_srli_epi64(A7, 56);
+  }
+  for (; i < width; ++i) out[i] = in[i] + out[i - 1];
+}
+
+static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in,
+                             uint8_t* out, int width) {
+  if (prev == NULL) {
+    HorizontalUnfilter(NULL, in, out, width);
+  } else {
+    int i;
+    const int max_pos = width & ~31;
+    assert(width >= 0);
+    for (i = 0; i < max_pos; i += 32) {
+      const __m128i A0 = _mm_loadu_si128((const __m128i*)&in[i +  0]);
+      const __m128i A1 = _mm_loadu_si128((const __m128i*)&in[i + 16]);
+      const __m128i B0 = _mm_loadu_si128((const __m128i*)&prev[i +  0]);
+      const __m128i B1 = _mm_loadu_si128((const __m128i*)&prev[i + 16]);
+      const __m128i C0 = _mm_add_epi8(A0, B0);
+      const __m128i C1 = _mm_add_epi8(A1, B1);
+      _mm_storeu_si128((__m128i*)&out[i +  0], C0);
+      _mm_storeu_si128((__m128i*)&out[i + 16], C1);
+    }
+    for (; i < width; ++i) out[i] = in[i] + prev[i];
+  }
+}
+
 static void GradientPredictInverse(const uint8_t* const in,
                                    const uint8_t* const top,
                                    uint8_t* const row, int length) {
@@ -232,25 +273,24 @@ static void GradientPredictInverse(const uint8_t* const in,
       const __m128i tmp1 = _mm_loadl_epi64((const __m128i*)&top[i - 1]);
       const __m128i B = _mm_unpacklo_epi8(tmp0, zero);
       const __m128i C = _mm_unpacklo_epi8(tmp1, zero);
-      const __m128i tmp2 = _mm_loadl_epi64((const __m128i*)&in[i]);
+      const __m128i D = _mm_loadl_epi64((const __m128i*)&in[i]);  // base input
-      const __m128i D = _mm_unpacklo_epi8(tmp2, zero);   // base input
       const __m128i E = _mm_sub_epi16(B, C);  // unclipped gradient basis B - C
       __m128i out = zero;                     // accumulator for output
       __m128i mask_hi = _mm_set_epi32(0, 0, 0, 0xff);
       int k = 8;
       while (1) {
-        const __m128i tmp3 = _mm_add_epi16(A, E);        // delta = A + B - C
+        const __m128i tmp3 = _mm_add_epi16(A, E);           // delta = A + B - C
-        const __m128i tmp4 = _mm_min_epi16(tmp3, mask_hi);
+        const __m128i tmp4 = _mm_packus_epi16(tmp3, zero);  // saturate delta
-        const __m128i tmp5 = _mm_max_epi16(tmp4, zero);  // clipped delta
+        const __m128i tmp5 = _mm_add_epi8(tmp4, D);         // add to in[]
-        const __m128i tmp6 = _mm_add_epi16(tmp5, D);     // add to in[] values
+        A = _mm_and_si128(tmp5, mask_hi);                   // 1-complement clip
-        A = _mm_and_si128(tmp6, mask_hi);                // 1-complement clip
+        out = _mm_or_si128(out, A);                         // accumulate output
-        out = _mm_or_si128(out, A);                      // accumulate output
         if (--k == 0) break;
-        A = _mm_slli_si128(A, 2);                        // rotate left sample
+        A = _mm_slli_si128(A, 1);                        // rotate left sample
-        mask_hi = _mm_slli_si128(mask_hi, 2);            // rotate mask
+        mask_hi = _mm_slli_si128(mask_hi, 1);            // rotate mask
+        A = _mm_unpacklo_epi8(A, zero);                  // convert 8b->16b
       }
-      A = _mm_srli_si128(A, 14);       // prepare left sample for next iteration
+      A = _mm_srli_si128(A, 7);       // prepare left sample for next iteration
-      _mm_storel_epi64((__m128i*)&row[i], _mm_packus_epi16(out, zero));
+      _mm_storel_epi64((__m128i*)&row[i], out);
     }
     for (; i < length; ++i) {
       row[i] = in[i] + GradientPredictorC(row[i - 1], top[i], top[i - 1]);
@@ -258,76 +298,14 @@ static void GradientPredictInverse(const uint8_t* const in,
   }
 }
 
-static WEBP_INLINE void DoGradientFilter(const uint8_t* in,
+static void GradientUnfilter(const uint8_t* prev, const uint8_t* in,
-                                         int width, int height, int stride,
+                             uint8_t* out, int width) {
-                                         int row, int num_rows,
+  if (prev == NULL) {
-                                         int inverse, uint8_t* out) {
+    HorizontalUnfilter(NULL, in, out, width);
-  const size_t start_offset = row * stride;
+  } else {
-  const int last_row = row + num_rows;
+    out[0] = in[0] + prev[0];  // predict from above
-  SANITY_CHECK(in, out);
+    GradientPredictInverse(in + 1, prev + 1, out + 1, width - 1);
-  in += start_offset;
-  out += start_offset;
-
-  // left prediction for top scan-line
-  if (row == 0) {
-    out[0] = in[0];
-    PredictLineLeft(in + 1, out + 1, width - 1, inverse);
-    row = 1;
-    in += stride;
-    out += stride;
   }
-
-  // Filter line-by-line.
-  while (row < last_row) {
-    if (inverse) {
-      PredictLineC(in, out - stride, out, 1, inverse);  // predict from above
-      GradientPredictInverse(in + 1, out + 1 - stride, out + 1, width - 1);
-    } else {
-      PredictLineC(in, in - stride, out, 1, inverse);
-      GradientPredictDirect(in + 1, in + 1 - stride, out + 1, width - 1);
-    }
-    ++row;
-    in += stride;
-    out += stride;
-  }
-}
-
-#undef SANITY_CHECK
-
-//------------------------------------------------------------------------------
-
-static void HorizontalFilter(const uint8_t* data, int width, int height,
-                             int stride, uint8_t* filtered_data) {
-  DoHorizontalFilter(data, width, height, stride, 0, height, 0, filtered_data);
-}
-
-static void VerticalFilter(const uint8_t* data, int width, int height,
-                           int stride, uint8_t* filtered_data) {
-  DoVerticalFilter(data, width, height, stride, 0, height, 0, filtered_data);
-}
-
-
-static void GradientFilter(const uint8_t* data, int width, int height,
-                           int stride, uint8_t* filtered_data) {
-  DoGradientFilter(data, width, height, stride, 0, height, 0, filtered_data);
-}
-
-
-//------------------------------------------------------------------------------
-
-static void VerticalUnfilter(int width, int height, int stride, int row,
-                             int num_rows, uint8_t* data) {
-  DoVerticalFilter(data, width, height, stride, row, num_rows, 1, data);
-}
-
-static void HorizontalUnfilter(int width, int height, int stride, int row,
-                               int num_rows, uint8_t* data) {
-  DoHorizontalFilter(data, width, height, stride, row, num_rows, 1, data);
-}
-
-static void GradientUnfilter(int width, int height, int stride, int row,
-                             int num_rows, uint8_t* data) {
-  DoGradientFilter(data, width, height, stride, row, num_rows, 1, data);
 }
 
 //------------------------------------------------------------------------------