perform alpha filtering within the decoding loop

Change-Id: I221ae20ba879cd4f7b48d012db1bc5443d968e17

src/dec/alpha.c

@@ -52,6 +52,7 @@ static int ALPHInit(ALPHDecoder* const dec, const uint8_t* data,
   const uint8_t* const alpha_data = data + ALPHA_HEADER_LEN;
   const size_t alpha_data_size = data_size - ALPHA_HEADER_LEN;
   int rsrv;
+  int filter;
   VP8Io* const io = &dec->io_;
 
   assert(data != NULL && output != NULL && src_io != NULL);
@@ -65,30 +66,25 @@ static int ALPHInit(ALPHDecoder* const dec, const uint8_t* data,
   }
 
   dec->method_ = (data[0] >> 0) & 0x03;
-  dec->filter_ = (data[0] >> 2) & 0x03;
+  filter = (data[0] >> 2) & 0x03;
   dec->pre_processing_ = (data[0] >> 4) & 0x03;
   rsrv = (data[0] >> 6) & 0x03;
   if (dec->method_ < ALPHA_NO_COMPRESSION ||
       dec->method_ > ALPHA_LOSSLESS_COMPRESSION ||
-      dec->filter_ >= WEBP_FILTER_LAST ||
+      filter >= WEBP_FILTER_LAST ||
       dec->pre_processing_ > ALPHA_PREPROCESSED_LEVELS ||
       rsrv != 0) {
     return 0;
   }
 
-  if (dec->method_ == ALPHA_NO_COMPRESSION) {
-    const size_t alpha_decoded_size = dec->width_ * dec->height_;
-    ok = (alpha_data_size >= alpha_decoded_size);
-  } else {
-    assert(dec->method_ == ALPHA_LOSSLESS_COMPRESSION);
-    ok = VP8LDecodeAlphaHeader(dec, alpha_data, alpha_data_size, output);
-  }
   VP8FiltersInit();
+  dec->unfilter_func_ = WebPUnfilters[filter];
+  dec->output_ = output;
 
   // Copy the necessary parameters from src_io to io
   VP8InitIo(io);
   WebPInitCustomIo(NULL, io);
-  io->opaque = output;      // output plane
+  io->opaque = dec;
   io->width = src_io->width;
   io->height = src_io->height;
 
@@ -99,6 +95,14 @@ static int ALPHInit(ALPHDecoder* const dec, const uint8_t* data,
   io->crop_bottom = src_io->crop_bottom;
   // No need to copy the scaling parameters.
 
+  if (dec->method_ == ALPHA_NO_COMPRESSION) {
+    const size_t alpha_decoded_size = dec->width_ * dec->height_;
+    ok = (alpha_data_size >= alpha_decoded_size);
+  } else {
+    assert(dec->method_ == ALPHA_LOSSLESS_COMPRESSION);
+    ok = VP8LDecodeAlphaHeader(dec, alpha_data, alpha_data_size);
+  }
+
   return ok;
 }
 
@@ -110,7 +114,6 @@ static int ALPHDecode(VP8Decoder* const dec, int row, int num_rows) {
   ALPHDecoder* const alph_dec = dec->alph_dec_;
   const int width = alph_dec->width_;
   const int height = alph_dec->height_;
-  WebPUnfilterFunc unfilter_func = WebPUnfilters[alph_dec->filter_];
   uint8_t* const output = dec->alpha_plane_;
   if (alph_dec->method_ == ALPHA_NO_COMPRESSION) {
     const size_t offset = row * width;
@@ -118,6 +121,9 @@ static int ALPHDecode(VP8Decoder* const dec, int row, int num_rows) {
     assert(dec->alpha_data_size_ >= ALPHA_HEADER_LEN + offset + num_pixels);
     memcpy(dec->alpha_plane_ + offset,
            dec->alpha_data_ + ALPHA_HEADER_LEN + offset, num_pixels);
+    if (alph_dec->unfilter_func_ != NULL) {
+      alph_dec->unfilter_func_(width, height, width, row, num_rows, output);
+    }
   } else {  // alph_dec->method_ == ALPHA_LOSSLESS_COMPRESSION
     assert(alph_dec->vp8l_dec_ != NULL);
     if (!VP8LDecodeAlphaImageStream(alph_dec, row + num_rows)) {
@@ -125,10 +131,6 @@ static int ALPHDecode(VP8Decoder* const dec, int row, int num_rows) {
     }
   }
 
-  if (unfilter_func != NULL) {
-    unfilter_func(width, height, width, row, num_rows, output);
-  }
-
   if (row + num_rows >= alph_dec->io_.crop_bottom) {
     dec->is_alpha_decoded_ = 1;
   }

src/dec/alphai.h

@@ -28,13 +28,14 @@ struct ALPHDecoder {
   int width_;
   int height_;
   int method_;
-  WEBP_FILTER_TYPE filter_;
+  WebPUnfilterFunc unfilter_func_;
   int pre_processing_;
   struct VP8LDecoder* vp8l_dec_;
   VP8Io io_;
   int use_8b_decode_;  // Although alpha channel requires only 1 byte per
                        // pixel, sometimes VP8LDecoder may need to allocate
                        // 4 bytes per pixel internally during decode.
+  uint8_t* output_;
 };
 
 //------------------------------------------------------------------------------

src/dec/vp8l.c

@@ -717,15 +717,22 @@ static void ApplyInverseTransforms(VP8LDecoder* const dec, int num_rows,
 // Special method for paletted alpha data.
 static void ApplyInverseTransformsAlpha(VP8LDecoder* const dec, int num_rows,
                                         const uint8_t* const rows) {
+  const ALPHDecoder* const alph_dec = (const ALPHDecoder*)dec->io_->opaque;
+  const int width = dec->io_->width;
+  uint8_t* const output = alph_dec->output_;
   const int start_row = dec->last_row_;
   const int end_row = start_row + num_rows;
   const uint8_t* rows_in = rows;
-  uint8_t* rows_out = (uint8_t*)dec->io_->opaque + dec->io_->width * start_row;
+  uint8_t* rows_out = output +  width * start_row;
   VP8LTransform* const transform = &dec->transforms_[0];
   assert(dec->next_transform_ == 1);
   assert(transform->type_ == COLOR_INDEXING_TRANSFORM);
   VP8LColorIndexInverseTransformAlpha(transform, start_row, end_row, rows_in,
                                       rows_out);
+  if (alph_dec->unfilter_func_ != NULL) {
+    alph_dec->unfilter_func_(width, dec->io_->height, width,
+                             start_row, num_rows, output);
+  }
 }
 
 // Processes (transforms, scales & color-converts) the rows decoded after the
@@ -1453,38 +1460,39 @@ static void ExtractAlphaRows(VP8LDecoder* const dec, int row) {
   assert(row <= dec->io_->crop_bottom);
   if (num_rows > 0) {
     // Extract alpha (which is stored in the green plane).
+    const ALPHDecoder* const alph_dec = (const ALPHDecoder*)dec->io_->opaque;
+    uint8_t* const output = alph_dec->output_;
     const int width = dec->io_->width;      // the final width (!= dec->width_)
     const int cache_pixs = width * num_rows;
-    uint8_t* const dst = (uint8_t*)dec->io_->opaque + width * dec->last_row_;
+    uint8_t* const dst = output + width * dec->last_row_;
     const uint32_t* const src = dec->argb_cache_;
     int i;
     ApplyInverseTransforms(dec, num_rows, in);
     for (i = 0; i < cache_pixs; ++i) dst[i] = (src[i] >> 8) & 0xff;
+    if (alph_dec->unfilter_func_ != NULL) {
+      alph_dec->unfilter_func_(width, dec->io_->height, width,
+                               dec->last_row_, num_rows, output);
+    }
   }
   dec->last_row_ = dec->last_out_row_ = row;
 }
 
 int VP8LDecodeAlphaHeader(ALPHDecoder* const alph_dec,
-                          const uint8_t* const data, size_t data_size,
+                          const uint8_t* const data, size_t data_size) {
-                          uint8_t* const output) {
   int ok = 0;
-  VP8LDecoder* dec;
+  VP8LDecoder* dec = VP8LNew();
-  VP8Io* io;
+
+  if (dec == NULL) return 0;
+
   assert(alph_dec != NULL);
-  alph_dec->vp8l_dec_ = VP8LNew();
+  alph_dec->vp8l_dec_ = dec;
-  if (alph_dec->vp8l_dec_ == NULL) return 0;
-  dec = alph_dec->vp8l_dec_;
 
   dec->width_ = alph_dec->width_;
   dec->height_ = alph_dec->height_;
   dec->io_ = &alph_dec->io_;
-  io = dec->io_;
+  dec->io_->opaque = alph_dec;
-
+  dec->io_->width = alph_dec->width_;
-  VP8InitIo(io);
+  dec->io_->height = alph_dec->height_;
-  WebPInitCustomIo(NULL, io);  // Just a sanity Init. io won't be used.
-  io->opaque = output;
-  io->width = alph_dec->width_;
-  io->height = alph_dec->height_;
 
   dec->status_ = VP8_STATUS_OK;
   VP8LInitBitReader(&dec->br_, data, data_size);

src/dec/vp8li.h

@@ -100,8 +100,7 @@ struct ALPHDecoder;  // Defined in dec/alphai.h.
 // Decodes image header for alpha data stored using lossless compression.
 // Returns false in case of error.
 int VP8LDecodeAlphaHeader(struct ALPHDecoder* const alph_dec,
-                          const uint8_t* const data, size_t data_size,
+                          const uint8_t* const data, size_t data_size);
-                          uint8_t* const output);
 
 // Decodes *at least* 'last_row' rows of alpha. If some of the initial rows are
 // already decoded in previous call(s), it will resume decoding from where it