From 2f5e8986cff7da5217de4409965de4d591860e1d Mon Sep 17 00:00:00 2001
From: Pascal Massimino <pascal.massimino@gmail.com>
Date: Wed, 17 Feb 2016 05:45:38 +0100
Subject: [PATCH] fix multiple allocation for transform buffer

We were not updating the current_width_, which is usually
not a problem, unless we use Delta Palette with small number
of colors
-> Addressed this re-entrancy problem by checking we have
enough capacity for transform buffer.

The problem is not currently visible, until we restrict
the number of gradient used in delta-palette to less than 16.
Then the buffers have different current_width_ and the problem
surfaces.

Change-Id: Icd84b919905d7789014bb6668bfb6813c93fb36e
---
 src/enc/vp8l.c  | 24 +++++++++++++++---------
 src/enc/vp8li.h |  3 +++
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/enc/vp8l.c b/src/enc/vp8l.c
index db94e78a..29a5a555 100644
--- a/src/enc/vp8l.c
+++ b/src/enc/vp8l.c
@@ -1114,6 +1114,12 @@ static WebPEncodingError WriteImage(const WebPPicture* const pic,
 
 // -----------------------------------------------------------------------------
 
+static void ClearTransformBuffer(VP8LEncoder* const enc) {
+  WebPSafeFree(enc->transform_mem_);
+  enc->transform_mem_ = NULL;
+  enc->transform_mem_size_ = 0;
+}
+
 // Allocates the memory for argb (W x H) buffer, 2 rows of context for
 // prediction and transform data.
 // Flags influencing the memory allocated:
@@ -1122,7 +1128,6 @@ static WebPEncodingError WriteImage(const WebPPicture* const pic,
 static WebPEncodingError AllocateTransformBuffer(VP8LEncoder* const enc,
                                                  int width, int height) {
   WebPEncodingError err = VP8_ENC_OK;
-  if (enc->argb_ == NULL) {
     const int tile_size = 1 << enc->transform_bits_;
     const uint64_t image_size = width * height;
     // Ensure enough size for tiles, as well as for two scanlines and two
@@ -1134,31 +1139,32 @@ static WebPEncodingError AllocateTransformBuffer(VP8LEncoder* const enc,
             ? VP8LSubSampleSize(width, enc->transform_bits_) *
               VP8LSubSampleSize(height, enc->transform_bits_)
             : 0;
-    const uint64_t total_size =
+  const uint64_t mem_size =
         image_size + WEBP_ALIGN_CST +
         argb_scratch_size + WEBP_ALIGN_CST +
         (uint64_t)transform_data_size;
-    uint32_t* mem = (uint32_t*)WebPSafeMalloc(total_size, sizeof(*mem));
+  uint32_t* mem = enc->transform_mem_;
+  if (mem == NULL || mem_size > enc->transform_mem_size_) {
+    ClearTransformBuffer(enc);
+    mem = (uint32_t*)WebPSafeMalloc(mem_size, sizeof(*mem));
     if (mem == NULL) {
       err = VP8_ENC_ERROR_OUT_OF_MEMORY;
       goto Error;
     }
+    enc->transform_mem_ = mem;
+    enc->transform_mem_size_ = (size_t)mem_size;
+  }
     enc->argb_ = mem;
     mem = (uint32_t*)WEBP_ALIGN(mem + image_size);
     enc->argb_scratch_ = mem;
     mem = (uint32_t*)WEBP_ALIGN(mem + argb_scratch_size);
     enc->transform_data_ = mem;
+
     enc->current_width_ = width;
-  }
  Error:
   return err;
 }
 
-static void ClearTransformBuffer(VP8LEncoder* const enc) {
-  WebPSafeFree(enc->argb_);
-  enc->argb_ = NULL;
-}
-
 static WebPEncodingError MakeInputImageCopy(VP8LEncoder* const enc) {
   WebPEncodingError err = VP8_ENC_OK;
   const WebPPicture* const picture = enc->pic_;
diff --git a/src/enc/vp8li.h b/src/enc/vp8li.h
index 6b6db127..7443238f 100644
--- a/src/enc/vp8li.h
+++ b/src/enc/vp8li.h
@@ -32,6 +32,9 @@ typedef struct {
   uint32_t* argb_scratch_;      // Scratch memory for argb rows
                                 // (used for prediction).
   uint32_t* transform_data_;    // Scratch memory for transform data.
+  uint32_t* transform_mem_;       // Currently allocated memory.
+  size_t    transform_mem_size_;  // Currently allocated memory size.
+
   int       current_width_;     // Corresponds to packed image width.
 
   // Encoding parameters derived from quality parameter.