add overflow check before calling malloc()

It's preferable to be over-strict and paranoid here.

Change-Id: Ia928b279b753fa8f836d17decb17a35e6dc441b3

src/dec/frame.c

@@ -409,6 +409,7 @@ static int InitThreadContext(VP8Decoder* const dec) {
 static int AllocateMemory(VP8Decoder* const dec) {
   const int num_caches = dec->num_caches_;
   const int mb_w = dec->mb_w_;
+  // Note: we use 'size_t' when there's no overflow risk, uint64_t otherwise.
   const size_t intra_pred_mode_size = 4 * mb_w * sizeof(uint8_t);
   const size_t top_size = (16 + 8 + 8) * mb_w;
   const size_t mb_info_size = (mb_w + 1) * sizeof(VP8MB);
@@ -421,24 +422,25 @@ static int AllocateMemory(VP8Decoder* const dec) {
   const size_t cache_height = (16 * num_caches
                             + kFilterExtraRows[dec->filter_type_]) * 3 / 2;
   const size_t cache_size = top_size * cache_height;
-  const size_t alpha_size =
-      (dec->alpha_data_ != NULL) ? dec->pic_hdr_.width_ * dec->pic_hdr_.height_
-                                 : 0;
-  const size_t needed = intra_pred_mode_size
-                      + top_size + mb_info_size + f_info_size
-                      + yuv_size + coeffs_size
-                      + cache_size + alpha_size + ALIGN_MASK;
+  // alpha_size is the only one that scales as width x height.
+  const uint64_t alpha_size = (dec->alpha_data_ != NULL) ?
+      (uint64_t)dec->pic_hdr_.width_ * dec->pic_hdr_.height_ : 0ULL;
+  const uint64_t needed = (uint64_t)intra_pred_mode_size
+                        + top_size + mb_info_size + f_info_size
+                        + yuv_size + coeffs_size
+                        + cache_size + alpha_size + ALIGN_MASK;
   uint8_t* mem;
 
+  if (needed != (size_t)needed) return 0;  // check for overflow
   if (needed > dec->mem_size_) {
     free(dec->mem_);
     dec->mem_size_ = 0;
-    dec->mem_ = (uint8_t*)malloc(needed);
+    dec->mem_ = (uint8_t*)malloc((size_t)needed);
     if (dec->mem_ == NULL) {
       return VP8SetError(dec, VP8_STATUS_OUT_OF_MEMORY,
                          "no memory during frame initialization.");
     }
-    dec->mem_size_ = needed;
+    dec->mem_size_ = (size_t)needed;
   }
 
   mem = (uint8_t*)dec->mem_;

src/dec/idec.c

@@ -135,6 +135,11 @@ static int AppendToMemBuffer(WebPIDecoder* const idec,
   MemBuffer* const mem = &idec->mem_;
   const uint8_t* const old_base = mem->buf_ + mem->start_;
   assert(mem->mode_ == MEM_MODE_APPEND);
+  if (data_size > MAX_CHUNK_PAYLOAD) {
+    // security safeguard: trying to allocate more than what the format
+    // allows for a chunk should be considered a smoke smell.
+    return 0;
+  }
 
   if (mem->end_ + data_size > mem->buf_size_) {  // Need some free memory
     const size_t current_size = MemDataSize(mem);
@@ -327,9 +332,9 @@ static int CopyParts0Data(WebPIDecoder* const idec) {
   const size_t psize = br->buf_end_ - br->buf_;
   MemBuffer* const mem = &idec->mem_;
   assert(!idec->is_lossless_);
-  assert(!mem->part0_buf_);
+  assert(mem->part0_buf_ == NULL);
   assert(psize > 0);
-  assert(psize <= mem->part0_size_);
+  assert(psize <= mem->part0_size_);  // Format limit: no need for runtime check
   if (mem->mode_ == MEM_MODE_APPEND) {
     // We copy and grab ownership of the partition #0 data.
     uint8_t* const part0_buf = (uint8_t*)malloc(psize);

src/dec/vp8l.c

@@ -334,6 +334,7 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize,
 
   if (br->error_) goto Error;
 
+  assert(num_htree_groups <= 0x10000);
   htree_groups = (HTreeGroup*)calloc(num_htree_groups, sizeof(*htree_groups));
   if (htree_groups == NULL) {
     dec->status_ = VP8_STATUS_OUT_OF_MEMORY;
@@ -372,14 +373,17 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) {
   const int out_width = io->scaled_width;
   const int in_height = io->mb_h;
   const int out_height = io->scaled_height;
-  const size_t work_size = 2 * num_channels * out_width;
+  const uint64_t work_size = 2 * num_channels * (uint64_t)out_width;
   int32_t* work;        // Rescaler work area.
-  const size_t scaled_data_size = num_channels * out_width;
+  const uint64_t scaled_data_size = num_channels * (uint64_t)out_width;
   uint32_t* scaled_data;  // Temporary storage for scaled BGRA data.
-  const size_t memory_size = sizeof(*dec->rescaler) +
-                             work_size * sizeof(*work) +
-                             scaled_data_size * sizeof(*scaled_data);
-  uint8_t* memory = calloc(1, memory_size);
+  const uint64_t memory_size = sizeof(*dec->rescaler) +
+                               work_size * sizeof(*work) +
+                               scaled_data_size * sizeof(*scaled_data);
+  uint8_t* memory;
+
+  if (memory_size != (size_t)memory_size) return 0;  // overflow check
+  memory = (uint8_t*)calloc(1, (size_t)memory_size);
   if (memory == NULL) {
     dec->status_ = VP8_STATUS_OUT_OF_MEMORY;
     return 0;
@@ -928,16 +932,20 @@ static int DecodeImageStream(int xsize, int ysize,
 // Allocate dec->argb_ and dec->argb_cache_ using dec->width_ and dec->height_
 
 static int AllocateARGBBuffers(VP8LDecoder* const dec, int final_width) {
-  const int num_pixels = dec->width_ * dec->height_;
+  const uint64_t num_pixels = (uint64_t)dec->width_ * dec->height_;
   // Scratch buffer corresponding to top-prediction row for transforming the
   // first row in the row-blocks.
-  const int cache_top_pixels = final_width;
+  const uint64_t cache_top_pixels = final_width;
   // Scratch buffer for temporary BGRA storage.
-  const int cache_pixels = final_width * NUM_ARGB_CACHE_ROWS;
-  const int total_num_pixels = num_pixels + cache_top_pixels + cache_pixels;
+  const uint64_t cache_pixels = (uint64_t)final_width * NUM_ARGB_CACHE_ROWS;
+  const uint64_t total_num_pixels =
+      num_pixels + cache_top_pixels + cache_pixels;
+  const uint64_t total_size = total_num_pixels * sizeof(*dec->argb_);
 
   assert(dec->width_ <= final_width);
-  dec->argb_ = (uint32_t*)malloc(total_num_pixels * sizeof(*dec->argb_));
+  // Check for overflow
+  if ((size_t)total_size != total_size) return 0;
+  dec->argb_ = (uint32_t*)malloc((size_t)total_size);
   if (dec->argb_ == NULL) {
     dec->argb_cache_ = NULL;
     dec->status_ = VP8_STATUS_OUT_OF_MEMORY;