prevent 32b overflow for very large canvas_width / height

some multiplies here and there needed some extra checks
and error reporting. Even if width * height is guaranteed
to be < 2**32, we were multiplying by num_channels and
triggering a 32b overflow.
Some multiplies were not using size_t or uint64_t, additionally.

Change-Id: If2a35b94c8af204135f4b88a7fd63850aa381bbf
(cherry picked from commit 1c36440094)

examples/anim_util.c

@@ -39,13 +39,24 @@ static int IsFullFrame(int width, int height,
   return (width == canvas_width && height == canvas_height);
 }
 
+static int CheckSizeForOverflow(uint64_t size) {
+  return (size == (size_t)size);
+}
+
 static int AllocateFrames(AnimatedImage* const image, uint32_t num_frames) {
   uint32_t i;
-  const size_t rgba_size =
+  uint8_t* mem = NULL;
-      image->canvas_width * kNumChannels * image->canvas_height;
+  DecodedFrame* frames = NULL;
-  uint8_t* const mem = (uint8_t*)malloc(num_frames * rgba_size * sizeof(*mem));
+  const uint64_t rgba_size =
-  DecodedFrame* const frames =
+      (uint64_t)image->canvas_width * kNumChannels * image->canvas_height;
-      (DecodedFrame*)malloc(num_frames * sizeof(*frames));
+  const uint64_t total_size = (uint64_t)num_frames * rgba_size * sizeof(*mem);
+  const uint64_t total_frame_size = (uint64_t)num_frames * sizeof(*frames);
+  if (!CheckSizeForOverflow(total_size) ||
+      !CheckSizeForOverflow(total_frame_size)) {
+    return 0;
+  }
+  mem = (uint8_t*)malloc(total_size);
+  frames = (DecodedFrame*)malloc(total_frame_size);
 
   if (mem == NULL || frames == NULL) {
     free(mem);

src/demux/anim_decode.c

@@ -112,18 +112,15 @@ WebPAnimDecoder* WebPAnimDecoderNewInternal(
   dec->info_.bgcolor = WebPDemuxGetI(dec->demux_, WEBP_FF_BACKGROUND_COLOR);
   dec->info_.frame_count = WebPDemuxGetI(dec->demux_, WEBP_FF_FRAME_COUNT);
 
-  {
+  // Note: calloc() because we fill frame with zeroes as well.
-    const int canvas_bytes =
+  dec->curr_frame_ = WebPSafeCalloc(
-        dec->info_.canvas_width * NUM_CHANNELS * dec->info_.canvas_height;
+      dec->info_.canvas_width * NUM_CHANNELS, dec->info_.canvas_height);
-    // Note: calloc() because we fill frame with zeroes as well.
+  if (dec->curr_frame_ == NULL) goto Error;
-    dec->curr_frame_ = WebPSafeCalloc(1ULL, canvas_bytes);
+  dec->prev_frame_disposed_ = WebPSafeCalloc(
-    if (dec->curr_frame_ == NULL) goto Error;
+      dec->info_.canvas_width * NUM_CHANNELS, dec->info_.canvas_height);
-    dec->prev_frame_disposed_ = WebPSafeCalloc(1ULL, canvas_bytes);
+  if (dec->prev_frame_disposed_ == NULL) goto Error;
-    if (dec->prev_frame_disposed_ == NULL) goto Error;
-  }
 
   WebPAnimDecoderReset(dec);
-
   return dec;
 
  Error:
@@ -144,9 +141,13 @@ static int IsFullFrame(int width, int height, int canvas_width,
 }
 
 // Clear the canvas to transparent.
-static void ZeroFillCanvas(uint8_t* buf, uint32_t canvas_width,
+static int ZeroFillCanvas(uint8_t* buf, uint32_t canvas_width,
-                           uint32_t canvas_height) {
+                          uint32_t canvas_height) {
-  memset(buf, 0, canvas_width * NUM_CHANNELS * canvas_height);
+  const uint64_t size =
+      (uint64_t)canvas_width * canvas_height * NUM_CHANNELS * sizeof(*buf);
+  if (size != (size_t)size) return 0;
+  memset(buf, 0, (size_t)size);
+  return 1;
 }
 
 // Clear given frame rectangle to transparent.
@@ -162,10 +163,13 @@ static void ZeroFillFrameRect(uint8_t* buf, int buf_stride, int x_offset,
 }
 
 // Copy width * height pixels from 'src' to 'dst'.
-static void CopyCanvas(const uint8_t* src, uint8_t* dst,
+static int CopyCanvas(const uint8_t* src, uint8_t* dst,
-                       uint32_t width, uint32_t height) {
+                      uint32_t width, uint32_t height) {
+  const uint64_t size = (uint64_t)width * height * NUM_CHANNELS;
+  if (size != (size_t)size) return 0;
   assert(src != NULL && dst != NULL);
-  memcpy(dst, src, width * NUM_CHANNELS * height);
+  memcpy(dst, src, (size_t)size);
+  return 1;
 }
 
 // Returns true if the current frame is a key-frame.
@@ -328,9 +332,14 @@ int WebPAnimDecoderGetNext(WebPAnimDecoder* dec,
   is_key_frame = IsKeyFrame(&iter, &dec->prev_iter_,
                             dec->prev_frame_was_keyframe_, width, height);
   if (is_key_frame) {
-    ZeroFillCanvas(dec->curr_frame_, width, height);
+    if (!ZeroFillCanvas(dec->curr_frame_, width, height)) {
+      goto Error;
+    }
   } else {
-    CopyCanvas(dec->prev_frame_disposed_, dec->curr_frame_, width, height);
+    if (!CopyCanvas(dec->prev_frame_disposed_, dec->curr_frame_,
+                    width, height)) {
+      goto Error;
+    }
   }
 
   // Decode.