harness some malloc/calloc to use WebPSafeMalloc and WebPSafeCalloc

quite a large security sweep.

Change-Id: If150dfbb46e6e9b56210473a109c8ad6ccd0cea4

src/dec/buffer.c

@@ -13,6 +13,7 @@
 
 #include "./vp8i.h"
 #include "./webpi.h"
+#include "../utils/utils.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -95,14 +96,11 @@ static VP8StatusCode AllocateBuffer(WebPDecBuffer* const buffer) {
     total_size = size + 2 * uv_size + a_size;
 
     // Security/sanity checks
-    if (((size_t)total_size != total_size) || (total_size >= (1ULL << 40))) {
+    output = (uint8_t*)WebPSafeMalloc(total_size, sizeof(*output));
-      return VP8_STATUS_INVALID_PARAM;
-    }
-
-    buffer->private_memory = output = (uint8_t*)malloc((size_t)total_size);
     if (output == NULL) {
       return VP8_STATUS_OUT_OF_MEMORY;
     }
+    buffer->private_memory = output;
 
     if (!WebPIsRGBMode(mode)) {   // YUVA initialization
       WebPYUVABuffer* const buf = &buffer->u.YUVA;

src/dec/frame.c

@@ -11,6 +11,7 @@
 
 #include <stdlib.h>
 #include "./vp8i.h"
+#include "../utils/utils.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -435,11 +436,12 @@ static int AllocateMemory(VP8Decoder* const dec) {
   if (needed > dec->mem_size_) {
     free(dec->mem_);
     dec->mem_size_ = 0;
-    dec->mem_ = (uint8_t*)malloc((size_t)needed);
+    dec->mem_ = WebPSafeMalloc(needed, sizeof(uint8_t));
     if (dec->mem_ == NULL) {
       return VP8SetError(dec, VP8_STATUS_OUT_OF_MEMORY,
                          "no memory during frame initialization.");
     }
+    // down-cast is ok, thanks to WebPSafeAlloc() above.
     dec->mem_size_ = (size_t)needed;
   }
 

src/dec/idec.c

@@ -15,6 +15,7 @@
 
 #include "./webpi.h"
 #include "./vp8i.h"
+#include "../utils/utils.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -143,14 +144,15 @@ static int AppendToMemBuffer(WebPIDecoder* const idec,
 
   if (mem->end_ + data_size > mem->buf_size_) {  // Need some free memory
     const size_t current_size = MemDataSize(mem);
-    const size_t new_size = current_size + data_size;
+    const uint64_t new_size = (uint64_t)current_size + data_size;
-    const size_t extra_size = (new_size + CHUNK_SIZE - 1) & ~(CHUNK_SIZE - 1);
+    const uint64_t extra_size = (new_size + CHUNK_SIZE - 1) & ~(CHUNK_SIZE - 1);
-    uint8_t* const new_buf = (uint8_t*)malloc(extra_size);
+    uint8_t* const new_buf =
+        (uint8_t*)WebPSafeMalloc(extra_size, sizeof(*new_buf));
     if (new_buf == NULL) return 0;
     memcpy(new_buf, old_base, current_size);
     free(mem->buf_);
     mem->buf_ = new_buf;
-    mem->buf_size_ = extra_size;
+    mem->buf_size_ = (size_t)extra_size;
     mem->start_ = 0;
     mem->end_ = current_size;
   }
@@ -534,7 +536,7 @@ static VP8StatusCode IDecode(WebPIDecoder* idec) {
 // Public functions
 
 WebPIDecoder* WebPINewDecoder(WebPDecBuffer* output_buffer) {
-  WebPIDecoder* idec = (WebPIDecoder*)calloc(1, sizeof(WebPIDecoder));
+  WebPIDecoder* idec = (WebPIDecoder*)calloc(1, sizeof(*idec));
   if (idec == NULL) {
     return NULL;
   }

src/dec/vp8.c

@@ -45,7 +45,7 @@ int VP8InitIoInternal(VP8Io* const io, int version) {
 }
 
 VP8Decoder* VP8New(void) {
-  VP8Decoder* const dec = (VP8Decoder*)calloc(1, sizeof(VP8Decoder));
+  VP8Decoder* const dec = (VP8Decoder*)calloc(1, sizeof(*dec));
   if (dec != NULL) {
     SetOk(dec);
     WebPWorkerInit(&dec->worker_);

src/dec/vp8l.c

@@ -15,6 +15,7 @@
 #include "./vp8li.h"
 #include "../dsp/lossless.h"
 #include "../utils/huffman.h"
+#include "../utils/utils.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -264,7 +265,8 @@ static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec,
       return 0;
     }
 
-    code_lengths = (int*)calloc(alphabet_size, sizeof(*code_lengths));
+    code_lengths =
+        (int*)WebPSafeCalloc((uint64_t)alphabet_size, sizeof(*code_lengths));
     if (code_lengths == NULL) {
       dec->status_ = VP8_STATUS_OUT_OF_MEMORY;
       return 0;
@@ -335,7 +337,9 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize,
   if (br->error_) goto Error;
 
   assert(num_htree_groups <= 0x10000);
-  htree_groups = (HTreeGroup*)calloc(num_htree_groups, sizeof(*htree_groups));
+  htree_groups =
+      (HTreeGroup*)WebPSafeCalloc((uint64_t)num_htree_groups,
+                                  sizeof(*htree_groups));
   if (htree_groups == NULL) {
     dec->status_ = VP8_STATUS_OUT_OF_MEMORY;
     goto Error;
@@ -380,10 +384,7 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) {
   const uint64_t memory_size = sizeof(*dec->rescaler) +
                                work_size * sizeof(*work) +
                                scaled_data_size * sizeof(*scaled_data);
-  uint8_t* memory;
+  uint8_t* memory = (uint8_t*)WebPSafeCalloc(memory_size, sizeof(*memory));
-
-  if (memory_size != (size_t)memory_size) return 0;  // overflow check
-  memory = (uint8_t*)calloc(1, (size_t)memory_size);
   if (memory == NULL) {
     dec->status_ = VP8_STATUS_OUT_OF_MEMORY;
     return 0;
@@ -700,7 +701,8 @@ static int ExpandColorMap(int num_colors, VP8LTransform* const transform) {
   int i;
   const int final_num_colors = 1 << (8 >> transform->bits_);
   uint32_t* const new_color_map =
-      (uint32_t*)malloc(final_num_colors * sizeof(*new_color_map));
+      (uint32_t*)WebPSafeMalloc((uint64_t)final_num_colors,
+                                sizeof(*new_color_map));
   if (new_color_map == NULL) {
     return 0;
   } else {
@@ -892,15 +894,8 @@ static int DecodeImageStream(int xsize, int ysize,
   }
 
   {
-    const uint64_t total_size =
+    const uint64_t total_size = (uint64_t)transform_xsize * transform_ysize;
-        transform_xsize * transform_ysize * sizeof(*data);
+    data = (uint32_t*)WebPSafeMalloc(total_size, sizeof(*data));
-    if (total_size != (size_t)total_size) {
-      // This shouldn't happen, because of transform_bits limit, but...
-      dec->status_ = VP8_STATUS_BITSTREAM_ERROR;
-      ok = 0;
-      goto End;
-    }
-    data = (uint32_t*)malloc((size_t)total_size);
     if (data == NULL) {
       dec->status_ = VP8_STATUS_OUT_OF_MEMORY;
       ok = 0;
@@ -951,12 +946,9 @@ static int AllocateARGBBuffers(VP8LDecoder* const dec, int final_width) {
   const uint64_t cache_pixels = (uint64_t)final_width * NUM_ARGB_CACHE_ROWS;
   const uint64_t total_num_pixels =
       num_pixels + cache_top_pixels + cache_pixels;
-  const uint64_t total_size = total_num_pixels * sizeof(*dec->argb_);
 
   assert(dec->width_ <= final_width);
-  // Check for overflow
+  dec->argb_ = (uint32_t*)WebPSafeMalloc(total_num_pixels, sizeof(*dec->argb_));
-  if ((size_t)total_size != total_size) return 0;
-  dec->argb_ = (uint32_t*)malloc((size_t)total_size);
   if (dec->argb_ == NULL) {
     dec->argb_cache_ = NULL;
     dec->status_ = VP8_STATUS_OUT_OF_MEMORY;

src/enc/alpha.c

@@ -111,6 +111,7 @@ static int EncodeAlphaInternal(const uint8_t* const data, int width, int height,
   size_t expected_size;
   const size_t data_size = width * height;
 
+  assert((uint64_t)data_size == (uint64_t)width * height);  // as per spec
   assert(filter >= 0 && filter < WEBP_FILTER_LAST);
   assert(method >= ALPHA_NO_COMPRESSION);
   assert(method <= ALPHA_LOSSLESS_COMPRESSION);
@@ -171,6 +172,7 @@ static int EncodeAlpha(VP8Encoder* const enc,
   const int reduce_levels = (quality < 100);
 
   // quick sanity checks
+  assert((uint64_t)data_size == (uint64_t)width * height);  // as per spec
   assert(enc != NULL && pic != NULL && pic->a != NULL);
   assert(output != NULL && output_size != NULL);
   assert(width > 0 && height > 0);

src/enc/analysis.c

@@ -15,6 +15,7 @@
 
 #include "./vp8enci.h"
 #include "./cost.h"
+#include "../utils/utils.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -35,7 +36,8 @@ static void SmoothSegmentMap(VP8Encoder* const enc) {
   const int w = enc->mb_w_;
   const int h = enc->mb_h_;
   const int majority_cnt_3_x_3_grid = 5;
-  uint8_t* const tmp = (uint8_t*)malloc(w * h * sizeof(uint8_t));
+  uint8_t* const tmp = (uint8_t*)WebPSafeMalloc((uint64_t)w * h, sizeof(*tmp));
+  assert((uint64_t)(w * h) == (uint64_t)w * h);   // no overflow, as per spec
 
   if (tmp == NULL) return;
   for (y = 1; y < h - 1; ++y) {

src/enc/backward_references.c

@@ -15,6 +15,7 @@
 #include "./backward_references.h"
 #include "./histogram.h"
 #include "../utils/color_cache.h"
+#include "../utils/utils.h"
 
 #define VALUES_IN_BYTE 256
 
@@ -93,7 +94,8 @@ int VP8LBackwardRefsAlloc(VP8LBackwardRefs* const refs, int max_size) {
   assert(refs != NULL);
   refs->size = 0;
   refs->max_size = 0;
-  refs->refs = (PixOrCopy*)malloc(max_size * sizeof(*refs->refs));
+  refs->refs = (PixOrCopy*)WebPSafeMalloc((uint64_t)max_size,
+                                          sizeof(*refs->refs));
   if (refs->refs == NULL) return 0;
   refs->max_size = max_size;
   return 1;
@@ -110,7 +112,7 @@ static WEBP_INLINE uint64_t GetPixPairHash64(const uint32_t* const argb) {
 
 static int HashChainInit(HashChain* const p, int size) {
   int i;
-  p->chain_ = (int*)malloc(size * sizeof(*p->chain_));
+  p->chain_ = (int*)WebPSafeMalloc((uint64_t)size, sizeof(*p->chain_));
   if (p->chain_ == NULL) {
     return 0;
   }
@@ -437,7 +439,8 @@ static int BackwardReferencesHashChainDistanceOnly(
   const int quality = 100;
   const int pix_count = xsize * ysize;
   const int use_color_cache = (cache_bits > 0);
-  double* const cost = (double*)malloc(pix_count * sizeof(*cost));
+  double* const cost =
+      (double*)WebPSafeMalloc((uint64_t)pix_count, sizeof(*cost));
   CostModel* cost_model = (CostModel*)malloc(sizeof(*cost_model));
   HashChain* hash_chain = (HashChain*)malloc(sizeof(*hash_chain));
   VP8LColorCache hashers;
@@ -564,7 +567,8 @@ static int TraceBackwards(const uint32_t* const dist_array,
   }
   // Allocate.
   *chosen_path_size = count;
-  *chosen_path = (uint32_t*)malloc(count * sizeof(*chosen_path));
+  *chosen_path =
+      (uint32_t*)WebPSafeMalloc((uint64_t)count, sizeof(**chosen_path));
   if (*chosen_path == NULL) return 0;
 
   // Write in reverse order.
@@ -658,7 +662,7 @@ static int BackwardReferencesTraceBackwards(int xsize, int ysize,
   uint32_t* chosen_path = NULL;
   int chosen_path_size = 0;
   uint32_t* dist_array =
-      (uint32_t*)malloc(dist_array_size * sizeof(*dist_array));
+      (uint32_t*)WebPSafeMalloc((uint64_t)dist_array_size, sizeof(*dist_array));
 
   if (dist_array == NULL) goto Error;
 

src/enc/histogram.c

@@ -17,6 +17,7 @@
 #include "./backward_references.h"
 #include "./histogram.h"
 #include "../dsp/lossless.h"
+#include "../utils/utils.h"
 
 #if defined(_MSC_VER) && !defined(NOT_HAVE_LOG2)
 # define NOT_HAVE_LOG2 1
@@ -65,10 +66,10 @@ VP8LHistogramSet* VP8LAllocateHistogramSet(int size, int cache_bits) {
   int i;
   VP8LHistogramSet* set;
   VP8LHistogram* bulk;
-  const size_t total_size = sizeof(*set)
+  const uint64_t total_size = (uint64_t)sizeof(*set)
                             + size * sizeof(*set->histograms)
                             + size * sizeof(**set->histograms);
-  uint8_t* memory = (uint8_t*)malloc(total_size);
+  uint8_t* memory = (uint8_t*)WebPSafeMalloc(total_size, sizeof(*memory));
   if (memory == NULL) return NULL;
 
   set = (VP8LHistogramSet*)memory;

src/enc/picture.c

@@ -15,6 +15,7 @@
 
 #include "./vp8enci.h"
 #include "../utils/rescaler.h"
+#include "../utils/utils.h"
 #include "../dsp/dsp.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
@@ -81,14 +82,12 @@ int WebPPictureAlloc(WebPPicture* picture) {
 
       // Security and validation checks
       if (width <= 0 || height <= 0 ||         // luma/alpha param error
-          uv_width < 0 || uv_height < 0 ||     // u/v param error
+          uv_width < 0 || uv_height < 0) {     // u/v param error
-          y_size >= (1ULL << 40) ||            // reasonable global size
-          (size_t)total_size != total_size) {  // overflow on 32bit
         return 0;
       }
       // Clear previous buffer and allocate a new one.
       WebPPictureFree(picture);   // erase previous buffer
-      mem = (uint8_t*)malloc((size_t)total_size);
+      mem = (uint8_t*)WebPSafeMalloc(total_size, sizeof(*mem));
       if (mem == NULL) return 0;
 
       // From now on, we're in the clear, we can no longer fail...
@@ -119,15 +118,12 @@ int WebPPictureAlloc(WebPPicture* picture) {
     } else {
       void* memory;
       const uint64_t argb_size = (uint64_t)width * height;
-      const uint64_t total_size = argb_size * sizeof(*picture->argb);
+      if (width <= 0 || height <= 0) {
-      if (width <= 0 || height <= 0 ||
-          argb_size >= (1ULL << 40) ||
-          (size_t)total_size != total_size) {
         return 0;
       }
       // Clear previous buffer and allocate a new one.
       WebPPictureFree(picture);   // erase previous buffer
-      memory = malloc((size_t)total_size);
+      memory = WebPSafeMalloc(argb_size, sizeof(*picture->argb));
       if (memory == NULL) return 0;
 
       // TODO(skal): align plane to cache line?
@@ -416,7 +412,7 @@ int WebPPictureRescale(WebPPicture* pic, int width, int height) {
   if (!WebPPictureAlloc(&tmp)) return 0;
 
   if (!pic->use_argb) {
-    work = (int32_t*)malloc(2 * width * sizeof(*work));
+    work = (int32_t*)WebPSafeMalloc(2ULL * width, sizeof(*work));
     if (work == NULL) {
       WebPPictureFree(&tmp);
       return 0;
@@ -449,7 +445,7 @@ int WebPPictureRescale(WebPPicture* pic, int width, int height) {
     }
 #endif
   } else {
-    work = (int32_t*)malloc(2 * width * 4 * sizeof(*work));
+    work = (int32_t*)WebPSafeMalloc(2ULL * width * 4, sizeof(*work));
     if (work == NULL) {
       WebPPictureFree(&tmp);
       return 0;
@@ -480,17 +476,17 @@ void WebPMemoryWriterInit(WebPMemoryWriter* writer) {
 int WebPMemoryWrite(const uint8_t* data, size_t data_size,
                     const WebPPicture* picture) {
   WebPMemoryWriter* const w = (WebPMemoryWriter*)picture->custom_ptr;
-  size_t next_size;
+  uint64_t next_size;
   if (w == NULL) {
     return 1;
   }
-  next_size = w->size + data_size;
+  next_size = (uint64_t)w->size + data_size;
   if (next_size > w->max_size) {
     uint8_t* new_mem;
-    size_t next_max_size = w->max_size * 2;
+    uint64_t next_max_size = 2ULL * w->max_size;
     if (next_max_size < next_size) next_max_size = next_size;
-    if (next_max_size < 8192) next_max_size = 8192;
+    if (next_max_size < 8192ULL) next_max_size = 8192ULL;
-    new_mem = (uint8_t*)malloc(next_max_size);
+    new_mem = (uint8_t*)WebPSafeMalloc(next_max_size, 1);
     if (new_mem == NULL) {
       return 0;
     }
@@ -499,7 +495,8 @@ int WebPMemoryWrite(const uint8_t* data, size_t data_size,
     }
     free(w->mem);
     w->mem = new_mem;
-    w->max_size = next_max_size;
+    // down-cast is ok, thanks to WebPSafeMalloc
+    w->max_size = (size_t)next_max_size;
   }
   if (data_size > 0) {
     memcpy(w->mem + w->size, data, data_size);

src/enc/vp8l.c

@@ -20,6 +20,7 @@
 #include "../dsp/lossless.h"
 #include "../utils/bit_writer.h"
 #include "../utils/huffman_encode.h"
+#include "../utils/utils.h"
 #include "../webp/format_constants.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
@@ -106,7 +107,8 @@ static int AnalyzeEntropy(const WebPPicture* const pic,
   uint32_t last_pix = argb[0];    // so we're sure that pix_diff == 0
 
   VP8LHistogram* nonpredicted = NULL;
-  VP8LHistogram* predicted = (VP8LHistogram*)malloc(2 * sizeof(*predicted));
+  VP8LHistogram* predicted =
+      (VP8LHistogram*)malloc(2 * sizeof(*predicted));
   if (predicted == NULL) return 0;
   nonpredicted = predicted + 1;
 
@@ -167,7 +169,7 @@ static int GetHuffBitLengthsAndCodes(
     HuffmanTreeCode* const huffman_codes) {
   int i, k;
   int ok = 1;
-  int total_length_size = 0;
+  uint64_t total_length_size = 0;
   uint8_t* mem_buf = NULL;
   const int histogram_image_size = histogram_image->size;
 
@@ -188,9 +190,8 @@ static int GetHuffBitLengthsAndCodes(
   {
     uint16_t* codes;
     uint8_t* lengths;
-    const size_t total_buf_size = total_length_size * sizeof(*lengths)
+    mem_buf = (uint8_t*)WebPSafeCalloc(total_length_size,
-                                + total_length_size * sizeof(*codes);
+                                       sizeof(*lengths) + sizeof(*codes));
-    mem_buf = (uint8_t*)calloc(total_buf_size, 1);
     if (mem_buf == NULL) {
       ok = 0;
       goto End;
@@ -292,7 +293,7 @@ static int StoreFullHuffmanCode(VP8LBitWriter* const bw,
   int num_tokens;
   HuffmanTreeCode huffman_code;
   HuffmanTreeToken* const tokens =
-      (HuffmanTreeToken*)malloc(max_tokens * sizeof(*tokens));
+      (HuffmanTreeToken*)WebPSafeMalloc((uint64_t)max_tokens, sizeof(*tokens));
   if (tokens == NULL) return 0;
 
   huffman_code.num_symbols = CODE_LENGTH_CODES;
@@ -499,21 +500,21 @@ static int EncodeImageInternal(VP8LBitWriter* const bw,
                                const uint32_t* const argb,
                                int width, int height, int quality,
                                int cache_bits, int histogram_bits) {
-  int i;
   int ok = 0;
   const int use_2d_locality = 1;
   const int use_color_cache = (cache_bits > 0);
-  const int histogram_image_xysize =
+  const uint32_t histogram_image_xysize =
       VP8LSubSampleSize(width, histogram_bits) *
       VP8LSubSampleSize(height, histogram_bits);
   VP8LHistogramSet* histogram_image =
       VP8LAllocateHistogramSet(histogram_image_xysize, 0);
   int histogram_image_size = 0;
-  int bit_array_size = 0;
+  size_t bit_array_size = 0;
   HuffmanTreeCode* huffman_codes = NULL;
   VP8LBackwardRefs refs;
   uint16_t* const histogram_symbols =
-      (uint16_t*)malloc(histogram_image_xysize * sizeof(*histogram_symbols));
+      (uint16_t*)WebPSafeMalloc((uint64_t)histogram_image_xysize,
+                                sizeof(*histogram_symbols));
   assert(histogram_bits >= MIN_HUFFMAN_BITS);
   assert(histogram_bits <= MAX_HUFFMAN_BITS);
   if (histogram_image == NULL || histogram_symbols == NULL) goto Error;
@@ -533,7 +534,7 @@ static int EncodeImageInternal(VP8LBitWriter* const bw,
   // Create Huffman bit lengths and codes for each histogram image.
   histogram_image_size = histogram_image->size;
   bit_array_size = 5 * histogram_image_size;
-  huffman_codes = (HuffmanTreeCode*)calloc(bit_array_size,
+  huffman_codes = (HuffmanTreeCode*)WebPSafeCalloc(bit_array_size,
                                                    sizeof(*huffman_codes));
   if (huffman_codes == NULL ||
       !GetHuffBitLengthsAndCodes(histogram_image, huffman_codes)) {
@@ -552,8 +553,10 @@ static int EncodeImageInternal(VP8LBitWriter* const bw,
     VP8LWriteBits(bw, 1, write_histogram_image);
     if (write_histogram_image) {
       uint32_t* const histogram_argb =
-          (uint32_t*)malloc(histogram_image_xysize * sizeof(*histogram_argb));
+          (uint32_t*)WebPSafeMalloc((uint64_t)histogram_image_xysize,
+                                    sizeof(*histogram_argb));
       int max_index = 0;
+      uint32_t i;
       if (histogram_argb == NULL) goto Error;
       for (i = 0; i < histogram_image_xysize; ++i) {
         const int index = histogram_symbols[i] & 0xffff;
@@ -575,14 +578,14 @@ static int EncodeImageInternal(VP8LBitWriter* const bw,
   }
 
   // Store Huffman codes.
+  {
+    int i;
     for (i = 0; i < 5 * histogram_image_size; ++i) {
       HuffmanTreeCode* const codes = &huffman_codes[i];
-    if (!StoreHuffmanCode(bw, codes)) {
+      if (!StoreHuffmanCode(bw, codes)) goto Error;
-      goto Error;
-    }
       ClearHuffmanTreeIfOnlyOneSymbol(codes);
     }
-
+  }
   // Free combined histograms.
   free(histogram_image);
   histogram_image = NULL;
@@ -769,14 +772,14 @@ static WebPEncodingError AllocateTransformBuffer(VP8LEncoder* const enc,
                                                  int width, int height) {
   WebPEncodingError err = VP8_ENC_OK;
   const int tile_size = 1 << enc->transform_bits_;
-  const size_t image_size = width * height;
+  const uint64_t image_size = width * height;
-  const size_t argb_scratch_size = tile_size * width + width;
+  const uint64_t argb_scratch_size = tile_size * width + width;
-  const size_t transform_data_size =
+  const uint64_t transform_data_size =
-      VP8LSubSampleSize(width, enc->transform_bits_) *
+      (uint64_t)VP8LSubSampleSize(width, enc->transform_bits_) *
-      VP8LSubSampleSize(height, enc->transform_bits_);
+      (uint64_t)VP8LSubSampleSize(height, enc->transform_bits_);
-  const size_t total_size =
+  const uint64_t total_size =
       image_size + argb_scratch_size + transform_data_size;
-  uint32_t* mem = (uint32_t*)malloc(total_size * sizeof(*mem));
+  uint32_t* mem = (uint32_t*)WebPSafeMalloc(total_size, sizeof(*mem));
   if (mem == NULL) {
     err = VP8_ENC_ERROR_OUT_OF_MEMORY;
     goto Error;

src/enc/webpenc.c

@@ -16,6 +16,7 @@
 
 #include "./vp8enci.h"
 #include "./vp8li.h"
+#include "../utils/utils.h"
 
 // #define PRINT_MEMORY_INFO
 
@@ -164,7 +165,8 @@ static VP8Encoder* InitVP8Encoder(const WebPConfig* const config,
       config->autofilter ? sizeof(LFStats) + ALIGN_CST : 0;
   VP8Encoder* enc;
   uint8_t* mem;
-  size_t size = sizeof(VP8Encoder) + ALIGN_CST  // main struct
+  const uint64_t size = (uint64_t)sizeof(VP8Encoder)   // main struct
+                      + ALIGN_CST                      // cache alignment
                       + cache_size                     // working caches
                       + info_size                      // modes info
                       + preds_size                     // prediction modes
@@ -198,7 +200,7 @@ static VP8Encoder* InitVP8Encoder(const WebPConfig* const config,
          mb_w * mb_h * 384 * sizeof(uint8_t));
   printf("===================================\n");
 #endif
-  mem = (uint8_t*)malloc(size);
+  mem = (uint8_t*)WebPSafeMalloc(size, sizeof(*mem));
   if (mem == NULL) {
     WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY);
     return NULL;

src/utils/color_cache.c

@@ -12,6 +12,7 @@
 #include <assert.h>
 #include <stdlib.h>
 #include "./color_cache.h"
+#include "../utils/utils.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
@@ -24,7 +25,8 @@ int VP8LColorCacheInit(VP8LColorCache* const cc, int hash_bits) {
   const int hash_size = 1 << hash_bits;
   assert(cc != NULL);
   assert(hash_bits > 0);
-  cc->colors_ = (uint32_t*)calloc(hash_size, sizeof(*cc->colors_));
+  cc->colors_ = (uint32_t*)WebPSafeCalloc((uint64_t)hash_size,
+                                          sizeof(*cc->colors_));
   if (cc->colors_ == NULL) return 0;
   cc->hash_shift_ = 32 - hash_bits;
   return 1;

src/utils/huffman.c

@@ -12,6 +12,7 @@
 #include <assert.h>
 #include <stdlib.h>
 #include "./huffman.h"
+#include "../utils/utils.h"
 #include "../webp/format_constants.h"
 
 #if defined(__cplusplus) || defined(c_plusplus)
@@ -49,8 +50,8 @@ static int TreeInit(HuffmanTree* const tree, int num_leaves) {
   // Note that a Huffman tree is a full binary tree; and in a full binary tree
   // with L leaves, the total number of nodes N = 2 * L - 1.
   tree->max_nodes_ = 2 * num_leaves - 1;
-  tree->root_ =
+  tree->root_ = (HuffmanTreeNode*)WebPSafeMalloc((uint64_t)tree->max_nodes_,
-      (HuffmanTreeNode*)malloc(tree->max_nodes_ * sizeof(*tree->root_));
+                                                 sizeof(*tree->root_));
   if (tree->root_ == NULL) return 0;
   TreeNodeInit(tree->root_);  // Initialize root.
   tree->num_nodes_ = 1;
@@ -173,7 +174,8 @@ int HuffmanTreeBuildImplicit(HuffmanTree* const tree,
     int ok = 0;
 
     // Get Huffman codes from the code lengths.
-    int* const codes = (int*)malloc(code_lengths_size * sizeof(*codes));
+    int* const codes =
+        (int*)WebPSafeMalloc((uint64_t)code_lengths_size, sizeof(*codes));
     if (codes == NULL) goto End;
 
     if (!HuffmanCodeLengthsToCodes(code_lengths, code_lengths_size, codes)) {

src/utils/huffman_encode.c

@@ -13,6 +13,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "./huffman_encode.h"
+#include "../utils/utils.h"
 #include "../webp/format_constants.h"
 
 // -----------------------------------------------------------------------------
@@ -196,7 +197,7 @@ static int GenerateOptimalTree(const int* const histogram, int histogram_size,
   // population and all the inserted nodes combining two existing nodes.
   // The tree pool needs 2 * (tree_size_orig - 1) entities, and the
   // tree needs exactly tree_size_orig entities.
-  tree = (HuffmanTree*)malloc(3 * tree_size_orig * sizeof(*tree));
+  tree = (HuffmanTree*)WebPSafeMalloc(3ULL * tree_size_orig, sizeof(*tree));
   if (tree == NULL) return 0;
   tree_pool = tree + tree_size_orig;