From 80237c4371b71aaa50a573aa1e73cb96dbe3c1d7 Mon Sep 17 00:00:00 2001 From: James Zern Date: Wed, 3 Oct 2012 12:14:44 -0700 Subject: [PATCH] GetHistoBits: fix integer overflow huff_image_size was a size_t (=32 bits with 32-bit builds) which could rollover causing an incorrectly sized allocation and a crash in lossless encoding. fixes issue #128 Change-Id: I175c8c6132ba9792034807c5c1028dfddfeb4ea5 --- src/enc/vp8l.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/enc/vp8l.c b/src/enc/vp8l.c index affd91ce..dcb8d35d 100644 --- a/src/enc/vp8l.c +++ b/src/enc/vp8l.c @@ -899,13 +899,13 @@ static int GetHistoBits(const WebPConfig* const config, const WebPPicture* const pic) { const int width = pic->width; const int height = pic->height; - const size_t hist_size = sizeof(VP8LHistogram); + const uint64_t hist_size = sizeof(VP8LHistogram); // Make tile size a function of encoding method (Range: 0 to 6). int histo_bits = 7 - config->method; while (1) { - const size_t huff_image_size = VP8LSubSampleSize(width, histo_bits) * - VP8LSubSampleSize(height, histo_bits) * - hist_size; + const uint64_t huff_image_size = VP8LSubSampleSize(width, histo_bits) * + VP8LSubSampleSize(height, histo_bits) * + hist_size; if (huff_image_size <= MAX_HUFF_IMAGE_SIZE) break; ++histo_bits; }