prevent 32b overflow for very large canvas_width / height

some multiplies here and there needed some extra checks
and error reporting. Even if width * height is guaranteed
to be < 2**32, we were multiplying by num_channels and
triggering a 32b overflow.
Some multiplies were not using size_t or uint64_t, additionally.

Change-Id: If2a35b94c8af204135f4b88a7fd63850aa381bbf
This commit is contained in:
Pascal Massimino
2016-09-23 06:56:11 +02:00
parent eee0cce158
commit 1c36440094
2 changed files with 43 additions and 23 deletions

View File

@ -39,13 +39,24 @@ static int IsFullFrame(int width, int height,
return (width == canvas_width && height == canvas_height);
}
static int CheckSizeForOverflow(uint64_t size) {
return (size == (size_t)size);
}
static int AllocateFrames(AnimatedImage* const image, uint32_t num_frames) {
uint32_t i;
const size_t rgba_size =
image->canvas_width * kNumChannels * image->canvas_height;
uint8_t* const mem = (uint8_t*)malloc(num_frames * rgba_size * sizeof(*mem));
DecodedFrame* const frames =
(DecodedFrame*)malloc(num_frames * sizeof(*frames));
uint8_t* mem = NULL;
DecodedFrame* frames = NULL;
const uint64_t rgba_size =
(uint64_t)image->canvas_width * kNumChannels * image->canvas_height;
const uint64_t total_size = (uint64_t)num_frames * rgba_size * sizeof(*mem);
const uint64_t total_frame_size = (uint64_t)num_frames * sizeof(*frames);
if (!CheckSizeForOverflow(total_size) ||
!CheckSizeForOverflow(total_frame_size)) {
return 0;
}
mem = (uint8_t*)malloc(total_size);
frames = (DecodedFrame*)malloc(total_frame_size);
if (mem == NULL || frames == NULL) {
free(mem);