From 73fee88c4a06ad1b6129c6549667b94e3899d884 Mon Sep 17 00:00:00 2001 From: James Zern Date: Wed, 4 Jun 2014 23:09:19 -0700 Subject: [PATCH] VP8RandomBits2: prevent signed int overflow 'diff' at its largest may be INT_MAX; << 1 of anything at or above 1 << 30 will overflow. Change-Id: Idb2b5a9b55acc2f6d5e32be8baaebee3f89919ad --- src/utils/random.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/random.h b/src/utils/random.h index 08a83e96..c392a615 100644 --- a/src/utils/random.h +++ b/src/utils/random.h @@ -45,7 +45,8 @@ static WEBP_INLINE int VP8RandomBits2(VP8Random* const rg, int num_bits, rg->tab_[rg->index1_] = diff; if (++rg->index1_ == VP8_RANDOM_TABLE_SIZE) rg->index1_ = 0; if (++rg->index2_ == VP8_RANDOM_TABLE_SIZE) rg->index2_ = 0; - diff = (diff << 1) >> (32 - num_bits); // sign-extend, 0-center + // sign-extend, 0-center + diff = (int)((uint32_t)diff << 1) >> (32 - num_bits); diff = (diff * amp) >> VP8_RANDOM_DITHER_FIX; // restrict range diff += 1 << (num_bits - 1); // shift back to 0.5-center return diff;