From 7416280d753c005e2bd500dd8b8f8562201fe528 Mon Sep 17 00:00:00 2001 From: hui su Date: Tue, 26 Jul 2016 13:55:09 -0700 Subject: [PATCH] Fix an unsigned integer overflow error in enc/cost.h Change-Id: I9774b59c417c185f09a61a115364b9642976a100 (cherry picked from commit 0b2c58a91cee8a8bdefa07c8b561f91ed4c96c47) --- src/enc/cost.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/enc/cost.h b/src/enc/cost.h index 95a4b075..ad7959fe 100644 --- a/src/enc/cost.h +++ b/src/enc/cost.h @@ -44,7 +44,9 @@ int VP8RecordCoeffs(int ctx, const VP8Residual* const res); // Record proba context used. static WEBP_INLINE int VP8RecordStats(int bit, proba_t* const stats) { proba_t p = *stats; - if (p >= 0xffff0000u) { // an overflow is inbound. + // An overflow is inbound. Note we handle this at 0xfffe0000u instead of + // 0xffff0000u to make sure p + 1u does not overflow. + if (p >= 0xfffe0000u) { p = ((p + 1u) >> 1) & 0x7fff7fffu; // -> divide the stats by 2. } // record bit count (lower 16 bits) and increment total count (upper 16 bits).