fix an overflow bug in LUT calculation

round(clip()) != clip(round())

Change-Id: Ia53f845b62e01bce672456cb7cdf8581f1a7ce44
This commit is contained in:
Pascal Massimino 2011-07-14 22:06:24 -07:00
parent d646d5c743
commit fb5d659bbd

View File

@ -24,6 +24,10 @@ uint8_t VP8kClip4Bits[YUV_RANGE_MAX - YUV_RANGE_MIN];
static int done = 0;
static inline uint8_t clip(int v, int max_value) {
return v < 0 ? 0 : v > max_value ? max_value : v;
}
void VP8YUVInit(void) {
int i;
if (done) {
@ -37,8 +41,8 @@ void VP8YUVInit(void) {
}
for (i = YUV_RANGE_MIN; i < YUV_RANGE_MAX; ++i) {
const int k = ((i - 16) * 76283 + YUV_HALF) >> YUV_FIX;
VP8kClip[i - YUV_RANGE_MIN] = (k < 0) ? 0 : (k > 255) ? 255 : k;
VP8kClip4Bits[i - YUV_RANGE_MIN] = (VP8kClip[i - YUV_RANGE_MIN] + 8) >> 4;
VP8kClip[i - YUV_RANGE_MIN] = clip(k, 255);
VP8kClip4Bits[i - YUV_RANGE_MIN] = clip((k + 8) >> 4, 15);
}
done = 1;
}