dany/libwebp
1
0
Fork 0
mirror of https://github.com/webmproject/libwebp.git synced 2024-12-27 06:08:21 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix bug in TrellisQuantize

Browse Source 
the *quantized* level should be clipped to 2047, not the
original coeff.
(similar problem was fixed in the regular quantize function
quite some time ago)

Change-Id: I2fd2f8d94561ff0204e60535321ab41a565e8f85
This commit is contained in:
Pascal Massimino 2013-12-12 15:31:50 +00:00 committed by James Zern
parent 605a712701
commit 495bef413d
1 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/enc/quant.c
View File

@ -598,11 +598,10 @@ static int TrellisQuantizeBlock(const VP8EncIterator* const it,
// note: it's important to take sign of the _original_ coeff, // note: it's important to take sign of the _original_ coeff,
// so we don't have to consider level < 0 afterward. // so we don't have to consider level < 0 afterward.
const int sign = (in[j] < 0); const int sign = (in[j] < 0);
int coeff0 = (sign ? -in[j] : in[j]) + mtx->sharpen_[j]; const int coeff0 = (sign ? -in[j] : in[j]) + mtx->sharpen_[j];
int level0; int level0 = QUANTDIV(coeff0, iQ, B);
if (coeff0 > 2047) coeff0 = 2047; if (level0 > MAX_LEVEL) level0 = MAX_LEVEL;
level0 = QUANTDIV(coeff0, iQ, B);
// test all alternate level values around level0. // test all alternate level values around level0.
for (m = -MIN_DELTA; m <= MAX_DELTA; ++m) { for (m = -MIN_DELTA; m <= MAX_DELTA; ++m) {
Node* const cur = &NODE(n, m); Node* const cur = &NODE(n, m);
@ -614,7 +613,7 @@ static int TrellisQuantizeBlock(const VP8EncIterator* const it,
cur->sign = sign; cur->sign = sign;
cur->level = level; cur->level = level;
cur->ctx = (level == 0) ? 0 : (level == 1) ? 1 : 2; cur->ctx = (level == 0) ? 0 : (level == 1) ? 1 : 2;
if (level >= 2048 || level < 0) { // node is dead? if (level > MAX_LEVEL || level < 0) { // node is dead?
cur->cost = MAX_COST; cur->cost = MAX_COST;
continue; continue;
} }