From 495bef413d15efd54d37a713818c75a1ffe79091 Mon Sep 17 00:00:00 2001 From: Pascal Massimino Date: Thu, 12 Dec 2013 15:31:50 +0000 Subject: [PATCH] fix bug in TrellisQuantize the *quantized* level should be clipped to 2047, not the original coeff. (similar problem was fixed in the regular quantize function quite some time ago) Change-Id: I2fd2f8d94561ff0204e60535321ab41a565e8f85 --- src/enc/quant.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/enc/quant.c b/src/enc/quant.c index 869e0331..e1d202b5 100644 --- a/src/enc/quant.c +++ b/src/enc/quant.c @@ -598,11 +598,10 @@ static int TrellisQuantizeBlock(const VP8EncIterator* const it, // note: it's important to take sign of the _original_ coeff, // so we don't have to consider level < 0 afterward. const int sign = (in[j] < 0); - int coeff0 = (sign ? -in[j] : in[j]) + mtx->sharpen_[j]; - int level0; - if (coeff0 > 2047) coeff0 = 2047; + const int coeff0 = (sign ? -in[j] : in[j]) + mtx->sharpen_[j]; + int level0 = QUANTDIV(coeff0, iQ, B); + if (level0 > MAX_LEVEL) level0 = MAX_LEVEL; - level0 = QUANTDIV(coeff0, iQ, B); // test all alternate level values around level0. for (m = -MIN_DELTA; m <= MAX_DELTA; ++m) { Node* const cur = &NODE(n, m); @@ -614,7 +613,7 @@ static int TrellisQuantizeBlock(const VP8EncIterator* const it, cur->sign = sign; cur->level = level; cur->ctx = (level == 0) ? 0 : (level == 1) ? 1 : 2; - if (level >= 2048 || level < 0) { // node is dead? + if (level > MAX_LEVEL || level < 0) { // node is dead? cur->cost = MAX_COST; continue; }