diff --git a/src/dsp/enc.c b/src/dsp/enc.c index ae2c830a..a6f05a5b 100644 --- a/src/dsp/enc.c +++ b/src/dsp/enc.c @@ -631,13 +631,13 @@ static int QuantizeBlock(int16_t in[16], int16_t out[16], for (; n < 16; ++n) { const int j = kZigzag[n]; const int sign = (in[j] < 0); - int coeff = (sign ? -in[j] : in[j]) + mtx->sharpen_[j]; - if (coeff > 2047) coeff = 2047; + const int coeff = (sign ? -in[j] : in[j]) + mtx->sharpen_[j]; if (coeff > mtx->zthresh_[j]) { const int Q = mtx->q_[j]; const int iQ = mtx->iq_[j]; const int B = mtx->bias_[j]; out[n] = QUANTDIV(coeff, iQ, B); + if (out[n] > MAX_LEVEL) out[n] = MAX_LEVEL; if (sign) out[n] = -out[n]; in[j] = out[n] * Q; if (out[n]) last = n; diff --git a/src/dsp/enc_sse2.c b/src/dsp/enc_sse2.c index c4148b56..619e6c5c 100644 --- a/src/dsp/enc_sse2.c +++ b/src/dsp/enc_sse2.c @@ -776,7 +776,7 @@ static int Disto16x16SSE2(const uint8_t* const a, const uint8_t* const b, // Simple quantization static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], int n, const VP8Matrix* const mtx) { - const __m128i max_coeff_2047 = _mm_set1_epi16(2047); + const __m128i max_coeff_2047 = _mm_set1_epi16(MAX_LEVEL); const __m128i zero = _mm_setzero_si128(); __m128i coeff0, coeff8; __m128i out0, out8; @@ -812,10 +812,6 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], coeff0 = _mm_add_epi16(coeff0, sharpen0); coeff8 = _mm_add_epi16(coeff8, sharpen8); - // if (coeff > 2047) coeff = 2047 - coeff0 = _mm_min_epi16(coeff0, max_coeff_2047); - coeff8 = _mm_min_epi16(coeff8, max_coeff_2047); - // out = (coeff * iQ + B) >> QFIX; { // doing calculations with 32b precision (QFIX=17) @@ -843,9 +839,14 @@ static int QuantizeBlockSSE2(int16_t in[16], int16_t out[16], out_04 = _mm_srai_epi32(out_04, QFIX); out_08 = _mm_srai_epi32(out_08, QFIX); out_12 = _mm_srai_epi32(out_12, QFIX); + // pack result as 16b out0 = _mm_packs_epi32(out_00, out_04); out8 = _mm_packs_epi32(out_08, out_12); + + // if (coeff > 2047) coeff = 2047 + out0 = _mm_min_epi16(out0, max_coeff_2047); + out8 = _mm_min_epi16(out8, max_coeff_2047); } // get sign back (if (sign[j]) out_n = -out_n) diff --git a/src/enc/cost.c b/src/enc/cost.c index 92e0cc71..dd528b8b 100644 --- a/src/enc/cost.c +++ b/src/enc/cost.c @@ -75,7 +75,7 @@ const uint16_t VP8LevelCodes[MAX_VARIABLE_LEVEL][2] = { // fixed costs for coding levels, deduce from the coding tree. // This is only the part that doesn't depend on the probability state. -const uint16_t VP8LevelFixedCosts[2048] = { +const uint16_t VP8LevelFixedCosts[MAX_LEVEL + 1] = { 0, 256, 256, 256, 256, 432, 618, 630, 731, 640, 640, 828, 901, 948, 1021, 1101, 1174, 1221, 1294, 1042, 1085, 1115, 1158, 1202, diff --git a/src/enc/cost.h b/src/enc/cost.h index 09b75b69..e264d321 100644 --- a/src/enc/cost.h +++ b/src/enc/cost.h @@ -18,7 +18,8 @@ extern "C" { #endif -extern const uint16_t VP8LevelFixedCosts[2048]; // approximate cost per level +// approximate cost per level: +extern const uint16_t VP8LevelFixedCosts[MAX_LEVEL + 1]; extern const uint16_t VP8EntropyCost[256]; // 8bit fixed-point log(p) // Cost of coding one event with probability 'proba'. diff --git a/src/enc/vp8enci.h b/src/enc/vp8enci.h index 6164703e..6aa3f436 100644 --- a/src/enc/vp8enci.h +++ b/src/enc/vp8enci.h @@ -55,8 +55,9 @@ enum { NUM_MB_SEGMENTS = 4, NUM_BANDS = 8, NUM_CTX = 3, NUM_PROBAS = 11, - MAX_LF_LEVELS = 64, // Maximum loop filter level - MAX_VARIABLE_LEVEL = 67 // last (inclusive) level with variable cost + MAX_LF_LEVELS = 64, // Maximum loop filter level + MAX_VARIABLE_LEVEL = 67, // last (inclusive) level with variable cost + MAX_LEVEL = 2047 // max level (note: max codable is 2047 + 67) }; typedef enum { // Rate-distortion optimization levels