Fix the oscillating prediction problem at low quality

For some exact resonance the over-quantization was exactly
compensating the under-quantization, leading to resonance
and strange patterns.

-> we special-handle the very flat blocks, hopefully for the
greater good (and not just the bad-resonance case).

For 'fast mode' (-m 3 or less), we just pay special attention
to the border of the image, where the oscillation / instability
usually starts. For the inner part of the image, since we're not
doing rd-opt, it's harder to fix anything.

Overall, on 'regular' images, the change is written the noise,
often leading to overall faster encoding (because of the short-cut).

BUG=webp:432

Change-Id: Ifaa8286499add80fd77daecf8e347abbff7c3a15

src/dsp/quant.h

@@ -10,6 +10,8 @@
 #ifndef WEBP_DSP_QUANT_H_
 #define WEBP_DSP_QUANT_H_
 
+#include <string.h>
+
 #include "src/dsp/dsp.h"
 #include "src/webp/types.h"
 
@@ -67,4 +69,17 @@ static WEBP_INLINE int IsFlat(const int16_t* levels, int num_blocks,
 #endif  // defined(WEBP_USE_NEON) && !defined(WEBP_ANDROID_NEON) &&
         // !defined(WEBP_HAVE_NEON_RTCD)
 
+static WEBP_INLINE int IsFlatSource16(const uint8_t* src) {
+  const uint32_t v = src[0] * 0x01010101u;
+  int i;
+  for (i = 0; i < 16; ++i) {
+    if (memcmp(src + 0, &v, 4) || memcmp(src +  4, &v, 4) ||
+        memcmp(src + 8, &v, 4) || memcmp(src + 12, &v, 4)) {
+      return 0;
+    }
+    src += BPS;
+  }
+  return 1;
+}
+
 #endif  // WEBP_DSP_QUANT_H_

src/enc/quant_enc.c

@@ -33,7 +33,7 @@
 
 // number of non-zero coeffs below which we consider the block very flat
 // (and apply a penalty to complex predictions)
-#define FLATNESS_LIMIT_I16 10      // I16 mode
+#define FLATNESS_LIMIT_I16 0       // I16 mode (special case)
 #define FLATNESS_LIMIT_I4  3       // I4 mode
 #define FLATNESS_LIMIT_UV  2       // UV mode
 #define FLATNESS_PENALTY   140     // roughly ~1bit per block
@@ -988,6 +988,7 @@ static void PickBestIntra16(VP8EncIterator* const it, VP8ModeScore* rd) {
   VP8ModeScore* rd_cur = &rd_tmp;
   VP8ModeScore* rd_best = rd;
   int mode;
+  int is_flat = IsFlatSource16(it->yuv_in_ + Y_OFF_ENC);
 
   rd->mode_i16 = -1;
   for (mode = 0; mode < NUM_PRED_MODES; ++mode) {
@@ -1003,10 +1004,14 @@ static void PickBestIntra16(VP8EncIterator* const it, VP8ModeScore* rd) {
         tlambda ? MULT_8B(tlambda, VP8TDisto16x16(src, tmp_dst, kWeightY)) : 0;
     rd_cur->H = VP8FixedCostsI16[mode];
     rd_cur->R = VP8GetCostLuma16(it, rd_cur);
-    if (mode > 0 &&
+    if (is_flat) {
-        IsFlat(rd_cur->y_ac_levels[0], kNumBlocks, FLATNESS_LIMIT_I16)) {
+      // refine the first impression (which was in pixel space)
-      // penalty to avoid flat area to be mispredicted by complex mode
+      is_flat = IsFlat(rd_cur->y_ac_levels[0], kNumBlocks, FLATNESS_LIMIT_I16);
-      rd_cur->R += FLATNESS_PENALTY * kNumBlocks;
+      if (is_flat) {
+        // Block is very flat. We put emphasis on the distortion being very low!
+        rd_cur->D *= 2;
+        rd_cur->SD *= 2;
+      }
     }
 
     // Since we always examine Intra16 first, we can overwrite *rd directly.
@@ -1087,7 +1092,8 @@ static int PickBestIntra4(VP8EncIterator* const it, VP8ModeScore* const rd) {
                   : 0;
       rd_tmp.H = mode_costs[mode];
 
-      // Add flatness penalty
+      // Add flatness penalty, to avoid flat area to be mispredicted
+      // by a complex mode.
       if (mode > 0 && IsFlat(tmp_levels, kNumBlocks, FLATNESS_LIMIT_I4)) {
         rd_tmp.R = FLATNESS_PENALTY * kNumBlocks;
       } else {
@@ -1242,11 +1248,19 @@ static void RefineUsingDistortion(VP8EncIterator* const it,
       if (mode > 0 && VP8FixedCostsI16[mode] > bit_limit) {
         continue;
       }
+
       if (score < best_score) {
         best_mode = mode;
         best_score = score;
       }
     }
+    if (it->x_ == 0 || it->y_ == 0) {
+      // avoid starting a checkerboard resonance from the border. See bug #432.
+      if (IsFlatSource16(src)) {
+        best_mode = (it->x_ == 0) ? 0 : 2;
+        try_both_modes = 0;  // stick to i16
+      }
+    }
     VP8SetIntra16Mode(it, best_mode);
     // we'll reconstruct later, if i16 mode actually gets selected
   }