Make the lossless predictors work on a batch of pixels.

Change-Id: Ieaee34f1f97c375b9e97ef7e9df60aed353dffa1

src/dsp/lossless.c

@@ -27,11 +27,6 @@
 //------------------------------------------------------------------------------
 // Image transforms.
 
-// In-place sum of each component with mod 256.
-static WEBP_INLINE void AddPixelsEq(uint32_t* a, uint32_t b) {
-  *a = VP8LAddPixels(*a, b);
-}
-
 static WEBP_INLINE uint32_t Average2(uint32_t a0, uint32_t a1) {
   return (((a0 ^ a1) & 0xfefefefeu) >> 1) + (a0 & a1);
 }
@@ -172,21 +167,33 @@ static uint32_t Predictor13(uint32_t left, const uint32_t* const top) {
   return pred;
 }
 
+GENERATE_PREDICTOR_ADD(0)
+GENERATE_PREDICTOR_ADD(1)
+GENERATE_PREDICTOR_ADD(2)
+GENERATE_PREDICTOR_ADD(3)
+GENERATE_PREDICTOR_ADD(4)
+GENERATE_PREDICTOR_ADD(5)
+GENERATE_PREDICTOR_ADD(6)
+GENERATE_PREDICTOR_ADD(7)
+GENERATE_PREDICTOR_ADD(8)
+GENERATE_PREDICTOR_ADD(9)
+GENERATE_PREDICTOR_ADD(10)
+GENERATE_PREDICTOR_ADD(11)
+GENERATE_PREDICTOR_ADD(12)
+GENERATE_PREDICTOR_ADD(13)
+
 //------------------------------------------------------------------------------
 
 // Inverse prediction.
 static void PredictorInverseTransform(const VP8LTransform* const transform,
-                                      int y_start, int y_end, uint32_t* data) {
+                                      int y_start, int y_end,
+                                      const uint32_t* in, uint32_t* out) {
   const int width = transform->xsize_;
   if (y_start == 0) {  // First Row follows the L (mode=1) mode.
-    int x;
+    PredictorAdd0(in, NULL, 1, out);
-    const uint32_t pred0 = Predictor0(data[-1], NULL);
+    PredictorAdd1(in + 1, NULL, width - 1, out + 1);
-    AddPixelsEq(data, pred0);
+    in += width;
-    for (x = 1; x < width; ++x) {
+    out += width;
-      const uint32_t pred1 = Predictor1(data[x - 1], NULL);
-      AddPixelsEq(data + x, pred1);
-    }
-    data += width;
     ++y_start;
   }
 
@@ -194,36 +201,26 @@ static void PredictorInverseTransform(const VP8LTransform* const transform,
     int y = y_start;
     const int tile_width = 1 << transform->bits_;
     const int mask = tile_width - 1;
-    const int safe_width = width & ~mask;
     const int tiles_per_row = VP8LSubSampleSize(width, transform->bits_);
     const uint32_t* pred_mode_base =
         transform->data_ + (y >> transform->bits_) * tiles_per_row;
 
     while (y < y_end) {
-      const uint32_t pred2 = Predictor2(data[-1], data - width);
       const uint32_t* pred_mode_src = pred_mode_base;
-      VP8LPredictorFunc pred_func;
       int x = 1;
-      int t = 1;
       // First pixel follows the T (mode=2) mode.
-      AddPixelsEq(data, pred2);
+      PredictorAdd2(in, out - width, 1, out);
       // .. the rest:
-      while (x < safe_width) {
+      while (x < width) {
-        pred_func = VP8LPredictors[((*pred_mode_src++) >> 8) & 0xf];
+        const VP8LPredictorAddSubFunc pred_func =
-        for (; t < tile_width; ++t, ++x) {
+            VP8LPredictorsAdd[((*pred_mode_src++) >> 8) & 0xf];
-          const uint32_t pred = pred_func(data[x - 1], data + x - width);
+        int x_end = (x & ~mask) + tile_width;
-          AddPixelsEq(data + x, pred);
+        if (x_end > width) x_end = width;
-        }
+        pred_func(in + x, out + x - width, x_end - x, out + x);
-        t = 0;
+        x = x_end;
       }
-      if (x < width) {
+      in += width;
-        pred_func = VP8LPredictors[((*pred_mode_src++) >> 8) & 0xf];
+      out += width;
-        for (; x < width; ++x) {
-          const uint32_t pred = pred_func(data[x - 1], data + x - width);
-          AddPixelsEq(data + x, pred);
-        }
-      }
-      data += width;
       ++y;
       if ((y & mask) == 0) {   // Use the same mask, since tiles are squares.
         pred_mode_base += tiles_per_row;
@@ -375,11 +372,7 @@ void VP8LInverseTransform(const VP8LTransform* const transform,
       VP8LAddGreenToBlueAndRed(in, (row_end - row_start) * width, out);
       break;
     case PREDICTOR_TRANSFORM:
-      // TODO(vrabaud): parallelize transform predictors.
+      PredictorInverseTransform(transform, row_start, row_end, in, out);
-      if (in != out) {
-        memcpy(out, in, (row_end - row_start) * width * sizeof(*out));
-      }
-      PredictorInverseTransform(transform, row_start, row_end, out);
       if (row_end != transform->ysize_) {
         // The last predicted row in this iteration will be the top-pred row
         // for the first row in next iteration.
@@ -566,6 +559,7 @@ void VP8LConvertFromBGRA(const uint32_t* const in_data, int num_pixels,
 //------------------------------------------------------------------------------
 
 VP8LProcessDecBlueAndRedFunc VP8LAddGreenToBlueAndRed;
+VP8LPredictorAddSubFunc VP8LPredictorsAdd[16];
 VP8LPredictorFunc VP8LPredictors[16];
 
 VP8LTransformColorInverseFunc VP8LTransformColorInverse;
@@ -607,6 +601,23 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInit(void) {
   VP8LPredictors[14] = Predictor0;     // <- padding security sentinels
   VP8LPredictors[15] = Predictor0;
 
+  VP8LPredictorsAdd[0] = PredictorAdd0;
+  VP8LPredictorsAdd[1] = PredictorAdd1;
+  VP8LPredictorsAdd[2] = PredictorAdd2;
+  VP8LPredictorsAdd[3] = PredictorAdd3;
+  VP8LPredictorsAdd[4] = PredictorAdd4;
+  VP8LPredictorsAdd[5] = PredictorAdd5;
+  VP8LPredictorsAdd[6] = PredictorAdd6;
+  VP8LPredictorsAdd[7] = PredictorAdd7;
+  VP8LPredictorsAdd[8] = PredictorAdd8;
+  VP8LPredictorsAdd[9] = PredictorAdd9;
+  VP8LPredictorsAdd[10] = PredictorAdd10;
+  VP8LPredictorsAdd[11] = PredictorAdd11;
+  VP8LPredictorsAdd[12] = PredictorAdd12;
+  VP8LPredictorsAdd[13] = PredictorAdd13;
+  VP8LPredictorsAdd[14] = PredictorAdd0;  // <- padding security sentinels
+  VP8LPredictorsAdd[15] = PredictorAdd0;
+
   VP8LAddGreenToBlueAndRed = VP8LAddGreenToBlueAndRed_C;
 
   VP8LTransformColorInverse = VP8LTransformColorInverse_C;

src/dsp/lossless.h

@@ -34,6 +34,10 @@ extern "C" {
 
 typedef uint32_t (*VP8LPredictorFunc)(uint32_t left, const uint32_t* const top);
 extern VP8LPredictorFunc VP8LPredictors[16];
+typedef void (*VP8LPredictorAddSubFunc)(const uint32_t* in,
+                                        const uint32_t* upper, int num_pixels,
+                                        uint32_t* out);
+extern VP8LPredictorAddSubFunc VP8LPredictorsAdd[16];
 
 typedef void (*VP8LProcessDecBlueAndRedFunc)(const uint32_t* src,
                                              int num_pixels, uint32_t* dst);
@@ -143,6 +147,8 @@ void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride,
                                       int green_to_blue, int red_to_blue,
                                       int histo[]);
 
+extern VP8LPredictorAddSubFunc VP8LPredictorsSub[16];
+
 // -----------------------------------------------------------------------------
 // Huffman-cost related functions.
 

src/dsp/lossless_common.h

@@ -174,6 +174,34 @@ uint32_t VP8LSubPixels(uint32_t a, uint32_t b) {
 }
 
 //------------------------------------------------------------------------------
+// Transform-related functions use din both encoding and decoding.
+
+// Macros used to create a batch predictor that iteratively uses a
+// one-pixel predictor.
+
+// The predictor is added to the output pixel (which
+// is therefore considered as a residual) to get the final prediction.
+#define GENERATE_PREDICTOR_ADD(X)                                        \
+  static void PredictorAdd##X(const uint32_t* in, const uint32_t* upper, \
+                              int num_pixels, uint32_t* out) {           \
+    int x;                                                               \
+    for (x = 0; x < num_pixels; ++x) {                                   \
+      const uint32_t pred = VP8LPredictors[(X)](out[x - 1], upper + x);  \
+      out[x] = VP8LAddPixels(in[x], pred);                               \
+    }                                                                    \
+  }
+
+// It subtracts the prediction from the input pixel and stores the residual
+// in the output pixel.
+#define GENERATE_PREDICTOR_SUB(X)                                        \
+  static void PredictorSub##X(const uint32_t* in, const uint32_t* upper, \
+                              int num_pixels, uint32_t* out) {           \
+    int x;                                                               \
+    for (x = 0; x < num_pixels; ++x) {                                   \
+      const uint32_t pred = VP8LPredictors[(X)](in[x - 1], upper + x);   \
+      out[x] = VP8LSubPixels(in[x], pred);                               \
+    }                                                                    \
+  }
 
 #ifdef __cplusplus
 }    // extern "C"

src/dsp/lossless_enc.c

@@ -665,6 +665,21 @@ static void HistogramAdd(const VP8LHistogram* const a,
 
 //------------------------------------------------------------------------------
 
+GENERATE_PREDICTOR_SUB(0)
+GENERATE_PREDICTOR_SUB(1)
+GENERATE_PREDICTOR_SUB(2)
+GENERATE_PREDICTOR_SUB(3)
+GENERATE_PREDICTOR_SUB(4)
+GENERATE_PREDICTOR_SUB(5)
+GENERATE_PREDICTOR_SUB(6)
+GENERATE_PREDICTOR_SUB(7)
+GENERATE_PREDICTOR_SUB(8)
+GENERATE_PREDICTOR_SUB(9)
+GENERATE_PREDICTOR_SUB(10)
+GENERATE_PREDICTOR_SUB(11)
+GENERATE_PREDICTOR_SUB(12)
+GENERATE_PREDICTOR_SUB(13)
+
 VP8LProcessEncBlueAndRedFunc VP8LSubtractGreenFromBlueAndRed;
 
 VP8LTransformColorFunc VP8LTransformColor;
@@ -686,6 +701,8 @@ VP8LHistogramAddFunc VP8LHistogramAdd;
 
 VP8LVectorMismatchFunc VP8LVectorMismatch;
 
+VP8LPredictorAddSubFunc VP8LPredictorsSub[16];
+
 extern void VP8LEncDspInitSSE2(void);
 extern void VP8LEncDspInitSSE41(void);
 extern void VP8LEncDspInitNEON(void);
@@ -722,6 +739,23 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInit(void) {
 
   VP8LVectorMismatch = VectorMismatch;
 
+  VP8LPredictorsSub[0] = PredictorSub0;
+  VP8LPredictorsSub[1] = PredictorSub1;
+  VP8LPredictorsSub[2] = PredictorSub2;
+  VP8LPredictorsSub[3] = PredictorSub3;
+  VP8LPredictorsSub[4] = PredictorSub4;
+  VP8LPredictorsSub[5] = PredictorSub5;
+  VP8LPredictorsSub[6] = PredictorSub6;
+  VP8LPredictorsSub[7] = PredictorSub7;
+  VP8LPredictorsSub[8] = PredictorSub8;
+  VP8LPredictorsSub[9] = PredictorSub9;
+  VP8LPredictorsSub[10] = PredictorSub10;
+  VP8LPredictorsSub[11] = PredictorSub11;
+  VP8LPredictorsSub[12] = PredictorSub12;
+  VP8LPredictorsSub[13] = PredictorSub13;
+  VP8LPredictorsSub[14] = PredictorSub0;  // <- padding security sentinels
+  VP8LPredictorsSub[15] = PredictorSub0;
+
   // If defined, use CPUInfo() to overwrite some pointers with faster versions.
   if (VP8GetCPUInfo != NULL) {
 #if defined(WEBP_USE_SSE2)

src/dsp/lossless_sse2.c

@@ -17,6 +17,7 @@
 
 #include "./common_sse2.h"
 #include "./lossless.h"
+#include "./lossless_common.h"
 #include <assert.h>
 #include <emmintrin.h>
 
@@ -154,6 +155,17 @@ static uint32_t Predictor13(uint32_t left, const uint32_t* const top) {
   return pred;
 }
 
+// TODO(vrabaud): implement those functions in SSE.
+GENERATE_PREDICTOR_ADD(5)
+GENERATE_PREDICTOR_ADD(6)
+GENERATE_PREDICTOR_ADD(7)
+GENERATE_PREDICTOR_ADD(8)
+GENERATE_PREDICTOR_ADD(9)
+GENERATE_PREDICTOR_ADD(10)
+GENERATE_PREDICTOR_ADD(11)
+GENERATE_PREDICTOR_ADD(12)
+GENERATE_PREDICTOR_ADD(13)
+
 //------------------------------------------------------------------------------
 // Subtract-Green Transform
 
@@ -394,6 +406,16 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitSSE2(void) {
   VP8LPredictors[12] = Predictor12;
   VP8LPredictors[13] = Predictor13;
 
+  VP8LPredictorsAdd[5] = PredictorAdd5;
+  VP8LPredictorsAdd[6] = PredictorAdd6;
+  VP8LPredictorsAdd[7] = PredictorAdd7;
+  VP8LPredictorsAdd[8] = PredictorAdd8;
+  VP8LPredictorsAdd[9] = PredictorAdd9;
+  VP8LPredictorsAdd[10] = PredictorAdd10;
+  VP8LPredictorsAdd[11] = PredictorAdd11;
+  VP8LPredictorsAdd[12] = PredictorAdd12;
+  VP8LPredictorsAdd[13] = PredictorAdd13;
+
   VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed;
   VP8LTransformColorInverse = TransformColorInverse;
 

src/enc/predictor.c

@@ -66,16 +66,27 @@ static WEBP_INLINE void UpdateHisto(int histo_argb[4][256], uint32_t argb) {
 //------------------------------------------------------------------------------
 // Spatial transform functions.
 
-static WEBP_INLINE uint32_t Predict(VP8LPredictorFunc pred_func,
+static WEBP_INLINE void PredictBatch(int mode, int x_start, int y,
-                                    int x, int y,
+                                     int num_pixels, const uint32_t* current,
-                                    const uint32_t* current_row,
+                                     const uint32_t* upper, uint32_t* out) {
-                                    const uint32_t* upper_row) {
+  if (x_start == 0) {
+    if (y == 0) {
+      // ARGB_BLACK.
+      VP8LPredictorsSub[0](current, NULL, 1, out);
+    } else {
+      // Top one.
+      VP8LPredictorsSub[2](current, upper, 1, out);
+    }
+    ++x_start;
+    ++out;
+    --num_pixels;
+  }
   if (y == 0) {
-    return (x == 0) ? ARGB_BLACK : current_row[x - 1];  // Left.
+    // Left one.
-  } else if (x == 0) {
+    VP8LPredictorsSub[1](current + x_start, NULL, num_pixels, out);
-    return upper_row[x];  // Top.
   } else {
-    return pred_func(current_row[x - 1], upper_row + x);
+    VP8LPredictorsSub[mode](current + x_start, upper + x_start, num_pixels,
+                            out);
   }
 }
 
@@ -208,42 +219,59 @@ static uint32_t NearLossless(uint32_t value, uint32_t predict,
   return ((uint32_t)a << 24) | ((uint32_t)r << 16) | ((uint32_t)g << 8) | b;
 }
 
-// Returns the difference between the pixel and its prediction. In case of a
+// Stores the difference between the pixel and its prediction in "out".
-// lossy encoding, updates the source image to avoid propagating the deviation
+// In case of a lossy encoding, updates the source image to avoid propagating
-// further to pixels which depend on the current pixel for their predictions.
+// the deviation further to pixels which depend on the current pixel for their
-static WEBP_INLINE uint32_t GetResidual(int width, int height,
+// predictions.
-                                        uint32_t* const upper_row,
+static WEBP_INLINE void GetResidual(
-                                        uint32_t* const current_row,
+    int width, int height, uint32_t* const upper_row,
-                                        const uint8_t* const max_diffs,
+    uint32_t* const current_row, const uint8_t* const max_diffs, int mode,
-                                        int mode, VP8LPredictorFunc pred_func,
+    int x_start, int x_end, int y, int max_quantization, int exact,
-                                        int x, int y, int max_quantization,
+    int used_subtract_green, uint32_t* const out) {
-                                        int exact, int used_subtract_green) {
+  if (exact) {
-  const uint32_t predict = Predict(pred_func, x, y, current_row, upper_row);
+    PredictBatch(mode, x_start, y, x_end - x_start, current_row, upper_row,
-  uint32_t residual;
+                 out);
-  if (max_quantization == 1 || mode == 0 || y == 0 || y == height - 1 ||
-      x == 0 || x == width - 1) {
-    residual = VP8LSubPixels(current_row[x], predict);
   } else {
-    residual = NearLossless(current_row[x], predict, max_quantization,
+    const VP8LPredictorFunc pred_func = VP8LPredictors[mode];
-                            max_diffs[x], used_subtract_green);
+    int x;
-    // Update the source image.
+    for (x = x_start; x < x_end; ++x) {
-    current_row[x] = VP8LAddPixels(predict, residual);
+      uint32_t predict;
-    // x is never 0 here so we do not need to update upper_row like below.
+      uint32_t residual;
+      if (y == 0) {
+        predict = (x == 0) ? ARGB_BLACK : current_row[x - 1];  // Left.
+      } else if (x == 0) {
+        predict = upper_row[x];  // Top.
+      } else {
+        predict = pred_func(current_row[x - 1], upper_row + x);
+      }
+      if (mode == 0 || y == 0 || y == height - 1 || x == 0 || x == width - 1) {
+        residual = VP8LSubPixels(current_row[x], predict);
+      } else {
+        residual = NearLossless(current_row[x], predict, max_quantization,
+                                max_diffs[x], used_subtract_green);
+        // Update the source image.
+        current_row[x] = VP8LAddPixels(predict, residual);
+        // x is never 0 here so we do not need to update upper_row like below.
+      }
+      if ((current_row[x] & kMaskAlpha) == 0) {
+        // If alpha is 0, cleanup RGB. We can choose the RGB values of the
+        // residual for best compression. The prediction of alpha itself can be
+        // non-zero and must be kept though. We choose RGB of the residual to be
+        // 0.
+        residual &= kMaskAlpha;
+        // Update the source image.
+        current_row[x] = predict & ~kMaskAlpha;
+        // The prediction for the rightmost pixel in a row uses the leftmost
+        // pixel
+        // in that row as its top-right context pixel. Hence if we change the
+        // leftmost pixel of current_row, the corresponding change must be
+        // applied
+        // to upper_row as well where top-right context is being read from.
+        if (x == 0 && y != 0) upper_row[width] = current_row[0];
+      }
+      out[x - x_start] = residual;
+    }
   }
-  if (!exact && (current_row[x] & kMaskAlpha) == 0) {
-    // If alpha is 0, cleanup RGB. We can choose the RGB values of the residual
-    // for best compression. The prediction of alpha itself can be non-zero and
-    // must be kept though. We choose RGB of the residual to be 0.
-    residual &= kMaskAlpha;
-    // Update the source image.
-    current_row[x] = predict & ~kMaskAlpha;
-    // The prediction for the rightmost pixel in a row uses the leftmost pixel
-    // in that row as its top-right context pixel. Hence if we change the
-    // leftmost pixel of current_row, the corresponding change must be applied
-    // to upper_row as well where top-right context is being read from.
-    if (x == 0 && y != 0) upper_row[width] = current_row[0];
-  }
-  return residual;
 }
 
 // Returns best predictor and updates the accumulated histogram.
@@ -293,9 +321,11 @@ static int GetBestPredictorForTile(int width, int height,
   int (*histo_argb)[256] = histo_stack_1;
   int (*best_histo)[256] = histo_stack_2;
   int i, j;
+  uint32_t residuals[1 << MAX_TRANSFORM_BITS];
+  assert(bits <= MAX_TRANSFORM_BITS);
+  assert(max_x <= (1 << MAX_TRANSFORM_BITS));
 
   for (mode = 0; mode < kNumPredModes; ++mode) {
-    const VP8LPredictorFunc pred_func = VP8LPredictors[mode];
     float cur_diff;
     int relative_y;
     memset(histo_argb, 0, sizeof(histo_stack_1));
@@ -326,12 +356,11 @@ static int GetBestPredictorForTile(int width, int height,
                        max_diffs + context_start_x, used_subtract_green);
       }
 
+      GetResidual(width, height, upper_row, current_row, max_diffs, mode,
+                  start_x, start_x + max_x, y, max_quantization, exact,
+                  used_subtract_green, residuals);
       for (relative_x = 0; relative_x < max_x; ++relative_x) {
-        const int x = start_x + relative_x;
+        UpdateHisto(histo_argb, residuals[relative_x]);
-        UpdateHisto(histo_argb,
-                    GetResidual(width, height, upper_row, current_row,
-                                max_diffs, mode, pred_func, x, y,
-                                max_quantization, exact, used_subtract_green));
       }
     }
     cur_diff = PredictionCostSpatialHistogram(
@@ -369,7 +398,6 @@ static void CopyImageWithPrediction(int width, int height,
                                     int low_effort, int max_quantization,
                                     int exact, int used_subtract_green) {
   const int tiles_per_row = VP8LSubSampleSize(width, bits);
-  const int mask = (1 << bits) - 1;
   // The width of upper_row and current_row is one pixel larger than image width
   // to allow the top right pixel to point to the leftmost pixel of the next row
   // when at the right edge.
@@ -378,8 +406,6 @@ static void CopyImageWithPrediction(int width, int height,
   uint8_t* current_max_diffs = (uint8_t*)(current_row + width + 1);
   uint8_t* lower_max_diffs = current_max_diffs + width;
   int y;
-  int mode = 0;
-  VP8LPredictorFunc pred_func = NULL;
 
   for (y = 0; y < height; ++y) {
     int x;
@@ -390,11 +416,8 @@ static void CopyImageWithPrediction(int width, int height,
            sizeof(*argb) * (width + (y + 1 < height)));
 
     if (low_effort) {
-      for (x = 0; x < width; ++x) {
+      PredictBatch(kPredLowEffort, 0, y, width, current_row, upper_row,
-        const uint32_t predict = Predict(VP8LPredictors[kPredLowEffort], x, y,
+                   argb + y * width);
-                                         current_row, upper_row);
-        argb[y * width + x] = VP8LSubPixels(current_row[x], predict);
-      }
     } else {
       if (max_quantization > 1) {
         // Compute max_diffs for the lower row now, because that needs the
@@ -408,14 +431,15 @@ static void CopyImageWithPrediction(int width, int height,
                          used_subtract_green);
         }
       }
-      for (x = 0; x < width; ++x) {
+      for (x = 0; x < width;) {
-        if ((x & mask) == 0) {
+        const int mode =
-          mode = (modes[(y >> bits) * tiles_per_row + (x >> bits)] >> 8) & 0xff;
+            (modes[(y >> bits) * tiles_per_row + (x >> bits)] >> 8) & 0xff;
-          pred_func = VP8LPredictors[mode];
+        int x_end = x + (1 << bits);
-        }
+        if (x_end > width) x_end = width;
-        argb[y * width + x] = GetResidual(
+        GetResidual(width, height, upper_row, current_row, current_max_diffs,
-            width, height, upper_row, current_row, current_max_diffs, mode,
+                    mode, x, x_end, y, max_quantization, exact,
-            pred_func, x, y, max_quantization, exact, used_subtract_green);
+                    used_subtract_green, argb + y * width + x);
+        x = x_end;
       }
     }
   }

src/enc/vp8l.c

@@ -319,7 +319,10 @@ static int GetHistoBits(int method, int use_palette, int width, int height) {
 
 static int GetTransformBits(int method, int histo_bits) {
   const int max_transform_bits = (method < 4) ? 6 : (method > 4) ? 4 : 5;
-  return (histo_bits > max_transform_bits) ? max_transform_bits : histo_bits;
+  const int res =
+      (histo_bits > max_transform_bits) ? max_transform_bits : histo_bits;
+  assert(res <= MAX_TRANSFORM_BITS);
+  return res;
 }
 
 static int AnalyzeAndInit(VP8LEncoder* const enc) {

src/enc/vp8li.h

@@ -24,6 +24,9 @@
 extern "C" {
 #endif
 
+// maximum value of transform_bits_ in VP8LEncoder.
+#define MAX_TRANSFORM_BITS 6
+
 typedef struct {
   const WebPConfig* config_;      // user configuration and parameters
   const WebPPicture* pic_;        // input picture.
@@ -39,7 +42,7 @@ typedef struct {
 
   // Encoding parameters derived from quality parameter.
   int histo_bits_;
-  int transform_bits_;
+  int transform_bits_;    // <= MAX_TRANSFORM_BITS.
   int cache_bits_;        // If equal to 0, don't use color cache.
 
   // Encoding parameters derived from image characteristics.