Revert "Increase the transform bits if possible."

This reverts commit 971a03d820.

Reason for revert:
This creates non-lossless encodes.

Original change's description:
> Increase the transform bits if possible.
>
> This brings minor size improvements because repetitive values in
> the transform images are easily explainable through LZ77. Still,
> it makes an upcoming pull request a bit more stable.
>
> Change-Id: I1c7135675cb59b5e27ca960738d74465f10d0deb

Bug: oss-fuzz:69109, b:341475869
Change-Id: I3b9f21a5498735eb3681e62fb35bf9f9c2ed4f9f

src/enc/predictor_enc.c

@@ -14,8 +14,6 @@
 //          Urvang Joshi (urvang@google.com)
 //          Vincent Rabaud (vrabaud@google.com)
 
-#include <string.h>
-
 #include "src/dsp/lossless.h"
 #include "src/dsp/lossless_common.h"
 #include "src/enc/vp8i_enc.h"
@@ -469,72 +467,6 @@ static void CopyImageWithPrediction(int width, int height, int bits,
   }
 }
 
-// Checks whether 'image' is sub-samplable by finding the biggest power of 2
-// squares (defined by 'best_bits') of uniform value it is made out of.
-static void OptimizeSampling(uint32_t* const image, int full_width,
-                             int full_height, int bits, int* best_bits_out) {
-  int width = VP8LSubSampleSize(full_width, bits);
-  int height = VP8LSubSampleSize(full_height, bits);
-  int old_width, x, y, square_size;
-  int best_bits = bits;
-  *best_bits_out = bits;
-  // Check rows first.
-  while (best_bits < MAX_TRANSFORM_BITS) {
-    const int new_square_size = 1 << (best_bits + 1 - bits);
-    int is_good = 1;
-    square_size = 1 << (best_bits - bits);
-    for (y = 0; y + new_square_size <= height; y += new_square_size) {
-      // Check the first lines of consecutive line groups.
-      if (memcmp(&image[y * width], &image[(y + square_size) * width],
-                 width * sizeof(*image)) != 0) {
-        is_good = 0;
-        break;
-      }
-    }
-    if (is_good) {
-      ++best_bits;
-    } else {
-      break;
-    }
-  }
-  if (best_bits == bits) return;
-
-  // Check columns.
-  while (best_bits > bits) {
-    int is_good = 1;
-    square_size = 1 << (best_bits - bits);
-    for (y = 0; is_good && y < height; ++y) {
-      for (x = 0; is_good && x + square_size <= width; x += square_size) {
-        int i;
-        for (i = 1; i < square_size; ++i) {
-          if (image[y * width + x + i] != image[y * width + x]) {
-            is_good = 0;
-            break;
-          }
-        }
-      }
-    }
-    if (is_good) {
-      break;
-    } else {
-      --best_bits;
-    }
-  }
-  if (best_bits == bits) return;
-
-  // Subsample the image.
-  old_width = width;
-  square_size = 1 << (best_bits - bits);
-  width = VP8LSubSampleSize(full_width, best_bits);
-  height = VP8LSubSampleSize(full_height, best_bits);
-  for (y = 0; y < height; ++y) {
-    for (x = 0; x < width; ++x) {
-      image[y * width + x] = image[square_size * (y * old_width + x)];
-    }
-  }
-  *best_bits_out = best_bits;
-}
-
 // Finds the best predictor for each tile, and converts the image to residuals
 // with respect to predictions. If near_lossless_quality < 100, applies
 // near lossless processing, shaving off more bits of residuals for lower
@@ -544,7 +476,7 @@ int VP8LResidualImage(int width, int height, int bits, int low_effort,
                       uint32_t* const image, int near_lossless_quality,
                       int exact, int used_subtract_green,
                       const WebPPicture* const pic, int percent_range,
-                      int* const percent, int* const best_bits) {
+                      int* const percent) {
   const int tiles_per_row = VP8LSubSampleSize(width, bits);
   const int tiles_per_col = VP8LSubSampleSize(height, bits);
   int percent_start = *percent;
@@ -554,7 +486,6 @@ int VP8LResidualImage(int width, int height, int bits, int low_effort,
     for (i = 0; i < tiles_per_row * tiles_per_col; ++i) {
       image[i] = ARGB_BLACK | (kPredLowEffort << 8);
     }
-    *best_bits = bits;
   } else {
     int tile_y;
     uint32_t histo[HISTO_SIZE] = { 0 };
@@ -573,10 +504,9 @@ int VP8LResidualImage(int width, int height, int bits, int low_effort,
         return 0;
       }
     }
-    OptimizeSampling(image, width, height, bits, best_bits);
   }
 
-  CopyImageWithPrediction(width, height, *best_bits, image, argb_scratch, argb,
+  CopyImageWithPrediction(width, height, bits, image, argb_scratch, argb,
                           low_effort, max_quantization, exact,
                           used_subtract_green);
   return WebPReportProgress(pic, percent_start + percent_range, percent);
@@ -794,7 +724,7 @@ static void CopyTileWithColorTransform(int xsize, int ysize,
 int VP8LColorSpaceTransform(int width, int height, int bits, int quality,
                             uint32_t* const argb, uint32_t* image,
                             const WebPPicture* const pic, int percent_range,
-                            int* const percent, int* best_bits) {
+                            int* const percent) {
   const int max_tile_size = 1 << bits;
   const int tile_xsize = VP8LSubSampleSize(width, bits);
   const int tile_ysize = VP8LSubSampleSize(height, bits);
@@ -854,6 +784,5 @@ int VP8LColorSpaceTransform(int width, int height, int bits, int quality,
       return 0;
     }
   }
-  OptimizeSampling(image, width, height, bits, best_bits);
   return 1;
 }

src/enc/vp8l_enc.c

@@ -1071,27 +1071,26 @@ static int ApplyPredictFilter(VP8LEncoder* const enc, int width, int height,
                               int quality, int low_effort,
                               int used_subtract_green, VP8LBitWriter* const bw,
                               int percent_range, int* const percent) {
-  const int min_bits = enc->predictor_transform_bits_;
+  const int pred_bits = enc->predictor_transform_bits_;
-  int best_bits;
+  const int transform_width = VP8LSubSampleSize(width, pred_bits);
+  const int transform_height = VP8LSubSampleSize(height, pred_bits);
   // we disable near-lossless quantization if palette is used.
   const int near_lossless_strength =
       enc->use_palette_ ? 100 : enc->config_->near_lossless;
 
-  if (!VP8LResidualImage(width, height, min_bits, low_effort, enc->argb_,
+  if (!VP8LResidualImage(
-                         enc->argb_scratch_, enc->transform_data_,
+          width, height, pred_bits, low_effort, enc->argb_, enc->argb_scratch_,
-                         near_lossless_strength, enc->config_->exact,
+          enc->transform_data_, near_lossless_strength, enc->config_->exact,
-                         used_subtract_green, enc->pic_, percent_range / 2,
+          used_subtract_green, enc->pic_, percent_range / 2, percent)) {
-                         percent, &best_bits)) {
     return 0;
   }
   VP8LPutBits(bw, TRANSFORM_PRESENT, 1);
   VP8LPutBits(bw, PREDICTOR_TRANSFORM, 2);
-  assert(best_bits >= MIN_TRANSFORM_BITS && best_bits <= MAX_TRANSFORM_BITS);
+  assert(pred_bits >= MIN_TRANSFORM_BITS && pred_bits <= MAX_TRANSFORM_BITS);
-  VP8LPutBits(bw, best_bits - MIN_TRANSFORM_BITS, NUM_TRANSFORM_BITS);
+  VP8LPutBits(bw, pred_bits - MIN_TRANSFORM_BITS, NUM_TRANSFORM_BITS);
-  enc->predictor_transform_bits_ = best_bits;
   return EncodeImageNoHuffman(
-      bw, enc->transform_data_, &enc->hash_chain_, &enc->refs_[0],
+      bw, enc->transform_data_, (VP8LHashChain*)&enc->hash_chain_,
-      VP8LSubSampleSize(width, best_bits), VP8LSubSampleSize(height, best_bits),
+      (VP8LBackwardRefs*)&enc->refs_[0], transform_width, transform_height,
       quality, low_effort, enc->pic_, percent_range - percent_range / 2,
       percent);
 }
@@ -1100,22 +1099,24 @@ static int ApplyCrossColorFilter(VP8LEncoder* const enc, int width, int height,
                                  int quality, int low_effort,
                                  VP8LBitWriter* const bw, int percent_range,
                                  int* const percent) {
-  const int min_bits = enc->cross_color_transform_bits_;
+  const int ccolor_transform_bits = enc->cross_color_transform_bits_;
-  int best_bits;
+  const int transform_width = VP8LSubSampleSize(width, ccolor_transform_bits);
+  const int transform_height = VP8LSubSampleSize(height, ccolor_transform_bits);
 
-  if (!VP8LColorSpaceTransform(width, height, min_bits, quality, enc->argb_,
+  if (!VP8LColorSpaceTransform(width, height, ccolor_transform_bits, quality,
-                               enc->transform_data_, enc->pic_,
+                               enc->argb_, enc->transform_data_, enc->pic_,
-                               percent_range / 2, percent, &best_bits)) {
+                               percent_range / 2, percent)) {
     return 0;
   }
   VP8LPutBits(bw, TRANSFORM_PRESENT, 1);
   VP8LPutBits(bw, CROSS_COLOR_TRANSFORM, 2);
-  assert(best_bits >= MIN_TRANSFORM_BITS && best_bits <= MAX_TRANSFORM_BITS);
+  assert(ccolor_transform_bits >= MIN_TRANSFORM_BITS &&
-  VP8LPutBits(bw, best_bits - MIN_TRANSFORM_BITS, NUM_TRANSFORM_BITS);
+         ccolor_transform_bits <= MAX_TRANSFORM_BITS);
-  enc->cross_color_transform_bits_ = best_bits;
+  VP8LPutBits(bw, ccolor_transform_bits - MIN_TRANSFORM_BITS,
+              NUM_TRANSFORM_BITS);
   return EncodeImageNoHuffman(
-      bw, enc->transform_data_, &enc->hash_chain_, &enc->refs_[0],
+      bw, enc->transform_data_, (VP8LHashChain*)&enc->hash_chain_,
-      VP8LSubSampleSize(width, best_bits), VP8LSubSampleSize(height, best_bits),
+      (VP8LBackwardRefs*)&enc->refs_[0], transform_width, transform_height,
       quality, low_effort, enc->pic_, percent_range - percent_range / 2,
       percent);
 }

src/enc/vp8li_enc.h

@@ -110,12 +110,12 @@ int VP8LResidualImage(int width, int height, int bits, int low_effort,
                       uint32_t* const image, int near_lossless_quality,
                       int exact, int used_subtract_green,
                       const WebPPicture* const pic, int percent_range,
-                      int* const percent, int* const best_bits);
+                      int* const percent);
 
 int VP8LColorSpaceTransform(int width, int height, int bits, int quality,
                             uint32_t* const argb, uint32_t* image,
                             const WebPPicture* const pic, int percent_range,
-                            int* const percent, int* const best_bits);
+                            int* const percent);
 
 //------------------------------------------------------------------------------