simplify the alpha-filter testing loop

Change-Id: Iacebae749c37edc87a3c94c76cd589a2565ee642

src/enc/alpha.c

@@ -97,6 +97,7 @@ static int EncodeLossless(const uint8_t* const data, int width, int height,
 
 // -----------------------------------------------------------------------------
 
+// This function always returns an initialized 'bw' object, even upon error.
 static int EncodeAlphaInternal(const uint8_t* const data, int width, int height,
                                int method, int filter, int reduce_levels,
                                int effort_level,  // in [0..6] range
@@ -175,6 +176,92 @@ static int GetNumColors(const uint8_t* data, int width, int height,
   return colors;
 }
 
+// Given the input 'filter' option, return an OR'd bit-set of filters to try.
+static uint32_t GetFilterMap(const uint8_t* alpha, int width, int height,
+                             int filter, int effort_level) {
+  uint32_t bit_map = 0U;
+  if (filter == WEBP_FILTER_FAST) {
+    // Quick estimate of the best candidate.
+    int try_filter_none = (effort_level > 3);
+    const int kMinColorsForFilterNone = 16;
+    const int kMaxColorsForFilterNone = 192;
+    const int num_colors = GetNumColors(alpha, width, height, width);
+    // For low number of colors, NONE yeilds better compression.
+    filter = (num_colors <= kMinColorsForFilterNone) ? WEBP_FILTER_NONE :
+             EstimateBestFilter(alpha, width, height, width);
+    bit_map |= 1 << filter;
+    // For large number of colors, try FILTER_NONE in addition to the best
+    // filter as well.
+    if (try_filter_none || num_colors > kMaxColorsForFilterNone) {
+      bit_map |= 1 << WEBP_FILTER_NONE;
+    }
+  } else if (filter == WEBP_FILTER_NONE) {
+    bit_map = 1 << WEBP_FILTER_NONE;
+  } else {  // WEBP_FILTER_BEST
+    bit_map = (1 << WEBP_FILTER_LAST) - 1;  // try all.
+  }
+  return bit_map;
+}
+
+// Small struct to hold the result of a filter mode compression attempt.
+typedef struct {
+  size_t score;
+  VP8BitWriter bw;
+  WebPAuxStats stats;
+} FilterTrial;
+
+static void InitFilterTrial(FilterTrial* const score) {
+  score->score = (size_t)~0U;
+  VP8BitWriterInit(&score->bw, 0);
+}
+
+static int ApplyFilters(const uint8_t* alpha, int width, int height,
+                        uint64_t data_size, int method, int filter,
+                        int reduce_levels, int effort_level,
+                        uint8_t** const output, size_t* const output_size,
+                        WebPAuxStats* const stats) {
+  int ok = 1;
+  uint8_t* filtered_alpha = NULL;
+  uint32_t try_map = GetFilterMap(alpha, width, height, filter, effort_level);
+
+  filtered_alpha = (uint8_t*)malloc(data_size);
+  ok = (filtered_alpha != NULL);
+
+  if (ok) {
+    FilterTrial best;
+    InitFilterTrial(&best);
+
+    for (filter = WEBP_FILTER_NONE; try_map; ++filter, try_map >>= 1) {
+      if (try_map & 1) {
+        FilterTrial trial;
+        ok = EncodeAlphaInternal(alpha, width, height, method, filter,
+                                 reduce_levels, effort_level, filtered_alpha,
+                                 &trial.bw, stats);
+        if (ok) {
+          trial.score = VP8BitWriterSize(&trial.bw);
+          if (trial.score < best.score) {
+            VP8BitWriterWipeOut(&best.bw);
+            best = trial;
+            if (stats != NULL) best.stats = *stats;
+          }
+        } else {
+          VP8BitWriterWipeOut(&trial.bw);
+          VP8BitWriterWipeOut(&best.bw);
+          break;
+        }
+      }
+    }
+
+    if (ok) {
+      if (stats != NULL) *stats = best.stats;
+      *output_size = VP8BitWriterSize(&best.bw);
+      *output = VP8BitWriterBuf(&best.bw);
+    }
+    free(filtered_alpha);
+  }
+  return ok;
+}
+
 static int EncodeAlpha(VP8Encoder* const enc,
                        int quality, int method, int filter,
                        int effort_level,
@@ -223,105 +310,19 @@ static int EncodeAlpha(VP8Encoder* const enc,
   }
 
   if (ok) {
-    VP8BitWriter bw;
+    ok = ApplyFilters(quant_alpha, width, height, data_size, method, filter,
-    int test_filter;
+                      reduce_levels, effort_level, output, output_size,
-    uint8_t* filtered_alpha = NULL;
+                      pic->stats);
-    int try_filter_none = (effort_level > 3);
+    if (pic->stats != NULL) {  // need stats?
-
+      pic->stats->coded_size += (int)(*output_size);
-    if (filter == WEBP_FILTER_FAST) {  // Quick estimate of the best candidate.
+      enc->sse_[3] = sse;
-      const int kMinColorsForFilterNone = 16;
-      const int kMaxColorsForFilterNone = 192;
-      const int num_colors = GetNumColors(quant_alpha, width, height, width);
-      // For low number of colors, NONE yeilds better compression.
-      filter = (num_colors <= kMinColorsForFilterNone) ? WEBP_FILTER_NONE :
-               EstimateBestFilter(quant_alpha, width, height, width);
-      // For large number of colors, try FILTER_NONE in addition to the best
-      // filter as well.
-      if (num_colors > kMaxColorsForFilterNone) {
-        try_filter_none = 1;
-      }
     }
-
-    // Test for WEBP_FILTER_NONE for higher effort levels.
-    if (try_filter_none || filter == WEBP_FILTER_NONE) {
-      ok = EncodeAlphaInternal(quant_alpha, width, height,
-                               method, WEBP_FILTER_NONE, reduce_levels,
-                               effort_level, NULL, &bw, pic->stats);
-
-      if (!ok) {
-        VP8BitWriterWipeOut(&bw);
-        goto End;
-      }
-    }
-    // Stop?
-    if (filter == WEBP_FILTER_NONE) {
-      goto Ok;
-    }
-
-    filtered_alpha = (uint8_t*)malloc(data_size);
-    ok = (filtered_alpha != NULL);
-    if (!ok) {
-      goto End;
-    }
-
-    // Try the other mode(s).
-    {
-      WebPAuxStats best_stats;
-      size_t best_score = try_filter_none ?
-                          VP8BitWriterSize(&bw) : (size_t)~0U;
-      int wipe_tmp_bw = try_filter_none;
-
-      memset(&best_stats, 0, sizeof(best_stats));  // prevent spurious warning
-      if (pic->stats != NULL) best_stats = *pic->stats;
-      for (test_filter =
-           try_filter_none ? WEBP_FILTER_HORIZONTAL : WEBP_FILTER_NONE;
-           ok && (test_filter <= WEBP_FILTER_GRADIENT);
-           ++test_filter) {
-        VP8BitWriter tmp_bw;
-        if (filter != WEBP_FILTER_BEST && test_filter != filter) {
-          continue;
-        }
-        ok = EncodeAlphaInternal(quant_alpha, width, height,
-                                 method, test_filter, reduce_levels,
-                                 effort_level, filtered_alpha, &tmp_bw,
-                                 pic->stats);
-        if (ok) {
-          const size_t score = VP8BitWriterSize(&tmp_bw);
-          if (score < best_score) {
-            // swap bitwriter objects.
-            VP8BitWriter tmp = tmp_bw;
-            tmp_bw = bw;
-            bw = tmp;
-            best_score = score;
-            if (pic->stats != NULL) best_stats = *pic->stats;
-          }
-        } else {
-          VP8BitWriterWipeOut(&bw);
-        }
-        if (wipe_tmp_bw) {
-          VP8BitWriterWipeOut(&tmp_bw);
-        }
-        wipe_tmp_bw = 1;  // For next filter trial for WEBP_FILTER_BEST.
-      }
-      if (pic->stats != NULL) *pic->stats = best_stats;
-    }
- Ok:
-    if (ok) {
-      *output_size = VP8BitWriterSize(&bw);
-      *output = VP8BitWriterBuf(&bw);
-      if (pic->stats != NULL) {         // need stats?
-        pic->stats->coded_size += (int)(*output_size);
-        enc->sse_[3] = sse;
-      }
-    }
-    free(filtered_alpha);
   }
- End:
+
   free(quant_alpha);
   return ok;
 }
 
-
 //------------------------------------------------------------------------------
 // Main calls