Unify some entropy functions.

The code and logic is unified when computing bit entropy + Huffman cost.

Speed-wise, we gain 8% for lossless encoding.
Logic-wise, the beginning/end of the distributions are handled properly
and the compression ratio does not change much.

Change-Id: Ifa91d7d3e667c9a9a421faec4e845ecb6479a633

src/dsp/lossless.h

@@ -208,12 +208,9 @@ typedef struct {        // small struct to hold counters
   int streaks[2][2];    // [zero/non-zero][streak<3 / streak>=3]
 } VP8LStreaks;
 
-typedef VP8LStreaks (*VP8LCostCountFunc)(const uint32_t* population,
-                                         int length);
 typedef VP8LStreaks (*VP8LCostCombinedCountFunc)(const uint32_t* X,
                                                  const uint32_t* Y, int length);
 
-extern VP8LCostCountFunc VP8LHuffmanCostCount;
 extern VP8LCostCombinedCountFunc VP8LHuffmanCostCombinedCount;
 
 typedef struct {            // small struct to hold bit entropy results
@@ -226,14 +223,28 @@ typedef struct {            // small struct to hold bit entropy results
 
 void VP8LBitEntropyInit(VP8LBitEntropy* const entropy);
 
-// Get the combined symbol entropy for the distributions 'X' and 'Y'.
+// Get the combined symbol bit entropy and Huffman cost stats for the
+// distributions 'X' and 'Y'. Those results can then be refined according to
+// codec specific heuristics.
 void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X,
                                      const uint32_t* const Y, int length,
-                                     VP8LBitEntropy* bit_entropy,
+                                     VP8LBitEntropy* const bit_entropy,
-                                     VP8LStreaks* stats);
+                                     VP8LStreaks* const stats);
+// Get the entropy for the distribution 'X'.
+void VP8LGetEntropyUnrefined(const uint32_t* const X, int length,
+                             VP8LBitEntropy* const bit_entropy,
+                             VP8LStreaks* const stats);
 
 void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n,
-                              VP8LBitEntropy* entropy);
+                              VP8LBitEntropy* const entropy);
+
+typedef void (*GetEntropyUnrefinedHelperFunc)(uint32_t val, int i,
+                                              uint32_t* const val_prev,
+                                              int* const i_prev,
+                                              VP8LBitEntropy* const bit_entropy,
+                                              VP8LStreaks* const stats);
+// Internal function used by VP8LGet*EntropyUnrefined.
+extern GetEntropyUnrefinedHelperFunc VP8LGetEntropyUnrefinedHelper;
 
 typedef void (*VP8LHistogramAddFunc)(const VP8LHistogram* const a,
                                      const VP8LHistogram* const b,

src/dsp/lossless_enc.c

@@ -449,7 +449,7 @@ void VP8LBitEntropyInit(VP8LBitEntropy* const entropy) {
 }
 
 void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n,
-                              VP8LBitEntropy* entropy) {
+                              VP8LBitEntropy* const entropy) {
   int i;
 
   VP8LBitEntropyInit(entropy);
@@ -468,69 +468,70 @@ void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n,
   entropy->entropy += VP8LFastSLog2(entropy->sum);
 }
 
-void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X,
+static WEBP_INLINE void GetEntropyUnrefinedHelper(
-                                     const uint32_t* const Y, int length,
+    uint32_t val, int i, uint32_t* const val_prev, int* const i_prev,
-                                     VP8LBitEntropy* bit_entropy,
+    VP8LBitEntropy* const bit_entropy, VP8LStreaks* const stats) {
-                                     VP8LStreaks* stats) {
+  const int streak = i - *i_prev;
-  int i;
-
-  // Bit entropy variables.
-  int i_prev;
-  uint32_t xy;
-
-  // Huffman cost variables.
-  int streak = 0;
-  uint32_t xy_prev;
-  memset(stats, 0, sizeof(*stats));
-
-  VP8LBitEntropyInit(bit_entropy);
-
-  // Treat the first value for the huffman cost: this is keeping the original
-  // behavior, even though there is no first streak.
-  // TODO(vrabaud): study proper behavior
-  xy = X[0] + Y[0];
-  ++stats->streaks[xy != 0][0];
-  xy_prev = xy;
-  i_prev = 0;
-
-  for (i = 1; i < length; ++i) {
-    xy = X[i] + Y[i];
-
-    // Process data by streaks for both bit entropy and huffman cost.
-    if (xy != xy_prev) {
-      streak = i - i_prev;
 
   // Gather info for the bit entropy.
-      if (xy_prev != 0) {
+  if (*val_prev != 0) {
-        bit_entropy->sum += xy_prev * streak;
+    bit_entropy->sum += (*val_prev) * streak;
     bit_entropy->nonzeros += streak;
-        bit_entropy->entropy -= VP8LFastSLog2(xy_prev) * streak;
+    bit_entropy->nonzero_code = *i_prev;
-        if (bit_entropy->max_val < xy_prev) {
+    bit_entropy->entropy -= VP8LFastSLog2(*val_prev) * streak;
-          bit_entropy->max_val = xy_prev;
+    if (bit_entropy->max_val < *val_prev) {
+      bit_entropy->max_val = *val_prev;
     }
   }
 
-      // Gather info for the huffman cost.
+  // Gather info for the Huffman cost.
-      stats->counts[xy != 0] += (streak > 3);
+  stats->counts[*val_prev != 0] += (streak > 3);
-      stats->streaks[xy != 0][(streak > 3)] += streak;
+  stats->streaks[*val_prev != 0][(streak > 3)] += streak;
 
-      xy_prev = xy;
+  *val_prev = val;
-      i_prev = i;
+  *i_prev = i;
-    }
 }
 
-  // Finish off the last streak for bit entropy.
+void VP8LGetEntropyUnrefined(const uint32_t* const X, int length,
-  if (xy != 0) {
+                             VP8LBitEntropy* const bit_entropy,
-    streak = i - i_prev;
+                             VP8LStreaks* const stats) {
-    bit_entropy->sum += xy * streak;
+  int i;
-    bit_entropy->nonzeros += streak;
+  int i_prev = 0;
-    bit_entropy->entropy -= VP8LFastSLog2(xy) * streak;
+  uint32_t x_prev = X[0];
-    if (bit_entropy->max_val < xy) {
+
-      bit_entropy->max_val = xy;
+  memset(stats, 0, sizeof(*stats));
+  VP8LBitEntropyInit(bit_entropy);
+
+  for (i = 1; i < length; ++i) {
+    const uint32_t x = X[i];
+    if (x != x_prev) {
+      VP8LGetEntropyUnrefinedHelper(x, i, &x_prev, &i_prev, bit_entropy, stats);
     }
   }
-  // Huffman cost is not updated with the last streak to keep original behavior.
+  VP8LGetEntropyUnrefinedHelper(0, i, &x_prev, &i_prev, bit_entropy, stats);
-  // TODO(vrabaud): study proper behavior
+
+  bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum);
+}
+
+void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X,
+                                     const uint32_t* const Y, int length,
+                                     VP8LBitEntropy* const bit_entropy,
+                                     VP8LStreaks* const stats) {
+  int i = 1;
+  int i_prev = 0;
+  uint32_t xy_prev = X[0] + Y[0];
+
+  memset(stats, 0, sizeof(*stats));
+  VP8LBitEntropyInit(bit_entropy);
+
+  for (i = 1; i < length; ++i) {
+    const uint32_t xy = X[i] + Y[i];
+    if (xy != xy_prev) {
+      VP8LGetEntropyUnrefinedHelper(xy, i, &xy_prev, &i_prev, bit_entropy,
+                                    stats);
+    }
+  }
+  VP8LGetEntropyUnrefinedHelper(0, i, &xy_prev, &i_prev, bit_entropy, stats);
 
   bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum);
 }
@@ -1100,27 +1101,6 @@ static double ExtraCostCombined(const uint32_t* X, const uint32_t* Y,
   return cost;
 }
 
-// Returns the various RLE counts
-static VP8LStreaks HuffmanCostCount(const uint32_t* population, int length) {
-  int i;
-  int streak = 0;
-  VP8LStreaks stats;
-  memset(&stats, 0, sizeof(stats));
-  for (i = 0; i < length - 1; ++i) {
-    ++streak;
-    if (population[i] == population[i + 1]) {
-      continue;
-    }
-    stats.counts[population[i] != 0] += (streak > 3);
-    stats.streaks[population[i] != 0][(streak > 3)] += streak;
-    streak = 0;
-  }
-  ++streak;
-  stats.counts[population[i] != 0] += (streak > 3);
-  stats.streaks[population[i] != 0][(streak > 3)] += streak;
-  return stats;
-}
-
 //------------------------------------------------------------------------------
 
 static void HistogramAdd(const VP8LHistogram* const a,
@@ -1172,7 +1152,7 @@ VP8LCostFunc VP8LExtraCost;
 VP8LCostCombinedFunc VP8LExtraCostCombined;
 VP8LCombinedShannonEntropyFunc VP8LCombinedShannonEntropy;
 
-VP8LCostCountFunc VP8LHuffmanCostCount;
+GetEntropyUnrefinedHelperFunc VP8LGetEntropyUnrefinedHelper;
 
 VP8LHistogramAddFunc VP8LHistogramAdd;
 
@@ -1204,7 +1184,7 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInit(void) {
   VP8LExtraCostCombined = ExtraCostCombined;
   VP8LCombinedShannonEntropy = CombinedShannonEntropy;
 
-  VP8LHuffmanCostCount = HuffmanCostCount;
+  VP8LGetEntropyUnrefinedHelper = GetEntropyUnrefinedHelper;
 
   VP8LHistogramAdd = HistogramAdd;
 

src/dsp/lossless_enc_mips32.c

@@ -217,51 +217,31 @@ static double ExtraCostCombined(const uint32_t* const X,
   );
 
 // Returns the various RLE counts
-static VP8LStreaks HuffmanCostCount(const uint32_t* population, int length) {
+static WEBP_INLINE void GetEntropyUnrefinedHelper(
-  int i;
+    uint32_t val, int i, uint32_t* const val_prev, int* const i_prev,
-  int streak = 0;
+    VP8LBitEntropy* const bit_entropy, VP8LStreaks* const stats) {
-  VP8LStreaks stats;
+  int* const pstreaks = &stats->streaks[0][0];
-  int* const pstreaks = &stats.streaks[0][0];
+  int* const pcnts = &stats->counts[0];
-  int* const pcnts = &stats.counts[0];
   int temp0, temp1, temp2, temp3;
-  memset(&stats, 0, sizeof(stats));
+  const int streak = i - *i_prev;
-  for (i = 0; i < length - 1; ++i) {
-    ++streak;
-    if (population[i] == population[i + 1]) {
-      continue;
-    }
-    temp0 = (population[i] != 0);
-    HUFFMAN_COST_PASS
-    streak = 0;
-  }
-  ++streak;
-  temp0 = (population[i] != 0);
-  HUFFMAN_COST_PASS
 
-  return stats;
+  // Gather info for the bit entropy.
+  if (*val_prev != 0) {
+    bit_entropy->sum += (*val_prev) * streak;
+    bit_entropy->nonzeros += streak;
+    bit_entropy->nonzero_code = *i_prev;
+    bit_entropy->entropy -= VP8LFastSLog2(*val_prev) * streak;
+    if (bit_entropy->max_val < *val_prev) {
+      bit_entropy->max_val = *val_prev;
+    }
   }
 
-static VP8LStreaks HuffmanCostCombinedCount(const uint32_t* X,
+  // Gather info for the Huffman cost.
-                                            const uint32_t* Y, int length) {
+  temp0 = (*val_prev != 0);
-  int i;
-  int streak = 0;
-  uint32_t xy_prev = 0xffffffff;
-  VP8LStreaks stats;
-  int* const pstreaks = &stats.streaks[0][0];
-  int* const pcnts = &stats.counts[0];
-  int temp0, temp1, temp2, temp3;
-  memset(&stats, 0, sizeof(stats));
-  for (i = 0; i < length; ++i) {
-    const uint32_t xy = X[i] + Y[i];
-    ++streak;
-    if (xy != xy_prev) {
-      temp0 = (xy != 0);
   HUFFMAN_COST_PASS
-      streak = 0;
+
-      xy_prev = xy;
+  *val_prev = val;
-    }
+  *i_prev = i;
-  }
-  return stats;
 }
 
 #define ASM_START                                       \
@@ -399,14 +379,7 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitMIPS32(void) {
   VP8LFastLog2Slow = FastLog2Slow;
   VP8LExtraCost = ExtraCost;
   VP8LExtraCostCombined = ExtraCostCombined;
-  VP8LHuffmanCostCount = HuffmanCostCount;
+  VP8LGetEntropyUnrefinedHelper = GetEntropyUnrefinedHelper;
-// TODO(mips team): rewrite VP8LGetCombinedEntropy (which used to use
-// HuffmanCostCombinedCount) with MIPS optimizations
-#if 0
-  VP8LHuffmanCostCombinedCount = HuffmanCostCombinedCount;
-#else
- (void)HuffmanCostCombinedCount;
-#endif
   VP8LHistogramAdd = HistogramAdd;
 }
 

src/enc/histogram.c

@@ -221,18 +221,19 @@ static double FinalHuffmanCost(const VP8LStreaks* const stats) {
   return retval;
 }
 
-// Trampolines
-static double HuffmanCost(const uint32_t* const population, int length) {
-  const VP8LStreaks stats = VP8LHuffmanCostCount(population, length);
-  return FinalHuffmanCost(&stats);
-}
-
 // Get the symbol entropy for the distribution 'population'.
 // Set 'trivial_sym', if there's only one symbol present in the distribution.
 static double PopulationCost(const uint32_t* const population, int length,
                              uint32_t* const trivial_sym) {
-  return VP8LBitsEntropy(population, length, trivial_sym) +
+  VP8LBitEntropy bit_entropy;
-         HuffmanCost(population, length);
+  VP8LStreaks stats;
+  VP8LGetEntropyUnrefined(population, length, &bit_entropy, &stats);
+  if (trivial_sym != NULL) {
+    *trivial_sym = (bit_entropy.nonzeros == 1) ? bit_entropy.nonzero_code
+                                               : VP8L_NON_TRIVIAL_SYM;
+  }
+
+  return BitsEntropyRefine(&bit_entropy) + FinalHuffmanCost(&stats);
 }
 
 static WEBP_INLINE double GetCombinedEntropy(const uint32_t* const X,
@@ -363,8 +364,8 @@ static void UpdateDominantCostRange(
 
 static void UpdateHistogramCost(VP8LHistogram* const h) {
   uint32_t alpha_sym, red_sym, blue_sym;
-  const double alpha_cost = PopulationCost(h->alpha_, NUM_LITERAL_CODES,
+  const double alpha_cost =
-                                           &alpha_sym);
+      PopulationCost(h->alpha_, NUM_LITERAL_CODES, &alpha_sym);
   const double distance_cost =
       PopulationCost(h->distance_, NUM_DISTANCE_CODES, NULL) +
       VP8LExtraCost(h->distance_, NUM_DISTANCE_CODES);