From 47ddd5a4cc3ce115db3bb11576074b36f07dc774 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Wed, 16 Dec 2015 18:52:05 +0100 Subject: [PATCH] Move some codec logic out of ./dsp . The functions containing magic constants are moved out of ./dsp . VP8LPopulationCost got put back in ./enc VP8LGetCombinedEntropy is now unrefined (refinement happening in ./enc) VP8LBitsEntropy is now unrefined (refinement happening in ./enc) VP8LHistogramEstimateBits got put back in ./enc VP8LHistogramEstimateBitsBulk got deleted. Change-Id: I09c4101eebbc6f174403157026fe4a23a5316beb --- src/dsp/lossless.h | 34 ++++---- src/dsp/lossless_enc.c | 177 +++++++++-------------------------------- src/enc/histogram.c | 131 ++++++++++++++++++++++++++---- src/enc/histogram.h | 13 +++ src/enc/vp8l.c | 1 + 5 files changed, 184 insertions(+), 172 deletions(-) diff --git a/src/dsp/lossless.h b/src/dsp/lossless.h index a44659cf..d6418898 100644 --- a/src/dsp/lossless.h +++ b/src/dsp/lossless.h @@ -29,9 +29,6 @@ extern "C" { #include "../enc/delta_palettization.h" #endif // WEBP_EXPERIMENTAL_FEATURES -// Not a trivial literal symbol. -#define VP8L_NON_TRIVIAL_SYM (0xffffffff) - //------------------------------------------------------------------------------ // Decoding @@ -219,25 +216,24 @@ typedef VP8LStreaks (*VP8LCostCombinedCountFunc)(const uint32_t* X, extern VP8LCostCountFunc VP8LHuffmanCostCount; extern VP8LCostCombinedCountFunc VP8LHuffmanCostCombinedCount; -// Get the symbol entropy for the distribution 'population'. -// Set 'trivial_sym', if there's only one symbol present in the distribution. -double VP8LPopulationCost(const uint32_t* const population, int length, - uint32_t* const trivial_sym); +typedef struct { // small struct to hold bit entropy results + double entropy; // entropy + uint32_t sum; // sum of the population + int nonzeros; // number of non-zero elements in the population + uint32_t max_val; // maximum value in the population + uint32_t nonzero_code; // index of the last non-zero in the population +} VP8LBitEntropy; + +void VP8LBitEntropyInit(VP8LBitEntropy* const entropy); // Get the combined symbol entropy for the distributions 'X' and 'Y'. -double VP8LGetCombinedEntropy(const uint32_t* const X, - const uint32_t* const Y, int length); +void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X, + const uint32_t* const Y, int length, + VP8LBitEntropy* bit_entropy, + VP8LStreaks* stats); -double VP8LBitsEntropy(const uint32_t* const array, int n, - uint32_t* const trivial_symbol); - -// Estimate how many bits the combined entropy of literals and distance -// approximately maps to. -double VP8LHistogramEstimateBits(const VP8LHistogram* const p); - -// This function estimates the cost in bits excluding the bits needed to -// represent the entropy code itself. -double VP8LHistogramEstimateBitsBulk(const VP8LHistogram* const p); +void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n, + VP8LBitEntropy* entropy); typedef void (*VP8LHistogramAddFunc)(const VP8LHistogram* const a, const VP8LHistogram* const b, diff --git a/src/dsp/lossless_enc.c b/src/dsp/lossless_enc.c index 1be75e24..e9dbe82e 100644 --- a/src/dsp/lossless_enc.c +++ b/src/dsp/lossless_enc.c @@ -440,125 +440,56 @@ static float PredictionCostSpatialHistogram(const int accumulated[4][256], return (float)retval; } -static WEBP_INLINE double BitsEntropyRefine(int nonzeros, int sum, int max_val, - double retval) { - double mix; - if (nonzeros < 5) { - if (nonzeros <= 1) { - return 0; - } - // Two symbols, they will be 0 and 1 in a Huffman code. - // Let's mix in a bit of entropy to favor good clustering when - // distributions of these are combined. - if (nonzeros == 2) { - return 0.99 * sum + 0.01 * retval; - } - // No matter what the entropy says, we cannot be better than min_limit - // with Huffman coding. I am mixing a bit of entropy into the - // min_limit since it produces much better (~0.5 %) compression results - // perhaps because of better entropy clustering. - if (nonzeros == 3) { - mix = 0.95; - } else { - mix = 0.7; // nonzeros == 4. - } - } else { - mix = 0.627; - } - - { - double min_limit = 2 * sum - max_val; - min_limit = mix * min_limit + (1.0 - mix) * retval; - return (retval < min_limit) ? min_limit : retval; - } +void VP8LBitEntropyInit(VP8LBitEntropy* const entropy) { + entropy->entropy = 0.; + entropy->sum = 0; + entropy->nonzeros = 0; + entropy->max_val = 0; + entropy->nonzero_code = VP8L_NON_TRIVIAL_SYM; } -// Returns the entropy for the symbols in the input array. -// Also sets trivial_symbol to the code value, if the array has only one code -// value. Otherwise, set it to VP8L_NON_TRIVIAL_SYM. -double VP8LBitsEntropy(const uint32_t* const array, int n, - uint32_t* const trivial_symbol) { - double retval = 0.; - uint32_t sum = 0; - uint32_t nonzero_code = VP8L_NON_TRIVIAL_SYM; - int nonzeros = 0; - uint32_t max_val = 0; +void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n, + VP8LBitEntropy* entropy) { int i; + + VP8LBitEntropyInit(entropy); + for (i = 0; i < n; ++i) { if (array[i] != 0) { - sum += array[i]; - nonzero_code = i; - ++nonzeros; - retval -= VP8LFastSLog2(array[i]); - if (max_val < array[i]) { - max_val = array[i]; + entropy->sum += array[i]; + entropy->nonzero_code = i; + ++entropy->nonzeros; + entropy->entropy -= VP8LFastSLog2(array[i]); + if (entropy->max_val < array[i]) { + entropy->max_val = array[i]; } } } - retval += VP8LFastSLog2(sum); - if (trivial_symbol != NULL) { - *trivial_symbol = (nonzeros == 1) ? nonzero_code : VP8L_NON_TRIVIAL_SYM; - } - return BitsEntropyRefine(nonzeros, sum, max_val, retval); + entropy->entropy += VP8LFastSLog2(entropy->sum); } -static double InitialHuffmanCost(void) { - // Small bias because Huffman code length is typically not stored in - // full length. - static const int kHuffmanCodeOfHuffmanCodeSize = CODE_LENGTH_CODES * 3; - static const double kSmallBias = 9.1; - return kHuffmanCodeOfHuffmanCodeSize - kSmallBias; -} - -// Finalize the Huffman cost based on streak numbers and length type (<3 or >=3) -static double FinalHuffmanCost(const VP8LStreaks* const stats) { - double retval = InitialHuffmanCost(); - retval += stats->counts[0] * 1.5625 + 0.234375 * stats->streaks[0][1]; - retval += stats->counts[1] * 2.578125 + 0.703125 * stats->streaks[1][1]; - retval += 1.796875 * stats->streaks[0][0]; - retval += 3.28125 * stats->streaks[1][0]; - return retval; -} - -// Trampolines -static double HuffmanCost(const uint32_t* const population, int length) { - const VP8LStreaks stats = VP8LHuffmanCostCount(population, length); - return FinalHuffmanCost(&stats); -} - -// Aggregated costs -double VP8LPopulationCost(const uint32_t* const population, int length, - uint32_t* const trivial_sym) { - return - VP8LBitsEntropy(population, length, trivial_sym) + - HuffmanCost(population, length); -} - -double VP8LGetCombinedEntropy(const uint32_t* const X, - const uint32_t* const Y, int length) { - double bits_entropy_combined; - double huffman_cost_combined; +void VP8LGetCombinedEntropyUnrefined(const uint32_t* const X, + const uint32_t* const Y, int length, + VP8LBitEntropy* bit_entropy, + VP8LStreaks* stats) { int i; // Bit entropy variables. - double retval = 0.; - int sum = 0; - int nonzeros = 0; - uint32_t max_val = 0; int i_prev; uint32_t xy; // Huffman cost variables. int streak = 0; uint32_t xy_prev; - VP8LStreaks stats; - memset(&stats, 0, sizeof(stats)); + memset(stats, 0, sizeof(*stats)); + + VP8LBitEntropyInit(bit_entropy); // Treat the first value for the huffman cost: this is keeping the original // behavior, even though there is no first streak. // TODO(vrabaud): study proper behavior xy = X[0] + Y[0]; - ++stats.streaks[xy != 0][0]; + ++stats->streaks[xy != 0][0]; xy_prev = xy; i_prev = 0; @@ -571,17 +502,17 @@ double VP8LGetCombinedEntropy(const uint32_t* const X, // Gather info for the bit entropy. if (xy_prev != 0) { - sum += xy_prev * streak; - nonzeros += streak; - retval -= VP8LFastSLog2(xy_prev) * streak; - if (max_val < xy_prev) { - max_val = xy_prev; + bit_entropy->sum += xy_prev * streak; + bit_entropy->nonzeros += streak; + bit_entropy->entropy -= VP8LFastSLog2(xy_prev) * streak; + if (bit_entropy->max_val < xy_prev) { + bit_entropy->max_val = xy_prev; } } // Gather info for the huffman cost. - stats.counts[xy != 0] += (streak > 3); - stats.streaks[xy != 0][(streak > 3)] += streak; + stats->counts[xy != 0] += (streak > 3); + stats->streaks[xy != 0][(streak > 3)] += streak; xy_prev = xy; i_prev = i; @@ -591,47 +522,17 @@ double VP8LGetCombinedEntropy(const uint32_t* const X, // Finish off the last streak for bit entropy. if (xy != 0) { streak = i - i_prev; - sum += xy * streak; - nonzeros += streak; - retval -= VP8LFastSLog2(xy) * streak; - if (max_val < xy) { - max_val = xy; + bit_entropy->sum += xy * streak; + bit_entropy->nonzeros += streak; + bit_entropy->entropy -= VP8LFastSLog2(xy) * streak; + if (bit_entropy->max_val < xy) { + bit_entropy->max_val = xy; } } // Huffman cost is not updated with the last streak to keep original behavior. // TODO(vrabaud): study proper behavior - retval += VP8LFastSLog2(sum); - bits_entropy_combined = BitsEntropyRefine(nonzeros, sum, max_val, retval); - - huffman_cost_combined = FinalHuffmanCost(&stats); - - return bits_entropy_combined + huffman_cost_combined; -} - -// Estimates the Entropy + Huffman + other block overhead size cost. -double VP8LHistogramEstimateBits(const VP8LHistogram* const p) { - return - VP8LPopulationCost( - p->literal_, VP8LHistogramNumCodes(p->palette_code_bits_), NULL) - + VP8LPopulationCost(p->red_, NUM_LITERAL_CODES, NULL) - + VP8LPopulationCost(p->blue_, NUM_LITERAL_CODES, NULL) - + VP8LPopulationCost(p->alpha_, NUM_LITERAL_CODES, NULL) - + VP8LPopulationCost(p->distance_, NUM_DISTANCE_CODES, NULL) - + VP8LExtraCost(p->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES) - + VP8LExtraCost(p->distance_, NUM_DISTANCE_CODES); -} - -double VP8LHistogramEstimateBitsBulk(const VP8LHistogram* const p) { - return - VP8LBitsEntropy(p->literal_, VP8LHistogramNumCodes(p->palette_code_bits_), - NULL) - + VP8LBitsEntropy(p->red_, NUM_LITERAL_CODES, NULL) - + VP8LBitsEntropy(p->blue_, NUM_LITERAL_CODES, NULL) - + VP8LBitsEntropy(p->alpha_, NUM_LITERAL_CODES, NULL) - + VP8LBitsEntropy(p->distance_, NUM_DISTANCE_CODES, NULL) - + VP8LExtraCost(p->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES) - + VP8LExtraCost(p->distance_, NUM_DISTANCE_CODES); + bit_entropy->entropy += VP8LFastSLog2(bit_entropy->sum); } static WEBP_INLINE void UpdateHisto(int histo_argb[4][256], uint32_t argb) { diff --git a/src/enc/histogram.c b/src/enc/histogram.c index d2057011..1952e0a2 100644 --- a/src/enc/histogram.c +++ b/src/enc/histogram.c @@ -156,6 +156,108 @@ void VP8LHistogramAddSinglePixOrCopy(VP8LHistogram* const histo, } } +// ----------------------------------------------------------------------------- +// Entropy-related functions. + +static WEBP_INLINE double BitsEntropyRefine(const VP8LBitEntropy* entropy) { + double mix; + if (entropy->nonzeros < 5) { + if (entropy->nonzeros <= 1) { + return 0; + } + // Two symbols, they will be 0 and 1 in a Huffman code. + // Let's mix in a bit of entropy to favor good clustering when + // distributions of these are combined. + if (entropy->nonzeros == 2) { + return 0.99 * entropy->sum + 0.01 * entropy->entropy; + } + // No matter what the entropy says, we cannot be better than min_limit + // with Huffman coding. I am mixing a bit of entropy into the + // min_limit since it produces much better (~0.5 %) compression results + // perhaps because of better entropy clustering. + if (entropy->nonzeros == 3) { + mix = 0.95; + } else { + mix = 0.7; // nonzeros == 4. + } + } else { + mix = 0.627; + } + + { + double min_limit = 2 * entropy->sum - entropy->max_val; + min_limit = mix * min_limit + (1.0 - mix) * entropy->entropy; + return (entropy->entropy < min_limit) ? min_limit : entropy->entropy; + } +} + +double VP8LBitsEntropy(const uint32_t* const array, int n, + uint32_t* const trivial_symbol) { + VP8LBitEntropy entropy; + VP8LBitsEntropyUnrefined(array, n, &entropy); + if (trivial_symbol != NULL) { + *trivial_symbol = + (entropy.nonzeros == 1) ? entropy.nonzero_code : VP8L_NON_TRIVIAL_SYM; + } + + return BitsEntropyRefine(&entropy); +} + +static double InitialHuffmanCost(void) { + // Small bias because Huffman code length is typically not stored in + // full length. + static const int kHuffmanCodeOfHuffmanCodeSize = CODE_LENGTH_CODES * 3; + static const double kSmallBias = 9.1; + return kHuffmanCodeOfHuffmanCodeSize - kSmallBias; +} + +// Finalize the Huffman cost based on streak numbers and length type (<3 or >=3) +static double FinalHuffmanCost(const VP8LStreaks* const stats) { + double retval = InitialHuffmanCost(); + retval += stats->counts[0] * 1.5625 + 0.234375 * stats->streaks[0][1]; + retval += stats->counts[1] * 2.578125 + 0.703125 * stats->streaks[1][1]; + retval += 1.796875 * stats->streaks[0][0]; + retval += 3.28125 * stats->streaks[1][0]; + return retval; +} + +// Trampolines +static double HuffmanCost(const uint32_t* const population, int length) { + const VP8LStreaks stats = VP8LHuffmanCostCount(population, length); + return FinalHuffmanCost(&stats); +} + +// Get the symbol entropy for the distribution 'population'. +// Set 'trivial_sym', if there's only one symbol present in the distribution. +static double PopulationCost(const uint32_t* const population, int length, + uint32_t* const trivial_sym) { + return VP8LBitsEntropy(population, length, trivial_sym) + + HuffmanCost(population, length); +} + +static WEBP_INLINE double GetCombinedEntropy(const uint32_t* const X, + const uint32_t* const Y, + int length) { + VP8LBitEntropy bit_entropy; + VP8LStreaks stats; + VP8LGetCombinedEntropyUnrefined(X, Y, length, &bit_entropy, &stats); + + return BitsEntropyRefine(&bit_entropy) + FinalHuffmanCost(&stats); +} + +// Estimates the Entropy + Huffman + other block overhead size cost. +double VP8LHistogramEstimateBits(const VP8LHistogram* const p) { + return + PopulationCost( + p->literal_, VP8LHistogramNumCodes(p->palette_code_bits_), NULL) + + PopulationCost(p->red_, NUM_LITERAL_CODES, NULL) + + PopulationCost(p->blue_, NUM_LITERAL_CODES, NULL) + + PopulationCost(p->alpha_, NUM_LITERAL_CODES, NULL) + + PopulationCost(p->distance_, NUM_DISTANCE_CODES, NULL) + + VP8LExtraCost(p->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES) + + VP8LExtraCost(p->distance_, NUM_DISTANCE_CODES); +} + // ----------------------------------------------------------------------------- // Various histogram combine/cost-eval functions @@ -165,26 +267,25 @@ static int GetCombinedHistogramEntropy(const VP8LHistogram* const a, double* cost) { const int palette_code_bits = a->palette_code_bits_; assert(a->palette_code_bits_ == b->palette_code_bits_); - *cost += VP8LGetCombinedEntropy(a->literal_, b->literal_, - VP8LHistogramNumCodes(palette_code_bits)); + *cost += GetCombinedEntropy(a->literal_, b->literal_, + VP8LHistogramNumCodes(palette_code_bits)); *cost += VP8LExtraCostCombined(a->literal_ + NUM_LITERAL_CODES, b->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES); if (*cost > cost_threshold) return 0; - *cost += VP8LGetCombinedEntropy(a->red_, b->red_, NUM_LITERAL_CODES); + *cost += GetCombinedEntropy(a->red_, b->red_, NUM_LITERAL_CODES); if (*cost > cost_threshold) return 0; - *cost += VP8LGetCombinedEntropy(a->blue_, b->blue_, NUM_LITERAL_CODES); + *cost += GetCombinedEntropy(a->blue_, b->blue_, NUM_LITERAL_CODES); if (*cost > cost_threshold) return 0; - *cost += VP8LGetCombinedEntropy(a->alpha_, b->alpha_, NUM_LITERAL_CODES); + *cost += GetCombinedEntropy(a->alpha_, b->alpha_, NUM_LITERAL_CODES); if (*cost > cost_threshold) return 0; - *cost += VP8LGetCombinedEntropy(a->distance_, b->distance_, - NUM_DISTANCE_CODES); - *cost += VP8LExtraCostCombined(a->distance_, b->distance_, - NUM_DISTANCE_CODES); + *cost += GetCombinedEntropy(a->distance_, b->distance_, NUM_DISTANCE_CODES); + *cost += + VP8LExtraCostCombined(a->distance_, b->distance_, NUM_DISTANCE_CODES); if (*cost > cost_threshold) return 0; return 1; @@ -262,17 +363,17 @@ static void UpdateDominantCostRange( static void UpdateHistogramCost(VP8LHistogram* const h) { uint32_t alpha_sym, red_sym, blue_sym; - const double alpha_cost = VP8LPopulationCost(h->alpha_, NUM_LITERAL_CODES, - &alpha_sym); + const double alpha_cost = PopulationCost(h->alpha_, NUM_LITERAL_CODES, + &alpha_sym); const double distance_cost = - VP8LPopulationCost(h->distance_, NUM_DISTANCE_CODES, NULL) + + PopulationCost(h->distance_, NUM_DISTANCE_CODES, NULL) + VP8LExtraCost(h->distance_, NUM_DISTANCE_CODES); const int num_codes = VP8LHistogramNumCodes(h->palette_code_bits_); - h->literal_cost_ = VP8LPopulationCost(h->literal_, num_codes, NULL) + + h->literal_cost_ = PopulationCost(h->literal_, num_codes, NULL) + VP8LExtraCost(h->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES); - h->red_cost_ = VP8LPopulationCost(h->red_, NUM_LITERAL_CODES, &red_sym); - h->blue_cost_ = VP8LPopulationCost(h->blue_, NUM_LITERAL_CODES, &blue_sym); + h->red_cost_ = PopulationCost(h->red_, NUM_LITERAL_CODES, &red_sym); + h->blue_cost_ = PopulationCost(h->blue_, NUM_LITERAL_CODES, &blue_sym); h->bit_cost_ = h->literal_cost_ + h->red_cost_ + h->blue_cost_ + alpha_cost + distance_cost; if ((alpha_sym | red_sym | blue_sym) == VP8L_NON_TRIVIAL_SYM) { diff --git a/src/enc/histogram.h b/src/enc/histogram.h index adb16c01..d303d1d5 100644 --- a/src/enc/histogram.h +++ b/src/enc/histogram.h @@ -24,6 +24,9 @@ extern "C" { #endif +// Not a trivial literal symbol. +#define VP8L_NON_TRIVIAL_SYM (0xffffffff) + // A simple container for histograms of data. typedef struct { // literal_ contains green literal, palette-code and @@ -103,6 +106,16 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize, VP8LHistogramSet* const tmp_histos, uint16_t* const histogram_symbols); +// Returns the entropy for the symbols in the input array. +// Also sets trivial_symbol to the code value, if the array has only one code +// value. Otherwise, set it to VP8L_NON_TRIVIAL_SYM. +double VP8LBitsEntropy(const uint32_t* const array, int n, + uint32_t* const trivial_symbol); + +// Estimate how many bits the combined entropy of literals and distance +// approximately maps to. +double VP8LHistogramEstimateBits(const VP8LHistogram* const p); + #ifdef __cplusplus } #endif diff --git a/src/enc/vp8l.c b/src/enc/vp8l.c index 84611a6f..db94e78a 100644 --- a/src/enc/vp8l.c +++ b/src/enc/vp8l.c @@ -16,6 +16,7 @@ #include #include "./backward_references.h" +#include "./histogram.h" #include "./vp8enci.h" #include "./vp8li.h" #include "../dsp/lossless.h"