From d0d88990d80396fac1a2c1dc49dacbf63420dc93 Mon Sep 17 00:00:00 2001 From: Pascal Massimino Date: Mon, 30 Apr 2012 19:46:22 +0000 Subject: [PATCH] more consolidation: introduce VP8LHistogramSet VP8LHistogramSet is container for pointers to histograms that we can shuffle around. Allocation is one big chunk of memory. Downside is that we don't de-allocate memory on-the-go during HistogramRefine(). + renamed HistogramRefine() into HistogramRemap(), so we don't confuse with "HistogramCombine" + made VP8LHistogramClear() static. Change-Id: Idf1a748a871c3b942cca5c8050072ccd82c7511d --- src/enc/histogram.c | 159 +++++++++++++++++++++----------------------- src/enc/histogram.h | 22 +++--- src/enc/vp8l.c | 66 +++++++++--------- 3 files changed, 121 insertions(+), 126 deletions(-) diff --git a/src/enc/histogram.c b/src/enc/histogram.c index 551bfec2..0958b6b8 100644 --- a/src/enc/histogram.c +++ b/src/enc/histogram.c @@ -17,20 +17,7 @@ #include "./histogram.h" #include "../dsp/lossless.h" -void VP8LHistogramCreate(VP8LHistogram* const p, - const VP8LBackwardRefs* const refs, - int palette_code_bits) { - int i; - if (palette_code_bits >= 0) { - p->palette_code_bits_ = palette_code_bits; - } - VP8LHistogramClear(p); - for (i = 0; i < refs->size; ++i) { - VP8LHistogramAddSinglePixOrCopy(p, &refs->refs[i]); - } -} - -void VP8LHistogramClear(VP8LHistogram* const p) { +static void HistogramClear(VP8LHistogram* const p) { memset(p->literal_, 0, sizeof(p->literal_)); memset(p->red_, 0, sizeof(p->red_)); memset(p->blue_, 0, sizeof(p->blue_)); @@ -39,38 +26,52 @@ void VP8LHistogramClear(VP8LHistogram* const p) { p->bit_cost_ = 0; } +void VP8LHistogramCreate(VP8LHistogram* const p, + const VP8LBackwardRefs* const refs, + int palette_code_bits) { + int i; + if (palette_code_bits >= 0) { + p->palette_code_bits_ = palette_code_bits; + } + HistogramClear(p); + for (i = 0; i < refs->size; ++i) { + VP8LHistogramAddSinglePixOrCopy(p, &refs->refs[i]); + } +} + void VP8LHistogramInit(VP8LHistogram* const p, int palette_code_bits) { p->palette_code_bits_ = palette_code_bits; - VP8LHistogramClear(p); + HistogramClear(p); } -VP8LHistogram** VP8LAllocateHistograms(int size, int cache_bits) { +VP8LHistogramSet* VP8LAllocateHistogramSet(int size, int cache_bits) { int i; - VP8LHistogram** const histos = - (VP8LHistogram**)calloc(size, sizeof(*histos)); - if (histos == NULL) return NULL; + VP8LHistogramSet* set; + VP8LHistogram* bulk; + const size_t total_size = sizeof(*set) + + size * sizeof(*set->histograms) + + size * sizeof(**set->histograms); + uint8_t* memory = (uint8_t*)malloc(total_size); + if (memory == NULL) return NULL; + + set = (VP8LHistogramSet*)memory; + memory += sizeof(*set); + set->histograms = (VP8LHistogram**)memory; + memory += size * sizeof(*set->histograms); + bulk = (VP8LHistogram*)memory; + set->max_size = size; + set->size = size; for (i = 0; i < size; ++i) { - histos[i] = (VP8LHistogram*)malloc(sizeof(**histos)); - if (histos[i] == NULL) { - VP8LDeleteHistograms(histos, i); - return NULL; - } - VP8LHistogramInit(histos[i], cache_bits); + set->histograms[i] = bulk + i; + VP8LHistogramInit(set->histograms[i], cache_bits); } - return histos; + return set; } -void VP8LDeleteHistograms(VP8LHistogram** const histograms, int size) { - if (histograms != NULL) { - int i; - for (i = 0; i < size; ++i) free(histograms[i]); - free(histograms); - } -} +// ----------------------------------------------------------------------------- void VP8LConvertPopulationCountTableToBitEstimates( - int num_symbols, - const int* const population_counts, + int num_symbols, const int* const population_counts, double* const output) { int sum = 0; int nonzeros = 0; @@ -241,7 +242,7 @@ double VP8LHistogramEstimateBitsHeader(const VP8LHistogram* const p) { static void HistogramBuildImage(int xsize, int histo_bits, const VP8LBackwardRefs* const backward_refs, - VP8LHistogram** const image) { + VP8LHistogramSet* const image) { int i; int x = 0, y = 0; const int histo_xsize = @@ -251,7 +252,7 @@ static void HistogramBuildImage(int xsize, int histo_bits, const int ix = (histo_bits > 0) ? (y >> histo_bits) * histo_xsize + (x >> histo_bits) : 0; - VP8LHistogramAddSinglePixOrCopy(image[ix], v); + VP8LHistogramAddSinglePixOrCopy(image->histograms[ix], v); x += PixOrCopyLength(v); while (x >= xsize) { x -= xsize; @@ -260,27 +261,25 @@ static void HistogramBuildImage(int xsize, int histo_bits, } } -static int HistogramCombine(VP8LHistogram* const * const in, int in_size, - int num_pairs, VP8LHistogram** const out, - int* const final_out_size) { +static int HistogramCombine(const VP8LHistogramSet* const in, + VP8LHistogramSet* const out, int num_pairs) { int ok = 0; int i, iter; unsigned int seed = 0; int tries_with_no_success = 0; const int min_cluster_size = 2; - int out_size = in_size; - const int outer_iters = in_size * 3; + int out_size = in->size; + const int outer_iters = in->size * 3; VP8LHistogram* const histos = (VP8LHistogram*)malloc(2 * sizeof(*histos)); VP8LHistogram* cur_combo = histos + 0; // trial merged histogram VP8LHistogram* best_combo = histos + 1; // best merged histogram so far if (histos == NULL) goto End; // Copy histograms from in[] to out[]. - for (i = 0; i < in_size; ++i) { - out[i] = (VP8LHistogram*)malloc(sizeof(*(out[i]))); - if (out[i] == NULL) goto End; - *(out[i]) = *(in[i]); - out[i]->bit_cost_ = in[i]->bit_cost_ = VP8LHistogramEstimateBits(out[i]); + assert(in->size <= out->size); + for (i = 0; i < in->size; ++i) { + in->histograms[i]->bit_cost_ = VP8LHistogramEstimateBits(in->histograms[i]); + *out->histograms[i] = *in->histograms[i]; } // Collapse similar histograms in 'out'. @@ -299,12 +298,13 @@ static int HistogramCombine(VP8LHistogram* const * const in, int in_size, if (idx1 == idx2) { continue; } - *cur_combo = *out[idx1]; - VP8LHistogramAdd(cur_combo, out[idx2]); + *cur_combo = *out->histograms[idx1]; + VP8LHistogramAdd(cur_combo, out->histograms[idx2]); cur_combo->bit_cost_ = VP8LHistogramEstimateBits(cur_combo); // Calculate cost reduction on combining. curr_cost_diff = cur_combo->bit_cost_ - - out[idx1]->bit_cost_ - out[idx2]->bit_cost_; + - out->histograms[idx1]->bit_cost_ + - out->histograms[idx2]->bit_cost_; if (best_cost_diff > curr_cost_diff) { // found a better pair? { // swap cur/best combo histograms VP8LHistogram* const tmp = cur_combo; @@ -318,13 +318,12 @@ static int HistogramCombine(VP8LHistogram* const * const in, int in_size, } if (best_cost_diff < 0.0) { - *out[best_idx1] = *best_combo; + *out->histograms[best_idx1] = *best_combo; // swap best_idx2 slot with last one (which is now unused) - free(out[best_idx2]); --out_size; if (best_idx2 != out_size) { - out[best_idx2] = out[out_size]; - out[out_size] = NULL; // just for sanity check. + out->histograms[best_idx2] = out->histograms[out_size]; + out->histograms[out_size] = NULL; // just for sanity check. } tries_with_no_success = 0; } @@ -332,7 +331,7 @@ static int HistogramCombine(VP8LHistogram* const * const in, int in_size, break; } } - *final_out_size = out_size; + out->size = out_size; ok = 1; End: @@ -359,18 +358,19 @@ static double HistogramDistance(const VP8LHistogram* const square_histogram, return new_bit_cost - previous_bit_cost; } -// Find the best 'out' histogram for each of the raw histograms. +// Find the best 'out' histogram for each of the 'in' histograms. // Note: we assume that out[]->bit_cost_ is already up-to-date. -static void HistogramRefine(VP8LHistogram* const * const raw, int raw_size, - uint16_t* const symbols, - VP8LHistogram** const out, int out_size) { +static void HistogramRemap(const VP8LHistogramSet* const in, + const VP8LHistogramSet* const out, + uint16_t* const symbols) { int i; - for (i = 0; i < raw_size; ++i) { + for (i = 0; i < in->size; ++i) { int best_out = 0; - double best_bits = HistogramDistance(raw[i], out[0]); + double best_bits = HistogramDistance(in->histograms[i], out->histograms[0]); int k; - for (k = 1; k < out_size; ++k) { - const double cur_bits = HistogramDistance(raw[i], out[k]); + for (k = 1; k < out->size; ++k) { + const double cur_bits = + HistogramDistance(in->histograms[i], out->histograms[k]); if (cur_bits < best_bits) { best_bits = cur_bits; best_out = k; @@ -380,47 +380,40 @@ static void HistogramRefine(VP8LHistogram* const * const raw, int raw_size, } // Recompute each out based on raw and symbols. - for (i = 0; i < out_size; ++i) { - VP8LHistogramClear(out[i]); + for (i = 0; i < out->size; ++i) { + HistogramClear(out->histograms[i]); } - for (i = 0; i < raw_size; ++i) { - VP8LHistogramAdd(out[symbols[i]], raw[i]); + for (i = 0; i < in->size; ++i) { + VP8LHistogramAdd(out->histograms[symbols[i]], in->histograms[i]); } } int VP8LGetHistoImageSymbols(int xsize, int ysize, const VP8LBackwardRefs* const refs, int quality, int histo_bits, int cache_bits, - VP8LHistogram** const histogram_image, - int* const histo_image_raw_size_ptr, + VP8LHistogramSet* const image_in, uint16_t* const histogram_symbols) { int ok = 0; const int histo_xsize = histo_bits ? VP8LSubSampleSize(xsize, histo_bits) : 1; const int histo_ysize = histo_bits ? VP8LSubSampleSize(ysize, histo_bits) : 1; const int num_histo_pairs = 10 + quality / 2; // For HistogramCombine(). const int histo_image_raw_size = histo_xsize * histo_ysize; - VP8LHistogram** const histo_image_raw = - VP8LAllocateHistograms(histo_image_raw_size, cache_bits); - if (histo_image_raw == NULL) return 0; - *histo_image_raw_size_ptr = histo_image_raw_size; // initial guess. + VP8LHistogramSet* const image_out = + VP8LAllocateHistogramSet(histo_image_raw_size, cache_bits); + if (image_out == NULL) return 0; // Build histogram image. - HistogramBuildImage(xsize, histo_bits, refs, histo_image_raw); + HistogramBuildImage(xsize, histo_bits, refs, image_out); // Collapse similar histograms. - if (!HistogramCombine(histo_image_raw, histo_image_raw_size, num_histo_pairs, - histogram_image, histo_image_raw_size_ptr)) { + if (!HistogramCombine(image_out, image_in, num_histo_pairs)) { goto Error; } - // Refine histogram image. - HistogramRefine(histo_image_raw, histo_image_raw_size, histogram_symbols, - histogram_image, *histo_image_raw_size_ptr); + // Find the optimal map from original histograms to the final ones. + HistogramRemap(image_out, image_in, histogram_symbols); ok = 1; Error: - if (!ok) { - VP8LDeleteHistograms(histogram_image, *histo_image_raw_size_ptr); - } - VP8LDeleteHistograms(histo_image_raw, histo_image_raw_size); + free(image_out); return ok; } diff --git a/src/enc/histogram.h b/src/enc/histogram.h index 7cf69e95..99b17659 100644 --- a/src/enc/histogram.h +++ b/src/enc/histogram.h @@ -41,6 +41,14 @@ typedef struct { double bit_cost_; // cached value of VP8LHistogramEstimateBits(this) } VP8LHistogram; +// Collection of histograms with fixed capacity, allocated as one +// big memory chunk. Can be destroyed by simply calling 'free()'. +typedef struct { + int size; // number of slots currently in use + int max_size; // maximum capacity + VP8LHistogram** histograms; +} VP8LHistogramSet; + // Create the histogram. // // The input data is the PixOrCopy data, which models the literals, stop @@ -50,18 +58,12 @@ void VP8LHistogramCreate(VP8LHistogram* const p, const VP8LBackwardRefs* const refs, int palette_code_bits); -// Reset the histogram's stats. -void VP8LHistogramClear(VP8LHistogram* const p); - // Set the palette_code_bits and reset the stats. void VP8LHistogramInit(VP8LHistogram* const p, int palette_code_bits); // Allocate an array of pointer to histograms, allocated and initialized // using 'cache_bits'. Return NULL in case of memory error. -VP8LHistogram** VP8LAllocateHistograms(int size, int cache_bits); - -// Destroy an array of histograms (and the array itself). -void VP8LDeleteHistograms(VP8LHistogram** const histograms, int size); +VP8LHistogramSet* VP8LAllocateHistogramSet(int size, int cache_bits); void VP8LHistogramAddSinglePixOrCopy(VP8LHistogram* const p, const PixOrCopy* const v); @@ -125,10 +127,8 @@ void VP8LConvertPopulationCountTableToBitEstimates( // Builds the histogram image. int VP8LGetHistoImageSymbols(int xsize, int ysize, const VP8LBackwardRefs* const refs, - int quality, int histogram_bits, - int cache_bits, - VP8LHistogram** const histogram_image, - int* const histogram_image_size, + int quality, int histogram_bits, int cache_bits, + VP8LHistogramSet* const image_in, uint16_t* const histogram_symbols); #if defined(__cplusplus) || defined(c_plusplus) diff --git a/src/enc/vp8l.c b/src/enc/vp8l.c index 709d48f7..e2fbdba3 100644 --- a/src/enc/vp8l.c +++ b/src/enc/vp8l.c @@ -103,7 +103,8 @@ static int AnalyzeAndCreatePalette(const uint32_t* const argb, int num_pix, } static int AnalyzeEntropy(const uint32_t const *argb, int xsize, int ysize, - int* nonpredicted_bits, int* predicted_bits) { + double* const nonpredicted_bits, + double* const predicted_bits) { int i; VP8LHistogram* nonpredicted = NULL; VP8LHistogram* predicted = (VP8LHistogram*)malloc(2 * sizeof(*predicted)); @@ -126,8 +127,8 @@ static int AnalyzeEntropy(const uint32_t const *argb, int xsize, int ysize, VP8LHistogramAddSinglePixOrCopy(predicted, &pix_diff_token); } } - *nonpredicted_bits = (int)VP8LHistogramEstimateBitsBulk(nonpredicted); - *predicted_bits = (int)VP8LHistogramEstimateBitsBulk(predicted); + *nonpredicted_bits = VP8LHistogramEstimateBitsBulk(nonpredicted); + *predicted_bits = VP8LHistogramEstimateBitsBulk(predicted); free(predicted); return 1; } @@ -140,13 +141,13 @@ static int VP8LEncAnalyze(VP8LEncoder* const enc) { AnalyzeAndCreatePalette(pic->argb, pic->width * pic->height, enc->palette_, &enc->palette_size_); if (!enc->use_palette_) { - int non_pred_entropy, pred_entropy; + double non_pred_entropy, pred_entropy; if (!AnalyzeEntropy(pic->argb, pic->width, pic->height, &non_pred_entropy, &pred_entropy)) { return 0; } - if (20 * pred_entropy < 19 * non_pred_entropy) { + if (pred_entropy < 0.95 * non_pred_entropy) { enc->use_predict_ = 1; enc->use_cross_color_ = 1; } @@ -157,7 +158,7 @@ static int VP8LEncAnalyze(VP8LEncoder* const enc) { // ----------------------------------------------------------------------------- static int GetBackwardReferences(int width, int height, - const uint32_t* argb, + const uint32_t* const argb, int quality, int use_color_cache, int cache_bits, int use_2d_locality, VP8LBackwardRefs* const best) { @@ -345,13 +346,15 @@ static int OptimizeHuffmanForRle(int length, int* counts) { // TODO(vikasa): Wrap bit_codes and bit_lengths in a Struct. static int GetHuffBitLengthsAndCodes( - int histogram_image_size, VP8LHistogram** histogram_image, + const VP8LHistogramSet* const histogram_image, int use_color_cache, int** bit_length_sizes, uint16_t*** bit_codes, uint8_t*** bit_lengths) { int i, k; int ok = 1; + const int histogram_image_size = histogram_image->size; for (i = 0; i < histogram_image_size; ++i) { - const int num_literals = VP8LHistogramNumCodes(histogram_image[i]); + VP8LHistogram* const histo = histogram_image->histograms[i]; + const int num_literals = VP8LHistogramNumCodes(histo); k = 0; // TODO(vikasa): Alloc one big buffer instead of allocating in the loop. (*bit_length_sizes)[5 * i] = num_literals; @@ -364,23 +367,21 @@ static int GetHuffBitLengthsAndCodes( } // For each component, optimize histogram for Huffman with RLE compression. - ok = ok && OptimizeHuffmanForRle(num_literals, - histogram_image[i]->literal_); + ok = ok && OptimizeHuffmanForRle(num_literals, histo->literal_); if (!use_color_cache) { // Implies that palette_bits == 0, // and so number of palette entries = (1 << 0) = 1. // Optimization might have smeared population count in this single // palette entry, so zero it out. - histogram_image[i]->literal_[256 + kLengthCodes] = 0; + histo->literal_[256 + kLengthCodes] = 0; } - ok = ok && OptimizeHuffmanForRle(256, histogram_image[i]->red_); - ok = ok && OptimizeHuffmanForRle(256, histogram_image[i]->blue_); - ok = ok && OptimizeHuffmanForRle(256, histogram_image[i]->alpha_); - ok = ok && OptimizeHuffmanForRle(DISTANCE_CODES_MAX, - histogram_image[i]->distance_); + ok = ok && OptimizeHuffmanForRle(256, histo->red_); + ok = ok && OptimizeHuffmanForRle(256, histo->blue_); + ok = ok && OptimizeHuffmanForRle(256, histo->alpha_); + ok = ok && OptimizeHuffmanForRle(DISTANCE_CODES_MAX, histo->distance_); // Create a Huffman tree (in the form of bit lengths) for each component. - ok = ok && VP8LCreateHuffmanTree(histogram_image[i]->literal_, num_literals, + ok = ok && VP8LCreateHuffmanTree(histo->literal_, num_literals, 15, (*bit_lengths)[5 * i]); for (k = 1; k < 5; ++k) { int val = 256; @@ -396,14 +397,14 @@ static int GetHuffBitLengthsAndCodes( goto Error; } } - ok = ok && VP8LCreateHuffmanTree(histogram_image[i]->red_, 256, 15, + ok = ok && + VP8LCreateHuffmanTree(histo->red_, 256, 15, (*bit_lengths)[5 * i + 1]) && - VP8LCreateHuffmanTree(histogram_image[i]->blue_, 256, 15, + VP8LCreateHuffmanTree(histo->blue_, 256, 15, (*bit_lengths)[5 * i + 2]) && - VP8LCreateHuffmanTree(histogram_image[i]->alpha_, 256, 15, + VP8LCreateHuffmanTree(histo->alpha_, 256, 15, (*bit_lengths)[5 * i + 3]) && - VP8LCreateHuffmanTree(histogram_image[i]->distance_, - DISTANCE_CODES_MAX, 15, + VP8LCreateHuffmanTree(histo->distance_, DISTANCE_CODES_MAX, 15, (*bit_lengths)[5 * i + 4]); // Create the actual bit codes for the bit lengths. for (k = 0; k < 5; ++k) { @@ -657,10 +658,11 @@ static int EncodeImageInternal(VP8LBitWriter* const bw, const int use_2d_locality = 1; const int use_color_cache = (cache_bits > 0); const int color_cache_size = use_color_cache ? (1 << cache_bits) : 0; - const int histogram_image_xysize = VP8LSubSampleSize(width, histogram_bits) * + const int histogram_image_xysize = + VP8LSubSampleSize(width, histogram_bits) * VP8LSubSampleSize(height, histogram_bits); - VP8LHistogram** histogram_image = - (VP8LHistogram**)calloc(histogram_image_xysize, sizeof(*histogram_image)); + VP8LHistogramSet* histogram_image = + VP8LAllocateHistogramSet(histogram_image_xysize, 0); int histogram_image_size; VP8LBackwardRefs refs; uint16_t* const histogram_symbols = @@ -677,11 +679,12 @@ static int EncodeImageInternal(VP8LBitWriter* const bw, // Build histogram image & symbols from backward references. if (!VP8LGetHistoImageSymbols(width, height, &refs, quality, histogram_bits, cache_bits, - histogram_image, &histogram_image_size, + histogram_image, histogram_symbols)) { goto Error; } // Create Huffman bit lengths & codes for each histogram image. + histogram_image_size = histogram_image->size; bit_lengths_sizes = (int*)calloc(5 * histogram_image_size, sizeof(*bit_lengths_sizes)); bit_lengths = (uint8_t**)calloc(5 * histogram_image_size, @@ -689,8 +692,8 @@ static int EncodeImageInternal(VP8LBitWriter* const bw, bit_codes = (uint16_t**)calloc(5 * histogram_image_size, sizeof(*bit_codes)); if (bit_lengths_sizes == NULL || bit_lengths == NULL || bit_codes == NULL || - !GetHuffBitLengthsAndCodes(histogram_image_size, histogram_image, - use_color_cache, &bit_lengths_sizes, + !GetHuffBitLengthsAndCodes(histogram_image, use_color_cache, + &bit_lengths_sizes, &bit_codes, &bit_lengths)) { goto Error; } @@ -742,7 +745,7 @@ static int EncodeImageInternal(VP8LBitWriter* const bw, } // Free combined histograms. - VP8LDeleteHistograms(histogram_image, histogram_image_size); + free(histogram_image); histogram_image = NULL; // Emit no bits if there is only one symbol in the histogram. @@ -757,9 +760,8 @@ static int EncodeImageInternal(VP8LBitWriter* const bw, ok = 1; Error: - if (!ok) { - VP8LDeleteHistograms(histogram_image, histogram_image_size); - } + if (!ok) free(histogram_image); + VP8LClearBackwardRefs(&refs); for (i = 0; i < 5 * histogram_image_size; ++i) { free(bit_lengths[i]);