From 9f566d1d36cd7b23280591fca943f2c25b05d93c Mon Sep 17 00:00:00 2001 From: Pascal Massimino Date: Thu, 10 May 2012 09:11:47 -0700 Subject: [PATCH] clean-up around Huffman-encode * refine doc * use LUT for bit-reversal * added assert * Introduce HuffmanTreeToken instead of separate code/extra_bits arrays Change-Id: I0fe0b50b55eb43a4be9f730b1abe40632a6fa7f0 --- src/enc/vp8l.c | 50 +++++----- src/utils/huffman_encode.c | 188 ++++++++++++++++++------------------- src/utils/huffman_encode.h | 38 ++++---- 3 files changed, 134 insertions(+), 142 deletions(-) diff --git a/src/enc/vp8l.c b/src/enc/vp8l.c index 45714e54..1eb0efde 100644 --- a/src/enc/vp8l.c +++ b/src/enc/vp8l.c @@ -387,25 +387,25 @@ static void StoreHuffmanTreeOfHuffmanTreeToBitMask( static void StoreHuffmanTreeToBitMask( VP8LBitWriter* const bw, - const uint8_t* huffman_tree, - const uint8_t* huffman_tree_extra_bits, - const int num_symbols, + const HuffmanTreeToken* const tokens, + const int num_tokens, const uint8_t* code_length_bitdepth, const uint16_t* code_length_bitdepth_symbols) { int i; - for (i = 0; i < num_symbols; ++i) { - const int ix = huffman_tree[i]; + for (i = 0; i < num_tokens; ++i) { + const int ix = tokens[i].code; + const int extra_bits = tokens[i].extra_bits; VP8LWriteBits(bw, code_length_bitdepth[ix], code_length_bitdepth_symbols[ix]); switch (ix) { case 16: - VP8LWriteBits(bw, 2, huffman_tree_extra_bits[i]); + VP8LWriteBits(bw, 2, extra_bits); break; case 17: - VP8LWriteBits(bw, 3, huffman_tree_extra_bits[i]); + VP8LWriteBits(bw, 3, extra_bits); break; case 18: - VP8LWriteBits(bw, 7, huffman_tree_extra_bits[i]); + VP8LWriteBits(bw, 7, extra_bits); break; } } @@ -415,27 +415,21 @@ static int StoreFullHuffmanCode(VP8LBitWriter* const bw, const uint8_t* const bit_lengths, int bit_lengths_size) { int ok = 0; - int huffman_tree_size = 0; uint8_t code_length_bitdepth[CODE_LENGTH_CODES] = { 0 }; uint16_t code_length_bitdepth_symbols[CODE_LENGTH_CODES] = { 0 }; - uint8_t* huffman_tree_extra_bits; - uint8_t* const huffman_tree = - (uint8_t*)malloc(bit_lengths_size * sizeof(*huffman_tree) + - bit_lengths_size * sizeof(*huffman_tree_extra_bits)); - - if (huffman_tree == NULL) return 0; - huffman_tree_extra_bits = - huffman_tree + bit_lengths_size * sizeof(*huffman_tree); + int num_tokens; + HuffmanTreeToken* const tokens = + (HuffmanTreeToken*)malloc(bit_lengths_size * sizeof(*tokens)); + if (tokens == NULL) return 0; VP8LWriteBits(bw, 1, 0); - VP8LCreateCompressedHuffmanTree(bit_lengths, bit_lengths_size, - &huffman_tree_size, huffman_tree, - huffman_tree_extra_bits); + num_tokens = VP8LCreateCompressedHuffmanTree(bit_lengths, bit_lengths_size, + tokens, bit_lengths_size); { int histogram[CODE_LENGTH_CODES] = { 0 }; int i; - for (i = 0; i < huffman_tree_size; ++i) { - ++histogram[huffman_tree[i]]; + for (i = 0; i < num_tokens; ++i) { + ++histogram[tokens[i].code]; } if (!VP8LCreateHuffmanTree(histogram, CODE_LENGTH_CODES, @@ -451,12 +445,12 @@ static int StoreFullHuffmanCode(VP8LBitWriter* const bw, code_length_bitdepth_symbols); { int trailing_zero_bits = 0; - int trimmed_length = huffman_tree_size; + int trimmed_length = num_tokens; int write_trimmed_length; int length; - int i = huffman_tree_size; + int i = num_tokens; while (i-- > 0) { - const int ix = huffman_tree[i]; + const int ix = tokens[i].code; if (ix == 0 || ix == 17 || ix == 18) { --trimmed_length; // discount trailing zeros trailing_zero_bits += code_length_bitdepth[ix]; @@ -470,7 +464,7 @@ static int StoreFullHuffmanCode(VP8LBitWriter* const bw, } } write_trimmed_length = (trimmed_length > 1 && trailing_zero_bits > 12); - length = write_trimmed_length ? trimmed_length : huffman_tree_size; + length = write_trimmed_length ? trimmed_length : num_tokens; VP8LWriteBits(bw, 1, write_trimmed_length); if (write_trimmed_length) { const int nbits = VP8LBitsLog2Ceiling(trimmed_length - 1); @@ -478,13 +472,13 @@ static int StoreFullHuffmanCode(VP8LBitWriter* const bw, VP8LWriteBits(bw, 3, nbitpairs - 1); VP8LWriteBits(bw, nbitpairs * 2, trimmed_length - 2); } - StoreHuffmanTreeToBitMask(bw, huffman_tree, huffman_tree_extra_bits, + StoreHuffmanTreeToBitMask(bw, tokens, length, code_length_bitdepth, code_length_bitdepth_symbols); } ok = 1; End: - free(huffman_tree); + free(tokens); return ok; } diff --git a/src/utils/huffman_encode.c b/src/utils/huffman_encode.c index d9576eee..aa3c2a04 100644 --- a/src/utils/huffman_encode.c +++ b/src/utils/huffman_encode.c @@ -7,11 +7,14 @@ // // Author: jyrki@google.com (Jyrki Alakuijala) // -// Flate like entropy encoding (Huffman) for webp lossless. +// Flate-like entropy encoding (Huffman) for webp lossless. #ifdef USE_LOSSLESS_ENCODER #include "./huffman_encode.h" + +#define MAX_BITS 16 // maximum allowed length for the codes + #include #include #include @@ -57,9 +60,7 @@ static void SetBitDepths(const HuffmanTree* const tree, // This function will create a Huffman tree. // -// The catch here is that the tree cannot be arbitrarily deep. -// Deflate specifies a maximum depth of 15 bits for "code trees" -// and 7 bits for "code length code trees." +// The catch here is that the tree cannot be arbitrarily deep // // count_limit is the value that is to be faked as the minimum value // and this minimum value is raised until the tree matches the @@ -165,149 +166,144 @@ int VP8LCreateHuffmanTree(const int* const histogram, int histogram_size, return 1; } -static void WriteHuffmanTreeRepetitions( - const int value, - const int prev_value, - int repetitions, - int* num_symbols, - uint8_t* tree, - uint8_t* extra_bits_data) { +// ----------------------------------------------------------------------------- +// Coding of the Huffman tree values + +static HuffmanTreeToken* CodeRepeatedValues(int repetitions, + HuffmanTreeToken* tokens, + int value, int prev_value) { + assert(value < MAX_BITS); if (value != prev_value) { - tree[*num_symbols] = value; - extra_bits_data[*num_symbols] = 0; - ++(*num_symbols); + tokens->code = value; + tokens->extra_bits = 0; + ++tokens; --repetitions; } while (repetitions >= 1) { if (repetitions < 3) { int i; for (i = 0; i < repetitions; ++i) { - tree[*num_symbols] = value; - extra_bits_data[*num_symbols] = 0; - ++(*num_symbols); + tokens->code = value; + tokens->extra_bits = 0; + ++tokens; } - return; + break; } else if (repetitions < 7) { - // 3 to 6 left - tree[*num_symbols] = 16; - extra_bits_data[*num_symbols] = repetitions - 3; - ++(*num_symbols); - return; + tokens->code = 16; + tokens->extra_bits = repetitions - 3; + ++tokens; + break; } else { - tree[*num_symbols] = 16; - extra_bits_data[*num_symbols] = 3; - ++(*num_symbols); + tokens->code = 16; + tokens->extra_bits = 3; + ++tokens; repetitions -= 6; } } + return tokens; } -static void WriteHuffmanTreeRepetitionsZeros( - const int value, - int repetitions, - int* num_symbols, - uint8_t* tree, - uint8_t* extra_bits_data) { +static HuffmanTreeToken* CodeRepeatedZeros(int repetitions, + HuffmanTreeToken* tokens) { while (repetitions >= 1) { if (repetitions < 3) { int i; for (i = 0; i < repetitions; ++i) { - tree[*num_symbols] = value; - extra_bits_data[*num_symbols] = 0; - ++(*num_symbols); + tokens->code = 0; // 0-value + tokens->extra_bits = 0; + ++tokens; } - return; + break; } else if (repetitions < 11) { - tree[*num_symbols] = 17; - extra_bits_data[*num_symbols] = repetitions - 3; - ++(*num_symbols); - return; + tokens->code = 17; + tokens->extra_bits = repetitions - 3; + ++tokens; + break; } else if (repetitions < 139) { - tree[*num_symbols] = 18; - extra_bits_data[*num_symbols] = repetitions - 11; - ++(*num_symbols); - return; + tokens->code = 18; + tokens->extra_bits = repetitions - 11; + ++tokens; + break; } else { - tree[*num_symbols] = 18; - extra_bits_data[*num_symbols] = 0x7f; // 138 repeated 0s - ++(*num_symbols); + tokens->code = 18; + tokens->extra_bits = 0x7f; // 138 repeated 0s + ++tokens; repetitions -= 138; } } + return tokens; } -void VP8LCreateCompressedHuffmanTree(const uint8_t* const depth, - int depth_size, - int* num_symbols, - uint8_t* tree, - uint8_t* extra_bits_data) { +int VP8LCreateCompressedHuffmanTree(const uint8_t* const depth, + int depth_size, + HuffmanTreeToken* tokens, + int max_tokens) { + HuffmanTreeToken* const starting_token = tokens; + HuffmanTreeToken* const ending_token = tokens + max_tokens; int prev_value = 8; // 8 is the initial value for rle. - int i; - for (i = 0; i < depth_size;) { + int i = 0; + while (i < depth_size) { const int value = depth[i]; - int reps = 1; - int k; - for (k = i + 1; k < depth_size && depth[k] == value; ++k) { - ++reps; - } + int k = i + 1; + int runs; + while (k < depth_size && depth[k] == value) ++k; + runs = k - i; if (value == 0) { - WriteHuffmanTreeRepetitionsZeros(value, reps, - num_symbols, - tree, extra_bits_data); + tokens = CodeRepeatedZeros(runs, tokens); } else { - WriteHuffmanTreeRepetitions(value, prev_value, reps, - num_symbols, - tree, extra_bits_data); + tokens = CodeRepeatedValues(runs, tokens, value, prev_value); prev_value = value; } - i += reps; + i += runs; + assert(tokens <= ending_token); } + (void)ending_token; // suppress 'unused variable' warning + return tokens - starting_token; } +// ----------------------------------------------------------------------------- + +// Pre-reversed 4-bit values. +static const uint8_t kReversedBits[16] = { + 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, + 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf +}; + static uint32_t ReverseBits(int num_bits, uint32_t bits) { uint32_t retval = 0; - int i; - for (i = 0; i < num_bits; ++i) { - retval <<= 1; - retval |= bits & 1; - bits >>= 1; + int i = 0; + while (i < num_bits) { + i += 4; + retval |= kReversedBits[bits & 0xf] << (MAX_BITS - i); + bits >>= 4; } + retval >>= (MAX_BITS - num_bits); return retval; } -void VP8LConvertBitDepthsToSymbols(const uint8_t* depth, int len, - uint16_t* bits) { - // This function is based on RFC 1951. - // - // In deflate, all bit depths are [1..15] - // 0 bit depth means that the symbol does not exist. - - // 0..15 are values for bits -#define MAX_BITS 16 - uint32_t next_code[MAX_BITS]; - uint32_t bl_count[MAX_BITS] = { 0 }; +void VP8LConvertBitDepthsToSymbols(const uint8_t* const depth, + int len, + uint16_t* const bits) { + // 0 bit-depth means that the symbol does not exist. int i; - { - for (i = 0; i < len; ++i) { - ++bl_count[depth[i]]; - } - bl_count[0] = 0; + uint32_t next_code[MAX_BITS]; + int depth_count[MAX_BITS] = { 0 }; + for (i = 0; i < len; ++i) { + assert(depth[i] < MAX_BITS); + ++depth_count[depth[i]]; } + depth_count[0] = 0; // ignore unused symbol next_code[0] = 0; { - int code = 0; - int bits; - for (bits = 1; bits < MAX_BITS; ++bits) { - code = (code + bl_count[bits - 1]) << 1; - next_code[bits] = code; + uint32_t code = 0; + for (i = 1; i < MAX_BITS; ++i) { + code = (code + depth_count[i - 1]) << 1; + next_code[i] = code; } } for (i = 0; i < len; ++i) { - if (depth[i]) { - bits[i] = ReverseBits(depth[i], next_code[depth[i]]++); - } + bits[i] = ReverseBits(depth[i], next_code[depth[i]]++); } } -#undef MAX_BITS #endif diff --git a/src/utils/huffman_encode.h b/src/utils/huffman_encode.h index da8d567c..7b3b9748 100644 --- a/src/utils/huffman_encode.h +++ b/src/utils/huffman_encode.h @@ -7,7 +7,7 @@ // // Author: jyrki@google.com (Jyrki Alakuijala) // -// Flate like entropy encoding (Huffman) for webp lossless +// Flate-like entropy encoding (Huffman) for webp lossless #ifndef WEBP_UTILS_HUFFMAN_ENCODE_H_ #define WEBP_UTILS_HUFFMAN_ENCODE_H_ @@ -20,27 +20,29 @@ extern "C" { #endif -// This function will create a Huffman tree. +// Create a Huffman tree. // -// 'histogram' contains the population counts. -// 'tree_depth_limit' is the maximum bit depth of the Huffman codes. -// The created tree is returned as 'bit_depths', which stores how many bits are -// used for each symbol. -// See http://en.wikipedia.org/wiki/Huffman_coding -// Returns 0 on memory error. -int VP8LCreateHuffmanTree(const int* const histogram, int histogram_size, - int tree_depth_limit, uint8_t* const bit_depths); +// (data,length): population counts. +// tree_limit: maximum bit depth (inclusive) of the codes. +// bit_depths[]: how many bits are used for the symbol. +// +// Returns 0 when an error has occured. +int VP8LCreateHuffmanTree(const int* data, const int length, + const int tree_limit, uint8_t* bit_depths); -// Write a huffman tree from bit depths. The generated Huffman tree is -// compressed once more using a Huffman tree. -void VP8LCreateCompressedHuffmanTree(const uint8_t* const depth, int len, - int* num_symbols, - uint8_t* tree, - uint8_t* extra_bits_data); +// Turn the Huffman tree into a token sequence. +// Returns the number of tokens used. +typedef struct { + uint8_t code; // value (0..15) or escape code (16,17,18) + uint8_t extra_bits; // extra bits for escape codes +} HuffmanTreeToken; + +int VP8LCreateCompressedHuffmanTree(const uint8_t* const depth, int len, + HuffmanTreeToken* tokens, int max_tokens); // Get the actual bit values for a tree of bit depths. -void VP8LConvertBitDepthsToSymbols(const uint8_t* depth, int len, - uint16_t* bits); +void VP8LConvertBitDepthsToSymbols(const uint8_t* const depth, int len, + uint16_t* const bits); #if defined(__cplusplus) || defined(c_plusplus) }