dany/libwebp
1
0
Fork 0
mirror of https://github.com/webmproject/libwebp.git synced 2024-12-27 06:08:21 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Speed up Huffman decoding for lossless

Browse Source 
speed-up is ~1.6% for photographic image to 10% for graphical image
(1000 images corpus was sped up by 5.8 %)

Code by akramarz@google.com and jyrki@google.com

Change-Id: Iceb2e50e6cc761b9315a3865d22ec9d19b8011c6
This commit is contained in:
skal 2014-08-28 08:35:19 -07:00 committed by Gerrit Code Review
parent 637b388809
commit f75dfbf23d
4 changed files with 259 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
src/dec/vp8l.c
View File

@ -72,6 +72,29 @@ static const uint8_t kCodeToPlane[CODE_TO_PLANE_CODES] = {
0x40, 0x72, 0x7e, 0x61, 0x6f, 0x50, 0x71, 0x7f, 0x60, 0x70
};
// Memory needed for lookup tables of one Huffman tree group. Red, blue, alpha
// and distance alphabets are constant (256 for red, blue and alpha, 40 for
// distance) and lookup table sizes for them in worst case are 630 and 410
// respectively. Size of green alphabet depends on color cache size and is equal
// to 256 (green component values) + 24 (length prefix values)
// + color_cache_size (between 0 and 2048).
// All values computed for 8-bit first level lookup with tests/tools/enough.cc.
#define FIXED_TABLE_SIZE (630 * 3 + 410)
static const int kTableSize[12] = {
FIXED_TABLE_SIZE + 654,
FIXED_TABLE_SIZE + 656,
FIXED_TABLE_SIZE + 658,
FIXED_TABLE_SIZE + 662,
FIXED_TABLE_SIZE + 670,
FIXED_TABLE_SIZE + 686,
FIXED_TABLE_SIZE + 718,
FIXED_TABLE_SIZE + 782,
FIXED_TABLE_SIZE + 912,
FIXED_TABLE_SIZE + 1168,
FIXED_TABLE_SIZE + 1680,
FIXED_TABLE_SIZE + 2704
};
static int DecodeImageStream(int xsize, int ysize,
int is_level0,
VP8LDecoder* const dec,
@ -151,31 +174,20 @@ static WEBP_INLINE int PlaneCodeToDistance(int xsize, int plane_code) {
// Decodes the next Huffman code from bit-stream.
// FillBitWindow(br) needs to be called at minimum every second call
// to ReadSymbol, in order to pre-fetch enough bits.
static WEBP_INLINE int ReadSymbol(const HuffmanTree* tree,
static WEBP_INLINE int ReadSymbol(const HuffmanCode* table,
VP8LBitReader* const br) {
const HuffmanTreeNode* node = tree->root_;
uint32_t bits = VP8LPrefetchBits(br);
int bitpos = br->bit_pos_;
// Check if we find the bit combination from the Huffman lookup table.
const int lut_ix = bits & (HUFF_LUT - 1);
const int lut_bits = tree->lut_bits_[lut_ix];
if (lut_bits <= HUFF_LUT_BITS) {
VP8LSetBitPos(br, bitpos + lut_bits);
return tree->lut_symbol_[lut_ix];
int nbits;
uint32_t val = VP8LPrefetchBits(br);
table += val & HUFFMAN_TABLE_MASK;
nbits = table->bits - HUFFMAN_TABLE_BITS;
if (nbits > 0) {
VP8LSetBitPos(br, br->bit_pos_ + HUFFMAN_TABLE_BITS);
val = VP8LPrefetchBits(br);
table += table->value;
table += val & ((1 << nbits) - 1);
}
node += tree->lut_jump_[lut_ix];
bitpos += HUFF_LUT_BITS;
bits >>= HUFF_LUT_BITS;
// Decode the value from a binary tree.
assert(node != NULL);
do {
node = HuffmanTreeNextNode(node, bits & 1);
bits >>= 1;
++bitpos;
} while (HuffmanTreeNodeIsNotLeaf(node));
VP8LSetBitPos(br, bitpos);
return node->symbol_;
VP8LSetBitPos(br, br->bit_pos_ + table->bits);
return table->value;
}
static int ReadHuffmanCodeLengths(
@ -186,11 +198,11 @@ static int ReadHuffmanCodeLengths(
int symbol;
int max_symbol;
int prev_code_len = DEFAULT_CODE_LENGTH;
HuffmanTree tree;
int huff_codes[NUM_CODE_LENGTH_CODES] = { 0 };
HuffmanCode table[1 << LENGTHS_TABLE_BITS];
if (!VP8LHuffmanTreeBuildImplicit(&tree, code_length_code_lengths,
huff_codes, NUM_CODE_LENGTH_CODES)) {
if (!VP8LBuildHuffmanTable(table, LENGTHS_TABLE_BITS,
code_length_code_lengths,
NUM_CODE_LENGTH_CODES)) {
dec->status_ = VP8_STATUS_BITSTREAM_ERROR;
return 0;
}
@ -208,10 +220,13 @@ static int ReadHuffmanCodeLengths(
symbol = 0;
while (symbol < num_symbols) {
const HuffmanCode* p = table;
int code_len;
if (max_symbol-- == 0) break;
VP8LFillBitWindow(br);
code_len = ReadSymbol(&tree, br);
p += VP8LPrefetchBits(br) & LENGTHS_TABLE_MASK;
VP8LSetBitPos(br, br->bit_pos_ + p->bits);
code_len = p->value;
if (code_len < kCodeLengthLiterals) {
code_lengths[symbol++] = code_len;
if (code_len != 0) prev_code_len = code_len;
@ -233,36 +248,31 @@ static int ReadHuffmanCodeLengths(
ok = 1;
End:
VP8LHuffmanTreeFree(&tree);
return ok;
}
// 'code_lengths' is pre-allocated temporary buffer, used for creating Huffman
// tree.
static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec,
int* const code_lengths, int* const huff_codes,
HuffmanTree* const tree) {
int* const code_lengths, HuffmanCode* const table) {
int ok = 0;
VP8LBitReader* const br = &dec->br_;
const int simple_code = VP8LReadBits(br, 1);
memset(code_lengths, 0, alphabet_size * sizeof(*code_lengths));
if (simple_code) { // Read symbols, codes & code lengths directly.
int symbols[2];
int codes[2];
const int num_symbols = VP8LReadBits(br, 1) + 1;
const int first_symbol_len_code = VP8LReadBits(br, 1);
// The first code is either 1 bit or 8 bit code.
symbols[0] = VP8LReadBits(br, (first_symbol_len_code == 0) ? 1 : 8);
codes[0] = 0;
code_lengths[0] = num_symbols - 1;
int symbol = VP8LReadBits(br, (first_symbol_len_code == 0) ? 1 : 8);
code_lengths[symbol] = 1;
// The second code (if present), is always 8 bit long.
if (num_symbols == 2) {
symbols[1] = VP8LReadBits(br, 8);
codes[1] = 1;
code_lengths[1] = num_symbols - 1;
symbol = VP8LReadBits(br, 8);
code_lengths[symbol] = 1;
}
ok = VP8LHuffmanTreeBuildExplicit(tree, code_lengths, codes, symbols,
alphabet_size, num_symbols);
ok = 1;
} else { // Decode Huffman-coded code lengths.
int i;
int code_length_code_lengths[NUM_CODE_LENGTH_CODES] = { 0 };
@ -272,22 +282,20 @@ static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec,
return 0;
}
memset(code_lengths, 0, alphabet_size * sizeof(*code_lengths));
for (i = 0; i < num_codes; ++i) {
code_length_code_lengths[kCodeLengthCodeOrder[i]] = VP8LReadBits(br, 3);
}
ok = ReadHuffmanCodeLengths(dec, code_length_code_lengths, alphabet_size,
code_lengths);
ok = ok && VP8LHuffmanTreeBuildImplicit(tree, code_lengths, huff_codes,
alphabet_size);
}
ok = ok && !br->error_;
if (!ok) {
dec->status_ = VP8_STATUS_BITSTREAM_ERROR;
return 0;
}
return 1;
return VP8LBuildHuffmanTable(table, HUFFMAN_TABLE_BITS,
code_lengths, alphabet_size);
}
static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize,
@ -297,10 +305,12 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize,
VP8LMetadata* const hdr = &dec->hdr_;
uint32_t* huffman_image = NULL;
HTreeGroup* htree_groups = NULL;
HuffmanCode* huffman_tables = NULL;
HuffmanCode* next = NULL;
int num_htree_groups = 1;
int max_alphabet_size = 0;
int* code_lengths = NULL;
int* huff_codes = NULL;
const int table_size = kTableSize[color_cache_bits];
if (allow_recursion && VP8LReadBits(br, 1)) {
// use meta Huffman codes.
@ -337,45 +347,48 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize,
}
}
huffman_tables = (HuffmanCode*)WebPSafeMalloc(num_htree_groups * table_size,
sizeof(HuffmanCode));
htree_groups = VP8LHtreeGroupsNew(num_htree_groups);
code_lengths =
(int*)WebPSafeCalloc((uint64_t)max_alphabet_size, sizeof(*code_lengths));
huff_codes =
(int*)WebPSafeMalloc((uint64_t)max_alphabet_size, sizeof(*huff_codes));
code_lengths = (int*)WebPSafeCalloc((uint64_t)max_alphabet_size,
sizeof(*code_lengths));
if (htree_groups == NULL || code_lengths == NULL || huff_codes == NULL) {
if (htree_groups == NULL || code_lengths == NULL || huffman_tables == NULL) {
dec->status_ = VP8_STATUS_OUT_OF_MEMORY;
goto Error;
}
next = huffman_tables;
for (i = 0; i < num_htree_groups; ++i) {
HuffmanTree* const htrees = htree_groups[i].htrees_;
HuffmanCode** const htrees = htree_groups[i].htrees;
int size;
for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) {
int alphabet_size = kAlphabetSize[j];
HuffmanTree* const htree = htrees + j;
htrees[j] = next;
if (j == 0 && color_cache_bits > 0) {
alphabet_size += 1 << color_cache_bits;
}
if (!ReadHuffmanCode(alphabet_size, dec, code_lengths, huff_codes,
htree)) {
size = ReadHuffmanCode(alphabet_size, dec, code_lengths, next);
next += size;
if (size == 0) {
goto Error;
}
}
}
WebPSafeFree(huff_codes);
WebPSafeFree(code_lengths);
// All OK. Finalize pointers and return.
hdr->huffman_image_ = huffman_image;
hdr->num_htree_groups_ = num_htree_groups;
hdr->htree_groups_ = htree_groups;
hdr->huffman_tables_ = huffman_tables;
return 1;
Error:
WebPSafeFree(huff_codes);
WebPSafeFree(code_lengths);
WebPSafeFree(huffman_image);
VP8LHtreeGroupsFree(htree_groups, num_htree_groups);
WebPSafeFree(huffman_tables);
VP8LHtreeGroupsFree(htree_groups);
return 0;
}
@ -715,10 +728,10 @@ static int Is8bOptimizable(const VP8LMetadata* const hdr) {
// When the Huffman tree contains only one symbol, we can skip the
// call to ReadSymbol() for red/blue/alpha channels.
for (i = 0; i < hdr->num_htree_groups_; ++i) {
const HuffmanTree* const htrees = hdr->htree_groups_[i].htrees_;
if (htrees[RED].num_nodes_ > 1) return 0;
if (htrees[BLUE].num_nodes_ > 1) return 0;
if (htrees[ALPHA].num_nodes_ > 1) return 0;
HuffmanCode** const htrees = hdr->htree_groups_[i].htrees;
if (htrees[RED][0].bits > 0) return 0;
if (htrees[BLUE][0].bits > 0) return 0;
if (htrees[ALPHA][0].bits > 0) return 0;
}
return 1;
}
@ -758,7 +771,7 @@ static int DecodeAlphaData(VP8LDecoder* const dec, uint8_t* const data,
htree_group = GetHtreeGroupForPos(hdr, col, row);
}
VP8LFillBitWindow(br);
code = ReadSymbol(&htree_group->htrees_[GREEN], br);
code = ReadSymbol(htree_group->htrees[GREEN], br);
if (code < NUM_LITERAL_CODES) { // Literal
data[pos] = code;
++pos;
@ -774,7 +787,7 @@ static int DecodeAlphaData(VP8LDecoder* const dec, uint8_t* const data,
int dist_code, dist;
const int length_sym = code - NUM_LITERAL_CODES;
const int length = GetCopyLength(length_sym, br);
const int dist_symbol = ReadSymbol(&htree_group->htrees_[DIST], br);
const int dist_symbol = ReadSymbol(htree_group->htrees[DIST], br);
VP8LFillBitWindow(br);
dist_code = GetCopyDistance(dist_symbol, br);
dist = PlaneCodeToDistance(width, dist_code);
@ -850,14 +863,14 @@ static int DecodeImageData(VP8LDecoder* const dec, uint32_t* const data,
htree_group = GetHtreeGroupForPos(hdr, col, row);
}
VP8LFillBitWindow(br);
code = ReadSymbol(&htree_group->htrees_[GREEN], br);
code = ReadSymbol(htree_group->htrees[GREEN], br);
if (code < NUM_LITERAL_CODES) { // Literal
int red, green, blue, alpha;
red = ReadSymbol(&htree_group->htrees_[RED], br);
red = ReadSymbol(htree_group->htrees[RED], br);
green = code;
VP8LFillBitWindow(br);
blue = ReadSymbol(&htree_group->htrees_[BLUE], br);
alpha = ReadSymbol(&htree_group->htrees_[ALPHA], br);
blue = ReadSymbol(htree_group->htrees[BLUE], br);
alpha = ReadSymbol(htree_group->htrees[ALPHA], br);
*src = ((uint32_t)alpha << 24) | (red << 16) | (green << 8) | blue;
AdvanceByOne:
++src;
@ -878,7 +891,7 @@ static int DecodeImageData(VP8LDecoder* const dec, uint32_t* const data,
int dist_code, dist;
const int length_sym = code - NUM_LITERAL_CODES;
const int length = GetCopyLength(length_sym, br);
const int dist_symbol = ReadSymbol(&htree_group->htrees_[DIST], br);
const int dist_symbol = ReadSymbol(htree_group->htrees[DIST], br);
VP8LFillBitWindow(br);
dist_code = GetCopyDistance(dist_symbol, br);
dist = PlaneCodeToDistance(width, dist_code);
@ -1035,7 +1048,8 @@ static void ClearMetadata(VP8LMetadata* const hdr) {
assert(hdr);
WebPSafeFree(hdr->huffman_image_);
VP8LHtreeGroupsFree(hdr->htree_groups_, hdr->num_htree_groups_);
WebPSafeFree(hdr->huffman_tables_);
VP8LHtreeGroupsFree(hdr->htree_groups_);
VP8LColorCacheClear(&hdr->color_cache_);
InitMetadata(hdr);
}

1
src/dec/vp8li.h
View File

@ -50,6 +50,7 @@ typedef struct {
uint32_t *huffman_image_;
int num_htree_groups_;
HTreeGroup *htree_groups_;
HuffmanCode *huffman_tables_;
} VP8LMetadata;
typedef struct VP8LDecoder VP8LDecoder;

412
src/utils/huffman.c
View File

@ -18,302 +18,192 @@
#include "../utils/utils.h"
#include "../webp/format_constants.h"
// Uncomment the following to use look-up table for ReverseBits()
// (might be faster on some platform)
// #define USE_LUT_REVERSE_BITS
// Huffman data read via DecodeImageStream is represented in two (red and green)
// bytes.
#define MAX_HTREE_GROUPS 0x10000
#define NON_EXISTENT_SYMBOL (-1)
static void TreeNodeInit(HuffmanTreeNode* const node) {
node->children_ = -1; // means: 'unassigned so far'
}
static int NodeIsEmpty(const HuffmanTreeNode* const node) {
return (node->children_ < 0);
}
static int IsFull(const HuffmanTree* const tree) {
return (tree->num_nodes_ == tree->max_nodes_);
}
static void AssignChildren(HuffmanTree* const tree,
HuffmanTreeNode* const node) {
HuffmanTreeNode* const children = tree->root_ + tree->num_nodes_;
node->children_ = (int)(children - node);
assert(children - node == (int)(children - node));
tree->num_nodes_ += 2;
TreeNodeInit(children + 0);
TreeNodeInit(children + 1);
}
// A Huffman tree is a full binary tree; and in a full binary tree with L
// leaves, the total number of nodes N = 2 * L - 1.
static int HuffmanTreeMaxNodes(int num_leaves) {
return (2 * num_leaves - 1);
}
static int HuffmanTreeAllocate(HuffmanTree* const tree, int num_nodes) {
assert(tree != NULL);
tree->root_ =
(HuffmanTreeNode*)WebPSafeMalloc(num_nodes, sizeof(*tree->root_));
return (tree->root_ != NULL);
}
static int TreeInit(HuffmanTree* const tree, int num_leaves) {
assert(tree != NULL);
if (num_leaves == 0) return 0;
tree->max_nodes_ = HuffmanTreeMaxNodes(num_leaves);
assert(tree->max_nodes_ < (1 << 16)); // limit for the lut_jump_ table
if (!HuffmanTreeAllocate(tree, tree->max_nodes_)) return 0;
TreeNodeInit(tree->root_); // Initialize root.
tree->num_nodes_ = 1;
memset(tree->lut_bits_, 255, sizeof(tree->lut_bits_));
memset(tree->lut_jump_, 0, sizeof(tree->lut_jump_));
return 1;
}
void VP8LHuffmanTreeFree(HuffmanTree* const tree) {
if (tree != NULL) {
WebPSafeFree(tree->root_);
tree->root_ = NULL;
tree->max_nodes_ = 0;
tree->num_nodes_ = 0;
}
}
HTreeGroup* VP8LHtreeGroupsNew(int num_htree_groups) {
HTreeGroup* const htree_groups =
(HTreeGroup*)WebPSafeCalloc(num_htree_groups, sizeof(*htree_groups));
assert(num_htree_groups <= MAX_HTREE_GROUPS);
(HTreeGroup*)WebPSafeMalloc(num_htree_groups, sizeof(*htree_groups));
if (htree_groups == NULL) {
return NULL;
}
assert(num_htree_groups <= MAX_HTREE_GROUPS);
return htree_groups;
}
void VP8LHtreeGroupsFree(HTreeGroup* htree_groups, int num_htree_groups) {
void VP8LHtreeGroupsFree(HTreeGroup* const htree_groups) {
if (htree_groups != NULL) {
int i, j;
for (i = 0; i < num_htree_groups; ++i) {
HuffmanTree* const htrees = htree_groups[i].htrees_;
for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) {
VP8LHuffmanTreeFree(&htrees[j]);
}
}
WebPSafeFree(htree_groups);
}
}
int VP8LHuffmanCodeLengthsToCodes(
const int* const code_lengths, int code_lengths_size,
int* const huff_codes) {
int symbol;
int code_len;
int code_length_hist[MAX_ALLOWED_CODE_LENGTH + 1] = { 0 };
int curr_code;
int next_codes[MAX_ALLOWED_CODE_LENGTH + 1] = { 0 };
int max_code_length = 0;
// Returns reverse(reverse(key, len) + 1, len), where reverse(key, len) is the
// bit-wise reversal of the len least significant bits of key.
static WEBP_INLINE uint32_t GetNextKey(uint32_t key, int len) {
uint32_t step = 1 << (len - 1);
while (key & step) {
step >>= 1;
}
return (key & (step - 1)) + step;
}
// Stores code in table[0], table[step], table[2*step], ..., table[end].
// Assumes that end is an integer multiple of step.
static WEBP_INLINE void ReplicateValue(HuffmanCode* table,
int step, int end,
HuffmanCode code) {
assert(end % step == 0);
do {
end -= step;
table[end] = code;
} while (end > 0);
}
// Returns the table width of the next 2nd level table. count is the histogram
// of bit lengths for the remaining symbols, len is the code length of the next
// processed symbol
static WEBP_INLINE int NextTableBitSize(const int* const count,
int len, int root_bits) {
int left = 1 << (len - root_bits);
while (len < MAX_ALLOWED_CODE_LENGTH) {
left -= count[len];
if (left <= 0) break;
++len;
left <<= 1;
}
return len - root_bits;
}
int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits,
const int* const code_lengths,
int code_lengths_size) {
HuffmanCode* table; // next available space in table
int len; // current code length
int symbol; // symbol index in original or sorted table
int total_size; // sum of root table size and 2nd level table sizes
int* sorted = NULL; // symbols sorted by code length
int count[MAX_ALLOWED_CODE_LENGTH + 1] = { 0 };
// number of codes of each length
int offset[MAX_ALLOWED_CODE_LENGTH + 1];
// offsets in sorted table for each length
assert(code_lengths_size != 0);
assert(code_lengths != NULL);
assert(code_lengths_size > 0);
assert(huff_codes != NULL);
assert(root_table != NULL);
assert(root_bits > 0);
// Calculate max code length.
// Build histogram of code lengths.
for (symbol = 0; symbol < code_lengths_size; ++symbol) {
if (code_lengths[symbol] > max_code_length) {
max_code_length = code_lengths[symbol];
}
}
if (max_code_length > MAX_ALLOWED_CODE_LENGTH) return 0;
// Calculate code length histogram.
for (symbol = 0; symbol < code_lengths_size; ++symbol) {
++code_length_hist[code_lengths[symbol]];
}
code_length_hist[0] = 0;
// Calculate the initial values of 'next_codes' for each code length.
// next_codes[code_len] denotes the code to be assigned to the next symbol
// of code length 'code_len'.
curr_code = 0;
next_codes[0] = -1; // Unused, as code length = 0 implies code doesn't exist.
for (code_len = 1; code_len <= max_code_length; ++code_len) {
curr_code = (curr_code + code_length_hist[code_len - 1]) << 1;
next_codes[code_len] = curr_code;
}
// Get symbols.
for (symbol = 0; symbol < code_lengths_size; ++symbol) {
if (code_lengths[symbol] > 0) {
huff_codes[symbol] = next_codes[code_lengths[symbol]]++;
} else {
huff_codes[symbol] = NON_EXISTENT_SYMBOL;
}
}
return 1;
}
#ifndef USE_LUT_REVERSE_BITS
static int ReverseBitsShort(int bits, int num_bits) {
int retval = 0;
int i;
assert(num_bits <= 8); // Not a hard requirement, just for coherency.
for (i = 0; i < num_bits; ++i) {
retval <<= 1;
retval |= bits & 1;
bits >>= 1;
}
return retval;
}
#else
static const uint8_t kReversedBits[16] = { // Pre-reversed 4-bit values.
0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe,
0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf
};
static int ReverseBitsShort(int bits, int num_bits) {
const uint8_t v = (kReversedBits[bits & 0xf] << 4) | kReversedBits[bits >> 4];
assert(num_bits <= 8);
return v >> (8 - num_bits);
}
#endif
static int TreeAddSymbol(HuffmanTree* const tree,
int symbol, int code, int code_length) {
int step = HUFF_LUT_BITS;
int base_code;
HuffmanTreeNode* node = tree->root_;
const HuffmanTreeNode* const max_node = tree->root_ + tree->max_nodes_;
assert(symbol == (int16_t)symbol);
if (code_length <= HUFF_LUT_BITS) {
int i;
base_code = ReverseBitsShort(code, code_length);
for (i = 0; i < (1 << (HUFF_LUT_BITS - code_length)); ++i) {
const int idx = base_code | (i << code_length);
tree->lut_symbol_[idx] = (int16_t)symbol;
tree->lut_bits_[idx] = code_length;
}
} else {
base_code = ReverseBitsShort((code >> (code_length - HUFF_LUT_BITS)),
HUFF_LUT_BITS);
}
while (code_length-- > 0) {
if (node >= max_node) {
if (code_lengths[symbol] > MAX_ALLOWED_CODE_LENGTH) {
return 0;
}
if (NodeIsEmpty(node)) {
if (IsFull(tree)) return 0; // error: too many symbols.
AssignChildren(tree, node);
} else if (!HuffmanTreeNodeIsNotLeaf(node)) {
return 0; // leaf is already occupied.
}
node += node->children_ + ((code >> code_length) & 1);
if (--step == 0) {
tree->lut_jump_[base_code] = (int16_t)(node - tree->root_);
}
}
if (NodeIsEmpty(node)) {
node->children_ = 0; // turn newly created node into a leaf.
} else if (HuffmanTreeNodeIsNotLeaf(node)) {
return 0; // trying to assign a symbol to already used code.
}
node->symbol_ = symbol; // Add symbol in this node.
return 1;
}
int VP8LHuffmanTreeBuildImplicit(HuffmanTree* const tree,
const int* const code_lengths,
int* const codes,
int code_lengths_size) {
int symbol;
int num_symbols = 0;
int root_symbol = 0;
assert(tree != NULL);
assert(code_lengths != NULL);
// Find out number of symbols and the root symbol.
for (symbol = 0; symbol < code_lengths_size; ++symbol) {
if (code_lengths[symbol] > 0) {
// Note: code length = 0 indicates non-existent symbol.
++num_symbols;
root_symbol = symbol;
}
++count[code_lengths[symbol]];
}
// Initialize the tree. Will fail for num_symbols = 0
if (!TreeInit(tree, num_symbols)) return 0;
// Error, all code lengths are zeros.
if (count[0] == code_lengths_size) {
return 0;
}
// Build tree.
if (num_symbols == 1) { // Trivial case.
const int max_symbol = code_lengths_size;
if (root_symbol < 0 || root_symbol >= max_symbol) {
VP8LHuffmanTreeFree(tree);
// Generate offsets into sorted symbol table by code length.
offset[1] = 0;
for (len = 1; len < MAX_ALLOWED_CODE_LENGTH; ++len) {
if (count[len] > (1 << len)) {
return 0;
}
return TreeAddSymbol(tree, root_symbol, 0, 0);
} else { // Normal case.
int ok = 0;
memset(codes, 0, code_lengths_size * sizeof(*codes));
offset[len + 1] = offset[len] + count[len];
}
if (!VP8LHuffmanCodeLengthsToCodes(code_lengths, code_lengths_size,
codes)) {
goto End;
sorted = (int*)WebPSafeMalloc(code_lengths_size, sizeof(*sorted));
if (sorted == NULL) {
return 0;
}
// Sort symbols by length, by symbol order within each length.
for (symbol = 0; symbol < code_lengths_size; ++symbol) {
const int symbol_code_length = code_lengths[symbol];
if (code_lengths[symbol] > 0) {
sorted[offset[symbol_code_length]++] = symbol;
}
}
table = root_table;
total_size = 1 << root_bits;
// Special case code with only one value.
if (offset[MAX_ALLOWED_CODE_LENGTH] == 1) {
HuffmanCode code;
code.bits = 0;
code.value = (uint16_t)sorted[0];
ReplicateValue(table, 1, total_size, code);
WebPSafeFree(sorted);
return total_size;
}
{
int step; // step size to replicate values in current table
uint32_t low = -1; // low bits for current root entry
uint32_t mask = total_size - 1; // mask for low bits
uint32_t key = 0; // reversed prefix code
int num_nodes = 1; // number of Huffman tree nodes
int num_open = 1; // number of open branches in current tree level
int table_bits = root_bits; // key length of current table
int table_size = 1 << table_bits; // size of current table
symbol = 0;
// Fill in root table.
for (len = 1, step = 2; len <= root_bits; ++len, step <<= 1) {
num_open <<= 1;
num_nodes += num_open;
num_open -= count[len];
if (num_open < 0) {
WebPSafeFree(sorted);
return 0;
}
for (; count[len] > 0; --count[len]) {
HuffmanCode code;
code.bits = (uint8_t)len;
code.value = (uint16_t)sorted[symbol++];
ReplicateValue(&table[key], step, table_size, code);
key = GetNextKey(key, len);
}
}
// Add symbols one-by-one.
for (symbol = 0; symbol < code_lengths_size; ++symbol) {
if (code_lengths[symbol] > 0) {
if (!TreeAddSymbol(tree, symbol, codes[symbol],
code_lengths[symbol])) {
goto End;
// Fill in 2nd level tables and add pointers to root table.
for (len = root_bits + 1, step = 2; len <= MAX_ALLOWED_CODE_LENGTH;
++len, step <<= 1) {
num_open <<= 1;
num_nodes += num_open;
num_open -= count[len];
if (num_open < 0) {
WebPSafeFree(sorted);
return 0;
}
for (; count[len] > 0; --count[len]) {
HuffmanCode code;
if ((key & mask) != low) {
table += table_size;
table_bits = NextTableBitSize(count, len, root_bits);
table_size = 1 << table_bits;
total_size += table_size;
low = key & mask;
root_table[low].bits = (uint8_t)(table_bits + root_bits);
root_table[low].value = (uint16_t)((table - root_table) - low);
}
code.bits = (uint8_t)(len - root_bits);
code.value = (uint16_t)sorted[symbol++];
ReplicateValue(&table[key >> root_bits], step, table_size, code);
key = GetNextKey(key, len);
}
}
ok = 1;
End:
ok = ok && IsFull(tree);
if (!ok) VP8LHuffmanTreeFree(tree);
return ok;
}
}
int VP8LHuffmanTreeBuildExplicit(HuffmanTree* const tree,
const int* const code_lengths,
const int* const codes,
const int* const symbols, int max_symbol,
int num_symbols) {
int ok = 0;
int i;
assert(tree != NULL);
assert(code_lengths != NULL);
assert(codes != NULL);
assert(symbols != NULL);
// Initialize the tree. Will fail if num_symbols = 0.
if (!TreeInit(tree, num_symbols)) return 0;
// Add symbols one-by-one.
for (i = 0; i < num_symbols; ++i) {
if (codes[i] != NON_EXISTENT_SYMBOL) {
if (symbols[i] < 0 || symbols[i] >= max_symbol) {
goto End;
}
if (!TreeAddSymbol(tree, symbols[i], codes[i], code_lengths[i])) {
goto End;
}
// Check if tree is full.
if (num_nodes != 2 * offset[MAX_ALLOWED_CODE_LENGTH] - 1) {
WebPSafeFree(sorted);
return 0;
}
}
ok = 1;
End:
ok = ok && IsFull(tree);
if (!ok) VP8LHuffmanTreeFree(tree);
return ok;
WebPSafeFree(sorted);
return total_size;
}

83
src/utils/huffman.h
View File

@ -22,78 +22,39 @@
extern "C" {
#endif
// A node of a Huffman tree.
#define HUFFMAN_TABLE_BITS 8
#define HUFFMAN_TABLE_MASK ((1 << HUFFMAN_TABLE_BITS) - 1)
#define LENGTHS_TABLE_BITS 7
#define LENGTHS_TABLE_MASK ((1 << LENGTHS_TABLE_BITS) - 1)
// Huffman lookup table entry
typedef struct {
int symbol_;
int children_; // delta offset to both children (contiguous) or 0 if leaf.
} HuffmanTreeNode;
uint8_t bits; // number of bits used for this symbol
uint16_t value; // symbol value or table offset
} HuffmanCode;
// Huffman Tree.
#define HUFF_LUT_BITS 7
#define HUFF_LUT (1U << HUFF_LUT_BITS)
typedef struct HuffmanTree HuffmanTree;
struct HuffmanTree {
// Fast lookup for short bit lengths.
uint8_t lut_bits_[HUFF_LUT];
int16_t lut_symbol_[HUFF_LUT];
int16_t lut_jump_[HUFF_LUT];
// Complete tree for lookups.
HuffmanTreeNode* root_; // all the nodes, starting at root.
int max_nodes_; // max number of nodes
int num_nodes_; // number of currently occupied nodes
};
// Huffman Tree group.
// Huffman table group.
typedef struct HTreeGroup HTreeGroup;
struct HTreeGroup {
HuffmanTree htrees_[HUFFMAN_CODES_PER_META_CODE];
HuffmanCode* htrees[HUFFMAN_CODES_PER_META_CODE];
};
// Returns true if the given node is not a leaf of the Huffman tree.
static WEBP_INLINE int HuffmanTreeNodeIsNotLeaf(
const HuffmanTreeNode* const node) {
return node->children_;
}
// Go down one level. Most critical function. 'right_child' must be 0 or 1.
static WEBP_INLINE const HuffmanTreeNode* HuffmanTreeNextNode(
const HuffmanTreeNode* node, int right_child) {
return node + node->children_ + right_child;
}
// Releases the nodes of the Huffman tree.
// Note: It does NOT free 'tree' itself.
void VP8LHuffmanTreeFree(HuffmanTree* const tree);
// Creates the instance of HTreeGroup with specified number of tree-groups.
HTreeGroup* VP8LHtreeGroupsNew(int num_htree_groups);
// Releases the memory allocated for HTreeGroup.
void VP8LHtreeGroupsFree(HTreeGroup* htree_groups, int num_htree_groups);
void VP8LHtreeGroupsFree(HTreeGroup* const htree_groups);
// Builds Huffman tree assuming code lengths are implicitly in symbol order.
// The 'huff_codes' and 'code_lengths' are pre-allocated temporary memory
// buffers, used for creating the huffman tree.
// Returns false in case of error (invalid tree or memory error).
int VP8LHuffmanTreeBuildImplicit(HuffmanTree* const tree,
const int* const code_lengths,
int* const huff_codes,
int code_lengths_size);
// Build a Huffman tree with explicitly given lists of code lengths, codes
// and symbols. Verifies that all symbols added are smaller than max_symbol.
// Returns false in case of an invalid symbol, invalid tree or memory error.
int VP8LHuffmanTreeBuildExplicit(HuffmanTree* const tree,
const int* const code_lengths,
const int* const codes,
const int* const symbols, int max_symbol,
int num_symbols);
// Utility: converts Huffman code lengths to corresponding Huffman codes.
// 'huff_codes' should be pre-allocated.
// Returns false in case of error (memory allocation, invalid codes).
int VP8LHuffmanCodeLengthsToCodes(const int* const code_lengths,
int code_lengths_size, int* const huff_codes);
// Builds Huffman lookup table assuming code lengths are in symbol order.
// The 'code_lengths' is pre-allocated temporary memory buffer used for creating
// the huffman table.
// Returns built table size or 0 in case of error (invalid tree or
// memory error).
int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits,
const int* const code_lengths,
int code_lengths_size);
#ifdef __cplusplus
} // extern "C"