clean-up around Huffman-encode

* refine doc
* use LUT for bit-reversal
* added assert
* Introduce HuffmanTreeToken instead of separate code/extra_bits arrays

Change-Id: I0fe0b50b55eb43a4be9f730b1abe40632a6fa7f0

src/enc/vp8l.c

@@ -387,25 +387,25 @@ static void StoreHuffmanTreeOfHuffmanTreeToBitMask(
 
 static void StoreHuffmanTreeToBitMask(
     VP8LBitWriter* const bw,
-    const uint8_t* huffman_tree,
-    const uint8_t* huffman_tree_extra_bits,
-    const int num_symbols,
+    const HuffmanTreeToken* const tokens,
+    const int num_tokens,
     const uint8_t* code_length_bitdepth,
     const uint16_t* code_length_bitdepth_symbols) {
   int i;
-  for (i = 0; i < num_symbols; ++i) {
-    const int ix = huffman_tree[i];
+  for (i = 0; i < num_tokens; ++i) {
+    const int ix = tokens[i].code;
+    const int extra_bits = tokens[i].extra_bits;
     VP8LWriteBits(bw, code_length_bitdepth[ix],
                   code_length_bitdepth_symbols[ix]);
     switch (ix) {
       case 16:
-        VP8LWriteBits(bw, 2, huffman_tree_extra_bits[i]);
+        VP8LWriteBits(bw, 2, extra_bits);
         break;
       case 17:
-        VP8LWriteBits(bw, 3, huffman_tree_extra_bits[i]);
+        VP8LWriteBits(bw, 3, extra_bits);
         break;
       case 18:
-        VP8LWriteBits(bw, 7, huffman_tree_extra_bits[i]);
+        VP8LWriteBits(bw, 7, extra_bits);
         break;
     }
   }
@@ -415,27 +415,21 @@ static int StoreFullHuffmanCode(VP8LBitWriter* const bw,
                                 const uint8_t* const bit_lengths,
                                 int bit_lengths_size) {
   int ok = 0;
-  int huffman_tree_size = 0;
   uint8_t code_length_bitdepth[CODE_LENGTH_CODES] = { 0 };
   uint16_t code_length_bitdepth_symbols[CODE_LENGTH_CODES] = { 0 };
-  uint8_t* huffman_tree_extra_bits;
-  uint8_t* const huffman_tree =
-      (uint8_t*)malloc(bit_lengths_size * sizeof(*huffman_tree) +
-                       bit_lengths_size * sizeof(*huffman_tree_extra_bits));
-
-  if (huffman_tree == NULL) return 0;
-  huffman_tree_extra_bits =
-      huffman_tree + bit_lengths_size * sizeof(*huffman_tree);
+  int num_tokens;
+  HuffmanTreeToken* const tokens =
+      (HuffmanTreeToken*)malloc(bit_lengths_size * sizeof(*tokens));
+  if (tokens == NULL) return 0;
 
   VP8LWriteBits(bw, 1, 0);
-  VP8LCreateCompressedHuffmanTree(bit_lengths, bit_lengths_size,
-                                  &huffman_tree_size, huffman_tree,
-                                  huffman_tree_extra_bits);
+  num_tokens = VP8LCreateCompressedHuffmanTree(bit_lengths, bit_lengths_size,
+                                               tokens, bit_lengths_size);
   {
     int histogram[CODE_LENGTH_CODES] = { 0 };
     int i;
-    for (i = 0; i < huffman_tree_size; ++i) {
-      ++histogram[huffman_tree[i]];
+    for (i = 0; i < num_tokens; ++i) {
+      ++histogram[tokens[i].code];
     }
 
     if (!VP8LCreateHuffmanTree(histogram, CODE_LENGTH_CODES,
@@ -451,12 +445,12 @@ static int StoreFullHuffmanCode(VP8LBitWriter* const bw,
                                   code_length_bitdepth_symbols);
   {
     int trailing_zero_bits = 0;
-    int trimmed_length = huffman_tree_size;
+    int trimmed_length = num_tokens;
     int write_trimmed_length;
     int length;
-    int i = huffman_tree_size;
+    int i = num_tokens;
     while (i-- > 0) {
-      const int ix = huffman_tree[i];
+      const int ix = tokens[i].code;
       if (ix == 0 || ix == 17 || ix == 18) {
         --trimmed_length;   // discount trailing zeros
         trailing_zero_bits += code_length_bitdepth[ix];
@@ -470,7 +464,7 @@ static int StoreFullHuffmanCode(VP8LBitWriter* const bw,
       }
     }
     write_trimmed_length = (trimmed_length > 1 && trailing_zero_bits > 12);
-    length = write_trimmed_length ? trimmed_length : huffman_tree_size;
+    length = write_trimmed_length ? trimmed_length : num_tokens;
     VP8LWriteBits(bw, 1, write_trimmed_length);
     if (write_trimmed_length) {
       const int nbits = VP8LBitsLog2Ceiling(trimmed_length - 1);
@@ -478,13 +472,13 @@ static int StoreFullHuffmanCode(VP8LBitWriter* const bw,
       VP8LWriteBits(bw, 3, nbitpairs - 1);
       VP8LWriteBits(bw, nbitpairs * 2, trimmed_length - 2);
     }
-    StoreHuffmanTreeToBitMask(bw, huffman_tree, huffman_tree_extra_bits,
+    StoreHuffmanTreeToBitMask(bw, tokens,
                               length, code_length_bitdepth,
                               code_length_bitdepth_symbols);
   }
   ok = 1;
  End:
-  free(huffman_tree);
+  free(tokens);
   return ok;
 }
 

src/utils/huffman_encode.c

@@ -7,11 +7,14 @@
 //
 // Author: jyrki@google.com (Jyrki Alakuijala)
 //
-// Flate like entropy encoding (Huffman) for webp lossless.
+// Flate-like entropy encoding (Huffman) for webp lossless.
 
 #ifdef USE_LOSSLESS_ENCODER
 
 #include "./huffman_encode.h"
+
+#define MAX_BITS 16   // maximum allowed length for the codes
+
 #include <assert.h>
 #include <stdint.h>
 #include <stdlib.h>
@@ -57,9 +60,7 @@ static void SetBitDepths(const HuffmanTree* const tree,
 
 // This function will create a Huffman tree.
 //
-// The catch here is that the tree cannot be arbitrarily deep.
-// Deflate specifies a maximum depth of 15 bits for "code trees"
-// and 7 bits for "code length code trees."
+// The catch here is that the tree cannot be arbitrarily deep
 //
 // count_limit is the value that is to be faked as the minimum value
 // and this minimum value is raised until the tree matches the
@@ -165,149 +166,144 @@ int VP8LCreateHuffmanTree(const int* const histogram, int histogram_size,
   return 1;
 }
 
-static void WriteHuffmanTreeRepetitions(
-    const int value,
-    const int prev_value,
-    int repetitions,
-    int* num_symbols,
-    uint8_t* tree,
-    uint8_t* extra_bits_data) {
+// -----------------------------------------------------------------------------
+// Coding of the Huffman tree values
+
+static HuffmanTreeToken* CodeRepeatedValues(int repetitions,
+                                            HuffmanTreeToken* tokens,
+                                            int value, int prev_value) {
+  assert(value < MAX_BITS);
   if (value != prev_value) {
-    tree[*num_symbols] = value;
-    extra_bits_data[*num_symbols] = 0;
-    ++(*num_symbols);
+    tokens->code = value;
+    tokens->extra_bits = 0;
+    ++tokens;
     --repetitions;
   }
   while (repetitions >= 1) {
     if (repetitions < 3) {
       int i;
       for (i = 0; i < repetitions; ++i) {
-        tree[*num_symbols] = value;
-        extra_bits_data[*num_symbols] = 0;
-        ++(*num_symbols);
+        tokens->code = value;
+        tokens->extra_bits = 0;
+        ++tokens;
       }
-      return;
+      break;
     } else if (repetitions < 7) {
-      // 3 to 6 left
-      tree[*num_symbols] = 16;
-      extra_bits_data[*num_symbols] = repetitions - 3;
-      ++(*num_symbols);
-      return;
+      tokens->code = 16;
+      tokens->extra_bits = repetitions - 3;
+      ++tokens;
+      break;
     } else {
-      tree[*num_symbols] = 16;
-      extra_bits_data[*num_symbols] = 3;
-      ++(*num_symbols);
+      tokens->code = 16;
+      tokens->extra_bits = 3;
+      ++tokens;
       repetitions -= 6;
     }
   }
+  return tokens;
 }
 
-static void WriteHuffmanTreeRepetitionsZeros(
-    const int value,
-    int repetitions,
-    int* num_symbols,
-    uint8_t* tree,
-    uint8_t* extra_bits_data) {
+static HuffmanTreeToken* CodeRepeatedZeros(int repetitions,
+                                           HuffmanTreeToken* tokens) {
   while (repetitions >= 1) {
     if (repetitions < 3) {
       int i;
       for (i = 0; i < repetitions; ++i) {
-        tree[*num_symbols] = value;
-        extra_bits_data[*num_symbols] = 0;
-        ++(*num_symbols);
+        tokens->code = 0;   // 0-value
+        tokens->extra_bits = 0;
+        ++tokens;
       }
-      return;
+      break;
     } else if (repetitions < 11) {
-      tree[*num_symbols] = 17;
-      extra_bits_data[*num_symbols] = repetitions - 3;
-      ++(*num_symbols);
-      return;
+      tokens->code = 17;
+      tokens->extra_bits = repetitions - 3;
+      ++tokens;
+      break;
     } else if (repetitions < 139) {
-      tree[*num_symbols] = 18;
-      extra_bits_data[*num_symbols] = repetitions - 11;
-      ++(*num_symbols);
-      return;
+      tokens->code = 18;
+      tokens->extra_bits = repetitions - 11;
+      ++tokens;
+      break;
     } else {
-      tree[*num_symbols] = 18;
-      extra_bits_data[*num_symbols] = 0x7f;  // 138 repeated 0s
-      ++(*num_symbols);
+      tokens->code = 18;
+      tokens->extra_bits = 0x7f;  // 138 repeated 0s
+      ++tokens;
       repetitions -= 138;
     }
   }
+  return tokens;
 }
 
-void VP8LCreateCompressedHuffmanTree(const uint8_t* const depth,
-                                     int depth_size,
-                                     int* num_symbols,
-                                     uint8_t* tree,
-                                     uint8_t* extra_bits_data) {
+int VP8LCreateCompressedHuffmanTree(const uint8_t* const depth,
+                                    int depth_size,
+                                    HuffmanTreeToken* tokens,
+                                    int max_tokens) {
+  HuffmanTreeToken* const starting_token = tokens;
+  HuffmanTreeToken* const ending_token = tokens + max_tokens;
   int prev_value = 8;  // 8 is the initial value for rle.
-  int i;
-  for (i = 0; i < depth_size;) {
+  int i = 0;
+  while (i < depth_size) {
     const int value = depth[i];
-    int reps = 1;
-    int k;
-    for (k = i + 1; k < depth_size && depth[k] == value; ++k) {
-      ++reps;
-    }
+    int k = i + 1;
+    int runs;
+    while (k < depth_size && depth[k] == value) ++k;
+    runs = k - i;
     if (value == 0) {
-      WriteHuffmanTreeRepetitionsZeros(value, reps,
-                                       num_symbols,
-                                       tree, extra_bits_data);
+      tokens = CodeRepeatedZeros(runs, tokens);
     } else {
-      WriteHuffmanTreeRepetitions(value, prev_value, reps,
-                                  num_symbols,
-                                  tree, extra_bits_data);
+      tokens = CodeRepeatedValues(runs, tokens, value, prev_value);
       prev_value = value;
     }
-    i += reps;
+    i += runs;
+    assert(tokens <= ending_token);
   }
+  (void)ending_token;    // suppress 'unused variable' warning
+  return tokens - starting_token;
 }
 
+// -----------------------------------------------------------------------------
+
+// Pre-reversed 4-bit values.
+static const uint8_t kReversedBits[16] = {
+  0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe,
+  0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf
+};
+
 static uint32_t ReverseBits(int num_bits, uint32_t bits) {
   uint32_t retval = 0;
-  int i;
-  for (i = 0; i < num_bits; ++i) {
-    retval <<= 1;
-    retval |= bits & 1;
-    bits >>= 1;
+  int i = 0;
+  while (i < num_bits) {
+    i += 4;
+    retval |= kReversedBits[bits & 0xf] << (MAX_BITS - i);
+    bits >>= 4;
   }
+  retval >>= (MAX_BITS - num_bits);
   return retval;
 }
 
-void VP8LConvertBitDepthsToSymbols(const uint8_t* depth, int len,
-                                   uint16_t* bits) {
-  // This function is based on RFC 1951.
-  //
-  // In deflate, all bit depths are [1..15]
-  // 0 bit depth means that the symbol does not exist.
-
-  // 0..15 are values for bits
-#define MAX_BITS 16
-  uint32_t next_code[MAX_BITS];
-  uint32_t bl_count[MAX_BITS] = { 0 };
+void VP8LConvertBitDepthsToSymbols(const uint8_t* const depth,
+                                   int len,
+                                   uint16_t* const bits) {
+  // 0 bit-depth means that the symbol does not exist.
   int i;
-  {
-    for (i = 0; i < len; ++i) {
-      ++bl_count[depth[i]];
-    }
-    bl_count[0] = 0;
+  uint32_t next_code[MAX_BITS];
+  int depth_count[MAX_BITS] = { 0 };
+  for (i = 0; i < len; ++i) {
+    assert(depth[i] < MAX_BITS);
+    ++depth_count[depth[i]];
   }
+  depth_count[0] = 0;  // ignore unused symbol
   next_code[0] = 0;
   {
-    int code = 0;
-    int bits;
-    for (bits = 1; bits < MAX_BITS; ++bits) {
-      code = (code + bl_count[bits - 1]) << 1;
-      next_code[bits] = code;
+    uint32_t code = 0;
+    for (i = 1; i < MAX_BITS; ++i) {
+      code = (code + depth_count[i - 1]) << 1;
+      next_code[i] = code;
     }
   }
   for (i = 0; i < len; ++i) {
-    if (depth[i]) {
-      bits[i] = ReverseBits(depth[i], next_code[depth[i]]++);
-    }
+    bits[i] = ReverseBits(depth[i], next_code[depth[i]]++);
   }
 }
-#undef MAX_BITS
 
 #endif

src/utils/huffman_encode.h

@@ -7,7 +7,7 @@
 //
 // Author: jyrki@google.com (Jyrki Alakuijala)
 //
-// Flate like entropy encoding (Huffman) for webp lossless
+// Flate-like entropy encoding (Huffman) for webp lossless
 
 #ifndef WEBP_UTILS_HUFFMAN_ENCODE_H_
 #define WEBP_UTILS_HUFFMAN_ENCODE_H_
@@ -20,27 +20,29 @@
 extern "C" {
 #endif
 
-// This function will create a Huffman tree.
+// Create a Huffman tree.
 //
-// 'histogram' contains the population counts.
-// 'tree_depth_limit' is the maximum bit depth of the Huffman codes.
-// The created tree is returned as 'bit_depths', which stores how many bits are
-// used for each symbol.
-// See http://en.wikipedia.org/wiki/Huffman_coding
-// Returns 0 on memory error.
-int VP8LCreateHuffmanTree(const int* const histogram, int histogram_size,
-                          int tree_depth_limit, uint8_t* const bit_depths);
+// (data,length): population counts.
+// tree_limit: maximum bit depth (inclusive) of the codes.
+// bit_depths[]: how many bits are used for the symbol.
+//
+// Returns 0 when an error has occured.
+int VP8LCreateHuffmanTree(const int* data, const int length,
+                          const int tree_limit, uint8_t* bit_depths);
 
-// Write a huffman tree from bit depths. The generated Huffman tree is
-// compressed once more using a Huffman tree.
-void VP8LCreateCompressedHuffmanTree(const uint8_t* const depth, int len,
-                                     int* num_symbols,
-                                     uint8_t* tree,
-                                     uint8_t* extra_bits_data);
+// Turn the Huffman tree into a token sequence.
+// Returns the number of tokens used.
+typedef struct {
+  uint8_t code;         // value (0..15) or escape code (16,17,18)
+  uint8_t extra_bits;   // extra bits for escape codes
+} HuffmanTreeToken;
+
+int VP8LCreateCompressedHuffmanTree(const uint8_t* const depth, int len,
+                                    HuffmanTreeToken* tokens, int max_tokens);
 
 // Get the actual bit values for a tree of bit depths.
-void VP8LConvertBitDepthsToSymbols(const uint8_t* depth, int len,
-                                   uint16_t* bits);
+void VP8LConvertBitDepthsToSymbols(const uint8_t* const depth, int len,
+                                   uint16_t* const bits);
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }