Merge branch 'lossless_encoder'

* lossless_encoder: (46 commits)
  split StoreHuffmanCode() into smaller functions
  more consolidation: introduce VP8LHistogramSet
  big code clean-up and refactoring and optimization
  Some cosmetics in histogram.c
  Approximate FastLog between value range [256, 8192]
  Forgot to update out_bit_costs to symbol_bit_costs at one instance.
  Evaluate output cluster's bit_costs once in HistogramRefine.
  Simple Huffman code changes.
  Lossless decoder: remove an unneeded param in ReadHuffmanCodeLengths().
  Reducing emerging palette size from 11 to 9 bits.
  Move GetHistImageSymbols to histogram.c
  Improve predict vs no-predict heuristic.
  code-moving and clean-up
  reduce memory usage by allocating only one histo
  Restrict histo_bits to ensure histo_image size is under 32MB
  further simplification for the meta-Huffman coding
  A quick pass of cleanup in backward reference code
  Make transform bits a function of encode method (-m).
  introduce -lossless option, protected by USE_LOSSLESS_ENCODER
  Run TraceBackwards for higher qualities.
  ...

Conflicts:
	src/enc/webpenc.c

Change-Id: I9a5d98cba0889ea91d10699466939cc283da345a
This commit is contained in:
James Zern
2012-05-07 14:27:17 -07:00
20 changed files with 4407 additions and 295 deletions

View File

@ -8,6 +8,7 @@
// Bit writing and boolean coder
//
// Author: Skal (pascal.massimino@gmail.com)
// Vikas Arora (vikaas.arora@gmail.com)
#include <assert.h>
#include <string.h> // for memcpy()
@ -186,6 +187,86 @@ void VP8BitWriterWipeOut(VP8BitWriter* const bw) {
}
}
#ifdef USE_LOSSLESS_ENCODER
//------------------------------------------------------------------------------
// VP8LBitWriter
// Returns 1 on success.
static int VP8LBitWriterResize(VP8LBitWriter* const bw, size_t extra_size) {
uint8_t* allocated_buf;
size_t allocated_size;
const size_t size_required = VP8LBitWriterNumBytes(bw) + extra_size;
if ((bw->max_bytes_ > 0) && (size_required <= bw->max_bytes_)) return 1;
allocated_size = (3 * bw->max_bytes_) >> 1;
if (allocated_size < size_required) {
allocated_size = size_required;
}
// Make Allocated size multiple of KBs
allocated_size = (((allocated_size >> 10) + 1) << 10);
allocated_buf = (uint8_t*)malloc(allocated_size);
if (allocated_buf == NULL) return 0;
memset(allocated_buf, 0, allocated_size);
if (bw->bit_pos_ > 0) {
memcpy(allocated_buf, bw->buf_, VP8LBitWriterNumBytes(bw));
}
free(bw->buf_);
bw->buf_ = allocated_buf;
bw->max_bytes_ = allocated_size;
return 1;
}
int VP8LBitWriterInit(VP8LBitWriter* const bw, size_t expected_size) {
memset(bw, 0, sizeof(*bw));
return VP8LBitWriterResize(bw, expected_size);
}
void VP8LBitWriterDestroy(VP8LBitWriter* const bw) {
if (bw != NULL) {
free(bw->buf_);
memset(bw, 0, sizeof(*bw));
}
}
void VP8LWriteBits(VP8LBitWriter* const bw, int n_bits, uint32_t bits) {
if (n_bits < 1) return;
#if !defined(__BIG_ENDIAN__)
// Technically, this branch of the code can write up to 25 bits at a time,
// but in deflate, the maximum number of bits written is 16 at a time.
{
uint8_t* p = &bw->buf_[bw->bit_pos_ >> 3];
uint32_t v = *(const uint32_t*)(p);
v |= bits << (bw->bit_pos_ & 7);
*(uint32_t*)(p) = v;
bw->bit_pos_ += n_bits;
}
#else // LITTLE_ENDIAN
// implicit & 0xff is assumed for uint8_t arithmetics
{
uint8_t* p = &bw->buf_[bw->bit_pos_ >> 3];
const int bits_reserved_in_first_byte = (bw->bit_pos_ & 7);
*p++ |= (bits << bits_reserved_in_first_byte);
const int bits_left_to_write = n_bits - 8 + bits_reserved_in_first_byte;
if (bits_left_to_write >= 1) {
*p++ = bits >> (8 - bits_reserved_in_first_byte);
if (bits_left_to_write >= 9) {
*p++ = bits >> (16 - bits_reserved_in_first_byte);
}
}
*p = 0;
bw->bit_pos_ += n_bits;
}
#endif // BIG_ENDIAN
if ((bw->bit_pos_ >> 3) > (bw->max_bytes_ - 8)) {
const size_t kAdditionalBuffer = 32768 + bw->max_bytes_;
if (!VP8LBitWriterResize(bw, kAdditionalBuffer)) {
bw->bit_pos_ = 0;
bw->error_ = 1;
}
}
}
#endif
//------------------------------------------------------------------------------
#if defined(__cplusplus) || defined(c_plusplus)

View File

@ -64,7 +64,59 @@ static WEBP_INLINE size_t VP8BitWriterSize(const VP8BitWriter* const bw) {
return bw->pos_;
}
#ifdef USE_LOSSLESS_ENCODER
//------------------------------------------------------------------------------
// VP8LBitWriter
// TODO(vikasa): VP8LBitWriter is copied as-is from lossless code. There's scope
// of re-using VP8BitWriter. Will evaluate once basic lossless encoder is
// implemented.
typedef struct {
uint8_t* buf_;
size_t bit_pos_;
size_t max_bytes_;
// After all bits are written, the caller must observe the state of
// error_. A value of 1 indicates that a memory allocation failure
// has happened during bit writing. A value of 0 indicates successful
// writing of bits.
int error_;
} VP8LBitWriter;
static WEBP_INLINE size_t VP8LBitWriterNumBytes(VP8LBitWriter* const bw) {
return (bw->bit_pos_ + 7) >> 3;
}
static WEBP_INLINE uint8_t* VP8LBitWriterFinish(VP8LBitWriter* const bw) {
return bw->buf_;
}
// Returns 0 in case of memory allocation error.
int VP8LBitWriterInit(VP8LBitWriter* const bw, size_t expected_size);
void VP8LBitWriterDestroy(VP8LBitWriter* const bw);
// This function writes bits into bytes in increasing addresses, and within
// a byte least-significant-bit first.
//
// The function can write up to 16 bits in one go with WriteBits
// Example: let's assume that 3 bits (Rs below) have been written already:
//
// BYTE-0 BYTE+1 BYTE+2
//
// 0000 0RRR 0000 0000 0000 0000
//
// Now, we could write 5 or less bits in MSB by just sifting by 3
// and OR'ing to BYTE-0.
//
// For n bits, we take the last 5 bytes, OR that with high bits in BYTE-0,
// and locate the rest in BYTE+1 and BYTE+2.
//
// VP8LBitWriter's error_ flag is set in case of memory allocation error.
void VP8LWriteBits(VP8LBitWriter* const bw, int n_bits, uint32_t bits);
//------------------------------------------------------------------------------
#endif
#if defined(__cplusplus) || defined(c_plusplus)
} // extern "C"

View File

@ -32,14 +32,17 @@ int VP8LColorCacheInit(VP8LColorCache* const cc, int hash_bits) {
return 1;
}
void VP8LColorCacheDelete(VP8LColorCache* const cc) {
void VP8LColorCacheClear(VP8LColorCache* const cc) {
if (cc != NULL) {
free(cc->colors_);
free(cc);
}
}
void VP8LColorCacheDelete(VP8LColorCache* const cc) {
VP8LColorCacheClear(cc);
free(cc);
}
#if defined(__cplusplus) || defined(c_plusplus)
}
#endif

View File

@ -39,6 +39,7 @@ static WEBP_INLINE void VP8LColorCacheInsert(const VP8LColorCache* const cc,
cc->colors_[key] = argb;
}
#ifdef USE_LOSSLESS_ENCODER
static WEBP_INLINE int VP8LColorCacheGetIndex(const VP8LColorCache* const cc,
uint32_t argb) {
return (kHashMul * argb) >> cc->hash_shift_;
@ -49,6 +50,7 @@ static WEBP_INLINE int VP8LColorCacheContains(const VP8LColorCache* const cc,
const uint32_t key = (kHashMul * argb) >> cc->hash_shift_;
return cc->colors_[key] == argb;
}
#endif
//------------------------------------------------------------------------------
@ -57,6 +59,9 @@ static WEBP_INLINE int VP8LColorCacheContains(const VP8LColorCache* const cc,
int VP8LColorCacheInit(VP8LColorCache* const color_cache, int hash_bits);
// Delete the color cache.
void VP8LColorCacheClear(VP8LColorCache* const color_cache);
// Delete the color_cache object.
void VP8LColorCacheDelete(VP8LColorCache* const color_cache);
//------------------------------------------------------------------------------

318
src/utils/huffman_encode.c Normal file
View File

@ -0,0 +1,318 @@
// Copyright 2011 Google Inc. All Rights Reserved.
//
// This code is licensed under the same terms as WebM:
// Software License Agreement: http://www.webmproject.org/license/software/
// Additional IP Rights Grant: http://www.webmproject.org/license/additional/
// -----------------------------------------------------------------------------
//
// Author: jyrki@google.com (Jyrki Alakuijala)
//
// Flate like entropy encoding (Huffman) for webp lossless.
#ifdef USE_LOSSLESS_ENCODER
#include "./huffman_encode.h"
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
typedef struct {
int total_count_;
int value_;
int pool_index_left_;
int pool_index_right_;
} HuffmanTree;
// Sort the root nodes, most popular first.
static int CompHuffmanTree(const void* vp0, const void* vp1) {
const HuffmanTree* v0 = (const HuffmanTree*)vp0;
const HuffmanTree* v1 = (const HuffmanTree*)vp1;
if (v0->total_count_ > v1->total_count_) {
return -1;
} else if (v0->total_count_ < v1->total_count_) {
return 1;
} else {
if (v0->value_ < v1->value_) {
return -1;
}
if (v0->value_ > v1->value_) {
return 1;
}
return 0;
}
}
static void SetDepth(const HuffmanTree* p,
HuffmanTree* pool,
uint8_t* depth,
const int level) {
if (p->pool_index_left_ >= 0) {
SetDepth(&pool[p->pool_index_left_], pool, depth, level + 1);
SetDepth(&pool[p->pool_index_right_], pool, depth, level + 1);
} else {
depth[p->value_] = level;
}
}
// This function will create a Huffman tree.
//
// The catch here is that the tree cannot be arbitrarily deep.
// Deflate specifies a maximum depth of 15 bits for "code trees"
// and 7 bits for "code length code trees."
//
// count_limit is the value that is to be faked as the minimum value
// and this minimum value is raised until the tree matches the
// maximum length requirement.
//
// This algorithm is not of excellent performance for very long data blocks,
// especially when population counts are longer than 2**tree_limit, but
// we are not planning to use this with extremely long blocks.
//
// See http://en.wikipedia.org/wiki/Huffman_coding
int VP8LCreateHuffmanTree(const int* const histogram, int histogram_size,
int tree_depth_limit,
uint8_t* const bit_depths) {
HuffmanTree* tree;
HuffmanTree* tree_pool;
int tree_pool_size;
// For block sizes with less than 64k symbols we never need to do a
// second iteration of this loop.
// If we actually start running inside this loop a lot, we would perhaps
// be better off with the Katajainen algorithm.
int count_limit;
for (count_limit = 1; ; count_limit *= 2) {
int tree_size = 0;
int i;
for (i = 0; i < histogram_size; ++i) {
if (histogram[i]) {
++tree_size;
}
}
// 3 * tree_size is enough to cover all the nodes representing a
// population and all the inserted nodes combining two existing nodes.
// The tree pool needs 2 * (tree_size - 1) entities, and the
// tree needs exactly tree_size entities.
tree = (HuffmanTree*)malloc(3 * tree_size * sizeof(*tree));
if (tree == NULL) {
return 0;
}
{
int j = 0;
int i;
for (i = 0; i < histogram_size; ++i) {
if (histogram[i]) {
const int count =
(histogram[i] < count_limit) ? count_limit : histogram[i];
tree[j].total_count_ = count;
tree[j].value_ = i;
tree[j].pool_index_left_ = -1;
tree[j].pool_index_right_ = -1;
++j;
}
}
}
qsort((void*)tree, tree_size, sizeof(*tree), CompHuffmanTree);
tree_pool = tree + tree_size;
tree_pool_size = 0;
if (tree_size >= 2) {
while (tree_size >= 2) { // Finish when we have only one root.
int count;
tree_pool[tree_pool_size] = tree[tree_size - 1];
++tree_pool_size;
tree_pool[tree_pool_size] = tree[tree_size - 2];
++tree_pool_size;
count =
tree_pool[tree_pool_size - 1].total_count_ +
tree_pool[tree_pool_size - 2].total_count_;
tree_size -= 2;
{
int k = 0;
// Search for the insertion point.
for (k = 0; k < tree_size; ++k) {
if (tree[k].total_count_ <= count) {
break;
}
}
memmove(tree + (k + 1), tree + k, (tree_size - k) * sizeof(*tree));
tree[k].total_count_ = count;
tree[k].value_ = -1;
tree[k].pool_index_left_ = tree_pool_size - 1;
tree[k].pool_index_right_ = tree_pool_size - 2;
tree_size = tree_size + 1;
}
}
SetDepth(&tree[0], tree_pool, bit_depths, 0);
} else {
if (tree_size == 1) {
// Only one element.
bit_depths[tree[0].value_] = 1;
}
}
free(tree);
// We need to pack the Huffman tree in tree_depth_limit bits.
// If this was not successful, add fake entities to the lowest values
// and retry.
{
int max_depth = bit_depths[0];
int j;
for (j = 1; j < histogram_size; ++j) {
if (max_depth < bit_depths[j]) {
max_depth = bit_depths[j];
}
}
if (max_depth <= tree_depth_limit) {
break;
}
}
}
return 1;
}
static void WriteHuffmanTreeRepetitions(
const int value,
const int prev_value,
int repetitions,
int* num_symbols,
uint8_t* tree,
uint8_t* extra_bits_data) {
if (value != prev_value) {
tree[*num_symbols] = value;
extra_bits_data[*num_symbols] = 0;
++(*num_symbols);
--repetitions;
}
while (repetitions >= 1) {
if (repetitions < 3) {
int i;
for (i = 0; i < repetitions; ++i) {
tree[*num_symbols] = value;
extra_bits_data[*num_symbols] = 0;
++(*num_symbols);
}
return;
} else if (repetitions < 7) {
// 3 to 6 left
tree[*num_symbols] = 16;
extra_bits_data[*num_symbols] = repetitions - 3;
++(*num_symbols);
return;
} else {
tree[*num_symbols] = 16;
extra_bits_data[*num_symbols] = 3;
++(*num_symbols);
repetitions -= 6;
}
}
}
static void WriteHuffmanTreeRepetitionsZeros(
const int value,
int repetitions,
int* num_symbols,
uint8_t* tree,
uint8_t* extra_bits_data) {
while (repetitions >= 1) {
if (repetitions < 3) {
int i;
for (i = 0; i < repetitions; ++i) {
tree[*num_symbols] = value;
extra_bits_data[*num_symbols] = 0;
++(*num_symbols);
}
return;
} else if (repetitions < 11) {
tree[*num_symbols] = 17;
extra_bits_data[*num_symbols] = repetitions - 3;
++(*num_symbols);
return;
} else if (repetitions < 139) {
tree[*num_symbols] = 18;
extra_bits_data[*num_symbols] = repetitions - 11;
++(*num_symbols);
return;
} else {
tree[*num_symbols] = 18;
extra_bits_data[*num_symbols] = 0x7f; // 138 repeated 0s
++(*num_symbols);
repetitions -= 138;
}
}
}
void VP8LCreateCompressedHuffmanTree(const uint8_t* const depth,
int depth_size,
int* num_symbols,
uint8_t* tree,
uint8_t* extra_bits_data) {
int prev_value = 8; // 8 is the initial value for rle.
int i;
for (i = 0; i < depth_size;) {
const int value = depth[i];
int reps = 1;
int k;
for (k = i + 1; k < depth_size && depth[k] == value; ++k) {
++reps;
}
if (value == 0) {
WriteHuffmanTreeRepetitionsZeros(value, reps,
num_symbols,
tree, extra_bits_data);
} else {
WriteHuffmanTreeRepetitions(value, prev_value, reps,
num_symbols,
tree, extra_bits_data);
prev_value = value;
}
i += reps;
}
}
static uint32_t ReverseBits(int num_bits, uint32_t bits) {
uint32_t retval = 0;
int i;
for (i = 0; i < num_bits; ++i) {
retval <<= 1;
retval |= bits & 1;
bits >>= 1;
}
return retval;
}
void VP8LConvertBitDepthsToSymbols(const uint8_t* depth, int len,
uint16_t* bits) {
// This function is based on RFC 1951.
//
// In deflate, all bit depths are [1..15]
// 0 bit depth means that the symbol does not exist.
// 0..15 are values for bits
#define MAX_BITS 16
uint32_t next_code[MAX_BITS];
uint32_t bl_count[MAX_BITS] = { 0 };
int i;
{
for (i = 0; i < len; ++i) {
++bl_count[depth[i]];
}
bl_count[0] = 0;
}
next_code[0] = 0;
{
int code = 0;
int bits;
for (bits = 1; bits < MAX_BITS; ++bits) {
code = (code + bl_count[bits - 1]) << 1;
next_code[bits] = code;
}
}
for (i = 0; i < len; ++i) {
if (depth[i]) {
bits[i] = ReverseBits(depth[i], next_code[depth[i]]++);
}
}
}
#undef MAX_BITS
#endif

View File

@ -0,0 +1,54 @@
// Copyright 2011 Google Inc. All Rights Reserved.
//
// This code is licensed under the same terms as WebM:
// Software License Agreement: http://www.webmproject.org/license/software/
// Additional IP Rights Grant: http://www.webmproject.org/license/additional/
// -----------------------------------------------------------------------------
//
// Author: jyrki@google.com (Jyrki Alakuijala)
//
// Flate like entropy encoding (Huffman) for webp lossless
#ifndef WEBP_UTILS_HUFFMAN_ENCODE_H_
#define WEBP_UTILS_HUFFMAN_ENCODE_H_
#ifdef USE_LOSSLESS_ENCODER
#include <stdint.h>
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
// This function will create a Huffman tree.
//
// The (data,length) contains the population counts.
// The tree_limit is the maximum bit depth of the Huffman codes.
//
// The depth contains the tree, i.e., how many bits are used for
// the symbol.
//
// See http://en.wikipedia.org/wiki/Huffman_coding
//
// Returns 0 when an error has occured.
int VP8LCreateHuffmanTree(const int* data, const int length,
const int tree_limit, uint8_t* depth);
// Write a huffman tree from bit depths. The generated Huffman tree is
// compressed once more using a Huffman tree.
void VP8LCreateCompressedHuffmanTree(const uint8_t* const depth, int len,
int* num_symbols,
uint8_t* tree,
uint8_t* extra_bits_data);
// Get the actual bit values for a tree of bit depths.
void VP8LConvertBitDepthsToSymbols(const uint8_t* depth, int len,
uint16_t* bits);
#if defined(__cplusplus) || defined(c_plusplus)
}
#endif
#endif
#endif // WEBP_UTILS_HUFFMAN_ENCODE_H_