From 09545eeadc7e889c15a9ccdf2dd23bf67fae2c5a Mon Sep 17 00:00:00 2001 From: Pascal Massimino Date: Sun, 1 Jun 2014 23:19:34 -0700 Subject: [PATCH] lossy bit-reader clean-up: * remove LEFT/RIGHT_JUSTIFY distinction. It's all RIGHT_JUSTIFY now. * simplify VP8GetSigned(), and add some masking branch-less code. Much faster on ARM (~13% speed-up). 8% on x86-64, 5% on MacBook. * split critical implementation into separate bit_reader_inl.h file that is only included where needed (vp8.c / tree.c / bit_reader.c) * bumped BITS value from 16 to 24 for x86-32b too, since it's a bit faster. Change-Id: If41ca1da3e5c3dadacf2379d1ba419b151e7fce8 --- makefile.unix | 5 + src/dec/tree.c | 1 + src/dec/vp8.c | 2 +- src/utils/Makefile.am | 1 + src/utils/bit_reader.c | 60 ++++----- src/utils/bit_reader.h | 258 ++++--------------------------------- src/utils/bit_reader_inl.h | 193 +++++++++++++++++++++++++++ 7 files changed, 246 insertions(+), 274 deletions(-) create mode 100644 src/utils/bit_reader_inl.h diff --git a/makefile.unix b/makefile.unix index 36892202..58dc01f2 100644 --- a/makefile.unix +++ b/makefile.unix @@ -244,6 +244,11 @@ all: ex $(EXTRA_EXAMPLES) $(EX_FORMAT_DEC_OBJS): %.o: %.h +# special dependencies for tree.c/vp8.c/bit_reader.c <-> bit_reader_inl.h +src/dec/tree.o: src/utils/bit_reader_inl.h +src/dec/vp8.o: src/utils/bit_reader_inl.h +src/utils/bit_reader.o: src/utils/bit_reader_inl.h + %.o: %.c $(HDRS) $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ diff --git a/src/dec/tree.c b/src/dec/tree.c index 22d83ee5..88238885 100644 --- a/src/dec/tree.c +++ b/src/dec/tree.c @@ -12,6 +12,7 @@ // Author: Skal (pascal.massimino@gmail.com) #include "vp8i.h" +#include "../utils/bit_reader_inl.h" #define USE_GENERIC_TREE diff --git a/src/dec/vp8.c b/src/dec/vp8.c index c5e280d9..d45dc4bf 100644 --- a/src/dec/vp8.c +++ b/src/dec/vp8.c @@ -17,7 +17,7 @@ #include "./vp8i.h" #include "./vp8li.h" #include "./webpi.h" -#include "../utils/bit_reader.h" +#include "../utils/bit_reader_inl.h" #include "../utils/utils.h" //------------------------------------------------------------------------------ diff --git a/src/utils/Makefile.am b/src/utils/Makefile.am index ac2db270..2c86206c 100644 --- a/src/utils/Makefile.am +++ b/src/utils/Makefile.am @@ -11,6 +11,7 @@ commondir = $(includedir)/webp COMMON_SOURCES = COMMON_SOURCES += bit_reader.c COMMON_SOURCES += bit_reader.h +COMMON_SOURCES += bit_reader_inl.h COMMON_SOURCES += color_cache.c COMMON_SOURCES += color_cache.h COMMON_SOURCES += filters.c diff --git a/src/utils/bit_reader.c b/src/utils/bit_reader.c index 46a913ec..1ee4d002 100644 --- a/src/utils/bit_reader.c +++ b/src/utils/bit_reader.c @@ -7,17 +7,11 @@ // be found in the AUTHORS file in the root of the source tree. // ----------------------------------------------------------------------------- // -// Boolean decoder +// Boolean decoder non-inlined methods // // Author: Skal (pascal.massimino@gmail.com) -#include "./bit_reader.h" - -#ifndef USE_RIGHT_JUSTIFY -#define MK(X) (((range_t)(X) << (BITS)) | (MASK)) -#else -#define MK(X) ((range_t)(X)) -#endif +#include "./bit_reader_inl.h" //------------------------------------------------------------------------------ // VP8BitReader @@ -27,12 +21,13 @@ void VP8InitBitReader(VP8BitReader* const br, assert(br != NULL); assert(start != NULL); assert(start <= end); - br->range_ = MK(255 - 1); + br->range_ = 255 - 1; br->buf_ = start; br->buf_end_ = end; br->value_ = 0; br->bits_ = -8; // to load the very first 8bits br->eof_ = 0; + VP8LoadNewBytes(br); } void VP8RemapBitReader(VP8BitReader* const br, ptrdiff_t offset) { @@ -54,45 +49,35 @@ const uint8_t kVP8Log2Range[128] = { 0 }; -// range = (range << kVP8Log2Range[range]) + trailing 1's +// range = ((range - 1) << kVP8Log2Range[range]) + 1 const range_t kVP8NewRange[128] = { - MK(127), MK(127), MK(191), MK(127), MK(159), MK(191), MK(223), MK(127), - MK(143), MK(159), MK(175), MK(191), MK(207), MK(223), MK(239), MK(127), - MK(135), MK(143), MK(151), MK(159), MK(167), MK(175), MK(183), MK(191), - MK(199), MK(207), MK(215), MK(223), MK(231), MK(239), MK(247), MK(127), - MK(131), MK(135), MK(139), MK(143), MK(147), MK(151), MK(155), MK(159), - MK(163), MK(167), MK(171), MK(175), MK(179), MK(183), MK(187), MK(191), - MK(195), MK(199), MK(203), MK(207), MK(211), MK(215), MK(219), MK(223), - MK(227), MK(231), MK(235), MK(239), MK(243), MK(247), MK(251), MK(127), - MK(129), MK(131), MK(133), MK(135), MK(137), MK(139), MK(141), MK(143), - MK(145), MK(147), MK(149), MK(151), MK(153), MK(155), MK(157), MK(159), - MK(161), MK(163), MK(165), MK(167), MK(169), MK(171), MK(173), MK(175), - MK(177), MK(179), MK(181), MK(183), MK(185), MK(187), MK(189), MK(191), - MK(193), MK(195), MK(197), MK(199), MK(201), MK(203), MK(205), MK(207), - MK(209), MK(211), MK(213), MK(215), MK(217), MK(219), MK(221), MK(223), - MK(225), MK(227), MK(229), MK(231), MK(233), MK(235), MK(237), MK(239), - MK(241), MK(243), MK(245), MK(247), MK(249), MK(251), MK(253), MK(127) + 127, 127, 191, 127, 159, 191, 223, 127, + 143, 159, 175, 191, 207, 223, 239, 127, + 135, 143, 151, 159, 167, 175, 183, 191, + 199, 207, 215, 223, 231, 239, 247, 127, + 131, 135, 139, 143, 147, 151, 155, 159, + 163, 167, 171, 175, 179, 183, 187, 191, + 195, 199, 203, 207, 211, 215, 219, 223, + 227, 231, 235, 239, 243, 247, 251, 127, + 129, 131, 133, 135, 137, 139, 141, 143, + 145, 147, 149, 151, 153, 155, 157, 159, + 161, 163, 165, 167, 169, 171, 173, 175, + 177, 179, 181, 183, 185, 187, 189, 191, + 193, 195, 197, 199, 201, 203, 205, 207, + 209, 211, 213, 215, 217, 219, 221, 223, + 225, 227, 229, 231, 233, 235, 237, 239, + 241, 243, 245, 247, 249, 251, 253, 127 }; -#undef MK - void VP8LoadFinalBytes(VP8BitReader* const br) { assert(br != NULL && br->buf_ != NULL); // Only read 8bits at a time if (br->buf_ < br->buf_end_) { -#ifndef USE_RIGHT_JUSTIFY - br->value_ |= (bit_t)(*br->buf_++) << ((BITS) - 8 - br->bits_); -#else - br->value_ = (bit_t)(*br->buf_++) | (br->value_ << 8); -#endif br->bits_ += 8; + br->value_ = (bit_t)(*br->buf_++) | (br->value_ << 8); } else if (!br->eof_) { -#ifdef USE_RIGHT_JUSTIFY - // These are not strictly needed, but it makes the behaviour - // consistent for both USE_RIGHT_JUSTIFY and !USE_RIGHT_JUSTIFY. br->value_ <<= 8; br->bits_ += 8; -#endif br->eof_ = 1; } } @@ -222,4 +207,3 @@ uint32_t VP8LReadBits(VP8LBitReader* const br, int n_bits) { } //------------------------------------------------------------------------------ - diff --git a/src/utils/bit_reader.h b/src/utils/bit_reader.h index f03c3ba8..1fb2bc9f 100644 --- a/src/utils/bit_reader.h +++ b/src/utils/bit_reader.h @@ -29,112 +29,52 @@ extern "C" { // However, since range_ is only 8bit, we only need an active window of 8 bits // for value_. Left bits (MSB) gets zeroed and shifted away when value_ falls // below 128, range_ is updated, and fresh bits read from the bitstream are -// brought in as LSB. -// To avoid reading the fresh bits one by one (slow), we cache a few of them -// ahead (actually, we cache BITS of them ahead. See below). There's two -// strategies regarding how to shift these looked-ahead fresh bits into the -// 8bit window of value_: either we shift them in, while keeping the position of -// the window fixed. Or we slide the window to the right while keeping the cache -// bits at a fixed, right-justified, position. +// brought in as LSB. To avoid reading the fresh bits one by one (slow), we +// cache BITS of them ahead. The total of (BITS + 8) bits must fit into a +// natural register (with type bit_t). To fetch BITS bits from bitstream we +// use a type lbit_t. // -// Example, for BITS=16: here is the content of value_ for both strategies: -// -// !USE_RIGHT_JUSTIFY || USE_RIGHT_JUSTIFY -// || -// <- 8b -><- 8b -><- BITS bits -> || <- 8b+3b -><- 8b -><- 13 bits -> -// [unused][value_][cached bits][0] || [unused...][value_][cached bits] -// [........00vvvvvvBBBBBBBBBBBBB000]LSB || [...........00vvvvvvBBBBBBBBBBBBB] -// || -// After calling VP8Shift(), where we need to shift away two zeros: -// [........vvvvvvvvBBBBBBBBBBB00000]LSB || [.............vvvvvvvvBBBBBBBBBBB] -// || -// Just before we need to call VP8LoadNewBytes(), the situation is: -// [........vvvvvv000000000000000000]LSB || [..........................vvvvvv] -// || -// And just after calling VP8LoadNewBytes(): -// [........vvvvvvvvBBBBBBBBBBBBBBBB]LSB || [........vvvvvvvvBBBBBBBBBBBBBBBB] -// -// -> we're back to eight active 'value_' bits (marked 'v') and BITS cached -// bits (marked 'B') -// -// The right-justify strategy tends to use less shifts and is often faster. - -//------------------------------------------------------------------------------ // BITS can be any multiple of 8 from 8 to 56 (inclusive). // Pick values that fit natural register size. -#if !defined(WEBP_REFERENCE_IMPLEMENTATION) - -#define USE_RIGHT_JUSTIFY - #if defined(__i386__) || defined(_M_IX86) // x86 32bit -#define BITS 16 +#define BITS 24 #elif defined(__x86_64__) || defined(_M_X64) // x86 64bit #define BITS 56 #elif defined(__arm__) || defined(_M_ARM) // ARM #define BITS 24 #elif defined(__mips__) // MIPS #define BITS 24 -#else // reasonable default -#define BITS 24 -#endif - -#else // reference choices - -#define USE_RIGHT_JUSTIFY -#define BITS 8 - -#endif - -// some endian fix (e.g.: mips-gcc doesn't define __BIG_ENDIAN__) -#if !defined(__BIG_ENDIAN__) && defined(__BYTE_ORDER__) && \ - (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -#define __BIG_ENDIAN__ +#else // reasonable default +#define BITS 24 // TODO(skal): test aarch64 and find the proper BITS value. #endif //------------------------------------------------------------------------------ -// Derived types and constants +// Derived types and constants: +// bit_t = natural register type for storing 'value_' (which is BITS+8 bits) +// range_t = register for 'range_' (which is 8bits only) -// bit_t = natural register type -// lbit_t = natural type for memory I/O - -#if (BITS > 32) +#if (BITS > 24) typedef uint64_t bit_t; -typedef uint64_t lbit_t; -#elif (BITS == 32) -typedef uint64_t bit_t; -typedef uint32_t lbit_t; -#elif (BITS == 24) -typedef uint32_t bit_t; -typedef uint32_t lbit_t; -#elif (BITS == 16) -typedef uint32_t bit_t; -typedef uint16_t lbit_t; #else typedef uint32_t bit_t; -typedef uint8_t lbit_t; #endif -#ifndef USE_RIGHT_JUSTIFY -typedef bit_t range_t; // type for storing range_ -#define MASK ((((bit_t)1) << (BITS)) - 1) -#else -typedef uint32_t range_t; // range_ only uses 8bits here. No need for bit_t. -#endif +typedef uint32_t range_t; //------------------------------------------------------------------------------ // Bitreader typedef struct VP8BitReader VP8BitReader; struct VP8BitReader { + // boolean decoder (keep the field ordering as is!) + range_t range_; // current range minus 1. In [127, 254] interval. + bit_t value_; // current value + int bits_; // number of valid bits left + // read buffer const uint8_t* buf_; // next byte to be read const uint8_t* buf_end_; // end of read buffer int eof_; // true if input is exhausted - - // boolean decoder - range_t range_; // current range minus 1. In [127, 254] interval. - bit_t value_; // current value - int bits_; // number of valid bits left }; // Initialize the bit reader and the boolean decoder. @@ -154,164 +94,12 @@ static WEBP_INLINE uint32_t VP8Get(VP8BitReader* const br) { // return the next value with sign-extension. int32_t VP8GetSignedValue(VP8BitReader* const br, int num_bits); -// Read a bit with proba 'prob'. Speed-critical function! -extern const uint8_t kVP8Log2Range[128]; -extern const range_t kVP8NewRange[128]; - -void VP8LoadFinalBytes(VP8BitReader* const br); // special case for the tail - -static WEBP_INLINE void VP8LoadNewBytes(VP8BitReader* const br) { - assert(br != NULL && br->buf_ != NULL); - // Read 'BITS' bits at a time if possible. - if (br->buf_ + sizeof(lbit_t) <= br->buf_end_) { - // convert memory type to register type (with some zero'ing!) - bit_t bits; -#if defined(__mips__) // MIPS - // This is needed because of un-aligned read. - lbit_t in_bits; - lbit_t* p_buf_ = (lbit_t*)br->buf_; - __asm__ volatile( - ".set push \n\t" - ".set at \n\t" - ".set macro \n\t" - "ulw %[in_bits], 0(%[p_buf_]) \n\t" - ".set pop \n\t" - : [in_bits]"=r"(in_bits) - : [p_buf_]"r"(p_buf_) - : "memory", "at" - ); -#else - const lbit_t in_bits = *(const lbit_t*)br->buf_; -#endif - br->buf_ += (BITS) >> 3; -#if !defined(__BIG_ENDIAN__) -#if (BITS > 32) -// gcc 4.3 has builtin functions for swap32/swap64 -#if defined(__GNUC__) && \ - (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) - bits = (bit_t)__builtin_bswap64(in_bits); -#elif defined(_MSC_VER) - bits = (bit_t)_byteswap_uint64(in_bits); -#elif defined(__x86_64__) - __asm__ volatile("bswapq %0" : "=r"(bits) : "0"(in_bits)); -#else // generic code for swapping 64-bit values (suggested by bdb@) - bits = (bit_t)in_bits; - bits = ((bits & 0xffffffff00000000ull) >> 32) | - ((bits & 0x00000000ffffffffull) << 32); - bits = ((bits & 0xffff0000ffff0000ull) >> 16) | - ((bits & 0x0000ffff0000ffffull) << 16); - bits = ((bits & 0xff00ff00ff00ff00ull) >> 8) | - ((bits & 0x00ff00ff00ff00ffull) << 8); -#endif - bits >>= 64 - BITS; -#elif (BITS >= 24) -#if defined(__i386__) || defined(__x86_64__) - { - lbit_t swapped_in_bits; - __asm__ volatile("bswap %k0" : "=r"(swapped_in_bits) : "0"(in_bits)); - bits = (bit_t)swapped_in_bits; // 24b/32b -> 32b/64b zero-extension - } -#elif defined(_MSC_VER) - bits = (bit_t)_byteswap_ulong(in_bits); -#else - bits = (bit_t)(in_bits >> 24) | ((in_bits >> 8) & 0xff00) - | ((in_bits << 8) & 0xff0000) | (in_bits << 24); -#endif // x86 - bits >>= (32 - BITS); -#elif (BITS == 16) - // gcc will recognize a 'rorw $8, ...' here: - bits = (bit_t)(in_bits >> 8) | ((in_bits & 0xff) << 8); -#else // BITS == 8 - bits = (bit_t)in_bits; -#endif -#else // BIG_ENDIAN - bits = (bit_t)in_bits; - if (BITS != 8 * sizeof(bit_t)) bits >>= (8 * sizeof(bit_t) - BITS); -#endif -#ifndef USE_RIGHT_JUSTIFY - br->value_ |= bits << (-br->bits_); -#else - br->value_ = bits | (br->value_ << (BITS)); -#endif - br->bits_ += (BITS); - } else { - VP8LoadFinalBytes(br); // no need to be inlined - } -} - -static WEBP_INLINE int VP8BitUpdate(VP8BitReader* const br, range_t split) { - if (br->bits_ < 0) { // Make sure we have a least BITS bits in 'value_' - VP8LoadNewBytes(br); - } -#ifndef USE_RIGHT_JUSTIFY - split |= (MASK); - if (br->value_ > split) { - br->range_ -= split + 1; - br->value_ -= split + 1; - return 1; - } else { - br->range_ = split; - return 0; - } -#else - { - const int pos = br->bits_; - const range_t value = (range_t)(br->value_ >> pos); - if (value > split) { - br->range_ -= split + 1; - br->value_ -= (bit_t)(split + 1) << pos; - return 1; - } else { - br->range_ = split; - return 0; - } - } -#endif -} - -static WEBP_INLINE void VP8Shift(VP8BitReader* const br) { -#ifndef USE_RIGHT_JUSTIFY - // range_ is in [0..127] interval here. - const bit_t idx = br->range_ >> (BITS); - const int shift = kVP8Log2Range[idx]; - br->range_ = kVP8NewRange[idx]; - br->value_ <<= shift; - br->bits_ -= shift; -#else - const int shift = kVP8Log2Range[br->range_]; - assert(br->range_ < (range_t)128); - br->range_ = kVP8NewRange[br->range_]; - br->bits_ -= shift; -#endif -} - -static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, int prob) { -#ifndef USE_RIGHT_JUSTIFY - // It's important to avoid generating a 64bit x 64bit multiply here. - // We just need an 8b x 8b after all. - const range_t split = - (range_t)((uint32_t)(br->range_ >> (BITS)) * prob) << ((BITS) - 8); - const int bit = VP8BitUpdate(br, split); - if (br->range_ <= (((range_t)0x7e << (BITS)) | (MASK))) { - VP8Shift(br); - } - return bit; -#else - const range_t split = (br->range_ * prob) >> 8; - const int bit = VP8BitUpdate(br, split); - if (br->range_ <= (range_t)0x7e) { - VP8Shift(br); - } - return bit; -#endif -} - -static WEBP_INLINE int VP8GetSigned(VP8BitReader* const br, int v) { - const range_t split = (br->range_ >> 1); - const int bit = VP8BitUpdate(br, split); - VP8Shift(br); - return bit ? -v : v; -} +// bit_reader_inl.h will implement the following methods: +// static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, int prob) +// static WEBP_INLINE int VP8GetSigned(VP8BitReader* const br, int v) +// and should be included by the .c files that actually need them. +// This is to avoid recompiling the whole library whenever this file is touched, +// and also allowing platform-specific ad-hoc hacks. // ----------------------------------------------------------------------------- // Bitreader for lossless format diff --git a/src/utils/bit_reader_inl.h b/src/utils/bit_reader_inl.h new file mode 100644 index 00000000..647f3c90 --- /dev/null +++ b/src/utils/bit_reader_inl.h @@ -0,0 +1,193 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Specific inlined methods for boolean decoder [VP8GetBit() ...] +// This file should be included by the .c sources that actually need to call +// these methods. +// +// Author: Skal (pascal.massimino@gmail.com) + +#ifndef WEBP_UTILS_BIT_READER_INL_H_ +#define WEBP_UTILS_BIT_READER_INL_H_ + +#include "./bit_reader.h" + +#ifdef __cplusplus +extern "C" { +#endif + +//------------------------------------------------------------------------------ +// Derived type lbit_t = natural type for memory I/O + +#if (BITS > 32) +typedef uint64_t lbit_t; +#elif (BITS > 16) +typedef uint32_t lbit_t; +#elif (BITS > 8) +typedef uint16_t lbit_t; +#else +typedef uint8_t lbit_t; +#endif + +// some endian fix (e.g.: mips-gcc doesn't define __BIG_ENDIAN__) +#if !defined(__BIG_ENDIAN__) && defined(__BYTE_ORDER__) && \ + (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#define __BIG_ENDIAN__ +#endif + +extern const uint8_t kVP8Log2Range[128]; +extern const range_t kVP8NewRange[128]; + +// special case for the tail byte-reading +void VP8LoadFinalBytes(VP8BitReader* const br); + +//------------------------------------------------------------------------------ +// Inlined critical functions + +// makes sure br->value_ has at least BITS bits worth of data +static WEBP_INLINE void VP8LoadNewBytes(VP8BitReader* const br) { + assert(br != NULL && br->buf_ != NULL); + // Read 'BITS' bits at a time if possible. + if (br->buf_ + sizeof(lbit_t) <= br->buf_end_) { + // convert memory type to register type (with some zero'ing!) + bit_t bits; +#if defined(__mips__) // MIPS + // This is needed because of un-aligned read. + lbit_t in_bits; + lbit_t* p_buf_ = (lbit_t*)br->buf_; + __asm__ volatile( + ".set push \n\t" + ".set at \n\t" + ".set macro \n\t" + "ulw %[in_bits], 0(%[p_buf_]) \n\t" + ".set pop \n\t" + : [in_bits]"=r"(in_bits) + : [p_buf_]"r"(p_buf_) + : "memory", "at" + ); +#else + const lbit_t in_bits = *(const lbit_t*)br->buf_; +#endif + br->buf_ += BITS >> 3; +#if !defined(__BIG_ENDIAN__) +#if (BITS > 32) +// gcc 4.3 has builtin functions for swap32/swap64 +#if defined(__GNUC__) && \ + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + bits = (bit_t)__builtin_bswap64(in_bits); +#elif defined(_MSC_VER) + bits = (bit_t)_byteswap_uint64(in_bits); +#elif defined(__x86_64__) + __asm__ volatile("bswapq %0" : "=r"(bits) : "0"(in_bits)); +#else // generic code for swapping 64-bit values (suggested by bdb@) + bits = (bit_t)in_bits; + bits = ((bits & 0xffffffff00000000ull) >> 32) | + ((bits & 0x00000000ffffffffull) << 32); + bits = ((bits & 0xffff0000ffff0000ull) >> 16) | + ((bits & 0x0000ffff0000ffffull) << 16); + bits = ((bits & 0xff00ff00ff00ff00ull) >> 8) | + ((bits & 0x00ff00ff00ff00ffull) << 8); +#endif + bits >>= 64 - BITS; +#elif (BITS >= 24) +#if defined(__i386__) || defined(__x86_64__) + { + lbit_t swapped_in_bits; + __asm__ volatile("bswap %k0" : "=r"(swapped_in_bits) : "0"(in_bits)); + bits = (bit_t)swapped_in_bits; // 24b/32b -> 32b/64b zero-extension + } +#elif defined(_MSC_VER) + bits = (bit_t)_byteswap_ulong(in_bits); +#else + bits = (bit_t)(in_bits >> 24) | ((in_bits >> 8) & 0xff00) + | ((in_bits << 8) & 0xff0000) | (in_bits << 24); +#endif // x86 + bits >>= (32 - BITS); +#elif (BITS == 16) + // gcc will recognize a 'rorw $8, ...' here: + bits = (bit_t)(in_bits >> 8) | ((in_bits & 0xff) << 8); +#else // BITS == 8 + bits = (bit_t)in_bits; +#endif +#else // BIG_ENDIAN + bits = (bit_t)in_bits; + if (BITS != 8 * sizeof(bit_t)) bits >>= (8 * sizeof(bit_t) - BITS); +#endif + br->value_ = bits | (br->value_ << BITS); + br->bits_ += BITS; + } else { + VP8LoadFinalBytes(br); // no need to be inlined + } +} + +// Read a bit with proba 'prob'. Speed-critical function! +static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, int prob) { + // Don't move this declaration! It makes a big speed difference to store + // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't + // alter br->range_ value. + range_t range = br->range_; + if (br->bits_ < 0) { + VP8LoadNewBytes(br); + } + { + const int pos = br->bits_; + const range_t split = (range * prob) >> 8; + const range_t value = (range_t)(br->value_ >> pos); +#if defined(__arm__) || defined(_M_ARM) // ARM-specific + const int bit = ((int)(split - value) >> 31) & 1; + if (value > split) { + range -= split + 1; + br->value_ -= (bit_t)(split + 1) << pos; + } else { + range = split; + } +#else // faster version on x86 + int bit; // Don't use 'const int bit = (value > split);", it's slower. + if (value > split) { + range -= split + 1; + br->value_ -= (bit_t)(split + 1) << pos; + bit = 1; + } else { + range = split; + bit = 0; + } +#endif + if (range <= (range_t)0x7e) { + const int shift = kVP8Log2Range[range]; + range = kVP8NewRange[range]; + br->bits_ -= shift; + } + br->range_ = range; + return bit; + } +} + +// simplified version of VP8GetBit() for prob=0x80 (note shift is always 1 here) +static WEBP_INLINE int VP8GetSigned(VP8BitReader* const br, int v) { + if (br->bits_ < 0) { + VP8LoadNewBytes(br); + } + { + const int pos = br->bits_; + const range_t split = br->range_ >> 1; + const range_t value = (range_t)(br->value_ >> pos); + const int32_t mask = (int32_t)(split - value) >> 31; // -1 or 0 + br->bits_ -= 1; + br->range_ += mask; + br->range_ |= 1; + br->value_ -= (bit_t)((split + 1) & mask) << pos; + return (v ^ mask) - mask; + } +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_UTILS_BIT_READER_INL_H_