diff --git a/src/dsp/lossless.h b/src/dsp/lossless.h index b2bbdfc9..f709cc86 100644 --- a/src/dsp/lossless.h +++ b/src/dsp/lossless.h @@ -163,7 +163,7 @@ extern VP8LCostCombinedFunc VP8LExtraCostCombined; extern VP8LCombinedShannonEntropyFunc VP8LCombinedShannonEntropy; typedef struct { // small struct to hold counters - int counts[2]; // index: 0=zero steak, 1=non-zero streak + int counts[2]; // index: 0=zero streak, 1=non-zero streak int streaks[2][2]; // [zero/non-zero][streak<3 / streak>=3] } VP8LStreaks; @@ -194,10 +194,14 @@ extern VP8LGetEntropyUnrefinedFunc VP8LGetEntropyUnrefined; void VP8LBitsEntropyUnrefined(const uint32_t* const array, int n, VP8LBitEntropy* const entropy); -typedef void (*VP8LHistogramAddFunc)(const VP8LHistogram* const a, - const VP8LHistogram* const b, - VP8LHistogram* const out); -extern VP8LHistogramAddFunc VP8LHistogramAdd; +typedef void (*VP8LAddVectorFunc)(const uint32_t* a, const uint32_t* b, + uint32_t* out, int size); +extern VP8LAddVectorFunc VP8LAddVector; +typedef void (*VP8LAddVectorEqFunc)(const uint32_t* a, uint32_t* out, int size); +extern VP8LAddVectorEqFunc VP8LAddVectorEq; +void VP8LHistogramAdd(const VP8LHistogram* const a, + const VP8LHistogram* const b, + VP8LHistogram* const out); // ----------------------------------------------------------------------------- // PrefixEncode() diff --git a/src/dsp/lossless_enc.c b/src/dsp/lossless_enc.c index d608326f..6ec9e46a 100644 --- a/src/dsp/lossless_enc.c +++ b/src/dsp/lossless_enc.c @@ -632,36 +632,34 @@ static double ExtraCostCombined_C(const uint32_t* X, const uint32_t* Y, //------------------------------------------------------------------------------ -static void HistogramAdd_C(const VP8LHistogram* const a, - const VP8LHistogram* const b, - VP8LHistogram* const out) { +static void AddVector_C(const uint32_t* a, const uint32_t* b, uint32_t* out, + int size) { int i; + for (i = 0; i < size; ++i) out[i] = a[i] + b[i]; +} + +static void AddVectorEq_C(const uint32_t* a, uint32_t* out, int size) { + int i; + for (i = 0; i < size; ++i) out[i] += a[i]; +} + +void VP8LHistogramAdd(const VP8LHistogram* const a, + const VP8LHistogram* const b, VP8LHistogram* const out) { const int literal_size = VP8LHistogramNumCodes(a->palette_code_bits_); assert(a->palette_code_bits_ == b->palette_code_bits_); if (b != out) { - for (i = 0; i < literal_size; ++i) { - out->literal_[i] = a->literal_[i] + b->literal_[i]; - } - for (i = 0; i < NUM_DISTANCE_CODES; ++i) { - out->distance_[i] = a->distance_[i] + b->distance_[i]; - } - for (i = 0; i < NUM_LITERAL_CODES; ++i) { - out->red_[i] = a->red_[i] + b->red_[i]; - out->blue_[i] = a->blue_[i] + b->blue_[i]; - out->alpha_[i] = a->alpha_[i] + b->alpha_[i]; - } + VP8LAddVector(a->literal_, b->literal_, out->literal_, literal_size); + VP8LAddVector(a->distance_, b->distance_, out->distance_, + NUM_DISTANCE_CODES); + VP8LAddVector(a->red_, b->red_, out->red_, NUM_LITERAL_CODES); + VP8LAddVector(a->blue_, b->blue_, out->blue_, NUM_LITERAL_CODES); + VP8LAddVector(a->alpha_, b->alpha_, out->alpha_, NUM_LITERAL_CODES); } else { - for (i = 0; i < literal_size; ++i) { - out->literal_[i] += a->literal_[i]; - } - for (i = 0; i < NUM_DISTANCE_CODES; ++i) { - out->distance_[i] += a->distance_[i]; - } - for (i = 0; i < NUM_LITERAL_CODES; ++i) { - out->red_[i] += a->red_[i]; - out->blue_[i] += a->blue_[i]; - out->alpha_[i] += a->alpha_[i]; - } + VP8LAddVectorEq(a->literal_, out->literal_, literal_size); + VP8LAddVectorEq(a->distance_, out->distance_, NUM_DISTANCE_CODES); + VP8LAddVectorEq(a->red_, out->red_, NUM_LITERAL_CODES); + VP8LAddVectorEq(a->blue_, out->blue_, NUM_LITERAL_CODES); + VP8LAddVectorEq(a->alpha_, out->alpha_, NUM_LITERAL_CODES); } } @@ -848,7 +846,8 @@ VP8LCombinedShannonEntropyFunc VP8LCombinedShannonEntropy; VP8LGetEntropyUnrefinedFunc VP8LGetEntropyUnrefined; VP8LGetCombinedEntropyUnrefinedFunc VP8LGetCombinedEntropyUnrefined; -VP8LHistogramAddFunc VP8LHistogramAdd; +VP8LAddVectorFunc VP8LAddVector; +VP8LAddVectorEqFunc VP8LAddVectorEq; VP8LVectorMismatchFunc VP8LVectorMismatch; VP8LBundleColorMapFunc VP8LBundleColorMap; @@ -885,7 +884,8 @@ WEBP_DSP_INIT_FUNC(VP8LEncDspInit) { VP8LGetEntropyUnrefined = GetEntropyUnrefined_C; VP8LGetCombinedEntropyUnrefined = GetCombinedEntropyUnrefined_C; - VP8LHistogramAdd = HistogramAdd_C; + VP8LAddVector = AddVector_C; + VP8LAddVectorEq = AddVectorEq_C; VP8LVectorMismatch = VectorMismatch_C; VP8LBundleColorMap = VP8LBundleColorMap_C; @@ -971,7 +971,8 @@ WEBP_DSP_INIT_FUNC(VP8LEncDspInit) { assert(VP8LCombinedShannonEntropy != NULL); assert(VP8LGetEntropyUnrefined != NULL); assert(VP8LGetCombinedEntropyUnrefined != NULL); - assert(VP8LHistogramAdd != NULL); + assert(VP8LAddVector != NULL); + assert(VP8LAddVectorEq != NULL); assert(VP8LVectorMismatch != NULL); assert(VP8LBundleColorMap != NULL); assert(VP8LPredictorsSub[0] != NULL); diff --git a/src/dsp/lossless_enc_mips32.c b/src/dsp/lossless_enc_mips32.c index e7b58f4e..0412a093 100644 --- a/src/dsp/lossless_enc_mips32.c +++ b/src/dsp/lossless_enc_mips32.c @@ -344,65 +344,29 @@ static void GetCombinedEntropyUnrefined_MIPS32(const uint32_t X[], ASM_END_COMMON_0 \ ASM_END_COMMON_1 -#define ADD_VECTOR(A, B, OUT, SIZE, EXTRA_SIZE) do { \ - const uint32_t* pa = (const uint32_t*)(A); \ - const uint32_t* pb = (const uint32_t*)(B); \ - uint32_t* pout = (uint32_t*)(OUT); \ - const uint32_t* const LoopEnd = pa + (SIZE); \ - assert((SIZE) % 4 == 0); \ - ASM_START \ - ADD_TO_OUT(0, 4, 8, 12, 1, pa, pb, pout) \ - ASM_END_0 \ - if ((EXTRA_SIZE) > 0) { \ - const int last = (EXTRA_SIZE); \ - int i; \ - for (i = 0; i < last; ++i) pout[i] = pa[i] + pb[i]; \ - } \ -} while (0) - -#define ADD_VECTOR_EQ(A, OUT, SIZE, EXTRA_SIZE) do { \ - const uint32_t* pa = (const uint32_t*)(A); \ - uint32_t* pout = (uint32_t*)(OUT); \ - const uint32_t* const LoopEnd = pa + (SIZE); \ - assert((SIZE) % 4 == 0); \ - ASM_START \ - ADD_TO_OUT(0, 4, 8, 12, 0, pa, pout, pout) \ - ASM_END_1 \ - if ((EXTRA_SIZE) > 0) { \ - const int last = (EXTRA_SIZE); \ - int i; \ - for (i = 0; i < last; ++i) pout[i] += pa[i]; \ - } \ -} while (0) - -static void HistogramAdd_MIPS32(const VP8LHistogram* const a, - const VP8LHistogram* const b, - VP8LHistogram* const out) { +static void AddVector_MIPS32(const uint32_t* pa, const uint32_t* pb, + uint32_t* pout, int size) { uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; - const int extra_cache_size = VP8LHistogramNumCodes(a->palette_code_bits_) - - (NUM_LITERAL_CODES + NUM_LENGTH_CODES); - assert(a->palette_code_bits_ == b->palette_code_bits_); - - if (b != out) { - ADD_VECTOR(a->literal_, b->literal_, out->literal_, - NUM_LITERAL_CODES + NUM_LENGTH_CODES, extra_cache_size); - ADD_VECTOR(a->distance_, b->distance_, out->distance_, - NUM_DISTANCE_CODES, 0); - ADD_VECTOR(a->red_, b->red_, out->red_, NUM_LITERAL_CODES, 0); - ADD_VECTOR(a->blue_, b->blue_, out->blue_, NUM_LITERAL_CODES, 0); - ADD_VECTOR(a->alpha_, b->alpha_, out->alpha_, NUM_LITERAL_CODES, 0); - } else { - ADD_VECTOR_EQ(a->literal_, out->literal_, - NUM_LITERAL_CODES + NUM_LENGTH_CODES, extra_cache_size); - ADD_VECTOR_EQ(a->distance_, out->distance_, NUM_DISTANCE_CODES, 0); - ADD_VECTOR_EQ(a->red_, out->red_, NUM_LITERAL_CODES, 0); - ADD_VECTOR_EQ(a->blue_, out->blue_, NUM_LITERAL_CODES, 0); - ADD_VECTOR_EQ(a->alpha_, out->alpha_, NUM_LITERAL_CODES, 0); - } + const uint32_t end = ((size) / 4) * 4; + const uint32_t* const LoopEnd = pa + end; + int i; + ASM_START + ADD_TO_OUT(0, 4, 8, 12, 1, pa, pb, pout) + ASM_END_0 + for (i = end; i < size; ++i) pout[i] = pa[i] + pb[i]; +} + +static void AddVectorEq_MIPS32(const uint32_t* pa, uint32_t* pout, int size) { + uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; + const uint32_t end = ((size) / 4) * 4; + const uint32_t* const LoopEnd = pa + end; + int i; + ASM_START + ADD_TO_OUT(0, 4, 8, 12, 0, pa, pout, pout) + ASM_END_1 + for (i = end; i < size; ++i) pout[i] += pa[i]; } -#undef ADD_VECTOR_EQ -#undef ADD_VECTOR #undef ASM_END_1 #undef ASM_END_0 #undef ASM_END_COMMON_1 @@ -422,7 +386,8 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitMIPS32(void) { VP8LExtraCostCombined = ExtraCostCombined_MIPS32; VP8LGetEntropyUnrefined = GetEntropyUnrefined_MIPS32; VP8LGetCombinedEntropyUnrefined = GetCombinedEntropyUnrefined_MIPS32; - VP8LHistogramAdd = HistogramAdd_MIPS32; + VP8LAddVector = AddVector_MIPS32; + VP8LAddVectorEq = AddVectorEq_MIPS32; } #else // !WEBP_USE_MIPS32 diff --git a/src/dsp/lossless_enc_sse2.c b/src/dsp/lossless_enc_sse2.c index f84a9909..36478c49 100644 --- a/src/dsp/lossless_enc_sse2.c +++ b/src/dsp/lossless_enc_sse2.c @@ -170,12 +170,13 @@ static void CollectColorRedTransforms_SSE2(const uint32_t* argb, int stride, //------------------------------------------------------------------------------ +// Note we are adding uint32_t's as *signed* int32's (using _mm_add_epi32). But +// that's ok since the histogram values are less than 1<<28 (max picture size). #define LINE_SIZE 16 // 8 or 16 static void AddVector_SSE2(const uint32_t* a, const uint32_t* b, uint32_t* out, int size) { int i; - assert(size % LINE_SIZE == 0); - for (i = 0; i < size; i += LINE_SIZE) { + for (i = 0; i + LINE_SIZE <= size; i += LINE_SIZE) { const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]); const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); #if (LINE_SIZE == 16) @@ -195,12 +196,14 @@ static void AddVector_SSE2(const uint32_t* a, const uint32_t* b, uint32_t* out, _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3)); #endif } + for (; i < size; ++i) { + out[i] = a[i] + b[i]; + } } static void AddVectorEq_SSE2(const uint32_t* a, uint32_t* out, int size) { int i; - assert(size % LINE_SIZE == 0); - for (i = 0; i < size; i += LINE_SIZE) { + for (i = 0; i + LINE_SIZE <= size; i += LINE_SIZE) { const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]); const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); #if (LINE_SIZE == 16) @@ -220,36 +223,12 @@ static void AddVectorEq_SSE2(const uint32_t* a, uint32_t* out, int size) { _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3)); #endif } + for (; i < size; ++i) { + out[i] += a[i]; + } } #undef LINE_SIZE -// Note we are adding uint32_t's as *signed* int32's (using _mm_add_epi32). But -// that's ok since the histogram values are less than 1<<28 (max picture size). -static void HistogramAdd_SSE2(const VP8LHistogram* const a, - const VP8LHistogram* const b, - VP8LHistogram* const out) { - int i; - const int literal_size = VP8LHistogramNumCodes(a->palette_code_bits_); - assert(a->palette_code_bits_ == b->palette_code_bits_); - if (b != out) { - AddVector_SSE2(a->literal_, b->literal_, out->literal_, NUM_LITERAL_CODES); - AddVector_SSE2(a->red_, b->red_, out->red_, NUM_LITERAL_CODES); - AddVector_SSE2(a->blue_, b->blue_, out->blue_, NUM_LITERAL_CODES); - AddVector_SSE2(a->alpha_, b->alpha_, out->alpha_, NUM_LITERAL_CODES); - } else { - AddVectorEq_SSE2(a->literal_, out->literal_, NUM_LITERAL_CODES); - AddVectorEq_SSE2(a->red_, out->red_, NUM_LITERAL_CODES); - AddVectorEq_SSE2(a->blue_, out->blue_, NUM_LITERAL_CODES); - AddVectorEq_SSE2(a->alpha_, out->alpha_, NUM_LITERAL_CODES); - } - for (i = NUM_LITERAL_CODES; i < literal_size; ++i) { - out->literal_[i] = a->literal_[i] + b->literal_[i]; - } - for (i = 0; i < NUM_DISTANCE_CODES; ++i) { - out->distance_[i] = a->distance_[i] + b->distance_[i]; - } -} - //------------------------------------------------------------------------------ // Entropy @@ -675,7 +654,8 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) { VP8LTransformColor = TransformColor_SSE2; VP8LCollectColorBlueTransforms = CollectColorBlueTransforms_SSE2; VP8LCollectColorRedTransforms = CollectColorRedTransforms_SSE2; - VP8LHistogramAdd = HistogramAdd_SSE2; + VP8LAddVector = AddVector_SSE2; + VP8LAddVectorEq = AddVectorEq_SSE2; VP8LCombinedShannonEntropy = CombinedShannonEntropy_SSE2; VP8LVectorMismatch = VectorMismatch_SSE2; VP8LBundleColorMap = BundleColorMap_SSE2;