From 2102ccd091bfac3b0eec4f4a1f7e4cdb0731755d Mon Sep 17 00:00:00 2001 From: Pascal Massimino Date: Thu, 14 Apr 2016 19:12:09 +0200 Subject: [PATCH] update the Unfilter API in dsp to process one row independently This will allow to work in-place on cropped area later. Also sped up the inverse gradient filtering in SSE2 (~4%) Change-Id: I463149eee95d36984328f163a1e17f8cabd87441 --- src/dec/alpha.c | 31 +++- src/dec/alphai.h | 1 + src/dec/vp8i.h | 1 + src/dec/vp8l.c | 55 +++--- src/dsp/dsp.h | 6 +- src/dsp/filters.c | 39 ++++- src/dsp/filters_mips_dsp_r2.c | 162 +++++++----------- src/dsp/filters_sse2.c | 310 ++++++++++++++++------------------ 8 files changed, 297 insertions(+), 308 deletions(-) diff --git a/src/dec/alpha.c b/src/dec/alpha.c index f31283c6..028eb3d5 100644 --- a/src/dec/alpha.c +++ b/src/dec/alpha.c @@ -111,17 +111,29 @@ static int ALPHDecode(VP8Decoder* const dec, int row, int num_rows) { ALPHDecoder* const alph_dec = dec->alph_dec_; const int width = alph_dec->width_; const int height = alph_dec->io_.crop_bottom; - uint8_t* const output = dec->alpha_plane_; if (alph_dec->method_ == ALPHA_NO_COMPRESSION) { - const size_t offset = row * width; - const size_t num_pixels = num_rows * width; - assert(dec->alpha_data_size_ >= ALPHA_HEADER_LEN + offset + num_pixels); - memcpy(dec->alpha_plane_ + offset, - dec->alpha_data_ + ALPHA_HEADER_LEN + offset, num_pixels); - if (WebPUnfilters[alph_dec->filter_] != NULL) { - WebPUnfilters[alph_dec->filter_](width, height, width, - row, num_rows, output); + int y; + const uint8_t* prev_line = dec->alpha_prev_line_; + const uint8_t* deltas = dec->alpha_data_ + ALPHA_HEADER_LEN + row * width; + uint8_t* dst = dec->alpha_plane_ + row * width; + assert(deltas <= &dec->alpha_data_[dec->alpha_data_size_]); + if (alph_dec->filter_ != WEBP_FILTER_NONE) { + assert(WebPUnfilters[alph_dec->filter_] != NULL); + for (y = 0; y < num_rows; ++y) { + WebPUnfilters[alph_dec->filter_](prev_line, deltas, dst, width); + prev_line = dst; + dst += width; + deltas += width; + } + } else { + for (y = 0; y < num_rows; ++y) { + memcpy(dst, deltas, width * sizeof(*dst)); + prev_line = dst; + dst += width; + deltas += width; + } } + dec->alpha_prev_line_ = prev_line; } else { // alph_dec->method_ == ALPHA_LOSSLESS_COMPRESSION assert(alph_dec->vp8l_dec_ != NULL); if (!VP8LDecodeAlphaImageStream(alph_dec, row + num_rows)) { @@ -146,6 +158,7 @@ static int AllocateAlphaPlane(VP8Decoder* const dec, const VP8Io* const io) { return 0; } dec->alpha_plane_ = dec->alpha_plane_mem_; + dec->alpha_prev_line_ = NULL; return 1; } diff --git a/src/dec/alphai.h b/src/dec/alphai.h index 1e817c81..69dd7c0f 100644 --- a/src/dec/alphai.h +++ b/src/dec/alphai.h @@ -36,6 +36,7 @@ struct ALPHDecoder { // pixel, sometimes VP8LDecoder may need to allocate // 4 bytes per pixel internally during decode. uint8_t* output_; + const uint8_t* prev_line_; // last output row (or NULL) }; //------------------------------------------------------------------------------ diff --git a/src/dec/vp8i.h b/src/dec/vp8i.h index ef14f4ec..fed2b279 100644 --- a/src/dec/vp8i.h +++ b/src/dec/vp8i.h @@ -261,6 +261,7 @@ struct VP8Decoder { int is_alpha_decoded_; // true if alpha_data_ is decoded in alpha_plane_ uint8_t* alpha_plane_mem_; // memory allocated for alpha_plane_ uint8_t* alpha_plane_; // output. Persistent, contains the whole data. + const uint8_t* alpha_prev_line_; // last decoded alpha row (or NULL) int alpha_dithering_; // derived from decoding options (0=off, 100=full) }; diff --git a/src/dec/vp8l.c b/src/dec/vp8l.c index 7736184d..164d2d03 100644 --- a/src/dec/vp8l.c +++ b/src/dec/vp8l.c @@ -771,35 +771,46 @@ static int Is8bOptimizable(const VP8LMetadata* const hdr) { return 1; } -static void ExtractPalettedAlphaRows(VP8LDecoder* const dec, int row) { +static void AlphaApplyFilter(ALPHDecoder* const alph_dec, + int first_row, int last_row, + uint8_t* out, int stride) { + if (alph_dec->filter_ != WEBP_FILTER_NONE) { + int y; + const uint8_t* prev_line = alph_dec->prev_line_; + assert(WebPUnfilters[alph_dec->filter_] != NULL); + for (y = first_row; y < last_row; ++y) { + WebPUnfilters[alph_dec->filter_](prev_line, out, out, stride); + prev_line = out; + out += stride; + } + alph_dec->prev_line_ = prev_line; + } +} + +static void ExtractPalettedAlphaRows(VP8LDecoder* const dec, int last_row) { // For vertical and gradient filtering, we need to decode the part above the // crop_top row, in order to have the correct spatial predictors. - const ALPHDecoder* const alph_dec = (const ALPHDecoder*)dec->io_->opaque; + ALPHDecoder* const alph_dec = (ALPHDecoder*)dec->io_->opaque; const int top_row = (alph_dec->filter_ == WEBP_FILTER_NONE || alph_dec->filter_ == WEBP_FILTER_HORIZONTAL) ? dec->io_->crop_top : dec->last_row_; const int first_row = (dec->last_row_ < top_row) ? top_row : dec->last_row_; - assert(row <= dec->io_->crop_bottom); - if (row > first_row) { + assert(last_row <= dec->io_->crop_bottom); + if (last_row > first_row) { // Special method for paletted alpha data. We only process the cropped area. const int width = dec->io_->width; - uint8_t* const out = alph_dec->output_ + width * first_row; + uint8_t* out = alph_dec->output_ + width * first_row; const uint8_t* const in = (uint8_t*)dec->pixels_ + dec->width_ * first_row; VP8LTransform* const transform = &dec->transforms_[0]; assert(dec->next_transform_ == 1); assert(transform->type_ == COLOR_INDEXING_TRANSFORM); - VP8LColorIndexInverseTransformAlpha(transform, first_row, row, + VP8LColorIndexInverseTransformAlpha(transform, first_row, last_row, in, out); - if (alph_dec->filter_ != WEBP_FILTER_NONE) { - assert(WebPUnfilters[alph_dec->filter_] != NULL); - WebPUnfilters[alph_dec->filter_](width, dec->io_->height, width, - first_row, row - first_row, - alph_dec->output_); - } + AlphaApplyFilter(alph_dec, first_row, last_row, out, width); } - dec->last_row_ = dec->last_out_row_ = row; + dec->last_row_ = dec->last_out_row_ = last_row; } //------------------------------------------------------------------------------ @@ -1452,29 +1463,25 @@ static int AllocateInternalBuffers8b(VP8LDecoder* const dec) { //------------------------------------------------------------------------------ // Special row-processing that only stores the alpha data. -static void ExtractAlphaRows(VP8LDecoder* const dec, int row) { - const int num_rows = row - dec->last_row_; +static void ExtractAlphaRows(VP8LDecoder* const dec, int last_row) { + const int num_rows = last_row - dec->last_row_; const uint32_t* const in = dec->pixels_ + dec->width_ * dec->last_row_; - assert(row <= dec->io_->crop_bottom); + assert(last_row <= dec->io_->crop_bottom); if (num_rows > 0) { // Extract alpha (which is stored in the green plane). - const ALPHDecoder* const alph_dec = (const ALPHDecoder*)dec->io_->opaque; + ALPHDecoder* const alph_dec = (ALPHDecoder*)dec->io_->opaque; uint8_t* const output = alph_dec->output_; const int width = dec->io_->width; // the final width (!= dec->width_) const int cache_pixs = width * num_rows; - uint8_t* const dst = output + width * dec->last_row_; + uint8_t* dst = output + width * dec->last_row_; const uint32_t* const src = dec->argb_cache_; int i; ApplyInverseTransforms(dec, num_rows, in); for (i = 0; i < cache_pixs; ++i) dst[i] = (src[i] >> 8) & 0xff; - if (alph_dec->filter_ != WEBP_FILTER_NONE) { - assert(WebPUnfilters[alph_dec->filter_] != NULL); - WebPUnfilters[alph_dec->filter_](width, dec->io_->height, width, - dec->last_row_, num_rows, output); - } + AlphaApplyFilter(alph_dec, dec->last_row_, last_row, dst, width); } - dec->last_row_ = dec->last_out_row_ = row; + dec->last_row_ = dec->last_out_row_ = last_row; } int VP8LDecodeAlphaHeader(ALPHDecoder* const alph_dec, diff --git a/src/dsp/dsp.h b/src/dsp/dsp.h index 10a949e6..10c16ee4 100644 --- a/src/dsp/dsp.h +++ b/src/dsp/dsp.h @@ -503,8 +503,10 @@ typedef enum { // Filter types. typedef void (*WebPFilterFunc)(const uint8_t* in, int width, int height, int stride, uint8_t* out); -typedef void (*WebPUnfilterFunc)(int width, int height, int stride, - int row, int num_rows, uint8_t* data); +// In-place un-filtering. +// Warning! 'prev_line' pointer can be equal to 'cur_line' or 'preds'. +typedef void (*WebPUnfilterFunc)(const uint8_t* prev_line, const uint8_t* preds, + uint8_t* cur_line, int width); // Filter the given data using the given predictor. // 'in' corresponds to a 2-dimensional pixel array of size (stride * height) diff --git a/src/dsp/filters.c b/src/dsp/filters.c index 5c30f2e4..9f04faf0 100644 --- a/src/dsp/filters.c +++ b/src/dsp/filters.c @@ -184,19 +184,40 @@ static void GradientFilter(const uint8_t* data, int width, int height, //------------------------------------------------------------------------------ -static void VerticalUnfilter(int width, int height, int stride, int row, - int num_rows, uint8_t* data) { - DoVerticalFilter(data, width, height, stride, row, num_rows, 1, data); +static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + uint8_t pred = (prev == NULL) ? 0 : prev[0]; + int i; + for (i = 0; i < width; ++i) { + out[i] = pred + in[i]; + pred = out[i]; + } } -static void HorizontalUnfilter(int width, int height, int stride, int row, - int num_rows, uint8_t* data) { - DoHorizontalFilter(data, width, height, stride, row, num_rows, 1, data); +static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + int i; + for (i = 0; i < width; ++i) out[i] = prev[i] + in[i]; + } } -static void GradientUnfilter(int width, int height, int stride, int row, - int num_rows, uint8_t* data) { - DoGradientFilter(data, width, height, stride, row, num_rows, 1, data); +static void GradientUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + uint8_t top = prev[0], top_left = top, left = top; + int i; + for (i = 0; i < width; ++i) { + top = prev[i]; // need to read this first, in case prev==out + left = in[i] + GradientPredictor(left, top, top_left); + top_left = top; + out[i] = left; + } + } } //------------------------------------------------------------------------------ diff --git a/src/dsp/filters_mips_dsp_r2.c b/src/dsp/filters_mips_dsp_r2.c index 8134af51..eb2f7b6d 100644 --- a/src/dsp/filters_mips_dsp_r2.c +++ b/src/dsp/filters_mips_dsp_r2.c @@ -33,12 +33,9 @@ assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \ (void)height; // Silence unused warning. -// if INVERSE -// preds == &dst[-1] == &src[-1] -// else // preds == &src[-1] != &dst[-1] -#define DO_PREDICT_LINE(SRC, DST, LENGTH, INVERSE) do { \ - const uint8_t* psrc = (uint8_t*)(SRC); \ +#define DO_PREDICT_LINE(SRC, DST, LENGTH) do { \ + const uint8_t* psrc = (const uint8_t*)(SRC); \ uint8_t* pdst = (uint8_t*)(DST); \ const int ilength = (int)(LENGTH); \ int temp0, temp1, temp2, temp3, temp4, temp5, temp6; \ @@ -48,25 +45,6 @@ "srl %[temp0], %[length], 0x2 \n\t" \ "beqz %[temp0], 4f \n\t" \ " andi %[temp6], %[length], 0x3 \n\t" \ - ".if " #INVERSE " \n\t" \ - "lbu %[temp1], -1(%[src]) \n\t" \ - "1: \n\t" \ - "lbu %[temp2], 0(%[src]) \n\t" \ - "lbu %[temp3], 1(%[src]) \n\t" \ - "lbu %[temp4], 2(%[src]) \n\t" \ - "lbu %[temp5], 3(%[src]) \n\t" \ - "addiu %[src], %[src], 4 \n\t" \ - "addiu %[temp0], %[temp0], -1 \n\t" \ - "addu %[temp2], %[temp2], %[temp1] \n\t" \ - "addu %[temp3], %[temp3], %[temp2] \n\t" \ - "addu %[temp4], %[temp4], %[temp3] \n\t" \ - "addu %[temp1], %[temp5], %[temp4] \n\t" \ - "sb %[temp2], -4(%[src]) \n\t" \ - "sb %[temp3], -3(%[src]) \n\t" \ - "sb %[temp4], -2(%[src]) \n\t" \ - "bnez %[temp0], 1b \n\t" \ - " sb %[temp1], -1(%[src]) \n\t" \ - ".else \n\t" \ "1: \n\t" \ "ulw %[temp1], -1(%[src]) \n\t" \ "ulw %[temp2], 0(%[src]) \n\t" \ @@ -76,7 +54,6 @@ "usw %[temp3], 0(%[dst]) \n\t" \ "bnez %[temp0], 1b \n\t" \ " addiu %[dst], %[dst], 4 \n\t" \ - ".endif \n\t" \ "4: \n\t" \ "beqz %[temp6], 3f \n\t" \ " nop \n\t" \ @@ -84,13 +61,8 @@ "lbu %[temp1], -1(%[src]) \n\t" \ "lbu %[temp2], 0(%[src]) \n\t" \ "addiu %[src], %[src], 1 \n\t" \ - ".if " #INVERSE " \n\t" \ - "addu %[temp3], %[temp1], %[temp2] \n\t" \ - "sb %[temp3], -1(%[src]) \n\t" \ - ".else \n\t" \ "subu %[temp3], %[temp1], %[temp2] \n\t" \ "sb %[temp3], 0(%[dst]) \n\t" \ - ".endif \n\t" \ "addiu %[temp6], %[temp6], -1 \n\t" \ "bnez %[temp6], 2b \n\t" \ " addiu %[dst], %[dst], 1 \n\t" \ @@ -105,12 +77,8 @@ } while (0) static WEBP_INLINE void PredictLine(const uint8_t* src, uint8_t* dst, - int length, int inverse) { - if (inverse) { - DO_PREDICT_LINE(src, dst, length, 1); - } else { - DO_PREDICT_LINE(src, dst, length, 0); - } + int length) { + DO_PREDICT_LINE(src, dst, length); } #define DO_PREDICT_LINE_VERTICAL(SRC, PRED, DST, LENGTH, INVERSE) do { \ @@ -172,16 +140,12 @@ static WEBP_INLINE void PredictLine(const uint8_t* src, uint8_t* dst, ); \ } while (0) -#define PREDICT_LINE_ONE_PASS(SRC, PRED, DST, INVERSE) do { \ +#define PREDICT_LINE_ONE_PASS(SRC, PRED, DST) do { \ int temp1, temp2, temp3; \ __asm__ volatile ( \ "lbu %[temp1], 0(%[src]) \n\t" \ "lbu %[temp2], 0(%[pred]) \n\t" \ - ".if " #INVERSE " \n\t" \ - "addu %[temp3], %[temp1], %[temp2] \n\t" \ - ".else \n\t" \ "subu %[temp3], %[temp1], %[temp2] \n\t" \ - ".endif \n\t" \ "sb %[temp3], 0(%[dst]) \n\t" \ : [temp1]"=&r"(temp1), [temp2]"=&r"(temp2), [temp3]"=&r"(temp3) \ : [pred]"r"((PRED)), [dst]"r"((DST)), [src]"r"((SRC)) \ @@ -192,10 +156,10 @@ static WEBP_INLINE void PredictLine(const uint8_t* src, uint8_t* dst, //------------------------------------------------------------------------------ // Horizontal filter. -#define FILTER_LINE_BY_LINE(INVERSE) do { \ +#define FILTER_LINE_BY_LINE do { \ while (row < last_row) { \ - PREDICT_LINE_ONE_PASS(in, preds - stride, out, INVERSE); \ - DO_PREDICT_LINE(in + 1, out + 1, width - 1, INVERSE); \ + PREDICT_LINE_ONE_PASS(in, preds - stride, out); \ + DO_PREDICT_LINE(in + 1, out + 1, width - 1); \ ++row; \ preds += stride; \ in += stride; \ @@ -206,19 +170,19 @@ static WEBP_INLINE void PredictLine(const uint8_t* src, uint8_t* dst, static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in, int width, int height, int stride, int row, int num_rows, - int inverse, uint8_t* out) { + uint8_t* out) { const uint8_t* preds; const size_t start_offset = row * stride; const int last_row = row + num_rows; SANITY_CHECK(in, out); in += start_offset; out += start_offset; - preds = inverse ? out : in; + preds = in; if (row == 0) { // Leftmost pixel is the same as input for topmost scanline. out[0] = in[0]; - PredictLine(in + 1, out + 1, width - 1, inverse); + PredictLine(in + 1, out + 1, width - 1); row = 1; preds += stride; in += stride; @@ -226,31 +190,21 @@ static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in, } // Filter line-by-line. - if (inverse) { - FILTER_LINE_BY_LINE(1); - } else { - FILTER_LINE_BY_LINE(0); - } + FILTER_LINE_BY_LINE; } - #undef FILTER_LINE_BY_LINE static void HorizontalFilter(const uint8_t* data, int width, int height, int stride, uint8_t* filtered_data) { - DoHorizontalFilter(data, width, height, stride, 0, height, 0, filtered_data); -} - -static void HorizontalUnfilter(int width, int height, int stride, int row, - int num_rows, uint8_t* data) { - DoHorizontalFilter(data, width, height, stride, row, num_rows, 1, data); + DoHorizontalFilter(data, width, height, stride, 0, height, filtered_data); } //------------------------------------------------------------------------------ // Vertical filter. -#define FILTER_LINE_BY_LINE(INVERSE) do { \ +#define FILTER_LINE_BY_LINE do { \ while (row < last_row) { \ - DO_PREDICT_LINE_VERTICAL(in, preds, out, width, INVERSE); \ + DO_PREDICT_LINE_VERTICAL(in, preds, out, width, 0); \ ++row; \ preds += stride; \ in += stride; \ @@ -260,21 +214,20 @@ static void HorizontalUnfilter(int width, int height, int stride, int row, static WEBP_INLINE void DoVerticalFilter(const uint8_t* in, int width, int height, int stride, - int row, int num_rows, - int inverse, uint8_t* out) { + int row, int num_rows, uint8_t* out) { const uint8_t* preds; const size_t start_offset = row * stride; const int last_row = row + num_rows; SANITY_CHECK(in, out); in += start_offset; out += start_offset; - preds = inverse ? out : in; + preds = in; if (row == 0) { // Very first top-left pixel is copied. out[0] = in[0]; // Rest of top scan-line is left-predicted. - PredictLine(in + 1, out + 1, width - 1, inverse); + PredictLine(in + 1, out + 1, width - 1); row = 1; in += stride; out += stride; @@ -284,24 +237,13 @@ static WEBP_INLINE void DoVerticalFilter(const uint8_t* in, } // Filter line-by-line. - if (inverse) { - FILTER_LINE_BY_LINE(1); - } else { - FILTER_LINE_BY_LINE(0); - } + FILTER_LINE_BY_LINE; } - #undef FILTER_LINE_BY_LINE -#undef DO_PREDICT_LINE_VERTICAL static void VerticalFilter(const uint8_t* data, int width, int height, int stride, uint8_t* filtered_data) { - DoVerticalFilter(data, width, height, stride, 0, height, 0, filtered_data); -} - -static void VerticalUnfilter(int width, int height, int stride, int row, - int num_rows, uint8_t* data) { - DoVerticalFilter(data, width, height, stride, row, num_rows, 1, data); + DoVerticalFilter(data, width, height, stride, 0, height, filtered_data); } //------------------------------------------------------------------------------ @@ -321,10 +263,10 @@ static WEBP_INLINE int GradientPredictor(uint8_t a, uint8_t b, uint8_t c) { return temp0; } -#define FILTER_LINE_BY_LINE(INVERSE, PREDS, OPERATION) do { \ +#define FILTER_LINE_BY_LINE(PREDS, OPERATION) do { \ while (row < last_row) { \ int w; \ - PREDICT_LINE_ONE_PASS(in, PREDS - stride, out, INVERSE); \ + PREDICT_LINE_ONE_PASS(in, PREDS - stride, out); \ for (w = 1; w < width; ++w) { \ const int pred = GradientPredictor(PREDS[w - 1], \ PREDS[w - stride], \ @@ -339,20 +281,19 @@ static WEBP_INLINE int GradientPredictor(uint8_t a, uint8_t b, uint8_t c) { static WEBP_INLINE void DoGradientFilter(const uint8_t* in, int width, int height, int stride, - int row, int num_rows, - int inverse, uint8_t* out) { + int row, int num_rows, uint8_t* out) { const uint8_t* preds; const size_t start_offset = row * stride; const int last_row = row + num_rows; SANITY_CHECK(in, out); in += start_offset; out += start_offset; - preds = inverse ? out : in; + preds = in; // left prediction for top scan-line if (row == 0) { out[0] = in[0]; - PredictLine(in + 1, out + 1, width - 1, inverse); + PredictLine(in + 1, out + 1, width - 1); row = 1; preds += stride; in += stride; @@ -360,25 +301,50 @@ static WEBP_INLINE void DoGradientFilter(const uint8_t* in, } // Filter line-by-line. - if (inverse) { - FILTER_LINE_BY_LINE(1, out, +); - } else { - FILTER_LINE_BY_LINE(0, in, -); - } + FILTER_LINE_BY_LINE(in, -); } - #undef FILTER_LINE_BY_LINE static void GradientFilter(const uint8_t* data, int width, int height, int stride, uint8_t* filtered_data) { - DoGradientFilter(data, width, height, stride, 0, height, 0, filtered_data); + DoGradientFilter(data, width, height, stride, 0, height, filtered_data); } -static void GradientUnfilter(int width, int height, int stride, int row, - int num_rows, uint8_t* data) { - DoGradientFilter(data, width, height, stride, row, num_rows, 1, data); +//------------------------------------------------------------------------------ + +static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + int i; + out[0] = in[0] + (prev == NULL ? 0 : prev[0]); + for (i = 1; i < width; ++i) out[i] = in[i] + out[i - 1]; } +static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + DO_PREDICT_LINE_VERTICAL(in, prev, out, width, 1); + } +} + +static void GradientUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + uint8_t top = prev[0], top_left = top, left = top; + int i; + for (i = 0; i < width; ++i) { + top = prev[i]; // need to read this first, in case prev==dst + left = in[i] + GradientPredictor(left, top, top_left); + top_left = top; + out[i] = left; + } + } +} + +#undef DO_PREDICT_LINE_VERTICAL #undef PREDICT_LINE_ONE_PASS #undef DO_PREDICT_LINE #undef SANITY_CHECK @@ -389,13 +355,13 @@ static void GradientUnfilter(int width, int height, int stride, int row, extern void VP8FiltersInitMIPSdspR2(void); WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitMIPSdspR2(void) { - WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter; - WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter; - WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter; - WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter; WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter; WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter; + + WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter; + WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter; + WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter; } #else // !WEBP_USE_MIPS_DSP_R2 diff --git a/src/dsp/filters_sse2.c b/src/dsp/filters_sse2.c index bf93342e..67f77999 100644 --- a/src/dsp/filters_sse2.c +++ b/src/dsp/filters_sse2.c @@ -33,82 +33,39 @@ (void)height; // Silence unused warning. static void PredictLineTop(const uint8_t* src, const uint8_t* pred, - uint8_t* dst, int length, int inverse) { + uint8_t* dst, int length) { int i; const int max_pos = length & ~31; assert(length >= 0); - if (inverse) { - for (i = 0; i < max_pos; i += 32) { - const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i + 0]); - const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]); - const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i + 0]); - const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]); - const __m128i C0 = _mm_add_epi8(A0, B0); - const __m128i C1 = _mm_add_epi8(A1, B1); - _mm_storeu_si128((__m128i*)&dst[i + 0], C0); - _mm_storeu_si128((__m128i*)&dst[i + 16], C1); - } - for (; i < length; ++i) dst[i] = src[i] + pred[i]; - } else { - for (i = 0; i < max_pos; i += 32) { - const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i + 0]); - const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]); - const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i + 0]); - const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]); - const __m128i C0 = _mm_sub_epi8(A0, B0); - const __m128i C1 = _mm_sub_epi8(A1, B1); - _mm_storeu_si128((__m128i*)&dst[i + 0], C0); - _mm_storeu_si128((__m128i*)&dst[i + 16], C1); - } - for (; i < length; ++i) dst[i] = src[i] - pred[i]; + for (i = 0; i < max_pos; i += 32) { + const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i + 0]); + const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]); + const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i + 0]); + const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]); + const __m128i C0 = _mm_sub_epi8(A0, B0); + const __m128i C1 = _mm_sub_epi8(A1, B1); + _mm_storeu_si128((__m128i*)&dst[i + 0], C0); + _mm_storeu_si128((__m128i*)&dst[i + 16], C1); } + for (; i < length; ++i) dst[i] = src[i] - pred[i]; } // Special case for left-based prediction (when preds==dst-1 or preds==src-1). -static void PredictLineLeft(const uint8_t* src, uint8_t* dst, int length, - int inverse) { +static void PredictLineLeft(const uint8_t* src, uint8_t* dst, int length) { int i; - if (length <= 0) return; - if (inverse) { - const int max_pos = length & ~7; - __m128i last = _mm_set_epi32(0, 0, 0, dst[-1]); - for (i = 0; i < max_pos; i += 8) { - const __m128i A0 = _mm_loadl_epi64((const __m128i*)(src + i)); - const __m128i A1 = _mm_add_epi8(A0, last); - const __m128i A2 = _mm_slli_si128(A1, 1); - const __m128i A3 = _mm_add_epi8(A1, A2); - const __m128i A4 = _mm_slli_si128(A3, 2); - const __m128i A5 = _mm_add_epi8(A3, A4); - const __m128i A6 = _mm_slli_si128(A5, 4); - const __m128i A7 = _mm_add_epi8(A5, A6); - _mm_storel_epi64((__m128i*)(dst + i), A7); - last = _mm_srli_epi64(A7, 56); - } - for (; i < length; ++i) dst[i] = src[i] + dst[i - 1]; - } else { - const int max_pos = length & ~31; - for (i = 0; i < max_pos; i += 32) { - const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i + 0 )); - const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i + 0 - 1)); - const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16 )); - const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1)); - const __m128i C0 = _mm_sub_epi8(A0, B0); - const __m128i C1 = _mm_sub_epi8(A1, B1); - _mm_storeu_si128((__m128i*)(dst + i + 0), C0); - _mm_storeu_si128((__m128i*)(dst + i + 16), C1); - } - for (; i < length; ++i) dst[i] = src[i] - src[i - 1]; - } -} - -static void PredictLineC(const uint8_t* src, const uint8_t* pred, - uint8_t* dst, int length, int inverse) { - int i; - if (inverse) { - for (i = 0; i < length; ++i) dst[i] = src[i] + pred[i]; - } else { - for (i = 0; i < length; ++i) dst[i] = src[i] - pred[i]; + const int max_pos = length & ~31; + assert(length >= 0); + for (i = 0; i < max_pos; i += 32) { + const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i + 0 )); + const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i + 0 - 1)); + const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16 )); + const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1)); + const __m128i C0 = _mm_sub_epi8(A0, B0); + const __m128i C1 = _mm_sub_epi8(A1, B1); + _mm_storeu_si128((__m128i*)(dst + i + 0), C0); + _mm_storeu_si128((__m128i*)(dst + i + 16), C1); } + for (; i < length; ++i) dst[i] = src[i] - src[i - 1]; } //------------------------------------------------------------------------------ @@ -117,21 +74,18 @@ static void PredictLineC(const uint8_t* src, const uint8_t* pred, static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in, int width, int height, int stride, int row, int num_rows, - int inverse, uint8_t* out) { - const uint8_t* preds; + uint8_t* out) { const size_t start_offset = row * stride; const int last_row = row + num_rows; SANITY_CHECK(in, out); in += start_offset; out += start_offset; - preds = inverse ? out : in; if (row == 0) { // Leftmost pixel is the same as input for topmost scanline. out[0] = in[0]; - PredictLineLeft(in + 1, out + 1, width - 1, inverse); + PredictLineLeft(in + 1, out + 1, width - 1); row = 1; - preds += stride; in += stride; out += stride; } @@ -139,10 +93,9 @@ static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in, // Filter line-by-line. while (row < last_row) { // Leftmost pixel is predicted from above. - PredictLineC(in, preds - stride, out, 1, inverse); - PredictLineLeft(in + 1, out + 1, width - 1, inverse); + out[0] = in[0] - in[-stride]; + PredictLineLeft(in + 1, out + 1, width - 1); ++row; - preds += stride; in += stride; out += stride; } @@ -153,34 +106,27 @@ static WEBP_INLINE void DoHorizontalFilter(const uint8_t* in, static WEBP_INLINE void DoVerticalFilter(const uint8_t* in, int width, int height, int stride, - int row, int num_rows, - int inverse, uint8_t* out) { - const uint8_t* preds; + int row, int num_rows, uint8_t* out) { const size_t start_offset = row * stride; const int last_row = row + num_rows; SANITY_CHECK(in, out); in += start_offset; out += start_offset; - preds = inverse ? out : in; if (row == 0) { // Very first top-left pixel is copied. out[0] = in[0]; // Rest of top scan-line is left-predicted. - PredictLineLeft(in + 1, out + 1, width - 1, inverse); + PredictLineLeft(in + 1, out + 1, width - 1); row = 1; in += stride; out += stride; - } else { - // We are starting from in-between. Make sure 'preds' points to prev row. - preds -= stride; } // Filter line-by-line. while (row < last_row) { - PredictLineTop(in, preds, out, width, inverse); + PredictLineTop(in, in - stride, out, width); ++row; - preds += stride; in += stride; out += stride; } @@ -219,6 +165,101 @@ static void GradientPredictDirect(const uint8_t* const row, } } +static WEBP_INLINE void DoGradientFilter(const uint8_t* in, + int width, int height, int stride, + int row, int num_rows, + uint8_t* out) { + const size_t start_offset = row * stride; + const int last_row = row + num_rows; + SANITY_CHECK(in, out); + in += start_offset; + out += start_offset; + + // left prediction for top scan-line + if (row == 0) { + out[0] = in[0]; + PredictLineLeft(in + 1, out + 1, width - 1); + row = 1; + in += stride; + out += stride; + } + + // Filter line-by-line. + while (row < last_row) { + out[0] = in[0] - in[-stride]; + GradientPredictDirect(in + 1, in + 1 - stride, out + 1, width - 1); + ++row; + in += stride; + out += stride; + } +} + +#undef SANITY_CHECK + +//------------------------------------------------------------------------------ + +static void HorizontalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoHorizontalFilter(data, width, height, stride, 0, height, filtered_data); +} + +static void VerticalFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoVerticalFilter(data, width, height, stride, 0, height, filtered_data); +} + +static void GradientFilter(const uint8_t* data, int width, int height, + int stride, uint8_t* filtered_data) { + DoGradientFilter(data, width, height, stride, 0, height, filtered_data); +} + +//------------------------------------------------------------------------------ +// Inverse transforms + +static void HorizontalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + int i; + __m128i last; + out[0] = in[0] + (prev == NULL ? 0 : prev[0]); + if (width <= 1) return; + last = _mm_set_epi32(0, 0, 0, out[0]); + for (i = 1; i + 8 <= width; i += 8) { + const __m128i A0 = _mm_loadl_epi64((const __m128i*)(in + i)); + const __m128i A1 = _mm_add_epi8(A0, last); + const __m128i A2 = _mm_slli_si128(A1, 1); + const __m128i A3 = _mm_add_epi8(A1, A2); + const __m128i A4 = _mm_slli_si128(A3, 2); + const __m128i A5 = _mm_add_epi8(A3, A4); + const __m128i A6 = _mm_slli_si128(A5, 4); + const __m128i A7 = _mm_add_epi8(A5, A6); + _mm_storel_epi64((__m128i*)(out + i), A7); + last = _mm_srli_epi64(A7, 56); + } + for (; i < width; ++i) out[i] = in[i] + out[i - 1]; +} + +static void VerticalUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + int i; + const int max_pos = width & ~31; + assert(width >= 0); + for (i = 0; i < max_pos; i += 32) { + const __m128i A0 = _mm_loadu_si128((const __m128i*)&in[i + 0]); + const __m128i A1 = _mm_loadu_si128((const __m128i*)&in[i + 16]); + const __m128i B0 = _mm_loadu_si128((const __m128i*)&prev[i + 0]); + const __m128i B1 = _mm_loadu_si128((const __m128i*)&prev[i + 16]); + const __m128i C0 = _mm_add_epi8(A0, B0); + const __m128i C1 = _mm_add_epi8(A1, B1); + _mm_storeu_si128((__m128i*)&out[i + 0], C0); + _mm_storeu_si128((__m128i*)&out[i + 16], C1); + } + for (; i < width; ++i) out[i] = in[i] + prev[i]; + } +} + static void GradientPredictInverse(const uint8_t* const in, const uint8_t* const top, uint8_t* const row, int length) { @@ -232,25 +273,24 @@ static void GradientPredictInverse(const uint8_t* const in, const __m128i tmp1 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); const __m128i B = _mm_unpacklo_epi8(tmp0, zero); const __m128i C = _mm_unpacklo_epi8(tmp1, zero); - const __m128i tmp2 = _mm_loadl_epi64((const __m128i*)&in[i]); - const __m128i D = _mm_unpacklo_epi8(tmp2, zero); // base input + const __m128i D = _mm_loadl_epi64((const __m128i*)&in[i]); // base input const __m128i E = _mm_sub_epi16(B, C); // unclipped gradient basis B - C __m128i out = zero; // accumulator for output __m128i mask_hi = _mm_set_epi32(0, 0, 0, 0xff); int k = 8; while (1) { - const __m128i tmp3 = _mm_add_epi16(A, E); // delta = A + B - C - const __m128i tmp4 = _mm_min_epi16(tmp3, mask_hi); - const __m128i tmp5 = _mm_max_epi16(tmp4, zero); // clipped delta - const __m128i tmp6 = _mm_add_epi16(tmp5, D); // add to in[] values - A = _mm_and_si128(tmp6, mask_hi); // 1-complement clip - out = _mm_or_si128(out, A); // accumulate output + const __m128i tmp3 = _mm_add_epi16(A, E); // delta = A + B - C + const __m128i tmp4 = _mm_packus_epi16(tmp3, zero); // saturate delta + const __m128i tmp5 = _mm_add_epi8(tmp4, D); // add to in[] + A = _mm_and_si128(tmp5, mask_hi); // 1-complement clip + out = _mm_or_si128(out, A); // accumulate output if (--k == 0) break; - A = _mm_slli_si128(A, 2); // rotate left sample - mask_hi = _mm_slli_si128(mask_hi, 2); // rotate mask + A = _mm_slli_si128(A, 1); // rotate left sample + mask_hi = _mm_slli_si128(mask_hi, 1); // rotate mask + A = _mm_unpacklo_epi8(A, zero); // convert 8b->16b } - A = _mm_srli_si128(A, 14); // prepare left sample for next iteration - _mm_storel_epi64((__m128i*)&row[i], _mm_packus_epi16(out, zero)); + A = _mm_srli_si128(A, 7); // prepare left sample for next iteration + _mm_storel_epi64((__m128i*)&row[i], out); } for (; i < length; ++i) { row[i] = in[i] + GradientPredictorC(row[i - 1], top[i], top[i - 1]); @@ -258,76 +298,14 @@ static void GradientPredictInverse(const uint8_t* const in, } } -static WEBP_INLINE void DoGradientFilter(const uint8_t* in, - int width, int height, int stride, - int row, int num_rows, - int inverse, uint8_t* out) { - const size_t start_offset = row * stride; - const int last_row = row + num_rows; - SANITY_CHECK(in, out); - in += start_offset; - out += start_offset; - - // left prediction for top scan-line - if (row == 0) { - out[0] = in[0]; - PredictLineLeft(in + 1, out + 1, width - 1, inverse); - row = 1; - in += stride; - out += stride; +static void GradientUnfilter(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + if (prev == NULL) { + HorizontalUnfilter(NULL, in, out, width); + } else { + out[0] = in[0] + prev[0]; // predict from above + GradientPredictInverse(in + 1, prev + 1, out + 1, width - 1); } - - // Filter line-by-line. - while (row < last_row) { - if (inverse) { - PredictLineC(in, out - stride, out, 1, inverse); // predict from above - GradientPredictInverse(in + 1, out + 1 - stride, out + 1, width - 1); - } else { - PredictLineC(in, in - stride, out, 1, inverse); - GradientPredictDirect(in + 1, in + 1 - stride, out + 1, width - 1); - } - ++row; - in += stride; - out += stride; - } -} - -#undef SANITY_CHECK - -//------------------------------------------------------------------------------ - -static void HorizontalFilter(const uint8_t* data, int width, int height, - int stride, uint8_t* filtered_data) { - DoHorizontalFilter(data, width, height, stride, 0, height, 0, filtered_data); -} - -static void VerticalFilter(const uint8_t* data, int width, int height, - int stride, uint8_t* filtered_data) { - DoVerticalFilter(data, width, height, stride, 0, height, 0, filtered_data); -} - - -static void GradientFilter(const uint8_t* data, int width, int height, - int stride, uint8_t* filtered_data) { - DoGradientFilter(data, width, height, stride, 0, height, 0, filtered_data); -} - - -//------------------------------------------------------------------------------ - -static void VerticalUnfilter(int width, int height, int stride, int row, - int num_rows, uint8_t* data) { - DoVerticalFilter(data, width, height, stride, row, num_rows, 1, data); -} - -static void HorizontalUnfilter(int width, int height, int stride, int row, - int num_rows, uint8_t* data) { - DoHorizontalFilter(data, width, height, stride, row, num_rows, 1, data); -} - -static void GradientUnfilter(int width, int height, int stride, int row, - int num_rows, uint8_t* data) { - DoGradientFilter(data, width, height, stride, row, num_rows, 1, data); } //------------------------------------------------------------------------------