dany/libwebp
1
0
Fork 0
mirror of https://github.com/webmproject/libwebp.git synced 2024-12-26 05:38:22 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix size_t overflow in WebPRescalerInit

Browse Source 
we need to surface potential error up, so lot of signature changes.

Change-Id: I7c11a46c4542564d06417203cd1158754e30a9e4
This commit is contained in:
Skal 2021-06-25 15:23:16 +02:00 committed by James Zern
parent b60d460318
commit 277d30749f
5 changed files with 92 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/dec/io_dec.c
View File

@ -330,21 +330,25 @@ static int InitYUVRescaler(const VP8Io* const io, WebPDecParams* const p) {
p->scaler_v = &scalers[2];
p->scaler_a = has_alpha ? &scalers[3] : NULL;
WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h,
if (!WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h,
buf->y, out_width, out_height, buf->y_stride, 1,
work);
WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height,
work) ||
!WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height,
buf->u, uv_out_width, uv_out_height, buf->u_stride, 1,
work + work_size);
WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height,
work + work_size) ||
!WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height,
buf->v, uv_out_width, uv_out_height, buf->v_stride, 1,
work + work_size + uv_work_size);
work + work_size + uv_work_size)) {
return 0;
}
p->emit = EmitRescaledYUV;
if (has_alpha) {
WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h,
if (!WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h,
buf->a, out_width, out_height, buf->a_stride, 1,
work + work_size + 2 * uv_work_size);
work + work_size + 2 * uv_work_size)) {
return 0;
}
p->emit_alpha = EmitRescaledAlphaYUV;
WebPInitAlphaProcessing();
}
@ -519,22 +523,26 @@ static int InitRGBRescaler(const VP8Io* const io, WebPDecParams* const p) {
p->scaler_v = &scalers[2];
p->scaler_a = has_alpha ? &scalers[3] : NULL;
WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h,
if (!WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h,
tmp + 0 * out_width, out_width, out_height, 0, 1,
work + 0 * work_size);
WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height,
work + 0 * work_size) ||
!WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height,
tmp + 1 * out_width, out_width, out_height, 0, 1,
work + 1 * work_size);
WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height,
work + 1 * work_size) ||
!WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height,
tmp + 2 * out_width, out_width, out_height, 0, 1,
work + 2 * work_size);
work + 2 * work_size)) {
return 0;
}
p->emit = EmitRescaledRGB;
WebPInitYUV444Converters();
if (has_alpha) {
WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h,
if (!WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h,
tmp + 3 * out_width, out_width, out_height, 0, 1,
work + 3 * work_size);
work + 3 * work_size)) {
return 0;
}
p->emit_alpha = EmitRescaledAlphaRGB;
if (p->output->colorspace == MODE_RGBA_4444 ||
p->output->colorspace == MODE_rgbA_4444) {

7
src/dec/vp8l_dec.c
View File

@ -559,8 +559,11 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) {
memory += work_size * sizeof(*work);
scaled_data = (uint32_t*)memory;
WebPRescalerInit(dec->rescaler, in_width, in_height, (uint8_t*)scaled_data,
out_width, out_height, 0, num_channels, work);
if (!WebPRescalerInit(dec->rescaler, in_width, in_height,
(uint8_t*)scaled_data, out_width, out_height,
0, num_channels, work)) {
return 0;
}
return 1;
}
#endif // WEBP_REDUCE_SIZE

39
src/enc/picture_rescale_enc.c
View File

@ -164,7 +164,7 @@ int WebPPictureCrop(WebPPicture* pic,
//------------------------------------------------------------------------------
// Simple picture rescaler
static void RescalePlane(const uint8_t* src,
static int RescalePlane(const uint8_t* src,
int src_width, int src_height, int src_stride,
uint8_t* dst,
int dst_width, int dst_height, int dst_stride,
@ -172,14 +172,17 @@ static void RescalePlane(const uint8_t* src,
int num_channels) {
WebPRescaler rescaler;
int y = 0;
WebPRescalerInit(&rescaler, src_width, src_height,
if (!WebPRescalerInit(&rescaler, src_width, src_height,
dst, dst_width, dst_height, dst_stride,
num_channels, work);
num_channels, work)) {
return 0;
}
while (y < src_height) {
y += WebPRescalerImport(&rescaler, src_height - y,
src + y * src_stride, src_stride);
WebPRescalerExport(&rescaler);
}
return 1;
}
static void AlphaMultiplyARGB(WebPPicture* const pic, int inverse) {
@ -222,25 +225,28 @@ int WebPPictureRescale(WebPPicture* pic, int width, int height) {
// If present, we need to rescale alpha first (for AlphaMultiplyY).
if (pic->a != NULL) {
WebPInitAlphaProcessing();
RescalePlane(pic->a, prev_width, prev_height, pic->a_stride,
tmp.a, width, height, tmp.a_stride, work, 1);
if (!RescalePlane(pic->a, prev_width, prev_height, pic->a_stride,
tmp.a, width, height, tmp.a_stride, work, 1)) {
return 0;
}
}
// We take transparency into account on the luma plane only. That's not
// totally exact blending, but still is a good approximation.
AlphaMultiplyY(pic, 0);
RescalePlane(pic->y, prev_width, prev_height, pic->y_stride,
tmp.y, width, height, tmp.y_stride, work, 1);
AlphaMultiplyY(&tmp, 1);
RescalePlane(pic->u,
if (!RescalePlane(pic->y, prev_width, prev_height, pic->y_stride,
tmp.y, width, height, tmp.y_stride, work, 1) ||
!RescalePlane(pic->u,
HALVE(prev_width), HALVE(prev_height), pic->uv_stride,
tmp.u,
HALVE(width), HALVE(height), tmp.uv_stride, work, 1);
RescalePlane(pic->v,
HALVE(width), HALVE(height), tmp.uv_stride, work, 1) ||
!RescalePlane(pic->v,
HALVE(prev_width), HALVE(prev_height), pic->uv_stride,
tmp.v,
HALVE(width), HALVE(height), tmp.uv_stride, work, 1);
HALVE(width), HALVE(height), tmp.uv_stride, work, 1)) {
return 0;
}
AlphaMultiplyY(&tmp, 1);
} else {
work = (rescaler_t*)WebPSafeMalloc(2ULL * width * 4, sizeof(*work));
if (work == NULL) {
@ -252,11 +258,12 @@ int WebPPictureRescale(WebPPicture* pic, int width, int height) {
// the premultiplication afterward (while preserving the alpha channel).
WebPInitAlphaProcessing();
AlphaMultiplyARGB(pic, 0);
RescalePlane((const uint8_t*)pic->argb, prev_width, prev_height,
if (!RescalePlane((const uint8_t*)pic->argb, prev_width, prev_height,
pic->argb_stride * 4,
(uint8_t*)tmp.argb, width, height,
tmp.argb_stride * 4,
work, 4);
tmp.argb_stride * 4, work, 4)) {
return 0;
}
AlphaMultiplyARGB(&tmp, 1);
}
WebPPictureFree(pic);

70
src/utils/rescaler_utils.c
View File

@ -17,63 +17,69 @@
#include <string.h>
#include "src/dsp/dsp.h"
#include "src/utils/rescaler_utils.h"
#include "src/utils/utils.h"
//------------------------------------------------------------------------------
void WebPRescalerInit(WebPRescaler* const wrk, int src_width, int src_height,
int WebPRescalerInit(WebPRescaler* const rescaler,
int src_width, int src_height,
uint8_t* const dst,
int dst_width, int dst_height, int dst_stride,
int num_channels, rescaler_t* const work) {
const int x_add = src_width, x_sub = dst_width;
const int y_add = src_height, y_sub = dst_height;
wrk->x_expand = (src_width < dst_width);
wrk->y_expand = (src_height < dst_height);
wrk->src_width = src_width;
wrk->src_height = src_height;
wrk->dst_width = dst_width;
wrk->dst_height = dst_height;
wrk->src_y = 0;
wrk->dst_y = 0;
wrk->dst = dst;
wrk->dst_stride = dst_stride;
wrk->num_channels = num_channels;
const uint64_t total_size = 2ull * dst_width * num_channels * sizeof(*work);
if (!CheckSizeOverflow(total_size)) return 0;
rescaler->x_expand = (src_width < dst_width);
rescaler->y_expand = (src_height < dst_height);
rescaler->src_width = src_width;
rescaler->src_height = src_height;
rescaler->dst_width = dst_width;
rescaler->dst_height = dst_height;
rescaler->src_y = 0;
rescaler->dst_y = 0;
rescaler->dst = dst;
rescaler->dst_stride = dst_stride;
rescaler->num_channels = num_channels;
// for 'x_expand', we use bilinear interpolation
wrk->x_add = wrk->x_expand ? (x_sub - 1) : x_add;
wrk->x_sub = wrk->x_expand ? (x_add - 1) : x_sub;
if (!wrk->x_expand) { // fx_scale is not used otherwise
wrk->fx_scale = WEBP_RESCALER_FRAC(1, wrk->x_sub);
rescaler->x_add = rescaler->x_expand ? (x_sub - 1) : x_add;
rescaler->x_sub = rescaler->x_expand ? (x_add - 1) : x_sub;
if (!rescaler->x_expand) { // fx_scale is not used otherwise
rescaler->fx_scale = WEBP_RESCALER_FRAC(1, rescaler->x_sub);
}
// vertical scaling parameters
wrk->y_add = wrk->y_expand ? y_add - 1 : y_add;
wrk->y_sub = wrk->y_expand ? y_sub - 1 : y_sub;
wrk->y_accum = wrk->y_expand ? wrk->y_sub : wrk->y_add;
if (!wrk->y_expand) {
rescaler->y_add = rescaler->y_expand ? y_add - 1 : y_add;
rescaler->y_sub = rescaler->y_expand ? y_sub - 1 : y_sub;
rescaler->y_accum = rescaler->y_expand ? rescaler->y_sub : rescaler->y_add;
if (!rescaler->y_expand) {
// This is WEBP_RESCALER_FRAC(dst_height, x_add * y_add) without the cast.
// Its value is <= WEBP_RESCALER_ONE, because dst_height <= wrk->y_add, and
// wrk->x_add >= 1;
// Its value is <= WEBP_RESCALER_ONE, because dst_height <= rescaler->y_add
// and rescaler->x_add >= 1;
const uint64_t num = (uint64_t)dst_height * WEBP_RESCALER_ONE;
const uint64_t den = (uint64_t)wrk->x_add * wrk->y_add;
const uint64_t den = (uint64_t)rescaler->x_add * rescaler->y_add;
const uint64_t ratio = num / den;
if (ratio != (uint32_t)ratio) {
// When ratio == WEBP_RESCALER_ONE, we can't represent the ratio with the
// current fixed-point precision. This happens when src_height ==
// wrk->y_add (which == src_height), and wrk->x_add == 1.
// rescaler->y_add (which == src_height), and rescaler->x_add == 1.
// => We special-case fxy_scale = 0, in WebPRescalerExportRow().
wrk->fxy_scale = 0;
rescaler->fxy_scale = 0;
} else {
wrk->fxy_scale = (uint32_t)ratio;
rescaler->fxy_scale = (uint32_t)ratio;
}
wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->y_sub);
rescaler->fy_scale = WEBP_RESCALER_FRAC(1, rescaler->y_sub);
} else {
wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->x_add);
// wrk->fxy_scale is unused here.
rescaler->fy_scale = WEBP_RESCALER_FRAC(1, rescaler->x_add);
// rescaler->fxy_scale is unused here.
}
wrk->irow = work;
wrk->frow = work + num_channels * dst_width;
memset(work, 0, 2 * dst_width * num_channels * sizeof(*work));
rescaler->irow = work;
rescaler->frow = work + num_channels * dst_width;
memset(work, 0, (size_t)total_size);
WebPRescalerDspInit();
return 1;
}
int WebPRescalerGetScaledDimensions(int src_width, int src_height,

3
src/utils/rescaler_utils.h
View File

@ -47,7 +47,8 @@ struct WebPRescaler {
};
// Initialize a rescaler given scratch area 'work' and dimensions of src & dst.
void WebPRescalerInit(WebPRescaler* const rescaler,
// Returns false in case of error.
int WebPRescalerInit(WebPRescaler* const rescaler,
int src_width, int src_height,
uint8_t* const dst,
int dst_width, int dst_height, int dst_stride,