VP8LEncodeStream: fix segfault on OOM

initialize bw_side before calling EncoderAnalyze() & EncoderInit() which
may fail; previously this would cause a free of an invalid pointer in
VP8LBitWriterWipeOut().

since at least:
v0.6.0-120-gf8c2ac15 Multi-thread the lossless cruncher.

Tested:
for i in `seq 1 639`; do
  export MALLOC_FAIL_AT=$i
  ./examples/cwebp -m 6 -q 100 -lossless jpeg_file
done

Bug: webp:565
Change-Id: I1c95883834b6e4b13aee890568ce3bad0f4266f0
(cherry picked from commit fe153fae98)
(cherry picked from commit ddd65f0d19)
(cherry picked from commit 5d805f7205)
This commit is contained in:
James Zern 2022-04-04 10:41:25 -07:00
parent a8f86c2ef1
commit 2ba7e11210

View File

@ -1692,11 +1692,16 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config,
const WebPWorkerInterface* const worker_interface = WebPGetWorkerInterface(); const WebPWorkerInterface* const worker_interface = WebPGetWorkerInterface();
int ok_main; int ok_main;
if (enc_main == NULL || !VP8LBitWriterInit(&bw_side, 0)) {
WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY);
VP8LEncoderDelete(enc_main);
return 0;
}
// Analyze image (entropy, num_palettes etc) // Analyze image (entropy, num_palettes etc)
if (enc_main == NULL || if (!EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main,
!EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main,
&red_and_blue_always_zero) || &red_and_blue_always_zero) ||
!EncoderInit(enc_main) || !VP8LBitWriterInit(&bw_side, 0)) { !EncoderInit(enc_main)) {
err = VP8_ENC_ERROR_OUT_OF_MEMORY; err = VP8_ENC_ERROR_OUT_OF_MEMORY;
goto Error; goto Error;
} }