From 2ba7e112103a9cff4d7b6ceb2985cdd6f92b793e Mon Sep 17 00:00:00 2001 From: James Zern Date: Mon, 4 Apr 2022 10:41:25 -0700 Subject: [PATCH] VP8LEncodeStream: fix segfault on OOM initialize bw_side before calling EncoderAnalyze() & EncoderInit() which may fail; previously this would cause a free of an invalid pointer in VP8LBitWriterWipeOut(). since at least: v0.6.0-120-gf8c2ac15 Multi-thread the lossless cruncher. Tested: for i in `seq 1 639`; do export MALLOC_FAIL_AT=$i ./examples/cwebp -m 6 -q 100 -lossless jpeg_file done Bug: webp:565 Change-Id: I1c95883834b6e4b13aee890568ce3bad0f4266f0 (cherry picked from commit fe153fae98a3fe4626ff537ec8d5f4477cec5739) (cherry picked from commit ddd65f0d19bc6a3acbc48e49d315140ccf099b9a) (cherry picked from commit 5d805f72051f0ae8aae61bbe1927c1a4bf9617ab) --- src/enc/vp8l_enc.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/enc/vp8l_enc.c b/src/enc/vp8l_enc.c index 2713edcd..4657c6d4 100644 --- a/src/enc/vp8l_enc.c +++ b/src/enc/vp8l_enc.c @@ -1692,11 +1692,16 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, const WebPWorkerInterface* const worker_interface = WebPGetWorkerInterface(); int ok_main; + if (enc_main == NULL || !VP8LBitWriterInit(&bw_side, 0)) { + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); + VP8LEncoderDelete(enc_main); + return 0; + } + // Analyze image (entropy, num_palettes etc) - if (enc_main == NULL || - !EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main, + if (!EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main, &red_and_blue_always_zero) || - !EncoderInit(enc_main) || !VP8LBitWriterInit(&bw_side, 0)) { + !EncoderInit(enc_main)) { err = VP8_ENC_ERROR_OUT_OF_MEMORY; goto Error; }