From e3cfafaf719c2e163d3548d7a415da96fdff714f Mon Sep 17 00:00:00 2001 From: James Zern Date: Mon, 4 Apr 2022 10:47:40 -0700 Subject: [PATCH] GetBackwardReferences: fail on alloc error previously failures in the call to VP8LBackwardReferencesTraceBackwards() would be ignored which, though it wouldn't result in a crash, would produce non-deterministic output Change-Id: Id9890a60883c3270ec75e968506d46eea32b76d4 --- src/enc/backward_references_enc.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/enc/backward_references_enc.c b/src/enc/backward_references_enc.c index 48993753..79c009c1 100644 --- a/src/enc/backward_references_enc.c +++ b/src/enc/backward_references_enc.c @@ -1007,15 +1007,16 @@ static int GetBackwardReferences(int width, int height, const VP8LHashChain* const hash_chain_tmp = (lz77_types_best[i] == kLZ77Standard) ? hash_chain : &hash_chain_box; const int cache_bits = (i == 1) ? 0 : *cache_bits_best; - if (VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits, - hash_chain_tmp, &refs[i], - refs_tmp)) { - double bit_cost_trace; - VP8LHistogramCreate(histo, refs_tmp, cache_bits); - bit_cost_trace = VP8LHistogramEstimateBits(histo); - if (bit_cost_trace < bit_costs_best[i]) { - BackwardRefsSwap(refs_tmp, &refs[i]); - } + double bit_cost_trace; + if (!VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits, + hash_chain_tmp, &refs[i], + refs_tmp)) { + goto Error; + } + VP8LHistogramCreate(histo, refs_tmp, cache_bits); + bit_cost_trace = VP8LHistogramEstimateBits(histo); + if (bit_cost_trace < bit_costs_best[i]) { + BackwardRefsSwap(refs_tmp, &refs[i]); } }