From 3991eb5fefe5c9e933ef38c59257353da538c9b9 Mon Sep 17 00:00:00 2001 From: James Zern Date: Mon, 4 Apr 2022 10:47:40 -0700 Subject: [PATCH] GetBackwardReferences: fail on alloc error previously failures in the call to VP8LBackwardReferencesTraceBackwards() would be ignored which, though it wouldn't result in a crash, would produce non-deterministic output Change-Id: Id9890a60883c3270ec75e968506d46eea32b76d4 (cherry picked from commit e3cfafaf719c2e163d3548d7a415da96fdff714f) (cherry picked from commit 20ef03ee351d4ff03fc5ff3ec4804a879d1b9d5c) (cherry picked from commit 89e226a3c70ae04fe8e1884aa0b291f39adb2fdf) --- src/enc/backward_references_enc.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/enc/backward_references_enc.c b/src/enc/backward_references_enc.c index 3ab7b0ac..1c22379d 100644 --- a/src/enc/backward_references_enc.c +++ b/src/enc/backward_references_enc.c @@ -911,13 +911,14 @@ static VP8LBackwardRefs* GetBackwardReferences( quality >= 25) { const VP8LHashChain* const hash_chain_tmp = (lz77_type_best == kLZ77Standard) ? hash_chain : &hash_chain_box; - if (VP8LBackwardReferencesTraceBackwards(width, height, argb, *cache_bits, - hash_chain_tmp, best, worst)) { - double bit_cost_trace; - VP8LHistogramCreate(histo, worst, *cache_bits); - bit_cost_trace = VP8LHistogramEstimateBits(histo); - if (bit_cost_trace < bit_cost_best) best = worst; + double bit_cost_trace; + if (!VP8LBackwardReferencesTraceBackwards(width, height, argb, *cache_bits, + hash_chain_tmp, best, worst)) { + goto Error; } + VP8LHistogramCreate(histo, worst, *cache_bits); + bit_cost_trace = VP8LHistogramEstimateBits(histo); + if (bit_cost_trace < bit_cost_best) best = worst; } BackwardReferences2DLocality(width, best);