GetBackwardReferences: fail on alloc error

previously failures in the call to
VP8LBackwardReferencesTraceBackwards() would be ignored which, though it
wouldn't result in a crash, would produce non-deterministic output

Change-Id: Id9890a60883c3270ec75e968506d46eea32b76d4
(cherry picked from commit e3cfafaf71)
(cherry picked from commit 20ef03ee35)
(cherry picked from commit 89e226a3c7)

src/dsp/alpha_processing_neon.c

@@ -83,7 +83,7 @@ static void ApplyAlphaMultiply_NEON(uint8_t* rgba, int alpha_first,
 static int DispatchAlpha_NEON(const uint8_t* alpha, int alpha_stride,
                               int width, int height,
                               uint8_t* dst, int dst_stride) {
-  uint32_t alpha_mask = 0xffffffffu;
+  uint32_t alpha_mask = 0xffu;
   uint8x8_t mask8 = vdup_n_u8(0xff);
   uint32_t tmp[2];
   int i, j;
@@ -107,6 +107,7 @@ static int DispatchAlpha_NEON(const uint8_t* alpha, int alpha_stride,
     dst += dst_stride;
   }
   vst1_u8((uint8_t*)tmp, mask8);
+  alpha_mask *= 0x01010101;
   alpha_mask &= tmp[0];
   alpha_mask &= tmp[1];
   return (alpha_mask != 0xffffffffu);
@@ -134,7 +135,7 @@ static void DispatchAlphaToGreen_NEON(const uint8_t* alpha, int alpha_stride,
 static int ExtractAlpha_NEON(const uint8_t* argb, int argb_stride,
                              int width, int height,
                              uint8_t* alpha, int alpha_stride) {
-  uint32_t alpha_mask = 0xffffffffu;
+  uint32_t alpha_mask = 0xffu;
   uint8x8_t mask8 = vdup_n_u8(0xff);
   uint32_t tmp[2];
   int i, j;
@@ -156,6 +157,7 @@ static int ExtractAlpha_NEON(const uint8_t* argb, int argb_stride,
     alpha += alpha_stride;
   }
   vst1_u8((uint8_t*)tmp, mask8);
+  alpha_mask *= 0x01010101;
   alpha_mask &= tmp[0];
   alpha_mask &= tmp[1];
   return (alpha_mask == 0xffffffffu);

src/dsp/lossless_enc_mips32.c

@@ -347,24 +347,24 @@ static void GetCombinedEntropyUnrefined_MIPS32(const uint32_t X[],
 static void AddVector_MIPS32(const uint32_t* pa, const uint32_t* pb,
                              uint32_t* pout, int size) {
   uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
-  const uint32_t end = ((size) / 4) * 4;
+  const int end = ((size) / 4) * 4;
   const uint32_t* const LoopEnd = pa + end;
   int i;
   ASM_START
   ADD_TO_OUT(0, 4, 8, 12, 1, pa, pb, pout)
   ASM_END_0
-  for (i = end; i < size; ++i) pout[i] = pa[i] + pb[i];
+  for (i = 0; i < size - end; ++i) pout[i] = pa[i] + pb[i];
 }
 
 static void AddVectorEq_MIPS32(const uint32_t* pa, uint32_t* pout, int size) {
   uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
-  const uint32_t end = ((size) / 4) * 4;
+  const int end = ((size) / 4) * 4;
   const uint32_t* const LoopEnd = pa + end;
   int i;
   ASM_START
   ADD_TO_OUT(0, 4, 8, 12, 0, pa, pout, pout)
   ASM_END_1
-  for (i = end; i < size; ++i) pout[i] += pa[i];
+  for (i = 0; i < size - end; ++i) pout[i] += pa[i];
 }
 
 #undef ASM_END_1

src/dsp/rescaler.c

@@ -109,8 +109,7 @@ void WebPRescalerExportRowExpand_C(WebPRescaler* const wrk) {
     for (x_out = 0; x_out < x_out_max; ++x_out) {
       const uint32_t J = frow[x_out];
       const int v = (int)MULT_FIX(J, wrk->fy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
     }
   } else {
     const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
@@ -120,8 +119,7 @@ void WebPRescalerExportRowExpand_C(WebPRescaler* const wrk) {
                        + (uint64_t)B * irow[x_out];
       const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
       const int v = (int)MULT_FIX(J, wrk->fy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
     }
   }
 }
@@ -138,17 +136,15 @@ void WebPRescalerExportRowShrink_C(WebPRescaler* const wrk) {
   assert(!wrk->y_expand);
   if (yscale) {
     for (x_out = 0; x_out < x_out_max; ++x_out) {
-      const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale);
-      const int v = (int)MULT_FIX_FLOOR(irow[x_out] - frac, wrk->fxy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      const uint32_t frac = (uint32_t)MULT_FIX_FLOOR(frow[x_out], yscale);
+      const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale);
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
       irow[x_out] = frac;   // new fractional start
     }
   } else {
     for (x_out = 0; x_out < x_out_max; ++x_out) {
       const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
       irow[x_out] = 0;
     }
   }

src/dsp/rescaler_mips_dsp_r2.c

@@ -107,10 +107,9 @@ static void ExportRowShrink_MIPSdspR2(WebPRescaler* const wrk) {
       );
     }
     for (i = 0; i < (x_out_max & 0x3); ++i) {
-      const uint32_t frac = (uint32_t)MULT_FIX(*frow++, yscale);
-      const int v = (int)MULT_FIX_FLOOR(*irow - frac, wrk->fxy_scale);
-      assert(v >= 0 && v <= 255);
-      *dst++ = v;
+      const uint32_t frac = (uint32_t)MULT_FIX_FLOOR(*frow++, yscale);
+      const int v = (int)MULT_FIX(*irow - frac, wrk->fxy_scale);
+      *dst++ = (v > 255) ? 255u : (uint8_t)v;
       *irow++ = frac;   // new fractional start
     }
   } else {
@@ -157,8 +156,7 @@ static void ExportRowShrink_MIPSdspR2(WebPRescaler* const wrk) {
     }
     for (i = 0; i < (x_out_max & 0x3); ++i) {
       const int v = (int)MULT_FIX_FLOOR(*irow, wrk->fxy_scale);
-      assert(v >= 0 && v <= 255);
-      *dst++ = v;
+      *dst++ = (v > 255) ? 255u : (uint8_t)v;
       *irow++ = 0;
     }
   }
@@ -219,8 +217,7 @@ static void ExportRowExpand_MIPSdspR2(WebPRescaler* const wrk) {
     for (i = 0; i < (x_out_max & 0x3); ++i) {
       const uint32_t J = *frow++;
       const int v = (int)MULT_FIX(J, wrk->fy_scale);
-      assert(v >= 0 && v <= 255);
-      *dst++ = v;
+      *dst++ = (v > 255) ? 255u : (uint8_t)v;
     }
   } else {
     const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
@@ -291,8 +288,7 @@ static void ExportRowExpand_MIPSdspR2(WebPRescaler* const wrk) {
                        + (uint64_t)B * *irow++;
       const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
       const int v = (int)MULT_FIX(J, wrk->fy_scale);
-      assert(v >= 0 && v <= 255);
-      *dst++ = v;
+      *dst++ = (v > 255) ? 255u : (uint8_t)v;
     }
   }
 }

src/dsp/rescaler_msa.c

@@ -166,8 +166,7 @@ static WEBP_INLINE void ExportRowExpand_0(const uint32_t* frow, uint8_t* dst,
     for (x_out = 0; x_out < length; ++x_out) {
       const uint32_t J = frow[x_out];
       const int v = (int)MULT_FIX(J, wrk->fy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
     }
   }
 }
@@ -241,8 +240,7 @@ static WEBP_INLINE void ExportRowExpand_1(const uint32_t* frow, uint32_t* irow,
                        + (uint64_t)B * irow[x_out];
       const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
       const int v = (int)MULT_FIX(J, wrk->fy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
     }
   }
 }
@@ -342,10 +340,9 @@ static WEBP_INLINE void ExportRowShrink_0(const uint32_t* frow, uint32_t* irow,
       length -= 4;
     }
     for (x_out = 0; x_out < length; ++x_out) {
-      const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale);
-      const int v = (int)MULT_FIX_FLOOR(irow[x_out] - frac, wrk->fxy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      const uint32_t frac = (uint32_t)MULT_FIX_FLOOR(frow[x_out], yscale);
+      const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale);
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
       irow[x_out] = frac;
     }
   }
@@ -406,8 +403,7 @@ static WEBP_INLINE void ExportRowShrink_1(uint32_t* irow, uint8_t* dst,
     }
     for (x_out = 0; x_out < length; ++x_out) {
       const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
       irow[x_out] = 0;
     }
   }

src/dsp/rescaler_neon.c

@@ -81,14 +81,13 @@ static void RescalerExportRowExpand_NEON(WebPRescaler* const wrk) {
       const uint32x4_t B1 = MULT_FIX(A1, fy_scale_half);
       const uint16x4_t C0 = vmovn_u32(B0);
       const uint16x4_t C1 = vmovn_u32(B1);
-      const uint8x8_t D = vmovn_u16(vcombine_u16(C0, C1));
+      const uint8x8_t D = vqmovn_u16(vcombine_u16(C0, C1));
       vst1_u8(dst + x_out, D);
     }
     for (; x_out < x_out_max; ++x_out) {
       const uint32_t J = frow[x_out];
       const int v = (int)MULT_FIX_C(J, fy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
     }
   } else {
     const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
@@ -102,7 +101,7 @@ static void RescalerExportRowExpand_NEON(WebPRescaler* const wrk) {
       const uint32x4_t D1 = MULT_FIX(C1, fy_scale_half);
       const uint16x4_t E0 = vmovn_u32(D0);
       const uint16x4_t E1 = vmovn_u32(D1);
-      const uint8x8_t F = vmovn_u16(vcombine_u16(E0, E1));
+      const uint8x8_t F = vqmovn_u16(vcombine_u16(E0, E1));
       vst1_u8(dst + x_out, F);
     }
     for (; x_out < x_out_max; ++x_out) {
@@ -110,8 +109,7 @@ static void RescalerExportRowExpand_NEON(WebPRescaler* const wrk) {
                        + (uint64_t)B * irow[x_out];
       const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
       const int v = (int)MULT_FIX_C(J, fy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
     }
   }
 }
@@ -135,23 +133,22 @@ static void RescalerExportRowShrink_NEON(WebPRescaler* const wrk) {
     for (x_out = 0; x_out < max_span; x_out += 8) {
       LOAD_32x8(frow + x_out, in0, in1);
       LOAD_32x8(irow + x_out, in2, in3);
-      const uint32x4_t A0 = MULT_FIX(in0, yscale_half);
-      const uint32x4_t A1 = MULT_FIX(in1, yscale_half);
+      const uint32x4_t A0 = MULT_FIX_FLOOR(in0, yscale_half);
+      const uint32x4_t A1 = MULT_FIX_FLOOR(in1, yscale_half);
       const uint32x4_t B0 = vqsubq_u32(in2, A0);
       const uint32x4_t B1 = vqsubq_u32(in3, A1);
-      const uint32x4_t C0 = MULT_FIX_FLOOR(B0, fxy_scale_half);
-      const uint32x4_t C1 = MULT_FIX_FLOOR(B1, fxy_scale_half);
+      const uint32x4_t C0 = MULT_FIX(B0, fxy_scale_half);
+      const uint32x4_t C1 = MULT_FIX(B1, fxy_scale_half);
       const uint16x4_t D0 = vmovn_u32(C0);
       const uint16x4_t D1 = vmovn_u32(C1);
-      const uint8x8_t E = vmovn_u16(vcombine_u16(D0, D1));
+      const uint8x8_t E = vqmovn_u16(vcombine_u16(D0, D1));
       vst1_u8(dst + x_out, E);
       STORE_32x8(A0, A1, irow + x_out);
     }
     for (; x_out < x_out_max; ++x_out) {
-      const uint32_t frac = (uint32_t)MULT_FIX_C(frow[x_out], yscale);
-      const int v = (int)MULT_FIX_FLOOR_C(irow[x_out] - frac, fxy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      const uint32_t frac = (uint32_t)MULT_FIX_FLOOR_C(frow[x_out], yscale);
+      const int v = (int)MULT_FIX_C(irow[x_out] - frac, fxy_scale);
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
       irow[x_out] = frac;   // new fractional start
     }
   } else {
@@ -161,14 +158,13 @@ static void RescalerExportRowShrink_NEON(WebPRescaler* const wrk) {
       const uint32x4_t A1 = MULT_FIX(in1, fxy_scale_half);
       const uint16x4_t B0 = vmovn_u32(A0);
       const uint16x4_t B1 = vmovn_u32(A1);
-      const uint8x8_t C = vmovn_u16(vcombine_u16(B0, B1));
+      const uint8x8_t C = vqmovn_u16(vcombine_u16(B0, B1));
       vst1_u8(dst + x_out, C);
       STORE_32x8(zero, zero, irow + x_out);
     }
     for (; x_out < x_out_max; ++x_out) {
       const int v = (int)MULT_FIX_C(irow[x_out], fxy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
       irow[x_out] = 0;
     }
   }

src/dsp/rescaler_sse2.c

@@ -225,35 +225,6 @@ static WEBP_INLINE void ProcessRow_SSE2(const __m128i* const A0,
   _mm_storel_epi64((__m128i*)dst, G);
 }
 
-static WEBP_INLINE void ProcessRow_Floor_SSE2(const __m128i* const A0,
-                                              const __m128i* const A1,
-                                              const __m128i* const A2,
-                                              const __m128i* const A3,
-                                              const __m128i* const mult,
-                                              uint8_t* const dst) {
-  const __m128i mask = _mm_set_epi32(0xffffffffu, 0, 0xffffffffu, 0);
-  const __m128i B0 = _mm_mul_epu32(*A0, *mult);
-  const __m128i B1 = _mm_mul_epu32(*A1, *mult);
-  const __m128i B2 = _mm_mul_epu32(*A2, *mult);
-  const __m128i B3 = _mm_mul_epu32(*A3, *mult);
-  const __m128i D0 = _mm_srli_epi64(B0, WEBP_RESCALER_RFIX);
-  const __m128i D1 = _mm_srli_epi64(B1, WEBP_RESCALER_RFIX);
-#if (WEBP_RESCALER_RFIX < 32)
-  const __m128i D2 =
-      _mm_and_si128(_mm_slli_epi64(B2, 32 - WEBP_RESCALER_RFIX), mask);
-  const __m128i D3 =
-      _mm_and_si128(_mm_slli_epi64(B3, 32 - WEBP_RESCALER_RFIX), mask);
-#else
-  const __m128i D2 = _mm_and_si128(B2, mask);
-  const __m128i D3 = _mm_and_si128(B3, mask);
-#endif
-  const __m128i E0 = _mm_or_si128(D0, D2);
-  const __m128i E1 = _mm_or_si128(D1, D3);
-  const __m128i F = _mm_packs_epi32(E0, E1);
-  const __m128i G = _mm_packus_epi16(F, F);
-  _mm_storel_epi64((__m128i*)dst, G);
-}
-
 static void RescalerExportRowExpand_SSE2(WebPRescaler* const wrk) {
   int x_out;
   uint8_t* const dst = wrk->dst;
@@ -274,8 +245,7 @@ static void RescalerExportRowExpand_SSE2(WebPRescaler* const wrk) {
     for (; x_out < x_out_max; ++x_out) {
       const uint32_t J = frow[x_out];
       const int v = (int)MULT_FIX(J, wrk->fy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
     }
   } else {
     const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
@@ -308,8 +278,7 @@ static void RescalerExportRowExpand_SSE2(WebPRescaler* const wrk) {
                        + (uint64_t)B * irow[x_out];
       const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
       const int v = (int)MULT_FIX(J, wrk->fy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
     }
   }
 }
@@ -328,20 +297,15 @@ static void RescalerExportRowShrink_SSE2(WebPRescaler* const wrk) {
     const int scale_xy = wrk->fxy_scale;
     const __m128i mult_xy = _mm_set_epi32(0, scale_xy, 0, scale_xy);
     const __m128i mult_y = _mm_set_epi32(0, yscale, 0, yscale);
-    const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER);
     for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) {
       __m128i A0, A1, A2, A3, B0, B1, B2, B3;
       LoadDispatchAndMult_SSE2(irow + x_out, NULL, &A0, &A1, &A2, &A3);
       LoadDispatchAndMult_SSE2(frow + x_out, &mult_y, &B0, &B1, &B2, &B3);
       {
-        const __m128i C0 = _mm_add_epi64(B0, rounder);
-        const __m128i C1 = _mm_add_epi64(B1, rounder);
-        const __m128i C2 = _mm_add_epi64(B2, rounder);
-        const __m128i C3 = _mm_add_epi64(B3, rounder);
-        const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX);   // = frac
-        const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX);
-        const __m128i D2 = _mm_srli_epi64(C2, WEBP_RESCALER_RFIX);
-        const __m128i D3 = _mm_srli_epi64(C3, WEBP_RESCALER_RFIX);
+        const __m128i D0 = _mm_srli_epi64(B0, WEBP_RESCALER_RFIX);   // = frac
+        const __m128i D1 = _mm_srli_epi64(B1, WEBP_RESCALER_RFIX);
+        const __m128i D2 = _mm_srli_epi64(B2, WEBP_RESCALER_RFIX);
+        const __m128i D3 = _mm_srli_epi64(B3, WEBP_RESCALER_RFIX);
         const __m128i E0 = _mm_sub_epi64(A0, D0);   // irow[x] - frac
         const __m128i E1 = _mm_sub_epi64(A1, D1);
         const __m128i E2 = _mm_sub_epi64(A2, D2);
@@ -352,14 +316,13 @@ static void RescalerExportRowShrink_SSE2(WebPRescaler* const wrk) {
         const __m128i G1 = _mm_or_si128(D1, F3);
         _mm_storeu_si128((__m128i*)(irow + x_out + 0), G0);
         _mm_storeu_si128((__m128i*)(irow + x_out + 4), G1);
-        ProcessRow_Floor_SSE2(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out);
+        ProcessRow_SSE2(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out);
       }
     }
     for (; x_out < x_out_max; ++x_out) {
-      const uint32_t frac = (int)MULT_FIX(frow[x_out], yscale);
-      const int v = (int)MULT_FIX_FLOOR(irow[x_out] - frac, wrk->fxy_scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      const uint32_t frac = (int)MULT_FIX_FLOOR(frow[x_out], yscale);
+      const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale);
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
       irow[x_out] = frac;   // new fractional start
     }
   } else {
@@ -375,8 +338,7 @@ static void RescalerExportRowShrink_SSE2(WebPRescaler* const wrk) {
     }
     for (; x_out < x_out_max; ++x_out) {
       const int v = (int)MULT_FIX(irow[x_out], scale);
-      assert(v >= 0 && v <= 255);
-      dst[x_out] = v;
+      dst[x_out] = (v > 255) ? 255u : (uint8_t)v;
       irow[x_out] = 0;
     }
   }

src/enc/backward_references_cost_enc.c

@@ -577,7 +577,7 @@ static int BackwardReferencesHashChainDistanceOnly(
       (CostModel*)WebPSafeCalloc(1ULL, cost_model_size);
   VP8LColorCache hashers;
   CostManager* cost_manager =
-      (CostManager*)WebPSafeMalloc(1ULL, sizeof(*cost_manager));
+      (CostManager*)WebPSafeCalloc(1ULL, sizeof(*cost_manager));
   int offset_prev = -1, len_prev = -1;
   double offset_cost = -1;
   int first_offset_is_constant = -1;  // initialized with 'impossible' value

src/enc/backward_references_enc.c

@@ -911,13 +911,14 @@ static VP8LBackwardRefs* GetBackwardReferences(
       quality >= 25) {
     const VP8LHashChain* const hash_chain_tmp =
         (lz77_type_best == kLZ77Standard) ? hash_chain : &hash_chain_box;
-    if (VP8LBackwardReferencesTraceBackwards(width, height, argb, *cache_bits,
-                                             hash_chain_tmp, best, worst)) {
-      double bit_cost_trace;
-      VP8LHistogramCreate(histo, worst, *cache_bits);
-      bit_cost_trace = VP8LHistogramEstimateBits(histo);
-      if (bit_cost_trace < bit_cost_best) best = worst;
+    double bit_cost_trace;
+    if (!VP8LBackwardReferencesTraceBackwards(width, height, argb, *cache_bits,
+                                              hash_chain_tmp, best, worst)) {
+      goto Error;
     }
+    VP8LHistogramCreate(histo, worst, *cache_bits);
+    bit_cost_trace = VP8LHistogramEstimateBits(histo);
+    if (bit_cost_trace < bit_cost_best) best = worst;
   }
 
   BackwardReferences2DLocality(width, best);

src/enc/vp8l_enc.c

@@ -1692,11 +1692,16 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config,
   const WebPWorkerInterface* const worker_interface = WebPGetWorkerInterface();
   int ok_main;
 
+  if (enc_main == NULL || !VP8LBitWriterInit(&bw_side, 0)) {
+    WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY);
+    VP8LEncoderDelete(enc_main);
+    return 0;
+  }
+
   // Analyze image (entropy, num_palettes etc)
-  if (enc_main == NULL ||
-      !EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main,
+  if (!EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main,
                       &red_and_blue_always_zero) ||
-      !EncoderInit(enc_main) || !VP8LBitWriterInit(&bw_side, 0)) {
+      !EncoderInit(enc_main)) {
     err = VP8_ENC_ERROR_OUT_OF_MEMORY;
     goto Error;
   }