GetBackwardReferences: fail on alloc error

previously failures in the call to
VP8LBackwardReferencesTraceBackwards() would be ignored which, though it
wouldn't result in a crash, would produce non-deterministic output

Change-Id: Id9890a60883c3270ec75e968506d46eea32b76d4
(cherry picked from commit e3cfafaf71)
(cherry picked from commit 20ef03ee35)

ChangeLog

@@ -1,3 +1,6 @@
+d9191588 fuzzer/*: normalize src/ includes
+53b6f762 fix indent
+731246ba update ChangeLog (tag: v1.2.1-rc2)
 d250f01d dsp/*: use WEBP_HAVE_* to determine Init availability
 3a4d3ecd update NEWS
 b2bc8093 bump version to 1.2.1

src/dec/io_dec.c

@@ -331,13 +331,13 @@ static int InitYUVRescaler(const VP8Io* const io, WebPDecParams* const p) {
   p->scaler_a = has_alpha ? &scalers[3] : NULL;
 
   if (!WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h,
-                   buf->y, out_width, out_height, buf->y_stride, 1,
+                        buf->y, out_width, out_height, buf->y_stride, 1,
                         work) ||
       !WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height,
-                   buf->u, uv_out_width, uv_out_height, buf->u_stride, 1,
+                        buf->u, uv_out_width, uv_out_height, buf->u_stride, 1,
                         work + work_size) ||
       !WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height,
-                   buf->v, uv_out_width, uv_out_height, buf->v_stride, 1,
+                        buf->v, uv_out_width, uv_out_height, buf->v_stride, 1,
                         work + work_size + uv_work_size)) {
     return 0;
   }
@@ -345,7 +345,7 @@ static int InitYUVRescaler(const VP8Io* const io, WebPDecParams* const p) {
 
   if (has_alpha) {
     if (!WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h,
-                     buf->a, out_width, out_height, buf->a_stride, 1,
+                          buf->a, out_width, out_height, buf->a_stride, 1,
                           work + work_size + 2 * uv_work_size)) {
       return 0;
     }
@@ -524,13 +524,13 @@ static int InitRGBRescaler(const VP8Io* const io, WebPDecParams* const p) {
   p->scaler_a = has_alpha ? &scalers[3] : NULL;
 
   if (!WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h,
-                   tmp + 0 * out_width, out_width, out_height, 0, 1,
+                        tmp + 0 * out_width, out_width, out_height, 0, 1,
                         work + 0 * work_size) ||
       !WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height,
-                   tmp + 1 * out_width, out_width, out_height, 0, 1,
+                        tmp + 1 * out_width, out_width, out_height, 0, 1,
                         work + 1 * work_size) ||
       !WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height,
-                   tmp + 2 * out_width, out_width, out_height, 0, 1,
+                        tmp + 2 * out_width, out_width, out_height, 0, 1,
                         work + 2 * work_size)) {
     return 0;
   }
@@ -539,7 +539,7 @@ static int InitRGBRescaler(const VP8Io* const io, WebPDecParams* const p) {
 
   if (has_alpha) {
     if (!WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h,
-                     tmp + 3 * out_width, out_width, out_height, 0, 1,
+                          tmp + 3 * out_width, out_width, out_height, 0, 1,
                           work + 3 * work_size)) {
       return 0;
     }

src/dsp/alpha_processing_neon.c

@@ -83,7 +83,7 @@ static void ApplyAlphaMultiply_NEON(uint8_t* rgba, int alpha_first,
 static int DispatchAlpha_NEON(const uint8_t* WEBP_RESTRICT alpha,
                               int alpha_stride, int width, int height,
                               uint8_t* WEBP_RESTRICT dst, int dst_stride) {
-  uint32_t alpha_mask = 0xffffffffu;
+  uint32_t alpha_mask = 0xffu;
   uint8x8_t mask8 = vdup_n_u8(0xff);
   uint32_t tmp[2];
   int i, j;
@@ -107,6 +107,7 @@ static int DispatchAlpha_NEON(const uint8_t* WEBP_RESTRICT alpha,
     dst += dst_stride;
   }
   vst1_u8((uint8_t*)tmp, mask8);
+  alpha_mask *= 0x01010101;
   alpha_mask &= tmp[0];
   alpha_mask &= tmp[1];
   return (alpha_mask != 0xffffffffu);
@@ -135,7 +136,7 @@ static void DispatchAlphaToGreen_NEON(const uint8_t* WEBP_RESTRICT alpha,
 static int ExtractAlpha_NEON(const uint8_t* WEBP_RESTRICT argb, int argb_stride,
                              int width, int height,
                              uint8_t* WEBP_RESTRICT alpha, int alpha_stride) {
-  uint32_t alpha_mask = 0xffffffffu;
+  uint32_t alpha_mask = 0xffu;
   uint8x8_t mask8 = vdup_n_u8(0xff);
   uint32_t tmp[2];
   int i, j;
@@ -157,6 +158,7 @@ static int ExtractAlpha_NEON(const uint8_t* WEBP_RESTRICT argb, int argb_stride,
     alpha += alpha_stride;
   }
   vst1_u8((uint8_t*)tmp, mask8);
+  alpha_mask *= 0x01010101;
   alpha_mask &= tmp[0];
   alpha_mask &= tmp[1];
   return (alpha_mask == 0xffffffffu);

src/dsp/lossless_enc_mips32.c

@@ -347,24 +347,24 @@ static void GetCombinedEntropyUnrefined_MIPS32(const uint32_t X[],
 static void AddVector_MIPS32(const uint32_t* pa, const uint32_t* pb,
                              uint32_t* pout, int size) {
   uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
-  const uint32_t end = ((size) / 4) * 4;
+  const int end = ((size) / 4) * 4;
   const uint32_t* const LoopEnd = pa + end;
   int i;
   ASM_START
   ADD_TO_OUT(0, 4, 8, 12, 1, pa, pb, pout)
   ASM_END_0
-  for (i = end; i < size; ++i) pout[i] = pa[i] + pb[i];
+  for (i = 0; i < size - end; ++i) pout[i] = pa[i] + pb[i];
 }
 
 static void AddVectorEq_MIPS32(const uint32_t* pa, uint32_t* pout, int size) {
   uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
-  const uint32_t end = ((size) / 4) * 4;
+  const int end = ((size) / 4) * 4;
   const uint32_t* const LoopEnd = pa + end;
   int i;
   ASM_START
   ADD_TO_OUT(0, 4, 8, 12, 0, pa, pout, pout)
   ASM_END_1
-  for (i = end; i < size; ++i) pout[i] += pa[i];
+  for (i = 0; i < size - end; ++i) pout[i] += pa[i];
 }
 
 #undef ASM_END_1

src/enc/backward_references_cost_enc.c

@@ -577,7 +577,7 @@ static int BackwardReferencesHashChainDistanceOnly(
       (CostModel*)WebPSafeCalloc(1ULL, cost_model_size);
   VP8LColorCache hashers;
   CostManager* cost_manager =
-      (CostManager*)WebPSafeMalloc(1ULL, sizeof(*cost_manager));
+      (CostManager*)WebPSafeCalloc(1ULL, sizeof(*cost_manager));
   int offset_prev = -1, len_prev = -1;
   double offset_cost = -1;
   int first_offset_is_constant = -1;  // initialized with 'impossible' value

src/enc/backward_references_enc.c

@@ -976,15 +976,16 @@ static int GetBackwardReferences(int width, int height,
       const VP8LHashChain* const hash_chain_tmp =
           (lz77_types_best[i] == kLZ77Standard) ? hash_chain : &hash_chain_box;
       const int cache_bits = (i == 1) ? 0 : *cache_bits_best;
-      if (VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits,
-                                               hash_chain_tmp, &refs[i],
-                                               refs_tmp)) {
-        double bit_cost_trace;
-        VP8LHistogramCreate(histo, refs_tmp, cache_bits);
-        bit_cost_trace = VP8LHistogramEstimateBits(histo);
-        if (bit_cost_trace < bit_costs_best[i]) {
-          BackwardRefsSwap(refs_tmp, &refs[i]);
-        }
+      double bit_cost_trace;
+      if (!VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits,
+                                                hash_chain_tmp, &refs[i],
+                                                refs_tmp)) {
+        goto Error;
+      }
+      VP8LHistogramCreate(histo, refs_tmp, cache_bits);
+      bit_cost_trace = VP8LHistogramEstimateBits(histo);
+      if (bit_cost_trace < bit_costs_best[i]) {
+        BackwardRefsSwap(refs_tmp, &refs[i]);
       }
     }
 

src/enc/picture_rescale_enc.c

@@ -165,15 +165,15 @@ int WebPPictureCrop(WebPPicture* pic,
 // Simple picture rescaler
 
 static int RescalePlane(const uint8_t* src,
-                         int src_width, int src_height, int src_stride,
-                         uint8_t* dst,
-                         int dst_width, int dst_height, int dst_stride,
-                         rescaler_t* const work,
-                         int num_channels) {
+                        int src_width, int src_height, int src_stride,
+                        uint8_t* dst,
+                        int dst_width, int dst_height, int dst_stride,
+                        rescaler_t* const work,
+                        int num_channels) {
   WebPRescaler rescaler;
   int y = 0;
   if (!WebPRescalerInit(&rescaler, src_width, src_height,
-                   dst, dst_width, dst_height, dst_stride,
+                        dst, dst_width, dst_height, dst_stride,
                         num_channels, work)) {
     return 0;
   }
@@ -237,12 +237,12 @@ int WebPPictureRescale(WebPPicture* pic, int width, int height) {
     if (!RescalePlane(pic->y, prev_width, prev_height, pic->y_stride,
                       tmp.y, width, height, tmp.y_stride, work, 1) ||
         !RescalePlane(pic->u,
-                 HALVE(prev_width), HALVE(prev_height), pic->uv_stride,
-                 tmp.u,
+                      HALVE(prev_width), HALVE(prev_height), pic->uv_stride,
+                      tmp.u,
                       HALVE(width), HALVE(height), tmp.uv_stride, work, 1) ||
         !RescalePlane(pic->v,
-                 HALVE(prev_width), HALVE(prev_height), pic->uv_stride,
-                 tmp.v,
+                      HALVE(prev_width), HALVE(prev_height), pic->uv_stride,
+                      tmp.v,
                       HALVE(width), HALVE(height), tmp.uv_stride, work, 1)) {
       return 0;
     }
@@ -259,8 +259,8 @@ int WebPPictureRescale(WebPPicture* pic, int width, int height) {
     WebPInitAlphaProcessing();
     AlphaMultiplyARGB(pic, 0);
     if (!RescalePlane((const uint8_t*)pic->argb, prev_width, prev_height,
-                 pic->argb_stride * 4,
-                 (uint8_t*)tmp.argb, width, height,
+                      pic->argb_stride * 4,
+                      (uint8_t*)tmp.argb, width, height,
                       tmp.argb_stride * 4, work, 4)) {
       return 0;
     }

src/enc/vp8l_enc.c

@@ -1912,11 +1912,16 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config,
   const WebPWorkerInterface* const worker_interface = WebPGetWorkerInterface();
   int ok_main;
 
+  if (enc_main == NULL || !VP8LBitWriterInit(&bw_side, 0)) {
+    WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY);
+    VP8LEncoderDelete(enc_main);
+    return 0;
+  }
+
   // Analyze image (entropy, num_palettes etc)
-  if (enc_main == NULL ||
-      !EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main,
+  if (!EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main,
                       &red_and_blue_always_zero) ||
-      !EncoderInit(enc_main) || !VP8LBitWriterInit(&bw_side, 0)) {
+      !EncoderInit(enc_main)) {
     err = VP8_ENC_ERROR_OUT_OF_MEMORY;
     goto Error;
   }

src/utils/rescaler_utils.h

@@ -49,11 +49,11 @@ struct WebPRescaler {
 // Initialize a rescaler given scratch area 'work' and dimensions of src & dst.
 // Returns false in case of error.
 int WebPRescalerInit(WebPRescaler* const rescaler,
-                      int src_width, int src_height,
-                      uint8_t* const dst,
-                      int dst_width, int dst_height, int dst_stride,
-                      int num_channels,
-                      rescaler_t* const work);
+                     int src_width, int src_height,
+                     uint8_t* const dst,
+                     int dst_width, int dst_height, int dst_stride,
+                     int num_channels,
+                     rescaler_t* const work);
 
 // If either 'scaled_width' or 'scaled_height' (but not both) is 0 the value
 // will be calculated preserving the aspect ratio, otherwise the values are

tests/fuzzer/advanced_api_fuzzer.c

@@ -17,7 +17,7 @@
 #include <string.h>
 
 #include "./fuzz_utils.h"
-#include "webp/decode.h"
+#include "src/webp/decode.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t* const data, size_t size) {
   int i;

tests/fuzzer/animation_api_fuzzer.c

@@ -15,9 +15,9 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "./fuzz_utils.h"
-#include "webp/decode.h"
-#include "webp/demux.h"
-#include "webp/mux_types.h"
+#include "src/webp/decode.h"
+#include "src/webp/demux.h"
+#include "src/webp/mux_types.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t* const data, size_t size) {
   WebPData webp_data;

tests/fuzzer/animdecoder_fuzzer.cc

@@ -16,7 +16,7 @@
 
 #include "examples/anim_util.h"
 #include "imageio/imageio_util.h"
-#include "webp/demux.h"
+#include "src/webp/demux.h"
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   // WebPAnimDecoderGetInfo() is too late to check the canvas size as

tests/fuzzer/mux_demux_api_fuzzer.c

@@ -15,8 +15,8 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "./fuzz_utils.h"
-#include "webp/demux.h"
-#include "webp/mux.h"
+#include "src/webp/demux.h"
+#include "src/webp/mux.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t* const data, size_t size) {
   WebPData webp_data;

tests/fuzzer/simple_api_fuzzer.c

@@ -15,7 +15,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "./fuzz_utils.h"
-#include "webp/decode.h"
+#include "src/webp/decode.h"
 
 int LLVMFuzzerTestOneInput(const uint8_t* const data, size_t size) {
   int w, h;