enc/*: add missing WebPEncodingSetError() calls

Users of the encoder (including anim_encode.c) and areas of the encoder
itself rely on the status returned via WebPPicture.

Change-Id: Id786176b8ac3b2329d1e41b9dacbb8dcc5d822e4

src/enc/alpha_enc.c

@@ -319,11 +319,11 @@ static int EncodeAlpha(VP8Encoder* const enc,
   assert(filter >= WEBP_FILTER_NONE && filter <= WEBP_FILTER_FAST);
 
   if (quality < 0 || quality > 100) {
-    return 0;
+    return WebPEncodingSetError(pic, VP8_ENC_ERROR_INVALID_CONFIGURATION);
   }
 
   if (method < ALPHA_NO_COMPRESSION || method > ALPHA_LOSSLESS_COMPRESSION) {
-    return 0;
+    return WebPEncodingSetError(pic, VP8_ENC_ERROR_INVALID_CONFIGURATION);
   }
 
   if (method == ALPHA_NO_COMPRESSION) {
@@ -333,7 +333,7 @@ static int EncodeAlpha(VP8Encoder* const enc,
 
   quant_alpha = (uint8_t*)WebPSafeMalloc(1ULL, data_size);
   if (quant_alpha == NULL) {
-    return 0;
+    return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);
   }
 
   // Extract alpha data (width x height) from raw_data (stride x height).
@@ -353,6 +353,9 @@ static int EncodeAlpha(VP8Encoder* const enc,
     ok = ApplyFiltersAndEncode(quant_alpha, width, height, data_size, method,
                                filter, reduce_levels, effort_level, output,
                                output_size, pic->stats);
+    if (!ok) {
+      WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);  // imprecise
+    }
 #if !defined(WEBP_DISABLE_STATS)
     if (pic->stats != NULL) {  // need stats?
       pic->stats->coded_size += (int)(*output_size);
@@ -412,7 +415,7 @@ int VP8EncStartAlpha(VP8Encoder* const enc) {
       WebPWorker* const worker = &enc->alpha_worker_;
       // Makes sure worker is good to go.
       if (!WebPGetWorkerInterface()->Reset(worker)) {
-        return 0;
+        return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY);
       }
       WebPGetWorkerInterface()->Launch(worker);
       return 1;

src/enc/analysis_enc.c

@@ -474,6 +474,10 @@ int VP8EncAnalyze(VP8Encoder* const enc) {
   } else {   // Use only one default segment.
     ResetAllMBInfo(enc);
   }
+  if (!ok) {
+    return WebPEncodingSetError(enc->pic_,
+                                VP8_ENC_ERROR_OUT_OF_MEMORY);  // imprecise
+  }
   return ok;
 }
 

src/enc/frame_enc.c

@@ -719,6 +719,7 @@ static int PostLoopFinalize(VP8EncIterator* const it, int ok) {
   } else {
     // Something bad happened -> need to do some memory cleanup.
     VP8EncFreeBitWriters(enc);
+    return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY);
   }
   return ok;
 }
@@ -754,6 +755,11 @@ int VP8EncLoop(VP8Encoder* const enc) {
     // *then* decide how to code the skip decision if there's one.
     if (!VP8Decimate(&it, &info, rd_opt) || dont_use_skip) {
       CodeResiduals(it.bw_, &it, &info);
+      if (it.bw_->error_) {
+        // enc->pic_->error_code is set in PostLoopFinalize().
+        ok = 0;
+        break;
+      }
     } else {   // reset predictors after a skip
       ResetAfterSkip(&it);
     }

src/enc/picture_csp_enc.c

@@ -535,7 +535,9 @@ static int ImportYUVAFromRGBA(const uint8_t* r_ptr,
     WebPInitConvertARGBToYUV();
     InitGammaTables();
 
-    if (tmp_rgb == NULL) return 0;  // malloc error
+    if (tmp_rgb == NULL) {
+      return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY);
+    }
 
     // Downsample Y/U/V planes, two rows at a time
     for (y = 0; y < (height >> 1); ++y) {

src/enc/picture_rescale_enc.c

@@ -137,7 +137,9 @@ int WebPPictureCrop(WebPPicture* pic,
   PictureGrabSpecs(pic, &tmp);
   tmp.width = width;
   tmp.height = height;
-  if (!WebPPictureAlloc(&tmp)) return 0;
+  if (!WebPPictureAlloc(&tmp)) {
+    return WebPEncodingSetError(pic, tmp.error_code);
+  }
 
   if (!pic->use_argb) {
     const int y_offset = top * pic->y_stride + left;
@@ -212,26 +214,28 @@ int WebPPictureRescale(WebPPicture* picture, int width, int height) {
   prev_height = picture->height;
   if (!WebPRescalerGetScaledDimensions(
           prev_width, prev_height, &width, &height)) {
-    return 0;
+    return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION);
   }
 
   PictureGrabSpecs(picture, &tmp);
   tmp.width = width;
   tmp.height = height;
-  if (!WebPPictureAlloc(&tmp)) return 0;
+  if (!WebPPictureAlloc(&tmp)) {
+    return WebPEncodingSetError(picture, tmp.error_code);
+  }
 
   if (!picture->use_argb) {
     work = (rescaler_t*)WebPSafeMalloc(2ULL * width, sizeof(*work));
     if (work == NULL) {
       WebPPictureFree(&tmp);
-      return 0;
+      return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY);
     }
     // If present, we need to rescale alpha first (for AlphaMultiplyY).
     if (picture->a != NULL) {
       WebPInitAlphaProcessing();
       if (!RescalePlane(picture->a, prev_width, prev_height, picture->a_stride,
                         tmp.a, width, height, tmp.a_stride, work, 1)) {
-        return 0;
+        return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION);
       }
     }
 
@@ -246,14 +250,14 @@ int WebPPictureRescale(WebPPicture* picture, int width, int height) {
         !RescalePlane(picture->v, HALVE(prev_width), HALVE(prev_height),
                       picture->uv_stride, tmp.v, HALVE(width), HALVE(height),
                       tmp.uv_stride, work, 1)) {
-      return 0;
+      return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION);
     }
     AlphaMultiplyY(&tmp, 1);
   } else {
     work = (rescaler_t*)WebPSafeMalloc(2ULL * width * 4, sizeof(*work));
     if (work == NULL) {
       WebPPictureFree(&tmp);
-      return 0;
+      return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY);
     }
     // In order to correctly interpolate colors, we need to apply the alpha
     // weighting first (black-matting), scale the RGB values, and remove
@@ -263,7 +267,7 @@ int WebPPictureRescale(WebPPicture* picture, int width, int height) {
     if (!RescalePlane((const uint8_t*)picture->argb, prev_width, prev_height,
                       picture->argb_stride * 4, (uint8_t*)tmp.argb, width,
                       height, tmp.argb_stride * 4, work, 4)) {
-      return 0;
+      return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION);
     }
     AlphaMultiplyARGB(&tmp, 1);
   }

src/enc/syntax_enc.c

@@ -258,7 +258,10 @@ static int EmitPartitionsSize(const VP8Encoder* const enc,
     buf[3 * p + 1] = (part_size >>  8) & 0xff;
     buf[3 * p + 2] = (part_size >> 16) & 0xff;
   }
-  return p ? pic->writer(buf, 3 * p, pic) : 1;
+  if (p && !pic->writer(buf, 3 * p, pic)) {
+    return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE);
+  }
+  return 1;
 }
 
 //------------------------------------------------------------------------------
@@ -381,6 +384,7 @@ int VP8EncWrite(VP8Encoder* const enc) {
 
   enc->coded_size_ = (int)(CHUNK_HEADER_SIZE + riff_size);
   ok = ok && WebPReportProgress(pic, final_percent, &enc->percent_);
+  if (!ok) WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE);
   return ok;
 }
 

src/enc/vp8l_enc.c

@@ -1298,7 +1298,10 @@ static int EncodeImageInternal(
           }
         }
         tokens = (HuffmanTreeToken*)WebPSafeMalloc(max_tokens, sizeof(*tokens));
-        if (tokens == NULL) goto Error;
+        if (tokens == NULL) {
+          WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);
+          goto Error;
+        }
         for (i = 0; i < 5 * histogram_image_size; ++i) {
           HuffmanTreeCode* const codes = &huffman_codes[i];
           StoreHuffmanCode(bw, huff_tree, tokens, codes);

src/enc/webp_enc.c

@@ -307,7 +307,10 @@ int WebPEncodingSetError(const WebPPicture* const pic,
                          WebPEncodingError error) {
   assert((int)error < VP8_ENC_ERROR_LAST);
   assert((int)error >= VP8_ENC_OK);
-  ((WebPPicture*)pic)->error_code = error;
+  // The oldest error reported takes precedence over the new one.
+  if (pic->error_code == VP8_ENC_OK) {
+    ((WebPPicture*)pic)->error_code = error;
+  }
   return 0;
 }
 
@@ -329,7 +332,7 @@ int WebPEncode(const WebPConfig* config, WebPPicture* pic) {
   int ok = 0;
   if (pic == NULL) return 0;
 
-  WebPEncodingSetError(pic, VP8_ENC_OK);  // all ok so far
+  pic->error_code = VP8_ENC_OK;  // all ok so far
   if (config == NULL) {  // bad params
     return WebPEncodingSetError(pic, VP8_ENC_ERROR_NULL_PARAMETER);
   }