From d48455502434f5e79d89033268ad2f854faf7640 Mon Sep 17 00:00:00 2001 From: Urvang Joshi Date: Tue, 14 Apr 2015 11:51:47 -0700 Subject: [PATCH] AnimEncoder API: Use timestamp instead of duration as input to Add(). When converting from video sources, the duration of current frame is often unavailable until the next frame. So, we internally convert timestamps to durations. Change-Id: I20ad86361c22e014be7eb91f00d5d40108281351 --- examples/gif2webp.c | 17 +++++-- src/mux/anim_encode.c | 108 ++++++++++++++++++++++++++++++++---------- src/webp/mux.h | 18 +++++-- 3 files changed, 110 insertions(+), 33 deletions(-) diff --git a/examples/gif2webp.c b/examples/gif2webp.c index 0400a43f..df27c1a6 100644 --- a/examples/gif2webp.c +++ b/examples/gif2webp.c @@ -91,7 +91,8 @@ int main(int argc, const char *argv[]) { const char *in_file = NULL, *out_file = NULL; FILE* out = NULL; GifFileType* gif = NULL; - int duration = 0; + int frame_duration = 0; + int frame_timestamp = 0; GIFDisposeMethod orig_dispose = GIF_DISPOSE_NONE; WebPPicture frame; // Frame rectangle only (not disposed). @@ -330,7 +331,7 @@ int main(int argc, const char *argv[]) { // Note that 'curr_canvas' is same as 'prev_canvas' at this point. GIFBlendFrames(&frame, &gif_rect, &curr_canvas); - if (!WebPAnimEncoderAdd(enc, &curr_canvas, duration, &config)) { + if (!WebPAnimEncoderAdd(enc, &curr_canvas, frame_timestamp, &config)) { fprintf(stderr, "Error! Cannot encode frame as WebP\n"); fprintf(stderr, "Error code: %d\n", curr_canvas.error_code); } @@ -340,11 +341,14 @@ int main(int argc, const char *argv[]) { GIFDisposeFrame(orig_dispose, &gif_rect, &prev_canvas, &curr_canvas); GIFCopyPixels(&curr_canvas, &prev_canvas); + // Update timestamp (for next frame). + frame_timestamp += frame_duration; + // In GIF, graphic control extensions are optional for a frame, so we // may not get one before reading the next frame. To handle this case, // we reset frame properties to reasonable defaults for the next frame. orig_dispose = GIF_DISPOSE_NONE; - duration = 0; + frame_duration = 0; transparent_index = GIF_INDEX_INVALID; break; } @@ -359,7 +363,7 @@ int main(int argc, const char *argv[]) { break; // Do nothing for now. } case GRAPHICS_EXT_FUNC_CODE: { - if (!GIFReadGraphicsExtension(data, &duration, &orig_dispose, + if (!GIFReadGraphicsExtension(data, &frame_duration, &orig_dispose, &transparent_index)) { goto End; } @@ -424,6 +428,11 @@ int main(int argc, const char *argv[]) { } } while (!done); + // Last NULL frame. + if (!WebPAnimEncoderAdd(enc, NULL, frame_timestamp, NULL)) { + fprintf(stderr, "Error flushing WebP muxer.\n"); + } + if (!WebPAnimEncoderAssemble(enc, &webp_data)) { // TODO(urvang): Print actual error code. fprintf(stderr, "ERROR assembling the WebP file.\n"); diff --git a/src/mux/anim_encode.c b/src/mux/anim_encode.c index 9fd50b00..bc7ff9e9 100644 --- a/src/mux/anim_encode.c +++ b/src/mux/anim_encode.c @@ -69,11 +69,16 @@ struct WebPAnimEncoder { // transparent pixels in a frame. int keyframe_; // Index of selected key-frame relative to 'start_'. int count_since_key_frame_; // Frames seen since the last key-frame. + + int first_timestamp_; // Timestamp of the first frame. + int prev_timestamp_; // Timestamp of the last added frame. int prev_candidate_undecided_; // True if it's not yet decided if previous // frame would be a sub-frame or a key-frame. // Misc. int is_first_frame_; // True if first frame is yet to be added/being added. + int got_null_frame_; // True if WebPAnimEncoderAdd() has already been called + // with a NULL frame. size_t frame_count_; // Number of frames added to mux so far. WebPMux* mux_; // Muxer to assemble the WebP bitstream. @@ -255,8 +260,11 @@ WebPAnimEncoder* WebPAnimEncoderNewInternal( if (enc->mux_ == NULL) goto Err; enc->count_since_key_frame_ = 0; + enc->first_timestamp_ = 0; + enc->prev_timestamp_ = 0; enc->prev_candidate_undecided_ = 0; enc->is_first_frame_ = 1; + enc->got_null_frame_ = 0; return enc; // All OK. @@ -634,7 +642,7 @@ typedef struct { static WebPEncodingError EncodeCandidate(WebPPicture* const sub_frame, const FrameRect* const rect, const WebPConfig* const config, - int use_blending, int duration, + int use_blending, Candidate* const candidate) { WebPEncodingError error_code = VP8_ENC_OK; assert(candidate != NULL); @@ -648,7 +656,7 @@ static WebPEncodingError EncodeCandidate(WebPPicture* const sub_frame, candidate->info_.dispose_method = WEBP_MUX_DISPOSE_NONE; // Set later. candidate->info_.blend_method = use_blending ? WEBP_MUX_BLEND : WEBP_MUX_NO_BLEND; - candidate->info_.duration = duration; + candidate->info_.duration = 0; // Set in next call to WebPAnimEncoderAdd(). // Encode picture. WebPMemoryWriterInit(&candidate->mem_); @@ -686,7 +694,7 @@ enum { static WebPEncodingError GenerateCandidates( WebPAnimEncoder* const enc, Candidate candidates[CANDIDATE_COUNT], WebPMuxAnimDispose dispose_method, int is_lossless, int is_key_frame, - const FrameRect* const rect, WebPPicture* sub_frame, int duration, + const FrameRect* const rect, WebPPicture* sub_frame, const WebPConfig* const config_ll, const WebPConfig* const config_lossy) { WebPEncodingError error_code = VP8_ENC_OK; const int is_dispose_none = (dispose_method == WEBP_MUX_DISPOSE_NONE); @@ -720,7 +728,7 @@ static WebPEncodingError GenerateCandidates( enc->curr_canvas_copy_modified_ = 1; } error_code = EncodeCandidate(sub_frame, rect, config_ll, use_blending, - duration, candidate_ll); + candidate_ll); if (error_code != VP8_ENC_OK) return error_code; } if (candidate_lossy->evaluate_) { @@ -730,7 +738,7 @@ static WebPEncodingError GenerateCandidates( enc->curr_canvas_copy_modified_ = 1; } error_code = EncodeCandidate(sub_frame, rect, config_lossy, use_blending, - duration, candidate_lossy); + candidate_lossy); if (error_code != VP8_ENC_OK) return error_code; } return error_code; @@ -764,6 +772,15 @@ static void SetPreviousDisposeMethod(WebPAnimEncoder* const enc, } } +// Sets duration of the previous frame to be 'duration'. +static void SetPreviousDuration(WebPAnimEncoder* const enc, int duration) { + const size_t position = enc->count_ - 1; + EncodedFrame* const prev_enc_frame = GetFrame(enc, position); + assert(enc->count_ >= 1); + prev_enc_frame->sub_frame_.duration = duration; + prev_enc_frame->key_frame_.duration = duration; +} + // Pick the candidate encoded frame with smallest size and release other // candidates. // TODO(later): Perhaps a rough SSIM/PSNR produced by the encoder should @@ -814,7 +831,7 @@ static void PickBestCandidate(WebPAnimEncoder* const enc, // Depending on the configuration, tries different compressions // (lossy/lossless), dispose methods, blending methods etc to encode the current // frame and outputs the best one in 'encoded_frame'. -static WebPEncodingError SetFrame(WebPAnimEncoder* const enc, int duration, +static WebPEncodingError SetFrame(WebPAnimEncoder* const enc, const WebPConfig* const config, int is_key_frame, EncodedFrame* const encoded_frame) { @@ -882,17 +899,16 @@ static WebPEncodingError SetFrame(WebPAnimEncoder* const enc, int duration, if (try_dispose_none) { error_code = GenerateCandidates( enc, candidates, WEBP_MUX_DISPOSE_NONE, is_lossless, is_key_frame, - &rect_none, &sub_frame_none, duration, &config_ll, &config_lossy); + &rect_none, &sub_frame_none, &config_ll, &config_lossy); if (error_code != VP8_ENC_OK) goto Err; } if (try_dispose_bg) { assert(!enc->is_first_frame_); assert(dispose_bg_possible); - error_code = - GenerateCandidates(enc, candidates, WEBP_MUX_DISPOSE_BACKGROUND, - is_lossless, is_key_frame, &rect_bg, &sub_frame_bg, - duration, &config_ll, &config_lossy); + error_code = GenerateCandidates( + enc, candidates, WEBP_MUX_DISPOSE_BACKGROUND, is_lossless, is_key_frame, + &rect_bg, &sub_frame_bg, &config_ll, &config_lossy); if (error_code != VP8_ENC_OK) goto Err; } @@ -920,7 +936,7 @@ static int64_t KeyFramePenalty(const EncodedFrame* const encoded_frame) { encoded_frame->sub_frame_.bitstream.size); } -static int CacheFrame(WebPAnimEncoder* const enc, int duration, +static int CacheFrame(WebPAnimEncoder* const enc, const WebPConfig* const config) { int ok = 0; WebPEncodingError error_code = VP8_ENC_OK; @@ -931,7 +947,7 @@ static int CacheFrame(WebPAnimEncoder* const enc, int duration, if (enc->is_first_frame_) { // Add this as a key-frame. error_code = - SetFrame(enc, duration, config, 1, encoded_frame); + SetFrame(enc, config, 1, encoded_frame); if (error_code != VP8_ENC_OK) { goto End; } @@ -944,7 +960,7 @@ static int CacheFrame(WebPAnimEncoder* const enc, int duration, ++enc->count_since_key_frame_; if (enc->count_since_key_frame_ <= enc->options_.kmin) { // Add this as a frame rectangle. - error_code = SetFrame(enc, duration, config, 0, encoded_frame); + error_code = SetFrame(enc, config, 0, encoded_frame); if (error_code != VP8_ENC_OK) { goto End; } @@ -955,11 +971,11 @@ static int CacheFrame(WebPAnimEncoder* const enc, int duration, int64_t curr_delta; // Add this as a frame rectangle to enc. - error_code = SetFrame(enc, duration, config, 0, encoded_frame); + error_code = SetFrame(enc, config, 0, encoded_frame); if (error_code != VP8_ENC_OK) goto End; // Add this as a key-frame to enc, too. - error_code = SetFrame(enc, duration, config, 1, encoded_frame); + error_code = SetFrame(enc, config, 1, encoded_frame); if (error_code != VP8_ENC_OK) goto End; // Analyze size difference of the two variants. @@ -1023,10 +1039,9 @@ static int FlushFrames(WebPAnimEncoder* const enc) { return 0; } if (enc->options_.verbose) { - fprintf(stderr, - "Added frame. offset:%d,%d duration:%d dispose:%d blend:%d\n", - info->x_offset, info->y_offset, info->duration, - info->dispose_method, info->blend_method); + fprintf(stderr, "Added frame. offset:%d,%d dispose:%d blend:%d\n", + info->x_offset, info->y_offset, info->dispose_method, + info->blend_method); } ++enc->frame_count_; FrameRelease(curr); @@ -1051,17 +1066,43 @@ static int FlushFrames(WebPAnimEncoder* const enc) { #undef DELTA_INFINITY #undef KEYFRAME_NONE -int WebPAnimEncoderAdd(WebPAnimEncoder* enc, WebPPicture* frame, int duration, +int WebPAnimEncoderAdd(WebPAnimEncoder* enc, WebPPicture* frame, int timestamp, const WebPConfig* encoder_config) { WebPConfig config; - if (enc == NULL || frame == NULL) { + + if (enc == NULL) { return 0; } + + if (!enc->is_first_frame_) { + // Make sure timestamps are non-decreasing (integer wrap-around is OK). + const uint32_t prev_frame_duration = + (uint32_t)timestamp - enc->prev_timestamp_; + if (prev_frame_duration >= MAX_DURATION) { + if (frame != NULL) { + frame->error_code = VP8_ENC_ERROR_INVALID_CONFIGURATION; + } + if (enc->options_.verbose) { + fprintf(stderr, + "ERROR adding frame: timestamps must be non-decreasing.\n"); + } + return 0; + } + SetPreviousDuration(enc, (int)prev_frame_duration); + } else { + enc->first_timestamp_ = timestamp; + } + + if (frame == NULL) { // Special: last call. + enc->got_null_frame_ = 1; + return 1; + } + if (frame->width != enc->canvas_width_ || - frame->height != enc->canvas_height_ || duration < 0) { + frame->height != enc->canvas_height_) { frame->error_code = VP8_ENC_ERROR_INVALID_CONFIGURATION; if (enc->options_.verbose) { - fprintf(stderr, "ERROR adding frame: Invalid input.\n"); + fprintf(stderr, "ERROR adding frame: Invalid frame dimensions.\n"); } return 0; } @@ -1088,7 +1129,7 @@ int WebPAnimEncoderAdd(WebPAnimEncoder* enc, WebPPicture* frame, int duration, assert(enc->curr_canvas_copy_modified_ == 1); CopyCurrentCanvas(enc); - if (!CacheFrame(enc, duration, &config)) { + if (!CacheFrame(enc, &config)) { return 0; } if (!FlushFrames(enc)) { @@ -1096,6 +1137,7 @@ int WebPAnimEncoderAdd(WebPAnimEncoder* enc, WebPPicture* frame, int duration, } enc->curr_canvas_ = NULL; enc->curr_canvas_copy_modified_ = 1; + enc->prev_timestamp_ = timestamp; return 1; } @@ -1206,6 +1248,7 @@ static WebPMuxError OptimizeSingleFrame(WebPAnimEncoder* const enc, int WebPAnimEncoderAssemble(WebPAnimEncoder* enc, WebPData* webp_data) { WebPMux* mux; WebPMuxError err; + int total_frames; // Muxed frames + cached (but not yet muxed) frames. if (enc == NULL) { return 0; @@ -1217,6 +1260,21 @@ int WebPAnimEncoderAssemble(WebPAnimEncoder* enc, WebPData* webp_data) { return 0; } + total_frames = enc->frame_count_ + enc->count_; + if (total_frames == 0) { + if (enc->options_.verbose) { + fprintf(stderr, "ERROR: No frames to assemble\n"); + } + return 0; + } + + if (!enc->got_null_frame_ && total_frames > 1 && enc->count_ > 0) { + // set duration of the last frame to be avg of durations of previous frames. + const int average_duration = + (enc->prev_timestamp_ - enc->first_timestamp_) / (total_frames - 1); + SetPreviousDuration(enc, average_duration); + } + // Flush any remaining frames. enc->flush_count_ = enc->count_; if (!FlushFrames(enc)) { diff --git a/src/webp/mux.h b/src/webp/mux.h index 2a0fc858..8c168dd1 100644 --- a/src/webp/mux.h +++ b/src/webp/mux.h @@ -22,7 +22,7 @@ extern "C" { #endif -#define WEBP_MUX_ABI_VERSION 0x0104 // MAJOR(8b) + MINOR(8b) +#define WEBP_MUX_ABI_VERSION 0x0105 // MAJOR(8b) + MINOR(8b) //------------------------------------------------------------------------------ // Mux API @@ -407,8 +407,9 @@ WEBP_EXTERN(WebPMuxError) WebPMuxAssemble(WebPMux* mux, WebPConfig config; WebPConfigInit(&config); // Tune 'config' as needed. - WebPAnimEncoderAdd(enc, frame, duration, &config); + WebPAnimEncoderAdd(enc, frame, timestamp_ms, &config); } + WebPAnimEncoderAdd(enc, NULL, timestamp_ms, NULL); WebPAnimEncoderAssemble(enc, webp_data); WebPAnimEncoderDelete(enc); // Write the 'webp_data' to a file, or re-mux it further. @@ -471,21 +472,30 @@ static WEBP_INLINE WebPAnimEncoder* WebPAnimEncoderNew( // Optimize the given frame for WebP, encode it and add it to the // WebPAnimEncoder object. +// The last call to 'WebPAnimEncoderAdd' should be with frame = NULL, which +// indicates that no more frames are to be added. This call is also used to +// determine the duration of the last frame. // Parameters: // enc - (in/out) object to which the frame is to be added. // frame - (in/out) frame data in ARGB or YUV(A) format. If it is in YUV(A) // format, it will be converted to ARGB, which incurs a small loss. -// duration - (in) frame duration +// timestamp_ms - (in) timestamp of this frame in milliseconds. +// Duration of a frame would be calculated as +// "timestamp of next frame - timestamp of this frame". +// Hence, timestamps should be in non-decreasing order. // config - (in) encoding options; can be passed NULL to pick // reasonable defaults. // Returns: // On error, returns false and frame->error_code is set appropriately. // Otherwise, returns true. WEBP_EXTERN(int) WebPAnimEncoderAdd( - WebPAnimEncoder* enc, WebPPicture* frame, int duration, + WebPAnimEncoder* enc, WebPPicture* frame, int timestamp_ms, const WebPConfig* config); // Assemble all frames added so far into a WebP bitstream. +// This call should be preceded by a call to 'WebPAnimEncoderAdd' with +// frame = NULL; if not, the duration of the last frame will be internally +// estimated. // Parameters: // enc - (in/out) object from which the frames are to be assembled. // webp_data - (out) generated WebP bitstream.