Introduce Token buffer (unused for now)

Later, will be useful to collect stats and code bitstream in one single pass

Change-Id: Icf866473958b5eff879e219dbb1d7e350f2268ba

src/enc/frame.c

@@ -85,6 +85,44 @@ static int FinalizeSkipProba(VP8Encoder* const enc) {
   return size;
 }
 
+//------------------------------------------------------------------------------
+// Token buffer
+
+#ifdef USE_TOKEN_BUFFER
+
+void VP8InitTBuffer(VP8TBuffer* const p) {
+  p->left_ = MAX_NUM_TOKEN;
+  p->error_ = 0;
+  p->next_ = NULL;
+}
+
+int VP8AllocTBuffer(VP8TBuffer** const p) {
+  VP8TBuffer* const page = malloc(sizeof(*page));
+  if (page == NULL) {
+    (*p)->error_ |= 1;
+    return 0;
+  }
+  VP8InitTBuffer(page);
+  (*p)->next_ = page;
+  *p = page;
+  return 1;
+}
+
+int VP8EmitTokens(const VP8TBuffer* const p, VP8BitWriter* const bw,
+                  const uint8_t* const probas) {
+  while (p != NULL) {
+    int n = MAX_NUM_TOKEN;
+    if (p->error_) return 0;
+    while (n-- > p->left) {
+      VP8PutBit(bw, (p->tokens_[n] >> 15) & 1, probas[p->tokens_[n] & 0x7fff]);
+    }
+    p = p->next_;
+  }
+  return 1;
+}
+
+#endif
+
 //------------------------------------------------------------------------------
 // Recording of token probabilities.
 

src/enc/vp8enci.h

@@ -310,6 +310,35 @@ void VP8SetSkip(const VP8EncIterator* const it, int skip);
 void VP8SetSegment(const VP8EncIterator* const it, int segment);
 void VP8IteratorResetCosts(VP8EncIterator* const it);
 
+//------------------------------------------------------------------------------
+// Paginated token buffer
+
+#ifdef USE_TOKEN_BUFFER
+
+#define MAX_NUM_TOKEN 2030
+
+typedef struct VP8TBuffer VP8TBuffer;
+struct VP8TBuffer {
+  uint16_t tokens_[MAX_NUM_TOKEN];   // bit#15: bit, bits 0..14: slot idx
+  int left_;
+  int error_;  // true in case of malloc error
+  VP8TBuffer* next_;
+};
+int VP8AllocTBuffer(VP8TBuffer** const p);    // allocate a new page
+void VP8InitTBuffer(VP8TBuffer* const p);
+void VP8EmitTokens(const VP8TBuffer* const p, VP8BitWriter* const bw,
+                   const uint8_t* const probas);
+
+static WEBP_INLINE int VP8AddToken(VP8TBuffer** p, int bit, int proba_idx) {
+  if ((*p)->left_ > 0 || VP8AllocTBuffer(p)) {
+    const int slot = --(*p)->left_;
+    (*p)->tokens_[slot] = (bit << 15) | proba_idx;
+  }
+  return bit;
+}
+
+#endif
+
 //------------------------------------------------------------------------------
 // VP8Encoder