diff --git a/src/enc/backward_references.c b/src/enc/backward_references.c
index 7e47d29a..4bbc8c07 100644
--- a/src/enc/backward_references.c
+++ b/src/enc/backward_references.c
@@ -453,9 +453,9 @@ Error:
 typedef struct {
   double alpha_[VALUES_IN_BYTE];
   double red_[VALUES_IN_BYTE];
-  double literal_[PIX_OR_COPY_CODES_MAX];
   double blue_[VALUES_IN_BYTE];
   double distance_[NUM_DISTANCE_CODES];
+  double* literal_;
 } CostModel;
 
 static int BackwardReferencesTraceBackwards(
@@ -547,7 +547,12 @@ static int BackwardReferencesHashChainDistanceOnly(
   const int use_color_cache = (cache_bits > 0);
   float* const cost =
       (float*)WebPSafeMalloc(pix_count, sizeof(*cost));
-  CostModel* cost_model = (CostModel*)WebPSafeMalloc(1ULL, sizeof(*cost_model));
+  const size_t literal_array_size = sizeof(double) *
+      (NUM_LITERAL_CODES + NUM_LENGTH_CODES +
+       ((cache_bits > 0) ? (1 << cache_bits) : 0));
+  const size_t cost_model_size = sizeof(CostModel) + literal_array_size;
+  CostModel* const cost_model =
+      (CostModel*)WebPSafeMalloc(1ULL, cost_model_size);
   VP8LColorCache hashers;
   const int min_distance_code = 2;  // TODO(vikasa): tune as function of quality
   int window_size = WINDOW_SIZE;
@@ -556,6 +561,7 @@ static int BackwardReferencesHashChainDistanceOnly(
 
   if (cost == NULL || cost_model == NULL) goto Error;
 
+  cost_model->literal_ = (double*)(cost_model + 1);
   if (use_color_cache) {
     cc_init = VP8LColorCacheInit(&hashers, cache_bits);
     if (!cc_init) goto Error;
diff --git a/src/enc/backward_references.h b/src/enc/backward_references.h
index c2c81c56..a5e35deb 100644
--- a/src/enc/backward_references.h
+++ b/src/enc/backward_references.h
@@ -26,10 +26,6 @@ extern "C" {
 // Having 9 instead of 11 only removes about 0.25 % of compression density.
 #define MAX_COLOR_CACHE_BITS 9
 
-// Max ever number of codes we'll use:
-#define PIX_OR_COPY_CODES_MAX \
-    (NUM_LITERAL_CODES + NUM_LENGTH_CODES + (1 << MAX_COLOR_CACHE_BITS))
-
 // -----------------------------------------------------------------------------
 // PixOrCopy