src/dsp/alpha_processing_sse2.c

@@ -16,8 +16,6 @@
 #if defined(WEBP_USE_SSE2)
 #include <emmintrin.h>
 
-#include "src/dsp/cpu.h"
-
 //------------------------------------------------------------------------------
 
 static int DispatchAlpha_SSE2(const uint8_t* WEBP_RESTRICT alpha,
@@ -28,44 +26,38 @@ static int DispatchAlpha_SSE2(const uint8_t* WEBP_RESTRICT alpha,
   uint32_t alpha_and = 0xff;
   int i, j;
   const __m128i zero = _mm_setzero_si128();
-  const __m128i alpha_mask = _mm_set1_epi32((int)0xff);  // to preserve A
+  const __m128i rgb_mask = _mm_set1_epi32((int)0xffffff00);  // to preserve RGB
-  const __m128i all_0xff = _mm_set1_epi8(0xff);
+  const __m128i all_0xff = _mm_set_epi32(0, 0, ~0, ~0);
-  __m128i all_alphas16 = all_0xff;
+  __m128i all_alphas = all_0xff;
-  __m128i all_alphas8 = all_0xff;
 
   // We must be able to access 3 extra bytes after the last written byte
   // 'dst[4 * width - 4]', because we don't know if alpha is the first or the
   // last byte of the quadruplet.
+  const int limit = (width - 1) & ~7;
+
   for (j = 0; j < height; ++j) {
-    char* ptr = (char*)dst;
+    __m128i* out = (__m128i*)dst;
-    for (i = 0; i + 16 <= width - 1; i += 16) {
+    for (i = 0; i < limit; i += 8) {
-      // load 16 alpha bytes
-      const __m128i a0 = _mm_loadu_si128((const __m128i*)&alpha[i]);
-      const __m128i a1_lo = _mm_unpacklo_epi8(a0, zero);
-      const __m128i a1_hi = _mm_unpackhi_epi8(a0, zero);
-      const __m128i a2_lo_lo = _mm_unpacklo_epi16(a1_lo, zero);
-      const __m128i a2_lo_hi = _mm_unpackhi_epi16(a1_lo, zero);
-      const __m128i a2_hi_lo = _mm_unpacklo_epi16(a1_hi, zero);
-      const __m128i a2_hi_hi = _mm_unpackhi_epi16(a1_hi, zero);
-      _mm_maskmoveu_si128(a2_lo_lo, alpha_mask, ptr + 0);
-      _mm_maskmoveu_si128(a2_lo_hi, alpha_mask, ptr + 16);
-      _mm_maskmoveu_si128(a2_hi_lo, alpha_mask, ptr + 32);
-      _mm_maskmoveu_si128(a2_hi_hi, alpha_mask, ptr + 48);
-      // accumulate 16 alpha 'and' in parallel
-      all_alphas16 = _mm_and_si128(all_alphas16, a0);
-      ptr += 64;
-    }
-    if (i + 8 <= width - 1) {
       // load 8 alpha bytes
       const __m128i a0 = _mm_loadl_epi64((const __m128i*)&alpha[i]);
       const __m128i a1 = _mm_unpacklo_epi8(a0, zero);
       const __m128i a2_lo = _mm_unpacklo_epi16(a1, zero);
       const __m128i a2_hi = _mm_unpackhi_epi16(a1, zero);
-      _mm_maskmoveu_si128(a2_lo, alpha_mask, ptr);
+      // load 8 dst pixels (32 bytes)
-      _mm_maskmoveu_si128(a2_hi, alpha_mask, ptr + 16);
+      const __m128i b0_lo = _mm_loadu_si128(out + 0);
-      // accumulate 8 alpha 'and' in parallel
+      const __m128i b0_hi = _mm_loadu_si128(out + 1);
-      all_alphas8 = _mm_and_si128(all_alphas8, a0);
+      // mask dst alpha values
-      i += 8;
+      const __m128i b1_lo = _mm_and_si128(b0_lo, rgb_mask);
+      const __m128i b1_hi = _mm_and_si128(b0_hi, rgb_mask);
+      // combine
+      const __m128i b2_lo = _mm_or_si128(b1_lo, a2_lo);
+      const __m128i b2_hi = _mm_or_si128(b1_hi, a2_hi);
+      // store
+      _mm_storeu_si128(out + 0, b2_lo);
+      _mm_storeu_si128(out + 1, b2_hi);
+      // accumulate eight alpha 'and' in parallel
+      all_alphas = _mm_and_si128(all_alphas, a0);
+      out += 2;
     }
     for (; i < width; ++i) {
       const uint32_t alpha_value = alpha[i];
@@ -76,9 +68,8 @@ static int DispatchAlpha_SSE2(const uint8_t* WEBP_RESTRICT alpha,
     dst += dst_stride;
   }
   // Combine the eight alpha 'and' into a 8-bit mask.
-  alpha_and &= _mm_movemask_epi8(_mm_cmpeq_epi8(all_alphas8, all_0xff)) & 0xff;
+  alpha_and &= _mm_movemask_epi8(_mm_cmpeq_epi8(all_alphas, all_0xff));
-  return (alpha_and != 0xff ||
+  return (alpha_and != 0xff);
-          _mm_movemask_epi8(_mm_cmpeq_epi8(all_alphas16, all_0xff)) != 0xffff);
 }
 
 static void DispatchAlphaToGreen_SSE2(const uint8_t* WEBP_RESTRICT alpha,

src/enc/histogram_enc.c

@@ -597,15 +597,17 @@ static void HistogramBuild(
 }
 
 // Copies the histograms and computes its bit_cost.
+static const uint32_t kInvalidHistogramSymbol = (uint32_t)(-1);
 static void HistogramCopyAndAnalyze(VP8LHistogramSet* const orig_histo,
                                     VP8LHistogramSet* const image_histo,
-                                    int* const num_used) {
+                                    int* const num_used,
-  int i;
+                                    uint32_t* const histogram_symbols) {
+  int i, cluster_id;
+  int num_used_orig = *num_used;
   VP8LHistogram** const orig_histograms = orig_histo->histograms;
   VP8LHistogram** const histograms = image_histo->histograms;
   assert(image_histo->max_size == orig_histo->max_size);
-  image_histo->size = 0;
+  for (cluster_id = 0, i = 0; i < orig_histo->max_size; ++i) {
-  for (i = 0; i < orig_histo->max_size; ++i) {
     VP8LHistogram* const histo = orig_histograms[i];
     UpdateHistogramCost(histo);
 
@@ -613,14 +615,18 @@ static void HistogramCopyAndAnalyze(VP8LHistogramSet* const orig_histo,
     // with no information (when they are skipped because of LZ77).
     if (!histo->is_used[0] && !histo->is_used[1] && !histo->is_used[2]
         && !histo->is_used[3] && !histo->is_used[4]) {
-      // The first histogram is always used.
+      // The first histogram is always used. If an histogram is empty, we set
+      // its id to be the same as the previous one: this will improve
+      // compressibility for later LZ77.
       assert(i > 0);
-      orig_histograms[i] = NULL;
+      HistogramSetRemoveHistogram(image_histo, i, num_used);
-      --*num_used;
+      HistogramSetRemoveHistogram(orig_histo, i, &num_used_orig);
+      histogram_symbols[i] = kInvalidHistogramSymbol;
     } else {
       // Copy histograms from orig_histo[] to image_histo[].
-      HistogramCopy(histo, histograms[image_histo->size]);
+      HistogramCopy(histo, histograms[i]);
-      ++image_histo->size;
+      histogram_symbols[i] = cluster_id++;
+      assert(cluster_id <= image_histo->max_size);
     }
   }
 }
@@ -628,6 +634,7 @@ static void HistogramCopyAndAnalyze(VP8LHistogramSet* const orig_histo,
 // Partition histograms to different entropy bins for three dominant (literal,
 // red and blue) symbol costs and compute the histogram aggregate bit_cost.
 static void HistogramAnalyzeEntropyBin(VP8LHistogramSet* const image_histo,
+                                       uint16_t* const bin_map,
                                        int low_effort) {
   int i;
   VP8LHistogram** const histograms = image_histo->histograms;
@@ -637,25 +644,28 @@ static void HistogramAnalyzeEntropyBin(VP8LHistogramSet* const image_histo,
 
   // Analyze the dominant (literal, red and blue) entropy costs.
   for (i = 0; i < histo_size; ++i) {
+    if (histograms[i] == NULL) continue;
     UpdateDominantCostRange(histograms[i], &cost_range);
   }
 
   // bin-hash histograms on three of the dominant (literal, red and blue)
   // symbol costs and store the resulting bin_id for each histogram.
   for (i = 0; i < histo_size; ++i) {
-    histograms[i]->bin_id =
+    // bin_map[i] is not set to a special value as its use will later be guarded
-        GetHistoBinIndex(histograms[i], &cost_range, low_effort);
+    // by another (histograms[i] == NULL).
+    if (histograms[i] == NULL) continue;
+    bin_map[i] = GetHistoBinIndex(histograms[i], &cost_range, low_effort);
   }
 }
 
 // Merges some histograms with same bin_id together if it's advantageous.
 // Sets the remaining histograms to NULL.
 // 'combine_cost_factor' has to be divided by 100.
-static void HistogramCombineEntropyBin(VP8LHistogramSet* const image_histo,
+static void HistogramCombineEntropyBin(
-                                       int* num_used, VP8LHistogram* cur_combo,
+    VP8LHistogramSet* const image_histo, int* num_used,
-                                       int num_bins,
+    const uint32_t* const clusters, uint16_t* const cluster_mappings,
-                                       int32_t combine_cost_factor,
+    VP8LHistogram* cur_combo, const uint16_t* const bin_map, int num_bins,
-                                       int low_effort) {
+    int32_t combine_cost_factor, int low_effort) {
   VP8LHistogram** const histograms = image_histo->histograms;
   int idx;
   struct {
@@ -670,14 +680,19 @@ static void HistogramCombineEntropyBin(VP8LHistogramSet* const image_histo,
     bin_info[idx].num_combine_failures = 0;
   }
 
+  // By default, a cluster matches itself.
+  for (idx = 0; idx < *num_used; ++idx) cluster_mappings[idx] = idx;
   for (idx = 0; idx < image_histo->size; ++idx) {
-    const int bin_id = histograms[idx]->bin_id;
+    int bin_id, first;
-    const int first = bin_info[bin_id].first;
+    if (histograms[idx] == NULL) continue;
+    bin_id = bin_map[idx];
+    first = bin_info[bin_id].first;
     if (first == -1) {
       bin_info[bin_id].first = idx;
     } else if (low_effort) {
       HistogramAdd(histograms[idx], histograms[first], histograms[first]);
       HistogramSetRemoveHistogram(image_histo, idx, num_used);
+      cluster_mappings[clusters[idx]] = clusters[first];
     } else {
       // try to merge #idx into #first (both share the same bin_id)
       const uint64_t bit_cost = histograms[idx]->bit_cost;
@@ -700,6 +715,7 @@ static void HistogramCombineEntropyBin(VP8LHistogramSet* const image_histo,
           // move the (better) merged histogram to its final slot
           HistogramSwap(&cur_combo, &histograms[first]);
           HistogramSetRemoveHistogram(image_histo, idx, num_used);
+          cluster_mappings[clusters[idx]] = clusters[first];
         } else {
           ++bin_info[bin_id].num_combine_failures;
         }
@@ -1106,6 +1122,60 @@ static int32_t GetCombineCostFactor(int histo_size, int quality) {
   return combine_cost_factor;
 }
 
+// Given a HistogramSet 'set', the mapping of clusters 'cluster_mapping' and the
+// current assignment of the cells in 'symbols', merge the clusters and
+// assign the smallest possible clusters values.
+static void OptimizeHistogramSymbols(const VP8LHistogramSet* const set,
+                                     uint16_t* const cluster_mappings,
+                                     uint32_t num_clusters,
+                                     uint16_t* const cluster_mappings_tmp,
+                                     uint32_t* const symbols) {
+  uint32_t i, cluster_max;
+  int do_continue = 1;
+  // First, assign the lowest cluster to each pixel.
+  while (do_continue) {
+    do_continue = 0;
+    for (i = 0; i < num_clusters; ++i) {
+      int k;
+      k = cluster_mappings[i];
+      while (k != cluster_mappings[k]) {
+        cluster_mappings[k] = cluster_mappings[cluster_mappings[k]];
+        k = cluster_mappings[k];
+      }
+      if (k != cluster_mappings[i]) {
+        do_continue = 1;
+        cluster_mappings[i] = k;
+      }
+    }
+  }
+  // Create a mapping from a cluster id to its minimal version.
+  cluster_max = 0;
+  memset(cluster_mappings_tmp, 0,
+         set->max_size * sizeof(*cluster_mappings_tmp));
+  assert(cluster_mappings[0] == 0);
+  // Re-map the ids.
+  for (i = 0; i < (uint32_t)set->max_size; ++i) {
+    int cluster;
+    if (symbols[i] == kInvalidHistogramSymbol) continue;
+    cluster = cluster_mappings[symbols[i]];
+    assert(symbols[i] < num_clusters);
+    if (cluster > 0 && cluster_mappings_tmp[cluster] == 0) {
+      ++cluster_max;
+      cluster_mappings_tmp[cluster] = cluster_max;
+    }
+    symbols[i] = cluster_mappings_tmp[cluster];
+  }
+
+  // Make sure all cluster values are used.
+  cluster_max = 0;
+  for (i = 0; i < (uint32_t)set->max_size; ++i) {
+    if (symbols[i] == kInvalidHistogramSymbol) continue;
+    if (symbols[i] <= cluster_max) continue;
+    ++cluster_max;
+    assert(symbols[i] == cluster_max);
+  }
+}
+
 static void RemoveEmptyHistograms(VP8LHistogramSet* const image_histo) {
   uint32_t size;
   int i;
@@ -1136,8 +1206,11 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize,
   // maximum quality q==100 (to preserve the compression gains at that level).
   const int entropy_combine_num_bins = low_effort ? NUM_PARTITIONS : BIN_SIZE;
   int entropy_combine;
+  uint16_t* const map_tmp =
+      (uint16_t*)WebPSafeMalloc(2 * image_histo_raw_size, sizeof(*map_tmp));
+  uint16_t* const cluster_mappings = map_tmp + image_histo_raw_size;
   int num_used = image_histo_raw_size;
-  if (orig_histo == NULL) {
+  if (orig_histo == NULL || map_tmp == NULL) {
     WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY);
     goto Error;
   }
@@ -1146,19 +1219,25 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize,
   HistogramBuild(xsize, histogram_bits, refs, orig_histo);
   // Copies the histograms and computes its bit_cost.
   // histogram_symbols is optimized
-  HistogramCopyAndAnalyze(orig_histo, image_histo, &num_used);
+  HistogramCopyAndAnalyze(orig_histo, image_histo, &num_used,
+                          histogram_symbols);
+
   entropy_combine =
       (num_used > entropy_combine_num_bins * 2) && (quality < 100);
 
   if (entropy_combine) {
+    uint16_t* const bin_map = map_tmp;
     const int32_t combine_cost_factor =
         GetCombineCostFactor(image_histo_raw_size, quality);
+    const uint32_t num_clusters = num_used;
 
-    HistogramAnalyzeEntropyBin(image_histo, low_effort);
+    HistogramAnalyzeEntropyBin(image_histo, bin_map, low_effort);
     // Collapse histograms with similar entropy.
-    HistogramCombineEntropyBin(image_histo, &num_used, tmp_histo,
+    HistogramCombineEntropyBin(
-                               entropy_combine_num_bins, combine_cost_factor,
+        image_histo, &num_used, histogram_symbols, cluster_mappings, tmp_histo,
-                               low_effort);
+        bin_map, entropy_combine_num_bins, combine_cost_factor, low_effort);
+    OptimizeHistogramSymbols(image_histo, cluster_mappings, num_clusters,
+                             map_tmp, histogram_symbols);
   }
 
   // Don't combine the histograms using stochastic and greedy heuristics for
@@ -1193,5 +1272,6 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize,
 
  Error:
   VP8LFreeHistogramSet(orig_histo);
+  WebPSafeFree(map_tmp);
   return (pic->error_code == VP8_ENC_OK);
 }

src/enc/histogram_enc.h

@@ -44,7 +44,6 @@ typedef struct {
   uint64_t red_cost;        // literal, red & blue.
   uint64_t blue_cost;
   uint8_t is_used[5];       // 5 for literal, red, blue, alpha, distance
-  uint16_t bin_id;          // entropy bin index.
 } VP8LHistogram;
 
 // Collection of histograms with fixed capacity, allocated as one