clapper: Cleanup cached harvests periodically

src/lib/clapper/clapper-cache-private.h

@@ -27,6 +27,9 @@ G_BEGIN_DECLS
 G_GNUC_INTERNAL
 void clapper_cache_initialize (void);
 
+G_GNUC_INTERNAL
+gboolean clapper_cache_is_disabled (void);
+
 G_GNUC_INTERNAL
 GMappedFile * clapper_cache_open (const gchar *filename, const gchar **data, GError **error);
 

src/lib/clapper/clapper-cache.c

@@ -45,6 +45,12 @@ clapper_cache_initialize (void)
   }
 }
 
+gboolean
+clapper_cache_is_disabled (void)
+{
+  return cache_disabled;
+}
+
 GMappedFile *
 clapper_cache_open (const gchar *filename, const gchar **data, GError **error)
 {

src/lib/clapper/gst/clapper-enhancer-director.c

@@ -17,12 +17,17 @@
  * Boston, MA 02110-1301, USA.
  */
 
+#include "config.h"
+
 #include <gst/gst.h>
 
 #include "clapper-enhancer-director-private.h"
+#include "../clapper-basic-functions.h"
+#include "../clapper-cache-private.h"
 #include "../clapper-enhancer-proxy-private.h"
 #include "../clapper-extractable-private.h"
 #include "../clapper-harvest-private.h"
+#include "../clapper-utils.h"
 #include "../../shared/clapper-shared-utils-private.h"
 
 #include "../clapper-functionalities-availability.h"
@@ -31,6 +36,8 @@
 #include "../clapper-enhancers-loader-private.h"
 #endif
 
+#define CLEANUP_INTERVAL 10800 // once every 3 hours
+
 #define GST_CAT_DEFAULT clapper_enhancer_director_debug
 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
 
@@ -131,6 +138,171 @@ clapper_enhancer_director_extract_in_thread (ClapperEnhancerDirectorData *data)
   return harvest;
 }
 
+static inline void
+_harvest_delete_if_expired (ClapperEnhancerDirector *self,
+    ClapperEnhancerProxy *proxy, GFile *file, const gint64 epoch_now)
+{
+  GMappedFile *mapped_file;
+  const gchar *data;
+  gchar *filename;
+  GError *error = NULL;
+  gboolean delete = TRUE;
+
+  filename = g_file_get_path (file);
+
+  if ((mapped_file = clapper_cache_open (filename, &data, &error))) {
+    /* Do not delete if versions match and not expired */
+    if (g_strcmp0 (clapper_cache_read_string (&data),
+        clapper_enhancer_proxy_get_version (proxy)) == 0
+        && clapper_cache_read_int64 (&data) > epoch_now) {
+      delete = FALSE;
+    }
+    g_mapped_file_unref (mapped_file);
+  } else if (error) {
+    if (error->domain == G_FILE_ERROR && error->code == G_FILE_ERROR_NOENT)
+      GST_DEBUG_OBJECT (self, "No cached harvest file found");
+    else
+      GST_ERROR_OBJECT (self, "Could not read cached harvest file, reason: %s", error->message);
+
+    g_clear_error (&error);
+  }
+
+  if (delete) {
+    if (G_LIKELY (g_file_delete (file, NULL, &error))) {
+      GST_TRACE_OBJECT (self, "Deleted cached harvest: \"%s\"", filename);
+    } else {
+      GST_ERROR_OBJECT (self, "Could not delete harvest: \"%s\", reason: %s",
+          filename, GST_STR_NULL (error->message));
+      g_error_free (error);
+    }
+  }
+
+  g_free (filename);
+}
+
+static inline void
+_cache_proxy_harvests_cleanup (ClapperEnhancerDirector *self,
+    ClapperEnhancerProxy *proxy, const gint64 epoch_now)
+{
+  GFile *dir;
+  GFileEnumerator *dir_enum;
+  GError *error = NULL;
+
+  dir = g_file_new_build_filename (g_get_user_cache_dir (), CLAPPER_API_NAME,
+      "enhancers", clapper_enhancer_proxy_get_module_name (proxy),
+      "harvests", NULL);
+
+  if ((dir_enum = g_file_enumerate_children (dir,
+      G_FILE_ATTRIBUTE_STANDARD_NAME "," G_FILE_ATTRIBUTE_STANDARD_TYPE,
+      G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL, &error))) {
+    while (TRUE) {
+      GFileInfo *info = NULL;
+      GFile *child = NULL;
+
+      if (!g_file_enumerator_iterate (dir_enum, &info,
+          &child, NULL, &error) || !info)
+        break;
+
+      if (G_LIKELY (g_file_info_get_file_type (info) == G_FILE_TYPE_REGULAR
+          && g_str_has_suffix (g_file_info_get_name (info), ".bin")))
+        _harvest_delete_if_expired (self, proxy, child, epoch_now);
+    }
+
+    g_object_unref (dir_enum);
+  }
+
+  if (error) {
+    if (error->domain != G_IO_ERROR || error->code != G_IO_ERROR_NOT_FOUND) {
+      gchar *path = g_file_get_path (dir);
+
+      GST_ERROR_OBJECT (self, "Could not cleanup in dir: \"%s\", reason: %s",
+          path, GST_STR_NULL (error->message));
+      g_free (path);
+    }
+
+    g_error_free (error);
+  }
+
+  g_object_unref (dir);
+}
+
+static gboolean
+_cache_cleanup_func (ClapperEnhancerDirector *self)
+{
+  GMappedFile *mapped_file;
+  GDateTime *date;
+  GError *error = NULL;
+  gchar *filename;
+  const gchar *data;
+  gint64 since_cleanup, epoch_now, epoch_last = 0;
+
+  date = g_date_time_new_now_utc ();
+  epoch_now = g_date_time_to_unix (date);
+  g_date_time_unref (date);
+
+  filename = g_build_filename (g_get_user_cache_dir (), CLAPPER_API_NAME,
+      "enhancers", "cleanup.bin", NULL);
+
+  if ((mapped_file = clapper_cache_open (filename, &data, &error))) {
+    epoch_last = clapper_cache_read_int64 (&data);
+    g_mapped_file_unref (mapped_file);
+  } else if (error) {
+    if (error->domain == G_FILE_ERROR && error->code == G_FILE_ERROR_NOENT)
+      GST_DEBUG_OBJECT (self, "No cache cleanup file found");
+    else
+      GST_ERROR_OBJECT (self, "Could not read cache cleanup file, reason: %s", error->message);
+
+    g_clear_error (&error);
+  }
+
+  since_cleanup = epoch_now - epoch_last;
+
+  if (since_cleanup >= CLEANUP_INTERVAL) {
+    ClapperEnhancerProxyList *proxies;
+    guint i, n_proxies;
+    GByteArray *bytes;
+
+    GST_TRACE_OBJECT (self, "Time for cache cleanup, last was %"
+        CLAPPER_TIME_FORMAT " ago", CLAPPER_TIME_ARGS (since_cleanup));
+
+    /* Start with writing to cache cleanup time,
+     * so other directors can find it earlier */
+    if ((bytes = clapper_cache_create ())) {
+      clapper_cache_store_int64 (bytes, epoch_now);
+
+      if (clapper_cache_write (filename, bytes, &error)) {
+        GST_TRACE_OBJECT (self, "Written data to cache cleanup file, cleanup time: %"
+            G_GINT64_FORMAT, epoch_now);
+      } else if (error) {
+        GST_ERROR_OBJECT (self, "Could not write cache cleanup data, reason: %s", error->message);
+        g_clear_error (&error);
+      }
+
+      g_byte_array_free (bytes, TRUE);
+    }
+
+    /* Now do cleanup */
+    proxies = clapper_get_global_enhancer_proxies ();
+    n_proxies = clapper_enhancer_proxy_list_get_n_proxies (proxies);
+
+    for (i = 0; i < n_proxies; ++i) {
+      ClapperEnhancerProxy *proxy = clapper_enhancer_proxy_list_peek_proxy (proxies, i);
+
+      if (!clapper_enhancer_proxy_target_has_interface (proxy, CLAPPER_TYPE_EXTRACTABLE))
+        continue;
+
+      _cache_proxy_harvests_cleanup (self, proxy, epoch_now);
+    }
+  } else {
+    GST_TRACE_OBJECT (self, "No cache cleanup yet, last was %"
+        CLAPPER_TIME_FORMAT " ago", CLAPPER_TIME_ARGS (since_cleanup));
+  }
+
+  g_free (filename);
+
+  return G_SOURCE_REMOVE;
+}
+
 /*
  * clapper_enhancer_director_new:
  *
@@ -153,6 +325,8 @@ clapper_enhancer_director_extract (ClapperEnhancerDirector *self,
     GCancellable *cancellable, GError **error)
 {
   ClapperEnhancerDirectorData *data = g_new (ClapperEnhancerDirectorData, 1);
+  GMainContext *context;
+  ClapperHarvest *harvest;
 
   data->director = self;
   data->filtered_proxies = filtered_proxies;
@@ -160,10 +334,18 @@ clapper_enhancer_director_extract (ClapperEnhancerDirector *self,
   data->cancellable = cancellable;
   data->error = error;
 
-  return CLAPPER_HARVEST_CAST (clapper_shared_utils_context_invoke_sync_full (
+  context = clapper_threaded_object_get_context (CLAPPER_THREADED_OBJECT_CAST (self));
-      clapper_threaded_object_get_context (CLAPPER_THREADED_OBJECT_CAST (self)),
+
+  harvest = CLAPPER_HARVEST_CAST (clapper_shared_utils_context_invoke_sync_full (context,
       (GThreadFunc) clapper_enhancer_director_extract_in_thread,
       data, (GDestroyNotify) g_free));
+
+  /* Run cleanup async. Since context belongs to "self", do not ref it.
+   * This ensures clean shutdown with thread stop function called. */
+  if (!g_cancellable_is_cancelled (cancellable) && !clapper_cache_is_disabled ())
+    g_main_context_invoke (context, (GSourceFunc) _cache_cleanup_func, self);
+
+  return harvest;
 }
 
 static void