clapper/lib/gst/plugin/gstclapperpaintable.c

/*
 * Copyright (C) 2022 Rafał Dzięgiel <rafostar.github@gmail.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include "gstclapperpaintable.h"
#include "gstclappergdkmemory.h"
#include "gstgtkutils.h"

#define DEFAULT_PAR_N               1
#define DEFAULT_PAR_D               1

#define GST_CAT_DEFAULT gst_clapper_paintable_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);

typedef struct
{
  GdkTexture *texture;
  GstVideoOverlayRectangle *rectangle;

  gint x, y;
  guint width, height;

  gboolean used;
} GstClapperGdkOverlay;

static void
gst_clapper_gdk_overlay_free (GstClapperGdkOverlay *overlay)
{
  GST_TRACE ("Freeing overlay: %" GST_PTR_FORMAT, overlay);

  g_object_unref (overlay->texture);
  gst_video_overlay_rectangle_unref (overlay->rectangle);
  g_slice_free (GstClapperGdkOverlay, overlay);
}

static void gst_clapper_paintable_iface_init (GdkPaintableInterface *iface);
static void gst_clapper_paintable_dispose (GObject *object);
static void gst_clapper_paintable_finalize (GObject *object);

#define parent_class gst_clapper_paintable_parent_class
G_DEFINE_TYPE_WITH_CODE (GstClapperPaintable, gst_clapper_paintable, G_TYPE_OBJECT,
    G_IMPLEMENT_INTERFACE (GDK_TYPE_PAINTABLE,
        gst_clapper_paintable_iface_init));

static void
gst_clapper_paintable_class_init (GstClapperPaintableClass *klass)
{
  GObjectClass *gobject_class = (GObjectClass *) klass;

  GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clapperpaintable", 0,
      "Clapper Paintable");

  gobject_class->dispose = gst_clapper_paintable_dispose;
  gobject_class->finalize = gst_clapper_paintable_finalize;
}

static void
gst_clapper_paintable_init (GstClapperPaintable *self)
{
  self->par_n = DEFAULT_PAR_N;
  self->par_d = DEFAULT_PAR_D;

  g_mutex_init (&self->lock);
  gst_video_info_init (&self->v_info);
  g_weak_ref_init (&self->widget, NULL);

  self->overlays = g_ptr_array_new_with_free_func (
      (GDestroyNotify) gst_clapper_gdk_overlay_free);

  gdk_rgba_parse (&self->bg, "black");
}

static void
gst_clapper_paintable_dispose (GObject *object)
{
  GstClapperPaintable *self = GST_CLAPPER_PAINTABLE (object);

  GST_CLAPPER_PAINTABLE_LOCK (self);

  if (self->draw_id > 0) {
    g_source_remove (self->draw_id);
    self->draw_id = 0;
  }

  GST_CLAPPER_PAINTABLE_UNLOCK (self);

  GST_CALL_PARENT (G_OBJECT_CLASS, dispose, (object));
}

static void
gst_clapper_paintable_finalize (GObject *object)
{
  GstClapperPaintable *self = GST_CLAPPER_PAINTABLE (object);

  GST_TRACE ("Finalize");

  GST_CLAPPER_PAINTABLE_LOCK (self);

  g_weak_ref_clear (&self->widget);

  gst_clear_buffer (&self->pending_buffer);
  gst_clear_buffer (&self->buffer);

  g_ptr_array_unref (self->overlays);

  GST_CLAPPER_PAINTABLE_UNLOCK (self);

  g_mutex_clear (&self->lock);

  GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object));
}

static gboolean
calculate_display_par (GstClapperPaintable *self, GstVideoInfo *info)
{
  gint width, height, par_n, par_d, req_par_n, req_par_d;
  gboolean success;

  width = GST_VIDEO_INFO_WIDTH (info);
  height = GST_VIDEO_INFO_HEIGHT (info);

  /* Cannot apply aspect ratio if there is no video */
  if (width == 0 || height == 0)
    return FALSE;

  par_n = GST_VIDEO_INFO_PAR_N (info);
  par_d = GST_VIDEO_INFO_PAR_D (info);

  req_par_n = self->par_n;
  req_par_d = self->par_d;

  if (par_n == 0)
    par_n = 1;

  /* Use defaults if user set zero */
  if (req_par_n == 0 || req_par_d == 0) {
    req_par_n = DEFAULT_PAR_N;
    req_par_d = DEFAULT_PAR_D;
  }

  GST_LOG_OBJECT (self, "PAR: %u/%u, DAR: %u/%u", par_n, par_d, req_par_n, req_par_d);

  if (!(success = gst_video_calculate_display_ratio (&self->display_ratio_num,
      &self->display_ratio_den, width, height, par_n, par_d,
      req_par_n, req_par_d))) {
    GST_ERROR_OBJECT (self, "Could not calculate display ratio values");
  }

  return success;
}

static void
invalidate_paintable_size_internal (GstClapperPaintable *self)
{
  gint video_width, video_height;
  guint display_ratio_num, display_ratio_den;

  GST_CLAPPER_PAINTABLE_LOCK (self);

  video_width = GST_VIDEO_INFO_WIDTH (&self->v_info);
  video_height = GST_VIDEO_INFO_HEIGHT (&self->v_info);

  display_ratio_num = self->display_ratio_num;
  display_ratio_den = self->display_ratio_den;

  GST_CLAPPER_PAINTABLE_UNLOCK (self);

  if (video_height % display_ratio_den == 0) {
    GST_LOG ("Keeping video height");

    self->display_width = (guint)
        gst_util_uint64_scale_int (video_height, display_ratio_num, display_ratio_den);
    self->display_height = video_height;
  } else if (video_width % display_ratio_num == 0) {
    GST_LOG ("Keeping video width");

    self->display_width = video_width;
    self->display_height = (guint)
        gst_util_uint64_scale_int (video_width, display_ratio_den, display_ratio_num);
  } else {
    GST_LOG ("Approximating while keeping video height");

    self->display_width = (guint)
        gst_util_uint64_scale_int (video_height, display_ratio_num, display_ratio_den);
    self->display_height = video_height;
  }

  self->display_aspect_ratio = ((gdouble) self->display_width
      / (gdouble) self->display_height);

  GST_DEBUG_OBJECT (self, "Invalidate paintable size, display: %dx%d",
      self->display_width, self->display_height);
  gdk_paintable_invalidate_size ((GdkPaintable *) self);
}

static gboolean
invalidate_paintable_size_on_main_cb (GstClapperPaintable *self)
{
  GST_CLAPPER_PAINTABLE_LOCK (self);
  self->draw_id = 0;
  GST_CLAPPER_PAINTABLE_UNLOCK (self);

  invalidate_paintable_size_internal (self);

  return G_SOURCE_REMOVE;
}

static void
comp_frame_unmap_and_free (GstVideoFrame *frame)
{
  gst_video_frame_unmap (frame);
  g_slice_free (GstVideoFrame, frame);
}

static GstClapperGdkOverlay *
_get_cached_overlay (GPtrArray *overlays, GstVideoOverlayRectangle *rectangle, guint *index)
{
  guint i;

  for (i = 0; i < overlays->len; i++) {
    GstClapperGdkOverlay *overlay = g_ptr_array_index (overlays, i);

    if (overlay->rectangle != rectangle)
      continue;

    *index = i;
    return overlay;
  }

  return NULL;
}

static void
gst_clapper_paintable_prepare_overlays (GstClapperPaintable *self)
{
  GstVideoOverlayCompositionMeta *comp_meta;
  guint num_overlays, i;

  /* As long as this is called from main thread, no need to lock here */
  if (G_UNLIKELY (!self->buffer)
      || !(comp_meta = gst_buffer_get_video_overlay_composition_meta (self->buffer))) {
    guint n_pending = self->overlays->len;

    /* Remove all cached overlays if new buffer does not have any */
    if (n_pending > 0) {
      GST_TRACE ("No overlays in buffer, removing all cached ones");
      g_ptr_array_remove_range (self->overlays, 0, n_pending);
    }

    return;
  }

  GST_LOG_OBJECT (self, "Preparing overlays...");

  /* Mark all old overlays as unused */
  for (i = 0; i < self->overlays->len; i++) {
    GstClapperGdkOverlay *overlay = g_ptr_array_index (self->overlays, i);
    overlay->used = FALSE;
  }

  num_overlays = gst_video_overlay_composition_n_rectangles (comp_meta->overlay);

  for (i = 0; i < num_overlays; i++) {
    GdkTexture *texture;
    GstBuffer *comp_buffer;
    GstVideoFrame *comp_frame;
    GstVideoMeta *vmeta;
    GstVideoInfo vinfo;
    GstVideoOverlayRectangle *rectangle;
    GstClapperGdkOverlay *overlay;
    GstVideoOverlayFormatFlags flags, alpha_flags = 0;
    gint comp_x, comp_y;
    guint comp_width, comp_height, cached_index;

    rectangle = gst_video_overlay_composition_get_rectangle (comp_meta->overlay, i);

    if ((overlay = _get_cached_overlay (self->overlays, rectangle, &cached_index))) {
      overlay->used = TRUE;

      /* Place overlay at expected position */
      if (i != cached_index) {
        GST_LOG ("Rearranging overlay position: %u => %u", cached_index, i);

        overlay = g_ptr_array_steal_index_fast (self->overlays, cached_index);
        g_ptr_array_insert (self->overlays, i, overlay);
      }

      GST_TRACE ("Reusing cached overlay: %" GST_PTR_FORMAT, overlay);
      continue;
    }

    if (G_UNLIKELY (!gst_video_overlay_rectangle_get_render_rectangle (rectangle,
        &comp_x, &comp_y, &comp_width, &comp_height))) {
      GST_WARNING ("Invalid overlay rectangle dimensions: %" GST_PTR_FORMAT, rectangle);
      continue;
    }

    flags = gst_video_overlay_rectangle_get_flags (rectangle);

    if (flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA)
      alpha_flags |= GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA;

    comp_buffer = gst_video_overlay_rectangle_get_pixels_unscaled_argb (rectangle, alpha_flags);
    comp_frame = g_slice_new (GstVideoFrame);

    /* Update overlay video info from video meta */
    if ((vmeta = gst_buffer_get_video_meta (comp_buffer))) {
      gst_video_info_set_format (&vinfo, vmeta->format, vmeta->width, vmeta->height);
      vinfo.stride[0] = vmeta->stride[0];
    }

    if (G_UNLIKELY (!gst_video_frame_map (comp_frame, &vinfo, comp_buffer, GST_MAP_READ))) {
      g_slice_free (GstVideoFrame, comp_frame);
      return;
    }

    if ((texture = gst_video_frame_into_gdk_texture (
        comp_frame, (GDestroyNotify) comp_frame_unmap_and_free))) {
      overlay = g_slice_new (GstClapperGdkOverlay);
      overlay->texture = texture;
      overlay->rectangle = gst_video_overlay_rectangle_ref (rectangle);
      overlay->x = comp_x;
      overlay->y = comp_y;
      overlay->width = comp_width;
      overlay->height = comp_height;
      overlay->used = TRUE;

      GST_TRACE ("Created overlay: %"
          GST_PTR_FORMAT ", x: %i, y: %i, width: %u, height: %u",
          overlay, overlay->x, overlay->y, overlay->width, overlay->height);

      g_ptr_array_insert (self->overlays, i, overlay);
    }
  }

  /* Remove all overlays that are not going to be used */
  for (i = self->overlays->len; i > 0; i--) {
    GstClapperGdkOverlay *overlay = g_ptr_array_index (self->overlays, i - 1);

    if (!overlay->used) {
      GST_TRACE ("Removing unused cached overlay: %" GST_PTR_FORMAT, overlay);
      g_ptr_array_remove (self->overlays, overlay);
    }
  }

  if (G_UNLIKELY (num_overlays != self->overlays->len)) {
    GST_WARNING_OBJECT (self, "Some overlays could not be prepared, %u != %u",
        num_overlays, self->overlays->len);
  }

  GST_LOG_OBJECT (self, "Prepared overlays: %u", self->overlays->len);
}

static gboolean
update_paintable_on_main_cb (GstClapperPaintable *self)
{
  gboolean size_changed;

  GST_CLAPPER_PAINTABLE_LOCK (self);

  /* Check if we will need to invalidate size */
  if ((size_changed = self->pending_resize))
    self->pending_resize = FALSE;

  gst_clear_buffer (&self->buffer);
  self->buffer = self->pending_buffer;
  self->pending_buffer = NULL;

  self->draw_id = 0;

  GST_CLAPPER_PAINTABLE_UNLOCK (self);

  gst_clapper_paintable_prepare_overlays (self);

  if (size_changed)
    invalidate_paintable_size_internal (self);

  GST_LOG_OBJECT (self, "Invalidate paintable contents");
  gdk_paintable_invalidate_contents ((GdkPaintable *) self);

  return G_SOURCE_REMOVE;
}

GstClapperPaintable *
gst_clapper_paintable_new (void)
{
  return g_object_new (GST_TYPE_CLAPPER_PAINTABLE, NULL);
}

void
gst_clapper_paintable_set_widget (GstClapperPaintable *self, GtkWidget *widget)
{
  g_weak_ref_set (&self->widget, widget);
}

void
gst_clapper_paintable_set_buffer (GstClapperPaintable *self, GstBuffer *buffer)
{
  GST_CLAPPER_PAINTABLE_LOCK (self);

  if (self->draw_id > 0) {
    GST_CLAPPER_PAINTABLE_UNLOCK (self);
    GST_TRACE ("Already have pending buffer, skipping %" GST_PTR_FORMAT, buffer);

    return;
  }

  gst_buffer_replace (&self->pending_buffer, buffer);
  self->draw_id = g_idle_add_full (G_PRIORITY_DEFAULT,
      (GSourceFunc) update_paintable_on_main_cb, self, NULL);

  GST_CLAPPER_PAINTABLE_UNLOCK (self);
}

gboolean
gst_clapper_paintable_set_video_info (GstClapperPaintable *self, GstVideoInfo *v_info)
{
  GST_CLAPPER_PAINTABLE_LOCK (self);

  if (gst_video_info_is_equal (&self->v_info, v_info)) {
    GST_CLAPPER_PAINTABLE_UNLOCK (self);
    return TRUE;
  }

  /* Reject info if values would cause integer overflow */
  if (G_UNLIKELY (!calculate_display_par (self, v_info))) {
    GST_CLAPPER_PAINTABLE_UNLOCK (self);
    return FALSE;
  }

  self->pending_resize = TRUE;
  self->v_info = *v_info;

  GST_CLAPPER_PAINTABLE_UNLOCK (self);

  return TRUE;
}

void
gst_clapper_paintable_set_pixel_aspect_ratio (GstClapperPaintable *self,
    gint par_n, gint par_d)
{
  gboolean success;

  GST_CLAPPER_PAINTABLE_LOCK (self);

  /* No change */
  if (self->par_n == par_n && self->par_d == par_d) {
    GST_CLAPPER_PAINTABLE_UNLOCK (self);
    return;
  }

  self->par_n = par_n;
  self->par_d = par_d;

  /* Check if we can accept new values. This will update
   * display `ratio_num` and `ratio_den` only when successful */
  success = calculate_display_par (self, &self->v_info);

  /* If paintable update is queued, wait for it, otherwise invalidate
   * size only for change to be applied even when paused */
  if (!success || self->draw_id > 0) {
    GST_CLAPPER_PAINTABLE_UNLOCK (self);
    return;
  }

  self->draw_id = g_idle_add_full (G_PRIORITY_DEFAULT,
      (GSourceFunc) invalidate_paintable_size_on_main_cb, self, NULL);

  GST_CLAPPER_PAINTABLE_UNLOCK (self);

  return;
}

/*
 * GdkPaintableInterface
 */
static void
gst_clapper_paintable_snapshot_internal (GstClapperPaintable *self,
    GdkSnapshot *snapshot, gdouble width, gdouble height,
    gint widget_width, gint widget_height)
{
  GstMemory *memory;
  GstMapInfo info;
  gfloat scale_x, scale_y;
  guint i;

  scale_x = (gfloat) width / self->display_width;
  scale_y = (gfloat) height / self->display_height;

  /* Apply black borders when keeping aspect ratio */
  if (scale_x == scale_y || abs (scale_x - scale_y) <= FLT_EPSILON) {
    if (widget_height - height > 0) {
      gdouble bar_height = (widget_height - height) / 2;

      gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, -bar_height));
      gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, height, width, bar_height + 0.5));
    } else if (widget_width - width > 0) {
      gdouble bar_width = (widget_width - width) / 2;

      gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, -bar_width, height));
      gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (width, 0, bar_width + 0.5, height));
    }
  }

  /* Buffer is accessed only from main thread, so no locking required */
  if (!self->buffer) {
    gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, height));
    return;
  }

  GST_TRACE ("Snapshot %" GST_PTR_FORMAT, self->buffer);

  memory = gst_buffer_peek_memory (self->buffer, 0);

  /* If we cannot map, just draw black */
  if (G_UNLIKELY (!gst_memory_map (memory, &info, GST_MAP_READ))) {
    gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, height));
    GST_WARNING_OBJECT (self, "Could not map %" GST_PTR_FORMAT, self->buffer);

    return;
  }

  gtk_snapshot_append_texture (snapshot,
      GST_CLAPPER_GDK_MEMORY_CAST (memory)->texture,
      &GRAPHENE_RECT_INIT (0, 0, width, height));
  gst_memory_unmap (memory, &info);

  /* FIXME: Draw black BG here when import format has-alpha */
  //gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, height));

  /* Finally append prepared overlays */
  for (i = 0; i < self->overlays->len; i++) {
    GstClapperGdkOverlay *overlay = g_ptr_array_index (self->overlays, i);

    gtk_snapshot_append_texture (snapshot, overlay->texture,
        &GRAPHENE_RECT_INIT (overlay->x * scale_x, overlay->y * scale_y,
            overlay->width * scale_x, overlay->height * scale_y));
  }
}

static void
gst_clapper_paintable_snapshot (GdkPaintable *paintable,
    GdkSnapshot *snapshot, gdouble width, gdouble height)
{
  GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable);
  GtkWidget *widget;
  gint widget_width = 0, widget_height = 0;

  if ((widget = g_weak_ref_get (&self->widget))) {
    widget_width = gtk_widget_get_width (widget);
    widget_height = gtk_widget_get_height (widget);

    g_object_unref (widget);
  }

  gst_clapper_paintable_snapshot_internal (self, snapshot,
      width, height, widget_width, widget_height);
}

static GdkPaintable *
gst_clapper_paintable_get_current_image (GdkPaintable *paintable)
{
  GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable);

  if (self->buffer) {
    GtkSnapshot *snapshot;
    GdkPaintable *ret;

    snapshot = gtk_snapshot_new ();

    /* Snapshot without widget size in order to get
     * paintable without black borders */
    gst_clapper_paintable_snapshot_internal (self, snapshot,
        self->display_width, self->display_height, 0, 0);

    if ((ret = gtk_snapshot_free_to_paintable (snapshot, NULL)))
      return ret;
  }

  return gdk_paintable_new_empty (0, 0);
}

static gint
gst_clapper_paintable_get_intrinsic_width (GdkPaintable *paintable)
{
  GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable);

  return self->display_width;
}

static gint
gst_clapper_paintable_get_intrinsic_height (GdkPaintable *paintable)
{
  GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable);

  return self->display_height;
}

static gdouble
gst_clapper_paintable_get_intrinsic_aspect_ratio (GdkPaintable *paintable)
{
  GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable);

  return self->display_aspect_ratio;
}

static void
gst_clapper_paintable_iface_init (GdkPaintableInterface *iface)
{
  iface->snapshot = gst_clapper_paintable_snapshot;
  iface->get_current_image = gst_clapper_paintable_get_current_image;
  iface->get_intrinsic_width = gst_clapper_paintable_get_intrinsic_width;
  iface->get_intrinsic_height = gst_clapper_paintable_get_intrinsic_height;
  iface->get_intrinsic_aspect_ratio = gst_clapper_paintable_get_intrinsic_aspect_ratio;
}