/* * Copyright (C) 2022 Rafał Dzięgiel * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include "gstclapperimporter.h" #include "gstgtkutils.h" #define GST_CAT_DEFAULT gst_clapper_importer_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); #define parent_class gst_clapper_importer_parent_class G_DEFINE_TYPE (GstClapperImporter, gst_clapper_importer, GST_TYPE_OBJECT); typedef struct { GdkTexture *texture; GstVideoOverlayRectangle *rectangle; gint x, y; guint width, height; gint index; gatomicrefcount ref_count; } GstClapperGdkOverlay; static GstClapperGdkOverlay * gst_clapper_gdk_overlay_new (GdkTexture *texture, GstVideoOverlayRectangle *rectangle, gint x, gint y, guint width, guint height, guint index) { GstClapperGdkOverlay *overlay = g_slice_new (GstClapperGdkOverlay); overlay->texture = g_object_ref (texture); overlay->rectangle = gst_video_overlay_rectangle_ref (rectangle); overlay->x = x; overlay->y = y; overlay->width = width; overlay->height = height; overlay->index = index; g_atomic_ref_count_init (&overlay->ref_count); return overlay; } static GstClapperGdkOverlay * gst_clapper_gdk_overlay_ref (GstClapperGdkOverlay *overlay) { g_atomic_ref_count_inc (&overlay->ref_count); return overlay; } static void gst_clapper_gdk_overlay_unref (GstClapperGdkOverlay *overlay) { if (g_atomic_ref_count_dec (&overlay->ref_count)) { GST_TRACE ("Freeing overlay: %" GST_PTR_FORMAT, overlay); g_object_unref (overlay->texture); gst_video_overlay_rectangle_unref (overlay->rectangle); g_slice_free (GstClapperGdkOverlay, overlay); } } static GstBufferPool * _default_create_pool (GstClapperImporter *self, GstStructure **config) { GST_FIXME_OBJECT (self, "Need to create buffer pool"); return NULL; } static GdkTexture * _default_generate_texture (GstClapperImporter *self, GstBuffer *buffer, GstVideoInfo *v_info) { GST_FIXME_OBJECT (self, "GdkTexture generation not implemented"); return NULL; } static void gst_clapper_importer_init (GstClapperImporter *self) { gst_video_info_init (&self->pending_v_info); gst_video_info_init (&self->v_info); self->pending_overlays = g_ptr_array_new_with_free_func ( (GDestroyNotify) gst_clapper_gdk_overlay_unref); self->overlays = g_ptr_array_new_with_free_func ( (GDestroyNotify) gst_clapper_gdk_overlay_unref); gdk_rgba_parse (&self->bg, "black"); } static void gst_clapper_importer_finalize (GObject *object) { GstClapperImporter *self = GST_CLAPPER_IMPORTER_CAST (object); GST_TRACE ("Finalize"); gst_clear_buffer (&self->pending_buffer); gst_clear_buffer (&self->buffer); g_ptr_array_unref (self->pending_overlays); g_ptr_array_unref (self->overlays); g_clear_object (&self->texture); GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object)); } static void gst_clapper_importer_class_init (GstClapperImporterClass *klass) { GObjectClass *gobject_class = (GObjectClass *) klass; GstClapperImporterClass *importer_class = (GstClapperImporterClass *) klass; GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clapperimporter", 0, "Clapper Importer"); gobject_class->finalize = gst_clapper_importer_finalize; importer_class->create_pool = _default_create_pool; importer_class->generate_texture = _default_generate_texture; } static GstClapperGdkOverlay * _get_cached_overlay (GPtrArray *overlays, GstVideoOverlayRectangle *rectangle) { guint i; for (i = 0; i < overlays->len; i++) { GstClapperGdkOverlay *overlay = g_ptr_array_index (overlays, i); if (overlay->rectangle == rectangle) return overlay; } return NULL; } static gint _sort_overlays_cb (gconstpointer a, gconstpointer b) { GstClapperGdkOverlay *overlay_a, *overlay_b; overlay_a = *((GstClapperGdkOverlay **) a); overlay_b = *((GstClapperGdkOverlay **) b); return (overlay_a->index - overlay_b->index); } /* * Prepares overlays to show with the next rendered buffer. * * In order for overlays caching to work correctly, this should be called for * every received buffer (even if its going to be disgarded), also must be * called together with pending buffer replacement within a single importer * locking, to make sure prepared overlays always match the pending buffer. */ static void gst_clapper_importer_prepare_overlays_locked (GstClapperImporter *self) { GstVideoOverlayCompositionMeta *comp_meta; guint num_overlays, i; if (G_UNLIKELY (!self->pending_buffer) || !(comp_meta = gst_buffer_get_video_overlay_composition_meta (self->pending_buffer))) { guint n_pending = self->pending_overlays->len; /* Remove all cached overlays if new buffer does not have any */ if (n_pending > 0) { GST_TRACE ("No overlays in buffer, removing all cached ones"); g_ptr_array_remove_range (self->pending_overlays, 0, n_pending); } return; } GST_LOG_OBJECT (self, "Preparing overlays..."); /* Mark all old overlays as unused by giving them negative index */ for (i = 0; i < self->pending_overlays->len; i++) { GstClapperGdkOverlay *overlay = g_ptr_array_index (self->pending_overlays, i); overlay->index = -1; } num_overlays = gst_video_overlay_composition_n_rectangles (comp_meta->overlay); for (i = 0; i < num_overlays; i++) { GdkTexture *texture; GstBuffer *comp_buffer; GstVideoFrame comp_frame; GstVideoMeta *v_meta; GstVideoInfo v_info; GstVideoOverlayRectangle *rectangle; GstClapperGdkOverlay *overlay; GstVideoOverlayFormatFlags flags, alpha_flags = 0; gint comp_x, comp_y; guint comp_width, comp_height; rectangle = gst_video_overlay_composition_get_rectangle (comp_meta->overlay, i); if ((overlay = _get_cached_overlay (self->pending_overlays, rectangle))) { overlay->index = i; GST_TRACE ("Reusing cached overlay: %" GST_PTR_FORMAT, overlay); continue; } if (G_UNLIKELY (!gst_video_overlay_rectangle_get_render_rectangle (rectangle, &comp_x, &comp_y, &comp_width, &comp_height))) { GST_WARNING ("Invalid overlay rectangle dimensions: %" GST_PTR_FORMAT, rectangle); continue; } flags = gst_video_overlay_rectangle_get_flags (rectangle); if (flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA) alpha_flags |= GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA; comp_buffer = gst_video_overlay_rectangle_get_pixels_unscaled_argb (rectangle, alpha_flags); /* Update overlay video info from video meta */ if ((v_meta = gst_buffer_get_video_meta (comp_buffer))) { gst_video_info_set_format (&v_info, v_meta->format, v_meta->width, v_meta->height); v_info.stride[0] = v_meta->stride[0]; } if (G_UNLIKELY (!gst_video_frame_map (&comp_frame, &v_info, comp_buffer, GST_MAP_READ))) return; if ((texture = gst_video_frame_into_gdk_texture (&comp_frame))) { overlay = gst_clapper_gdk_overlay_new (texture, rectangle, comp_x, comp_y, comp_width, comp_height, i); g_object_unref (texture); GST_TRACE_OBJECT (self, "Created overlay: %" GST_PTR_FORMAT ", x: %i, y: %i, width: %u, height: %u", overlay, overlay->x, overlay->y, overlay->width, overlay->height); g_ptr_array_insert (self->pending_overlays, i, overlay); } gst_video_frame_unmap (&comp_frame); } /* Remove all overlays that are not going to be used */ for (i = self->pending_overlays->len; i > 0; i--) { GstClapperGdkOverlay *overlay = g_ptr_array_index (self->pending_overlays, i - 1); if (overlay->index < 0) { GST_TRACE ("Removing unused cached overlay: %" GST_PTR_FORMAT, overlay); g_ptr_array_remove (self->pending_overlays, overlay); } } /* Sort remaining overlays */ if (self->pending_overlays->len > 1) { GST_LOG_OBJECT (self, "Sorting overlays"); g_ptr_array_sort (self->pending_overlays, (GCompareFunc) _sort_overlays_cb); } if (G_UNLIKELY (num_overlays != self->pending_overlays->len)) { GST_WARNING_OBJECT (self, "Some overlays could not be prepared, %u != %u", num_overlays, self->pending_overlays->len); } GST_LOG_OBJECT (self, "Prepared overlays: %u", self->pending_overlays->len); } gboolean gst_clapper_importer_prepare (GstClapperImporter *self) { GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); if (importer_class->prepare) { if (!importer_class->prepare (self)) return FALSE; } GST_DEBUG_OBJECT (self, "Importer prepared"); return TRUE; } void gst_clapper_importer_share_data (GstClapperImporter *self, GstClapperImporter *dest) { GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); if (importer_class->share_data) importer_class->share_data (self, dest); } void gst_clapper_importer_set_caps (GstClapperImporter *self, GstCaps *caps) { GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); GST_OBJECT_LOCK (self); self->has_pending_v_info = gst_video_info_from_caps (&self->pending_v_info, caps); GST_OBJECT_UNLOCK (self); if (importer_class->set_caps) importer_class->set_caps (self, caps); } void gst_clapper_importer_set_buffer (GstClapperImporter *self, GstBuffer *buffer) { /* Both overlays and pending buffer must be * set within a single importer locking */ GST_OBJECT_LOCK (self); gst_buffer_replace (&self->pending_buffer, buffer); gst_clapper_importer_prepare_overlays_locked (self); GST_OBJECT_UNLOCK (self); } GstBufferPool * gst_clapper_importer_create_pool (GstClapperImporter *self, GstStructure **config) { GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); return importer_class->create_pool (self, config); } void gst_clapper_importer_add_allocation_metas (GstClapperImporter *self, GstQuery *query) { GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); if (importer_class->add_allocation_metas) importer_class->add_allocation_metas (self, query); } gboolean gst_clapper_importer_handle_context_query (GstClapperImporter *self, GstBaseSink *bsink, GstQuery *query) { GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); if (!importer_class->handle_context_query) return FALSE; return importer_class->handle_context_query (self, bsink, query); } void gst_clapper_importer_snapshot (GstClapperImporter *self, GdkSnapshot *snapshot, gdouble width, gdouble height, gfloat scale_x, gfloat scale_y) { guint i; gboolean buffer_changed; /* Collect all data that we need to snapshot pending buffer, * lock ourselves to make sure everything matches */ GST_OBJECT_LOCK (self); buffer_changed = gst_buffer_replace (&self->buffer, self->pending_buffer); /* Only replace v_info when buffer changed, this way * we still use old (correct) v_info when resizing */ if (buffer_changed && self->has_pending_v_info) { self->v_info = self->pending_v_info; self->has_pending_v_info = FALSE; } /* Ref overlays associated with current buffer */ for (i = 0; i < self->pending_overlays->len; i++) { GstClapperGdkOverlay *overlay = g_ptr_array_index (self->pending_overlays, i); g_ptr_array_insert (self->overlays, i, gst_clapper_gdk_overlay_ref (overlay)); } GST_OBJECT_UNLOCK (self); /* Draw black BG when no buffer or imported format has alpha */ if (!self->buffer || GST_VIDEO_INFO_HAS_ALPHA (&self->v_info)) gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, height)); if (self->buffer) { if (buffer_changed || !self->texture) { GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); GST_TRACE_OBJECT (self, "Importing %" GST_PTR_FORMAT, self->buffer); g_clear_object (&self->texture); self->texture = importer_class->generate_texture (self, self->buffer, &self->v_info); } else { GST_TRACE_OBJECT (self, "Reusing texture from %" GST_PTR_FORMAT, self->buffer); } if (G_LIKELY (self->texture)) { gtk_snapshot_append_texture (snapshot, self->texture, &GRAPHENE_RECT_INIT (0, 0, width, height)); for (i = 0; i < self->overlays->len; i++) { GstClapperGdkOverlay *overlay = g_ptr_array_index (self->overlays, i); gtk_snapshot_append_texture (snapshot, overlay->texture, &GRAPHENE_RECT_INIT (overlay->x * scale_x, overlay->y * scale_y, overlay->width * scale_x, overlay->height * scale_y)); } } else { GST_ERROR_OBJECT (self, "Failed import of %" GST_PTR_FORMAT, self->buffer); /* Draw black instead of texture on failure if not drawn already */ if (!GST_VIDEO_INFO_HAS_ALPHA (&self->v_info)) gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, height)); } } /* Unref all used overlays */ if (self->overlays->len > 0) g_ptr_array_remove_range (self->overlays, 0, self->overlays->len); }