From 044710f97e684063aeea76014088117e681ee459 Mon Sep 17 00:00:00 2001 From: Rafostar <40623528+Rafostar@users.noreply.github.com> Date: Sun, 1 May 2022 16:48:23 +0200 Subject: [PATCH 1/6] plugin: Add clapper GStreamer plugin Add new GStreamer plugin with custom video sink for Clapper video player. The main difference is that unlike the old one, this does not operate using `GtkGLArea` anymore, but processes and displays `GdkTextures` directly through `GtkPicture` widget. Also this one is installed like any other GStreamer plugin, thus can be used via `gst-launch-1.0` binary or even by other GTK4 apps if they wish to integrate it. In order to not depend on GL stuff at build time, this plugin uses seperate GModules called "importers" in order to import different kind of memories into GdkTexture. This allows expanding its capabilities further then we were able to do before. --- lib/gst/meson.build | 8 +- lib/gst/plugin/gstclapperimporter.c | 439 +++++++++++ lib/gst/plugin/gstclapperimporter.h | 102 +++ lib/gst/plugin/gstclapperimporterloader.c | 416 ++++++++++ lib/gst/plugin/gstclapperimporterloader.h | 36 + lib/gst/plugin/gstclapperpaintable.c | 428 ++++++++++ lib/gst/plugin/gstclapperpaintable.h | 78 ++ lib/gst/plugin/gstclappersink.c | 910 ++++++++++++++++++++++ lib/gst/plugin/gstclappersink.h | 72 ++ lib/gst/plugin/gstgdkformats.h | 37 + lib/gst/plugin/gstgtkutils.c | 132 ++++ lib/gst/plugin/gstgtkutils.h | 37 + lib/gst/plugin/gstplugin.c | 43 + lib/gst/plugin/meson.build | 70 ++ lib/meson.build | 24 +- meson.build | 4 +- meson_options.txt | 6 + 17 files changed, 2824 insertions(+), 18 deletions(-) create mode 100644 lib/gst/plugin/gstclapperimporter.c create mode 100644 lib/gst/plugin/gstclapperimporter.h create mode 100644 lib/gst/plugin/gstclapperimporterloader.c create mode 100644 lib/gst/plugin/gstclapperimporterloader.h create mode 100644 lib/gst/plugin/gstclapperpaintable.c create mode 100644 lib/gst/plugin/gstclapperpaintable.h create mode 100644 lib/gst/plugin/gstclappersink.c create mode 100644 lib/gst/plugin/gstclappersink.h create mode 100644 lib/gst/plugin/gstgdkformats.h create mode 100644 lib/gst/plugin/gstgtkutils.c create mode 100644 lib/gst/plugin/gstgtkutils.h create mode 100644 lib/gst/plugin/gstplugin.c create mode 100644 lib/gst/plugin/meson.build diff --git a/lib/gst/meson.build b/lib/gst/meson.build index 90cf6581..7311e7be 100644 --- a/lib/gst/meson.build +++ b/lib/gst/meson.build @@ -1 +1,7 @@ -subdir('clapper') +if get_option('lib') + subdir('clapper') +endif + +if not get_option('gst-plugin').disabled() + subdir('plugin') +endif diff --git a/lib/gst/plugin/gstclapperimporter.c b/lib/gst/plugin/gstclapperimporter.c new file mode 100644 index 00000000..9ee4fa5f --- /dev/null +++ b/lib/gst/plugin/gstclapperimporter.c @@ -0,0 +1,439 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstclapperimporter.h" +#include "gstgtkutils.h" + +#define GST_CAT_DEFAULT gst_clapper_importer_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +#define parent_class gst_clapper_importer_parent_class +G_DEFINE_TYPE (GstClapperImporter, gst_clapper_importer, GST_TYPE_OBJECT); + +typedef struct +{ + GdkTexture *texture; + GstVideoOverlayRectangle *rectangle; + + gint x, y; + guint width, height; + + gint index; + gatomicrefcount ref_count; +} GstClapperGdkOverlay; + +static GstClapperGdkOverlay * +gst_clapper_gdk_overlay_new (GdkTexture *texture, GstVideoOverlayRectangle *rectangle, + gint x, gint y, guint width, guint height, guint index) +{ + GstClapperGdkOverlay *overlay = g_slice_new (GstClapperGdkOverlay); + + overlay->texture = g_object_ref (texture); + overlay->rectangle = gst_video_overlay_rectangle_ref (rectangle); + overlay->x = x; + overlay->y = y; + overlay->width = width; + overlay->height = height; + overlay->index = index; + + g_atomic_ref_count_init (&overlay->ref_count); + + return overlay; +} + +static GstClapperGdkOverlay * +gst_clapper_gdk_overlay_ref (GstClapperGdkOverlay *overlay) +{ + g_atomic_ref_count_inc (&overlay->ref_count); + + return overlay; +} + +static void +gst_clapper_gdk_overlay_unref (GstClapperGdkOverlay *overlay) +{ + if (g_atomic_ref_count_dec (&overlay->ref_count)) { + GST_TRACE ("Freeing overlay: %" GST_PTR_FORMAT, overlay); + + g_object_unref (overlay->texture); + gst_video_overlay_rectangle_unref (overlay->rectangle); + g_slice_free (GstClapperGdkOverlay, overlay); + } +} + +static GstBufferPool * +_default_create_pool (GstClapperImporter *self, GstStructure **config) +{ + GST_FIXME_OBJECT (self, "Need to create buffer pool"); + + return NULL; +} + +static GdkTexture * +_default_generate_texture (GstClapperImporter *self, + GstBuffer *buffer, GstVideoInfo *v_info) +{ + GST_FIXME_OBJECT (self, "GdkTexture generation not implemented"); + + return NULL; +} + +static void +gst_clapper_importer_init (GstClapperImporter *self) +{ + gst_video_info_init (&self->pending_v_info); + gst_video_info_init (&self->v_info); + + self->pending_overlays = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_clapper_gdk_overlay_unref); + self->overlays = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_clapper_gdk_overlay_unref); + + gdk_rgba_parse (&self->bg, "black"); +} + +static void +gst_clapper_importer_finalize (GObject *object) +{ + GstClapperImporter *self = GST_CLAPPER_IMPORTER_CAST (object); + + GST_TRACE ("Finalize"); + + gst_clear_buffer (&self->pending_buffer); + gst_clear_buffer (&self->buffer); + + g_ptr_array_unref (self->pending_overlays); + g_ptr_array_unref (self->overlays); + + g_clear_object (&self->texture); + + GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object)); +} + +static void +gst_clapper_importer_class_init (GstClapperImporterClass *klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstClapperImporterClass *importer_class = (GstClapperImporterClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clapperimporter", 0, + "Clapper Importer"); + + gobject_class->finalize = gst_clapper_importer_finalize; + + importer_class->create_pool = _default_create_pool; + importer_class->generate_texture = _default_generate_texture; +} + +static GstClapperGdkOverlay * +_get_cached_overlay (GPtrArray *overlays, GstVideoOverlayRectangle *rectangle) +{ + guint i; + + for (i = 0; i < overlays->len; i++) { + GstClapperGdkOverlay *overlay = g_ptr_array_index (overlays, i); + + if (overlay->rectangle == rectangle) + return overlay; + } + + return NULL; +} + +static gint +_sort_overlays_cb (gconstpointer a, gconstpointer b) +{ + GstClapperGdkOverlay *overlay_a, *overlay_b; + + overlay_a = *((GstClapperGdkOverlay **) a); + overlay_b = *((GstClapperGdkOverlay **) b); + + return (overlay_a->index - overlay_b->index); +} + +/* + * Prepares overlays to show with the next rendered buffer. + * + * In order for overlays caching to work correctly, this should be called for + * every received buffer (even if its going to be disgarded), also must be + * called together with pending buffer replacement within a single importer + * locking, to make sure prepared overlays always match the pending buffer. + */ +static void +gst_clapper_importer_prepare_overlays_locked (GstClapperImporter *self) +{ + GstVideoOverlayCompositionMeta *comp_meta; + guint num_overlays, i; + + if (G_UNLIKELY (!self->pending_buffer) + || !(comp_meta = gst_buffer_get_video_overlay_composition_meta (self->pending_buffer))) { + guint n_pending = self->pending_overlays->len; + + /* Remove all cached overlays if new buffer does not have any */ + if (n_pending > 0) { + GST_TRACE ("No overlays in buffer, removing all cached ones"); + g_ptr_array_remove_range (self->pending_overlays, 0, n_pending); + } + + return; + } + + GST_LOG_OBJECT (self, "Preparing overlays..."); + + /* Mark all old overlays as unused by giving them negative index */ + for (i = 0; i < self->pending_overlays->len; i++) { + GstClapperGdkOverlay *overlay = g_ptr_array_index (self->pending_overlays, i); + overlay->index = -1; + } + + num_overlays = gst_video_overlay_composition_n_rectangles (comp_meta->overlay); + + for (i = 0; i < num_overlays; i++) { + GdkTexture *texture; + GstBuffer *comp_buffer; + GstVideoFrame comp_frame; + GstVideoMeta *v_meta; + GstVideoInfo v_info; + GstVideoOverlayRectangle *rectangle; + GstClapperGdkOverlay *overlay; + GstVideoOverlayFormatFlags flags, alpha_flags = 0; + gint comp_x, comp_y; + guint comp_width, comp_height; + + rectangle = gst_video_overlay_composition_get_rectangle (comp_meta->overlay, i); + + if ((overlay = _get_cached_overlay (self->pending_overlays, rectangle))) { + overlay->index = i; + + GST_TRACE ("Reusing cached overlay: %" GST_PTR_FORMAT, overlay); + continue; + } + + if (G_UNLIKELY (!gst_video_overlay_rectangle_get_render_rectangle (rectangle, + &comp_x, &comp_y, &comp_width, &comp_height))) { + GST_WARNING ("Invalid overlay rectangle dimensions: %" GST_PTR_FORMAT, rectangle); + continue; + } + + flags = gst_video_overlay_rectangle_get_flags (rectangle); + + if (flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA) + alpha_flags |= GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA; + + comp_buffer = gst_video_overlay_rectangle_get_pixels_unscaled_argb (rectangle, alpha_flags); + + /* Update overlay video info from video meta */ + if ((v_meta = gst_buffer_get_video_meta (comp_buffer))) { + gst_video_info_set_format (&v_info, v_meta->format, v_meta->width, v_meta->height); + v_info.stride[0] = v_meta->stride[0]; + } + + if (G_UNLIKELY (!gst_video_frame_map (&comp_frame, &v_info, comp_buffer, GST_MAP_READ))) + return; + + if ((texture = gst_video_frame_into_gdk_texture (&comp_frame))) { + overlay = gst_clapper_gdk_overlay_new (texture, rectangle, comp_x, comp_y, + comp_width, comp_height, i); + g_object_unref (texture); + + GST_TRACE_OBJECT (self, "Created overlay: %" + GST_PTR_FORMAT ", x: %i, y: %i, width: %u, height: %u", + overlay, overlay->x, overlay->y, overlay->width, overlay->height); + + g_ptr_array_insert (self->pending_overlays, i, overlay); + } + + gst_video_frame_unmap (&comp_frame); + } + + /* Remove all overlays that are not going to be used */ + for (i = self->pending_overlays->len; i > 0; i--) { + GstClapperGdkOverlay *overlay = g_ptr_array_index (self->pending_overlays, i - 1); + + if (overlay->index < 0) { + GST_TRACE ("Removing unused cached overlay: %" GST_PTR_FORMAT, overlay); + g_ptr_array_remove (self->pending_overlays, overlay); + } + } + + /* Sort remaining overlays */ + if (self->pending_overlays->len > 1) { + GST_LOG_OBJECT (self, "Sorting overlays"); + g_ptr_array_sort (self->pending_overlays, (GCompareFunc) _sort_overlays_cb); + } + + if (G_UNLIKELY (num_overlays != self->pending_overlays->len)) { + GST_WARNING_OBJECT (self, "Some overlays could not be prepared, %u != %u", + num_overlays, self->pending_overlays->len); + } + + GST_LOG_OBJECT (self, "Prepared overlays: %u", self->pending_overlays->len); +} + +gboolean +gst_clapper_importer_prepare (GstClapperImporter *self) +{ + GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); + + if (importer_class->prepare) { + if (!importer_class->prepare (self)) + return FALSE; + } + + GST_DEBUG_OBJECT (self, "Importer prepared"); + + return TRUE; +} + +void +gst_clapper_importer_share_data (GstClapperImporter *self, GstClapperImporter *dest) +{ + GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); + + if (importer_class->share_data) + importer_class->share_data (self, dest); +} + +void +gst_clapper_importer_set_caps (GstClapperImporter *self, GstCaps *caps) +{ + GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); + + GST_OBJECT_LOCK (self); + self->has_pending_v_info = gst_video_info_from_caps (&self->pending_v_info, caps); + GST_OBJECT_UNLOCK (self); + + if (importer_class->set_caps) + importer_class->set_caps (self, caps); +} + +void +gst_clapper_importer_set_buffer (GstClapperImporter *self, GstBuffer *buffer) +{ + /* Both overlays and pending buffer must be + * set within a single importer locking */ + GST_OBJECT_LOCK (self); + + gst_buffer_replace (&self->pending_buffer, buffer); + gst_clapper_importer_prepare_overlays_locked (self); + + GST_OBJECT_UNLOCK (self); +} + +GstBufferPool * +gst_clapper_importer_create_pool (GstClapperImporter *self, GstStructure **config) +{ + GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); + + return importer_class->create_pool (self, config); +} + +void +gst_clapper_importer_add_allocation_metas (GstClapperImporter *self, GstQuery *query) +{ + GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); + + if (importer_class->add_allocation_metas) + importer_class->add_allocation_metas (self, query); +} + +gboolean +gst_clapper_importer_handle_context_query (GstClapperImporter *self, + GstBaseSink *bsink, GstQuery *query) +{ + GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); + + if (!importer_class->handle_context_query) + return FALSE; + + return importer_class->handle_context_query (self, bsink, query); +} + +void +gst_clapper_importer_snapshot (GstClapperImporter *self, GdkSnapshot *snapshot, + gdouble width, gdouble height, gfloat scale_x, gfloat scale_y) +{ + guint i; + gboolean buffer_changed; + + /* Collect all data that we need to snapshot pending buffer, + * lock ourselves to make sure everything matches */ + GST_OBJECT_LOCK (self); + + buffer_changed = gst_buffer_replace (&self->buffer, self->pending_buffer); + + /* Only replace v_info when buffer changed, this way + * we still use old (correct) v_info when resizing */ + if (buffer_changed && self->has_pending_v_info) { + self->v_info = self->pending_v_info; + self->has_pending_v_info = FALSE; + } + + /* Ref overlays associated with current buffer */ + for (i = 0; i < self->pending_overlays->len; i++) { + GstClapperGdkOverlay *overlay = g_ptr_array_index (self->pending_overlays, i); + + g_ptr_array_insert (self->overlays, i, gst_clapper_gdk_overlay_ref (overlay)); + } + + GST_OBJECT_UNLOCK (self); + + /* Draw black BG when no buffer or imported format has alpha */ + if (!self->buffer || GST_VIDEO_INFO_HAS_ALPHA (&self->v_info)) + gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, height)); + + if (self->buffer) { + if (buffer_changed || !self->texture) { + GstClapperImporterClass *importer_class = GST_CLAPPER_IMPORTER_GET_CLASS (self); + + GST_TRACE_OBJECT (self, "Importing %" GST_PTR_FORMAT, self->buffer); + + g_clear_object (&self->texture); + self->texture = importer_class->generate_texture (self, self->buffer, &self->v_info); + } else { + GST_TRACE_OBJECT (self, "Reusing texture from %" GST_PTR_FORMAT, self->buffer); + } + + if (G_LIKELY (self->texture)) { + gtk_snapshot_append_texture (snapshot, self->texture, &GRAPHENE_RECT_INIT (0, 0, width, height)); + + for (i = 0; i < self->overlays->len; i++) { + GstClapperGdkOverlay *overlay = g_ptr_array_index (self->overlays, i); + + gtk_snapshot_append_texture (snapshot, overlay->texture, + &GRAPHENE_RECT_INIT (overlay->x * scale_x, overlay->y * scale_y, + overlay->width * scale_x, overlay->height * scale_y)); + } + } else { + GST_ERROR_OBJECT (self, "Failed import of %" GST_PTR_FORMAT, self->buffer); + + /* Draw black instead of texture on failure if not drawn already */ + if (!GST_VIDEO_INFO_HAS_ALPHA (&self->v_info)) + gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, height)); + } + } + + /* Unref all used overlays */ + if (self->overlays->len > 0) + g_ptr_array_remove_range (self->overlays, 0, self->overlays->len); +} diff --git a/lib/gst/plugin/gstclapperimporter.h b/lib/gst/plugin/gstclapperimporter.h new file mode 100644 index 00000000..6477f441 --- /dev/null +++ b/lib/gst/plugin/gstclapperimporter.h @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_CLAPPER_IMPORTER (gst_clapper_importer_get_type()) +#define GST_IS_CLAPPER_IMPORTER(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_CLAPPER_IMPORTER)) +#define GST_IS_CLAPPER_IMPORTER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_CLAPPER_IMPORTER)) +#define GST_CLAPPER_IMPORTER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_CLAPPER_IMPORTER, GstClapperImporterClass)) +#define GST_CLAPPER_IMPORTER(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_CLAPPER_IMPORTER, GstClapperImporter)) +#define GST_CLAPPER_IMPORTER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_CLAPPER_IMPORTER, GstClapperImporterClass)) +#define GST_CLAPPER_IMPORTER_CAST(obj) ((GstClapperImporter *)(obj)) + +#define GST_CLAPPER_IMPORTER_DEFINE(camel,lower,type) \ +G_DEFINE_TYPE (camel, lower, type) \ +G_MODULE_EXPORT GstClapperImporter *make_importer (void); \ +G_MODULE_EXPORT GstCaps *make_caps (GstRank *rank, GStrv *context_types); + +typedef struct _GstClapperImporter GstClapperImporter; +typedef struct _GstClapperImporterClass GstClapperImporterClass; + +#ifdef G_DEFINE_AUTOPTR_CLEANUP_FUNC +G_DEFINE_AUTOPTR_CLEANUP_FUNC (GstClapperImporter, gst_object_unref) +#endif + +struct _GstClapperImporter +{ + GstObject parent; + + GstBuffer *pending_buffer, *buffer; + GPtrArray *pending_overlays, *overlays; + GstVideoInfo pending_v_info, v_info; + gboolean has_pending_v_info; + + GdkTexture *texture; + + GdkRGBA bg; +}; + +struct _GstClapperImporterClass +{ + GstObjectClass parent_class; + + gboolean (* prepare) (GstClapperImporter *importer); + + void (* share_data) (GstClapperImporter *src, + GstClapperImporter *dest); + + void (* set_caps) (GstClapperImporter *importer, + GstCaps *caps); + + gboolean (* handle_context_query) (GstClapperImporter *importer, + GstBaseSink *bsink, + GstQuery *query); + + GstBufferPool * (* create_pool) (GstClapperImporter *importer, + GstStructure **config); + + void (* add_allocation_metas) (GstClapperImporter *importer, + GstQuery *query); + + GdkTexture * (* generate_texture) (GstClapperImporter *importer, + GstBuffer *buffer, + GstVideoInfo *v_info); +}; + +GType gst_clapper_importer_get_type (void); + +gboolean gst_clapper_importer_prepare (GstClapperImporter *importer); +void gst_clapper_importer_share_data (GstClapperImporter *importer, GstClapperImporter *dest); +gboolean gst_clapper_importer_handle_context_query (GstClapperImporter *importer, GstBaseSink *bsink, GstQuery *query); +GstBufferPool * gst_clapper_importer_create_pool (GstClapperImporter *importer, GstStructure **config); +void gst_clapper_importer_add_allocation_metas (GstClapperImporter *importer, GstQuery *query); + +void gst_clapper_importer_set_caps (GstClapperImporter *importer, GstCaps *caps); +void gst_clapper_importer_set_buffer (GstClapperImporter *importer, GstBuffer *buffer); + +void gst_clapper_importer_snapshot (GstClapperImporter *importer, GdkSnapshot *snapshot, gdouble width, gdouble height, gfloat scale_x, gfloat scale_y); + +G_END_DECLS diff --git a/lib/gst/plugin/gstclapperimporterloader.c b/lib/gst/plugin/gstclapperimporterloader.c new file mode 100644 index 00000000..c9eb99a8 --- /dev/null +++ b/lib/gst/plugin/gstclapperimporterloader.c @@ -0,0 +1,416 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "gstclapperimporterloader.h" +#include "gstclapperimporter.h" + +#define GST_CAT_DEFAULT gst_clapper_importer_loader_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +typedef GstClapperImporter* (* MakeImporter) (void); +typedef GstCaps* (* MakeCaps) (GstRank *rank, GStrv *context_types); + +typedef struct +{ + gchar *module_path; + GModule *open_module; + GstCaps *caps; + GstRank rank; + GStrv context_types; +} GstClapperImporterData; + +static void +gst_clapper_importer_data_free (GstClapperImporterData *data) +{ + g_free (data->module_path); + + if (data->open_module) + g_module_close (data->open_module); + + gst_clear_caps (&data->caps); + g_strfreev (data->context_types); + g_free (data); +} + +static gboolean +_open_importer (GstClapperImporterData *data) +{ + g_return_val_if_fail (data && data->module_path, FALSE); + + /* Already open */ + if (data->open_module) + return TRUE; + + GST_DEBUG ("Opening module: %s", data->module_path); + data->open_module = g_module_open (data->module_path, G_MODULE_BIND_LAZY); + + if (!data->open_module) { + GST_WARNING ("Could not load importer: %s, reason: %s", + data->module_path, g_module_error ()); + return FALSE; + } + GST_DEBUG ("Opened importer module"); + + /* Make sure module stays loaded. Seems to be needed for + * reusing exported symbols from the same module again */ + g_module_make_resident (data->open_module); + + return TRUE; +} + +static void +_close_importer (GstClapperImporterData *data) +{ + if (!data || !data->open_module) + return; + + if (G_LIKELY (g_module_close (data->open_module))) + GST_DEBUG ("Closed module: %s", data->module_path); + else + GST_WARNING ("Could not close importer module"); + + data->open_module = NULL; +} + +static GstClapperImporter * +_obtain_importer_internal (GstClapperImporterData *data) +{ + MakeImporter make_importer; + GstClapperImporter *importer = NULL; + + if (!_open_importer (data)) + goto finish; + + if (!g_module_symbol (data->open_module, "make_importer", (gpointer *) &make_importer) + || make_importer == NULL) { + GST_WARNING ("Make function missing in importer"); + goto fail; + } + + /* Do not close the module, we are gonna continue using it */ + if ((importer = make_importer ())) + goto finish; + +fail: + _close_importer (data); + +finish: + return importer; +} + +static GstClapperImporterData * +_fill_importer_data (const gchar *module_path) +{ + MakeCaps make_caps; + GstClapperImporterData *data; + + data = g_new0 (GstClapperImporterData, 1); + data->module_path = g_strdup (module_path); + data->open_module = g_module_open (data->module_path, G_MODULE_BIND_LAZY); + + if (!data->open_module) + goto fail; + + if (!g_module_symbol (data->open_module, "make_caps", (gpointer *) &make_caps) + || make_caps == NULL) { + GST_WARNING ("Make caps function missing in importer"); + goto fail; + } + + data->caps = make_caps (&data->rank, &data->context_types); + GST_DEBUG ("Caps reading %ssuccessful", data->caps ? "" : "un"); + + if (!data->caps) + goto fail; + + /* Once we obtain importer data, close module afterwards */ + _close_importer (data); + + return data; + +fail: + gst_clapper_importer_data_free (data); + + return NULL; +} + +static gint +_sort_importers_cb (gconstpointer a, gconstpointer b) +{ + GstClapperImporterData *data_a, *data_b; + + data_a = *((GstClapperImporterData **) a); + data_b = *((GstClapperImporterData **) b); + + return (data_b->rank - data_a->rank); +} + +static gpointer +_obtain_available_importers (G_GNUC_UNUSED gpointer data) +{ + GPtrArray *importers; + GFile *dir; + GFileEnumerator *dir_enum; + GError *error = NULL; + + importers = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_clapper_importer_data_free); + + GST_INFO ("Checking available clapper sink importers"); + + dir = g_file_new_for_path (CLAPPER_SINK_IMPORTER_PATH); + + if ((dir_enum = g_file_enumerate_children (dir, + G_FILE_ATTRIBUTE_STANDARD_NAME, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL, &error))) { + while (TRUE) { + GFileInfo *info = NULL; + GstClapperImporterData *data; + gchar *module_path; + const gchar *module_name; + + if (!g_file_enumerator_iterate (dir_enum, &info, + NULL, NULL, &error) || !info) + break; + + module_name = g_file_info_get_name (info); + + if (!g_str_has_suffix (module_name, G_MODULE_SUFFIX)) + continue; + + module_path = g_module_build_path (CLAPPER_SINK_IMPORTER_PATH, module_name); + data = _fill_importer_data (module_path); + g_free (module_path); + + if (!data) { + GST_WARNING ("Could not read importer data: %s", module_name); + continue; + } + + GST_INFO ("Found importer: %s, caps: %" GST_PTR_FORMAT, module_name, data->caps); + g_ptr_array_add (importers, data); + } + } + + g_object_unref (dir); + g_object_unref (dir_enum); + + if (error) { + GST_ERROR ("Could not load importer, reason: %s", + (error->message) ? error->message : "unknown"); + g_error_free (error); + } + + g_ptr_array_sort (importers, (GCompareFunc) _sort_importers_cb); + + return importers; +} + +static const GPtrArray * +gst_clapper_importer_loader_get_available_importers (void) +{ + static GOnce once = G_ONCE_INIT; + + g_once (&once, _obtain_available_importers, NULL); + return (const GPtrArray *) once.retval; +} + +static GstClapperImporterData * +_find_open_importer_data (const GPtrArray *importers) +{ + guint i; + + for (i = 0; i < importers->len; i++) { + GstClapperImporterData *data = g_ptr_array_index (importers, i); + + if (data->open_module) + return data; + } + + return NULL; +} + +static GstClapperImporterData * +_get_importer_data_for_caps (const GPtrArray *importers, const GstCaps *caps) +{ + guint i; + + for (i = 0; i < importers->len; i++) { + GstClapperImporterData *data = g_ptr_array_index (importers, i); + + if (!gst_caps_is_always_compatible (caps, data->caps)) + continue; + + return data; + } + + return NULL; +} + +static GstClapperImporterData * +_get_importer_data_for_context_type (const GPtrArray *importers, const gchar *context_type) +{ + guint i; + + for (i = 0; i < importers->len; i++) { + GstClapperImporterData *data = g_ptr_array_index (importers, i); + guint j; + + if (!data->context_types) + continue; + + for (j = 0; data->context_types[j]; j++) { + if (strcmp (context_type, data->context_types[j])) + continue; + + return data; + } + } + + return NULL; +} + +void +gst_clapper_importer_loader_unload_all (void) +{ + const GPtrArray *importers; + guint i; + + importers = gst_clapper_importer_loader_get_available_importers (); + GST_TRACE ("Unloading all open modules"); + + for (i = 0; i < importers->len; i++) { + GstClapperImporterData *data = g_ptr_array_index (importers, i); + + _close_importer (data); + } +} + +GstPadTemplate * +gst_clapper_importer_loader_make_sink_pad_template (void) +{ + const GPtrArray *importers; + GstCaps *sink_caps; + GstPadTemplate *templ; + guint i; + + /* This is only called once from sink class init function */ + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clapperimporterloader", 0, + "Clapper Importer Loader"); + + importers = gst_clapper_importer_loader_get_available_importers (); + sink_caps = gst_caps_new_empty (); + + for (i = 0; i < importers->len; i++) { + GstClapperImporterData *data = g_ptr_array_index (importers, i); + GstCaps *copied_caps; + + copied_caps = gst_caps_copy (data->caps); + gst_caps_append (sink_caps, copied_caps); + } + + if (G_UNLIKELY (gst_caps_is_empty (sink_caps))) + gst_caps_append (sink_caps, gst_caps_new_any ()); + + templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, sink_caps); + gst_caps_unref (sink_caps); + + return templ; +} + +static gboolean +_find_importer_internal (GstCaps *caps, GstQuery *query, GstClapperImporter **importer) +{ + const GPtrArray *importers; + GstClapperImporterData *old_data = NULL, *new_data = NULL; + GstClapperImporter *found_importer = NULL; + + importers = gst_clapper_importer_loader_get_available_importers (); + old_data = _find_open_importer_data (importers); + + if (caps) { + GST_DEBUG ("Requested importer for caps: %" GST_PTR_FORMAT, caps); + new_data = _get_importer_data_for_caps (importers, caps); + } else if (query) { + const gchar *context_type; + + gst_query_parse_context_type (query, &context_type); + + GST_DEBUG ("Requested importer for context: %s", context_type); + new_data = _get_importer_data_for_context_type (importers, context_type); + + /* In case missing importer for context query, leave the old one. + * We should allow some queries to go through unresponded */ + if (!new_data) + new_data = old_data; + } + GST_LOG ("Old importer path: %s, new path: %s", + (old_data != NULL) ? old_data->module_path : NULL, + (new_data != NULL) ? new_data->module_path : NULL); + + if (old_data == new_data) { + GST_DEBUG ("No importer change"); + + if (*importer && caps) + gst_clapper_importer_set_caps (*importer, caps); + + return (*importer != NULL); + } + + if (new_data) { + found_importer = _obtain_importer_internal (new_data); + + if (*importer && found_importer) + gst_clapper_importer_share_data (*importer, found_importer); + } + + gst_clear_object (importer); + _close_importer (old_data); + + if (found_importer && gst_clapper_importer_prepare (found_importer)) { + if (caps) + gst_clapper_importer_set_caps (found_importer, caps); + + *importer = found_importer; + return TRUE; + } + + gst_clear_object (&found_importer); + _close_importer (new_data); + + return FALSE; +} + +gboolean +gst_clapper_importer_loader_find_importer_for_caps (GstCaps *caps, GstClapperImporter **importer) +{ + return _find_importer_internal (caps, NULL, importer); +} + +gboolean +gst_clapper_importer_loader_find_importer_for_context_query (GstQuery *query, GstClapperImporter **importer) +{ + return _find_importer_internal (NULL, query, importer); +} diff --git a/lib/gst/plugin/gstclapperimporterloader.h b/lib/gst/plugin/gstclapperimporterloader.h new file mode 100644 index 00000000..0a596540 --- /dev/null +++ b/lib/gst/plugin/gstclapperimporterloader.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include + +#include "gstclapperimporter.h" + +G_BEGIN_DECLS + +GstPadTemplate * gst_clapper_importer_loader_make_sink_pad_template (void); + +gboolean gst_clapper_importer_loader_find_importer_for_caps (GstCaps *caps, GstClapperImporter **importer); + +gboolean gst_clapper_importer_loader_find_importer_for_context_query (GstQuery *query, GstClapperImporter **importer); + +void gst_clapper_importer_loader_unload_all (void); + +G_END_DECLS diff --git a/lib/gst/plugin/gstclapperpaintable.c b/lib/gst/plugin/gstclapperpaintable.c new file mode 100644 index 00000000..494e0275 --- /dev/null +++ b/lib/gst/plugin/gstclapperpaintable.c @@ -0,0 +1,428 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstclapperpaintable.h" + +#define DEFAULT_PAR_N 1 +#define DEFAULT_PAR_D 1 + +#define GST_CAT_DEFAULT gst_clapper_paintable_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +static void gst_clapper_paintable_iface_init (GdkPaintableInterface *iface); +static void gst_clapper_paintable_dispose (GObject *object); +static void gst_clapper_paintable_finalize (GObject *object); + +#define parent_class gst_clapper_paintable_parent_class +G_DEFINE_TYPE_WITH_CODE (GstClapperPaintable, gst_clapper_paintable, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (GDK_TYPE_PAINTABLE, + gst_clapper_paintable_iface_init)); + +static void +gst_clapper_paintable_class_init (GstClapperPaintableClass *klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clapperpaintable", 0, + "Clapper Paintable"); + + gobject_class->dispose = gst_clapper_paintable_dispose; + gobject_class->finalize = gst_clapper_paintable_finalize; +} + +static void +gst_clapper_paintable_init (GstClapperPaintable *self) +{ + self->par_n = DEFAULT_PAR_N; + self->par_d = DEFAULT_PAR_D; + self->pixel_aspect = ((gdouble) self->par_d / self->par_n); + + g_mutex_init (&self->lock); + gst_video_info_init (&self->v_info); + g_weak_ref_init (&self->widget, NULL); + g_weak_ref_init (&self->importer, NULL); + + gdk_rgba_parse (&self->bg, "black"); +} + +static void +gst_clapper_paintable_dispose (GObject *object) +{ + GstClapperPaintable *self = GST_CLAPPER_PAINTABLE (object); + + GST_CLAPPER_PAINTABLE_LOCK (self); + + if (self->draw_id > 0) { + g_source_remove (self->draw_id); + self->draw_id = 0; + } + + GST_CLAPPER_PAINTABLE_UNLOCK (self); + + GST_CALL_PARENT (G_OBJECT_CLASS, dispose, (object)); +} + +static void +gst_clapper_paintable_finalize (GObject *object) +{ + GstClapperPaintable *self = GST_CLAPPER_PAINTABLE (object); + + GST_TRACE ("Finalize"); + + g_weak_ref_clear (&self->widget); + g_weak_ref_clear (&self->importer); + g_mutex_clear (&self->lock); + + GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object)); +} + +static gboolean +calculate_display_par (GstClapperPaintable *self, const GstVideoInfo *info) +{ + gint width, height, par_n, par_d, req_par_n, req_par_d; + gboolean success; + + width = GST_VIDEO_INFO_WIDTH (info); + height = GST_VIDEO_INFO_HEIGHT (info); + + /* Cannot apply aspect ratio if there is no video */ + if (width == 0 || height == 0) + return FALSE; + + par_n = GST_VIDEO_INFO_PAR_N (info); + par_d = GST_VIDEO_INFO_PAR_D (info); + + req_par_n = self->par_n; + req_par_d = self->par_d; + + if (par_n == 0) + par_n = 1; + + /* Use defaults if user set zero */ + if (req_par_n == 0 || req_par_d == 0) { + req_par_n = DEFAULT_PAR_N; + req_par_d = DEFAULT_PAR_D; + } + + GST_LOG_OBJECT (self, "PAR: %u/%u, DAR: %u/%u", par_n, par_d, req_par_n, req_par_d); + + if (!(success = gst_video_calculate_display_ratio (&self->display_ratio_num, + &self->display_ratio_den, width, height, par_n, par_d, + req_par_n, req_par_d))) { + GST_ERROR_OBJECT (self, "Could not calculate display ratio values"); + } + + return success; +} + +static void +invalidate_paintable_size_internal (GstClapperPaintable *self) +{ + gint video_width, video_height; + guint display_ratio_num, display_ratio_den; + + GST_CLAPPER_PAINTABLE_LOCK (self); + + video_width = GST_VIDEO_INFO_WIDTH (&self->v_info); + video_height = GST_VIDEO_INFO_HEIGHT (&self->v_info); + + display_ratio_num = self->display_ratio_num; + display_ratio_den = self->display_ratio_den; + + self->pixel_aspect = ((gdouble) self->par_d / self->par_n); + + GST_CLAPPER_PAINTABLE_UNLOCK (self); + + if (video_height % display_ratio_den == 0) { + GST_LOG ("Keeping video height"); + + self->display_width = (guint) + gst_util_uint64_scale_int (video_height, display_ratio_num, display_ratio_den); + self->display_height = video_height; + } else if (video_width % display_ratio_num == 0) { + GST_LOG ("Keeping video width"); + + self->display_width = video_width; + self->display_height = (guint) + gst_util_uint64_scale_int (video_width, display_ratio_den, display_ratio_num); + } else { + GST_LOG ("Approximating while keeping video height"); + + self->display_width = (guint) + gst_util_uint64_scale_int (video_height, display_ratio_num, display_ratio_den); + self->display_height = video_height; + } + + self->display_aspect_ratio = ((gdouble) self->display_width + / (gdouble) self->display_height); + + GST_DEBUG_OBJECT (self, "Invalidate paintable size, display: %dx%d", + self->display_width, self->display_height); + gdk_paintable_invalidate_size ((GdkPaintable *) self); +} + +static gboolean +invalidate_paintable_size_on_main_cb (GstClapperPaintable *self) +{ + GST_CLAPPER_PAINTABLE_LOCK (self); + self->draw_id = 0; + GST_CLAPPER_PAINTABLE_UNLOCK (self); + + invalidate_paintable_size_internal (self); + + return G_SOURCE_REMOVE; +} + +static gboolean +update_paintable_on_main_cb (GstClapperPaintable *self) +{ + gboolean size_changed; + + GST_CLAPPER_PAINTABLE_LOCK (self); + + /* Check if we will need to invalidate size */ + if ((size_changed = self->pending_resize)) + self->pending_resize = FALSE; + + self->draw_id = 0; + + GST_CLAPPER_PAINTABLE_UNLOCK (self); + + if (size_changed) + invalidate_paintable_size_internal (self); + + GST_LOG_OBJECT (self, "Invalidate paintable contents"); + gdk_paintable_invalidate_contents ((GdkPaintable *) self); + + return G_SOURCE_REMOVE; +} + +GstClapperPaintable * +gst_clapper_paintable_new (void) +{ + return g_object_new (GST_TYPE_CLAPPER_PAINTABLE, NULL); +} + +void +gst_clapper_paintable_set_widget (GstClapperPaintable *self, GtkWidget *widget) +{ + g_weak_ref_set (&self->widget, widget); +} + +void +gst_clapper_paintable_set_importer (GstClapperPaintable *self, GstClapperImporter *importer) +{ + g_weak_ref_set (&self->importer, importer); +} + +void +gst_clapper_paintable_queue_draw (GstClapperPaintable *self) +{ + GST_CLAPPER_PAINTABLE_LOCK (self); + + if (self->draw_id > 0) { + GST_CLAPPER_PAINTABLE_UNLOCK (self); + GST_TRACE ("Already have pending draw"); + + return; + } + + self->draw_id = g_idle_add_full (G_PRIORITY_DEFAULT, + (GSourceFunc) update_paintable_on_main_cb, self, NULL); + + GST_CLAPPER_PAINTABLE_UNLOCK (self); +} + +gboolean +gst_clapper_paintable_set_video_info (GstClapperPaintable *self, const GstVideoInfo *v_info) +{ + GST_CLAPPER_PAINTABLE_LOCK (self); + + if (gst_video_info_is_equal (&self->v_info, v_info)) { + GST_CLAPPER_PAINTABLE_UNLOCK (self); + return TRUE; + } + + /* Reject info if values would cause integer overflow */ + if (G_UNLIKELY (!calculate_display_par (self, v_info))) { + GST_CLAPPER_PAINTABLE_UNLOCK (self); + return FALSE; + } + + self->pending_resize = TRUE; + self->v_info = *v_info; + + GST_CLAPPER_PAINTABLE_UNLOCK (self); + + return TRUE; +} + +void +gst_clapper_paintable_set_pixel_aspect_ratio (GstClapperPaintable *self, + gint par_n, gint par_d) +{ + gboolean success; + + GST_CLAPPER_PAINTABLE_LOCK (self); + + /* No change */ + if (self->par_n == par_n && self->par_d == par_d) { + GST_CLAPPER_PAINTABLE_UNLOCK (self); + return; + } + + self->par_n = par_n; + self->par_d = par_d; + + /* Check if we can accept new values. This will update + * display `ratio_num` and `ratio_den` only when successful */ + success = calculate_display_par (self, &self->v_info); + + /* If paintable update is queued, wait for it, otherwise invalidate + * size only for change to be applied even when paused */ + if (!success || self->draw_id > 0) { + self->pending_resize = success; + GST_CLAPPER_PAINTABLE_UNLOCK (self); + + return; + } + + self->draw_id = g_idle_add_full (G_PRIORITY_DEFAULT, + (GSourceFunc) invalidate_paintable_size_on_main_cb, self, NULL); + + GST_CLAPPER_PAINTABLE_UNLOCK (self); +} + +/* + * GdkPaintableInterface + */ +static void +gst_clapper_paintable_snapshot_internal (GstClapperPaintable *self, + GdkSnapshot *snapshot, gdouble width, gdouble height, + gint widget_width, gint widget_height) +{ + GstClapperImporter *importer; + gfloat scale_x, scale_y; + + GST_LOG_OBJECT (self, "Snapshot"); + + scale_x = (gfloat) width / self->display_width; + scale_y = (gfloat) height / self->display_height; + + /* Apply black borders when keeping aspect ratio */ + if (scale_x == scale_y || abs (scale_x - scale_y) <= FLT_EPSILON) { + if (widget_height - height > 0) { + /* XXX: Top uses integer to work with GTK rounding (not going offscreen) */ + gint top_bar_height = (widget_height - height) / 2; + gdouble bottom_bar_height = (widget_height - top_bar_height - height); + + gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, -top_bar_height)); + gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, height, width, bottom_bar_height)); + } else if (widget_width - width > 0) { + gint left_bar_width = (widget_width - width) / 2; + gdouble right_bar_width = (widget_width - left_bar_width - width); + + gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, -left_bar_width, height)); + gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (width, 0, right_bar_width, height)); + } + } + + if ((importer = g_weak_ref_get (&self->importer))) { + gst_clapper_importer_snapshot (importer, snapshot, width, height, + scale_x * self->pixel_aspect, scale_y); + g_object_unref (importer); + } else { + GST_LOG_OBJECT (self, "No texture importer, drawing black"); + gtk_snapshot_append_color (snapshot, &self->bg, &GRAPHENE_RECT_INIT (0, 0, width, height)); + } +} + +static void +gst_clapper_paintable_snapshot (GdkPaintable *paintable, + GdkSnapshot *snapshot, gdouble width, gdouble height) +{ + GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable); + GtkWidget *widget; + gint widget_width = 0, widget_height = 0; + + if ((widget = g_weak_ref_get (&self->widget))) { + gint scale_factor; + + scale_factor = gtk_widget_get_scale_factor (widget); + widget_width = gtk_widget_get_width (widget) * scale_factor; + widget_height = gtk_widget_get_height (widget) * scale_factor; + + g_object_unref (widget); + } + + gst_clapper_paintable_snapshot_internal (self, snapshot, + width, height, widget_width, widget_height); +} + +static GdkPaintable * +gst_clapper_paintable_get_current_image (GdkPaintable *paintable) +{ + GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable); + GtkSnapshot *snapshot = gtk_snapshot_new (); + + /* Snapshot without widget size in order to get + * paintable without black borders */ + gst_clapper_paintable_snapshot_internal (self, snapshot, + self->display_width, self->display_height, 0, 0); + + return gtk_snapshot_free_to_paintable (snapshot, NULL); +} + +static gint +gst_clapper_paintable_get_intrinsic_width (GdkPaintable *paintable) +{ + GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable); + + return self->display_width; +} + +static gint +gst_clapper_paintable_get_intrinsic_height (GdkPaintable *paintable) +{ + GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable); + + return self->display_height; +} + +static gdouble +gst_clapper_paintable_get_intrinsic_aspect_ratio (GdkPaintable *paintable) +{ + GstClapperPaintable *self = GST_CLAPPER_PAINTABLE_CAST (paintable); + + return self->display_aspect_ratio; +} + +static void +gst_clapper_paintable_iface_init (GdkPaintableInterface *iface) +{ + iface->snapshot = gst_clapper_paintable_snapshot; + iface->get_current_image = gst_clapper_paintable_get_current_image; + iface->get_intrinsic_width = gst_clapper_paintable_get_intrinsic_width; + iface->get_intrinsic_height = gst_clapper_paintable_get_intrinsic_height; + iface->get_intrinsic_aspect_ratio = gst_clapper_paintable_get_intrinsic_aspect_ratio; +} diff --git a/lib/gst/plugin/gstclapperpaintable.h b/lib/gst/plugin/gstclapperpaintable.h new file mode 100644 index 00000000..604e5e8e --- /dev/null +++ b/lib/gst/plugin/gstclapperpaintable.h @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include + +#include "gstclapperimporter.h" + +G_BEGIN_DECLS + +#define GST_TYPE_CLAPPER_PAINTABLE (gst_clapper_paintable_get_type()) +G_DECLARE_FINAL_TYPE (GstClapperPaintable, gst_clapper_paintable, GST, CLAPPER_PAINTABLE, GObject) + +#define GST_CLAPPER_PAINTABLE_CAST(obj) ((GstClapperPaintable *)(obj)) + +#define GST_CLAPPER_PAINTABLE_GET_LOCK(obj) (&GST_CLAPPER_PAINTABLE_CAST(obj)->lock) +#define GST_CLAPPER_PAINTABLE_LOCK(obj) g_mutex_lock (GST_CLAPPER_PAINTABLE_GET_LOCK(obj)) +#define GST_CLAPPER_PAINTABLE_UNLOCK(obj) g_mutex_unlock (GST_CLAPPER_PAINTABLE_GET_LOCK(obj)) + +struct _GstClapperPaintable +{ + GObject parent; + + GMutex lock; + + GstVideoInfo v_info; + + GdkRGBA bg; + + GWeakRef widget, importer; + + /* Sink properties */ + gint par_n, par_d; + + /* For drawing overlays */ + gdouble pixel_aspect; + + /* Resize */ + gboolean pending_resize; + guint display_ratio_num; + guint display_ratio_den; + + /* GdkPaintableInterface */ + gint display_width; + gint display_height; + gdouble display_aspect_ratio; + + /* Pending draw signal id */ + guint draw_id; +}; + +GstClapperPaintable * gst_clapper_paintable_new (void); +void gst_clapper_paintable_queue_draw (GstClapperPaintable *paintable); +void gst_clapper_paintable_set_widget (GstClapperPaintable *paintable, GtkWidget *widget); +void gst_clapper_paintable_set_importer (GstClapperPaintable *paintable, GstClapperImporter *importer); +gboolean gst_clapper_paintable_set_video_info (GstClapperPaintable *paintable, const GstVideoInfo *v_info); +void gst_clapper_paintable_set_pixel_aspect_ratio (GstClapperPaintable *paintable, gint par_n, gint par_d); + +G_END_DECLS diff --git a/lib/gst/plugin/gstclappersink.c b/lib/gst/plugin/gstclappersink.c new file mode 100644 index 00000000..7efdb51c --- /dev/null +++ b/lib/gst/plugin/gstclappersink.c @@ -0,0 +1,910 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstclappersink.h" +#include "gstclapperimporterloader.h" +#include "gstgtkutils.h" + +#define DEFAULT_FORCE_ASPECT_RATIO TRUE +#define DEFAULT_PAR_N 1 +#define DEFAULT_PAR_D 1 +#define DEFAULT_KEEP_LAST_FRAME FALSE + +#define WINDOW_CSS_CLASS_NAME "clappersinkwindow" + +enum +{ + PROP_0, + PROP_WIDGET, + PROP_FORCE_ASPECT_RATIO, + PROP_PIXEL_ASPECT_RATIO, + PROP_KEEP_LAST_FRAME, + PROP_LAST +}; + +#define GST_CAT_DEFAULT gst_clapper_sink_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +static void gst_clapper_sink_navigation_interface_init ( + GstNavigationInterface *iface); + +#define parent_class gst_clapper_sink_parent_class +G_DEFINE_TYPE_WITH_CODE (GstClapperSink, gst_clapper_sink, GST_TYPE_VIDEO_SINK, + G_IMPLEMENT_INTERFACE (GST_TYPE_NAVIGATION, + gst_clapper_sink_navigation_interface_init)); +GST_ELEMENT_REGISTER_DEFINE (clappersink, "clappersink", GST_RANK_NONE, + GST_TYPE_CLAPPER_SINK); + +static void +window_clear_no_lock (GstClapperSink *self) +{ + if (!self->window) + return; + + GST_TRACE_OBJECT (self, "Window clear"); + + if (self->window_destroy_id) { + g_signal_handler_disconnect (self->window, self->window_destroy_id); + self->window_destroy_id = 0; + } + self->window = NULL; + self->presented_window = FALSE; +} + +static void +widget_clear_no_lock (GstClapperSink *self) +{ + if (!self->widget) + return; + + GST_TRACE_OBJECT (self, "Widget clear"); + + if (self->widget_destroy_id) { + g_signal_handler_disconnect (self->widget, self->widget_destroy_id); + self->widget_destroy_id = 0; + } + g_clear_object (&self->widget); +} + +static void +widget_destroy_cb (GtkWidget *widget, GstClapperSink *self) +{ + GST_CLAPPER_SINK_LOCK (self); + widget_clear_no_lock (self); + GST_CLAPPER_SINK_UNLOCK (self); +} + +static void +window_destroy_cb (GtkWidget *window, GstClapperSink *self) +{ + GST_DEBUG_OBJECT (self, "Window destroy"); + + GST_CLAPPER_SINK_LOCK (self); + + widget_clear_no_lock (self); + window_clear_no_lock (self); + + GST_CLAPPER_SINK_UNLOCK (self); +} + +static void +calculate_stream_coords (GstClapperSink *self, GtkWidget *widget, + gdouble x, gdouble y, gdouble *stream_x, gdouble *stream_y) +{ + GstVideoRectangle result; + gint scaled_width, scaled_height, scale_factor; + gint video_width, video_height; + gboolean force_aspect_ratio; + + GST_CLAPPER_SINK_LOCK (self); + + video_width = GST_VIDEO_INFO_WIDTH (&self->v_info); + video_height = GST_VIDEO_INFO_HEIGHT (&self->v_info); + force_aspect_ratio = self->force_aspect_ratio; + + GST_CLAPPER_SINK_UNLOCK (self); + + scale_factor = gtk_widget_get_scale_factor (widget); + scaled_width = gtk_widget_get_width (widget) * scale_factor; + scaled_height = gtk_widget_get_height (widget) * scale_factor; + + if (force_aspect_ratio) { + GstVideoRectangle src, dst; + + src.x = 0; + src.y = 0; + src.w = gdk_paintable_get_intrinsic_width ((GdkPaintable *) self->paintable); + src.h = gdk_paintable_get_intrinsic_height ((GdkPaintable *) self->paintable); + + dst.x = 0; + dst.y = 0; + dst.w = scaled_width; + dst.h = scaled_height; + + gst_video_center_rect (&src, &dst, &result, TRUE); + } else { + result.x = 0; + result.y = 0; + result.w = scaled_width; + result.h = scaled_height; + } + + /* Display coordinates to stream coordinates */ + *stream_x = (result.w > 0) + ? (x - result.x) / result.w * video_width + : 0; + *stream_y = (result.h > 0) + ? (y - result.y) / result.h * video_height + : 0; + + /* Clip to stream size */ + *stream_x = CLAMP (*stream_x, 0, video_width); + *stream_y = CLAMP (*stream_y, 0, video_height); + + GST_LOG ("Transform coords %fx%f => %fx%f", x, y, *stream_x, *stream_y); +} + +static void +gst_clapper_sink_widget_motion_event (GtkEventControllerMotion *motion, + gdouble x, gdouble y, GstClapperSink *self) +{ + GtkWidget *widget; + gdouble stream_x, stream_y; + gboolean is_inactive; + + if (x == self->last_pos_x && y == self->last_pos_y) + return; + + GST_OBJECT_LOCK (self); + is_inactive = (GST_STATE (self) < GST_STATE_PLAYING); + GST_OBJECT_UNLOCK (self); + + if (is_inactive) + return; + + self->last_pos_x = x; + self->last_pos_y = y; + + widget = gtk_event_controller_get_widget ((GtkEventController *) motion); + calculate_stream_coords (self, widget, x, y, &stream_x, &stream_y); + GST_LOG ("Event \"mouse-move\", x: %f, y: %f", stream_x, stream_y); + + gst_navigation_send_mouse_event ((GstNavigation *) self, "mouse-move", + 0, stream_x, stream_y); +} + +static void +gst_clapper_sink_widget_button_event (GtkGestureClick *click, + gint n_press, gdouble x, gdouble y, GstClapperSink *self) +{ + GtkWidget *widget; + GdkEvent *event; + GdkEventType event_type; + const gchar *event_name; + gdouble stream_x, stream_y; + gboolean is_inactive; + + GST_OBJECT_LOCK (self); + is_inactive = (GST_STATE (self) < GST_STATE_PLAYING); + GST_OBJECT_UNLOCK (self); + + if (is_inactive) + return; + + event = gtk_event_controller_get_current_event ((GtkEventController *) click); + event_type = gdk_event_get_event_type (event); + + /* FIXME: Touchscreen handling should probably use new touch events from GStreamer 1.22 */ + event_name = (event_type == GDK_BUTTON_PRESS || event_type == GDK_TOUCH_BEGIN) + ? "mouse-button-press" + : (event_type == GDK_BUTTON_RELEASE || event_type == GDK_TOUCH_END) + ? "mouse-button-release" + : NULL; + + /* Can be NULL on touch */ + if (!event_name) + return; + + widget = gtk_event_controller_get_widget ((GtkEventController *) click); + calculate_stream_coords (self, widget, x, y, &stream_x, &stream_y); + GST_LOG ("Event \"%s\", x: %f, y: %f", event_name, stream_x, stream_y); + + /* Gesture is set to handle only primary button, so we do not have to check */ + gst_navigation_send_mouse_event ((GstNavigation *) self, event_name, + 1, stream_x, stream_y); +} + +/* Must call from main thread only with a lock */ +static GtkWidget * +gst_clapper_sink_get_widget (GstClapperSink *self) +{ + if (G_UNLIKELY (!self->widget)) { + GtkEventController *controller; + GtkGesture *gesture; + + /* Make sure GTK is initialized */ + if (!gtk_init_check ()) { + GST_ERROR_OBJECT (self, "Could not ensure GTK initialization"); + return NULL; + } + + self->widget = gtk_picture_new (); + + /* Otherwise widget in grid will appear as a 1x1px + * video which might be misleading for users */ + gtk_widget_set_hexpand (self->widget, TRUE); + gtk_widget_set_vexpand (self->widget, TRUE); + + gtk_widget_set_focusable (self->widget, TRUE); + gtk_widget_set_can_focus (self->widget, TRUE); + + controller = gtk_event_controller_motion_new (); + g_signal_connect (controller, "motion", + G_CALLBACK (gst_clapper_sink_widget_motion_event), self); + gtk_widget_add_controller (self->widget, controller); + + gesture = gtk_gesture_click_new (); + gtk_gesture_single_set_button (GTK_GESTURE_SINGLE (gesture), 1); + g_signal_connect (gesture, "pressed", + G_CALLBACK (gst_clapper_sink_widget_button_event), self); + g_signal_connect (gesture, "released", + G_CALLBACK (gst_clapper_sink_widget_button_event), self); + gtk_widget_add_controller (self->widget, GTK_EVENT_CONTROLLER (gesture)); + + /* TODO: Implement touch events once we depend on GStreamer 1.22 */ + + /* Take floating ref */ + g_object_ref_sink (self->widget); + + /* Set widget back pointer */ + gst_clapper_paintable_set_widget (self->paintable, self->widget); + + /* Set earlier remembered property */ + gtk_picture_set_keep_aspect_ratio (GTK_PICTURE (self->widget), + self->force_aspect_ratio); + + gtk_picture_set_paintable (GTK_PICTURE (self->widget), GDK_PAINTABLE (self->paintable)); + + self->widget_destroy_id = g_signal_connect (self->widget, + "destroy", G_CALLBACK (widget_destroy_cb), self); + } + + return self->widget; +} + +static GtkWidget * +gst_clapper_sink_obtain_widget (GstClapperSink *self) +{ + GtkWidget *widget; + + GST_CLAPPER_SINK_LOCK (self); + widget = gst_clapper_sink_get_widget (self); + if (widget) + g_object_ref (widget); + GST_CLAPPER_SINK_UNLOCK (self); + + return widget; +} + +static void +gst_clapper_sink_get_property (GObject *object, guint prop_id, + GValue *value, GParamSpec *pspec) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (object); + + GST_CLAPPER_SINK_LOCK (self); + + switch (prop_id) { + case PROP_WIDGET: + if (self->widget) { + g_value_set_object (value, self->widget); + } else { + GtkWidget *widget; + + GST_CLAPPER_SINK_UNLOCK (self); + widget = gst_gtk_invoke_on_main ((GThreadFunc) gst_clapper_sink_obtain_widget, self); + GST_CLAPPER_SINK_LOCK (self); + + g_value_set_object (value, widget); + g_object_unref (widget); + } + break; + case PROP_FORCE_ASPECT_RATIO: + g_value_set_boolean (value, self->force_aspect_ratio); + break; + case PROP_PIXEL_ASPECT_RATIO: + gst_value_set_fraction (value, self->par_n, self->par_d); + break; + case PROP_KEEP_LAST_FRAME: + g_value_set_boolean (value, self->keep_last_frame); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + + GST_CLAPPER_SINK_UNLOCK (self); +} + +static void +gst_clapper_sink_set_property (GObject *object, guint prop_id, + const GValue *value, GParamSpec *pspec) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (object); + + GST_CLAPPER_SINK_LOCK (self); + + switch (prop_id) { + case PROP_FORCE_ASPECT_RATIO: + self->force_aspect_ratio = g_value_get_boolean (value); + + if (self->widget) { + gtk_picture_set_keep_aspect_ratio (GTK_PICTURE (self->widget), + self->force_aspect_ratio); + } + break; + case PROP_PIXEL_ASPECT_RATIO: + self->par_n = gst_value_get_fraction_numerator (value); + self->par_d = gst_value_get_fraction_denominator (value); + + gst_clapper_paintable_set_pixel_aspect_ratio (self->paintable, + self->par_n, self->par_d); + break; + case PROP_KEEP_LAST_FRAME: + self->keep_last_frame = g_value_get_boolean (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + + GST_CLAPPER_SINK_UNLOCK (self); +} + +static void +gst_clapper_sink_navigation_send_event (GstNavigation *navigation, + GstStructure *structure) +{ + GstClapperSink *sink = GST_CLAPPER_SINK_CAST (navigation); + GstEvent *event; + + GST_TRACE_OBJECT (sink, "Navigation event: %" GST_PTR_FORMAT, structure); + event = gst_event_new_navigation (structure); + + if (G_LIKELY (event)) { + GstPad *pad; + + pad = gst_pad_get_peer (GST_VIDEO_SINK_PAD (sink)); + + if (G_LIKELY (pad)) { + if (!gst_pad_send_event (pad, gst_event_ref (event))) { + /* If upstream didn't handle the event we'll post a message with it + * for the application in case it wants to do something with it */ + gst_element_post_message (GST_ELEMENT_CAST (sink), + gst_navigation_message_new_event (GST_OBJECT_CAST (sink), event)); + } + gst_object_unref (pad); + } + gst_event_unref (event); + } +} + +static gboolean +gst_clapper_sink_propose_allocation (GstBaseSink *bsink, GstQuery *query) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (bsink); + GstBufferPool *pool = NULL; + GstCaps *caps; + GstVideoInfo info; + guint size, min_buffers; + gboolean need_pool; + + gst_query_parse_allocation (query, &caps, &need_pool); + + if (!caps) { + GST_DEBUG_OBJECT (self, "No caps specified"); + return FALSE; + } + + if (!gst_video_info_from_caps (&info, caps)) { + GST_DEBUG_OBJECT (self, "Invalid caps specified"); + return FALSE; + } + + /* Normal size of a frame */ + size = GST_VIDEO_INFO_SIZE (&info); + + /* We keep around current buffer and a pending one */ + min_buffers = 3; + + if (need_pool) { + GstStructure *config = NULL; + + GST_DEBUG_OBJECT (self, "Need to create buffer pool"); + + GST_CLAPPER_SINK_LOCK (self); + pool = gst_clapper_importer_create_pool (self->importer, &config); + GST_CLAPPER_SINK_UNLOCK (self); + + if (pool) { + /* If we did not get config, use default one */ + if (!config) + config = gst_buffer_pool_get_config (pool); + + gst_buffer_pool_config_set_params (config, caps, size, min_buffers, 0); + + if (!gst_buffer_pool_set_config (pool, config)) { + gst_object_unref (pool); + + GST_ERROR_OBJECT (self, "Failed to set config"); + return FALSE; + } + } else if (config) { + GST_WARNING_OBJECT (self, "Got config without a pool to apply it"); + gst_structure_free (config); + } + } + + gst_query_add_allocation_pool (query, pool, size, min_buffers, 0); + if (pool) + gst_object_unref (pool); + + GST_CLAPPER_SINK_LOCK (self); + gst_clapper_importer_add_allocation_metas (self->importer, query); + GST_CLAPPER_SINK_UNLOCK (self); + + return TRUE; +} + +static gboolean +gst_clapper_sink_query (GstBaseSink *bsink, GstQuery *query) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (bsink); + gboolean res = FALSE; + + GST_CLAPPER_SINK_LOCK (self); + + if (GST_QUERY_TYPE (query) == GST_QUERY_CONTEXT) { + gboolean is_inactive; + + GST_OBJECT_LOCK (self); + is_inactive = (GST_STATE (self) < GST_STATE_PAUSED); + GST_OBJECT_UNLOCK (self); + + /* Some random context query in the middle of playback + * should not trigger importer replacement */ + if (is_inactive) + gst_clapper_importer_loader_find_importer_for_context_query (query, &self->importer); + if (self->importer) + res = gst_clapper_importer_handle_context_query (self->importer, bsink, query); + } + + GST_CLAPPER_SINK_UNLOCK (self); + + if (res) + return TRUE; + + return GST_BASE_SINK_CLASS (parent_class)->query (bsink, query); +} + +static gboolean +gst_clapper_sink_start_on_main (GstClapperSink *self) +{ + GtkWidget *widget; + + GST_CLAPPER_SINK_LOCK (self); + + /* Make sure widget is created */ + if (!(widget = gst_clapper_sink_get_widget (self))) { + GST_CLAPPER_SINK_UNLOCK (self); + + return FALSE; + } + + /* When no toplevel window, make our own */ + if (G_UNLIKELY (!gtk_widget_get_root (widget) && !self->window)) { + GtkWidget *toplevel, *parent; + GtkCssProvider *provider; + gchar *win_title; + + if ((parent = gtk_widget_get_parent (widget))) { + GtkWidget *temp_parent; + + while ((temp_parent = gtk_widget_get_parent (parent))) + parent = temp_parent; + } + toplevel = (parent) ? parent : widget; + + self->window = (GtkWindow *) gtk_window_new (); + gtk_widget_add_css_class (GTK_WIDGET (self->window), WINDOW_CSS_CLASS_NAME); + + provider = gtk_css_provider_new (); + gtk_css_provider_load_from_data (provider, + "." WINDOW_CSS_CLASS_NAME " { background: none; }", -1); + gtk_style_context_add_provider_for_display ( + gdk_display_get_default (), GTK_STYLE_PROVIDER (provider), + GTK_STYLE_PROVIDER_PRIORITY_APPLICATION); + g_object_unref (provider); + + win_title = g_strdup_printf ("Clapper Sink - GTK %u.%u.%u Window", + gtk_get_major_version (), + gtk_get_minor_version (), + gtk_get_micro_version ()); + + /* Set some common default size, adding stock headerbar height + * to it in order to display 4:3 aspect video widget */ + gtk_window_set_default_size (self->window, 640, 480 + 37); + gtk_window_set_title (self->window, win_title); + gtk_window_set_child (self->window, toplevel); + + g_free (win_title); + + self->window_destroy_id = g_signal_connect (self->window, + "destroy", G_CALLBACK (window_destroy_cb), self); + } + + GST_CLAPPER_SINK_UNLOCK (self); + + return TRUE; +} + +static gboolean +window_present_on_main_idle (GtkWindow *window) +{ + GST_INFO ("Presenting window"); + gtk_window_present (window); + + return G_SOURCE_REMOVE; +} + +static gboolean +gst_clapper_sink_start (GstBaseSink *bsink) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (bsink); + + GST_INFO_OBJECT (self, "Start"); + + if (G_UNLIKELY (!(! !gst_gtk_invoke_on_main ((GThreadFunc) (GCallback) + gst_clapper_sink_start_on_main, self)))) { + GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, + ("GtkWidget could not be created"), (NULL)); + + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_clapper_sink_stop_on_main (GstClapperSink *self) +{ + GtkWindow *window = NULL; + + GST_CLAPPER_SINK_LOCK (self); + if (self->window) + window = g_object_ref (self->window); + GST_CLAPPER_SINK_UNLOCK (self); + + if (window) { + gtk_window_destroy (window); + g_object_unref (window); + } + + return TRUE; +} + +static gboolean +gst_clapper_sink_stop (GstBaseSink *bsink) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (bsink); + gboolean has_window; + + GST_INFO_OBJECT (self, "Stop"); + + GST_CLAPPER_SINK_LOCK (self); + has_window = (self->window != NULL); + GST_CLAPPER_SINK_UNLOCK (self); + + if (G_UNLIKELY (has_window)) { + return (! !gst_gtk_invoke_on_main ((GThreadFunc) (GCallback) + gst_clapper_sink_stop_on_main, self)); + } + + return TRUE; +} + +static GstStateChangeReturn +gst_clapper_sink_change_state (GstElement *element, GstStateChange transition) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (element); + + GST_DEBUG_OBJECT (self, "Changing state: %s => %s", + gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), + gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + GST_CLAPPER_SINK_LOCK (self); + if (!self->keep_last_frame && self->importer) { + gst_clapper_importer_set_buffer (self->importer, NULL); + gst_clapper_paintable_queue_draw (self->paintable); + } + GST_CLAPPER_SINK_UNLOCK (self); + break; + case GST_STATE_CHANGE_PAUSED_TO_PLAYING: + GST_CLAPPER_SINK_LOCK (self); + if (G_UNLIKELY (self->window && !self->presented_window)) { + g_idle_add_full (G_PRIORITY_DEFAULT, + (GSourceFunc) window_present_on_main_idle, + g_object_ref (self->window), (GDestroyNotify) g_object_unref); + self->presented_window = TRUE; + } + GST_CLAPPER_SINK_UNLOCK (self); + break; + default: + break; + } + + return GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); +} + +static void +gst_clapper_sink_get_times (GstBaseSink *bsink, GstBuffer *buffer, + GstClockTime *start, GstClockTime *end) +{ + if (!GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) + return; + + *start = GST_BUFFER_TIMESTAMP (buffer); + + if (GST_BUFFER_DURATION_IS_VALID (buffer)) { + *end = *start + GST_BUFFER_DURATION (buffer); + } else { + GstClapperSink *self = GST_CLAPPER_SINK_CAST (bsink); + gint fps_n, fps_d; + + GST_CLAPPER_SINK_LOCK (self); + fps_n = GST_VIDEO_INFO_FPS_N (&self->v_info); + fps_d = GST_VIDEO_INFO_FPS_D (&self->v_info); + GST_CLAPPER_SINK_UNLOCK (self); + + if (fps_n > 0) + *end = *start + gst_util_uint64_scale_int (GST_SECOND, fps_d, fps_n); + } +} + +static GstCaps * +gst_clapper_sink_get_caps (GstBaseSink *bsink, GstCaps *filter) +{ + GstCaps *result, *tmp; + + tmp = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (bsink)); + + if (filter) { + GST_DEBUG ("Intersecting with filter caps: %" GST_PTR_FORMAT, filter); + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + } else { + result = tmp; + } + GST_DEBUG ("Returning caps: %" GST_PTR_FORMAT, result); + + return result; +} + +static gboolean +gst_clapper_sink_set_caps (GstBaseSink *bsink, GstCaps *caps) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (bsink); + + GST_INFO_OBJECT (self, "Set caps: %" GST_PTR_FORMAT, caps); + GST_CLAPPER_SINK_LOCK (self); + + if (G_UNLIKELY (!self->widget)) { + GST_CLAPPER_SINK_UNLOCK (self); + GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, + ("Output widget was destroyed"), (NULL)); + + return FALSE; + } + + if (!gst_clapper_importer_loader_find_importer_for_caps (caps, &self->importer)) { + GST_CLAPPER_SINK_UNLOCK (self); + GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, + ("No importer for given caps found"), (NULL)); + + return FALSE; + } + gst_clapper_paintable_set_importer (self->paintable, self->importer); + + GST_CLAPPER_SINK_UNLOCK (self); + + return GST_BASE_SINK_CLASS (parent_class)->set_caps (bsink, caps); +} + +static gboolean +gst_clapper_sink_set_info (GstVideoSink *vsink, GstCaps *caps, const GstVideoInfo *info) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (vsink); + gboolean res; + + GST_CLAPPER_SINK_LOCK (self); + + self->v_info = *info; + GST_DEBUG_OBJECT (self, "Video info changed"); + + res = gst_clapper_paintable_set_video_info (self->paintable, info); + GST_CLAPPER_SINK_UNLOCK (self); + + return res; +} + +static GstFlowReturn +gst_clapper_sink_show_frame (GstVideoSink *vsink, GstBuffer *buffer) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (vsink); + + GST_TRACE ("Got %" GST_PTR_FORMAT, buffer); + GST_CLAPPER_SINK_LOCK (self); + + if (G_UNLIKELY (!self->widget)) { + GST_CLAPPER_SINK_UNLOCK (self); + GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, + ("Output widget was destroyed"), (NULL)); + + return GST_FLOW_ERROR; + } + + gst_clapper_importer_set_buffer (self->importer, buffer); + gst_clapper_paintable_queue_draw (self->paintable); + + GST_CLAPPER_SINK_UNLOCK (self); + + return GST_FLOW_OK; +} + +static void +gst_clapper_sink_init (GstClapperSink *self) +{ + GObjectClass *gobject_class; + + gobject_class = (GObjectClass *) GST_CLAPPER_SINK_GET_CLASS (self); + + /* HACK: install here instead of class init to avoid GStreamer + * plugin scanner GObject type conflicts with older GTK versions */ + if (!g_object_class_find_property (gobject_class, "widget")) { + g_object_class_install_property (gobject_class, PROP_WIDGET, + g_param_spec_object ("widget", "GTK Widget", + "The GtkWidget to place in the widget hierarchy", + GTK_TYPE_WIDGET, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + } + + self->force_aspect_ratio = DEFAULT_FORCE_ASPECT_RATIO; + self->par_n = DEFAULT_PAR_N; + self->par_d = DEFAULT_PAR_D; + self->keep_last_frame = DEFAULT_KEEP_LAST_FRAME; + + g_mutex_init (&self->lock); + gst_video_info_init (&self->v_info); + + self->paintable = gst_clapper_paintable_new (); +} + +static void +gst_clapper_sink_dispose (GObject *object) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (object); + + GST_CLAPPER_SINK_LOCK (self); + + window_clear_no_lock (self); + widget_clear_no_lock (self); + + g_clear_object (&self->paintable); + gst_clear_object (&self->importer); + + GST_CLAPPER_SINK_UNLOCK (self); + + GST_CALL_PARENT (G_OBJECT_CLASS, dispose, (object)); +} + +static void +gst_clapper_sink_finalize (GObject *object) +{ + GstClapperSink *self = GST_CLAPPER_SINK_CAST (object); + + GST_TRACE ("Finalize"); + + gst_clapper_importer_loader_unload_all (); + g_mutex_clear (&self->lock); + + GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object)); +} + +static void +gst_clapper_sink_class_init (GstClapperSinkClass *klass) +{ + GstPadTemplate *sink_pad_templ; + + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *gstelement_class = (GstElementClass *) klass; + GstBaseSinkClass *gstbasesink_class = (GstBaseSinkClass *) klass; + GstVideoSinkClass *gstvideosink_class = (GstVideoSinkClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clappersink", 0, + "Clapper Sink"); + + gobject_class->get_property = gst_clapper_sink_get_property; + gobject_class->set_property = gst_clapper_sink_set_property; + gobject_class->dispose = gst_clapper_sink_dispose; + gobject_class->finalize = gst_clapper_sink_finalize; + + g_object_class_install_property (gobject_class, PROP_FORCE_ASPECT_RATIO, + g_param_spec_boolean ("force-aspect-ratio", "Force aspect ratio", + "When enabled, scaling will respect original aspect ratio", + DEFAULT_FORCE_ASPECT_RATIO, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PIXEL_ASPECT_RATIO, + gst_param_spec_fraction ("pixel-aspect-ratio", "Pixel Aspect Ratio", + "The pixel aspect ratio of the device", + DEFAULT_PAR_N, DEFAULT_PAR_D, + G_MAXINT, 1, 1, 1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_KEEP_LAST_FRAME, + g_param_spec_boolean ("keep-last-frame", "Keep last frame", + "Keep showing last video frame after playback instead of black screen", + DEFAULT_KEEP_LAST_FRAME, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gstelement_class->change_state = gst_clapper_sink_change_state; + + gstbasesink_class->get_caps = gst_clapper_sink_get_caps; + gstbasesink_class->set_caps = gst_clapper_sink_set_caps; + gstbasesink_class->get_times = gst_clapper_sink_get_times; + gstbasesink_class->propose_allocation = gst_clapper_sink_propose_allocation; + gstbasesink_class->query = gst_clapper_sink_query; + gstbasesink_class->start = gst_clapper_sink_start; + gstbasesink_class->stop = gst_clapper_sink_stop; + + gstvideosink_class->set_info = gst_clapper_sink_set_info; + gstvideosink_class->show_frame = gst_clapper_sink_show_frame; + + gst_element_class_set_metadata (gstelement_class, + "Clapper video sink", + "Sink/Video", "A GTK4 video sink used by Clapper media player", + "Rafał Dzięgiel "); + + sink_pad_templ = gst_clapper_importer_loader_make_sink_pad_template (); + gst_element_class_add_pad_template (gstelement_class, sink_pad_templ); +} + +/* + * GstNavigationInterface + */ +static void +gst_clapper_sink_navigation_interface_init (GstNavigationInterface *iface) +{ + /* TODO: Port to "send_event_simple" once we depend on GStreamer 1.22 */ + iface->send_event = gst_clapper_sink_navigation_send_event; +} diff --git a/lib/gst/plugin/gstclappersink.h b/lib/gst/plugin/gstclappersink.h new file mode 100644 index 00000000..d18a2754 --- /dev/null +++ b/lib/gst/plugin/gstclappersink.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include +#include + +#include "gstclapperpaintable.h" +#include "gstclapperimporter.h" + +G_BEGIN_DECLS + +#define GST_TYPE_CLAPPER_SINK (gst_clapper_sink_get_type()) +G_DECLARE_FINAL_TYPE (GstClapperSink, gst_clapper_sink, GST, CLAPPER_SINK, GstVideoSink) + +#define GST_CLAPPER_SINK_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_CLAPPER_SINK, GstClapperSinkClass)) +#define GST_CLAPPER_SINK_CAST(obj) ((GstClapperSink *)(obj)) + +#define GST_CLAPPER_SINK_GET_LOCK(obj) (&GST_CLAPPER_SINK_CAST(obj)->lock) +#define GST_CLAPPER_SINK_LOCK(obj) g_mutex_lock (GST_CLAPPER_SINK_GET_LOCK(obj)) +#define GST_CLAPPER_SINK_UNLOCK(obj) g_mutex_unlock (GST_CLAPPER_SINK_GET_LOCK(obj)) + +struct _GstClapperSink +{ + GstVideoSink parent; + + GMutex lock; + + GstClapperPaintable *paintable; + GstClapperImporter *importer; + GstVideoInfo v_info; + + GtkWidget *widget; + GtkWindow *window; + + gboolean presented_window; + + /* Properties */ + gboolean force_aspect_ratio; + gint par_n, par_d; + gboolean keep_last_frame; + + /* Position coords */ + gdouble last_pos_x; + gdouble last_pos_y; + + gulong widget_destroy_id; + gulong window_destroy_id; +}; + +GST_ELEMENT_REGISTER_DECLARE (clappersink); + +G_END_DECLS diff --git a/lib/gst/plugin/gstgdkformats.h b/lib/gst/plugin/gstgdkformats.h new file mode 100644 index 00000000..a284bac6 --- /dev/null +++ b/lib/gst/plugin/gstgdkformats.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include + +#if G_BYTE_ORDER == G_LITTLE_ENDIAN +#define GST_GDK_MEMORY_ENDIAN_FORMATS "RGBA64_LE" +#define GST_GDK_GL_TEXTURE_ENDIAN_FORMATS "RGBA64_LE" +#elif G_BYTE_ORDER == G_BIG_ENDIAN +#define GST_GDK_MEMORY_ENDIAN_FORMATS "RGBA64_BE" +#define GST_GDK_GL_TEXTURE_ENDIAN_FORMATS "RGBA64_BE" +#endif + +#define GST_GDK_MEMORY_FORMATS \ + GST_GDK_MEMORY_ENDIAN_FORMATS ", " \ + "ABGR, BGRA, ARGB, RGBA, BGRx, RGBx, BGR, RGB" + +/* Formats that `GdkGLTexture` supports */ +#define GST_GDK_GL_TEXTURE_FORMATS \ + GST_GDK_GL_TEXTURE_ENDIAN_FORMATS ", " \ + "RGBA, RGBx, RGB" diff --git a/lib/gst/plugin/gstgtkutils.c b/lib/gst/plugin/gstgtkutils.c new file mode 100644 index 00000000..4100a9d7 --- /dev/null +++ b/lib/gst/plugin/gstgtkutils.c @@ -0,0 +1,132 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * Copyright (C) 2015 Thibault Saunier + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstgtkutils.h" + +struct invoke_context +{ + GThreadFunc func; + gpointer data; + GMutex lock; + GCond cond; + gboolean fired; + + gpointer res; +}; + +static gboolean +gst_gtk_invoke_func (struct invoke_context *info) +{ + g_mutex_lock (&info->lock); + info->res = info->func (info->data); + info->fired = TRUE; + g_cond_signal (&info->cond); + g_mutex_unlock (&info->lock); + + return G_SOURCE_REMOVE; +} + +gpointer +gst_gtk_invoke_on_main (GThreadFunc func, gpointer data) +{ + GMainContext *main_context = g_main_context_default (); + struct invoke_context info; + + g_mutex_init (&info.lock); + g_cond_init (&info.cond); + info.fired = FALSE; + info.func = func; + info.data = data; + + g_main_context_invoke (main_context, (GSourceFunc) gst_gtk_invoke_func, + &info); + + g_mutex_lock (&info.lock); + while (!info.fired) + g_cond_wait (&info.cond, &info.lock); + g_mutex_unlock (&info.lock); + + g_mutex_clear (&info.lock); + g_cond_clear (&info.cond); + + return info.res; +} + +/* For use with `GdkMemoryTexture` only! */ +GdkMemoryFormat +gst_video_format_to_gdk_memory_format (GstVideoFormat format) +{ + switch (format) { + case GST_VIDEO_FORMAT_RGBA64_LE: + case GST_VIDEO_FORMAT_RGBA64_BE: + return GDK_MEMORY_R16G16B16A16_PREMULTIPLIED; + case GST_VIDEO_FORMAT_RGBA: + return GDK_MEMORY_R8G8B8A8; + case GST_VIDEO_FORMAT_BGRA: + return GDK_MEMORY_B8G8R8A8; + case GST_VIDEO_FORMAT_ARGB: + return GDK_MEMORY_A8R8G8B8; + case GST_VIDEO_FORMAT_ABGR: + return GDK_MEMORY_A8B8G8R8; + case GST_VIDEO_FORMAT_RGBx: + return GDK_MEMORY_R8G8B8A8_PREMULTIPLIED; + case GST_VIDEO_FORMAT_BGRx: + return GDK_MEMORY_B8G8R8A8_PREMULTIPLIED; + case GST_VIDEO_FORMAT_RGB: + return GDK_MEMORY_R8G8B8; + case GST_VIDEO_FORMAT_BGR: + return GDK_MEMORY_B8G8R8; + default: + break; + } + + /* This should never happen as long as above switch statement + * is updated when new formats are added to caps */ + g_assert_not_reached (); + + /* Fallback format */ + return GDK_MEMORY_R8G8B8A8_PREMULTIPLIED; +} + +GdkTexture * +gst_video_frame_into_gdk_texture (GstVideoFrame *frame) +{ + GdkTexture *texture; + GBytes *bytes; + + bytes = g_bytes_new_with_free_func ( + GST_VIDEO_FRAME_PLANE_DATA (frame, 0), + GST_VIDEO_FRAME_HEIGHT (frame) * GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0), + (GDestroyNotify) gst_buffer_unref, + gst_buffer_ref (frame->buffer)); + + texture = gdk_memory_texture_new ( + GST_VIDEO_FRAME_WIDTH (frame), + GST_VIDEO_FRAME_HEIGHT (frame), + gst_video_format_to_gdk_memory_format (GST_VIDEO_FRAME_FORMAT (frame)), + bytes, + GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0)); + + g_bytes_unref (bytes); + + return texture; +} diff --git a/lib/gst/plugin/gstgtkutils.h b/lib/gst/plugin/gstgtkutils.h new file mode 100644 index 00000000..5ef26144 --- /dev/null +++ b/lib/gst/plugin/gstgtkutils.h @@ -0,0 +1,37 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * Copyright (C) 2015 Thibault Saunier + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include + +G_BEGIN_DECLS + +gpointer gst_gtk_invoke_on_main (GThreadFunc func, gpointer data); + +GdkMemoryFormat gst_video_format_to_gdk_memory_format (GstVideoFormat format); + +GdkTexture * gst_video_frame_into_gdk_texture (GstVideoFrame *frame); + +G_END_DECLS diff --git a/lib/gst/plugin/gstplugin.c b/lib/gst/plugin/gstplugin.c new file mode 100644 index 00000000..9362ceab --- /dev/null +++ b/lib/gst/plugin/gstplugin.c @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "gstclappersink.h" + +static gboolean +plugin_init (GstPlugin *plugin) +{ + if (!g_module_supported ()) + return FALSE; + + gst_plugin_add_dependency_simple (plugin, + NULL, CLAPPER_SINK_IMPORTER_PATH, NULL, + GST_PLUGIN_DEPENDENCY_FLAG_NONE); + + return GST_ELEMENT_REGISTER (clappersink, plugin); +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, + clapper, "Clapper elements", plugin_init, VERSION, "LGPL", + GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/lib/gst/plugin/meson.build b/lib/gst/plugin/meson.build new file mode 100644 index 00000000..3694d46d --- /dev/null +++ b/lib/gst/plugin/meson.build @@ -0,0 +1,70 @@ +gst_plugins_libdir = join_paths(prefix, libdir, 'gstreamer-1.0') + +gst_clapper_plugin_args = [ + '-DHAVE_CONFIG_H', + '-DGST_USE_UNSTABLE_API', +] +gtk4_dep = dependency('gtk4', version: '>=4.6.0', required: false) + +gmodule_dep = dependency('gmodule-2.0', + version: glib_req, + required: false, + fallback: ['glib', 'libgmodule_dep'], +) + +gst_clapper_plugin_deps = [ + gtk4_dep, + gst_dep, + gstbase_dep, + gstvideo_dep, + gmodule_dep, +] + +foreach dep : gst_clapper_plugin_deps + if not dep.found() + if gst_clapper_plugin_option.enabled() + error('GStreamer plugin was enabled, but required dependencies were not found') + endif + subdir_done() + endif +endforeach + +if get_option('default_library') == 'static' + gst_clapper_plugin_args += ['-DGST_STATIC_COMPILATION'] +endif + +gst_clapper_plugin_option = get_option('gst-plugin') +if gst_clapper_plugin_option.disabled() + subdir_done() +endif + +foreach dep : gst_clapper_plugin_deps + if not dep.found() + if gst_clapper_plugin_option.enabled() + error('GStreamer plugin was enabled, but required dependencies were not found') + endif + subdir_done() + endif +endforeach + +gst_clapper_plugin_sources = [ + 'gstclappersink.c', + 'gstclapperpaintable.c', + 'gstgtkutils.c', + 'gstplugin.c', + 'gstclapperimporter.c', + 'gstclapperimporterloader.c', +] + +gst_clapper_sink_dep = declare_dependency( + link_with: library('gstclapper', + gst_clapper_plugin_sources, + c_args: gst_clapper_plugin_args, + include_directories: configinc, + dependencies: gst_clapper_plugin_deps, + install: true, + install_dir: gst_plugins_libdir, + ), + include_directories: configinc, + dependencies: gst_clapper_plugin_deps, +) diff --git a/lib/meson.build b/lib/meson.build index 588ddb94..2cb9b331 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -1,5 +1,5 @@ -glib_req = '>= 2.56.0' -gst_req = '>= 1.18.0' +glib_req = '>= 2.68.0' +gst_req = '>= 1.20.0' api_version = '1.0' libversion = meson.project_version() @@ -42,10 +42,6 @@ endif # Symbol visibility if cc.get_id() == 'msvc' export_define = '__declspec(dllexport) extern' -elif cc.has_argument('-fvisibility=hidden') - add_project_arguments('-fvisibility=hidden', language: 'c') - add_project_arguments('-fvisibility=hidden', language: 'cpp') - export_define = 'extern __attribute__ ((visibility ("default")))' else export_define = 'extern' endif @@ -132,7 +128,7 @@ cdata.set('SIZEOF_SHORT', cc.sizeof('short')) cdata.set('SIZEOF_VOIDP', cc.sizeof('void*')) cdata.set_quoted('VERSION', libversion) -cdata.set_quoted('PACKAGE', 'gst-plugins-clapper') +cdata.set_quoted('PACKAGE', 'clapper') cdata.set_quoted('PACKAGE_VERSION', libversion) cdata.set_quoted('PACKAGE_BUGREPORT', 'https://github.com/Rafostar/clapper/issues/new') cdata.set_quoted('PACKAGE_NAME', 'GStreamer Clapper Libs') @@ -184,7 +180,7 @@ foreach extra_arg : warning_flags endif endforeach -cdata.set_quoted('GST_PACKAGE_NAME', 'GStreamer Plugins Clapper') +cdata.set_quoted('GST_PACKAGE_NAME', 'gst-plugin-clapper') cdata.set_quoted('GST_PACKAGE_ORIGIN', 'https://github.com/Rafostar/clapper') # Mandatory GST deps @@ -251,21 +247,21 @@ giounix_dep = dependency('gio-unix-2.0', version: glib_req, fallback: ['glib', ' cdata.set('DISABLE_ORC', 1) cdata.set('GST_ENABLE_EXTRA_CHECKS', get_option('devel-checks')) -cdata.set_quoted('GST_PACKAGE_RELEASE_DATETIME', 'Unknown') configinc = include_directories('.') libsinc = include_directories('gst') -gir = find_program('g-ir-scanner', required: true) -if not gir.found() - error('Clapper requires GI bindings to be compiled') -endif - +gir = find_program('g-ir-scanner') gir_init_section = ['--add-init-section=extern void gst_init(gint*,gchar**);' + \ 'g_setenv("GST_REGISTRY_1.0", "@0@", TRUE);'.format(meson.current_build_dir() + '/gir_empty_registry.reg') + \ 'g_setenv("GST_PLUGIN_PATH_1_0", "", TRUE);' + \ 'g_setenv("GST_PLUGIN_SYSTEM_PATH_1_0", "", TRUE);' + \ 'gst_init(NULL,NULL);', '--quiet' ] + +gst_clapper_plugin_libdir = join_paths(get_option('prefix'), libdir, 'clapper-0.0', 'gst', 'plugin') +gst_clapper_importers_libdir = join_paths(gst_clapper_plugin_libdir, 'importers') +cdata.set_quoted('CLAPPER_SINK_IMPORTER_PATH', gst_clapper_importers_libdir) + subdir('gst') configure_file(output: 'config.h', configuration: cdata) diff --git a/meson.build b/meson.build index b962661d..e84117b9 100644 --- a/meson.build +++ b/meson.build @@ -19,9 +19,7 @@ datadir = join_paths(get_option('prefix'), get_option('datadir')) pkglibdir = join_paths(libdir, meson.project_name()) pkgdatadir = join_paths(datadir, meson.project_name()) -if get_option('lib') - subdir('lib') -endif +subdir('lib') if get_option('player') subdir('bin') diff --git a/meson_options.txt b/meson_options.txt index 1e0d763a..78817aff 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -8,6 +8,12 @@ option('lib', value: true, description: 'Build GstClapper lib' ) +option('gst-plugin', + type: 'feature', + value: 'enabled', + description: 'Build GStreamer plugin (includes GTK video sink element)' +) + option('devel-checks', type: 'boolean', value: false, From 5b7b7085e4d5b34a66adae27d1429df964f57289 Mon Sep 17 00:00:00 2001 From: Rafostar <40623528+Rafostar@users.noreply.github.com> Date: Sun, 1 May 2022 16:58:34 +0200 Subject: [PATCH 2/6] plugin: Add RAW system memory importer Add "clapperrawimporter" module which allows importing RAW system memory mapped frames --- .../plugin/importers/gstclapperrawimporter.c | 111 ++++++++++++++++++ .../plugin/importers/gstclapperrawimporter.h | 36 ++++++ lib/gst/plugin/importers/meson.build | 11 ++ lib/gst/plugin/meson.build | 2 + meson_options.txt | 6 + 5 files changed, 166 insertions(+) create mode 100644 lib/gst/plugin/importers/gstclapperrawimporter.c create mode 100644 lib/gst/plugin/importers/gstclapperrawimporter.h create mode 100644 lib/gst/plugin/importers/meson.build diff --git a/lib/gst/plugin/importers/gstclapperrawimporter.c b/lib/gst/plugin/importers/gstclapperrawimporter.c new file mode 100644 index 00000000..67de1f58 --- /dev/null +++ b/lib/gst/plugin/importers/gstclapperrawimporter.c @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstclapperrawimporter.h" +#include "gst/plugin/gstgtkutils.h" +#include "gst/plugin/gstgdkformats.h" + +#define GST_CAT_DEFAULT gst_clapper_raw_importer_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +#define parent_class gst_clapper_raw_importer_parent_class +GST_CLAPPER_IMPORTER_DEFINE (GstClapperRawImporter, gst_clapper_raw_importer, GST_TYPE_CLAPPER_IMPORTER); + +static GstBufferPool * +gst_clapper_raw_importer_create_pool (GstClapperImporter *importer, GstStructure **config) +{ + GstClapperRawImporter *self = GST_CLAPPER_RAW_IMPORTER_CAST (importer); + GstBufferPool *pool; + + GST_DEBUG_OBJECT (self, "Creating new buffer pool"); + + pool = gst_video_buffer_pool_new (); + *config = gst_buffer_pool_get_config (pool); + + gst_buffer_pool_config_add_option (*config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + return pool; +} + +static void +gst_clapper_raw_importer_add_allocation_metas (GstClapperImporter *importer, GstQuery *query) +{ + gst_query_add_allocation_meta (query, GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL); + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); +} + +static GdkTexture * +gst_clapper_raw_importer_generate_texture (GstClapperImporter *importer, + GstBuffer *buffer, GstVideoInfo *v_info) +{ + GdkTexture *texture; + GstVideoFrame frame; + + if (G_UNLIKELY (!gst_video_frame_map (&frame, v_info, buffer, GST_MAP_READ))) { + GST_ERROR_OBJECT (importer, "Could not map input buffer for reading"); + return NULL; + } + + texture = gst_video_frame_into_gdk_texture (&frame); + gst_video_frame_unmap (&frame); + + return texture; +} + +static void +gst_clapper_raw_importer_init (GstClapperRawImporter *self) +{ +} + +static void +gst_clapper_raw_importer_class_init (GstClapperRawImporterClass *klass) +{ + GstClapperImporterClass *importer_class = (GstClapperImporterClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clapperrawimporter", 0, + "Clapper RAW Importer"); + + importer_class->create_pool = gst_clapper_raw_importer_create_pool; + importer_class->add_allocation_metas = gst_clapper_raw_importer_add_allocation_metas; + importer_class->generate_texture = gst_clapper_raw_importer_generate_texture; +} + +GstClapperImporter * +make_importer (void) +{ + return g_object_new (GST_TYPE_CLAPPER_RAW_IMPORTER, NULL); +} + +GstCaps * +make_caps (GstRank *rank, GStrv *context_types) +{ + *rank = GST_RANK_MARGINAL; + + return gst_caps_from_string ( + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY ", " + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + "{ " GST_GDK_MEMORY_FORMATS " }") + "; " + GST_VIDEO_CAPS_MAKE ( + "{ " GST_GDK_MEMORY_FORMATS " }")); +} diff --git a/lib/gst/plugin/importers/gstclapperrawimporter.h b/lib/gst/plugin/importers/gstclapperrawimporter.h new file mode 100644 index 00000000..811b52b7 --- /dev/null +++ b/lib/gst/plugin/importers/gstclapperrawimporter.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include "gst/plugin/gstclapperimporter.h" + +G_BEGIN_DECLS + +#define GST_TYPE_CLAPPER_RAW_IMPORTER (gst_clapper_raw_importer_get_type()) +G_DECLARE_FINAL_TYPE (GstClapperRawImporter, gst_clapper_raw_importer, GST, CLAPPER_RAW_IMPORTER, GstClapperImporter) + +#define GST_CLAPPER_RAW_IMPORTER_CAST(obj) ((GstClapperRawImporter *)(obj)) + +struct _GstClapperRawImporter +{ + GstClapperImporter parent; +}; + +G_END_DECLS diff --git a/lib/gst/plugin/importers/meson.build b/lib/gst/plugin/importers/meson.build new file mode 100644 index 00000000..2796afce --- /dev/null +++ b/lib/gst/plugin/importers/meson.build @@ -0,0 +1,11 @@ +if not get_option('rawimporter').disabled() + library( + 'gstclapperrawimporter', + 'gstclapperrawimporter.c', + dependencies: gst_clapper_sink_dep, + include_directories: configinc, + c_args: gst_clapper_plugin_args, + install: true, + install_dir: gst_clapper_importers_libdir, + ) +endif diff --git a/lib/gst/plugin/meson.build b/lib/gst/plugin/meson.build index 3694d46d..3a0e7ee5 100644 --- a/lib/gst/plugin/meson.build +++ b/lib/gst/plugin/meson.build @@ -68,3 +68,5 @@ gst_clapper_sink_dep = declare_dependency( include_directories: configinc, dependencies: gst_clapper_plugin_deps, ) + +subdir('importers') diff --git a/meson_options.txt b/meson_options.txt index 78817aff..87c8eec1 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -14,6 +14,12 @@ option('gst-plugin', description: 'Build GStreamer plugin (includes GTK video sink element)' ) +option('rawimporter', + type: 'feature', + value: 'auto', + description: 'Build RAW system memory importer for clappersink' +) + option('devel-checks', type: 'boolean', value: false, From 3e0a0e05555e49d0a7066d15738eb0f9aa440dd4 Mon Sep 17 00:00:00 2001 From: Rafostar <40623528+Rafostar@users.noreply.github.com> Date: Sun, 1 May 2022 17:04:35 +0200 Subject: [PATCH 3/6] plugin: Add GL memory importer Add "clapperglimporter" and a base class for creating GL importers. This module allows importing GL memory into GdkTexture. --- .../importers/gstclapperglbaseimporter.c | 549 ++++++++++++++++++ .../importers/gstclapperglbaseimporter.h | 75 +++ .../plugin/importers/gstclapperglimporter.c | 70 +++ .../plugin/importers/gstclapperglimporter.h | 36 ++ lib/gst/plugin/importers/meson.build | 72 +++ meson_options.txt | 5 + 6 files changed, 807 insertions(+) create mode 100644 lib/gst/plugin/importers/gstclapperglbaseimporter.c create mode 100644 lib/gst/plugin/importers/gstclapperglbaseimporter.h create mode 100644 lib/gst/plugin/importers/gstclapperglimporter.c create mode 100644 lib/gst/plugin/importers/gstclapperglimporter.h diff --git a/lib/gst/plugin/importers/gstclapperglbaseimporter.c b/lib/gst/plugin/importers/gstclapperglbaseimporter.c new file mode 100644 index 00000000..4c29fd0a --- /dev/null +++ b/lib/gst/plugin/importers/gstclapperglbaseimporter.c @@ -0,0 +1,549 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstclapperglbaseimporter.h" +#include "gst/plugin/gstgdkformats.h" +#include "gst/plugin/gstgtkutils.h" + +#include + +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_WAYLAND +#include +#include +#endif + +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11 +#include +#endif +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11_GLX +#include +#endif +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11_EGL +#include +#endif + +#define GST_CAT_DEFAULT gst_clapper_gl_base_importer_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +#define parent_class gst_clapper_gl_base_importer_parent_class +G_DEFINE_TYPE (GstClapperGLBaseImporter, gst_clapper_gl_base_importer, GST_TYPE_CLAPPER_IMPORTER); + +static GstGLContext * +wrap_current_gl (GstGLDisplay *display, GdkGLAPI gdk_gl_api, GstGLPlatform platform) +{ + GstGLAPI gst_gl_api = GST_GL_API_NONE; + + switch (gdk_gl_api) { + case GDK_GL_API_GL: + gst_gl_api = GST_GL_API_OPENGL | GST_GL_API_OPENGL3; + break; + case GDK_GL_API_GLES: + gst_gl_api = GST_GL_API_GLES2; + break; + default: + g_assert_not_reached (); + break; + } + + if (gst_gl_api != GST_GL_API_NONE) { + guintptr gl_handle; + + gst_gl_display_filter_gl_api (display, gst_gl_api); + + if ((gl_handle = gst_gl_context_get_current_gl_context (platform))) + return gst_gl_context_new_wrapped (display, gl_handle, platform, gst_gl_api); + } + + return NULL; +} + +static gboolean +retrieve_gl_context_on_main (GstClapperGLBaseImporter *self) +{ + GstClapperGLBaseImporterClass *gl_bi_class = GST_CLAPPER_GL_BASE_IMPORTER_GET_CLASS (self); + GdkDisplay *gdk_display; + GdkGLContext *gdk_context; + GError *error = NULL; + GdkGLAPI gdk_gl_api; + GstGLPlatform platform = GST_GL_PLATFORM_NONE; + gint gl_major = 0, gl_minor = 0; + + if (!gtk_init_check ()) { + GST_ERROR_OBJECT (self, "Could not ensure GTK initialization"); + return FALSE; + } + + /* Make sure we are clean here, otherwise data sharing + * between GL-based importers may lead to leaks */ + gst_clear_object (&self->wrapped_context); + g_clear_object (&self->gdk_context); + gst_clear_object (&self->gst_display); + + gdk_display = gdk_display_get_default (); + + if (!(gdk_context = gdk_display_create_gl_context (gdk_display, &error))) { + GST_ERROR_OBJECT (self, "Error creating Gdk GL context: %s", + error ? error->message : "No error set by Gdk"); + g_clear_error (&error); + + return FALSE; + } + + if (!gl_bi_class->gdk_context_realize (self, gdk_context)) { + GST_ERROR_OBJECT (self, "Could not realize Gdk context: %" GST_PTR_FORMAT, + gdk_context); + g_object_unref (gdk_context); + + return FALSE; + } + gdk_gl_api = gdk_gl_context_get_api (gdk_context); + + GST_OBJECT_LOCK (self); + + self->gdk_context = gdk_context; + +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_WAYLAND + if (GDK_IS_WAYLAND_DISPLAY (gdk_display)) { + struct wl_display *wayland_display = + gdk_wayland_display_get_wl_display (gdk_display); + self->gst_display = (GstGLDisplay *) + gst_gl_display_wayland_new_with_display (wayland_display); + } +#endif + +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11 + if (GDK_IS_X11_DISPLAY (gdk_display)) { + gpointer display_ptr; +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11_EGL + display_ptr = gdk_x11_display_get_egl_display (gdk_display); + if (display_ptr) { + self->gst_display = (GstGLDisplay *) + gst_gl_display_egl_new_with_egl_display (display_ptr); + } +#endif +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11_GLX + if (!self->gst_display) { + display_ptr = gdk_x11_display_get_xdisplay (gdk_display); + self->gst_display = (GstGLDisplay *) + gst_gl_display_x11_new_with_display (display_ptr); + } + } +#endif +#endif + + /* Fallback to generic display */ + if (G_UNLIKELY (!self->gst_display)) { + GST_WARNING_OBJECT (self, "Unknown Gdk display!"); + self->gst_display = gst_gl_display_new (); + } + +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_WAYLAND + if (GST_IS_GL_DISPLAY_WAYLAND (self->gst_display)) { + platform = GST_GL_PLATFORM_EGL; + GST_INFO_OBJECT (self, "Using EGL on Wayland"); + goto have_display; + } +#endif +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11_EGL + if (GST_IS_GL_DISPLAY_EGL (self->gst_display)) { + platform = GST_GL_PLATFORM_EGL; + GST_INFO_OBJECT (self, "Using EGL on x11"); + goto have_display; + } +#endif +#if GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11_GLX + if (GST_IS_GL_DISPLAY_X11 (self->gst_display)) { + platform = GST_GL_PLATFORM_GLX; + GST_INFO_OBJECT (self, "Using GLX on x11"); + goto have_display; + } +#endif + + g_clear_object (&self->gdk_context); + gst_clear_object (&self->gst_display); + + GST_OBJECT_UNLOCK (self); + + GST_ERROR_OBJECT (self, "Unsupported GL platform"); + return FALSE; + +have_display: + gdk_gl_context_make_current (self->gdk_context); + + self->wrapped_context = wrap_current_gl (self->gst_display, gdk_gl_api, platform); + if (!self->wrapped_context) { + GST_ERROR ("Could not retrieve Gdk OpenGL context"); + gdk_gl_context_clear_current (); + + g_clear_object (&self->gdk_context); + gst_clear_object (&self->gst_display); + + GST_OBJECT_UNLOCK (self); + + return FALSE; + } + + GST_INFO ("Retrieved Gdk OpenGL context %" GST_PTR_FORMAT, self->wrapped_context); + gst_gl_context_activate (self->wrapped_context, TRUE); + + if (!gst_gl_context_fill_info (self->wrapped_context, &error)) { + GST_ERROR ("Failed to fill Gdk context info: %s", error->message); + g_clear_error (&error); + + gst_gl_context_activate (self->wrapped_context, FALSE); + + gst_clear_object (&self->wrapped_context); + g_clear_object (&self->gdk_context); + gst_clear_object (&self->gst_display); + + GST_OBJECT_UNLOCK (self); + + return FALSE; + } + + gst_gl_context_get_gl_version (self->wrapped_context, &gl_major, &gl_minor); + GST_INFO ("Using OpenGL%s %i.%i", (gdk_gl_api == GDK_GL_API_GLES) ? " ES" : "", + gl_major, gl_minor); + + /* Deactivate in both places */ + gst_gl_context_activate (self->wrapped_context, FALSE); + gdk_gl_context_clear_current (); + + GST_OBJECT_UNLOCK (self); + + return TRUE; +} + +static gboolean +retrieve_gst_context (GstClapperGLBaseImporter *self) +{ + GstGLDisplay *gst_display = NULL; + GstGLContext *gst_context = NULL; + GError *error = NULL; + + GST_OBJECT_LOCK (self); + + gst_display = gst_object_ref (self->gst_display); + + /* GstGLDisplay operations require display object lock to be held */ + GST_OBJECT_LOCK (gst_display); + + if (!self->gst_context) { + GST_TRACE_OBJECT (self, "Creating new GstGLContext"); + + if (!gst_gl_display_create_context (gst_display, self->wrapped_context, + &self->gst_context, &error)) { + GST_WARNING ("Could not create OpenGL context: %s", + error ? error->message : "Unknown"); + g_clear_error (&error); + + GST_OBJECT_UNLOCK (gst_display); + GST_OBJECT_UNLOCK (self); + + return FALSE; + } + } + + gst_context = gst_object_ref (self->gst_context); + + GST_OBJECT_UNLOCK (self); + + gst_gl_display_add_context (gst_display, gst_context); + + GST_OBJECT_UNLOCK (gst_display); + + gst_object_unref (gst_display); + gst_object_unref (gst_context); + + return TRUE; +} + +static gboolean +gst_clapper_gl_base_importer_prepare (GstClapperImporter *importer) +{ + GstClapperGLBaseImporter *self = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); + gboolean need_invoke; + + GST_OBJECT_LOCK (self); + need_invoke = (!self->gdk_context || !self->gst_display || !self->wrapped_context); + GST_OBJECT_UNLOCK (self); + + if (need_invoke) { + if (!(! !gst_gtk_invoke_on_main ((GThreadFunc) (GCallback) + retrieve_gl_context_on_main, self))) + return FALSE; + } + + if (!retrieve_gst_context (self)) + return FALSE; + + if (!GST_CLAPPER_IMPORTER_CLASS (parent_class)->prepare) + return TRUE; + + return GST_CLAPPER_IMPORTER_CLASS (parent_class)->prepare (importer); +} + +static void +gst_clapper_gl_base_importer_share_data (GstClapperImporter *importer, GstClapperImporter *dest_importer) +{ + GstClapperGLBaseImporter *self = GST_CLAPPER_GL_BASE_IMPORTER (importer); + + if (GST_IS_CLAPPER_GL_BASE_IMPORTER (dest_importer)) { + GstClapperGLBaseImporter *dest = GST_CLAPPER_GL_BASE_IMPORTER (dest_importer); + + GST_OBJECT_LOCK (self); + GST_OBJECT_LOCK (dest); + + /* Successfully prepared GL importer should have all three */ + if (self->gdk_context && self->gst_display && self->wrapped_context) { + g_clear_object (&dest->gdk_context); + dest->gdk_context = g_object_ref (self->gdk_context); + + gst_clear_object (&dest->gst_display); + dest->gst_display = gst_object_ref (self->gst_display); + + gst_clear_object (&dest->wrapped_context); + dest->wrapped_context = gst_object_ref (self->wrapped_context); + } + + /* This context is not required, we can create it ourselves + * using gst_display and wrapped_context */ + if (self->gst_context) { + gst_clear_object (&dest->gst_context); + dest->gst_context = gst_object_ref (self->gst_context); + } + + GST_OBJECT_UNLOCK (dest); + GST_OBJECT_UNLOCK (self); + } + + if (GST_CLAPPER_IMPORTER_CLASS (parent_class)->share_data) + GST_CLAPPER_IMPORTER_CLASS (parent_class)->share_data (importer, dest_importer); +} + +static gboolean +gst_clapper_gl_base_importer_handle_context_query (GstClapperImporter *importer, + GstBaseSink *bsink, GstQuery *query) +{ + GstClapperGLBaseImporter *self = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); + gboolean res; + + GST_OBJECT_LOCK (self); + res = gst_gl_handle_context_query (GST_ELEMENT_CAST (bsink), query, + self->gst_display, self->gst_context, self->wrapped_context); + GST_OBJECT_UNLOCK (self); + + return res; +} + +static GstBufferPool * +gst_clapper_gl_base_importer_create_pool (GstClapperImporter *importer, GstStructure **config) +{ + GstClapperGLBaseImporter *self = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); + GstBufferPool *pool; + + GST_DEBUG_OBJECT (self, "Creating new GL buffer pool"); + + GST_OBJECT_LOCK (self); + pool = gst_gl_buffer_pool_new (self->gst_context); + GST_OBJECT_UNLOCK (self); + + *config = gst_buffer_pool_get_config (pool); + + gst_buffer_pool_config_add_option (*config, GST_BUFFER_POOL_OPTION_VIDEO_META); + gst_buffer_pool_config_add_option (*config, GST_BUFFER_POOL_OPTION_GL_SYNC_META); + + return pool; +} + +static void +gst_clapper_gl_base_importer_add_allocation_metas (GstClapperImporter *importer, GstQuery *query) +{ + GstClapperGLBaseImporter *self = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); + + gst_query_add_allocation_meta (query, GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL); + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + + GST_OBJECT_LOCK (self); + if (self->gst_context->gl_vtable->FenceSync) + gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL); + GST_OBJECT_UNLOCK (self); +} + +static gboolean +gst_clapper_gl_base_importer_gdk_context_realize (GstClapperGLBaseImporter *self, GdkGLContext *gdk_context) +{ + GdkGLAPI allowed_apis; + GError *error = NULL; + const gchar *gl_env; + gboolean success; + + GST_DEBUG_OBJECT (self, "Realizing GdkGLContext with default implementation"); + + /* Use single "GST_GL_API" env to also influence Gdk GL selection */ + gl_env = g_getenv ("GST_GL_API"); + allowed_apis = (!gl_env || g_str_has_prefix (gl_env, "gles")) + ? GDK_GL_API_GLES + : (g_str_has_prefix (gl_env, "opengl")) + ? GDK_GL_API_GL + : GDK_GL_API_GL | GDK_GL_API_GLES; + + gdk_gl_context_set_allowed_apis (gdk_context, allowed_apis); + if (!(success = gdk_gl_context_realize (gdk_context, &error))) { + GST_WARNING_OBJECT (self, "Could not realize Gdk context with %s: %s", + (allowed_apis & GDK_GL_API_GL) ? "GL" : "GLES", error->message); + g_clear_error (&error); + } + if (!success && !gl_env) { + gdk_gl_context_set_allowed_apis (gdk_context, GDK_GL_API_GL); + if (!(success = gdk_gl_context_realize (gdk_context, &error))) { + GST_WARNING_OBJECT (self, "Could not realize Gdk context with GL: %s", error->message); + g_clear_error (&error); + } + } + + return success; +} + +static void +gst_clapper_gl_base_importer_init (GstClapperGLBaseImporter *self) +{ +} + +static void +gst_clapper_gl_base_importer_finalize (GObject *object) +{ + GstClapperGLBaseImporter *self = GST_CLAPPER_GL_BASE_IMPORTER_CAST (object); + + g_clear_object (&self->gdk_context); + + gst_clear_object (&self->gst_display); + gst_clear_object (&self->wrapped_context); + gst_clear_object (&self->gst_context); + + GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object)); +} + +static void +gst_clapper_gl_base_importer_class_init (GstClapperGLBaseImporterClass *klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstClapperImporterClass *importer_class = (GstClapperImporterClass *) klass; + GstClapperGLBaseImporterClass *gl_bi_class = (GstClapperGLBaseImporterClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clapperglbaseimporter", 0, + "Clapper GL Base Importer"); + + gobject_class->finalize = gst_clapper_gl_base_importer_finalize; + + importer_class->prepare = gst_clapper_gl_base_importer_prepare; + importer_class->share_data = gst_clapper_gl_base_importer_share_data; + importer_class->handle_context_query = gst_clapper_gl_base_importer_handle_context_query; + importer_class->create_pool = gst_clapper_gl_base_importer_create_pool; + importer_class->add_allocation_metas = gst_clapper_gl_base_importer_add_allocation_metas; + + gl_bi_class->gdk_context_realize = gst_clapper_gl_base_importer_gdk_context_realize; +} + +GstCaps * +gst_clapper_gl_base_importer_make_supported_gdk_gl_caps (void) +{ + GstCaps *caps, *tmp; + + tmp = gst_caps_from_string ( + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "{ " GST_GDK_GL_TEXTURE_FORMATS " }") ", " + "texture-target = (string) { " GST_GL_TEXTURE_TARGET_2D_STR " }"); + + caps = gst_caps_copy (tmp); + gst_caps_set_features_simple (caps, gst_caps_features_new ( + GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, NULL)); + + gst_caps_append (caps, tmp); + + return caps; +} + +GStrv +gst_clapper_gl_base_importer_make_gl_context_types (void) +{ + GStrv context_types; + GStrvBuilder *builder = g_strv_builder_new (); + + g_strv_builder_add (builder, GST_GL_DISPLAY_CONTEXT_TYPE); + g_strv_builder_add (builder, "gst.gl.app_context"); + g_strv_builder_add (builder, "gst.gl.local_context"); + + context_types = g_strv_builder_end (builder); + g_strv_builder_unref (builder); + + return context_types; +} + +GdkTexture * +gst_clapper_gl_base_importer_make_gl_texture (GstClapperGLBaseImporter *self, + GstBuffer *buffer, GstVideoInfo *v_info) +{ + GdkTexture *texture; + GstGLSyncMeta *sync_meta; + GstVideoFrame frame; + + if (G_UNLIKELY (!gst_video_frame_map (&frame, v_info, buffer, GST_MAP_READ | GST_MAP_GL))) { + GST_ERROR_OBJECT (self, "Could not map input buffer for reading"); + return NULL; + } + + GST_OBJECT_LOCK (self); + + /* Must have context active here for both sync meta + * and Gdk texture format auto-detection to work */ + gdk_gl_context_make_current (self->gdk_context); + gst_gl_context_activate (self->wrapped_context, TRUE); + + sync_meta = gst_buffer_get_gl_sync_meta (buffer); + + /* Wait for all previous OpenGL commands to complete, + * before we start using the input texture */ + if (sync_meta) { + gst_gl_sync_meta_set_sync_point (sync_meta, self->gst_context); + gst_gl_sync_meta_wait (sync_meta, self->wrapped_context); + } + + texture = gdk_gl_texture_new ( + self->gdk_context, + *(guint *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0), + GST_VIDEO_FRAME_WIDTH (&frame), + GST_VIDEO_FRAME_HEIGHT (&frame), + (GDestroyNotify) gst_buffer_unref, + gst_buffer_ref (buffer)); + + gst_gl_context_activate (self->wrapped_context, FALSE); + gdk_gl_context_clear_current (); + + GST_OBJECT_UNLOCK (self); + + gst_video_frame_unmap (&frame); + + return texture; +} diff --git a/lib/gst/plugin/importers/gstclapperglbaseimporter.h b/lib/gst/plugin/importers/gstclapperglbaseimporter.h new file mode 100644 index 00000000..79fda32c --- /dev/null +++ b/lib/gst/plugin/importers/gstclapperglbaseimporter.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include + +#include "gst/plugin/gstclapperimporter.h" + +G_BEGIN_DECLS + +#define GST_TYPE_CLAPPER_GL_BASE_IMPORTER (gst_clapper_gl_base_importer_get_type()) +#define GST_IS_CLAPPER_GL_BASE_IMPORTER(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_CLAPPER_GL_BASE_IMPORTER)) +#define GST_IS_CLAPPER_GL_BASE_IMPORTER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_CLAPPER_GL_BASE_IMPORTER)) +#define GST_CLAPPER_GL_BASE_IMPORTER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_CLAPPER_GL_BASE_IMPORTER, GstClapperGLBaseImporterClass)) +#define GST_CLAPPER_GL_BASE_IMPORTER(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_CLAPPER_GL_BASE_IMPORTER, GstClapperGLBaseImporter)) +#define GST_CLAPPER_GL_BASE_IMPORTER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_CLAPPER_GL_BASE_IMPORTER, GstClapperGLBaseImporterClass)) +#define GST_CLAPPER_GL_BASE_IMPORTER_CAST(obj) ((GstClapperGLBaseImporter *)(obj)) + +#define GST_CLAPPER_GL_BASE_IMPORTER_HAVE_WAYLAND (GST_GL_HAVE_WINDOW_WAYLAND && defined (GDK_WINDOWING_WAYLAND)) +#define GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11 (GST_GL_HAVE_WINDOW_X11 && defined (GDK_WINDOWING_X11)) +#define GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11_GLX (GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11 && GST_GL_HAVE_PLATFORM_GLX) +#define GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11_EGL (GST_CLAPPER_GL_BASE_IMPORTER_HAVE_X11 && GST_GL_HAVE_PLATFORM_EGL) + +typedef struct _GstClapperGLBaseImporter GstClapperGLBaseImporter; +typedef struct _GstClapperGLBaseImporterClass GstClapperGLBaseImporterClass; + +#ifdef G_DEFINE_AUTOPTR_CLEANUP_FUNC +G_DEFINE_AUTOPTR_CLEANUP_FUNC (GstClapperGLBaseImporter, gst_object_unref) +#endif + +struct _GstClapperGLBaseImporter +{ + GstClapperImporter parent; + + GdkGLContext *gdk_context; + + GstGLDisplay *gst_display; + GstGLContext *wrapped_context; + GstGLContext *gst_context; +}; + +struct _GstClapperGLBaseImporterClass +{ + GstClapperImporterClass parent_class; + + gboolean (* gdk_context_realize) (GstClapperGLBaseImporter *gl_bi, + GdkGLContext *gdk_context); +}; + +GType gst_clapper_gl_base_importer_get_type (void); + +GstCaps * gst_clapper_gl_base_importer_make_supported_gdk_gl_caps (void); + +GStrv gst_clapper_gl_base_importer_make_gl_context_types (void); + +GdkTexture * gst_clapper_gl_base_importer_make_gl_texture (GstClapperGLBaseImporter *self, GstBuffer *buffer, GstVideoInfo *v_info); + +G_END_DECLS diff --git a/lib/gst/plugin/importers/gstclapperglimporter.c b/lib/gst/plugin/importers/gstclapperglimporter.c new file mode 100644 index 00000000..9cdfcbee --- /dev/null +++ b/lib/gst/plugin/importers/gstclapperglimporter.c @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstclapperglimporter.h" + +#define GST_CAT_DEFAULT gst_clapper_gl_importer_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +#define parent_class gst_clapper_gl_importer_parent_class +GST_CLAPPER_IMPORTER_DEFINE (GstClapperGLImporter, gst_clapper_gl_importer, GST_TYPE_CLAPPER_GL_BASE_IMPORTER); + +static GdkTexture * +gst_clapper_gl_importer_generate_texture (GstClapperImporter *importer, + GstBuffer *buffer, GstVideoInfo *v_info) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); + + return gst_clapper_gl_base_importer_make_gl_texture (gl_bi, buffer, v_info); +} + +static void +gst_clapper_gl_importer_init (GstClapperGLImporter *self) +{ +} + +static void +gst_clapper_gl_importer_class_init (GstClapperGLImporterClass *klass) +{ + GstClapperImporterClass *importer_class = (GstClapperImporterClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clapperglimporter", 0, + "Clapper GL Importer"); + + importer_class->generate_texture = gst_clapper_gl_importer_generate_texture; +} + +GstClapperImporter * +make_importer (void) +{ + return g_object_new (GST_TYPE_CLAPPER_GL_IMPORTER, NULL); +} + +GstCaps * +make_caps (GstRank *rank, GStrv *context_types) +{ + *rank = GST_RANK_SECONDARY; + *context_types = gst_clapper_gl_base_importer_make_gl_context_types (); + + return gst_clapper_gl_base_importer_make_supported_gdk_gl_caps (); +} diff --git a/lib/gst/plugin/importers/gstclapperglimporter.h b/lib/gst/plugin/importers/gstclapperglimporter.h new file mode 100644 index 00000000..7c9cb3d3 --- /dev/null +++ b/lib/gst/plugin/importers/gstclapperglimporter.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include "gstclapperglbaseimporter.h" + +G_BEGIN_DECLS + +#define GST_TYPE_CLAPPER_GL_IMPORTER (gst_clapper_gl_importer_get_type()) +G_DECLARE_FINAL_TYPE (GstClapperGLImporter, gst_clapper_gl_importer, GST, CLAPPER_GL_IMPORTER, GstClapperGLBaseImporter) + +#define GST_CLAPPER_GL_IMPORTER_CAST(obj) ((GstClapperGLImporter *)(obj)) + +struct _GstClapperGLImporter +{ + GstClapperGLBaseImporter parent; +}; + +G_END_DECLS diff --git a/lib/gst/plugin/importers/meson.build b/lib/gst/plugin/importers/meson.build index 2796afce..a8daaaa8 100644 --- a/lib/gst/plugin/importers/meson.build +++ b/lib/gst/plugin/importers/meson.build @@ -1,3 +1,5 @@ +gst_clapper_gl_base_importer_dep = dependency('', required: false) + if not get_option('rawimporter').disabled() library( 'gstclapperrawimporter', @@ -9,3 +11,73 @@ if not get_option('rawimporter').disabled() install_dir: gst_clapper_importers_libdir, ) endif + +plugin_needs_gl_base = ( + not get_option('glimporter').disabled() +) + +gst_plugin_gl_deps = [gstgl_dep, gstglproto_dep] +have_gtk_gl_windowing = false + +if gst_gl_have_window_x11 and (gst_gl_have_platform_egl or gst_gl_have_platform_glx) + gtk_x11_dep = dependency('gtk4-x11', required: false) + if gtk_x11_dep.found() + gst_plugin_gl_deps += gtk_x11_dep + if gst_gl_have_platform_glx + gst_plugin_gl_deps += gstglx11_dep + endif + have_gtk_gl_windowing = true + endif +endif + +if gst_gl_have_window_wayland and gst_gl_have_platform_egl + gtk_wayland_dep = dependency('gtk4-wayland', required: false) + if gtk_wayland_dep.found() + gst_plugin_gl_deps += [gtk_wayland_dep, gstglwayland_dep] + have_gtk_gl_windowing = true + endif +endif + +if gst_gl_have_platform_egl + gst_plugin_gl_deps += gstglegl_dep +endif + +gst_clapper_gl_base_importer_deps = [ + gst_clapper_sink_dep +] + gst_plugin_gl_deps + +foreach dep : gst_clapper_gl_base_importer_deps + if not dep.found() or not have_gtk_gl_windowing + if get_option('glimporter').enabled() + error('GL importer was enabled, but required dependencies were not found') + endif + plugin_needs_gl_base = false + endif +endforeach + +if plugin_needs_gl_base + gst_clapper_gl_base_importer_dep = declare_dependency( + link_with: library('gstclapperglbaseimporter', + 'gstclapperglbaseimporter.c', + c_args: gst_clapper_plugin_args, + include_directories: configinc, + dependencies: gst_clapper_gl_base_importer_deps, + version: libversion, + install: true, + ), + include_directories: configinc, + dependencies: gst_clapper_gl_base_importer_deps, + ) +endif + +if not get_option('glimporter').disabled() and gst_clapper_gl_base_importer_dep.found() + library( + 'gstclapperglimporter', + 'gstclapperglimporter.c', + dependencies: gst_clapper_gl_base_importer_dep, + include_directories: configinc, + c_args: gst_clapper_plugin_args, + install: true, + install_dir: gst_clapper_importers_libdir, + ) +endif diff --git a/meson_options.txt b/meson_options.txt index 87c8eec1..a5fc377b 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -19,6 +19,11 @@ option('rawimporter', value: 'auto', description: 'Build RAW system memory importer for clappersink' ) +option('glimporter', + type: 'feature', + value: 'auto', + description: 'Build GL memory importer for clappersink' +) option('devel-checks', type: 'boolean', From 0410c6e9b524d63431ebe658f562e286931748b3 Mon Sep 17 00:00:00 2001 From: Rafostar <40623528+Rafostar@users.noreply.github.com> Date: Sun, 1 May 2022 17:08:48 +0200 Subject: [PATCH 4/6] plugin: Add GL uploader Add "clappergluploader" module which uses "glupload" and "glcolorconvert" internally, allowing either uploading RAW system memory into GPU with GL colorspace conversion or importing DMABufs into GL memory and making a GdkTexture out of them at one go. --- .../plugin/importers/gstclappergluploader.c | 232 ++++++++++++++++++ .../plugin/importers/gstclappergluploader.h | 42 ++++ lib/gst/plugin/importers/meson.build | 15 ++ meson_options.txt | 5 + 4 files changed, 294 insertions(+) create mode 100644 lib/gst/plugin/importers/gstclappergluploader.c create mode 100644 lib/gst/plugin/importers/gstclappergluploader.h diff --git a/lib/gst/plugin/importers/gstclappergluploader.c b/lib/gst/plugin/importers/gstclappergluploader.c new file mode 100644 index 00000000..03e318ee --- /dev/null +++ b/lib/gst/plugin/importers/gstclappergluploader.c @@ -0,0 +1,232 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstclappergluploader.h" + +#define GST_CAT_DEFAULT gst_clapper_gl_uploader_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +#define parent_class gst_clapper_gl_uploader_parent_class +GST_CLAPPER_IMPORTER_DEFINE (GstClapperGLUploader, gst_clapper_gl_uploader, GST_TYPE_CLAPPER_GL_BASE_IMPORTER); + +static void +_update_elements_caps_locked (GstClapperGLUploader *self, GstCaps *upload_sink_caps) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self); + GstCaps *upload_src_caps, *color_sink_caps, *color_src_caps, *gdk_sink_caps; + + GST_INFO_OBJECT (self, "Input caps: %" GST_PTR_FORMAT, upload_sink_caps); + + upload_src_caps = gst_gl_upload_transform_caps (self->upload, gl_bi->gst_context, + GST_PAD_SINK, upload_sink_caps, NULL); + upload_src_caps = gst_caps_fixate (upload_src_caps); + + GST_INFO_OBJECT (self, "GLUpload caps: %" GST_PTR_FORMAT, upload_src_caps); + gst_gl_upload_set_caps (self->upload, upload_sink_caps, upload_src_caps); + + gdk_sink_caps = gst_clapper_gl_base_importer_make_supported_gdk_gl_caps (); + color_sink_caps = gst_gl_color_convert_transform_caps (gl_bi->gst_context, + GST_PAD_SRC, upload_src_caps, gdk_sink_caps); + gst_caps_unref (gdk_sink_caps); + + /* Second caps arg is transfer-full */ + color_src_caps = gst_gl_color_convert_fixate_caps (gl_bi->gst_context, + GST_PAD_SINK, upload_src_caps, color_sink_caps); + + GST_INFO_OBJECT (self, "GLColorConvert caps: %" GST_PTR_FORMAT, color_src_caps); + gst_gl_color_convert_set_caps (self->color_convert, upload_src_caps, color_src_caps); + + self->has_pending_v_info = gst_video_info_from_caps (&self->pending_v_info, color_src_caps); + + gst_caps_unref (upload_src_caps); + gst_caps_unref (color_src_caps); +} + +static void +gst_clapper_gl_uploader_set_caps (GstClapperImporter *importer, GstCaps *caps) +{ + GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (importer); + + GST_OBJECT_LOCK (self); + _update_elements_caps_locked (self, caps); + GST_OBJECT_UNLOCK (self); +} + +static void +_uploader_reconfigure_locked (GstClapperGLUploader *self) +{ + GstCaps *in_caps = NULL; + + GST_DEBUG_OBJECT (self, "Reconfiguring upload"); + + gst_gl_upload_get_caps (self->upload, &in_caps, NULL); + + if (G_LIKELY (in_caps)) { + _update_elements_caps_locked (self, in_caps); + gst_caps_unref (in_caps); + } +} + +static gboolean +gst_clapper_gl_uploader_prepare (GstClapperImporter *importer) +{ + gboolean res = GST_CLAPPER_IMPORTER_CLASS (parent_class)->prepare (importer); + + if (res) { + GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (importer); + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); + + GST_OBJECT_LOCK (self); + + if (!self->upload) + self->upload = gst_gl_upload_new (gl_bi->gst_context); + if (!self->color_convert) + self->color_convert = gst_gl_color_convert_new (gl_bi->gst_context); + + GST_OBJECT_UNLOCK (self); + } + + return res; +} + +static GstBuffer * +_upload_perform_locked (GstClapperGLUploader *self, GstBuffer *buffer) +{ + GstBuffer *upload_buf = NULL; + GstGLUploadReturn ret; + + ret = gst_gl_upload_perform_with_buffer (self->upload, buffer, &upload_buf); + + if (G_UNLIKELY (ret != GST_GL_UPLOAD_DONE)) { + switch (ret) { + case GST_GL_UPLOAD_RECONFIGURE: + _uploader_reconfigure_locked (self); + /* Retry with the same buffer after reconfiguring */ + return _upload_perform_locked (self, buffer); + default: + GST_ERROR_OBJECT (self, "Could not upload input buffer, returned: %i", ret); + break; + } + } + + return upload_buf; +} + +static GdkTexture * +gst_clapper_gl_uploader_generate_texture (GstClapperImporter *importer, + GstBuffer *buffer, GstVideoInfo *v_info) +{ + GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (importer); + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); + GstBuffer *upload_buf, *color_buf; + GdkTexture *texture; + + /* XXX: We both upload and perform color conversion here, thus we skip + * upload for buffers that are not going to be shown and gain more free + * CPU time to prepare the next one. Improves performance on weak HW. */ + + GST_LOG_OBJECT (self, "Uploading %" GST_PTR_FORMAT, buffer); + + GST_OBJECT_LOCK (self); + + upload_buf = _upload_perform_locked (self, buffer); + + if (G_UNLIKELY (!upload_buf)) { + GST_ERROR_OBJECT (self, "Could not perform upload on input buffer"); + GST_OBJECT_UNLOCK (self); + + return NULL; + } + GST_LOG_OBJECT (self, "Uploaded into %" GST_PTR_FORMAT, upload_buf); + + color_buf = gst_gl_color_convert_perform (self->color_convert, upload_buf); + gst_buffer_unref (upload_buf); + + /* Use video info associated with converted buffer */ + if (self->has_pending_v_info) { + self->v_info = self->pending_v_info; + self->has_pending_v_info = FALSE; + } + + GST_OBJECT_UNLOCK (self); + + if (G_UNLIKELY (!color_buf)) { + GST_ERROR_OBJECT (self, "Could not perform color conversion on input buffer"); + return NULL; + } + GST_LOG_OBJECT (self, "Color converted into %" GST_PTR_FORMAT, color_buf); + + texture = gst_clapper_gl_base_importer_make_gl_texture (gl_bi, color_buf, &self->v_info); + gst_buffer_unref (color_buf); + + return texture; +} + +static void +gst_clapper_gl_uploader_init (GstClapperGLUploader *self) +{ + gst_video_info_init (&self->pending_v_info); + gst_video_info_init (&self->v_info); +} + +static void +gst_clapper_gl_uploader_finalize (GObject *object) +{ + GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (object); + + gst_clear_object (&self->upload); + gst_clear_object (&self->color_convert); + + GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object)); +} + +static void +gst_clapper_gl_uploader_class_init (GstClapperGLUploaderClass *klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstClapperImporterClass *importer_class = (GstClapperImporterClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clappergluploader", 0, + "Clapper GL Uploader"); + + gobject_class->finalize = gst_clapper_gl_uploader_finalize; + + importer_class->prepare = gst_clapper_gl_uploader_prepare; + importer_class->set_caps = gst_clapper_gl_uploader_set_caps; + importer_class->generate_texture = gst_clapper_gl_uploader_generate_texture; +} + +GstClapperImporter * +make_importer (void) +{ + return g_object_new (GST_TYPE_CLAPPER_GL_UPLOADER, NULL); +} + +GstCaps * +make_caps (GstRank *rank, GStrv *context_types) +{ + *rank = GST_RANK_MARGINAL + 1; + *context_types = gst_clapper_gl_base_importer_make_gl_context_types (); + + return gst_gl_upload_get_input_template_caps (); +} diff --git a/lib/gst/plugin/importers/gstclappergluploader.h b/lib/gst/plugin/importers/gstclappergluploader.h new file mode 100644 index 00000000..aa76f9c2 --- /dev/null +++ b/lib/gst/plugin/importers/gstclappergluploader.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include "gstclapperglbaseimporter.h" + +G_BEGIN_DECLS + +#define GST_TYPE_CLAPPER_GL_UPLOADER (gst_clapper_gl_uploader_get_type()) +G_DECLARE_FINAL_TYPE (GstClapperGLUploader, gst_clapper_gl_uploader, GST, CLAPPER_GL_UPLOADER, GstClapperGLBaseImporter) + +#define GST_CLAPPER_GL_UPLOADER_CAST(obj) ((GstClapperGLUploader *)(obj)) + +struct _GstClapperGLUploader +{ + GstClapperGLBaseImporter parent; + + GstGLUpload *upload; + GstGLColorConvert *color_convert; + + GstVideoInfo pending_v_info, v_info; + gboolean has_pending_v_info; +}; + +G_END_DECLS diff --git a/lib/gst/plugin/importers/meson.build b/lib/gst/plugin/importers/meson.build index a8daaaa8..a435604d 100644 --- a/lib/gst/plugin/importers/meson.build +++ b/lib/gst/plugin/importers/meson.build @@ -14,6 +14,7 @@ endif plugin_needs_gl_base = ( not get_option('glimporter').disabled() + or not get_option('gluploader').disabled() ) gst_plugin_gl_deps = [gstgl_dep, gstglproto_dep] @@ -50,6 +51,8 @@ foreach dep : gst_clapper_gl_base_importer_deps if not dep.found() or not have_gtk_gl_windowing if get_option('glimporter').enabled() error('GL importer was enabled, but required dependencies were not found') + elif get_option('gluploader').enabled() + error('GL uploader was enabled, but required dependencies were not found') endif plugin_needs_gl_base = false endif @@ -81,3 +84,15 @@ if not get_option('glimporter').disabled() and gst_clapper_gl_base_importer_dep. install_dir: gst_clapper_importers_libdir, ) endif + +if not get_option('gluploader').disabled() and gst_clapper_gl_base_importer_dep.found() + library( + 'gstclappergluploader', + 'gstclappergluploader.c', + dependencies: gst_clapper_gl_base_importer_dep, + include_directories: configinc, + c_args: gst_clapper_plugin_args, + install: true, + install_dir: gst_clapper_importers_libdir, + ) +endif diff --git a/meson_options.txt b/meson_options.txt index a5fc377b..67e9c482 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -24,6 +24,11 @@ option('glimporter', value: 'auto', description: 'Build GL memory importer for clappersink' ) +option('gluploader', + type: 'feature', + value: 'auto', + description: 'Build GL uploader for clappersink' +) option('devel-checks', type: 'boolean', From b37ab432d78097904f600ffb992547aa93ed627e Mon Sep 17 00:00:00 2001 From: Rafostar <40623528+Rafostar@users.noreply.github.com> Date: Sun, 1 May 2022 17:45:19 +0200 Subject: [PATCH 5/6] player: Use new "clappersink" element Use brand new clapper video sink for video output. Also add "CLAPPER_USE_LEGACY_SINK" env to still allow the usage of old video sink if any problems arise. --- src/player.js | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/player.js b/src/player.js index f20eee86..02bdc3a9 100644 --- a/src/player.js +++ b/src/player.js @@ -1,4 +1,4 @@ -const { Adw, Gdk, Gio, GObject, Gst, GstClapper, Gtk } = imports.gi; +const { Adw, Gdk, Gio, GLib, GObject, Gst, GstClapper, Gtk } = imports.gi; const ByteArray = imports.byteArray; const Debug = imports.src.debug; const Misc = imports.src.misc; @@ -16,14 +16,28 @@ class ClapperPlayer extends GstClapper.Clapper { _init() { - const gtk4plugin = new GstClapper.ClapperGtk4Plugin(); - const glsinkbin = Gst.ElementFactory.make('glsinkbin', null); - glsinkbin.sink = gtk4plugin.video_sink; + let vsink = null; + const use_legacy_sink = GLib.getenv('CLAPPER_USE_LEGACY_SINK'); + + if(!use_legacy_sink || use_legacy_sink != '1') { + vsink = Gst.ElementFactory.make('clappersink', null); + this.clappersink = vsink; + } + + if(!vsink) { + vsink = Gst.ElementFactory.make('glsinkbin', null); + const gtk4plugin = new GstClapper.ClapperGtk4Plugin(); + + warn('using legacy video sink'); + + this.clappersink = gtk4plugin.video_sink; + vsink.sink = this.clappersink; + } super._init({ signal_dispatcher: new GstClapper.ClapperGMainContextSignalDispatcher(), video_renderer: new GstClapper.ClapperVideoOverlayVideoRenderer({ - video_sink: glsinkbin, + video_sink: vsink, }), mpris: new GstClapper.ClapperMpris({ own_name: `org.mpris.MediaPlayer2.${Misc.appName}`, @@ -36,7 +50,7 @@ class ClapperPlayer extends GstClapper.Clapper use_pipewire: settings.get_boolean('use-pipewire'), }); - this.widget = gtk4plugin.video_sink.widget; + this.widget = this.clappersink.widget; this.widget.add_css_class('videowidget'); this.visualization_enabled = false; @@ -615,7 +629,7 @@ class ClapperPlayer extends GstClapper.Clapper switch(key) { case 'after-playback': - this.widget.keep_last_frame = (settings.get_int(key) === 1); + this.clappersink.keep_last_frame = (settings.get_int(key) === 1); break; case 'seeking-mode': switch(settings.get_int(key)) { From 5b9e7eacba5705b12df6bbc10755e7dc38eb5683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Dzi=C4=99giel?= Date: Fri, 6 May 2022 11:34:46 +0200 Subject: [PATCH 6/6] meson: Do not auto build rawimporter when building gluploader No need to auto build rawimporter if we are building gluploader as it will be always loaded on startup, but never used. Skip the build of it when its meson option is set to "auto". --- lib/gst/plugin/importers/meson.build | 60 +++++++++++++++++++--------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/lib/gst/plugin/importers/meson.build b/lib/gst/plugin/importers/meson.build index a435604d..baa11cf5 100644 --- a/lib/gst/plugin/importers/meson.build +++ b/lib/gst/plugin/importers/meson.build @@ -1,21 +1,13 @@ gst_clapper_gl_base_importer_dep = dependency('', required: false) -if not get_option('rawimporter').disabled() - library( - 'gstclapperrawimporter', - 'gstclapperrawimporter.c', - dependencies: gst_clapper_sink_dep, - include_directories: configinc, - c_args: gst_clapper_plugin_args, - install: true, - install_dir: gst_clapper_importers_libdir, - ) -endif - plugin_needs_gl_base = ( not get_option('glimporter').disabled() or not get_option('gluploader').disabled() ) +plugin_gl_support_required = ( + get_option('glimporter').enabled() + or get_option('gluploader').enabled() +) gst_plugin_gl_deps = [gstgl_dep, gstglproto_dep] have_gtk_gl_windowing = false @@ -39,6 +31,10 @@ if gst_gl_have_window_wayland and gst_gl_have_platform_egl endif endif +if plugin_gl_support_required and not have_gtk_gl_windowing + error('GL-based importer was enabled, but support for current GL windowing is missing') +endif + if gst_gl_have_platform_egl gst_plugin_gl_deps += gstglegl_dep endif @@ -48,11 +44,9 @@ gst_clapper_gl_base_importer_deps = [ ] + gst_plugin_gl_deps foreach dep : gst_clapper_gl_base_importer_deps - if not dep.found() or not have_gtk_gl_windowing - if get_option('glimporter').enabled() - error('GL importer was enabled, but required dependencies were not found') - elif get_option('gluploader').enabled() - error('GL uploader was enabled, but required dependencies were not found') + if not dep.found() + if plugin_gl_support_required + error('GL-based importer was enabled, but required dependencies were not found') endif plugin_needs_gl_base = false endif @@ -73,7 +67,12 @@ if plugin_needs_gl_base ) endif -if not get_option('glimporter').disabled() and gst_clapper_gl_base_importer_dep.found() +build_glimporter = ( + not get_option('glimporter').disabled() + and gst_clapper_gl_base_importer_dep.found() +) + +if build_glimporter library( 'gstclapperglimporter', 'gstclapperglimporter.c', @@ -85,7 +84,12 @@ if not get_option('glimporter').disabled() and gst_clapper_gl_base_importer_dep. ) endif -if not get_option('gluploader').disabled() and gst_clapper_gl_base_importer_dep.found() +build_gluploader = ( + not get_option('gluploader').disabled() + and gst_clapper_gl_base_importer_dep.found() +) + +if build_gluploader library( 'gstclappergluploader', 'gstclappergluploader.c', @@ -96,3 +100,21 @@ if not get_option('gluploader').disabled() and gst_clapper_gl_base_importer_dep. install_dir: gst_clapper_importers_libdir, ) endif + +# No need to auto build rawimporter if we are building gluploader +build_rawimporter = ( + not get_option('rawimporter').disabled() + and (not build_gluploader or get_option('rawimporter').enabled()) +) + +if build_rawimporter + library( + 'gstclapperrawimporter', + 'gstclapperrawimporter.c', + dependencies: gst_clapper_sink_dep, + include_directories: configinc, + c_args: gst_clapper_plugin_args, + install: true, + install_dir: gst_clapper_importers_libdir, + ) +endif