From 2e776ddf7b1354cb8c843366cc9674aa2a852bb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Dzi=C4=99giel?= Date: Fri, 2 Jul 2021 18:24:59 +0200 Subject: [PATCH] sink: Support rendering external-oes textures This significantly improves performance on mobile devices by removing the texture copy from an OES texture to 2D texture. --- lib/gst/clapper/gtk4/gstclapperglsink.c | 27 +++- lib/gst/clapper/gtk4/gstclapperglutils.c | 103 +++++++++++++ lib/gst/clapper/gtk4/gstclapperglutils.h | 29 ++++ lib/gst/clapper/gtk4/gtkclapperglwidget.c | 170 +++++++++++++++++++--- lib/gst/clapper/gtk4/gtkclapperglwidget.h | 3 +- lib/gst/clapper/meson.build | 1 + 6 files changed, 303 insertions(+), 30 deletions(-) create mode 100644 lib/gst/clapper/gtk4/gstclapperglutils.c create mode 100644 lib/gst/clapper/gtk4/gstclapperglutils.h diff --git a/lib/gst/clapper/gtk4/gstclapperglsink.c b/lib/gst/clapper/gtk4/gstclapperglsink.c index 57511c50..cadcacc0 100644 --- a/lib/gst/clapper/gtk4/gstclapperglsink.c +++ b/lib/gst/clapper/gtk4/gstclapperglsink.c @@ -37,15 +37,27 @@ GST_DEBUG_CATEGORY (gst_debug_clapper_gl_sink); #define GST_CAT_DEFAULT gst_debug_clapper_gl_sink +#define GST_CLAPPER_GL_SINK_CAPS \ + "video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " \ + "format = (string) RGBA, " \ + "width = " GST_VIDEO_SIZE_RANGE ", " \ + "height = " GST_VIDEO_SIZE_RANGE ", " \ + "framerate = " GST_VIDEO_FPS_RANGE ", " \ + "texture-target = (string) { 2D, external-oes } " \ + " ; " \ + "video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "," \ + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION "), " \ + "format = (string) RGBA, " \ + "width = " GST_VIDEO_SIZE_RANGE ", " \ + "height = " GST_VIDEO_SIZE_RANGE ", " \ + "framerate = " GST_VIDEO_FPS_RANGE ", " \ + "texture-target = (string) { 2D, external-oes } " + static GstStaticPadTemplate gst_clapper_gl_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA") "; " - GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY ", " - GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, "RGBA"))); + GST_STATIC_CAPS (GST_CLAPPER_GL_SINK_CAPS)); static void gst_clapper_gl_sink_finalize (GObject * object); static void gst_clapper_gl_sink_set_property (GObject * object, guint prop_id, @@ -672,6 +684,7 @@ static gboolean gst_clapper_gl_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) { GstClapperGLSink *clapper_sink = GST_CLAPPER_GL_SINK (bsink); + gboolean res = FALSE; GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps); @@ -691,9 +704,11 @@ gst_clapper_gl_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) GST_OBJECT_UNLOCK (clapper_sink); return FALSE; } + + res = gtk_clapper_gl_widget_update_output_format (clapper_sink->widget, caps); GST_OBJECT_UNLOCK (clapper_sink); - return TRUE; + return res; } static GstFlowReturn diff --git a/lib/gst/clapper/gtk4/gstclapperglutils.c b/lib/gst/clapper/gtk4/gstclapperglutils.c new file mode 100644 index 00000000..d74732cb --- /dev/null +++ b/lib/gst/clapper/gtk4/gstclapperglutils.c @@ -0,0 +1,103 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * Copyright (C) 2015 Thibault Saunier + * Copyright (C) 2021 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/* FIXME: remove these once their headers are public in gstreamer: + * https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/804 + */ + +#include "gstclapperglutils.h" + +static const gfloat identity_matrix[] = { + 1.0, 0.0, 0.0, 0.0, + 0.0, 1.0, 0.0, 0.0, + 0.0, 0.0, 1.0, 0.0, + 0.0, 0.0, 0.0, 1.0, +}; + +static const gfloat from_ndc_matrix[] = { + 0.5, 0.0, 0.0, 0.0, + 0.0, 0.5, 0.0, 0.0, + 0.0, 0.0, 0.5, 0.0, + 0.5, 0.5, 0.5, 1.0, +}; + +static const gfloat to_ndc_matrix[] = { + 2.0, 0.0, 0.0, 0.0, + 0.0, 2.0, 0.0, 0.0, + 0.0, 0.0, 2.0, 0.0, + -1.0, -1.0, -1.0, 1.0, +}; + +/* multiplies two 4x4 matrices, @a X @b, and stores the result in @result + * https://en.wikipedia.org/wiki/Matrix_multiplication + */ +static void +gst_gl_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result) +{ + int i, j, k; + gfloat tmp[16] = { 0.0f }; + + if (!a || !b || !result) + return; + for (i = 0; i < 4; i++) { /* column */ + for (j = 0; j < 4; j++) { /* row */ + for (k = 0; k < 4; k++) { + tmp[j + (i * 4)] += a[k + (i * 4)] * b[j + (k * 4)]; + } + } + } + + for (i = 0; i < 16; i++) + result[i] = tmp[i]; +} + +/* + * gst_clapper_gl_get_affine_transformation_meta_as_ndc: + * @meta: (nullable): a #GstVideoAffineTransformationMeta + * @matrix: (out): result of the 4x4 matrix + * + * Retrieves the stored 4x4 affine transformation matrix stored in @meta in + * NDC coordinates. if @meta is NULL, an identity matrix is returned. + * + * NDC is a left-handed coordinate system + * - x - [-1, 1] - +ve X moves right + * - y - [-1, 1] - +ve Y moves up + * - z - [-1, 1] - +ve Z moves into + */ +void +gst_clapper_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta * + meta, gfloat * matrix) +{ + if (!meta) { + int i; + + for (i = 0; i < 16; i++) { + matrix[i] = identity_matrix[i]; + } + } else { + float tmp[16]; + + /* change of basis multiplications */ + gst_gl_multiply_matrix4 (from_ndc_matrix, meta->matrix, tmp); + gst_gl_multiply_matrix4 (tmp, to_ndc_matrix, matrix); + } +} diff --git a/lib/gst/clapper/gtk4/gstclapperglutils.h b/lib/gst/clapper/gtk4/gstclapperglutils.h new file mode 100644 index 00000000..f0d51fa1 --- /dev/null +++ b/lib/gst/clapper/gtk4/gstclapperglutils.h @@ -0,0 +1,29 @@ +/* + * GStreamer + * Copyright (C) 2016 Matthew Waters + * Copyright (C) 2021 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_CLAPPER_GL_UTILS_H__ +#define __GST_CLAPPER_GL_UTILS_H__ + +#include + +void gst_clapper_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta *meta, gfloat *matrix); + +#endif /* __GST_CLAPPER_GL_UTILS_H__ */ diff --git a/lib/gst/clapper/gtk4/gtkclapperglwidget.c b/lib/gst/clapper/gtk4/gtkclapperglwidget.c index bb1cfcd5..bca815b5 100644 --- a/lib/gst/clapper/gtk4/gtkclapperglwidget.c +++ b/lib/gst/clapper/gtk4/gtkclapperglwidget.c @@ -26,9 +26,11 @@ #include #include #include +#include #include "gtkclapperglwidget.h" #include "gstgtkutils.h" +#include "gstclapperglutils.h" #if GST_GL_HAVE_WINDOW_X11 && defined (GDK_WINDOWING_X11) #include @@ -60,17 +62,24 @@ GST_DEBUG_CATEGORY (gst_debug_clapper_gl_widget); struct _GtkClapperGLWidgetPrivate { gboolean initiated; + GstGLDisplay *display; GdkGLContext *gdk_context; GstGLContext *other_context; GstGLContext *context; + + GstGLTextureTarget texture_target; + guint gl_target; + GstGLUpload *upload; GstGLShader *shader; + GLuint vao; GLuint vertex_buffer; GLint attr_position; GLint attr_texture; GLuint current_tex; + GstGLOverlayCompositor *overlay_compositor; }; @@ -503,9 +512,42 @@ gtk_clapper_gl_widget_init_redisplay (GtkClapperGLWidget * clapper_widget) GtkClapperGLWidgetPrivate *priv = clapper_widget->priv; const GstGLFuncs *gl = priv->context->gl_vtable; GError *error = NULL; + GstGLSLStage *frag_stage, *vert_stage; gst_gl_insert_debug_marker (priv->other_context, "initializing redisplay"); - if (!(priv->shader = gst_gl_shader_new_default (priv->context, &error))) { + + vert_stage = gst_glsl_stage_new_with_string (priv->context, + GL_VERTEX_SHADER, GST_GLSL_VERSION_NONE, + GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY, + gst_gl_shader_string_vertex_mat4_vertex_transform); + if (priv->texture_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) { + gchar *frag_str; + + frag_str = + gst_gl_shader_string_fragment_external_oes_get_default + (priv->context, GST_GLSL_VERSION_NONE, + GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY); + frag_stage = gst_glsl_stage_new_with_string (priv->context, + GL_FRAGMENT_SHADER, GST_GLSL_VERSION_NONE, + GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY, frag_str); + + g_free (frag_str); + } else { + frag_stage = gst_glsl_stage_new_default_fragment (priv->context); + } + + if (!vert_stage || !frag_stage) { + GST_ERROR ("Failed to retrieve fragment shader for texture target"); + if (vert_stage) + gst_object_unref (vert_stage); + if (frag_stage) + gst_object_unref (frag_stage); + return; + } + + if (!(priv->shader = + gst_gl_shader_new_link_with_stages (priv->context, &error, + vert_stage, frag_stage, NULL))) { GST_ERROR ("Failed to initialize shader: %s", error->message); return; } @@ -532,8 +574,9 @@ gtk_clapper_gl_widget_init_redisplay (GtkClapperGLWidget * clapper_widget) gl->BindBuffer (GL_ARRAY_BUFFER, 0); - priv->overlay_compositor = - gst_gl_overlay_compositor_new (priv->other_context); + if (!priv->overlay_compositor) + priv->overlay_compositor = + gst_gl_overlay_compositor_new (priv->other_context); priv->initiated = TRUE; } @@ -564,6 +607,9 @@ gtk_clapper_gl_widget_render (GtkGLArea * widget, GdkGLContext * context) GtkClapperGLWidgetPrivate *priv = clapper_widget->priv; const GstGLFuncs *gl; + GstVideoAffineTransformationMeta *af_meta; + gfloat matrix[16]; + GTK_CLAPPER_GL_WIDGET_LOCK (clapper_widget); /* Draw black with GDK context when priv is not available yet. @@ -651,9 +697,15 @@ gtk_clapper_gl_widget_render (GtkGLArea * widget, GdkGLContext * context) gtk_clapper_gl_widget_bind_buffer (clapper_widget); gl->ActiveTexture (GL_TEXTURE0); - gl->BindTexture (GL_TEXTURE_2D, priv->current_tex); + gl->BindTexture (priv->gl_target, priv->current_tex); gst_gl_shader_set_uniform_1i (priv->shader, "tex", 0); + af_meta = gst_buffer_get_video_affine_transformation_meta ( + clapper_widget->buffer); + gst_clapper_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix); + gst_gl_shader_set_uniform_matrix_4fv (priv->shader, + "u_transformation", 1, FALSE, matrix); + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); if (gl->BindVertexArray) @@ -661,7 +713,7 @@ gtk_clapper_gl_widget_render (GtkGLArea * widget, GdkGLContext * context) else gtk_clapper_gl_widget_unbind_buffer (clapper_widget); - gl->BindTexture (GL_TEXTURE_2D, 0); + gl->BindTexture (priv->gl_target, 0); /* Draw subtitles */ gst_gl_overlay_compositor_draw_overlays (priv->overlay_compositor); @@ -675,10 +727,34 @@ done: } static void -_reset_gl (GtkClapperGLWidget * clapper_widget) +_cleanup_gl_private (GtkClapperGLWidgetPrivate * priv) +{ + const GstGLFuncs *gl = priv->other_context->gl_vtable; + + if (priv->vao) { + gl->DeleteVertexArrays (1, &priv->vao); + priv->vao = 0; + } + if (priv->vertex_buffer) { + gl->DeleteBuffers (1, &priv->vertex_buffer); + priv->vertex_buffer = 0; + } + if (priv->upload) { + gst_object_unref (priv->upload); + priv->upload = NULL; + } + if (priv->shader) { + gst_object_unref (priv->shader); + priv->shader = NULL; + } + if (priv->overlay_compositor) + gst_gl_overlay_compositor_free_overlays (priv->overlay_compositor); +} + +static void +_cleanup_gl_thread (GtkClapperGLWidget * clapper_widget) { GtkClapperGLWidgetPrivate *priv = clapper_widget->priv; - const GstGLFuncs *gl = priv->other_context->gl_vtable; if (!priv->gdk_context) priv->gdk_context = gtk_gl_area_get_context (GTK_GL_AREA (clapper_widget)); @@ -689,25 +765,29 @@ _reset_gl (GtkClapperGLWidget * clapper_widget) gdk_gl_context_make_current (priv->gdk_context); gst_gl_context_activate (priv->other_context, TRUE); - if (priv->vao) { - gl->DeleteVertexArrays (1, &priv->vao); - priv->vao = 0; - } + _cleanup_gl_private (priv); - if (priv->vertex_buffer) { - gl->DeleteBuffers (1, &priv->vertex_buffer); - priv->vertex_buffer = 0; - } + gst_gl_context_activate (priv->other_context, FALSE); + gdk_gl_context_clear_current (); - if (priv->upload) { - gst_object_unref (priv->upload); - priv->upload = NULL; - } + priv->initiated = FALSE; +} - if (priv->shader) { - gst_object_unref (priv->shader); - priv->shader = NULL; - } +static void +_reset_gl (GtkClapperGLWidget * clapper_widget) +{ + GtkClapperGLWidgetPrivate *priv = clapper_widget->priv; + + if (!priv->gdk_context) + priv->gdk_context = gtk_gl_area_get_context (GTK_GL_AREA (clapper_widget)); + + if (priv->gdk_context == NULL) + return; + + gdk_gl_context_make_current (priv->gdk_context); + gst_gl_context_activate (priv->other_context, TRUE); + + _cleanup_gl_private (priv); if (priv->overlay_compositor) gst_object_unref (priv->overlay_compositor); @@ -1010,6 +1090,9 @@ gtk_clapper_gl_widget_init (GtkClapperGLWidget * clapper_widget) GST_INFO ("Created %" GST_PTR_FORMAT, priv->display); + priv->texture_target = GST_GL_TEXTURE_TARGET_NONE; + priv->gl_target = 0; + gtk_gl_area_set_auto_render (GTK_GL_AREA (widget), FALSE); g_signal_connect_swapped (gtk_widget_get_settings (widget), "notify", @@ -1094,3 +1177,44 @@ gtk_clapper_gl_widget_get_display (GtkClapperGLWidget * clapper_widget) return gst_object_ref (clapper_widget->priv->display); } + +gboolean +gtk_clapper_gl_widget_update_output_format (GtkClapperGLWidget * clapper_widget, + GstCaps * caps) +{ + GtkClapperGLWidgetPrivate *priv; + GstGLTextureTarget previous_target; + GstStructure *structure; + const gchar *target_str; + gboolean cleanup_gl; + + GTK_CLAPPER_GL_WIDGET_LOCK (clapper_widget); + priv = clapper_widget->priv; + + previous_target = priv->texture_target; + structure = gst_caps_get_structure (caps, 0); + target_str = gst_structure_get_string (structure, "texture-target"); + + if (!target_str) + target_str = GST_GL_TEXTURE_TARGET_2D_STR; + + priv->texture_target = gst_gl_texture_target_from_string (target_str); + if (!priv->texture_target) + goto fail; + + GST_DEBUG_OBJECT (clapper_widget, "Using texture-target: %s", target_str); + priv->gl_target = gst_gl_texture_target_to_gl (priv->texture_target); + + cleanup_gl = (previous_target != GST_GL_TEXTURE_TARGET_NONE && + priv->texture_target != previous_target); + + GTK_CLAPPER_GL_WIDGET_UNLOCK (clapper_widget); + if (cleanup_gl) + gst_gtk_invoke_on_main ((GThreadFunc) (GCallback) _cleanup_gl_thread, clapper_widget); + + return TRUE; + +fail: + GTK_CLAPPER_GL_WIDGET_UNLOCK (clapper_widget); + return FALSE; +} diff --git a/lib/gst/clapper/gtk4/gtkclapperglwidget.h b/lib/gst/clapper/gtk4/gtkclapperglwidget.h index acfbe412..f9f2d303 100644 --- a/lib/gst/clapper/gtk4/gtkclapperglwidget.h +++ b/lib/gst/clapper/gtk4/gtkclapperglwidget.h @@ -100,12 +100,13 @@ gboolean gtk_clapper_gl_widget_set_format (GtkClapperGLWidget * void gtk_clapper_gl_widget_set_buffer (GtkClapperGLWidget * widget, GstBuffer * buffer); void gtk_clapper_gl_widget_set_element (GtkClapperGLWidget * widget, GstElement * element); -GtkWidget * gtk_clapper_gl_widget_new (void); +GtkWidget * gtk_clapper_gl_widget_new (void); gboolean gtk_clapper_gl_widget_init_winsys (GtkClapperGLWidget * widget); GstGLDisplay * gtk_clapper_gl_widget_get_display (GtkClapperGLWidget * widget); GstGLContext * gtk_clapper_gl_widget_get_context (GtkClapperGLWidget * widget); GstGLContext * gtk_clapper_gl_widget_get_gtk_context (GtkClapperGLWidget * widget); +gboolean gtk_clapper_gl_widget_update_output_format (GtkClapperGLWidget * widget, GstCaps * caps); G_END_DECLS diff --git a/lib/gst/clapper/meson.build b/lib/gst/clapper/meson.build index 517fc672..ed4e4efe 100644 --- a/lib/gst/clapper/meson.build +++ b/lib/gst/clapper/meson.build @@ -14,6 +14,7 @@ gstclapper_sources = [ 'gtk4/gstclapperglsink.c', 'gtk4/gstgtkutils.c', 'gtk4/gtkclapperglwidget.c', + 'gtk4/gstclapperglutils.c', ] gstclapper_headers = [ 'clapper.h',