From 0410c6e9b524d63431ebe658f562e286931748b3 Mon Sep 17 00:00:00 2001 From: Rafostar <40623528+Rafostar@users.noreply.github.com> Date: Sun, 1 May 2022 17:08:48 +0200 Subject: [PATCH] plugin: Add GL uploader Add "clappergluploader" module which uses "glupload" and "glcolorconvert" internally, allowing either uploading RAW system memory into GPU with GL colorspace conversion or importing DMABufs into GL memory and making a GdkTexture out of them at one go. --- .../plugin/importers/gstclappergluploader.c | 232 ++++++++++++++++++ .../plugin/importers/gstclappergluploader.h | 42 ++++ lib/gst/plugin/importers/meson.build | 15 ++ meson_options.txt | 5 + 4 files changed, 294 insertions(+) create mode 100644 lib/gst/plugin/importers/gstclappergluploader.c create mode 100644 lib/gst/plugin/importers/gstclappergluploader.h diff --git a/lib/gst/plugin/importers/gstclappergluploader.c b/lib/gst/plugin/importers/gstclappergluploader.c new file mode 100644 index 00000000..03e318ee --- /dev/null +++ b/lib/gst/plugin/importers/gstclappergluploader.c @@ -0,0 +1,232 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstclappergluploader.h" + +#define GST_CAT_DEFAULT gst_clapper_gl_uploader_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +#define parent_class gst_clapper_gl_uploader_parent_class +GST_CLAPPER_IMPORTER_DEFINE (GstClapperGLUploader, gst_clapper_gl_uploader, GST_TYPE_CLAPPER_GL_BASE_IMPORTER); + +static void +_update_elements_caps_locked (GstClapperGLUploader *self, GstCaps *upload_sink_caps) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self); + GstCaps *upload_src_caps, *color_sink_caps, *color_src_caps, *gdk_sink_caps; + + GST_INFO_OBJECT (self, "Input caps: %" GST_PTR_FORMAT, upload_sink_caps); + + upload_src_caps = gst_gl_upload_transform_caps (self->upload, gl_bi->gst_context, + GST_PAD_SINK, upload_sink_caps, NULL); + upload_src_caps = gst_caps_fixate (upload_src_caps); + + GST_INFO_OBJECT (self, "GLUpload caps: %" GST_PTR_FORMAT, upload_src_caps); + gst_gl_upload_set_caps (self->upload, upload_sink_caps, upload_src_caps); + + gdk_sink_caps = gst_clapper_gl_base_importer_make_supported_gdk_gl_caps (); + color_sink_caps = gst_gl_color_convert_transform_caps (gl_bi->gst_context, + GST_PAD_SRC, upload_src_caps, gdk_sink_caps); + gst_caps_unref (gdk_sink_caps); + + /* Second caps arg is transfer-full */ + color_src_caps = gst_gl_color_convert_fixate_caps (gl_bi->gst_context, + GST_PAD_SINK, upload_src_caps, color_sink_caps); + + GST_INFO_OBJECT (self, "GLColorConvert caps: %" GST_PTR_FORMAT, color_src_caps); + gst_gl_color_convert_set_caps (self->color_convert, upload_src_caps, color_src_caps); + + self->has_pending_v_info = gst_video_info_from_caps (&self->pending_v_info, color_src_caps); + + gst_caps_unref (upload_src_caps); + gst_caps_unref (color_src_caps); +} + +static void +gst_clapper_gl_uploader_set_caps (GstClapperImporter *importer, GstCaps *caps) +{ + GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (importer); + + GST_OBJECT_LOCK (self); + _update_elements_caps_locked (self, caps); + GST_OBJECT_UNLOCK (self); +} + +static void +_uploader_reconfigure_locked (GstClapperGLUploader *self) +{ + GstCaps *in_caps = NULL; + + GST_DEBUG_OBJECT (self, "Reconfiguring upload"); + + gst_gl_upload_get_caps (self->upload, &in_caps, NULL); + + if (G_LIKELY (in_caps)) { + _update_elements_caps_locked (self, in_caps); + gst_caps_unref (in_caps); + } +} + +static gboolean +gst_clapper_gl_uploader_prepare (GstClapperImporter *importer) +{ + gboolean res = GST_CLAPPER_IMPORTER_CLASS (parent_class)->prepare (importer); + + if (res) { + GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (importer); + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); + + GST_OBJECT_LOCK (self); + + if (!self->upload) + self->upload = gst_gl_upload_new (gl_bi->gst_context); + if (!self->color_convert) + self->color_convert = gst_gl_color_convert_new (gl_bi->gst_context); + + GST_OBJECT_UNLOCK (self); + } + + return res; +} + +static GstBuffer * +_upload_perform_locked (GstClapperGLUploader *self, GstBuffer *buffer) +{ + GstBuffer *upload_buf = NULL; + GstGLUploadReturn ret; + + ret = gst_gl_upload_perform_with_buffer (self->upload, buffer, &upload_buf); + + if (G_UNLIKELY (ret != GST_GL_UPLOAD_DONE)) { + switch (ret) { + case GST_GL_UPLOAD_RECONFIGURE: + _uploader_reconfigure_locked (self); + /* Retry with the same buffer after reconfiguring */ + return _upload_perform_locked (self, buffer); + default: + GST_ERROR_OBJECT (self, "Could not upload input buffer, returned: %i", ret); + break; + } + } + + return upload_buf; +} + +static GdkTexture * +gst_clapper_gl_uploader_generate_texture (GstClapperImporter *importer, + GstBuffer *buffer, GstVideoInfo *v_info) +{ + GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (importer); + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); + GstBuffer *upload_buf, *color_buf; + GdkTexture *texture; + + /* XXX: We both upload and perform color conversion here, thus we skip + * upload for buffers that are not going to be shown and gain more free + * CPU time to prepare the next one. Improves performance on weak HW. */ + + GST_LOG_OBJECT (self, "Uploading %" GST_PTR_FORMAT, buffer); + + GST_OBJECT_LOCK (self); + + upload_buf = _upload_perform_locked (self, buffer); + + if (G_UNLIKELY (!upload_buf)) { + GST_ERROR_OBJECT (self, "Could not perform upload on input buffer"); + GST_OBJECT_UNLOCK (self); + + return NULL; + } + GST_LOG_OBJECT (self, "Uploaded into %" GST_PTR_FORMAT, upload_buf); + + color_buf = gst_gl_color_convert_perform (self->color_convert, upload_buf); + gst_buffer_unref (upload_buf); + + /* Use video info associated with converted buffer */ + if (self->has_pending_v_info) { + self->v_info = self->pending_v_info; + self->has_pending_v_info = FALSE; + } + + GST_OBJECT_UNLOCK (self); + + if (G_UNLIKELY (!color_buf)) { + GST_ERROR_OBJECT (self, "Could not perform color conversion on input buffer"); + return NULL; + } + GST_LOG_OBJECT (self, "Color converted into %" GST_PTR_FORMAT, color_buf); + + texture = gst_clapper_gl_base_importer_make_gl_texture (gl_bi, color_buf, &self->v_info); + gst_buffer_unref (color_buf); + + return texture; +} + +static void +gst_clapper_gl_uploader_init (GstClapperGLUploader *self) +{ + gst_video_info_init (&self->pending_v_info); + gst_video_info_init (&self->v_info); +} + +static void +gst_clapper_gl_uploader_finalize (GObject *object) +{ + GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (object); + + gst_clear_object (&self->upload); + gst_clear_object (&self->color_convert); + + GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object)); +} + +static void +gst_clapper_gl_uploader_class_init (GstClapperGLUploaderClass *klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstClapperImporterClass *importer_class = (GstClapperImporterClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "clappergluploader", 0, + "Clapper GL Uploader"); + + gobject_class->finalize = gst_clapper_gl_uploader_finalize; + + importer_class->prepare = gst_clapper_gl_uploader_prepare; + importer_class->set_caps = gst_clapper_gl_uploader_set_caps; + importer_class->generate_texture = gst_clapper_gl_uploader_generate_texture; +} + +GstClapperImporter * +make_importer (void) +{ + return g_object_new (GST_TYPE_CLAPPER_GL_UPLOADER, NULL); +} + +GstCaps * +make_caps (GstRank *rank, GStrv *context_types) +{ + *rank = GST_RANK_MARGINAL + 1; + *context_types = gst_clapper_gl_base_importer_make_gl_context_types (); + + return gst_gl_upload_get_input_template_caps (); +} diff --git a/lib/gst/plugin/importers/gstclappergluploader.h b/lib/gst/plugin/importers/gstclappergluploader.h new file mode 100644 index 00000000..aa76f9c2 --- /dev/null +++ b/lib/gst/plugin/importers/gstclappergluploader.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2022 Rafał Dzięgiel + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include "gstclapperglbaseimporter.h" + +G_BEGIN_DECLS + +#define GST_TYPE_CLAPPER_GL_UPLOADER (gst_clapper_gl_uploader_get_type()) +G_DECLARE_FINAL_TYPE (GstClapperGLUploader, gst_clapper_gl_uploader, GST, CLAPPER_GL_UPLOADER, GstClapperGLBaseImporter) + +#define GST_CLAPPER_GL_UPLOADER_CAST(obj) ((GstClapperGLUploader *)(obj)) + +struct _GstClapperGLUploader +{ + GstClapperGLBaseImporter parent; + + GstGLUpload *upload; + GstGLColorConvert *color_convert; + + GstVideoInfo pending_v_info, v_info; + gboolean has_pending_v_info; +}; + +G_END_DECLS diff --git a/lib/gst/plugin/importers/meson.build b/lib/gst/plugin/importers/meson.build index a8daaaa8..a435604d 100644 --- a/lib/gst/plugin/importers/meson.build +++ b/lib/gst/plugin/importers/meson.build @@ -14,6 +14,7 @@ endif plugin_needs_gl_base = ( not get_option('glimporter').disabled() + or not get_option('gluploader').disabled() ) gst_plugin_gl_deps = [gstgl_dep, gstglproto_dep] @@ -50,6 +51,8 @@ foreach dep : gst_clapper_gl_base_importer_deps if not dep.found() or not have_gtk_gl_windowing if get_option('glimporter').enabled() error('GL importer was enabled, but required dependencies were not found') + elif get_option('gluploader').enabled() + error('GL uploader was enabled, but required dependencies were not found') endif plugin_needs_gl_base = false endif @@ -81,3 +84,15 @@ if not get_option('glimporter').disabled() and gst_clapper_gl_base_importer_dep. install_dir: gst_clapper_importers_libdir, ) endif + +if not get_option('gluploader').disabled() and gst_clapper_gl_base_importer_dep.found() + library( + 'gstclappergluploader', + 'gstclappergluploader.c', + dependencies: gst_clapper_gl_base_importer_dep, + include_directories: configinc, + c_args: gst_clapper_plugin_args, + install: true, + install_dir: gst_clapper_importers_libdir, + ) +endif diff --git a/meson_options.txt b/meson_options.txt index a5fc377b..67e9c482 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -24,6 +24,11 @@ option('glimporter', value: 'auto', description: 'Build GL memory importer for clappersink' ) +option('gluploader', + type: 'feature', + value: 'auto', + description: 'Build GL uploader for clappersink' +) option('devel-checks', type: 'boolean',