diff --git a/lib/gst/plugin/importers/gstclappergluploader.c b/lib/gst/plugin/importers/gstclappergluploader.c index 03e318ee..a68428ce 100644 --- a/lib/gst/plugin/importers/gstclappergluploader.c +++ b/lib/gst/plugin/importers/gstclappergluploader.c @@ -22,6 +22,36 @@ #endif #include "gstclappergluploader.h" +#include "gst/plugin/gstgtkutils.h" + +#include +#include + +static const GLfloat vertices[] = { + 1.0f, 1.0f, 0.0f, 1.0f, 0.0f, + -1.0f, 1.0f, 0.0f, 0.0f, 0.0f, + -1.0f, -1.0f, 0.0f, 0.0f, 1.0f, + 1.0f, -1.0f, 0.0f, 1.0f, 1.0f +}; +static const GLushort indices[] = { + 0, 1, 2, 0, 2, 3 +}; + +/* GTK4 renders things upside down ¯\_(ツ)_/¯ */ +static const gfloat vertical_flip_matrix[] = { + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, +}; + +typedef struct +{ + GstClapperGLUploader *dmabuf_bi; + GLuint id; + guint width; + guint height; +} GstClapperDmabufTexData; #define GST_CAT_DEFAULT gst_clapper_gl_uploader_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); @@ -29,6 +59,350 @@ GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); #define parent_class gst_clapper_gl_uploader_parent_class GST_CLAPPER_IMPORTER_DEFINE (GstClapperGLUploader, gst_clapper_gl_uploader, GST_TYPE_CLAPPER_GL_BASE_IMPORTER); +static void +gst_clapper_gl_uploader_bind_buffer (GstClapperGLUploader *self) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self); + const GstGLFuncs *gl = gl_bi->gst_context->gl_vtable; + + gl->BindBuffer (GL_ARRAY_BUFFER, self->vertex_buffer); + + /* Load the vertex position */ + gl->VertexAttribPointer (self->attr_position, 3, GL_FLOAT, GL_FALSE, + 5 * sizeof (GLfloat), (void *) 0); + + /* Load the texture coordinate */ + gl->VertexAttribPointer (self->attr_texture, 2, GL_FLOAT, GL_FALSE, + 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat))); + + gl->EnableVertexAttribArray (self->attr_position); + gl->EnableVertexAttribArray (self->attr_texture); +} + +static void +gst_clapper_gl_uploader_unbind_buffer (GstClapperGLUploader *self) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self); + const GstGLFuncs *gl = gl_bi->gst_context->gl_vtable; + + gl->BindBuffer (GL_ARRAY_BUFFER, 0); + gl->DisableVertexAttribArray (self->attr_position); + gl->DisableVertexAttribArray (self->attr_texture); +} + +static gboolean +prepare_dmabuf_support_on_main (GstClapperGLUploader *self) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self); + GstGLSLStage *frag_stage, *vert_stage; + GError *error = NULL; + gchar *frag_str; + const GstGLFuncs *gl; + + GST_OBJECT_LOCK (self); + + /* FIXME: Return if already prepared */ + + gdk_gl_context_make_current (gl_bi->gdk_context); + gst_gl_context_activate (gl_bi->gst_context, TRUE); + + if (!((vert_stage = gst_glsl_stage_new_with_string (gl_bi->gst_context, + GL_VERTEX_SHADER, GST_GLSL_VERSION_NONE, + GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY, + gst_gl_shader_string_vertex_mat4_vertex_transform)))) { + gdk_gl_context_make_current (gl_bi->gdk_context); + gst_gl_context_activate (gl_bi->gst_context, TRUE); + + GST_OBJECT_UNLOCK (self); + GST_ERROR ("Failed to retrieve vertex shader for texture target"); + + return FALSE; + } + + frag_str = gst_gl_shader_string_fragment_external_oes_get_default ( + gl_bi->gst_context, GST_GLSL_VERSION_NONE, + GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY); + frag_stage = gst_glsl_stage_new_with_string (gl_bi->gst_context, + GL_FRAGMENT_SHADER, GST_GLSL_VERSION_NONE, + GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY, frag_str); + + g_free (frag_str); + + if (!frag_stage) { + gst_gl_context_activate (gl_bi->gst_context, FALSE); + gdk_gl_context_clear_current (); + + GST_OBJECT_UNLOCK (self); + GST_ERROR ("Failed to retrieve fragment shader for texture target"); + + return FALSE; + } + + if (!((self->shader = gst_gl_shader_new_link_with_stages (gl_bi->gst_context, + &error, vert_stage, frag_stage, NULL)))) { + gst_gl_context_activate (gl_bi->gst_context, FALSE); + gdk_gl_context_clear_current (); + + GST_OBJECT_UNLOCK (self); + + GST_ERROR ("Failed to initialize shader: %s", error->message); + g_clear_error (&error); + + return FALSE; + } + + self->attr_position = + gst_gl_shader_get_attribute_location (self->shader, "a_position"); + self->attr_texture = + gst_gl_shader_get_attribute_location (self->shader, "a_texcoord"); + + gl = gl_bi->gst_context->gl_vtable; + + if (gl->GenVertexArrays) { + gl->GenVertexArrays (1, &self->vao); + gl->BindVertexArray (self->vao); + } + + gl->GenBuffers (1, &self->vertex_buffer); + gl->BindBuffer (GL_ARRAY_BUFFER, self->vertex_buffer); + gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices, GL_STATIC_DRAW); + + if (gl->GenVertexArrays) { + gst_clapper_gl_uploader_bind_buffer (self); + gl->BindVertexArray (0); + } + + gl->BindBuffer (GL_ARRAY_BUFFER, 0); + //self->prepared = TRUE; + + gst_gl_context_activate (gl_bi->gst_context, FALSE); + gdk_gl_context_clear_current (); + + GST_OBJECT_UNLOCK (self); + + return TRUE; +} + +static void +_tex_data_free (GstClapperDmabufTexData *tex_data) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (tex_data->dmabuf_bi); + + if (G_LIKELY (tex_data->id > 0)) { + const GstGLFuncs *gl; + + GST_OBJECT_LOCK (gl_bi); + + gl = gl_bi->gst_context->gl_vtable; + + gdk_gl_context_make_current (gl_bi->gdk_context); + gst_gl_context_activate (gl_bi->gst_context, TRUE); + + gl->DeleteTextures (1, &tex_data->id); + + gst_gl_context_activate (gl_bi->gst_context, FALSE); + gdk_gl_context_clear_current (); + + GST_OBJECT_UNLOCK (gl_bi); + } + + gst_object_unref (tex_data->dmabuf_bi); + g_slice_free (GstClapperDmabufTexData, tex_data); +} + +static gboolean +_dmabuf_into_texture (GstClapperGLUploader *self, gint *fds, GstVideoInfo *v_info, + gsize *offsets, GstClapperDmabufTexData *tex_data) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self); + GstEGLImage *image; + const GstGLFuncs *gl; + + image = gst_egl_image_from_dmabuf_direct_target (gl_bi->gst_context, + fds, offsets, v_info, self->gst_tex_target); + + /* If HW colorspace conversion failed and there is only one + * plane, we can just make it into single EGLImage as is */ + if (!image && GST_VIDEO_INFO_N_PLANES (v_info) == 1) + image = gst_egl_image_from_dmabuf (gl_bi->gst_context, + fds[0], v_info, 0, offsets[0]); + + if (!image) + return FALSE; + + gl = gl_bi->gst_context->gl_vtable; + + gl->GenTextures (1, &tex_data->id); + tex_data->width = GST_VIDEO_INFO_WIDTH (v_info); + tex_data->height = GST_VIDEO_INFO_HEIGHT (v_info); + + gl->BindTexture (self->gl_tex_target, tex_data->id); + + gl->TexParameteri (self->gl_tex_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->TexParameteri (self->gl_tex_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + + gl->TexParameteri (self->gl_tex_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->TexParameteri (self->gl_tex_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + gl->EGLImageTargetTexture2D (self->gl_tex_target, gst_egl_image_get_image (image)); + + gl->BindTexture (GL_TEXTURE_2D, 0); + gst_egl_image_unref (image); + + return TRUE; +} + +static gboolean +_oes_texture_into_2d (GstClapperGLUploader *self, GstClapperDmabufTexData *tex_data) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self); + GLuint framebuffer, tex_id; + GLenum status; + const GstGLFuncs *gl; + + gl = gl_bi->gst_context->gl_vtable; + + gl->GenFramebuffers (1, &framebuffer); + gl->BindFramebuffer (GL_FRAMEBUFFER, framebuffer); + + gl->GenTextures (1, &tex_id); + gl->BindTexture (GL_TEXTURE_2D, tex_id); + + gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + + gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + gl->TexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, tex_data->width, tex_data->height, 0, + GL_RGBA, GL_UNSIGNED_BYTE, NULL); + + gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, tex_id, 0); + + status = gl->CheckFramebufferStatus (GL_FRAMEBUFFER); + if (G_UNLIKELY (status != GL_FRAMEBUFFER_COMPLETE)) { + GST_ERROR ("Invalid framebuffer status: %u", status); + + gl->BindTexture (GL_TEXTURE_2D, 0); + gl->DeleteTextures (1, &tex_id); + + gl->BindFramebuffer (GL_FRAMEBUFFER, 0); + gl->DeleteFramebuffers (1, &framebuffer); + + return FALSE; + } + + gl->Viewport (0, 0, tex_data->width, tex_data->height); + + gst_gl_shader_use (self->shader); + + if (gl->BindVertexArray) + gl->BindVertexArray (self->vao); + + gst_clapper_gl_uploader_bind_buffer (self); + + gl->ActiveTexture (GL_TEXTURE0); + gl->BindTexture (self->gl_tex_target, tex_data->id); + + gst_gl_shader_set_uniform_1i (self->shader, "tex", 0); + gst_gl_shader_set_uniform_matrix_4fv (self->shader, + "u_transformation", 1, FALSE, vertical_flip_matrix); + + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); + + if (gl->BindVertexArray) + gl->BindVertexArray (0); + else + gst_clapper_gl_uploader_unbind_buffer (self); + + gl->BindTexture (self->gl_tex_target, 0); + + /* Replace External OES texture with newly created 2D */ + gl->DeleteTextures (1, &tex_data->id); + tex_data->id = tex_id; + + gl->BindFramebuffer (GL_FRAMEBUFFER, 0); + gl->DeleteFramebuffers (1, &framebuffer); + + return TRUE; +} + +static GdkTexture * +dmabuf_into_gdk_texture (GstClapperGLUploader *self, GstVideoInfo *v_info, gint *fds, gsize *offsets) +{ + GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self); + GdkTexture *texture = NULL; + GstClapperDmabufTexData *tex_data; + + tex_data = g_slice_new (GstClapperDmabufTexData); + tex_data->dmabuf_bi = gst_object_ref (self); + + GST_OBJECT_LOCK (self); + + gdk_gl_context_make_current (gl_bi->gdk_context); + gst_gl_context_activate (gl_bi->gst_context, TRUE); + + if (!_dmabuf_into_texture (self, fds, v_info, offsets, tex_data)) + goto finish; + + /* GTK4 does not support External OES textures. + * Make it into 2D using framebuffer + shader */ + if (self->gst_tex_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) { + if (G_UNLIKELY (!_oes_texture_into_2d (self, tex_data))) + goto finish; + } + + texture = gdk_gl_texture_new (gl_bi->gdk_context, + tex_data->id, + tex_data->width, + tex_data->height, + (GDestroyNotify) _tex_data_free, + tex_data); + +finish: + gst_gl_context_activate (gl_bi->gst_context, FALSE); + gdk_gl_context_clear_current (); + + GST_OBJECT_UNLOCK (self); + + return texture; +} + +static gboolean +verify_dmabuf_memory (GstBuffer *buffer, GstVideoInfo *info, gint *fds, gsize *offsets) +{ + guint i, n_planes = GST_VIDEO_INFO_N_PLANES (info); + + for (i = 0; i < n_planes; i++) { + GstMemory *memory; + gsize plane_size, mem_skip; + guint mem_idx, length; + + plane_size = gst_gl_get_plane_data_size (info, NULL, i); + + if (!gst_buffer_find_memory (buffer, + GST_VIDEO_INFO_PLANE_OFFSET (info, i), + plane_size, &mem_idx, &length, &mem_skip)) { + GST_DEBUG ("Could not find memory %u", i); + return FALSE; + } + + /* We cannot have more then one DMABuf per plane */ + if (length != 1) { + GST_DEBUG ("Data for plane %u spans %u memories", i, length); + return FALSE; + } + + memory = gst_buffer_peek_memory (buffer, mem_idx); + + offsets[i] = memory->offset + mem_skip; + fds[i] = gst_dmabuf_memory_get_fd (memory); + } + + return TRUE; +} + static void _update_elements_caps_locked (GstClapperGLUploader *self, GstCaps *upload_sink_caps) { @@ -104,6 +478,14 @@ gst_clapper_gl_uploader_prepare (GstClapperImporter *importer) self->color_convert = gst_gl_color_convert_new (gl_bi->gst_context); GST_OBJECT_UNLOCK (self); + + if (!(! !gst_gtk_invoke_on_main ( + (GThreadFunc) (GCallback) prepare_dmabuf_support_on_main, self))) { + GST_WARNING_OBJECT (self, "Could not prepare DMABuf support"); + + /* FIXME: Continue to allow using glupload/cc as fallback */ + return FALSE; + } } return res; @@ -139,12 +521,38 @@ gst_clapper_gl_uploader_generate_texture (GstClapperImporter *importer, GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (importer); GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer); GstBuffer *upload_buf, *color_buf; + GstVideoMeta *meta; GdkTexture *texture; /* XXX: We both upload and perform color conversion here, thus we skip * upload for buffers that are not going to be shown and gain more free * CPU time to prepare the next one. Improves performance on weak HW. */ + if ((meta = gst_buffer_get_video_meta (buffer))) { + guint i; + + GST_VIDEO_INFO_WIDTH (v_info) = meta->width; + GST_VIDEO_INFO_HEIGHT (v_info) = meta->height; + + for (i = 0; i < meta->n_planes; i++) { + GST_VIDEO_INFO_PLANE_OFFSET (v_info, i) = meta->offset[i]; + GST_VIDEO_INFO_PLANE_STRIDE (v_info, i) = meta->stride[i]; + } + } + + /* FIXME: if can do dmabuf and seems like we have dmabuf here */ + { + gint fds[GST_VIDEO_MAX_PLANES]; + gsize offsets[GST_VIDEO_MAX_PLANES]; + + if (verify_dmabuf_memory (buffer, v_info, fds, offsets)) { + if ((texture = dmabuf_into_gdk_texture (self, v_info, fds, offsets))) { + GST_TRACE_OBJECT (self, "Got texture from DMABuf, skipping upload of %" GST_PTR_FORMAT, buffer); + goto done; + } + } + } + GST_LOG_OBJECT (self, "Uploading %" GST_PTR_FORMAT, buffer); GST_OBJECT_LOCK (self); @@ -179,6 +587,7 @@ gst_clapper_gl_uploader_generate_texture (GstClapperImporter *importer, texture = gst_clapper_gl_base_importer_make_gl_texture (gl_bi, color_buf, &self->v_info); gst_buffer_unref (color_buf); +done: return texture; } @@ -187,6 +596,9 @@ gst_clapper_gl_uploader_init (GstClapperGLUploader *self) { gst_video_info_init (&self->pending_v_info); gst_video_info_init (&self->v_info); + + self->gst_tex_target = GST_GL_TEXTURE_TARGET_EXTERNAL_OES; + self->gl_tex_target = gst_gl_texture_target_to_gl (self->gst_tex_target); } static void @@ -197,6 +609,8 @@ gst_clapper_gl_uploader_finalize (GObject *object) gst_clear_object (&self->upload); gst_clear_object (&self->color_convert); + gst_clear_object (&self->shader); + GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object)); } diff --git a/lib/gst/plugin/importers/gstclappergluploader.h b/lib/gst/plugin/importers/gstclappergluploader.h index aa76f9c2..3386fe22 100644 --- a/lib/gst/plugin/importers/gstclappergluploader.h +++ b/lib/gst/plugin/importers/gstclappergluploader.h @@ -21,6 +21,8 @@ #include "gstclapperglbaseimporter.h" +#include + G_BEGIN_DECLS #define GST_TYPE_CLAPPER_GL_UPLOADER (gst_clapper_gl_uploader_get_type()) @@ -37,6 +39,17 @@ struct _GstClapperGLUploader GstVideoInfo pending_v_info, v_info; gboolean has_pending_v_info; + + /* DMABuf fast-path */ + GstGLTextureTarget gst_tex_target; + guint gl_tex_target; + + GstGLShader *shader; + + GLuint vao; + GLuint vertex_buffer; + GLint attr_position; + GLint attr_texture; }; G_END_DECLS diff --git a/lib/gst/plugin/importers/meson.build b/lib/gst/plugin/importers/meson.build index 5f190b54..d945c8e3 100644 --- a/lib/gst/plugin/importers/meson.build +++ b/lib/gst/plugin/importers/meson.build @@ -100,11 +100,25 @@ build_gluploader = ( and gst_clapper_gl_base_importer_dep.found() ) +gluploader_deps = [ + gst_clapper_gl_base_importer_dep, + gstallocators_dep, +] + +foreach dep : gluploader_deps + if not dep.found() + if get_option('gluploader').enabled() + error('GL uploader was enabled, but required dependencies were not found') + endif + build_gluploader = false + endif +endforeach + if build_gluploader library( 'gstclappergluploader', 'gstclappergluploader.c', - dependencies: gst_clapper_gl_base_importer_dep, + dependencies: gluploader_deps, include_directories: configinc, c_args: gst_clapper_plugin_args, install: true, diff --git a/lib/meson.build b/lib/meson.build index 45b94330..dbe29a74 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -196,6 +196,8 @@ gsttag_dep = dependency('gstreamer-tag-1.0', version: gst_req, fallback: ['gst-plugins-base', 'tag_dep']) gstvideo_dep = dependency('gstreamer-video-1.0', version: gst_req, fallback: ['gst-plugins-base', 'video_dep']) +gstallocators_dep = dependency('gstreamer-allocators-1.0', version: gst_req, + fallback : ['gst-plugins-base', 'allocators_dep']) # GStreamer OpenGL gstgl_dep = dependency('gstreamer-gl-1.0', version: gst_req,