gluploader: Handle DMABufs directly

lib/gst/plugin/importers/gstclappergluploader.c

@@ -22,6 +22,36 @@
 #endif
 
 #include "gstclappergluploader.h"
+#include "gst/plugin/gstgtkutils.h"
+
+#include <gst/gl/egl/gsteglimage.h>
+#include <gst/allocators/gstdmabuf.h>
+
+static const GLfloat vertices[] = {
+  1.0f, 1.0f, 0.0f, 1.0f, 0.0f,
+  -1.0f, 1.0f, 0.0f, 0.0f, 0.0f,
+  -1.0f, -1.0f, 0.0f, 0.0f, 1.0f,
+  1.0f, -1.0f, 0.0f, 1.0f, 1.0f
+};
+static const GLushort indices[] = {
+  0, 1, 2, 0, 2, 3
+};
+
+/* GTK4 renders things upside down ¯\_(ツ)_/¯ */
+static const gfloat vertical_flip_matrix[] = {
+  1.0f, 0.0f, 0.0f, 0.0f,
+  0.0f, -1.0f, 0.0f, 0.0f,
+  0.0f, 0.0f, 1.0f, 0.0f,
+  0.0f, 0.0f, 0.0f, 1.0f,
+};
+
+typedef struct
+{
+  GstClapperGLUploader *dmabuf_bi;
+  GLuint id;
+  guint width;
+  guint height;
+} GstClapperDmabufTexData;
 
 #define GST_CAT_DEFAULT gst_clapper_gl_uploader_debug
 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
@@ -29,6 +59,350 @@ GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
 #define parent_class gst_clapper_gl_uploader_parent_class
 GST_CLAPPER_IMPORTER_DEFINE (GstClapperGLUploader, gst_clapper_gl_uploader, GST_TYPE_CLAPPER_GL_BASE_IMPORTER);
 
+static void
+gst_clapper_gl_uploader_bind_buffer (GstClapperGLUploader *self)
+{
+  GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self);
+  const GstGLFuncs *gl = gl_bi->gst_context->gl_vtable;
+
+  gl->BindBuffer (GL_ARRAY_BUFFER, self->vertex_buffer);
+
+  /* Load the vertex position */
+  gl->VertexAttribPointer (self->attr_position, 3, GL_FLOAT, GL_FALSE,
+      5 * sizeof (GLfloat), (void *) 0);
+
+  /* Load the texture coordinate */
+  gl->VertexAttribPointer (self->attr_texture, 2, GL_FLOAT, GL_FALSE,
+      5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
+
+  gl->EnableVertexAttribArray (self->attr_position);
+  gl->EnableVertexAttribArray (self->attr_texture);
+}
+
+static void
+gst_clapper_gl_uploader_unbind_buffer (GstClapperGLUploader *self)
+{
+  GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self);
+  const GstGLFuncs *gl = gl_bi->gst_context->gl_vtable;
+
+  gl->BindBuffer (GL_ARRAY_BUFFER, 0);
+  gl->DisableVertexAttribArray (self->attr_position);
+  gl->DisableVertexAttribArray (self->attr_texture);
+}
+
+static gboolean
+prepare_dmabuf_support_on_main (GstClapperGLUploader *self)
+{
+  GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self);
+  GstGLSLStage *frag_stage, *vert_stage;
+  GError *error = NULL;
+  gchar *frag_str;
+  const GstGLFuncs *gl;
+
+  GST_OBJECT_LOCK (self);
+
+  /* FIXME: Return if already prepared */
+
+  gdk_gl_context_make_current (gl_bi->gdk_context);
+  gst_gl_context_activate (gl_bi->gst_context, TRUE);
+
+  if (!((vert_stage = gst_glsl_stage_new_with_string (gl_bi->gst_context,
+      GL_VERTEX_SHADER, GST_GLSL_VERSION_NONE,
+      GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY,
+      gst_gl_shader_string_vertex_mat4_vertex_transform)))) {
+    gdk_gl_context_make_current (gl_bi->gdk_context);
+    gst_gl_context_activate (gl_bi->gst_context, TRUE);
+
+    GST_OBJECT_UNLOCK (self);
+    GST_ERROR ("Failed to retrieve vertex shader for texture target");
+
+    return FALSE;
+  }
+
+  frag_str = gst_gl_shader_string_fragment_external_oes_get_default (
+      gl_bi->gst_context, GST_GLSL_VERSION_NONE,
+      GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY);
+  frag_stage = gst_glsl_stage_new_with_string (gl_bi->gst_context,
+      GL_FRAGMENT_SHADER, GST_GLSL_VERSION_NONE,
+      GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY, frag_str);
+
+  g_free (frag_str);
+
+  if (!frag_stage) {
+    gst_gl_context_activate (gl_bi->gst_context, FALSE);
+    gdk_gl_context_clear_current ();
+
+    GST_OBJECT_UNLOCK (self);
+    GST_ERROR ("Failed to retrieve fragment shader for texture target");
+
+    return FALSE;
+  }
+
+  if (!((self->shader = gst_gl_shader_new_link_with_stages (gl_bi->gst_context,
+      &error, vert_stage, frag_stage, NULL)))) {
+    gst_gl_context_activate (gl_bi->gst_context, FALSE);
+    gdk_gl_context_clear_current ();
+
+    GST_OBJECT_UNLOCK (self);
+
+    GST_ERROR ("Failed to initialize shader: %s", error->message);
+    g_clear_error (&error);
+
+    return FALSE;
+  }
+
+  self->attr_position =
+      gst_gl_shader_get_attribute_location (self->shader, "a_position");
+  self->attr_texture =
+      gst_gl_shader_get_attribute_location (self->shader, "a_texcoord");
+
+  gl = gl_bi->gst_context->gl_vtable;
+
+  if (gl->GenVertexArrays) {
+    gl->GenVertexArrays (1, &self->vao);
+    gl->BindVertexArray (self->vao);
+  }
+
+  gl->GenBuffers (1, &self->vertex_buffer);
+  gl->BindBuffer (GL_ARRAY_BUFFER, self->vertex_buffer);
+  gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices, GL_STATIC_DRAW);
+
+  if (gl->GenVertexArrays) {
+    gst_clapper_gl_uploader_bind_buffer (self);
+    gl->BindVertexArray (0);
+  }
+
+  gl->BindBuffer (GL_ARRAY_BUFFER, 0);
+  //self->prepared = TRUE;
+
+  gst_gl_context_activate (gl_bi->gst_context, FALSE);
+  gdk_gl_context_clear_current ();
+
+  GST_OBJECT_UNLOCK (self);
+
+  return TRUE;
+}
+
+static void
+_tex_data_free (GstClapperDmabufTexData *tex_data)
+{
+  GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (tex_data->dmabuf_bi);
+
+  if (G_LIKELY (tex_data->id > 0)) {
+    const GstGLFuncs *gl;
+
+    GST_OBJECT_LOCK (gl_bi);
+
+    gl = gl_bi->gst_context->gl_vtable;
+
+    gdk_gl_context_make_current (gl_bi->gdk_context);
+    gst_gl_context_activate (gl_bi->gst_context, TRUE);
+
+    gl->DeleteTextures (1, &tex_data->id);
+
+    gst_gl_context_activate (gl_bi->gst_context, FALSE);
+    gdk_gl_context_clear_current ();
+
+    GST_OBJECT_UNLOCK (gl_bi);
+  }
+
+  gst_object_unref (tex_data->dmabuf_bi);
+  g_slice_free (GstClapperDmabufTexData, tex_data);
+}
+
+static gboolean
+_dmabuf_into_texture (GstClapperGLUploader *self, gint *fds, GstVideoInfo *v_info,
+    gsize *offsets, GstClapperDmabufTexData *tex_data)
+{
+  GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self);
+  GstEGLImage *image;
+  const GstGLFuncs *gl;
+
+  image = gst_egl_image_from_dmabuf_direct_target (gl_bi->gst_context,
+      fds, offsets, v_info, self->gst_tex_target);
+
+  /* If HW colorspace conversion failed and there is only one
+   * plane, we can just make it into single EGLImage as is */
+  if (!image && GST_VIDEO_INFO_N_PLANES (v_info) == 1)
+    image = gst_egl_image_from_dmabuf (gl_bi->gst_context,
+        fds[0], v_info, 0, offsets[0]);
+
+  if (!image)
+    return FALSE;
+
+  gl = gl_bi->gst_context->gl_vtable;
+
+  gl->GenTextures (1, &tex_data->id);
+  tex_data->width = GST_VIDEO_INFO_WIDTH (v_info);
+  tex_data->height = GST_VIDEO_INFO_HEIGHT (v_info);
+
+  gl->BindTexture (self->gl_tex_target, tex_data->id);
+
+  gl->TexParameteri (self->gl_tex_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+  gl->TexParameteri (self->gl_tex_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+
+  gl->TexParameteri (self->gl_tex_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  gl->TexParameteri (self->gl_tex_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+
+  gl->EGLImageTargetTexture2D (self->gl_tex_target, gst_egl_image_get_image (image));
+
+  gl->BindTexture (GL_TEXTURE_2D, 0);
+  gst_egl_image_unref (image);
+
+  return TRUE;
+}
+
+static gboolean
+_oes_texture_into_2d (GstClapperGLUploader *self, GstClapperDmabufTexData *tex_data)
+{
+  GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self);
+  GLuint framebuffer, tex_id;
+  GLenum status;
+  const GstGLFuncs *gl;
+
+  gl = gl_bi->gst_context->gl_vtable;
+
+  gl->GenFramebuffers (1, &framebuffer);
+  gl->BindFramebuffer (GL_FRAMEBUFFER, framebuffer);
+
+  gl->GenTextures (1, &tex_id);
+  gl->BindTexture (GL_TEXTURE_2D, tex_id);
+
+  gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+  gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+
+  gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+
+  gl->TexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, tex_data->width, tex_data->height, 0,
+      GL_RGBA, GL_UNSIGNED_BYTE, NULL);
+
+  gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
+      GL_TEXTURE_2D, tex_id, 0);
+
+  status = gl->CheckFramebufferStatus (GL_FRAMEBUFFER);
+  if (G_UNLIKELY (status != GL_FRAMEBUFFER_COMPLETE)) {
+    GST_ERROR ("Invalid framebuffer status: %u", status);
+
+    gl->BindTexture (GL_TEXTURE_2D, 0);
+    gl->DeleteTextures (1, &tex_id);
+
+    gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
+    gl->DeleteFramebuffers (1, &framebuffer);
+
+    return FALSE;
+  }
+
+  gl->Viewport (0, 0, tex_data->width, tex_data->height);
+
+  gst_gl_shader_use (self->shader);
+
+  if (gl->BindVertexArray)
+    gl->BindVertexArray (self->vao);
+
+  gst_clapper_gl_uploader_bind_buffer (self);
+
+  gl->ActiveTexture (GL_TEXTURE0);
+  gl->BindTexture (self->gl_tex_target, tex_data->id);
+
+  gst_gl_shader_set_uniform_1i (self->shader, "tex", 0);
+  gst_gl_shader_set_uniform_matrix_4fv (self->shader,
+      "u_transformation", 1, FALSE, vertical_flip_matrix);
+
+  gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices);
+
+  if (gl->BindVertexArray)
+    gl->BindVertexArray (0);
+  else
+    gst_clapper_gl_uploader_unbind_buffer (self);
+
+  gl->BindTexture (self->gl_tex_target, 0);
+
+  /* Replace External OES texture with newly created 2D */
+  gl->DeleteTextures (1, &tex_data->id);
+  tex_data->id = tex_id;
+
+  gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
+  gl->DeleteFramebuffers (1, &framebuffer);
+
+  return TRUE;
+}
+
+static GdkTexture *
+dmabuf_into_gdk_texture (GstClapperGLUploader *self, GstVideoInfo *v_info, gint *fds, gsize *offsets)
+{
+  GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (self);
+  GdkTexture *texture = NULL;
+  GstClapperDmabufTexData *tex_data;
+
+  tex_data = g_slice_new (GstClapperDmabufTexData);
+  tex_data->dmabuf_bi = gst_object_ref (self);
+
+  GST_OBJECT_LOCK (self);
+
+  gdk_gl_context_make_current (gl_bi->gdk_context);
+  gst_gl_context_activate (gl_bi->gst_context, TRUE);
+
+  if (!_dmabuf_into_texture (self, fds, v_info, offsets, tex_data))
+    goto finish;
+
+  /* GTK4 does not support External OES textures.
+   * Make it into 2D using framebuffer + shader */
+  if (self->gst_tex_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) {
+    if (G_UNLIKELY (!_oes_texture_into_2d (self, tex_data)))
+      goto finish;
+  }
+
+  texture = gdk_gl_texture_new (gl_bi->gdk_context,
+      tex_data->id,
+      tex_data->width,
+      tex_data->height,
+      (GDestroyNotify) _tex_data_free,
+      tex_data);
+
+finish:
+  gst_gl_context_activate (gl_bi->gst_context, FALSE);
+  gdk_gl_context_clear_current ();
+
+  GST_OBJECT_UNLOCK (self);
+
+  return texture;
+}
+
+static gboolean
+verify_dmabuf_memory (GstBuffer *buffer, GstVideoInfo *info, gint *fds, gsize *offsets)
+{
+  guint i, n_planes = GST_VIDEO_INFO_N_PLANES (info);
+
+  for (i = 0; i < n_planes; i++) {
+    GstMemory *memory;
+    gsize plane_size, mem_skip;
+    guint mem_idx, length;
+
+    plane_size = gst_gl_get_plane_data_size (info, NULL, i);
+
+    if (!gst_buffer_find_memory (buffer,
+        GST_VIDEO_INFO_PLANE_OFFSET (info, i),
+        plane_size, &mem_idx, &length, &mem_skip)) {
+      GST_DEBUG ("Could not find memory %u", i);
+      return FALSE;
+    }
+
+    /* We cannot have more then one DMABuf per plane */
+    if (length != 1) {
+      GST_DEBUG ("Data for plane %u spans %u memories", i, length);
+      return FALSE;
+    }
+
+    memory = gst_buffer_peek_memory (buffer, mem_idx);
+
+    offsets[i] = memory->offset + mem_skip;
+    fds[i] = gst_dmabuf_memory_get_fd (memory);
+  }
+
+  return TRUE;
+}
+
 static void
 _update_elements_caps_locked (GstClapperGLUploader *self, GstCaps *upload_sink_caps)
 {
@@ -104,6 +478,14 @@ gst_clapper_gl_uploader_prepare (GstClapperImporter *importer)
       self->color_convert = gst_gl_color_convert_new (gl_bi->gst_context);
 
     GST_OBJECT_UNLOCK (self);
+
+    if (!(! !gst_gtk_invoke_on_main (
+        (GThreadFunc) (GCallback) prepare_dmabuf_support_on_main, self))) {
+      GST_WARNING_OBJECT (self, "Could not prepare DMABuf support");
+
+      /* FIXME: Continue to allow using glupload/cc as fallback */
+      return FALSE;
+    }
   }
 
   return res;
@@ -139,12 +521,38 @@ gst_clapper_gl_uploader_generate_texture (GstClapperImporter *importer,
   GstClapperGLUploader *self = GST_CLAPPER_GL_UPLOADER_CAST (importer);
   GstClapperGLBaseImporter *gl_bi = GST_CLAPPER_GL_BASE_IMPORTER_CAST (importer);
   GstBuffer *upload_buf, *color_buf;
+  GstVideoMeta *meta;
   GdkTexture *texture;
 
   /* XXX: We both upload and perform color conversion here, thus we skip
    * upload for buffers that are not going to be shown and gain more free
    * CPU time to prepare the next one. Improves performance on weak HW. */
 
+  if ((meta = gst_buffer_get_video_meta (buffer))) {
+    guint i;
+
+    GST_VIDEO_INFO_WIDTH (v_info) = meta->width;
+    GST_VIDEO_INFO_HEIGHT (v_info) = meta->height;
+
+    for (i = 0; i < meta->n_planes; i++) {
+      GST_VIDEO_INFO_PLANE_OFFSET (v_info, i) = meta->offset[i];
+      GST_VIDEO_INFO_PLANE_STRIDE (v_info, i) = meta->stride[i];
+    }
+  }
+
+  /* FIXME: if can do dmabuf and seems like we have dmabuf here */
+  {
+    gint fds[GST_VIDEO_MAX_PLANES];
+    gsize offsets[GST_VIDEO_MAX_PLANES];
+
+    if (verify_dmabuf_memory (buffer, v_info, fds, offsets)) {
+      if ((texture = dmabuf_into_gdk_texture (self, v_info, fds, offsets))) {
+        GST_TRACE_OBJECT (self, "Got texture from DMABuf, skipping upload of %" GST_PTR_FORMAT, buffer);
+        goto done;
+      }
+    }
+  }
+
   GST_LOG_OBJECT (self, "Uploading %" GST_PTR_FORMAT, buffer);
 
   GST_OBJECT_LOCK (self);
@@ -179,6 +587,7 @@ gst_clapper_gl_uploader_generate_texture (GstClapperImporter *importer,
   texture = gst_clapper_gl_base_importer_make_gl_texture (gl_bi, color_buf, &self->v_info);
   gst_buffer_unref (color_buf);
 
+done:
   return texture;
 }
 
@@ -187,6 +596,9 @@ gst_clapper_gl_uploader_init (GstClapperGLUploader *self)
 {
   gst_video_info_init (&self->pending_v_info);
   gst_video_info_init (&self->v_info);
+
+  self->gst_tex_target = GST_GL_TEXTURE_TARGET_EXTERNAL_OES;
+  self->gl_tex_target = gst_gl_texture_target_to_gl (self->gst_tex_target);
 }
 
 static void
@@ -197,6 +609,8 @@ gst_clapper_gl_uploader_finalize (GObject *object)
   gst_clear_object (&self->upload);
   gst_clear_object (&self->color_convert);
 
+  gst_clear_object (&self->shader);
+
   GST_CALL_PARENT (G_OBJECT_CLASS, finalize, (object));
 }
 

lib/gst/plugin/importers/gstclappergluploader.h

@@ -21,6 +21,8 @@
 
 #include "gstclapperglbaseimporter.h"
 
+#include <gst/gl/gstglfuncs.h>
+
 G_BEGIN_DECLS
 
 #define GST_TYPE_CLAPPER_GL_UPLOADER (gst_clapper_gl_uploader_get_type())
@@ -37,6 +39,17 @@ struct _GstClapperGLUploader
 
   GstVideoInfo pending_v_info, v_info;
   gboolean has_pending_v_info;
+
+  /* DMABuf fast-path */
+  GstGLTextureTarget gst_tex_target;
+  guint gl_tex_target;
+
+  GstGLShader *shader;
+
+  GLuint vao;
+  GLuint vertex_buffer;
+  GLint attr_position;
+  GLint attr_texture;
 };
 
 G_END_DECLS

lib/gst/plugin/importers/meson.build

@@ -100,11 +100,25 @@ build_gluploader = (
   and gst_clapper_gl_base_importer_dep.found()
 )
 
+gluploader_deps = [
+  gst_clapper_gl_base_importer_dep,
+  gstallocators_dep,
+]
+
+foreach dep : gluploader_deps
+  if not dep.found()
+    if get_option('gluploader').enabled()
+      error('GL uploader was enabled, but required dependencies were not found')
+    endif
+    build_gluploader = false
+  endif
+endforeach
+
 if build_gluploader
   library(
     'gstclappergluploader',
     'gstclappergluploader.c',
-    dependencies: gst_clapper_gl_base_importer_dep,
+    dependencies: gluploader_deps,
     include_directories: configinc,
     c_args: gst_clapper_plugin_args,
     install: true,

lib/meson.build

@@ -196,6 +196,8 @@ gsttag_dep = dependency('gstreamer-tag-1.0', version: gst_req,
   fallback: ['gst-plugins-base', 'tag_dep'])
 gstvideo_dep = dependency('gstreamer-video-1.0', version: gst_req,
   fallback: ['gst-plugins-base', 'video_dep'])
+gstallocators_dep = dependency('gstreamer-allocators-1.0', version: gst_req,
+  fallback : ['gst-plugins-base', 'allocators_dep'])
 
 # GStreamer OpenGL
 gstgl_dep = dependency('gstreamer-gl-1.0', version: gst_req,