Video for GTK4

main/src/ui/call_window/call_window.vala

@@ -149,7 +149,7 @@ namespace Dino.Ui {
         }
 
         public void set_own_video(Widget? widget_) {
-//            own_video_box.foreach((widget) => { own_video_box.remove(widget); });
+            unset_own_video();
 
             own_video = widget_;
             if (own_video == null) {
@@ -170,7 +170,11 @@ namespace Dino.Ui {
         }
 
         public void unset_own_video() {
-//            own_video_box.foreach((widget) => { own_video_box.remove(widget); });
+            Widget to_remove = own_video_box.get_first_child();
+            while (to_remove != null) {
+                own_video_box.remove(to_remove);
+                to_remove = own_video_box.get_first_child();
+            }
         }
 
         public void set_status(string participant_id, string state) {

main/src/ui/call_window/call_window_controller.vala

@@ -364,7 +364,7 @@ public class Dino.Ui.CallWindowController : Object {
         participant_widgets.keys.@foreach((peer_id) => { remove_participant(peer_id); return true; });
 
         call_window_handler_ids = bottom_bar_handler_ids = new ulong[0];
-
+        own_video.detach();
         base.dispose();
     }
 }

plugins/rtp/CMakeLists.txt

@@ -11,6 +11,7 @@ find_packages(RTP_PACKAGES REQUIRED
     Gst
     GstApp
     GstAudio
+    GstVideo
 )
 
 set(RTP_DEFINITIONS)
@@ -23,8 +24,8 @@ if(GstRtp_VERSION VERSION_GREATER "1.18")
     set(RTP_DEFINITIONS ${RTP_DEFINITIONS} GST_1_18)
 endif()
 
-if(GLib_VERSION VERSION_GREATER "2.64")
+if(GstRtp_VERSION VERSION_GREATER "1.20")
-    set(RTP_DEFINITIONS ${RTP_DEFINITIONS} GLIB_2_64)
+    set(RTP_DEFINITIONS ${RTP_DEFINITIONS} GST_1_20)
 endif()
 
 set(RTP_ENABLE_VP9 "no" CACHE BOOL "Enable VP9 support")
@@ -84,7 +85,7 @@ DEFINITIONS
 )
 
 add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src)
-add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX})
+add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX} src/gst_fixes.c)
 target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 ${RTP_VOICE_PROCESSOR_LIB})
 set_target_properties(rtp PROPERTIES PREFIX "")
 set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/)

plugins/rtp/src/gst_fixes.c

@@ -0,0 +1,10 @@
+#include <gst/video/video.h>
+
+GstVideoInfo *gst_video_frame_get_video_info(GstVideoFrame *frame) {
+    return &frame->info;
+}
+
+void *gst_video_frame_get_data(GstVideoFrame *frame, size_t* length) {
+    *length = frame->info.height * frame->info.stride[0];
+    return frame->data[0];
+}

plugins/rtp/src/video_widget.vala

@@ -1,16 +1,164 @@
-#if !VALA_0_52
+private static extern unowned Gst.Video.Info gst_video_frame_get_video_info(Gst.Video.Frame frame);
-[CCode (cheader_filename = "gst/gst.h")]
+private static extern unowned uint8[] gst_video_frame_get_data(Gst.Video.Frame frame);
-private static extern void gst_value_set_fraction(ref GLib.Value value, int numerator, int denominator);
+
+public class Dino.Plugins.Rtp.Paintable : Gdk.Paintable, Object {
+    private Gdk.Paintable image;
+    private double pixel_aspect_ratio;
+
+    public override Gdk.PaintableFlags get_flags() {
+        return 0;
+    }
+
+    public void snapshot(Gdk.Snapshot snapshot, double width, double height) {
+        if (image != null) image.snapshot(snapshot, width, height);
+    }
+
+    public override Gdk.Paintable get_current_image() {
+        if (image != null) return image;
+        return Gdk.Paintable.new_empty(0, 0);
+    }
+
+    public override int get_intrinsic_width() {
+        if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_width());
+        return 0;
+    }
+
+    public override int get_intrinsic_height() {
+        if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_height());
+        return 0;
+    }
+
+    public override double get_intrinsic_aspect_ratio() {
+        if (image != null) return pixel_aspect_ratio * image.get_intrinsic_aspect_ratio();
+        return 0.0;
+    }
+
+    public override void dispose() {
+        image = null;
+        base.dispose();
+    }
+
+    private void set_paintable(Gdk.Paintable paintable, double pixel_aspect_ratio) {
+        if (paintable == image) return;
+        bool size_changed = image == null ||
+                this.pixel_aspect_ratio * image.get_intrinsic_width() != pixel_aspect_ratio * paintable.get_intrinsic_width() ||
+                image.get_intrinsic_height() != paintable.get_intrinsic_height() ||
+                image.get_intrinsic_aspect_ratio() != paintable.get_intrinsic_aspect_ratio();
+
+        this.image.dispose();
+        this.image = paintable;
+        this.pixel_aspect_ratio = pixel_aspect_ratio;
+
+        if (size_changed) invalidate_size();
+        invalidate_contents();
+    }
+
+    public void queue_set_texture(Gdk.Texture texture, double pixel_aspect_ratio) {
+        Idle.add(() => {
+            set_paintable(texture, pixel_aspect_ratio);
+            return Source.REMOVE;
+        }, Priority.DEFAULT);
+    }
+}
+
+public class Dino.Plugins.Rtp.Sink : Gst.Video.Sink {
+    internal Paintable paintable = new Paintable();
+    private Gst.Video.Info info = new Gst.Video.Info();
+
+    class construct {
+        set_metadata("Dino Gtk Video Sink", "Sink/Video", "The video sink used by Dino", "Dino Team <team@dino.im>");
+        add_pad_template(new Gst.PadTemplate("sink", Gst.PadDirection.SINK, Gst.PadPresence.ALWAYS, Gst.Caps.from_string(@"video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }")));
+    }
+
+    construct {
+        set_drop_out_of_segment(false);
+    }
+
+#if GST_1_20
+    public override bool set_info(Gst.Caps caps, Gst.Video.Info info) {
+        this.info = info;
+        return true;
+    }
+#else
+    public override bool set_caps(Gst.Caps caps) {
+        base.set_caps(caps);
+        return info.from_caps(caps);
+    }
 #endif
 
+    public override void get_times(Gst.Buffer buffer, out Gst.ClockTime start, out Gst.ClockTime end) {
+        if (buffer.pts != -1) {
+            start = buffer.pts;
+            if (buffer.duration != -1) {
+                end = start + buffer.duration;
+            } else if (info.fps_n > 0) {
+                end = start + Gst.Util.uint64_scale_int(Gst.SECOND, info.fps_d, info.fps_n);
+            }
+        }
+    }
+
+    public override Gst.Caps get_caps(Gst.Caps? filter) {
+        Gst.Caps caps = Gst.Caps.from_string("video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }");
+
+        if (filter != null) {
+            return filter.intersect(caps, Gst.CapsIntersectMode.FIRST);
+        } else {
+            return caps;
+        }
+    }
+
+    private Gdk.MemoryFormat memory_format_from_video(Gst.Video.Format format) {
+        switch (format) {
+            case Gst.Video.Format.BGRA: return Gdk.MemoryFormat.B8G8R8A8;
+            case Gst.Video.Format.ARGB: return Gdk.MemoryFormat.A8R8G8B8;
+            case Gst.Video.Format.RGBA: return Gdk.MemoryFormat.R8G8B8A8;
+            case Gst.Video.Format.ABGR: return Gdk.MemoryFormat.A8B8G8R8;
+            case Gst.Video.Format.RGB: return Gdk.MemoryFormat.R8G8B8;
+            case Gst.Video.Format.BGR: return Gdk.MemoryFormat.B8G8R8;
+            default:
+                warning("Unsupported video format: %s", format.to_string());
+                return Gdk.MemoryFormat.A8R8G8B8;
+        }
+    }
+
+    private Gdk.Texture texture_from_buffer(Gst.Buffer buffer, out double pixel_aspect_ratio) {
+        Gst.Video.Frame frame;
+        Gdk.Texture texture;
+
+        if (Gst.Video.frame_map(out frame, info, buffer, Gst.MapFlags.READ)) {
+            unowned Gst.Video.Info info = gst_video_frame_get_video_info(frame);
+            Bytes bytes = new Bytes.take(gst_video_frame_get_data(frame));
+            texture = new Gdk.MemoryTexture(info.width, info.height, memory_format_from_video(info.finfo.format), bytes, info.stride[0]);
+            pixel_aspect_ratio = ((double) info.par_n) / ((double) info.par_d);
+            frame.unmap();
+        } else {
+            texture = null;
+        }
+        return texture;
+    }
+
+    private void queue_buffer(Gst.Buffer buf) {
+        double pixel_aspect_ratio;
+        Gdk.Texture texture = texture_from_buffer(buf, out pixel_aspect_ratio);
+        if (texture != null) {
+            paintable.queue_set_texture(texture, pixel_aspect_ratio);
+        }
+    }
+
+    public override Gst.FlowReturn show_frame(Gst.Buffer buf) {
+        @lock.lock();
+        queue_buffer(buf);
+        @lock.unlock();
+
+        return Gst.FlowReturn.OK;
+    }
+}
+
 public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWidget {
     private const int RECAPS_AFTER_CHANGE = 5;
     private static uint last_id = 0;
 
     public uint id { get; private set; }
-    public Gst.Base.Sink sink { get; private set; }
-    public Gtk.Widget widget { get; private set; }
-
     public Plugin plugin { get; private set; }
     public Gst.Pipeline pipe { get {
         return plugin.pipe;
@@ -24,26 +172,17 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWi
     private Gst.Caps last_input_caps;
     private Gst.Caps last_caps;
     private int recaps_since_change;
+    private Sink sink;
+    private Gtk.Picture widget;
 
     public VideoWidget(Plugin plugin) {
         this.plugin = plugin;
+        this.layout_manager = new Gtk.BinLayout();
 
         id = last_id++;
-//        sink = Gst.ElementFactory.make("gtksink", @"video_widget_$id") as Gst.Base.Sink;
+        sink = new Sink() { async = false, sync = true };
-        if (sink != null) {
+        widget = new Gtk.Picture.for_paintable(sink.paintable);
-            Gtk.Widget widget;
+        widget.insert_after(this, null);
-            sink.@get("widget", out widget);
-            sink.@set("async", false);
-            sink.@set("sync", true);
-            sink.@set("ignore-alpha", false);
-            this.widget = widget;
-//            this.widget.draw.connect_after(fix_caps_issues);
-            this.widget.insert_after(this, null);
-            this.widget.visible = true;
-        } else {
-            warning("Could not create GTK video sink. Won't display videos.");
-        }
-//        size_allocate.connect_after(after_size_allocate);
     }
 
     public void input_caps_changed(GLib.Object pad, ParamSpec spec) {
@@ -61,75 +200,6 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWi
         last_input_caps = caps;
     }
 
-    public void processed_input_caps_changed(GLib.Object pad, ParamSpec spec) {
-        Gst.Caps? caps = ((Gst.Pad)pad).caps;
-        if (caps == null) {
-            debug("Processed input: No caps");
-            return;
-        }
-
-        int width, height;
-        caps.get_structure(0).get_int("width", out width);
-        caps.get_structure(0).get_int("height", out height);
-        debug("Processed resolution changed: %ix%i", width, height);
-        sink.set_caps(caps);
-        last_caps = caps;
-        recaps_since_change = 0;
-    }
-
-    public void after_size_allocate(Gtk.Allocation allocation) {
-        if (prepare != null) {
-            Gst.Element crop = ((Gst.Bin)prepare).get_by_name(@"video_widget_$(id)_crop");
-            if (crop != null) {
-                int output_width = allocation.width;
-                int output_height = allocation.height;
-                int target_num, target_den;
-                if (last_input_caps != null) {
-                    int input_width, input_height;
-                    last_input_caps.get_structure(0).get_int("width", out input_width);
-                    last_input_caps.get_structure(0).get_int("height", out input_height);
-                    double target_ratio = 3.0/2.0;
-                    double ratio = (double)(output_width*input_height)/(double)(input_width*output_height);
-                    if (ratio > target_ratio) {
-                        target_num = (int)((double)input_width * target_ratio);
-                        target_den = input_height;
-                        sink.@set("force-aspect-ratio", true);
-                    } else if (ratio < 1.0/target_ratio) {
-                        target_num = input_width;
-                        target_den = (int)((double)input_height * target_ratio);;
-                        sink.@set("force-aspect-ratio", true);
-                    } else {
-                        target_num = output_width;
-                        target_den = output_height;
-                        sink.@set("force-aspect-ratio", false);
-                    }
-                } else {
-                    target_num = output_width;
-                    target_den = output_height;
-                    sink.@set("force-aspect-ratio", false);
-                }
-                Value ratio = Value(typeof(Gst.Fraction));
-#if VALA_0_52
-                Gst.Value.set_fraction(ref ratio, target_num, target_den);
-#else
-                gst_value_set_fraction(ref ratio, target_num, target_den);
-#endif
-                crop.set_property("aspect-ratio", ratio);
-            }
-        }
-    }
-
-    public bool fix_caps_issues() {
-        // FIXME: Detect if draw would fail and do something better
-        if (last_caps != null && recaps_since_change++ < RECAPS_AFTER_CHANGE) {
-            Gst.Caps? temp = last_caps.copy();
-            temp.set_simple("width", typeof(int), 1, "height", typeof(int), 1, null);
-            sink.set_caps(temp);
-            sink.set_caps(last_caps);
-        }
-        return false;
-    }
-
     public void display_stream(Xmpp.Xep.JingleRtp.Stream? stream, Xmpp.Jid jid) {
         if (sink == null) return;
         detach();
@@ -138,10 +208,9 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWi
         if (connected_stream == null) return;
         plugin.pause();
         pipe.add(sink);
-        prepare = Gst.parse_bin_from_description(@"aspectratiocrop aspect-ratio=4/3 name=video_widget_$(id)_crop ! videoconvert name=video_widget_$(id)_convert", true);
+        prepare = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true);
         prepare.name = @"video_widget_$(id)_prepare";
         prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed);
-        prepare.get_static_pad("src").notify["caps"].connect(processed_input_caps_changed);
         pipe.add(prepare);
         connected_stream.add_output(prepare);
         prepare.link(sink);
@@ -157,7 +226,7 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWi
         if (connected_device == null) return;
         plugin.pause();
         pipe.add(sink);
-        prepare = Gst.parse_bin_from_description(@"aspectratiocrop aspect-ratio=4/3 name=video_widget_$(id)_crop ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true);
+        prepare = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true);
         prepare.name = @"video_widget_$(id)_prepare";
         prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed);
         pipe.add(prepare);
@@ -172,6 +241,7 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWi
     public void detach() {
         if (sink == null) return;
         if (attached) {
+            debug("Detaching");
             if (connected_stream != null) {
                 connected_stream.remove_output(prepare);
                 connected_stream = null;
@@ -195,6 +265,7 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWi
 
     public override void dispose() {
         detach();
+        if (widget != null) widget.unparent();
         widget = null;
         sink = null;
     }