anotherim/anotherim-desktop
2
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Video optimizations

Browse source
This commit is contained in:
Marvin W 2021-04-29 15:46:06 +02:00
parent 328c3cf37f
commit 3880628de4
No known key found for this signature in database
GPG key ID: 072E9235DB996F2A
13 changed files with 1126 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
cmake/FindGstRtp.cmake Normal file
View file

@ -0,0 +1,14 @@
include(PkgConfigWithFallback)
find_pkg_config_with_fallback(GstRtp
PKG_CONFIG_NAME gstreamer-rtp-1.0
LIB_NAMES gstrtp
LIB_DIR_HINTS gstreamer-1.0
INCLUDE_NAMES gst/rtp/rtp.h
INCLUDE_DIR_SUFFIXES gstreamer-1.0 gstreamer-1.0/include gstreamer-rtp-1.0 gstreamer-rtp-1.0/include
DEPENDS Gst
)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(GstRtp
REQUIRED_VARS GstRtp_LIBRARY
VERSION_VAR GstRtp_VERSION)

4
plugins/rtp/CMakeLists.txt
View file

@ -1,3 +1,4 @@
find_package(GstRtp REQUIRED)
find_packages(RTP_PACKAGES REQUIRED find_packages(RTP_PACKAGES REQUIRED
Gee Gee
GLib GLib
@ -27,6 +28,7 @@ CUSTOM_VAPIS
${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi
${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi
${CMAKE_BINARY_DIR}/exports/qlite.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi
${CMAKE_CURRENT_SOURCE_DIR}/vapi/gstreamer-rtp-1.0.vapi
PACKAGES PACKAGES
${RTP_PACKAGES} ${RTP_PACKAGES}
DEFINITIONS DEFINITIONS
@ -35,7 +37,7 @@ DEFINITIONS
add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src)
add_library(rtp SHARED ${RTP_VALA_C}) add_library(rtp SHARED ${RTP_VALA_C})
target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES}) target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0)
set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES PREFIX "")
set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/)

115
plugins/rtp/src/codec_util.vala
View file

@ -6,7 +6,7 @@ public class Dino.Plugins.Rtp.CodecUtil {
private Set<string> supported_elements = new HashSet<string>(); private Set<string> supported_elements = new HashSet<string>();
private Set<string> unsupported_elements = new HashSet<string>(); private Set<string> unsupported_elements = new HashSet<string>();
public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type) { public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type, bool incoming) {
Gst.Caps caps = new Gst.Caps.simple("application/x-rtp", Gst.Caps caps = new Gst.Caps.simple("application/x-rtp",
"media", typeof(string), media, "media", typeof(string), media,
"payload", typeof(int), payload_type.id); "payload", typeof(int), payload_type.id);
@ -19,6 +19,15 @@ public class Dino.Plugins.Rtp.CodecUtil {
if (payload_type.name != null) { if (payload_type.name != null) {
s.set("encoding-name", typeof(string), payload_type.name.up()); s.set("encoding-name", typeof(string), payload_type.name.up());
} }
if (incoming) {
foreach (JingleRtp.RtcpFeedback rtcp_fb in payload_type.rtcp_fbs) {
if (rtcp_fb.subtype == null) {
s.set(@"rtcp-fb-$(rtcp_fb.type_)", typeof(bool), true);
} else {
s.set(@"rtcp-fb-$(rtcp_fb.type_)-$(rtcp_fb.subtype)", typeof(bool), true);
}
}
}
return caps; return caps;
} }
@ -122,32 +131,82 @@ public class Dino.Plugins.Rtp.CodecUtil {
return new string[0]; return new string[0];
} }
public static string? get_encode_prefix(string media, string codec, string encode) { public static string? get_encode_prefix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
if (encode == "msdkh264enc") return "video/x-raw,format=NV12 ! "; if (encode == "msdkh264enc") return "video/x-raw,format=NV12 ! ";
if (encode == "vaapih264enc") return "video/x-raw,format=NV12 ! "; if (encode == "vaapih264enc") return "video/x-raw,format=NV12 ! ";
return null; return null;
} }
public static string? get_encode_suffix(string media, string codec, string encode) { public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
// H264 // H264
const string h264_suffix = " ! video/x-h264,profile=constrained-baseline ! h264parse"; if (encode == "msdkh264enc") return @" rate-control=vbr";
if (encode == "msdkh264enc") return @" bitrate=256 rate-control=vbr target-usage=7$h264_suffix"; if (encode == "vaapih264enc") return @" tune=low-power";
if (encode == "vaapih264enc") return @" bitrate=256 quality-level=7 tune=low-power$h264_suffix"; if (encode == "x264enc") return @" byte-stream=1 profile=baseline speed-preset=ultrafast tune=zerolatency";
if (encode == "x264enc") return @" byte-stream=1 bitrate=256 profile=baseline speed-preset=ultrafast tune=zerolatency$h264_suffix";
if (media == "video" && codec == "h264") return h264_suffix;
// VP8 // VP8
if (encode == "msdkvp8enc") return " bitrate=256 rate-control=vbr target-usage=7"; if (encode == "msdkvp8enc") return " rate-control=vbr";
if (encode == "vaapivp8enc") return " bitrate=256 rate-control=vbr quality-level=7"; if (encode == "vaapivp8enc") return " rate-control=vbr";
if (encode == "vp8enc") return " target-bitrate=256000 deadline=1 error-resilient=1"; if (encode == "vp8enc") return " deadline=1 error-resilient=1";
// OPUS // OPUS
if (encode == "opusenc") return " audio-type=voice"; if (encode == "opusenc") {
if (payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " audio-type=voice inband-fec=true";
return " audio-type=voice";
}
return null; return null;
} }
public static string? get_decode_prefix(string media, string codec, string decode) { public static string? get_encode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
// H264
if (media == "video" && codec == "h264") return " ! video/x-h264,profile=constrained-baseline ! h264parse";
return null;
}
public uint update_bitrate(string media, JingleRtp.PayloadType payload_type, Gst.Element encode_element, uint bitrate) {
Gst.Bin? encode_bin = encode_element as Gst.Bin;
if (encode_bin == null) return 0;
string? codec = get_codec_from_payload(media, payload_type);
string? encode_name = get_encode_element_name(media, codec);
if (encode_name == null) return 0;
Gst.Element encode = encode_bin.get_by_name(@"$(encode_bin.name)_encode");
bitrate = uint.min(2048000, bitrate);
switch (encode_name) {
case "msdkh264enc":
case "vaapih264enc":
case "x264enc":
case "msdkvp8enc":
case "vaapivp8enc":
bitrate = uint.min(2048000, bitrate);
encode.set("bitrate", bitrate);
return bitrate;
case "vp8enc":
bitrate = uint.min(2147483, bitrate);
encode.set("target-bitrate", bitrate * 1000);
return bitrate;
}
return 0;
}
public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) {
return null;
}
public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) {
if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true";
if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100";
return null;
}
public static string? get_decode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
return null;
}
public static string? get_depay_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
if (codec == "vp8") return " wait-for-keyframe=true";
return null; return null;
} }
@ -195,21 +254,24 @@ public class Dino.Plugins.Rtp.CodecUtil {
unsupported_elements.add(element_name); unsupported_elements.add(element_name);
} }
public string? get_decode_bin_description(string media, string? codec, string? element_name = null, string? name = null) { public string? get_decode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) {
if (codec == null) return null; if (codec == null) return null;
string base_name = name ?? @"encode-$codec-$(Random.next_int())"; string base_name = name ?? @"encode-$codec-$(Random.next_int())";
string depay = get_depay_element_name(media, codec); string depay = get_depay_element_name(media, codec);
string decode = element_name ?? get_decode_element_name(media, codec); string decode = element_name ?? get_decode_element_name(media, codec);
if (depay == null || decode == null) return null; if (depay == null || decode == null) return null;
string decode_prefix = get_decode_prefix(media, codec, decode) ?? ""; string decode_prefix = get_decode_prefix(media, codec, decode, payload_type) ?? "";
string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; string decode_args = get_decode_args(media, codec, decode, payload_type) ?? "";
return @"$depay name=$base_name-rtp-depay ! $decode_prefix$decode name=$base_name-decode ! $(media)convert name=$base_name-convert$resample"; string decode_suffix = get_decode_suffix(media, codec, decode, payload_type) ?? "";
string depay_args = get_depay_args(media, codec, decode, payload_type) ?? "";
string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : "";
return @"$depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample";
} }
public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) {
string? codec = get_codec_from_payload(media, payload_type); string? codec = get_codec_from_payload(media, payload_type);
string base_name = name ?? @"encode-$codec-$(Random.next_int())"; string base_name = name ?? @"encode-$codec-$(Random.next_int())";
string? desc = get_decode_bin_description(media, codec, null, base_name); string? desc = get_decode_bin_description(media, codec, payload_type, null, base_name);
if (desc == null) return null; if (desc == null) return null;
debug("Pipeline to decode %s %s: %s", media, codec, desc); debug("Pipeline to decode %s %s: %s", media, codec, desc);
Gst.Element bin = Gst.parse_bin_from_description(desc, true); Gst.Element bin = Gst.parse_bin_from_description(desc, true);
@ -217,22 +279,23 @@ public class Dino.Plugins.Rtp.CodecUtil {
return bin; return bin;
} }
public string? get_encode_bin_description(string media, string? codec, string? element_name = null, uint pt = 96, string? name = null) { public string? get_encode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) {
if (codec == null) return null; if (codec == null) return null;
string base_name = name ?? @"encode-$codec-$(Random.next_int())"; string base_name = name ?? @"encode_$(codec)_$(Random.next_int())";
string pay = get_pay_element_name(media, codec); string pay = get_pay_element_name(media, codec);
string encode = element_name ?? get_encode_element_name(media, codec); string encode = element_name ?? get_encode_element_name(media, codec);
if (pay == null || encode == null) return null; if (pay == null || encode == null) return null;
string encode_prefix = get_encode_prefix(media, codec, encode) ?? ""; string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? "";
string encode_suffix = get_encode_suffix(media, codec, encode) ?? ""; string encode_args = get_encode_args(media, codec, encode, payload_type) ?? "";
string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? "";
return @"$(media)convert name=$base_name-convert$resample ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : "";
return @"$(media)convert name=$(base_name)_convert$resample ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix ! $pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay";
} }
public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) {
string? codec = get_codec_from_payload(media, payload_type); string? codec = get_codec_from_payload(media, payload_type);
string base_name = name ?? @"encode-$codec-$(Random.next_int())"; string base_name = name ?? @"encode_$(codec)_$(Random.next_int())";
string? desc = get_encode_bin_description(media, codec, null, payload_type.id, base_name); string? desc = get_encode_bin_description(media, codec, payload_type, null, base_name);
if (desc == null) return null; if (desc == null) return null;
debug("Pipeline to encode %s %s: %s", media, codec, desc); debug("Pipeline to encode %s %s: %s", media, codec, desc);
Gst.Element bin = Gst.parse_bin_from_description(desc, true); Gst.Element bin = Gst.parse_bin_from_description(desc, true);

9
plugins/rtp/src/device.vala
View file

@ -126,19 +126,20 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
element = device.create_element(id); element = device.create_element(id);
pipe.add(element); pipe.add(element);
if (is_source) { if (is_source) {
filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); element.@set("do-timestamp", true);
filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
filter.@set("caps", get_best_caps()); filter.@set("caps", get_best_caps());
pipe.add(filter); pipe.add(filter);
element.link(filter); element.link(filter);
if (media == "audio" && plugin.echoprobe != null) { if (media == "audio" && plugin.echoprobe != null) {
dsp = Gst.ElementFactory.make("webrtcdsp", @"$id-dsp"); dsp = Gst.ElementFactory.make("webrtcdsp", @"dsp_$id");
if (dsp != null) { if (dsp != null) {
dsp.@set("probe", plugin.echoprobe.name); dsp.@set("probe", plugin.echoprobe.name);
pipe.add(dsp); pipe.add(dsp);
filter.link(dsp); filter.link(dsp);
} }
} }
tee = Gst.ElementFactory.make("tee", @"$id-tee"); tee = Gst.ElementFactory.make("tee", @"tee_$id");
tee.@set("allow-not-linked", true); tee.@set("allow-not-linked", true);
pipe.add(tee); pipe.add(tee);
(dsp ?? filter).link(tee); (dsp ?? filter).link(tee);
@ -148,7 +149,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
element.@set("sync", false); element.@set("sync", false);
} }
if (is_sink && media == "audio") { if (is_sink && media == "audio") {
filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
filter.@set("caps", get_best_caps()); filter.@set("caps", get_best_caps());
pipe.add(filter); pipe.add(filter);
if (plugin.echoprobe != null) { if (plugin.echoprobe != null) {

133
plugins/rtp/src/module.vala
View file

@ -63,7 +63,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module {
return supported; return supported;
} }
private async bool supports(string media, JingleRtp.PayloadType payload_type) { private async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) {
string codec = CodecUtil.get_codec_from_payload(media, payload_type); string codec = CodecUtil.get_codec_from_payload(media, payload_type);
if (codec == null) return false; if (codec == null) return false;
if (unsupported_codecs.contains(codec)) return false; if (unsupported_codecs.contains(codec)) return false;
@ -77,7 +77,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module {
return false; return false;
} }
string encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); string encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element);
while (!(yield pipeline_works(media, encode_bin))) { while (!(yield pipeline_works(media, encode_bin))) {
debug("%s not suited for encoding %s", encode_element, codec); debug("%s not suited for encoding %s", encode_element, codec);
codec_util.mark_element_unsupported(encode_element); codec_util.mark_element_unsupported(encode_element);
@ -87,11 +87,11 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module {
unsupported_codecs.add(codec); unsupported_codecs.add(codec);
return false; return false;
} }
encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element);
} }
debug("using %s to encode %s", encode_element, codec); debug("using %s to encode %s", encode_element, codec);
string decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); string decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element);
while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) { while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) {
debug("%s not suited for decoding %s", decode_element, codec); debug("%s not suited for decoding %s", decode_element, codec);
codec_util.mark_element_unsupported(decode_element); codec_util.mark_element_unsupported(decode_element);
@ -101,7 +101,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module {
unsupported_codecs.add(codec); unsupported_codecs.add(codec);
return false; return false;
} }
decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element);
} }
debug("using %s to decode %s", decode_element, codec); debug("using %s to decode %s", decode_element, codec);
@ -109,8 +109,21 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module {
return true; return true;
} }
public override bool is_header_extension_supported(string media, JingleRtp.HeaderExtension ext) {
if (media == "video" && ext.uri == "urn:3gpp:video-orientation") return true;
return false;
}
public override Gee.List<JingleRtp.HeaderExtension> get_suggested_header_extensions(string media) {
Gee.List<JingleRtp.HeaderExtension> exts = new ArrayList<JingleRtp.HeaderExtension>();
if (media == "video") {
exts.add(new JingleRtp.HeaderExtension(1, "urn:3gpp:video-orientation"));
}
return exts;
}
public async void add_if_supported(Gee.List<JingleRtp.PayloadType> list, string media, JingleRtp.PayloadType payload_type) { public async void add_if_supported(Gee.List<JingleRtp.PayloadType> list, string media, JingleRtp.PayloadType payload_type) {
if (yield supports(media, payload_type)) { if (yield is_payload_supported(media, payload_type)) {
list.add(payload_type); list.add(payload_type);
} }
} }
@ -118,58 +131,34 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module {
public override async Gee.List<JingleRtp.PayloadType> get_supported_payloads(string media) { public override async Gee.List<JingleRtp.PayloadType> get_supported_payloads(string media) {
Gee.List<JingleRtp.PayloadType> list = new ArrayList<JingleRtp.PayloadType>(JingleRtp.PayloadType.equals_func); Gee.List<JingleRtp.PayloadType> list = new ArrayList<JingleRtp.PayloadType>(JingleRtp.PayloadType.equals_func);
if (media == "audio") { if (media == "audio") {
yield add_if_supported(list, media, new JingleRtp.PayloadType() { var opus = new JingleRtp.PayloadType() { channels = 2, clockrate = 48000, name = "opus", id = 99 };
channels = 2, opus.parameters["useinbandfec"] = "1";
clockrate = 48000, var speex32 = new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", id = 100 };
name = "opus", var speex16 = new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", id = 101 };
id = 99 var speex8 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "speex", id = 102 };
}); var pcmu = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMU", id = 0 };
yield add_if_supported(list, media, new JingleRtp.PayloadType() { var pcma = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMA", id = 8 };
channels = 1, yield add_if_supported(list, media, opus);
clockrate = 32000, yield add_if_supported(list, media, speex32);
name = "speex", yield add_if_supported(list, media, speex16);
id = 100 yield add_if_supported(list, media, speex8);
}); yield add_if_supported(list, media, pcmu);
yield add_if_supported(list, media, new JingleRtp.PayloadType() { yield add_if_supported(list, media, pcma);
channels = 1,
clockrate = 16000,
name = "speex",
id = 101
});
yield add_if_supported(list, media, new JingleRtp.PayloadType() {
channels = 1,
clockrate = 8000,
name = "speex",
id = 102
});
yield add_if_supported(list, media, new JingleRtp.PayloadType() {
channels = 1,
clockrate = 8000,
name = "PCMU",
id = 0
});
yield add_if_supported(list, media, new JingleRtp.PayloadType() {
channels = 1,
clockrate = 8000,
name = "PCMA",
id = 8
});
} else if (media == "video") { } else if (media == "video") {
yield add_if_supported(list, media, new JingleRtp.PayloadType() { var h264 = new JingleRtp.PayloadType() { clockrate = 90000, name = "H264", id = 96 };
clockrate = 90000, var vp9 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP9", id = 97 };
name = "H264", var vp8 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP8", id = 98 };
id = 96 var rtcp_fbs = new ArrayList<JingleRtp.RtcpFeedback>();
}); rtcp_fbs.add(new JingleRtp.RtcpFeedback("goog-remb"));
yield add_if_supported(list, media, new JingleRtp.PayloadType() { rtcp_fbs.add(new JingleRtp.RtcpFeedback("ccm", "fir"));
clockrate = 90000, rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack"));
name = "VP9", rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack", "pli"));
id = 97 h264.rtcp_fbs.add_all(rtcp_fbs);
}); vp9.rtcp_fbs.add_all(rtcp_fbs);
yield add_if_supported(list, media, new JingleRtp.PayloadType() { vp8.rtcp_fbs.add_all(rtcp_fbs);
clockrate = 90000, yield add_if_supported(list, media, h264);
name = "VP8", yield add_if_supported(list, media, vp9);
id = 98 yield add_if_supported(list, media, vp8);
});
} else { } else {
warning("Unsupported media type: %s", media); warning("Unsupported media type: %s", media);
} }
@ -179,11 +168,15 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module {
public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List<JingleRtp.PayloadType> payloads) { public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List<JingleRtp.PayloadType> payloads) {
if (media == "audio") { if (media == "audio") {
foreach (JingleRtp.PayloadType type in payloads) { foreach (JingleRtp.PayloadType type in payloads) {
if (yield supports(media, type)) return type; if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone());
} }
} else if (media == "video") { } else if (media == "video") {
// We prefer H.264 (best support for hardware acceleration and good overall codec quality)
JingleRtp.PayloadType? h264 = payloads.first_match((it) => it.name.up() == "H264");
if (h264 != null && yield is_payload_supported(media, h264)) return adjust_payload_type(media, h264.clone());
// Take first of the list that we do support otherwise
foreach (JingleRtp.PayloadType type in payloads) { foreach (JingleRtp.PayloadType type in payloads) {
if (yield supports(media, type)) return type; if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone());
} }
} else { } else {
warning("Unsupported media type: %s", media); warning("Unsupported media type: %s", media);
@ -191,6 +184,28 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module {
return null; return null;
} }
public JingleRtp.PayloadType adjust_payload_type(string media, JingleRtp.PayloadType type) {
var iter = type.rtcp_fbs.iterator();
while (iter.next()) {
var fb = iter.@get();
switch (fb.type_) {
case "goog-remb":
if (fb.subtype != null) iter.remove();
break;
case "ccm":
if (fb.subtype != "fir") iter.remove();
break;
case "nack":
if (fb.subtype != null && fb.subtype != "pli") iter.remove();
break;
default:
iter.remove();
break;
}
}
return type;
}
public override JingleRtp.Stream create_stream(Jingle.Content content) { public override JingleRtp.Stream create_stream(Jingle.Content content) {
return plugin.open_stream(content); return plugin.open_stream(content);
} }

14
plugins/rtp/src/plugin.vala
View file

@ -65,6 +65,9 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
} }
rtpbin.pad_added.connect(on_rtp_pad_added); rtpbin.pad_added.connect(on_rtp_pad_added);
rtpbin.@set("latency", 100); rtpbin.@set("latency", 100);
rtpbin.@set("do-lost", true);
rtpbin.@set("do-sync-event", true);
rtpbin.@set("drop-on-latency", true);
rtpbin.connect("signal::request-pt-map", request_pt_map, this); rtpbin.connect("signal::request-pt-map", request_pt_map, this);
pipe.add(rtpbin); pipe.add(rtpbin);
@ -160,6 +163,17 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
case Gst.MessageType.QOS: case Gst.MessageType.QOS:
// Ignore // Ignore
break; break;
case Gst.MessageType.LATENCY:
if (message.src != null && message.src.name != null && message.src is Gst.Element) {
Gst.Query latency_query = new Gst.Query.latency();
if (((Gst.Element)message.src).query(latency_query)) {
bool live;
Gst.ClockTime min_latency, max_latency;
latency_query.parse_latency(out live, out min_latency, out max_latency);
debug("Latency message from %s: live=%s, min_latency=%s, max_latency=%s", message.src.name, live.to_string(), min_latency.to_string(), max_latency.to_string());
}
}
break;
default: default:
debug("Pipe bus message: %s", message.type.to_string()); debug("Pipe bus message: %s", message.type.to_string());
break; break;

220
plugins/rtp/src/stream.vala
View file

@ -19,9 +19,12 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
private Gst.App.Src recv_rtp; private Gst.App.Src recv_rtp;
private Gst.App.Src recv_rtcp; private Gst.App.Src recv_rtcp;
private Gst.Element encode; private Gst.Element encode;
private Gst.RTP.BasePayload encode_pay;
private Gst.Element decode; private Gst.Element decode;
private Gst.RTP.BaseDepayload decode_depay;
private Gst.Element input; private Gst.Element input;
private Gst.Element output; private Gst.Element output;
private Gst.Element session;
private Device _input_device; private Device _input_device;
public Device input_device { get { return _input_device; } set { public Device input_device { get { return _input_device; } set {
@ -85,15 +88,15 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
} }
// Create app elements // Create app elements
send_rtp = Gst.ElementFactory.make("appsink", @"rtp-sink-$rtpid") as Gst.App.Sink; send_rtp = Gst.ElementFactory.make("appsink", @"rtp_sink_$rtpid") as Gst.App.Sink;
send_rtp.async = false; send_rtp.async = false;
send_rtp.caps = CodecUtil.get_caps(media, payload_type); send_rtp.caps = CodecUtil.get_caps(media, payload_type, false);
send_rtp.emit_signals = true; send_rtp.emit_signals = true;
send_rtp.sync = false; send_rtp.sync = false;
send_rtp.new_sample.connect(on_new_sample); send_rtp.new_sample.connect(on_new_sample);
pipe.add(send_rtp); pipe.add(send_rtp);
send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp-sink-$rtpid") as Gst.App.Sink; send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink;
send_rtcp.async = false; send_rtcp.async = false;
send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp");
send_rtcp.emit_signals = true; send_rtcp.emit_signals = true;
@ -101,14 +104,14 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
send_rtcp.new_sample.connect(on_new_sample); send_rtcp.new_sample.connect(on_new_sample);
pipe.add(send_rtcp); pipe.add(send_rtcp);
recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp-src-$rtpid") as Gst.App.Src; recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src;
recv_rtp.caps = CodecUtil.get_caps(media, payload_type); recv_rtp.caps = CodecUtil.get_caps(media, payload_type, true);
recv_rtp.do_timestamp = true; recv_rtp.do_timestamp = true;
recv_rtp.format = Gst.Format.TIME; recv_rtp.format = Gst.Format.TIME;
recv_rtp.is_live = true; recv_rtp.is_live = true;
pipe.add(recv_rtp); pipe.add(recv_rtp);
recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp-src-$rtpid") as Gst.App.Src; recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src;
recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp");
recv_rtcp.do_timestamp = true; recv_rtcp.do_timestamp = true;
recv_rtcp.format = Gst.Format.TIME; recv_rtcp.format = Gst.Format.TIME;
@ -122,7 +125,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad);
// Connect input // Connect input
encode = codec_util.get_encode_bin(media, payload_type, @"encode-$rtpid"); encode = codec_util.get_encode_bin(media, payload_type, @"encode_$rtpid");
encode_pay = (Gst.RTP.BasePayload)((Gst.Bin)encode).get_by_name(@"encode_$(rtpid)_rtp_pay");
pipe.add(encode); pipe.add(encode);
send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid");
encode.get_static_pad("src").link(send_rtp_sink_pad); encode.get_static_pad("src").link(send_rtp_sink_pad);
@ -131,7 +135,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
} }
// Connect output // Connect output
decode = codec_util.get_decode_bin(media, payload_type, @"decode-$rtpid"); decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid");
decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)encode).get_by_name(@"decode_$(rtpid)_rtp_depay");
pipe.add(decode); pipe.add(decode);
if (output != null) { if (output != null) {
decode.link(output); decode.link(output);
@ -144,6 +149,110 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
created = true; created = true;
push_recv_data = true; push_recv_data = true;
plugin.unpause(); plugin.unpause();
GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session);
if (session != null && payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb")) {
Object internal_session;
session.@get("internal-session", out internal_session);
if (internal_session != null) {
internal_session.connect("signal::on-feedback-rtcp", on_feedback_rtcp, this);
}
Timeout.add(1000, () => remb_adjust());
}
if (media == "video") {
codec_util.update_bitrate(media, payload_type, encode, 256);
}
}
private uint remb = 256;
private int last_packets_lost = -1;
private uint64 last_packets_received;
private uint64 last_octets_received;
private bool remb_adjust() {
unowned Gst.Structure? stats;
if (session == null) {
debug("Session for %u finished, turning off remb adjustment", rtpid);
return Source.REMOVE;
}
session.get("stats", out stats);
if (stats == null) {
warning("No stats for session %u", rtpid);
return Source.REMOVE;
}
unowned ValueArray? source_stats;
stats.get("source-stats", typeof(ValueArray), out source_stats);
if (source_stats == null) {
warning("No source-stats for session %u", rtpid);
return Source.REMOVE;
}
foreach (Value value in source_stats.values) {
unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed();
uint ssrc;
if (!source_stat.get_uint("ssrc", out ssrc)) continue;
if (ssrc.to_string() == participant_ssrc) {
int packets_lost;
uint64 packets_received, octets_received;
source_stat.get_int("packets-lost", out packets_lost);
source_stat.get_uint64("packets-received", out packets_received);
source_stat.get_uint64("octets-received", out octets_received);
int new_lost = packets_lost - last_packets_lost;
uint64 new_received = packets_received - last_packets_received;
uint64 new_octets = octets_received - last_octets_received;
if (new_received == 0) continue;
last_packets_lost = packets_lost;
last_packets_received = packets_received;
last_octets_received = octets_received;
double loss_rate = (double)new_lost / (double)(new_lost + new_received);
if (new_lost <= 0 || loss_rate < 0.02) {
remb = (uint)(1.08 * (double)remb);
} else if (loss_rate > 0.1) {
remb = (uint)((1.0 - 0.5 * loss_rate) * (double)remb);
}
remb = uint.max(remb, (uint)((new_octets * 8) / 1000));
remb = uint.max(16, remb); // Never go below 16
uint8[] data = new uint8[] {
143, 206, 0, 5,
0, 0, 0, 0,
0, 0, 0, 0,
'R', 'E', 'M', 'B',
1, 0, 0, 0,
0, 0, 0, 0
};
data[4] = (uint8)((encode_pay.ssrc >> 24) & 0xff);
data[5] = (uint8)((encode_pay.ssrc >> 16) & 0xff);
data[6] = (uint8)((encode_pay.ssrc >> 8) & 0xff);
data[7] = (uint8)(encode_pay.ssrc & 0xff);
uint8 br_exp = 0;
uint32 br_mant = remb * 1000;
uint8 bits = (uint8)Math.log2(br_mant);
if (bits > 16) {
br_exp = (uint8)bits - 16;
br_mant = br_mant >> br_exp;
}
data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3));
data[18] = (uint8)((br_mant >> 8) & 0xff);
data[19] = (uint8)(br_mant & 0xff);
data[20] = (uint8)((ssrc >> 24) & 0xff);
data[21] = (uint8)((ssrc >> 16) & 0xff);
data[22] = (uint8)((ssrc >> 8) & 0xff);
data[23] = (uint8)(ssrc & 0xff);
encrypt_and_send_rtcp(data);
}
}
return Source.CONTINUE;
}
private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) {
if (type == 206 && fbtype == 15 && fci != null && sender_ssrc.to_string() == self.participant_ssrc) {
// https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03
uint8[] data;
fci.extract_dup(0, fci.get_size(), out data);
if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return;
uint8 br_exp = data[5] >> 2;
uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7];
uint bitrate = (br_mant << br_exp) / 1000;
self.codec_util.update_bitrate(self.media, self.payload_type, self.encode, bitrate * 8);
}
} }
private void prepare_local_crypto() { private void prepare_local_crypto() {
@ -167,22 +276,26 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
if (crypto_session.has_encrypt) { if (crypto_session.has_encrypt) {
data = crypto_session.encrypt_rtp(data); data = crypto_session.encrypt_rtp(data);
} }
on_send_rtp_data(new Bytes.take(data)); on_send_rtp_data(new Bytes.take((owned) data));
} else if (sink == send_rtcp) { } else if (sink == send_rtcp) {
if (crypto_session.has_encrypt) { encrypt_and_send_rtcp((owned) data);
data = crypto_session.encrypt_rtcp(data);
}
if (rtcp_mux) {
on_send_rtp_data(new Bytes.take(data));
} else {
on_send_rtcp_data(new Bytes.take(data));
}
} else { } else {
warning("unknown sample"); warning("unknown sample");
} }
return Gst.FlowReturn.OK; return Gst.FlowReturn.OK;
} }
private void encrypt_and_send_rtcp(owned uint8[] data) {
if (crypto_session.has_encrypt) {
data = crypto_session.encrypt_rtcp(data);
}
if (rtcp_mux) {
on_send_rtp_data(new Bytes.take((owned) data));
} else {
on_send_rtcp_data(new Bytes.take((owned) data));
}
}
private static Gst.PadProbeReturn drop_probe() { private static Gst.PadProbeReturn drop_probe() {
return Gst.PadProbeReturn.DROP; return Gst.PadProbeReturn.DROP;
} }
@ -211,6 +324,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
encode.get_static_pad("src").unlink(send_rtp_sink_pad); encode.get_static_pad("src").unlink(send_rtp_sink_pad);
pipe.remove(encode); pipe.remove(encode);
encode = null; encode = null;
encode_pay = null;
// Disconnect RTP sending // Disconnect RTP sending
if (send_rtp_src_pad != null) { if (send_rtp_src_pad != null) {
@ -243,6 +357,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
decode.set_state(Gst.State.NULL); decode.set_state(Gst.State.NULL);
pipe.remove(decode); pipe.remove(decode);
decode = null; decode = null;
decode_depay = null;
output = null; output = null;
// Disconnect output device // Disconnect output device
@ -276,6 +391,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
send_rtcp_src_pad = null; send_rtcp_src_pad = null;
send_rtp_src_pad = null; send_rtp_src_pad = null;
recv_rtp_src_pad = null; recv_rtp_src_pad = null;
session = null;
} }
private void prepare_remote_crypto() { private void prepare_remote_crypto() {
@ -285,6 +402,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
} }
} }
private uint16 previous_video_orientation_degree = uint16.MAX;
public signal void video_orientation_changed(uint16 degree);
public override void on_recv_rtp_data(Bytes bytes) { public override void on_recv_rtp_data(Bytes bytes) {
if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) { if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) {
on_recv_rtcp_data(bytes); on_recv_rtcp_data(bytes);
@ -301,6 +421,33 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
} }
if (push_recv_data) { if (push_recv_data) {
Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data);
Gst.RTP.Buffer rtp_buffer;
if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) {
if (rtp_buffer.get_extension()) {
Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation");
if (ext != null) {
unowned uint8[] extension_data;
if (rtp_buffer.get_extension_onebyte_header(ext.id, 0, out extension_data) && extension_data.length == 1) {
bool camera = (extension_data[0] & 0x8) > 0;
bool flip = (extension_data[0] & 0x4) > 0;
uint8 rotation = extension_data[0] & 0x3;
uint16 rotation_degree = uint16.MAX;
switch(rotation) {
case 0: rotation_degree = 0; break;
case 1: rotation_degree = 90; break;
case 2: rotation_degree = 180; break;
case 3: rotation_degree = 270; break;
}
if (rotation_degree != previous_video_orientation_degree) {
video_orientation_changed(rotation_degree);
previous_video_orientation_degree = rotation_degree;
}
}
}
}
rtp_buffer.unmap();
}
// FIXME: VAPI file in Vala < 0.49.1 has a bug that results in broken ownership of buffer in push_buffer() // FIXME: VAPI file in Vala < 0.49.1 has a bug that results in broken ownership of buffer in push_buffer()
// We workaround by using the plain signal. The signal unfortunately will cause an unnecessary copy of // We workaround by using the plain signal. The signal unfortunately will cause an unnecessary copy of
// the underlying buffer, so and some point we should move over to the new version (once we require // the underlying buffer, so and some point we should move over to the new version (once we require
@ -449,6 +596,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
public class Dino.Plugins.Rtp.VideoStream : Stream { public class Dino.Plugins.Rtp.VideoStream : Stream {
private Gee.List<Gst.Element> outputs = new ArrayList<Gst.Element>(); private Gee.List<Gst.Element> outputs = new ArrayList<Gst.Element>();
private Gst.Element output_tee; private Gst.Element output_tee;
private Gst.Element rotate;
private ulong video_orientation_changed_handler;
public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) {
base(plugin, content); base(plugin, content);
@ -456,11 +605,15 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
} }
public override void create() { public override void create() {
video_orientation_changed_handler = video_orientation_changed.connect(on_video_orientation_changed);
plugin.pause(); plugin.pause();
output_tee = Gst.ElementFactory.make("tee", null); rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid");
pipe.add(rotate);
output_tee = Gst.ElementFactory.make("tee", @"video_tee_$rtpid");
output_tee.@set("allow-not-linked", true); output_tee.@set("allow-not-linked", true);
pipe.add(output_tee); pipe.add(output_tee);
add_output(output_tee); rotate.link(output_tee);
add_output(rotate);
base.create(); base.create();
foreach (Gst.Element output in outputs) { foreach (Gst.Element output in outputs) {
output_tee.link(output); output_tee.link(output);
@ -468,19 +621,44 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
plugin.unpause(); plugin.unpause();
} }
private void on_video_orientation_changed(uint16 degree) {
if (rotate != null) {
switch (degree) {
case 0:
rotate.@set("method", 0);
break;
case 90:
rotate.@set("method", 1);
break;
case 180:
rotate.@set("method", 2);
break;
case 270:
rotate.@set("method", 3);
break;
}
}
}
public override void destroy() { public override void destroy() {
foreach (Gst.Element output in outputs) { foreach (Gst.Element output in outputs) {
output_tee.unlink(output); output_tee.unlink(output);
} }
base.destroy(); base.destroy();
rotate.set_locked_state(true);
rotate.set_state(Gst.State.NULL);
rotate.unlink(output_tee);
pipe.remove(rotate);
rotate = null;
output_tee.set_locked_state(true); output_tee.set_locked_state(true);
output_tee.set_state(Gst.State.NULL); output_tee.set_state(Gst.State.NULL);
pipe.remove(output_tee); pipe.remove(output_tee);
output_tee = null; output_tee = null;
disconnect(video_orientation_changed_handler);
} }
public override void add_output(Gst.Element element) { public override void add_output(Gst.Element element) {
if (element == output_tee) { if (element == output_tee || element == rotate) {
base.add_output(element); base.add_output(element);
return; return;
} }
@ -491,7 +669,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
} }
public override void remove_output(Gst.Element element) { public override void remove_output(Gst.Element element) {
if (element == output_tee) { if (element == output_tee || element == rotate) {
base.remove_output(element); base.remove_output(element);
return; return;
} }

10
plugins/rtp/src/video_widget.vala
View file

@ -19,7 +19,7 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge
this.plugin = plugin; this.plugin = plugin;
id = last_id++; id = last_id++;
element = Gst.ElementFactory.make("gtksink", @"video-widget-$id"); element = Gst.ElementFactory.make("gtksink", @"video_widget_$id");
if (element != null) { if (element != null) {
Gtk.Widget widget; Gtk.Widget widget;
element.@get("widget", out widget); element.@get("widget", out widget);
@ -51,8 +51,8 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge
if (connected_stream == null) return; if (connected_stream == null) return;
plugin.pause(); plugin.pause();
pipe.add(element); pipe.add(element);
convert = Gst.parse_bin_from_description(@"videoconvert name=video-widget-$id-convert", true); convert = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true);
convert.name = @"video-widget-$id-prepare"; convert.name = @"video_widget_$(id)_prepare";
pipe.add(convert); pipe.add(convert);
convert.link(element); convert.link(element);
connected_stream.add_output(convert); connected_stream.add_output(convert);
@ -68,8 +68,8 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge
if (connected_device == null) return; if (connected_device == null) return;
plugin.pause(); plugin.pause();
pipe.add(element); pipe.add(element);
convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video-widget-$id-flip ! videoconvert name=video-widget-$id-convert", true); convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true);
convert.name = @"video-widget-$id-prepare"; convert.name = @"video_widget_$(id)_prepare";
pipe.add(convert); pipe.add(convert);
convert.link(element); convert.link(element);
connected_device.link_source().link(convert); connected_device.link_source().link(convert);

625
plugins/rtp/vapi/gstreamer-rtp-1.0.vapi Normal file
View file

@ -0,0 +1,625 @@
// Fixme: This is fetched from development code of Vala upstream which fixed a few bugs.
/* gstreamer-rtp-1.0.vapi generated by vapigen, do not modify. */
[CCode (cprefix = "Gst", gir_namespace = "GstRtp", gir_version = "1.0", lower_case_cprefix = "gst_")]
namespace Gst {
namespace RTCP {
[CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)]
[GIR (name = "RTCPBuffer")]
public struct Buffer {
public weak Gst.Buffer buffer;
public bool add_packet (Gst.RTCP.Type type, Gst.RTCP.Packet packet);
public bool get_first_packet (Gst.RTCP.Packet packet);
public uint get_packet_count ();
public static bool map (Gst.Buffer buffer, Gst.MapFlags flags, out Gst.RTCP.Buffer rtcp);
public static Gst.Buffer @new (uint mtu);
public static Gst.Buffer new_copy_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data);
public static Gst.Buffer new_take_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] owned uint8[] data);
public bool unmap ();
public static bool validate (Gst.Buffer buffer);
public static bool validate_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data);
[Version (since = "1.6")]
public static bool validate_data_reduced ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data);
[Version (since = "1.6")]
public static bool validate_reduced (Gst.Buffer buffer);
}
[CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)]
[GIR (name = "RTCPPacket")]
public struct Packet {
public weak Gst.RTCP.Buffer? rtcp;
public uint offset;
[Version (since = "1.10")]
public bool add_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data);
public bool add_rb (uint32 ssrc, uint8 fractionlost, int32 packetslost, uint32 exthighestseq, uint32 jitter, uint32 lsr, uint32 dlsr);
[Version (since = "1.10")]
public uint8 app_get_data ();
[Version (since = "1.10")]
public uint16 app_get_data_length ();
[Version (since = "1.10")]
public unowned string app_get_name ();
[Version (since = "1.10")]
public uint32 app_get_ssrc ();
[Version (since = "1.10")]
public uint8 app_get_subtype ();
[Version (since = "1.10")]
public bool app_set_data_length (uint16 wordlen);
[Version (since = "1.10")]
public void app_set_name (string name);
[Version (since = "1.10")]
public void app_set_ssrc (uint32 ssrc);
[Version (since = "1.10")]
public void app_set_subtype (uint8 subtype);
public bool bye_add_ssrc (uint32 ssrc);
public bool bye_add_ssrcs ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint32[] ssrc);
public uint32 bye_get_nth_ssrc (uint nth);
public string bye_get_reason ();
public uint8 bye_get_reason_len ();
public uint bye_get_ssrc_count ();
public bool bye_set_reason (string reason);
[Version (since = "1.10")]
public bool copy_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] out uint8[] data);
public uint8 fb_get_fci ();
public uint16 fb_get_fci_length ();
public uint32 fb_get_media_ssrc ();
public uint32 fb_get_sender_ssrc ();
public Gst.RTCP.FBType fb_get_type ();
public bool fb_set_fci_length (uint16 wordlen);
public void fb_set_media_ssrc (uint32 ssrc);
public void fb_set_sender_ssrc (uint32 ssrc);
public void fb_set_type (Gst.RTCP.FBType type);
public uint8 get_count ();
public uint16 get_length ();
public bool get_padding ();
[Version (since = "1.10")]
public bool get_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] out unowned uint8[] data);
[Version (since = "1.10")]
public uint16 get_profile_specific_ext_length ();
public void get_rb (uint nth, out uint32 ssrc, out uint8 fractionlost, out int32 packetslost, out uint32 exthighestseq, out uint32 jitter, out uint32 lsr, out uint32 dlsr);
public uint get_rb_count ();
public Gst.RTCP.Type get_type ();
public bool move_to_next ();
public bool remove ();
public uint32 rr_get_ssrc ();
public void rr_set_ssrc (uint32 ssrc);
public bool sdes_add_entry (Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] uint8[] data);
public bool sdes_add_item (uint32 ssrc);
public bool sdes_copy_entry (out Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] out uint8[] data);
public bool sdes_first_entry ();
public bool sdes_first_item ();
public bool sdes_get_entry (out Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] out unowned uint8[] data);
public uint sdes_get_item_count ();
public uint32 sdes_get_ssrc ();
public bool sdes_next_entry ();
public bool sdes_next_item ();
public void set_rb (uint nth, uint32 ssrc, uint8 fractionlost, int32 packetslost, uint32 exthighestseq, uint32 jitter, uint32 lsr, uint32 dlsr);
public void sr_get_sender_info (out uint32 ssrc, out uint64 ntptime, out uint32 rtptime, out uint32 packet_count, out uint32 octet_count);
public void sr_set_sender_info (uint32 ssrc, uint64 ntptime, uint32 rtptime, uint32 packet_count, uint32 octet_count);
[Version (since = "1.16")]
public bool xr_first_rb ();
[Version (since = "1.16")]
public uint16 xr_get_block_length ();
[Version (since = "1.16")]
public Gst.RTCP.XRType xr_get_block_type ();
[Version (since = "1.16")]
public bool xr_get_dlrr_block (uint nth, out uint32 ssrc, out uint32 last_rr, out uint32 delay);
[Version (since = "1.16")]
public bool xr_get_prt_by_seq (uint16 seq, out uint32 receipt_time);
[Version (since = "1.16")]
public bool xr_get_prt_info (out uint32 ssrc, out uint8 thinning, out uint16 begin_seq, out uint16 end_seq);
[Version (since = "1.16")]
public bool xr_get_rle_info (out uint32 ssrc, out uint8 thinning, out uint16 begin_seq, out uint16 end_seq, out uint32 chunk_count);
[Version (since = "1.16")]
public bool xr_get_rle_nth_chunk (uint nth, out uint16 chunk);
[Version (since = "1.16")]
public bool xr_get_rrt (out uint64 timestamp);
[Version (since = "1.16")]
public uint32 xr_get_ssrc ();
[Version (since = "1.16")]
public bool xr_get_summary_info (out uint32 ssrc, out uint16 begin_seq, out uint16 end_seq);
[Version (since = "1.16")]
public bool xr_get_summary_jitter (out uint32 min_jitter, out uint32 max_jitter, out uint32 mean_jitter, out uint32 dev_jitter);
[Version (since = "1.16")]
public bool xr_get_summary_pkt (out uint32 lost_packets, out uint32 dup_packets);
[Version (since = "1.16")]
public bool xr_get_summary_ttl (out bool is_ipv4, out uint8 min_ttl, out uint8 max_ttl, out uint8 mean_ttl, out uint8 dev_ttl);
[Version (since = "1.16")]
public bool xr_get_voip_burst_metrics (out uint8 burst_density, out uint8 gap_density, out uint16 burst_duration, out uint16 gap_duration);
[Version (since = "1.16")]
public bool xr_get_voip_configuration_params (out uint8 gmin, out uint8 rx_config);
[Version (since = "1.16")]
public bool xr_get_voip_delay_metrics (out uint16 roundtrip_delay, out uint16 end_system_delay);
[Version (since = "1.16")]
public bool xr_get_voip_jitter_buffer_params (out uint16 jb_nominal, out uint16 jb_maximum, out uint16 jb_abs_max);
[Version (since = "1.16")]
public bool xr_get_voip_metrics_ssrc (out uint32 ssrc);
[Version (since = "1.16")]
public bool xr_get_voip_packet_metrics (out uint8 loss_rate, out uint8 discard_rate);
[Version (since = "1.16")]
public bool xr_get_voip_quality_metrics (out uint8 r_factor, out uint8 ext_r_factor, out uint8 mos_lq, out uint8 mos_cq);
[Version (since = "1.16")]
public bool xr_get_voip_signal_metrics (out uint8 signal_level, out uint8 noise_level, out uint8 rerl, out uint8 gmin);
[Version (since = "1.16")]
public bool xr_next_rb ();
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_", type_id = "gst_rtcpfb_type_get_type ()")]
[GIR (name = "RTCPFBType")]
public enum FBType {
FB_TYPE_INVALID,
RTPFB_TYPE_NACK,
RTPFB_TYPE_TMMBR,
RTPFB_TYPE_TMMBN,
RTPFB_TYPE_RTCP_SR_REQ,
RTPFB_TYPE_TWCC,
PSFB_TYPE_PLI,
PSFB_TYPE_SLI,
PSFB_TYPE_RPSI,
PSFB_TYPE_AFB,
PSFB_TYPE_FIR,
PSFB_TYPE_TSTR,
PSFB_TYPE_TSTN,
PSFB_TYPE_VBCN
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_SDES_", type_id = "gst_rtcpsdes_type_get_type ()")]
[GIR (name = "RTCPSDESType")]
public enum SDESType {
INVALID,
END,
CNAME,
NAME,
EMAIL,
PHONE,
LOC,
TOOL,
NOTE,
PRIV;
[CCode (cname = "gst_rtcp_sdes_name_to_type")]
public static Gst.RTCP.SDESType from_string (string name);
[CCode (cname = "gst_rtcp_sdes_type_to_name")]
public unowned string to_string ();
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_TYPE_", type_id = "gst_rtcp_type_get_type ()")]
[GIR (name = "RTCPType")]
public enum Type {
INVALID,
SR,
RR,
SDES,
BYE,
APP,
RTPFB,
PSFB,
XR
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_XR_TYPE_", type_id = "gst_rtcpxr_type_get_type ()")]
[GIR (name = "RTCPXRType")]
[Version (since = "1.16")]
public enum XRType {
INVALID,
LRLE,
DRLE,
PRT,
RRT,
DLRR,
SSUMM,
VOIP_METRICS
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_BYE_SSRC_COUNT")]
public const int MAX_BYE_SSRC_COUNT;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_RB_COUNT")]
public const int MAX_RB_COUNT;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_SDES")]
public const int MAX_SDES;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_SDES_ITEM_COUNT")]
public const int MAX_SDES_ITEM_COUNT;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_REDUCED_SIZE_VALID_MASK")]
public const int REDUCED_SIZE_VALID_MASK;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VALID_MASK")]
public const int VALID_MASK;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VALID_VALUE")]
public const int VALID_VALUE;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VERSION")]
public const int VERSION;
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static uint64 ntp_to_unix (uint64 ntptime);
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static uint64 unix_to_ntp (uint64 unixtime);
}
namespace RTP {
[CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_audio_payload_get_type ()")]
[GIR (name = "RTPBaseAudioPayload")]
public class BaseAudioPayload : Gst.RTP.BasePayload {
public Gst.ClockTime base_ts;
public int frame_duration;
public int frame_size;
public int sample_size;
[CCode (has_construct_function = false)]
protected BaseAudioPayload ();
public Gst.FlowReturn flush (uint payload_len, Gst.ClockTime timestamp);
public Gst.Base.Adapter get_adapter ();
public Gst.FlowReturn push ([CCode (array_length_cname = "payload_len", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, Gst.ClockTime timestamp);
public void set_frame_based ();
public void set_frame_options (int frame_duration, int frame_size);
public void set_sample_based ();
public void set_sample_options (int sample_size);
public void set_samplebits_options (int sample_size);
[NoAccessorMethod]
public bool buffer_list { get; set; }
}
[CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_depayload_get_type ()")]
[GIR (name = "RTPBaseDepayload")]
public abstract class BaseDepayload : Gst.Element {
public uint clock_rate;
public bool need_newsegment;
public weak Gst.Segment segment;
public weak Gst.Pad sinkpad;
public weak Gst.Pad srcpad;
[CCode (has_construct_function = false)]
protected BaseDepayload ();
[NoWrapper]
public virtual bool handle_event (Gst.Event event);
[Version (since = "1.16")]
public bool is_source_info_enabled ();
[NoWrapper]
public virtual bool packet_lost (Gst.Event event);
[NoWrapper]
public virtual Gst.Buffer process (Gst.Buffer @in);
[NoWrapper]
public virtual Gst.Buffer process_rtp_packet (Gst.RTP.Buffer rtp_buffer);
public Gst.FlowReturn push (Gst.Buffer out_buf);
public Gst.FlowReturn push_list (Gst.BufferList out_list);
[NoWrapper]
public virtual bool set_caps (Gst.Caps caps);
[Version (since = "1.16")]
public void set_source_info_enabled (bool enable);
[NoAccessorMethod]
[Version (since = "1.20")]
public bool auto_header_extension { get; set; }
[NoAccessorMethod]
[Version (since = "1.18")]
public int max_reorder { get; set; }
[NoAccessorMethod]
[Version (since = "1.16")]
public bool source_info { get; set; }
[NoAccessorMethod]
public Gst.Structure stats { owned get; }
[Version (since = "1.20")]
public signal void add_extension (owned Gst.RTP.HeaderExtension ext);
[Version (since = "1.20")]
public signal void clear_extensions ();
[Version (since = "1.20")]
public signal Gst.RTP.HeaderExtension request_extension (uint ext_id, string? ext_uri);
}
[CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_payload_get_type ()")]
[GIR (name = "RTPBasePayload")]
public abstract class BasePayload : Gst.Element {
[CCode (has_construct_function = false)]
protected BasePayload ();
[Version (since = "1.16")]
public Gst.Buffer allocate_output_buffer (uint payload_len, uint8 pad_len, uint8 csrc_count);
[NoWrapper]
public virtual Gst.Caps get_caps (Gst.Pad pad, Gst.Caps filter);
[Version (since = "1.16")]
public uint get_source_count (Gst.Buffer buffer);
[NoWrapper]
public virtual Gst.FlowReturn handle_buffer (Gst.Buffer buffer);
public bool is_filled (uint size, Gst.ClockTime duration);
[Version (since = "1.16")]
public bool is_source_info_enabled ();
public Gst.FlowReturn push (Gst.Buffer buffer);
public Gst.FlowReturn push_list (Gst.BufferList list);
[NoWrapper]
public virtual bool query (Gst.Pad pad, Gst.Query query);
[NoWrapper]
public virtual bool set_caps (Gst.Caps caps);
public void set_options (string media, bool @dynamic, string encoding_name, uint32 clock_rate);
[Version (since = "1.20")]
public bool set_outcaps_structure (Gst.Structure? s);
[Version (since = "1.16")]
public void set_source_info_enabled (bool enable);
[NoWrapper]
public virtual bool sink_event (Gst.Event event);
[NoWrapper]
public virtual bool src_event (Gst.Event event);
[NoAccessorMethod]
[Version (since = "1.20")]
public bool auto_header_extension { get; set; }
[NoAccessorMethod]
public int64 max_ptime { get; set; }
[NoAccessorMethod]
public int64 min_ptime { get; set; }
[NoAccessorMethod]
public uint mtu { get; set; }
[NoAccessorMethod]
[Version (since = "1.16")]
public bool onvif_no_rate_control { get; set; }
[NoAccessorMethod]
public bool perfect_rtptime { get; set; }
[NoAccessorMethod]
public uint pt { get; set; }
[NoAccessorMethod]
public int64 ptime_multiple { get; set; }
[NoAccessorMethod]
[Version (since = "1.18")]
public bool scale_rtptime { get; set; }
[NoAccessorMethod]
public uint seqnum { get; }
[NoAccessorMethod]
public int seqnum_offset { get; set; }
[NoAccessorMethod]
[Version (since = "1.16")]
public bool source_info { get; set; }
[NoAccessorMethod]
public uint ssrc { get; set; }
[NoAccessorMethod]
public Gst.Structure stats { owned get; }
[NoAccessorMethod]
public uint timestamp { get; }
[NoAccessorMethod]
public uint timestamp_offset { get; set; }
[Version (since = "1.20")]
public signal void add_extension (owned Gst.RTP.HeaderExtension ext);
[Version (since = "1.20")]
public signal void clear_extensions ();
[Version (since = "1.20")]
public signal Gst.RTP.HeaderExtension request_extension (uint ext_id, string ext_uri);
}
[CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_header_extension_get_type ()")]
[GIR (name = "RTPHeaderExtension")]
[Version (since = "1.20")]
public abstract class HeaderExtension : Gst.Element {
public uint ext_id;
[CCode (has_construct_function = false)]
protected HeaderExtension ();
public static Gst.RTP.HeaderExtension? create_from_uri (string uri);
public uint get_id ();
public virtual size_t get_max_size (Gst.Buffer input_meta);
public string get_sdp_caps_field_name ();
public virtual Gst.RTP.HeaderExtensionFlags get_supported_flags ();
public unowned string get_uri ();
public virtual bool read (Gst.RTP.HeaderExtensionFlags read_flags, [CCode (array_length_cname = "size", array_length_pos = 2.5, array_length_type = "gsize", type = "const guint8*")] uint8[] data, Gst.Buffer buffer);
public virtual bool set_attributes_from_caps (Gst.Caps caps);
public bool set_attributes_from_caps_simple_sdp (Gst.Caps caps);
public virtual bool set_caps_from_attributes (Gst.Caps caps);
public bool set_caps_from_attributes_simple_sdp (Gst.Caps caps);
public void set_id (uint ext_id);
public virtual bool set_non_rtp_sink_caps (Gst.Caps caps);
[CCode (cname = "gst_rtp_header_extension_class_set_uri")]
public class void set_uri (string uri);
public void set_wants_update_non_rtp_src_caps (bool state);
public virtual bool update_non_rtp_src_caps (Gst.Caps caps);
public virtual size_t write (Gst.Buffer input_meta, Gst.RTP.HeaderExtensionFlags write_flags, Gst.Buffer output, [CCode (array_length_cname = "size", array_length_pos = 4.1, array_length_type = "gsize", type = "guint8*")] uint8[] data);
}
[CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)]
[GIR (name = "RTPBuffer")]
public struct Buffer {
public weak Gst.Buffer buffer;
public uint state;
[CCode (array_length = false)]
public weak void* data[4];
[CCode (array_length = false)]
public weak size_t size[4];
public bool add_extension_onebyte_header (uint8 id, [CCode (array_length_cname = "size", array_length_pos = 2.1, array_length_type = "guint")] uint8[] data);
public bool add_extension_twobytes_header (uint8 appbits, uint8 id, [CCode (array_length_cname = "size", array_length_pos = 3.1, array_length_type = "guint")] uint8[] data);
[CCode (cname = "gst_buffer_add_rtp_source_meta")]
[Version (since = "1.16")]
public static unowned Gst.RTP.SourceMeta? add_rtp_source_meta (Gst.Buffer buffer, uint32? ssrc, uint32? csrc, uint csrc_count);
public static void allocate_data (Gst.Buffer buffer, uint payload_len, uint8 pad_len, uint8 csrc_count);
public static uint calc_header_len (uint8 csrc_count);
public static uint calc_packet_len (uint payload_len, uint8 pad_len, uint8 csrc_count);
public static uint calc_payload_len (uint packet_len, uint8 pad_len, uint8 csrc_count);
public static int compare_seqnum (uint16 seqnum1, uint16 seqnum2);
public static uint32 default_clock_rate (uint8 payload_type);
public static uint64 ext_timestamp (ref uint64 exttimestamp, uint32 timestamp);
public uint32 get_csrc (uint8 idx);
public uint8 get_csrc_count ();
public bool get_extension ();
[Version (since = "1.2")]
public GLib.Bytes get_extension_bytes (out uint16 bits);
public bool get_extension_data (out uint16 bits, [CCode (array_length = false)] out unowned uint8[] data, out uint wordlen);
public bool get_extension_onebyte_header (uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 3.1, array_length_type = "guint")] out unowned uint8[] data);
[Version (since = "1.18")]
public static bool get_extension_onebyte_header_from_bytes (GLib.Bytes bytes, uint16 bit_pattern, uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 5.1, array_length_type = "guint")] out unowned uint8[] data);
public bool get_extension_twobytes_header (out uint8 appbits, uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 4.1, array_length_type = "guint")] out unowned uint8[] data);
public uint get_header_len ();
public bool get_marker ();
public uint get_packet_len ();
public bool get_padding ();
[CCode (array_length = false)]
public unowned uint8[] get_payload ();
public Gst.Buffer get_payload_buffer ();
[Version (since = "1.2")]
public GLib.Bytes get_payload_bytes ();
public uint get_payload_len ();
public Gst.Buffer get_payload_subbuffer (uint offset, uint len);
public uint8 get_payload_type ();
[CCode (cname = "gst_buffer_get_rtp_source_meta")]
[Version (since = "1.16")]
public static unowned Gst.RTP.SourceMeta? get_rtp_source_meta (Gst.Buffer buffer);
public uint16 get_seq ();
public uint32 get_ssrc ();
public uint32 get_timestamp ();
public uint8 get_version ();
public static bool map (Gst.Buffer buffer, Gst.MapFlags flags, out Gst.RTP.Buffer rtp);
public static Gst.Buffer new_allocate (uint payload_len, uint8 pad_len, uint8 csrc_count);
public static Gst.Buffer new_allocate_len (uint packet_len, uint8 pad_len, uint8 csrc_count);
public static Gst.Buffer new_copy_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "gsize")] uint8[] data);
public static Gst.Buffer new_take_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "gsize")] owned uint8[] data);
public void pad_to (uint len);
public void set_csrc (uint8 idx, uint32 csrc);
public void set_extension (bool extension);
public bool set_extension_data (uint16 bits, uint16 length);
public void set_marker (bool marker);
public void set_packet_len (uint len);
public void set_padding (bool padding);
public void set_payload_type (uint8 payload_type);
public void set_seq (uint16 seq);
public void set_ssrc (uint32 ssrc);
public void set_timestamp (uint32 timestamp);
public void set_version (uint8 version);
public void unmap ();
}
[CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)]
[GIR (name = "RTPPayloadInfo")]
public struct PayloadInfo {
public uint8 payload_type;
public weak string media;
public weak string encoding_name;
public uint clock_rate;
public weak string encoding_parameters;
public uint bitrate;
}
[CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)]
[GIR (name = "RTPSourceMeta")]
[Version (since = "1.16")]
public struct SourceMeta {
public Gst.Meta meta;
public uint32 ssrc;
public bool ssrc_valid;
[CCode (array_length = false)]
public weak uint32 csrc[15];
public uint csrc_count;
public bool append_csrc ([CCode (array_length_cname = "csrc_count", array_length_pos = 1.1, array_length_type = "guint", type = "const guint32*")] uint32[] csrc);
public uint get_source_count ();
public bool set_ssrc (uint32? ssrc);
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_BUFFER_FLAG_", type_id = "gst_rtp_buffer_flags_get_type ()")]
[Flags]
[GIR (name = "RTPBufferFlags")]
[Version (since = "1.10")]
public enum BufferFlags {
RETRANSMISSION,
REDUNDANT,
LAST
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_BUFFER_MAP_FLAG_", type_id = "gst_rtp_buffer_map_flags_get_type ()")]
[Flags]
[GIR (name = "RTPBufferMapFlags")]
[Version (since = "1.6.1")]
public enum BufferMapFlags {
SKIP_PADDING,
LAST
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_HEADER_EXTENSION_", type_id = "gst_rtp_header_extension_flags_get_type ()")]
[Flags]
[GIR (name = "RTPHeaderExtensionFlags")]
[Version (since = "1.20")]
public enum HeaderExtensionFlags {
ONE_BYTE,
TWO_BYTE
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_PAYLOAD_", type_id = "gst_rtp_payload_get_type ()")]
[GIR (name = "RTPPayload")]
public enum Payload {
PCMU,
@1016,
G721,
GSM,
G723,
DVI4_8000,
DVI4_16000,
LPC,
PCMA,
G722,
L16_STEREO,
L16_MONO,
QCELP,
CN,
MPA,
G728,
DVI4_11025,
DVI4_22050,
G729,
CELLB,
JPEG,
NV,
H261,
MPV,
MP2T,
H263;
public const string @1016_STRING;
public const string CELLB_STRING;
public const string CN_STRING;
public const string DVI4_11025_STRING;
public const string DVI4_16000_STRING;
public const string DVI4_22050_STRING;
public const string DVI4_8000_STRING;
public const string DYNAMIC_STRING;
public const string G721_STRING;
public const string G722_STRING;
public const int G723_53;
public const string G723_53_STRING;
public const int G723_63;
public const string G723_63_STRING;
public const string G723_STRING;
public const string G728_STRING;
public const string G729_STRING;
public const string GSM_STRING;
public const string H261_STRING;
public const string H263_STRING;
public const string JPEG_STRING;
public const string L16_MONO_STRING;
public const string L16_STEREO_STRING;
public const string LPC_STRING;
public const string MP2T_STRING;
public const string MPA_STRING;
public const string MPV_STRING;
public const string NV_STRING;
public const string PCMA_STRING;
public const string PCMU_STRING;
public const string QCELP_STRING;
public const int TS41;
public const string TS41_STRING;
public const int TS48;
public const string TS48_STRING;
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_PROFILE_", type_id = "gst_rtp_profile_get_type ()")]
[GIR (name = "RTPProfile")]
[Version (since = "1.6")]
public enum Profile {
UNKNOWN,
AVP,
SAVP,
AVPF,
SAVPF
}
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_BASE")]
public const string HDREXT_BASE;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_ELEMENT_CLASS")]
[Version (since = "1.20")]
public const string HDREXT_ELEMENT_CLASS;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_56")]
public const string HDREXT_NTP_56;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_56_SIZE")]
public const int HDREXT_NTP_56_SIZE;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_64")]
public const string HDREXT_NTP_64;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_64_SIZE")]
public const int HDREXT_NTP_64_SIZE;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HEADER_EXTENSION_URI_METADATA_KEY")]
[Version (since = "1.20")]
public const string HEADER_EXTENSION_URI_METADATA_KEY;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_SOURCE_META_MAX_CSRC_COUNT")]
public const int SOURCE_META_MAX_CSRC_COUNT;
[CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_VERSION")]
public const int VERSION;
[CCode (cheader_filename = "gst/rtp/rtp.h")]
[Version (since = "1.20")]
public static GLib.List<Gst.RTP.HeaderExtension> get_header_extension_list ();
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static bool hdrext_get_ntp_56 ([CCode (array_length_cname = "size", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, out uint64 ntptime);
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static bool hdrext_get_ntp_64 ([CCode (array_length_cname = "size", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, out uint64 ntptime);
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static bool hdrext_set_ntp_56 (void* data, uint size, uint64 ntptime);
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static bool hdrext_set_ntp_64 (void* data, uint size, uint64 ntptime);
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static unowned Gst.RTP.PayloadInfo? payload_info_for_name (string media, string encoding_name);
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static unowned Gst.RTP.PayloadInfo? payload_info_for_pt (uint8 payload_type);
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static GLib.Type source_meta_api_get_type ();
[CCode (cheader_filename = "gst/rtp/rtp.h")]
public static unowned Gst.MetaInfo? source_meta_get_info ();
}
}

40
xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala
View file

@ -17,6 +17,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object {
public bool encryption_required { get; private set; default = false; } public bool encryption_required { get; private set; default = false; }
public PayloadType? agreed_payload_type { get; private set; } public PayloadType? agreed_payload_type { get; private set; }
public Gee.List<PayloadType> payload_types = new ArrayList<PayloadType>(PayloadType.equals_func); public Gee.List<PayloadType> payload_types = new ArrayList<PayloadType>(PayloadType.equals_func);
public Gee.List<HeaderExtension> header_extensions = new ArrayList<HeaderExtension>();
public Gee.List<Crypto> remote_cryptos = new ArrayList<Crypto>(); public Gee.List<Crypto> remote_cryptos = new ArrayList<Crypto>();
public Crypto? local_crypto = null; public Crypto? local_crypto = null;
public Crypto? remote_crypto = null; public Crypto? remote_crypto = null;
@ -54,9 +55,12 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object {
this.remote_cryptos.add(Crypto.parse(crypto)); this.remote_cryptos.add(Crypto.parse(crypto));
} }
} }
foreach (StanzaNode payloadType in node.get_subnodes("payload-type")) { foreach (StanzaNode payloadType in node.get_subnodes(PayloadType.NAME)) {
this.payload_types.add(PayloadType.parse(payloadType)); this.payload_types.add(PayloadType.parse(payloadType));
} }
foreach (StanzaNode subnode in node.get_subnodes(HeaderExtension.NAME, HeaderExtension.NS_URI)) {
this.header_extensions.add(HeaderExtension.parse(subnode));
}
} }
public async void handle_proposed_content(XmppStream stream, Jingle.Session session, Jingle.Content content) { public async void handle_proposed_content(XmppStream stream, Jingle.Session session, Jingle.Content content) {
@ -66,6 +70,11 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object {
content.reject(); content.reject();
return; return;
} }
// Drop unsupported header extensions
var iter = header_extensions.iterator();
while(iter.next()) {
if (!parent.is_header_extension_supported(media, iter.@get())) iter.remove();
}
remote_crypto = parent.pick_remote_crypto(remote_cryptos); remote_crypto = parent.pick_remote_crypto(remote_cryptos);
if (local_crypto == null && remote_crypto != null) { if (local_crypto == null && remote_crypto != null) {
local_crypto = parent.pick_local_crypto(remote_crypto); local_crypto = parent.pick_local_crypto(remote_crypto);
@ -151,7 +160,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object {
Gee.List<StanzaNode> crypto_nodes = description_node.get_deep_subnodes("encryption", "crypto"); Gee.List<StanzaNode> crypto_nodes = description_node.get_deep_subnodes("encryption", "crypto");
if (crypto_nodes.size == 0) { if (crypto_nodes.size == 0) {
warning("Counterpart didn't include any cryptos"); debug("Counterpart didn't include any cryptos");
if (encryption_required) { if (encryption_required) {
return; return;
} }
@ -182,6 +191,9 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object {
ret.put_node(payload_type.to_xml()); ret.put_node(payload_type.to_xml());
} }
} }
foreach (HeaderExtension ext in header_extensions) {
ret.put_node(ext.to_xml());
}
if (local_crypto != null) { if (local_crypto != null) {
ret.put_node(new StanzaNode.build("encryption", NS_URI) ret.put_node(new StanzaNode.build("encryption", NS_URI)
.put_node(local_crypto.to_xml())); .put_node(local_crypto.to_xml()));
@ -192,3 +204,27 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object {
return ret; return ret;
} }
} }
public class Xmpp.Xep.JingleRtp.HeaderExtension {
public const string NS_URI = "urn:xmpp:jingle:apps:rtp:rtp-hdrext:0";
public const string NAME = "rtp-hdrext";
public uint8 id { get; private set; }
public string uri { get; private set; }
public HeaderExtension(uint8 id, string uri) {
this.id = id;
this.uri = uri;
}
public static HeaderExtension parse(StanzaNode node) {
return new HeaderExtension((uint8) node.get_attribute_int("id"), node.get_attribute("uri"));
}
public StanzaNode to_xml() {
return new StanzaNode.build(NAME, NS_URI)
.add_self_xmlns()
.put_attribute("id", id.to_string())
.put_attribute("uri", uri);
}
}

5
xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala
View file

@ -24,6 +24,8 @@ public abstract class Module : XmppStreamModule {
public abstract Crypto? pick_remote_crypto(Gee.List<Crypto> cryptos); public abstract Crypto? pick_remote_crypto(Gee.List<Crypto> cryptos);
public abstract Crypto? pick_local_crypto(Crypto? remote); public abstract Crypto? pick_local_crypto(Crypto? remote);
public abstract Stream create_stream(Jingle.Content content); public abstract Stream create_stream(Jingle.Content content);
public abstract bool is_header_extension_supported(string media, HeaderExtension ext);
public abstract Gee.List<HeaderExtension> get_suggested_header_extensions(string media);
public abstract void close_stream(Stream stream); public abstract void close_stream(Stream stream);
public async Jingle.Session start_call(XmppStream stream, Jid receiver_full_jid, bool video, string? sid = null) throws Jingle.Error { public async Jingle.Session start_call(XmppStream stream, Jid receiver_full_jid, bool video, string? sid = null) throws Jingle.Error {
@ -40,6 +42,7 @@ public abstract class Module : XmppStreamModule {
// Create audio content // Create audio content
Parameters audio_content_parameters = new Parameters(this, "audio", yield get_supported_payloads("audio")); Parameters audio_content_parameters = new Parameters(this, "audio", yield get_supported_payloads("audio"));
audio_content_parameters.local_crypto = generate_local_crypto(); audio_content_parameters.local_crypto = generate_local_crypto();
audio_content_parameters.header_extensions.add_all(get_suggested_header_extensions("audio"));
Jingle.Transport? audio_transport = yield jingle_module.select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); Jingle.Transport? audio_transport = yield jingle_module.select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty());
if (audio_transport == null) { if (audio_transport == null) {
throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable audio transports"); throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable audio transports");
@ -57,6 +60,7 @@ public abstract class Module : XmppStreamModule {
// Create video content // Create video content
Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video"));
video_content_parameters.local_crypto = generate_local_crypto(); video_content_parameters.local_crypto = generate_local_crypto();
video_content_parameters.header_extensions.add_all(get_suggested_header_extensions("video"));
Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty());
if (video_transport == null) { if (video_transport == null) {
throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports");
@ -98,6 +102,7 @@ public abstract class Module : XmppStreamModule {
// Content for video does not yet exist -> create it // Content for video does not yet exist -> create it
Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video"));
video_content_parameters.local_crypto = generate_local_crypto(); video_content_parameters.local_crypto = generate_local_crypto();
video_content_parameters.header_extensions.add_all(get_suggested_header_extensions("video"));
Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty());
if (video_transport == null) { if (video_transport == null) {
throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports");

49
xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala
View file

@ -3,6 +3,8 @@ using Xmpp;
using Xmpp.Xep; using Xmpp.Xep;
public class Xmpp.Xep.JingleRtp.PayloadType { public class Xmpp.Xep.JingleRtp.PayloadType {
public const string NAME = "payload-type";
public uint8 id { get; set; } public uint8 id { get; set; }
public string? name { get; set; } public string? name { get; set; }
public uint8 channels { get; set; default = 1; } public uint8 channels { get; set; default = 1; }
@ -10,6 +12,7 @@ public class Xmpp.Xep.JingleRtp.PayloadType {
public uint32 maxptime { get; set; } public uint32 maxptime { get; set; }
public uint32 ptime { get; set; } public uint32 ptime { get; set; }
public Map<string, string> parameters = new HashMap<string, string>(); public Map<string, string> parameters = new HashMap<string, string>();
public Gee.List<RtcpFeedback> rtcp_fbs = new ArrayList<RtcpFeedback>();
public static PayloadType parse(StanzaNode node) { public static PayloadType parse(StanzaNode node) {
PayloadType payloadType = new PayloadType(); PayloadType payloadType = new PayloadType();
@ -22,11 +25,14 @@ public class Xmpp.Xep.JingleRtp.PayloadType {
foreach (StanzaNode parameter in node.get_subnodes("parameter")) { foreach (StanzaNode parameter in node.get_subnodes("parameter")) {
payloadType.parameters[parameter.get_attribute("name")] = parameter.get_attribute("value"); payloadType.parameters[parameter.get_attribute("name")] = parameter.get_attribute("value");
} }
foreach (StanzaNode subnode in node.get_subnodes(RtcpFeedback.NAME, RtcpFeedback.NS_URI)) {
payloadType.rtcp_fbs.add(RtcpFeedback.parse(subnode));
}
return payloadType; return payloadType;
} }
public StanzaNode to_xml() { public StanzaNode to_xml() {
StanzaNode node = new StanzaNode.build("payload-type", NS_URI) StanzaNode node = new StanzaNode.build(NAME, NS_URI)
.put_attribute("id", id.to_string()); .put_attribute("id", id.to_string());
if (channels != 1) node.put_attribute("channels", channels.to_string()); if (channels != 1) node.put_attribute("channels", channels.to_string());
if (clockrate != 0) node.put_attribute("clockrate", clockrate.to_string()); if (clockrate != 0) node.put_attribute("clockrate", clockrate.to_string());
@ -38,9 +44,25 @@ public class Xmpp.Xep.JingleRtp.PayloadType {
.put_attribute("name", parameter) .put_attribute("name", parameter)
.put_attribute("value", parameters[parameter])); .put_attribute("value", parameters[parameter]));
} }
foreach (RtcpFeedback rtcp_fb in rtcp_fbs) {
node.put_node(rtcp_fb.to_xml());
}
return node; return node;
} }
public PayloadType clone() {
PayloadType clone = new PayloadType();
clone.id = id;
clone.name = name;
clone.channels = channels;
clone.clockrate = clockrate;
clone.maxptime = maxptime;
clone.ptime = ptime;
clone.parameters.set_all(parameters);
clone.rtcp_fbs.add_all(rtcp_fbs);
return clone;
}
public static bool equals_func(PayloadType a, PayloadType b) { public static bool equals_func(PayloadType a, PayloadType b) {
return a.id == b.id && return a.id == b.id &&
a.name == b.name && a.name == b.name &&
@ -50,3 +72,28 @@ public class Xmpp.Xep.JingleRtp.PayloadType {
a.ptime == b.ptime; a.ptime == b.ptime;
} }
} }
public class Xmpp.Xep.JingleRtp.RtcpFeedback {
public const string NS_URI = "urn:xmpp:jingle:apps:rtp:rtcp-fb:0";
public const string NAME = "rtcp-fb";
public string type_ { get; private set; }
public string? subtype { get; private set; }
public RtcpFeedback(string type, string? subtype = null) {
this.type_ = type;
this.subtype = subtype;
}
public static RtcpFeedback parse(StanzaNode node) {
return new RtcpFeedback(node.get_attribute("type"), node.get_attribute("subtype"));
}
public StanzaNode to_xml() {
StanzaNode node = new StanzaNode.build(NAME, NS_URI)
.add_self_xmlns()
.put_attribute("type", type_);
if (subtype != null) node.put_attribute("subtype", subtype);
return node;
}
}

7
xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala
View file

@ -33,6 +33,13 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object {
} }
return null; return null;
}} }}
public Gee.List<JingleRtp.HeaderExtension>? header_extensions { get {
var content_params = content.content_params;
if (content_params is Parameters) {
return ((Parameters)content_params).header_extensions;
}
return null;
}}
public bool sending { get { public bool sending { get {
return content.session.senders_include_us(content.senders); return content.session.senders_include_us(content.senders);
}} }}