anotherim/anotherim-desktop
2
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

RTP: Improve screen resolution update logic

Browse source
This commit is contained in:
Marvin W 2021-12-19 22:38:00 +01:00
parent b1c1751cc4
commit cd6d501c23
2 changed files with 30 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
plugins/rtp/src/device.vala
View file

@ -2,24 +2,19 @@ using Xmpp.Xep.JingleRtp;
using Gee; using Gee;
public class Dino.Plugins.Rtp.Device : MediaDevice, Object { public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
private const int[] common_widths = {320, 360, 400, 480, 640, 960, 1280, 1920, 2560, 3840};
public Plugin plugin { get; private set; } public Plugin plugin { get; private set; }
public CodecUtil codec_util { get { return plugin.codec_util; } } public CodecUtil codec_util { get { return plugin.codec_util; } }
public Gst.Device device { get; private set; } public Gst.Device device { get; private set; }
private string device_name; public string id { get { return device_name; }}
public string id { get { public string display_name { get { return device_display_name; }}
return device_name;
}}
private string device_display_name;
public string display_name { get {
return device_display_name;
}}
public string detail_name { get { public string detail_name { get {
return device.properties.get_string("alsa.card_name") ?? device.properties.get_string("alsa.id") ?? id; return device.properties.get_string("alsa.card_name") ?? device.properties.get_string("alsa.id") ?? id;
}} }}
public Gst.Pipeline pipe { get {
return plugin.pipe; public Gst.Pipeline pipe { get { return plugin.pipe; }}
}}
public string? media { get { public string? media { get {
if (device.has_classes("Audio")) { if (device.has_classes("Audio")) {
return "audio"; return "audio";
@ -29,12 +24,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
return null; return null;
} }
}} }}
public bool is_source { get { public bool is_source { get { return device.has_classes("Source"); }}
return device.has_classes("Source"); public bool is_sink { get { return device.has_classes("Sink"); }}
}}
public bool is_sink { get { private string device_name;
return device.has_classes("Sink"); private string device_display_name;
}}
private Gst.Caps device_caps; private Gst.Caps device_caps;
private Gst.Element element; private Gst.Element element;
@ -96,11 +90,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
return element; return element;
} }
public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1, uint32 timestamp_offset = 0) { public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = 0, int seqnum_offset = -1, uint32 timestamp_offset = 0) {
if (!is_source) return null; if (!is_source) return null;
if (element == null) create(); if (element == null) create();
links++; links++;
if (payload_type != null && tee != null) { if (payload_type != null && ssrc != 0 && tee != null) {
bool new_codec = false; bool new_codec = false;
string? codec = CodecUtil.get_codec_from_payload(media, payload_type); string? codec = CodecUtil.get_codec_from_payload(media, payload_type);
if (!codecs.has_key(payload_type)) { if (!codecs.has_key(payload_type)) {
@ -172,16 +166,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
return br; return br;
} }
private const int[] common_widths = {320, 480, 640, 960, 1280, 1920, 2560, 3840};
private Gst.Caps get_active_caps(PayloadType payload_type) { private Gst.Caps get_active_caps(PayloadType payload_type) {
return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps; return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps;
} }
private void apply_caps(PayloadType payload_type, Gst.Caps caps) { private void apply_caps(PayloadType payload_type, Gst.Caps caps) {
plugin.pause(); plugin.pause();
debug("Set scaled caps to %s", caps.to_string()); debug("Set scaled caps to %s", caps.to_string());
codec_util.update_rescale_caps(codecs[payload_type], caps); codec_util.update_rescale_caps(codecs[payload_type], caps);
plugin.unpause(); plugin.unpause();
} }
private void apply_width(PayloadType payload_type, int new_width, uint bitrate) { private void apply_width(PayloadType payload_type, int new_width, uint bitrate) {
int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den; int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den;
device_caps.get_structure(0).get_int("width", out device_caps_width); device_caps.get_structure(0).get_int("width", out device_caps_width);
@ -197,9 +192,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
int new_height = device_caps_height * new_width / device_caps_width; int new_height = device_caps_height * new_width / device_caps_width;
Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null); Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null);
double required_bitrate = get_target_bitrate(new_caps); double required_bitrate = get_target_bitrate(new_caps);
if (bitrate < required_bitrate) return; debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate);
if (bitrate < required_bitrate && new_width > active_caps_width) return;
apply_caps(payload_type, new_caps); apply_caps(payload_type, new_caps);
} }
public void update_bitrate(PayloadType payload_type, uint bitrate) { public void update_bitrate(PayloadType payload_type, uint bitrate) {
if (codecs.has_key(payload_type)) { if (codecs.has_key(payload_type)) {
lock(codec_bitrates); lock(codec_bitrates);
@ -233,12 +230,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) { if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) {
// Lower video resolution // Lower video resolution
int i = 1; int i = 1;
for(; i < common_widths.length && common_widths[i] < active_caps_width; i++); for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);if (common_widths[i] != active_caps_width) {
debug("Decrease resolution to ensure target bitrate (%u) is in reach (current resolution target bitrate is %f)", bitrate, current_target_bitrate);
}
apply_width(payload_type, common_widths[i-1], bitrate); apply_width(payload_type, common_widths[i-1], bitrate);
} else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) { } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) {
// Higher video resolution // Higher video resolution
int i = 0; int i = 0;
for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++); for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++);
if (common_widths[i] != active_caps_width) {
debug("Increase resolution to make use of available bandwidth of target bitrate (%u) (current resolution target bitrate is %f)", bitrate, current_target_bitrate);
}
if (common_widths[i] > device_caps_width) { if (common_widths[i] > device_caps_width) {
// We never scale up, so just stick with what the device gives // We never scale up, so just stick with what the device gives
apply_width(payload_type, device_caps_width, bitrate); apply_width(payload_type, device_caps_width, bitrate);

9
plugins/rtp/src/stream.vala
View file

@ -169,7 +169,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
Timeout.add(1000, () => remb_adjust()); Timeout.add(1000, () => remb_adjust());
} }
if (input_device != null && media == "video") { if (input_device != null && media == "video") {
input_device.update_bitrate(payload_type, 256); input_device.update_bitrate(payload_type, target_send_bitrate);
} }
} }
@ -214,16 +214,18 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
if (packets_received < last_packets_received) new_received = 0; if (packets_received < last_packets_received) new_received = 0;
uint64 new_octets = octets_received - last_octets_received; uint64 new_octets = octets_received - last_octets_received;
if (octets_received < last_octets_received) octets_received = 0; if (octets_received < last_octets_received) octets_received = 0;
if (new_received == 0) continue;
last_packets_lost = packets_lost; last_packets_lost = packets_lost;
last_packets_received = packets_received; last_packets_received = packets_received;
last_octets_received = octets_received; last_octets_received = octets_received;
if (new_received == 0) continue;
double loss_rate = (double)new_lost / (double)(new_lost + new_received); double loss_rate = (double)new_lost / (double)(new_lost + new_received);
uint new_target_receive_bitrate = 256; uint new_target_receive_bitrate;
if (new_lost <= 0 || loss_rate < 0.02) { if (new_lost <= 0 || loss_rate < 0.02) {
new_target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); new_target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate);
} else if (loss_rate > 0.1) { } else if (loss_rate > 0.1) {
new_target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); new_target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate);
} else {
new_target_receive_bitrate = target_receive_bitrate;
} }
if (last_remb_time == 0) { if (last_remb_time == 0) {
last_remb_time = get_monotonic_time(); last_remb_time = get_monotonic_time();
@ -670,6 +672,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
public void unpause() { public void unpause() {
if (!paused) return; if (!paused) return;
set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false); set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false);
input_device.update_bitrate(payload_type, target_send_bitrate);
} }
public uint get_participant_ssrc(Xmpp.Jid participant) { public uint get_participant_ssrc(Xmpp.Jid participant) {