anotherim/anotherim-desktop
2
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity
anotherim-desktop/plugins/rtp/src/device.vala

551 lines
25 KiB
Vala
Raw Normal View History

RTP: Encode with device
2021-11-09 21:06:47 +00:00
using Xmpp.Xep.JingleRtp;
using Gee;
Calls: Device picker
2022-02-08 20:57:48 +00:00
public enum Dino.Plugins.Rtp.DeviceProtocol {
OTHER,
PIPEWIRE,
V4L2,
PULSEAUDIO,
ALSA
}
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
private const int[] common_widths = {320, 360, 400, 480, 640, 960, 1280, 1920, 2560, 3840};
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
public Plugin plugin { get; private set; }
RTP: Encode with device
2021-11-09 21:06:47 +00:00
public CodecUtil codec_util { get { return plugin.codec_util; } }
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
public Gst.Device device { get; private set; }
Calls: Device picker
2022-02-08 20:57:48 +00:00
public string id { owned get { return device_name; }}
public string display_name { owned get { return device_display_name; }}
Calls: Fix device selector for multi-party calls, allow picking device before call started
2022-02-12 13:35:44 +00:00
public string? detail_name { owned get {
if (device.properties.has_field("alsa.card_name")) return device.properties.get_string("alsa.card_name");
if (device.properties.has_field("alsa.name")) return device.properties.get_string("alsa.name");
if (device.properties.has_field("alsa.id")) return device.properties.get_string("alsa.id");
if (device.properties.has_field("api.v4l2.cap.card")) return device.properties.get_string("api.v4l2.cap.card");
return null;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
}}
Calls: Fix device selector for multi-party calls, allow picking device before call started
2022-02-12 13:35:44 +00:00
public string? media { owned get {
RTP: Encode with device
2021-11-09 21:06:47 +00:00
if (device.has_classes("Audio")) {
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
return "audio";
RTP: Encode with device
2021-11-09 21:06:47 +00:00
} else if (device.has_classes("Video")) {
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
return "video";
} else {
return null;
}
}}
Calls: Fix device selector for multi-party calls, allow picking device before call started
2022-02-12 13:35:44 +00:00
public bool incoming { get { return is_sink; } }
public Gst.Pipeline pipe { get { return plugin.pipe; }}
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
public bool is_source { get { return device.has_classes("Source"); }}
public bool is_sink { get { return device.has_classes("Sink"); }}
Calls: Device picker
2022-02-08 20:57:48 +00:00
public bool is_monitor { get { return device.properties.get_string("device.class") == "monitor" || (protocol == DeviceProtocol.PIPEWIRE && device.has_classes("Stream")); } }
public bool is_default { get {
bool ret;
device.properties.get_boolean("is-default", out ret);
return ret;
}}
public DeviceProtocol protocol { get {
if (device.properties.has_name("pulse-proplist")) return DeviceProtocol.PULSEAUDIO;
if (device.properties.has_name("pipewire-proplist")) return DeviceProtocol.PIPEWIRE;
if (device.properties.has_name("v4l2deviceprovider")) return DeviceProtocol.V4L2;
return DeviceProtocol.OTHER;
}}
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
private string device_name;
private string device_display_name;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
private Gst.Caps device_caps;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
private Gst.Element element;
private Gst.Element tee;
private Gst.Element dsp;
RTP: Encode with device
2021-11-09 21:06:47 +00:00
private Gst.Base.Aggregator mixer;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
private Gst.Element filter;
RTP: Encode with device
2021-11-09 21:06:47 +00:00
private int links;
// Codecs
private Gee.Map<PayloadType, Gst.Element> codecs = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gst.Element> codec_tees = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
// Payloaders
RTP: Encode with device
2021-11-09 21:06:47 +00:00
private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloaders = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloader_tees = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gee.Map<uint, uint>> payloader_links = new HashMap<PayloadType, Gee.Map<uint, uint>>(PayloadType.hash_func, PayloadType.equals_func);
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
// Bitrate
RTP: Encode with device
2021-11-09 21:06:47 +00:00
private Gee.Map<PayloadType, Gee.List<CodecBitrate>> codec_bitrates = new HashMap<PayloadType, Gee.List<CodecBitrate>>(PayloadType.hash_func, PayloadType.equals_func);
private class CodecBitrate {
public uint bitrate;
public int64 timestamp;
public CodecBitrate(uint bitrate) {
this.bitrate = bitrate;
this.timestamp = get_monotonic_time();
}
}
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
public Device(Plugin plugin, Gst.Device device) {
this.plugin = plugin;
update(device);
}
public bool matches(Gst.Device device) {
if (this.device.name == device.name) return true;
return false;
}
public void update(Gst.Device device) {
this.device = device;
this.device_name = device.name;
this.device_display_name = device.display_name;
}
public Gst.Element? link_sink() {
RTP: Encode with device
2021-11-09 21:06:47 +00:00
if (!is_sink) return null;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
if (element == null) create();
links++;
RTP: Encode with device
2021-11-09 21:06:47 +00:00
if (mixer != null) {
Gst.Element rate = Gst.ElementFactory.make("audiorate", @"$(id)_rate_$(Random.next_int())");
pipe.add(rate);
rate.link(mixer);
return rate;
}
if (media == "audio") return filter;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
return element;
}
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = 0, int seqnum_offset = -1, uint32 timestamp_offset = 0) {
RTP: Encode with device
2021-11-09 21:06:47 +00:00
if (!is_source) return null;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
if (element == null) create();
links++;
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
if (payload_type != null && ssrc != 0 && tee != null) {
RTP: Encode with device
2021-11-09 21:06:47 +00:00
bool new_codec = false;
string? codec = CodecUtil.get_codec_from_payload(media, payload_type);
if (!codecs.has_key(payload_type)) {
codecs[payload_type] = codec_util.get_encode_bin_without_payloader(media, payload_type, @"$(id)_$(codec)_encoder");
pipe.add(codecs[payload_type]);
new_codec = true;
}
if (!codec_tees.has_key(payload_type)) {
codec_tees[payload_type] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_tee");
codec_tees[payload_type].@set("allow-not-linked", true);
pipe.add(codec_tees[payload_type]);
codecs[payload_type].link(codec_tees[payload_type]);
}
if (!payloaders.has_key(payload_type)) {
payloaders[payload_type] = new HashMap<uint, Gst.Element>();
}
if (!payloaders[payload_type].has_key(ssrc)) {
payloaders[payload_type][ssrc] = codec_util.get_payloader_bin(media, payload_type, @"$(id)_$(codec)_$(ssrc)");
var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay");
payload.ssrc = ssrc;
payload.seqnum_offset = seqnum_offset;
RTP: Correctly handle timestamp after re-enabling a stream
2021-12-18 20:43:12 +00:00
if (timestamp_offset != 0) {
payload.timestamp_offset = timestamp_offset;
}
RTP: Encode with device
2021-11-09 21:06:47 +00:00
pipe.add(payloaders[payload_type][ssrc]);
codec_tees[payload_type].link(payloaders[payload_type][ssrc]);
RTP: Correctly handle timestamp after re-enabling a stream
2021-12-18 20:43:12 +00:00
debug("Payload for %s with %s using ssrc %u, seqnum_offset %u, timestamp_offset %u", media, codec, ssrc, seqnum_offset, timestamp_offset);
RTP: Encode with device
2021-11-09 21:06:47 +00:00
}
if (!payloader_tees.has_key(payload_type)) {
payloader_tees[payload_type] = new HashMap<uint, Gst.Element>();
}
if (!payloader_tees[payload_type].has_key(ssrc)) {
payloader_tees[payload_type][ssrc] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_$(ssrc)_tee");
payloader_tees[payload_type][ssrc].@set("allow-not-linked", true);
pipe.add(payloader_tees[payload_type][ssrc]);
payloaders[payload_type][ssrc].link(payloader_tees[payload_type][ssrc]);
}
if (!payloader_links.has_key(payload_type)) {
payloader_links[payload_type] = new HashMap<uint, uint>();
}
if (!payloader_links[payload_type].has_key(ssrc)) {
payloader_links[payload_type][ssrc] = 1;
} else {
payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] + 1;
}
if (new_codec) {
tee.link(codecs[payload_type]);
}
return payloader_tees[payload_type][ssrc];
}
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
if (tee != null) return tee;
return element;
}
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
private static double get_target_bitrate(Gst.Caps caps) {
if (caps == null || caps.get_size() == 0) return uint.MAX;
unowned Gst.Structure? that = caps.get_structure(0);
int num = 0, den = 0, width = 0, height = 0;
if (!that.has_field("width") || !that.get_int("width", out width)) return uint.MAX;
if (!that.has_field("height") || !that.get_int("height", out height)) return uint.MAX;
if (!that.has_field("framerate")) return uint.MAX;
Value framerate = that.get_value("framerate");
if (framerate.type() != typeof(Gst.Fraction)) return uint.MAX;
num = Gst.Value.get_fraction_numerator(framerate);
den = Gst.Value.get_fraction_denominator(framerate);
double pxs = ((double)num/(double)den) * (double)width * (double)height;
double br = Math.sqrt(Math.sqrt(pxs)) * 100.0 - 3700.0;
if (br < 128.0) return 128.0;
return br;
}
private Gst.Caps get_active_caps(PayloadType payload_type) {
return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps;
}
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
private void apply_caps(PayloadType payload_type, Gst.Caps caps) {
plugin.pause();
debug("Set scaled caps to %s", caps.to_string());
codec_util.update_rescale_caps(codecs[payload_type], caps);
plugin.unpause();
}
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
private void apply_width(PayloadType payload_type, int new_width, uint bitrate) {
int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den;
device_caps.get_structure(0).get_int("width", out device_caps_width);
device_caps.get_structure(0).get_int("height", out device_caps_height);
device_caps.get_structure(0).get_fraction("framerate", out device_caps_framerate_num, out device_caps_framerate_den);
Gst.Caps active_caps = get_active_caps(payload_type);
if (active_caps != null && active_caps.get_size() > 0) {
active_caps.get_structure(0).get_int("width", out active_caps_width);
} else {
active_caps_width = device_caps_width;
}
if (new_width == active_caps_width) return;
int new_height = device_caps_height * new_width / device_caps_width;
Fix video for cameras with rotated image
2023-04-21 15:38:15 +00:00
Gst.Caps new_caps;
if (device_caps_framerate_den != 0) {
new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null);
} else {
new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, null);
}
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
double required_bitrate = get_target_bitrate(new_caps);
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate);
if (bitrate < required_bitrate && new_width > active_caps_width) return;
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
apply_caps(payload_type, new_caps);
}
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
RTP: Encode with device
2021-11-09 21:06:47 +00:00
public void update_bitrate(PayloadType payload_type, uint bitrate) {
if (codecs.has_key(payload_type)) {
lock(codec_bitrates);
if (!codec_bitrates.has_key(payload_type)) {
codec_bitrates[payload_type] = new ArrayList<CodecBitrate>();
}
codec_bitrates[payload_type].add(new CodecBitrate(bitrate));
var remove = new ArrayList<CodecBitrate>();
foreach (CodecBitrate rate in codec_bitrates[payload_type]) {
if (rate.timestamp < get_monotonic_time() - 5000000L) {
remove.add(rate);
continue;
}
if (rate.bitrate < bitrate) {
bitrate = rate.bitrate;
}
}
codec_bitrates[payload_type].remove_all(remove);
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
if (media == "video") {
if (bitrate < 128) bitrate = 128;
Gst.Caps active_caps = get_active_caps(payload_type);
double max_bitrate = get_target_bitrate(device_caps) * 2;
double current_target_bitrate = get_target_bitrate(active_caps);
int device_caps_width, active_caps_width;
device_caps.get_structure(0).get_int("width", out device_caps_width);
if (active_caps != null && active_caps.get_size() > 0) {
active_caps.get_structure(0).get_int("width", out active_caps_width);
} else {
active_caps_width = device_caps_width;
}
if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) {
// Lower video resolution
int i = 1;
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);if (common_widths[i] != active_caps_width) {
debug("Decrease resolution to ensure target bitrate (%u) is in reach (current resolution target bitrate is %f)", bitrate, current_target_bitrate);
}
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
apply_width(payload_type, common_widths[i-1], bitrate);
} else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) {
// Higher video resolution
int i = 0;
for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++);
RTP: Improve screen resolution update logic
2021-12-19 21:38:00 +00:00
if (common_widths[i] != active_caps_width) {
debug("Increase resolution to make use of available bandwidth of target bitrate (%u) (current resolution target bitrate is %f)", bitrate, current_target_bitrate);
}
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
if (common_widths[i] > device_caps_width) {
// We never scale up, so just stick with what the device gives
apply_width(payload_type, device_caps_width, bitrate);
} else if (common_widths[i] != active_caps_width) {
apply_width(payload_type, common_widths[i], bitrate);
}
}
if (bitrate > max_bitrate) bitrate = (uint) max_bitrate;
}
RTP: Encode with device
2021-11-09 21:06:47 +00:00
codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate);
unlock(codec_bitrates);
}
}
public void unlink(Gst.Element? link = null) {
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
if (links <= 0) {
critical("Link count below zero.");
return;
}
RTP: Encode with device
2021-11-09 21:06:47 +00:00
if (link != null && is_source && tee != null) {
PayloadType payload_type = payloader_tees.first_match((entry) => entry.value.any_match((entry) => entry.value == link)).key;
uint ssrc = payloader_tees[payload_type].first_match((entry) => entry.value == link).key;
payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] - 1;
if (payloader_links[payload_type][ssrc] == 0) {
plugin.pause();
codec_tees[payload_type].unlink(payloaders[payload_type][ssrc]);
payloaders[payload_type][ssrc].set_locked_state(true);
payloaders[payload_type][ssrc].set_state(Gst.State.NULL);
payloaders[payload_type][ssrc].unlink(payloader_tees[payload_type][ssrc]);
pipe.remove(payloaders[payload_type][ssrc]);
payloaders[payload_type].unset(ssrc);
payloader_tees[payload_type][ssrc].set_locked_state(true);
payloader_tees[payload_type][ssrc].set_state(Gst.State.NULL);
pipe.remove(payloader_tees[payload_type][ssrc]);
payloader_tees[payload_type].unset(ssrc);
payloader_links[payload_type].unset(ssrc);
plugin.unpause();
}
if (payloader_links[payload_type].size == 0) {
plugin.pause();
tee.unlink(codecs[payload_type]);
codecs[payload_type].set_locked_state(true);
codecs[payload_type].set_state(Gst.State.NULL);
codecs[payload_type].unlink(codec_tees[payload_type]);
pipe.remove(codecs[payload_type]);
codecs.unset(payload_type);
codec_tees[payload_type].set_locked_state(true);
codec_tees[payload_type].set_state(Gst.State.NULL);
pipe.remove(codec_tees[payload_type]);
codec_tees.unset(payload_type);
payloaders.unset(payload_type);
payloader_tees.unset(payload_type);
payloader_links.unset(payload_type);
plugin.unpause();
}
}
if (link != null && is_sink && mixer != null) {
plugin.pause();
link.set_locked_state(true);
Gst.Base.AggregatorPad mixer_sink_pad = (Gst.Base.AggregatorPad) link.get_static_pad("src").get_peer();
link.get_static_pad("src").unlink(mixer_sink_pad);
mixer_sink_pad.set_active(false);
link.set_state(Gst.State.NULL);
pipe.remove(link);
mixer.release_request_pad(mixer_sink_pad);
plugin.unpause();
}
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
links--;
if (links == 0) {
destroy();
}
}
private Gst.Caps get_best_caps() {
if (media == "audio") {
return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1");
} else if (media == "video" && device.caps.get_size() > 0) {
Fix video for cameras with rotated image
2023-04-21 15:38:15 +00:00
int best_index = -1;
Fix webcam framerate selection
2021-05-01 15:27:55 +00:00
Value? best_fraction = null;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
int best_fps = 0;
int best_width = 0;
int best_height = 0;
for (int i = 0; i < device.caps.get_size(); i++) {
unowned Gst.Structure? that = device.caps.get_structure(i);
Fix calculation of best camera framerate When the algorithm iterates over all the available v4l2 capabilities it tries to determine the best framerate for each one of YUYV (video/x-raw) modes presented by the hardware (best_fraction, line 357 in device.vala). Regardless of what's determined to be the "best" YUYV mode from within the conditional right after (line 385) the best fractional framerate will always point to the last iterated framerate, which might be an extremely low one, like 7 or 5 FPS. When the framerate is then stored in the Gst.Structure (line 394) the fractional framerate will always be that last value which might be different than the correct one as calculated by best_fps (line 386). This workaround solves this issue by only updating best_fraction when the conditional in line 385 is satisfied. from issue #1195
2022-05-17 12:02:12 +00:00
Value? best_fraction_now = null;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
if (!that.has_name("video/x-raw")) continue;
int num = 0, den = 0, width = 0, height = 0;
Fix webcam framerate selection
2021-05-01 15:27:55 +00:00
if (!that.has_field("framerate")) continue;
Value framerate = that.get_value("framerate");
if (framerate.type() == typeof(Gst.Fraction)) {
num = Gst.Value.get_fraction_numerator(framerate);
den = Gst.Value.get_fraction_denominator(framerate);
} else if (framerate.type() == typeof(Gst.ValueList)) {
for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) {
Value fraction = Gst.ValueList.get_value(framerate, j);
int in_num = Gst.Value.get_fraction_numerator(fraction);
int in_den = Gst.Value.get_fraction_denominator(fraction);
int fps = den > 0 ? (num/den) : 0;
int in_fps = in_den > 0 ? (in_num/in_den) : 0;
if (in_fps > fps) {
Fix calculation of best camera framerate When the algorithm iterates over all the available v4l2 capabilities it tries to determine the best framerate for each one of YUYV (video/x-raw) modes presented by the hardware (best_fraction, line 357 in device.vala). Regardless of what's determined to be the "best" YUYV mode from within the conditional right after (line 385) the best fractional framerate will always point to the last iterated framerate, which might be an extremely low one, like 7 or 5 FPS. When the framerate is then stored in the Gst.Structure (line 394) the fractional framerate will always be that last value which might be different than the correct one as calculated by best_fps (line 386). This workaround solves this issue by only updating best_fraction when the conditional in line 385 is satisfied. from issue #1195
2022-05-17 12:02:12 +00:00
best_fraction_now = fraction;
Fix webcam framerate selection
2021-05-01 15:27:55 +00:00
num = in_num;
den = in_den;
}
}
} else {
debug("Unknown type for framerate: %s", framerate.type_name());
}
if (den == 0) continue;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
if (!that.has_field("width") || !that.get_int("width", out width)) continue;
if (!that.has_field("height") || !that.get_int("height", out height)) continue;
int fps = num/den;
if (best_fps < fps || best_fps == fps && best_width < width || best_fps == fps && best_width == width && best_height < height) {
best_fps = fps;
best_width = width;
best_height = height;
best_index = i;
Fix calculation of best camera framerate When the algorithm iterates over all the available v4l2 capabilities it tries to determine the best framerate for each one of YUYV (video/x-raw) modes presented by the hardware (best_fraction, line 357 in device.vala). Regardless of what's determined to be the "best" YUYV mode from within the conditional right after (line 385) the best fractional framerate will always point to the last iterated framerate, which might be an extremely low one, like 7 or 5 FPS. When the framerate is then stored in the Gst.Structure (line 394) the fractional framerate will always be that last value which might be different than the correct one as calculated by best_fps (line 386). This workaround solves this issue by only updating best_fraction when the conditional in line 385 is satisfied. from issue #1195
2022-05-17 12:02:12 +00:00
best_fraction = best_fraction_now;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
}
}
Fix video for cameras with rotated image
2023-04-21 15:38:15 +00:00
if (best_index == -1) {
// No caps in first round, try without framerate
for (int i = 0; i < device.caps.get_size(); i++) {
unowned Gst.Structure? that = device.caps.get_structure(i);
if (!that.has_name("video/x-raw")) continue;
int width = 0, height = 0;
if (!that.has_field("width") || !that.get_int("width", out width)) continue;
if (!that.has_field("height") || !that.get_int("height", out height)) continue;
if (best_width < width || best_width == width && best_height < height) {
best_width = width;
best_height = height;
best_index = i;
}
}
}
Fix webcam framerate selection
2021-05-01 15:27:55 +00:00
Gst.Caps res = caps_copy_nth(device.caps, best_index);
unowned Gst.Structure? that = res.get_structure(0);
Fix video for cameras with rotated image
2023-04-21 15:38:15 +00:00
Value? framerate = that.get_value("framerate");
if (framerate != null && framerate.type() == typeof(Gst.ValueList) && best_fraction != null) {
Fix webcam framerate selection
2021-05-01 15:27:55 +00:00
that.set_value("framerate", best_fraction);
}
debug("Selected caps %s", res.to_string());
return res;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
} else if (device.caps.get_size() > 0) {
RTP: Backport gst_caps_copy_nth from GStreamer 1.16
2021-03-21 14:42:58 +00:00
return caps_copy_nth(device.caps, 0);
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
} else {
return new Gst.Caps.any();
}
}
RTP: Backport gst_caps_copy_nth from GStreamer 1.16
2021-03-21 14:42:58 +00:00
// Backport from gst_caps_copy_nth added in GStreamer 1.16
private static Gst.Caps caps_copy_nth(Gst.Caps source, uint index) {
Gst.Caps target = new Gst.Caps.empty();
target.flags = source.flags;
target.append_structure_full(source.get_structure(index).copy(), source.get_features(index).copy());
return target;
}
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
private void create() {
debug("Creating device %s", id);
plugin.pause();
element = device.create_element(id);
RTP: Encode with device
2021-11-09 21:06:47 +00:00
if (is_sink) {
element.@set("async", false);
element.@set("sync", false);
}
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
pipe.add(element);
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
device_caps = get_best_caps();
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
if (is_source) {
Video optimizations
2021-04-29 13:46:06 +00:00
element.@set("do-timestamp", true);
filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
filter.@set("caps", device_caps);
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
pipe.add(filter);
element.link(filter);
Correctly handle missing webrtc-audio-processing
2021-05-01 14:00:37 +00:00
#if WITH_VOICE_PROCESSOR
GStreamer compat
2021-04-11 10:31:03 +00:00
if (media == "audio" && plugin.echoprobe != null) {
Correctly handle missing webrtc-audio-processing
2021-05-01 14:00:37 +00:00
dsp = new VoiceProcessor(plugin.echoprobe as EchoProbe, element as Gst.Audio.StreamVolume);
Echo Cancellation
2021-05-01 13:19:05 +00:00
dsp.name = @"dsp_$id";
pipe.add(dsp);
filter.link(dsp);
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
}
Correctly handle missing webrtc-audio-processing
2021-05-01 14:00:37 +00:00
#endif
Video optimizations
2021-04-29 13:46:06 +00:00
tee = Gst.ElementFactory.make("tee", @"tee_$id");
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
tee.@set("allow-not-linked", true);
pipe.add(tee);
(dsp ?? filter).link(tee);
}
if (is_sink && media == "audio") {
RTP: Encode with device
2021-11-09 21:06:47 +00:00
mixer = (Gst.Base.Aggregator) Gst.ElementFactory.make("audiomixer", @"mixer_$id");
pipe.add(mixer);
if (plugin.echoprobe != null && !plugin.echoprobe.get_static_pad("src").is_linked()) {
mixer.link(plugin.echoprobe);
GStreamer compat
2021-04-11 10:31:03 +00:00
plugin.echoprobe.link(element);
} else {
RTP: Encode with device
2021-11-09 21:06:47 +00:00
filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
Add maximum bitrate and adjust video resolution based on bitrate
2021-11-15 21:49:44 +00:00
filter.@set("caps", device_caps);
RTP: Encode with device
2021-11-09 21:06:47 +00:00
pipe.add(filter);
mixer.link(filter);
GStreamer compat
2021-04-11 10:31:03 +00:00
filter.link(element);
}
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
}
plugin.unpause();
}
private void destroy() {
RTP: Encode with device
2021-11-09 21:06:47 +00:00
if (is_sink) {
if (mixer != null) {
int linked_sink_pads = 0;
mixer.foreach_sink_pad((_, pad) => {
if (pad.is_linked()) linked_sink_pads++;
return true;
});
if (linked_sink_pads > 0) {
warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads);
}
mixer.unlink(plugin.echoprobe ?? element);
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
}
Resample audio data for common 48k sample rate
2021-03-23 14:09:06 +00:00
if (filter != null) {
filter.set_locked_state(true);
filter.set_state(Gst.State.NULL);
RTP: Encode with device
2021-11-09 21:06:47 +00:00
filter.unlink(element);
Resample audio data for common 48k sample rate
2021-03-23 14:09:06 +00:00
pipe.remove(filter);
filter = null;
}
GStreamer compat
2021-04-11 10:31:03 +00:00
if (plugin.echoprobe != null) {
plugin.echoprobe.unlink(element);
}
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
}
element.set_locked_state(true);
element.set_state(Gst.State.NULL);
if (filter != null) element.unlink(filter);
else if (is_source) element.unlink(tee);
pipe.remove(element);
element = null;
RTP: Encode with device
2021-11-09 21:06:47 +00:00
if (mixer != null) {
mixer.set_locked_state(true);
mixer.set_state(Gst.State.NULL);
pipe.remove(mixer);
mixer = null;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
}
RTP: Encode with device
2021-11-09 21:06:47 +00:00
if (is_source) {
if (filter != null) {
filter.set_locked_state(true);
filter.set_state(Gst.State.NULL);
filter.unlink(dsp ?? tee);
pipe.remove(filter);
filter = null;
}
if (dsp != null) {
dsp.set_locked_state(true);
dsp.set_state(Gst.State.NULL);
dsp.unlink(tee);
pipe.remove(dsp);
dsp = null;
}
if (tee != null) {
int linked_src_pads = 0;
tee.foreach_src_pad((_, pad) => {
if (pad.is_linked()) linked_src_pads++;
return true;
});
if (linked_src_pads != 0) {
warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads);
}
tee.set_locked_state(true);
tee.set_state(Gst.State.NULL);
pipe.remove(tee);
tee = null;
Add RTP implementation as plugin
2021-03-21 11:41:38 +00:00
}
}
debug("Destroyed device %s", id);
}
RTP: Encode with device
2021-11-09 21:06:47 +00:00
}
Reference in a new issue Copy permalink