add SDP Offer / Answer support

This commit is contained in:
Daniel Gultsch 2023-11-13 12:36:20 +01:00
parent 38ca53fcac
commit 96dcc75ac3
No known key found for this signature in database
GPG key ID: F43D18AD2A0982C2
5 changed files with 128 additions and 34 deletions

View file

@ -69,4 +69,5 @@ public final class Namespace {
public static final String UNIFIED_PUSH = "http://gultsch.de/xmpp/drafts/unified-push"; public static final String UNIFIED_PUSH = "http://gultsch.de/xmpp/drafts/unified-push";
public static final String REPORTING = "urn:xmpp:reporting:1"; public static final String REPORTING = "urn:xmpp:reporting:1";
public static final String REPORTING_REASON_SPAM = "urn:xmpp:reporting:spam"; public static final String REPORTING_REASON_SPAM = "urn:xmpp:reporting:spam";
public static final String SDP_OFFER_ANSWER = "urn:ietf:rfc:3264";
} }

View file

@ -635,7 +635,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
} }
} }
private void resendCandidatesFromSdp(final SessionDescription answer) { private static ImmutableMultimap<String, IceUdpTransportInfo.Candidate> parseCandidates(final SessionDescription answer) {
final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>(); final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
for(final SessionDescription.Media media : answer.media) { for(final SessionDescription.Media media : answer.media) {
final String mid = Iterables.getFirst(media.attributes.get("mid"), null); final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
@ -649,8 +649,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
} }
} }
} }
final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build(); return candidateBuilder.build();
sendTransportInfo(candidates);
} }
private void receiveContentReject(final JinglePacket jinglePacket) { private void receiveContentReject(final JinglePacket jinglePacket) {
@ -1406,8 +1405,9 @@ public class JingleRtpConnection extends AbstractJingleConnection
+ ": ICE servers got discovered when session was already terminated. nothing to do."); + ": ICE servers got discovered when session was already terminated. nothing to do.");
return; return;
} }
final boolean includeCandidates = remoteHasSdpOfferAnswer();
try { try {
setupWebRTC(media, iceServers); setupWebRTC(media, iceServers, !includeCandidates);
} catch (final WebRTCWrapper.InitializationException e) { } catch (final WebRTCWrapper.InitializationException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC"); Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
webRTCWrapper.close(); webRTCWrapper.close();
@ -1421,8 +1421,8 @@ public class JingleRtpConnection extends AbstractJingleConnection
this.webRTCWrapper.setRemoteDescription(sdp).get(); this.webRTCWrapper.setRemoteDescription(sdp).get();
addIceCandidatesFromBlackLog(); addIceCandidatesFromBlackLog();
org.webrtc.SessionDescription webRTCSessionDescription = org.webrtc.SessionDescription webRTCSessionDescription =
this.webRTCWrapper.setLocalDescription().get(); this.webRTCWrapper.setLocalDescription(includeCandidates).get();
prepareSessionAccept(webRTCSessionDescription); prepareSessionAccept(webRTCSessionDescription, includeCandidates);
} catch (final Exception e) { } catch (final Exception e) {
failureToAcceptSession(e); failureToAcceptSession(e);
} }
@ -1459,10 +1459,16 @@ public class JingleRtpConnection extends AbstractJingleConnection
} }
private void prepareSessionAccept( private void prepareSessionAccept(
final org.webrtc.SessionDescription webRTCSessionDescription) { final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates) {
final SessionDescription sessionDescription = final SessionDescription sessionDescription =
SessionDescription.parse(webRTCSessionDescription.description); SessionDescription.parse(webRTCSessionDescription.description);
final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false); final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
if (includeCandidates) {
candidates = parseCandidates(sessionDescription);
} else {
candidates = ImmutableMultimap.of();
}
this.responderRtpContentMap = respondingRtpContentMap; this.responderRtpContentMap = respondingRtpContentMap;
storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip()); storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
final ListenableFuture<RtpContentMap> outgoingContentMapFuture = final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
@ -1472,8 +1478,18 @@ public class JingleRtpConnection extends AbstractJingleConnection
new FutureCallback<RtpContentMap>() { new FutureCallback<RtpContentMap>() {
@Override @Override
public void onSuccess(final RtpContentMap outgoingContentMap) { public void onSuccess(final RtpContentMap outgoingContentMap) {
sendSessionAccept(outgoingContentMap); if (includeCandidates) {
webRTCWrapper.setIsReadyToReceiveIceCandidates(true); Log.d(
Config.LOGTAG,
"including "
+ candidates.size()
+ " candidates in session accept");
sendSessionAccept(outgoingContentMap.withCandidates(candidates));
webRTCWrapper.resetPendingCandidates();
} else {
sendSessionAccept(outgoingContentMap);
webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
}
} }
@Override @Override
@ -1871,8 +1887,9 @@ public class JingleRtpConnection extends AbstractJingleConnection
+ ": ICE servers got discovered when session was already terminated. nothing to do."); + ": ICE servers got discovered when session was already terminated. nothing to do.");
return; return;
} }
final boolean includeCandidates = remoteHasSdpOfferAnswer();
try { try {
setupWebRTC(media, iceServers); setupWebRTC(media, iceServers, !includeCandidates);
} catch (final WebRTCWrapper.InitializationException e) { } catch (final WebRTCWrapper.InitializationException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC"); Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC");
webRTCWrapper.close(); webRTCWrapper.close();
@ -1881,8 +1898,8 @@ public class JingleRtpConnection extends AbstractJingleConnection
} }
try { try {
org.webrtc.SessionDescription webRTCSessionDescription = org.webrtc.SessionDescription webRTCSessionDescription =
this.webRTCWrapper.setLocalDescription().get(); this.webRTCWrapper.setLocalDescription(includeCandidates).get();
prepareSessionInitiate(webRTCSessionDescription, targetState); prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState);
} catch (final Exception e) { } catch (final Exception e) {
// TODO sending the error text is worthwhile as well. Especially for FailureToSet // TODO sending the error text is worthwhile as well. Especially for FailureToSet
// exceptions // exceptions
@ -1915,10 +1932,16 @@ public class JingleRtpConnection extends AbstractJingleConnection
} }
private void prepareSessionInitiate( private void prepareSessionInitiate(
final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) { final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates, final State targetState) {
final SessionDescription sessionDescription = final SessionDescription sessionDescription =
SessionDescription.parse(webRTCSessionDescription.description); SessionDescription.parse(webRTCSessionDescription.description);
final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true); final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates;
if (includeCandidates) {
candidates = parseCandidates(sessionDescription);
} else {
candidates = ImmutableMultimap.of();
}
this.initiatorRtpContentMap = rtpContentMap; this.initiatorRtpContentMap = rtpContentMap;
final ListenableFuture<RtpContentMap> outgoingContentMapFuture = final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
encryptSessionInitiate(rtpContentMap); encryptSessionInitiate(rtpContentMap);
@ -1927,8 +1950,18 @@ public class JingleRtpConnection extends AbstractJingleConnection
new FutureCallback<RtpContentMap>() { new FutureCallback<RtpContentMap>() {
@Override @Override
public void onSuccess(final RtpContentMap outgoingContentMap) { public void onSuccess(final RtpContentMap outgoingContentMap) {
sendSessionInitiate(outgoingContentMap, targetState); if (includeCandidates) {
webRTCWrapper.setIsReadyToReceiveIceCandidates(true); Log.d(
Config.LOGTAG,
"including "
+ candidates.size()
+ " candidates in session initiate");
sendSessionInitiate(outgoingContentMap.withCandidates(candidates), targetState);
webRTCWrapper.resetPendingCandidates();
} else {
sendSessionInitiate(outgoingContentMap, targetState);
webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
}
} }
@Override @Override
@ -2031,11 +2064,6 @@ public class JingleRtpConnection extends AbstractJingleConnection
send(jinglePacket); send(jinglePacket);
} }
private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
// TODO send all candidates in one transport-info
}
private void send(final JinglePacket jinglePacket) { private void send(final JinglePacket jinglePacket) {
jinglePacket.setTo(id.with); jinglePacket.setTo(id.with);
xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse); xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);
@ -2400,10 +2428,10 @@ public class JingleRtpConnection extends AbstractJingleConnection
finish(); finish();
} }
private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException { private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers, final boolean trickle) throws WebRTCWrapper.InitializationException {
this.jingleConnectionManager.ensureConnectionIsRegistered(this); this.jingleConnectionManager.ensureConnectionIsRegistered(this);
this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media)); this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
this.webRTCWrapper.initializePeerConnection(media, iceServers); this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle);
} }
private void acceptCallFromProposed() { private void acceptCallFromProposed() {
@ -2736,7 +2764,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
private SessionDescription setLocalSessionDescription() private SessionDescription setLocalSessionDescription()
throws ExecutionException, InterruptedException { throws ExecutionException, InterruptedException {
final org.webrtc.SessionDescription sessionDescription = final org.webrtc.SessionDescription sessionDescription =
this.webRTCWrapper.setLocalDescription().get(); this.webRTCWrapper.setLocalDescription(false).get();
return SessionDescription.parse(sessionDescription.description); return SessionDescription.parse(sessionDescription.description);
} }
@ -3024,6 +3052,14 @@ public class JingleRtpConnection extends AbstractJingleConnection
} }
private boolean remoteHasVideoFeature() { private boolean remoteHasVideoFeature() {
return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO);
}
private boolean remoteHasSdpOfferAnswer() {
return remoteHasFeature(Namespace.SDP_OFFER_ANSWER);
}
private boolean remoteHasFeature(final String feature) {
final Contact contact = id.getContact(); final Contact contact = id.getContact();
final Presence presence = final Presence presence =
contact.getPresences().get(Strings.nullToEmpty(id.with.getResource())); contact.getPresences().get(Strings.nullToEmpty(id.with.getResource()));
@ -3031,7 +3067,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
presence == null ? null : presence.getServiceDiscoveryResult(); presence == null ? null : presence.getServiceDiscoveryResult();
final List<String> features = final List<String> features =
serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures(); serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures();
return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO); return features != null && features.contains(feature);
} }
private interface OnIceServersDiscovered { private interface OnIceServersDiscovered {

View file

@ -8,6 +8,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Collections2; import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
@ -196,6 +197,24 @@ public class RtpContentMap {
dt.senders, null, dt.transport.cloneWrapper()))); dt.senders, null, dt.transport.cloneWrapper())));
} }
RtpContentMap withCandidates(
ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates) {
final ImmutableMap.Builder<String, DescriptionTransport> contentBuilder =
new ImmutableMap.Builder<>();
for (final Map.Entry<String, DescriptionTransport> entry : this.contents.entrySet()) {
final String name = entry.getKey();
final DescriptionTransport descriptionTransport = entry.getValue();
final var transport = descriptionTransport.transport;
contentBuilder.put(
name,
new DescriptionTransport(
descriptionTransport.senders,
descriptionTransport.description,
transport.withCandidates(candidates.get(name))));
}
return new RtpContentMap(group, contentBuilder.build());
}
public IceUdpTransportInfo.Credentials getDistinctCredentials() { public IceUdpTransportInfo.Credentials getDistinctCredentials() {
final Set<IceUdpTransportInfo.Credentials> allCredentials = getCredentials(); final Set<IceUdpTransportInfo.Credentials> allCredentials = getCredentials();
final IceUdpTransportInfo.Credentials credentials = final IceUdpTransportInfo.Credentials credentials =

View file

@ -94,6 +94,8 @@ public class WebRTCWrapper {
private TrackWrapper<AudioTrack> localAudioTrack = null; private TrackWrapper<AudioTrack> localAudioTrack = null;
private TrackWrapper<VideoTrack> localVideoTrack = null; private TrackWrapper<VideoTrack> localVideoTrack = null;
private VideoTrack remoteVideoTrack = null; private VideoTrack remoteVideoTrack = null;
private final SettableFuture<Void> iceGatheringComplete = SettableFuture.create();
private final PeerConnection.Observer peerConnectionObserver = private final PeerConnection.Observer peerConnectionObserver =
new PeerConnection.Observer() { new PeerConnection.Observer() {
@Override @Override
@ -128,8 +130,11 @@ public class WebRTCWrapper {
@Override @Override
public void onIceGatheringChange( public void onIceGatheringChange(
PeerConnection.IceGatheringState iceGatheringState) { final PeerConnection.IceGatheringState iceGatheringState) {
Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")"); Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
if (iceGatheringState == PeerConnection.IceGatheringState.COMPLETE) {
iceGatheringComplete.set(null);
}
} }
@Override @Override
@ -256,7 +261,9 @@ public class WebRTCWrapper {
} }
synchronized void initializePeerConnection( synchronized void initializePeerConnection(
final Set<Media> media, final List<PeerConnection.IceServer> iceServers) final Set<Media> media,
final List<PeerConnection.IceServer> iceServers,
final boolean trickle)
throws InitializationException { throws InitializationException {
Preconditions.checkState(this.eglBase != null); Preconditions.checkState(this.eglBase != null);
Preconditions.checkNotNull(media); Preconditions.checkNotNull(media);
@ -283,7 +290,7 @@ public class WebRTCWrapper {
.createAudioDeviceModule()) .createAudioDeviceModule())
.createPeerConnectionFactory(); .createPeerConnectionFactory();
final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers); final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers, trickle);
final PeerConnection peerConnection = final PeerConnection peerConnection =
requirePeerConnectionFactory() requirePeerConnectionFactory()
.createPeerConnection(rtcConfig, peerConnectionObserver); .createPeerConnection(rtcConfig, peerConnectionObserver);
@ -398,21 +405,27 @@ public class WebRTCWrapper {
} }
private static PeerConnection.RTCConfiguration buildConfiguration( private static PeerConnection.RTCConfiguration buildConfiguration(
final List<PeerConnection.IceServer> iceServers) { final List<PeerConnection.IceServer> iceServers, final boolean trickle) {
final PeerConnection.RTCConfiguration rtcConfig = final PeerConnection.RTCConfiguration rtcConfig =
new PeerConnection.RTCConfiguration(iceServers); new PeerConnection.RTCConfiguration(iceServers);
rtcConfig.tcpCandidatePolicy = rtcConfig.tcpCandidatePolicy =
PeerConnection.TcpCandidatePolicy.DISABLED; // XEP-0176 doesn't support tcp PeerConnection.TcpCandidatePolicy.DISABLED; // XEP-0176 doesn't support tcp
rtcConfig.continualGatheringPolicy = if (trickle) {
PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; rtcConfig.continualGatheringPolicy =
PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
} else {
rtcConfig.continualGatheringPolicy =
PeerConnection.ContinualGatheringPolicy.GATHER_ONCE;
}
rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN;
rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE; rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE;
rtcConfig.enableImplicitRollback = true; rtcConfig.enableImplicitRollback = true;
return rtcConfig; return rtcConfig;
} }
void reconfigurePeerConnection(final List<PeerConnection.IceServer> iceServers) { void reconfigurePeerConnection(
requirePeerConnection().setConfiguration(buildConfiguration(iceServers)); final List<PeerConnection.IceServer> iceServers, final boolean trickle) {
requirePeerConnection().setConfiguration(buildConfiguration(iceServers, trickle));
} }
void restartIceAsync() { void restartIceAsync() {
@ -443,6 +456,11 @@ public class WebRTCWrapper {
"setIsReadyToReceiveCandidates(" + ready + ") was=" + was + " is=" + is); "setIsReadyToReceiveCandidates(" + ready + ") was=" + was + " is=" + is);
} }
public void resetPendingCandidates() {
this.readyToReceivedIceCandidates.set(true);
this.iceCandidates.clear();
}
synchronized void close() { synchronized void close() {
final PeerConnection peerConnection = this.peerConnection; final PeerConnection peerConnection = this.peerConnection;
final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory; final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory;
@ -561,7 +579,7 @@ public class WebRTCWrapper {
throw new IllegalStateException("Local video track does not exist"); throw new IllegalStateException("Local video track does not exist");
} }
synchronized ListenableFuture<SessionDescription> setLocalDescription() { synchronized ListenableFuture<SessionDescription> setLocalDescription(final boolean waitForCandidates) {
this.setIsReadyToReceiveIceCandidates(false); this.setIsReadyToReceiveIceCandidates(false);
return Futures.transformAsync( return Futures.transformAsync(
getPeerConnectionFuture(), getPeerConnectionFuture(),
@ -575,7 +593,16 @@ public class WebRTCWrapper {
new SetSdpObserver() { new SetSdpObserver() {
@Override @Override
public void onSetSuccess() { public void onSetSuccess() {
future.setFuture(getLocalDescriptionFuture()); final var delay =
waitForCandidates
? iceGatheringComplete
: Futures.immediateVoidFuture();
final var delayedSessionDescription =
Futures.transformAsync(
delay,
v -> getLocalDescriptionFuture(),
MoreExecutors.directExecutor());
future.setFuture(delayedSessionDescription);
} }
@Override @Override

View file

@ -12,6 +12,7 @@ import com.google.common.base.Splitter;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Collections2; import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
@ -152,6 +153,16 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
return transportInfo; return transportInfo;
} }
public IceUdpTransportInfo withCandidates(ImmutableCollection<Candidate> candidates) {
final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo();
transportInfo.setAttributes(new Hashtable<>(getAttributes()));
transportInfo.setChildren(this.getChildren());
for(final Candidate candidate : candidates) {
transportInfo.addChild(candidate);
}
return transportInfo;
}
public static class Credentials { public static class Credentials {
public final String ufrag; public final String ufrag;
public final String password; public final String password;