From 96dcc75ac3297272e3929e638b8e86577431b943 Mon Sep 17 00:00:00 2001 From: Daniel Gultsch Date: Mon, 13 Nov 2023 12:36:20 +0100 Subject: [PATCH] add SDP Offer / Answer support --- .../eu/siacs/conversations/xml/Namespace.java | 1 + .../xmpp/jingle/JingleRtpConnection.java | 84 +++++++++++++------ .../xmpp/jingle/RtpContentMap.java | 19 +++++ .../xmpp/jingle/WebRTCWrapper.java | 47 ++++++++--- .../jingle/stanzas/IceUdpTransportInfo.java | 11 +++ 5 files changed, 128 insertions(+), 34 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/xml/Namespace.java b/src/main/java/eu/siacs/conversations/xml/Namespace.java index 667fa6457..e17508077 100644 --- a/src/main/java/eu/siacs/conversations/xml/Namespace.java +++ b/src/main/java/eu/siacs/conversations/xml/Namespace.java @@ -69,4 +69,5 @@ public final class Namespace { public static final String UNIFIED_PUSH = "http://gultsch.de/xmpp/drafts/unified-push"; public static final String REPORTING = "urn:xmpp:reporting:1"; public static final String REPORTING_REASON_SPAM = "urn:xmpp:reporting:spam"; + public static final String SDP_OFFER_ANSWER = "urn:ietf:rfc:3264"; } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index b8a1483b0..c3c2dd1b0 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -635,7 +635,7 @@ public class JingleRtpConnection extends AbstractJingleConnection } } - private void resendCandidatesFromSdp(final SessionDescription answer) { + private static ImmutableMultimap parseCandidates(final SessionDescription answer) { final ImmutableMultimap.Builder candidateBuilder = new ImmutableMultimap.Builder<>(); for(final SessionDescription.Media media : answer.media) { final String mid = Iterables.getFirst(media.attributes.get("mid"), null); @@ -649,8 +649,7 @@ public class JingleRtpConnection extends AbstractJingleConnection } } } - final ImmutableMultimap candidates = candidateBuilder.build(); - sendTransportInfo(candidates); + return candidateBuilder.build(); } private void receiveContentReject(final JinglePacket jinglePacket) { @@ -1406,8 +1405,9 @@ public class JingleRtpConnection extends AbstractJingleConnection + ": ICE servers got discovered when session was already terminated. nothing to do."); return; } + final boolean includeCandidates = remoteHasSdpOfferAnswer(); try { - setupWebRTC(media, iceServers); + setupWebRTC(media, iceServers, !includeCandidates); } catch (final WebRTCWrapper.InitializationException e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC"); webRTCWrapper.close(); @@ -1421,8 +1421,8 @@ public class JingleRtpConnection extends AbstractJingleConnection this.webRTCWrapper.setRemoteDescription(sdp).get(); addIceCandidatesFromBlackLog(); org.webrtc.SessionDescription webRTCSessionDescription = - this.webRTCWrapper.setLocalDescription().get(); - prepareSessionAccept(webRTCSessionDescription); + this.webRTCWrapper.setLocalDescription(includeCandidates).get(); + prepareSessionAccept(webRTCSessionDescription, includeCandidates); } catch (final Exception e) { failureToAcceptSession(e); } @@ -1459,10 +1459,16 @@ public class JingleRtpConnection extends AbstractJingleConnection } private void prepareSessionAccept( - final org.webrtc.SessionDescription webRTCSessionDescription) { + final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates) { final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description); final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false); + final ImmutableMultimap candidates; + if (includeCandidates) { + candidates = parseCandidates(sessionDescription); + } else { + candidates = ImmutableMultimap.of(); + } this.responderRtpContentMap = respondingRtpContentMap; storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip()); final ListenableFuture outgoingContentMapFuture = @@ -1472,8 +1478,18 @@ public class JingleRtpConnection extends AbstractJingleConnection new FutureCallback() { @Override public void onSuccess(final RtpContentMap outgoingContentMap) { - sendSessionAccept(outgoingContentMap); - webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + if (includeCandidates) { + Log.d( + Config.LOGTAG, + "including " + + candidates.size() + + " candidates in session accept"); + sendSessionAccept(outgoingContentMap.withCandidates(candidates)); + webRTCWrapper.resetPendingCandidates(); + } else { + sendSessionAccept(outgoingContentMap); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + } } @Override @@ -1871,8 +1887,9 @@ public class JingleRtpConnection extends AbstractJingleConnection + ": ICE servers got discovered when session was already terminated. nothing to do."); return; } + final boolean includeCandidates = remoteHasSdpOfferAnswer(); try { - setupWebRTC(media, iceServers); + setupWebRTC(media, iceServers, !includeCandidates); } catch (final WebRTCWrapper.InitializationException e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC"); webRTCWrapper.close(); @@ -1881,8 +1898,8 @@ public class JingleRtpConnection extends AbstractJingleConnection } try { org.webrtc.SessionDescription webRTCSessionDescription = - this.webRTCWrapper.setLocalDescription().get(); - prepareSessionInitiate(webRTCSessionDescription, targetState); + this.webRTCWrapper.setLocalDescription(includeCandidates).get(); + prepareSessionInitiate(webRTCSessionDescription, includeCandidates, targetState); } catch (final Exception e) { // TODO sending the error text is worthwhile as well. Especially for FailureToSet // exceptions @@ -1915,10 +1932,16 @@ public class JingleRtpConnection extends AbstractJingleConnection } private void prepareSessionInitiate( - final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) { + final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates, final State targetState) { final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description); final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true); + final ImmutableMultimap candidates; + if (includeCandidates) { + candidates = parseCandidates(sessionDescription); + } else { + candidates = ImmutableMultimap.of(); + } this.initiatorRtpContentMap = rtpContentMap; final ListenableFuture outgoingContentMapFuture = encryptSessionInitiate(rtpContentMap); @@ -1927,8 +1950,18 @@ public class JingleRtpConnection extends AbstractJingleConnection new FutureCallback() { @Override public void onSuccess(final RtpContentMap outgoingContentMap) { - sendSessionInitiate(outgoingContentMap, targetState); - webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + if (includeCandidates) { + Log.d( + Config.LOGTAG, + "including " + + candidates.size() + + " candidates in session initiate"); + sendSessionInitiate(outgoingContentMap.withCandidates(candidates), targetState); + webRTCWrapper.resetPendingCandidates(); + } else { + sendSessionInitiate(outgoingContentMap, targetState); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + } } @Override @@ -2031,11 +2064,6 @@ public class JingleRtpConnection extends AbstractJingleConnection send(jinglePacket); } - private void sendTransportInfo(final Multimap candidates) { - // TODO send all candidates in one transport-info - } - - private void send(final JinglePacket jinglePacket) { jinglePacket.setTo(id.with); xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse); @@ -2400,10 +2428,10 @@ public class JingleRtpConnection extends AbstractJingleConnection finish(); } - private void setupWebRTC(final Set media, final List iceServers) throws WebRTCWrapper.InitializationException { + private void setupWebRTC(final Set media, final List iceServers, final boolean trickle) throws WebRTCWrapper.InitializationException { this.jingleConnectionManager.ensureConnectionIsRegistered(this); this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media)); - this.webRTCWrapper.initializePeerConnection(media, iceServers); + this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle); } private void acceptCallFromProposed() { @@ -2736,7 +2764,7 @@ public class JingleRtpConnection extends AbstractJingleConnection private SessionDescription setLocalSessionDescription() throws ExecutionException, InterruptedException { final org.webrtc.SessionDescription sessionDescription = - this.webRTCWrapper.setLocalDescription().get(); + this.webRTCWrapper.setLocalDescription(false).get(); return SessionDescription.parse(sessionDescription.description); } @@ -3024,6 +3052,14 @@ public class JingleRtpConnection extends AbstractJingleConnection } private boolean remoteHasVideoFeature() { + return remoteHasFeature(Namespace.JINGLE_FEATURE_VIDEO); + } + + private boolean remoteHasSdpOfferAnswer() { + return remoteHasFeature(Namespace.SDP_OFFER_ANSWER); + } + + private boolean remoteHasFeature(final String feature) { final Contact contact = id.getContact(); final Presence presence = contact.getPresences().get(Strings.nullToEmpty(id.with.getResource())); @@ -3031,7 +3067,7 @@ public class JingleRtpConnection extends AbstractJingleConnection presence == null ? null : presence.getServiceDiscoveryResult(); final List features = serviceDiscoveryResult == null ? null : serviceDiscoveryResult.getFeatures(); - return features != null && features.contains(Namespace.JINGLE_FEATURE_VIDEO); + return features != null && features.contains(feature); } private interface OnIceServersDiscovered { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java index 5ffd74ee3..cfd4bef78 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java @@ -8,6 +8,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; @@ -196,6 +197,24 @@ public class RtpContentMap { dt.senders, null, dt.transport.cloneWrapper()))); } + RtpContentMap withCandidates( + ImmutableMultimap candidates) { + final ImmutableMap.Builder contentBuilder = + new ImmutableMap.Builder<>(); + for (final Map.Entry entry : this.contents.entrySet()) { + final String name = entry.getKey(); + final DescriptionTransport descriptionTransport = entry.getValue(); + final var transport = descriptionTransport.transport; + contentBuilder.put( + name, + new DescriptionTransport( + descriptionTransport.senders, + descriptionTransport.description, + transport.withCandidates(candidates.get(name)))); + } + return new RtpContentMap(group, contentBuilder.build()); + } + public IceUdpTransportInfo.Credentials getDistinctCredentials() { final Set allCredentials = getCredentials(); final IceUdpTransportInfo.Credentials credentials = diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java index 5b841b88a..e09d7d242 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -94,6 +94,8 @@ public class WebRTCWrapper { private TrackWrapper localAudioTrack = null; private TrackWrapper localVideoTrack = null; private VideoTrack remoteVideoTrack = null; + + private final SettableFuture iceGatheringComplete = SettableFuture.create(); private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() { @Override @@ -128,8 +130,11 @@ public class WebRTCWrapper { @Override public void onIceGatheringChange( - PeerConnection.IceGatheringState iceGatheringState) { + final PeerConnection.IceGatheringState iceGatheringState) { Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")"); + if (iceGatheringState == PeerConnection.IceGatheringState.COMPLETE) { + iceGatheringComplete.set(null); + } } @Override @@ -256,7 +261,9 @@ public class WebRTCWrapper { } synchronized void initializePeerConnection( - final Set media, final List iceServers) + final Set media, + final List iceServers, + final boolean trickle) throws InitializationException { Preconditions.checkState(this.eglBase != null); Preconditions.checkNotNull(media); @@ -283,7 +290,7 @@ public class WebRTCWrapper { .createAudioDeviceModule()) .createPeerConnectionFactory(); - final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers); + final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers, trickle); final PeerConnection peerConnection = requirePeerConnectionFactory() .createPeerConnection(rtcConfig, peerConnectionObserver); @@ -398,21 +405,27 @@ public class WebRTCWrapper { } private static PeerConnection.RTCConfiguration buildConfiguration( - final List iceServers) { + final List iceServers, final boolean trickle) { final PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers); rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; // XEP-0176 doesn't support tcp - rtcConfig.continualGatheringPolicy = - PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; + if (trickle) { + rtcConfig.continualGatheringPolicy = + PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; + } else { + rtcConfig.continualGatheringPolicy = + PeerConnection.ContinualGatheringPolicy.GATHER_ONCE; + } rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE; rtcConfig.enableImplicitRollback = true; return rtcConfig; } - void reconfigurePeerConnection(final List iceServers) { - requirePeerConnection().setConfiguration(buildConfiguration(iceServers)); + void reconfigurePeerConnection( + final List iceServers, final boolean trickle) { + requirePeerConnection().setConfiguration(buildConfiguration(iceServers, trickle)); } void restartIceAsync() { @@ -443,6 +456,11 @@ public class WebRTCWrapper { "setIsReadyToReceiveCandidates(" + ready + ") was=" + was + " is=" + is); } + public void resetPendingCandidates() { + this.readyToReceivedIceCandidates.set(true); + this.iceCandidates.clear(); + } + synchronized void close() { final PeerConnection peerConnection = this.peerConnection; final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory; @@ -561,7 +579,7 @@ public class WebRTCWrapper { throw new IllegalStateException("Local video track does not exist"); } - synchronized ListenableFuture setLocalDescription() { + synchronized ListenableFuture setLocalDescription(final boolean waitForCandidates) { this.setIsReadyToReceiveIceCandidates(false); return Futures.transformAsync( getPeerConnectionFuture(), @@ -575,7 +593,16 @@ public class WebRTCWrapper { new SetSdpObserver() { @Override public void onSetSuccess() { - future.setFuture(getLocalDescriptionFuture()); + final var delay = + waitForCandidates + ? iceGatheringComplete + : Futures.immediateVoidFuture(); + final var delayedSessionDescription = + Futures.transformAsync( + delay, + v -> getLocalDescriptionFuture(), + MoreExecutors.directExecutor()); + future.setFuture(delayedSessionDescription); } @Override diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java index 026adbd02..ccaba56a6 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java @@ -12,6 +12,7 @@ import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -152,6 +153,16 @@ public class IceUdpTransportInfo extends GenericTransportInfo { return transportInfo; } + public IceUdpTransportInfo withCandidates(ImmutableCollection candidates) { + final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo(); + transportInfo.setAttributes(new Hashtable<>(getAttributes())); + transportInfo.setChildren(this.getChildren()); + for(final Candidate candidate : candidates) { + transportInfo.addChild(candidate); + } + return transportInfo; + } + public static class Credentials { public final String ufrag; public final String password;