From 4e8ceadfbf9f38cc9a53b7e912c9a7eb926dc2df Mon Sep 17 00:00:00 2001 From: Daniel Gultsch Date: Mon, 28 Nov 2022 08:59:23 +0100 Subject: [PATCH] prepare JingleRtpConnection for content-adds --- .../services/AppRTCAudioManager.java | 23 +- .../xmpp/jingle/ContentAddition.java | 88 +++ .../xmpp/jingle/JingleRtpConnection.java | 536 +++++++++++++++++- .../conversations/xmpp/jingle/Media.java | 15 + .../xmpp/jingle/RtpContentMap.java | 69 ++- .../xmpp/jingle/RtpEndUserState.java | 1 + .../xmpp/jingle/SessionDescription.java | 2 +- .../xmpp/jingle/ToneManager.java | 31 +- .../xmpp/jingle/TrackWrapper.java | 53 +- .../xmpp/jingle/WebRTCWrapper.java | 77 ++- .../xmpp/jingle/stanzas/RtpDescription.java | 136 +++-- 11 files changed, 921 insertions(+), 110 deletions(-) create mode 100644 src/main/java/eu/siacs/conversations/xmpp/jingle/ContentAddition.java diff --git a/src/main/java/eu/siacs/conversations/services/AppRTCAudioManager.java b/src/main/java/eu/siacs/conversations/services/AppRTCAudioManager.java index e1fe854ff..3bed4eaba 100644 --- a/src/main/java/eu/siacs/conversations/services/AppRTCAudioManager.java +++ b/src/main/java/eu/siacs/conversations/services/AppRTCAudioManager.java @@ -33,6 +33,7 @@ import java.util.concurrent.CountDownLatch; import eu.siacs.conversations.Config; import eu.siacs.conversations.utils.AppRTCUtils; +import eu.siacs.conversations.xmpp.jingle.Media; /** * AppRTCAudioManager manages all audio related parts of the AppRTC demo. @@ -44,7 +45,7 @@ public class AppRTCAudioManager { private final Context apprtcContext; // Contains speakerphone setting: auto, true or false @Nullable - private final SpeakerPhonePreference speakerPhonePreference; + private SpeakerPhonePreference speakerPhonePreference; // Handles all tasks related to Bluetooth headset devices. private final AppRTCBluetoothManager bluetoothManager; @Nullable @@ -110,6 +111,16 @@ public class AppRTCAudioManager { AppRTCUtils.logDeviceInfo(Config.LOGTAG); } + public void switchSpeakerPhonePreference(final SpeakerPhonePreference speakerPhonePreference) { + this.speakerPhonePreference = speakerPhonePreference; + if (speakerPhonePreference == SpeakerPhonePreference.EARPIECE && hasEarpiece()) { + defaultAudioDevice = AudioDevice.EARPIECE; + } else { + defaultAudioDevice = AudioDevice.SPEAKER_PHONE; + } + updateAudioDeviceState(); + } + /** * Construction. */ @@ -587,7 +598,15 @@ public class AppRTCAudioManager { } public enum SpeakerPhonePreference { - AUTO, EARPIECE, SPEAKER + AUTO, EARPIECE, SPEAKER; + + public static SpeakerPhonePreference of(final Set media) { + if (media.contains(Media.VIDEO)) { + return SPEAKER; + } else { + return EARPIECE; + } + } } /** diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/ContentAddition.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/ContentAddition.java new file mode 100644 index 000000000..97bf802fd --- /dev/null +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/ContentAddition.java @@ -0,0 +1,88 @@ +package eu.siacs.conversations.xmpp.jingle; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +import eu.siacs.conversations.xmpp.jingle.stanzas.Content; + +public final class ContentAddition { + + public final Direction direction; + public final Set summary; + + private ContentAddition(Direction direction, Set summary) { + this.direction = direction; + this.summary = summary; + } + + public Set media() { + return ImmutableSet.copyOf(Collections2.transform(summary, s -> s.media)); + } + + public static ContentAddition of(final Direction direction, final RtpContentMap rtpContentMap) { + return new ContentAddition(direction, summary(rtpContentMap)); + } + + public static Set summary(final RtpContentMap rtpContentMap) { + return ImmutableSet.copyOf( + Collections2.transform( + rtpContentMap.contents.entrySet(), + e -> { + final RtpContentMap.DescriptionTransport dt = e.getValue(); + return new Summary(e.getKey(), dt.description.getMedia(), dt.senders); + })); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("direction", direction) + .add("summary", summary) + .toString(); + } + + public enum Direction { + OUTGOING, + INCOMING + } + + public static final class Summary { + public final String name; + public final Media media; + public final Content.Senders senders; + + private Summary(final String name, final Media media, final Content.Senders senders) { + this.name = name; + this.media = media; + this.senders = senders; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Summary summary = (Summary) o; + return Objects.equal(name, summary.name) + && media == summary.media + && senders == summary.senders; + } + + @Override + public int hashCode() { + return Objects.hashCode(name, media, senders); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("name", name) + .add("media", media) + .add("senders", senders) + .toString(); + } + } +} diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index ad5aeac0e..6e14fc56e 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -5,16 +5,16 @@ import android.util.Log; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.base.Preconditions; -import com.google.common.base.Predicates; import com.google.common.base.Stopwatch; import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.FutureCallback; @@ -39,6 +39,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import eu.siacs.conversations.BuildConfig; import eu.siacs.conversations.Config; import eu.siacs.conversations.crypto.axolotl.AxolotlService; import eu.siacs.conversations.crypto.axolotl.CryptoFailedException; @@ -53,6 +54,7 @@ import eu.siacs.conversations.utils.IP; import eu.siacs.conversations.xml.Element; import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xmpp.Jid; +import eu.siacs.conversations.xmpp.jingle.stanzas.Content; import eu.siacs.conversations.xmpp.jingle.stanzas.Group; import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; @@ -163,6 +165,8 @@ public class JingleRtpConnection extends AbstractJingleConnection private Set proposedMedia; private RtpContentMap initiatorRtpContentMap; private RtpContentMap responderRtpContentMap; + private RtpContentMap incomingContentAdd; + private RtpContentMap outgoingContentAdd; private IceUdpTransportInfo.Setup peerDtlsSetup; private final Stopwatch sessionDuration = Stopwatch.createUnstarted(); private final Queue stateHistory = new LinkedList<>(); @@ -218,6 +222,18 @@ public class JingleRtpConnection extends AbstractJingleConnection case SESSION_TERMINATE: receiveSessionTerminate(jinglePacket); break; + case CONTENT_ADD: + receiveContentAdd(jinglePacket); + break; + case CONTENT_ACCEPT: + receiveContentAccept(jinglePacket); + break; + case CONTENT_REJECT: + receiveContentReject(jinglePacket); + break; + case CONTENT_REMOVE: + receiveContentRemove(jinglePacket); + break; default: respondOk(jinglePacket); Log.d( @@ -346,6 +362,405 @@ public class JingleRtpConnection extends AbstractJingleConnection } } + private void receiveContentAdd(final JinglePacket jinglePacket) { + final RtpContentMap modification; + try { + modification = RtpContentMap.of(jinglePacket); + modification.requireContentDescriptions(); + } catch (final RuntimeException e) { + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + ": improperly formatted contents", + Throwables.getRootCause(e)); + respondOk(jinglePacket); + webRTCWrapper.close(); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + if (isInState(State.SESSION_ACCEPTED)) { + receiveContentAdd(jinglePacket, modification); + } else { + terminateWithOutOfOrder(jinglePacket); + } + } + + private void receiveContentAdd( + final JinglePacket jinglePacket, final RtpContentMap modification) { + final RtpContentMap remote = getRemoteContentMap(); + if (!Collections.disjoint(modification.getNames(), remote.getNames())) { + respondOk(jinglePacket); + this.webRTCWrapper.close(); + sendSessionTerminate( + Reason.FAILED_APPLICATION, + String.format( + "contents with names %s already exists", + Joiner.on(", ").join(modification.getNames()))); + return; + } + final ContentAddition contentAddition = + ContentAddition.of(ContentAddition.Direction.INCOMING, modification); + + final RtpContentMap outgoing = this.outgoingContentAdd; + final Set outgoingContentAddSummary = + outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing); + + if (outgoingContentAddSummary.equals(contentAddition.summary)) { + if (isInitiator()) { + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": respond with tie break to matching content-add offer"); + respondWithTieBreak(jinglePacket); + } else { + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": automatically accept matching content-add offer"); + acceptContentAdd(contentAddition.summary, modification); + } + return; + } + + // once we can display multiple video tracks we can be more loose with this condition + // theoretically it should also be fine to automatically accept audio only contents + if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) { + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + ": received " + contentAddition); + this.incomingContentAdd = modification; + respondOk(jinglePacket); + updateEndUserState(); + } else { + respondOk(jinglePacket); + // TODO do we want to add a reason? + rejectContentAdd(modification); + } + } + + private void receiveContentAccept(final JinglePacket jinglePacket) { + final RtpContentMap receivedContentAccept; + try { + receivedContentAccept = RtpContentMap.of(jinglePacket); + receivedContentAccept.requireContentDescriptions(); + } catch (final RuntimeException e) { + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + ": improperly formatted contents", + Throwables.getRootCause(e)); + respondOk(jinglePacket); + webRTCWrapper.close(); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + + final RtpContentMap outgoingContentAdd = this.outgoingContentAdd; + if (outgoingContentAdd == null) { + Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add"); + terminateWithOutOfOrder(jinglePacket); + return; + } + final Set ourSummary = ContentAddition.summary(outgoingContentAdd); + if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) { + this.outgoingContentAdd = null; + respondOk(jinglePacket); + receiveContentAccept(receivedContentAccept); + } else { + Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add"); + terminateWithOutOfOrder(jinglePacket); + } + } + + private void receiveContentAccept(final RtpContentMap receivedContentAccept) { + final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup(); + final RtpContentMap modifiedContentMap = + getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup); + + setRemoteContentMap(modifiedContentMap); + + final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator()); + + final org.webrtc.SessionDescription sdp = + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.ANSWER, answer.toString()); + + try { + this.webRTCWrapper.setRemoteDescription(sdp).get(); + } catch (final Exception e) { + final Throwable cause = Throwables.getRootCause(e); + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": unable to set remote description after receiving content-accept", + cause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); + return; + } + updateEndUserState(); + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": remote has accepted content-add " + + ContentAddition.summary(receivedContentAccept)); + } + + private void receiveContentReject(final JinglePacket jinglePacket) { + final RtpContentMap receivedContentReject; + try { + receivedContentReject = RtpContentMap.of(jinglePacket); + } catch (final RuntimeException e) { + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + ": improperly formatted contents", + Throwables.getRootCause(e)); + respondOk(jinglePacket); + this.webRTCWrapper.close(); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + + final RtpContentMap outgoingContentAdd = this.outgoingContentAdd; + if (outgoingContentAdd == null) { + Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add"); + terminateWithOutOfOrder(jinglePacket); + return; + } + final Set ourSummary = ContentAddition.summary(outgoingContentAdd); + if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) { + this.outgoingContentAdd = null; + respondOk(jinglePacket); + Log.d(Config.LOGTAG,jinglePacket.toString()); + receiveContentReject(ourSummary); + } else { + Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add"); + terminateWithOutOfOrder(jinglePacket); + } + } + + private void receiveContentReject(final Set summary) { + try { + this.webRTCWrapper.removeTrack(Media.VIDEO); + final RtpContentMap localContentMap = customRollback(); + modifyLocalContentMap(localContentMap); + } catch (final Exception e) { + final Throwable cause = Throwables.getRootCause(e); + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": unable to rollback local description after receiving content-reject", + cause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); + return; + } + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": remote has rejected our content-add " + + summary); + } + + private void receiveContentRemove(final JinglePacket jinglePacket) { + final RtpContentMap receivedContentRemove; + try { + receivedContentRemove = RtpContentMap.of(jinglePacket); + receivedContentRemove.requireContentDescriptions(); + } catch (final RuntimeException e) { + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + ": improperly formatted contents", + Throwables.getRootCause(e)); + respondOk(jinglePacket); + this.webRTCWrapper.close(); + sendSessionTerminate(Reason.of(e), e.getMessage()); + return; + } + respondOk(jinglePacket); + receiveContentRemove(receivedContentRemove); + } + + private void receiveContentRemove(final RtpContentMap receivedContentRemove) { + final RtpContentMap incomingContentAdd = this.incomingContentAdd; + final Set contentAddSummary = + incomingContentAdd == null + ? Collections.emptySet() + : ContentAddition.summary(incomingContentAdd); + final Set removeSummary = + ContentAddition.summary(receivedContentRemove); + if (contentAddSummary.equals(removeSummary)) { + this.incomingContentAdd = null; + updateEndUserState(); + } else { + webRTCWrapper.close(); + sendSessionTerminate( + Reason.FAILED_APPLICATION, + String.format( + "%s only supports %s as a means to retract a not yet accepted %s", + BuildConfig.APP_NAME, + JinglePacket.Action.CONTENT_REMOVE, + JinglePacket.Action.CONTENT_ACCEPT)); + } + } + + public synchronized void retractContentAdd() { + final RtpContentMap outgoingContentAdd = this.outgoingContentAdd; + if (outgoingContentAdd == null) { + throw new IllegalStateException("Not outgoing content add"); + } + try { + webRTCWrapper.removeTrack(Media.VIDEO); + final RtpContentMap localContentMap = customRollback(); + modifyLocalContentMap(localContentMap); + } catch (final Exception e) { + final Throwable cause = Throwables.getRootCause(e); + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": unable to rollback local description after trying to retract content-add", + cause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); + return; + } + this.outgoingContentAdd = null; + final JinglePacket retract = + outgoingContentAdd + .toStub() + .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId); + this.send(retract); + Log.d( + Config.LOGTAG, + id.getAccount().getJid() + + ": retract content-add " + + ContentAddition.summary(outgoingContentAdd)); + } + + private RtpContentMap customRollback() throws ExecutionException, InterruptedException { + final SessionDescription sdp = setLocalSessionDescription(); + final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator()); + final SessionDescription answer = generateFakeResponse(localRtpContentMap); + this.webRTCWrapper + .setRemoteDescription( + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.ANSWER, answer.toString())) + .get(); + return localRtpContentMap; + } + + private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) { + final RtpContentMap currentRemote = getRemoteContentMap(); + final RtpContentMap.Diff diff = currentRemote.diff(localContentMap); + if (diff.isEmpty()) { + throw new IllegalStateException( + "Unexpected rollback condition. No difference between local and remote"); + } + final RtpContentMap patch = localContentMap.toContentModification(diff.added); + if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) { + final RtpContentMap nextRemote = + currentRemote.addContent( + patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup()); + return SessionDescription.of(nextRemote, !isInitiator()); + } + throw new IllegalStateException( + "Unexpected rollback condition. Senders were not uniformly none"); + } + + public synchronized void acceptContentAdd(@NonNull final Set contentAddition) { + final RtpContentMap incomingContentAdd = this.incomingContentAdd; + if (incomingContentAdd == null) { + throw new IllegalStateException("No incoming content add"); + } + + if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) { + this.incomingContentAdd = null; + acceptContentAdd(contentAddition, incomingContentAdd); + } else { + throw new IllegalStateException("Accepted content add does not match pending content-add"); + } + } + + private void acceptContentAdd(@NonNull final Set contentAddition, final RtpContentMap incomingContentAdd) { + final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup(); + final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup); + this.setRemoteContentMap(modifiedContentMap); + + final SessionDescription offer; + try { + offer = SessionDescription.of(modifiedContentMap, !isInitiator()); + } catch (final IllegalArgumentException | NullPointerException e) { + Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); + return; + } + this.incomingContentAdd = null; + acceptContentAdd(contentAddition, offer); + } + + private void acceptContentAdd( + final Set contentAddition, final SessionDescription offer) { + final org.webrtc.SessionDescription sdp = + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.OFFER, offer.toString()); + try { + this.webRTCWrapper.setRemoteDescription(sdp).get(); + + // TODO add tracks for 'media' where contentAddition.senders matches + + // TODO if senders.sending(isInitiator()) + + this.webRTCWrapper.addTrack(Media.VIDEO); + + // TODO add additional transceivers for recv only cases + + final SessionDescription answer = setLocalSessionDescription(); + final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator()); + + final RtpContentMap contentAcceptMap = + rtpContentMap.toContentModification( + Collections2.transform(contentAddition, ca -> ca.name)); + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": sending content-accept " + + ContentAddition.summary(contentAcceptMap)); + modifyLocalContentMap(rtpContentMap); + sendContentAccept(contentAcceptMap); + } catch (final Exception e) { + Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e)); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION); + } + } + + private void sendContentAccept(final RtpContentMap contentAcceptMap) { + final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId); + send(jinglePacket); + } + + public synchronized void rejectContentAdd() { + final RtpContentMap incomingContentAdd = this.incomingContentAdd; + if (incomingContentAdd == null) { + throw new IllegalStateException("No incoming content add"); + } + this.incomingContentAdd = null; + updateEndUserState(); + rejectContentAdd(incomingContentAdd); + } + + private void rejectContentAdd(final RtpContentMap contentMap) { + final JinglePacket jinglePacket = + contentMap + .toStub() + .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId); + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": rejecting content " + + ContentAddition.summary(contentMap)); + send(jinglePacket); + } + private boolean checkForIceRestart( final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) { final RtpContentMap existing = getRemoteContentMap(); @@ -1534,6 +1949,10 @@ public class JingleRtpConnection extends AbstractJingleConnection return RtpEndUserState.CONNECTING; } case SESSION_ACCEPTED: + final ContentAddition ca = getPendingContentAddition(); + if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) { + return RtpEndUserState.INCOMING_CONTENT_ADD; + } return getPeerConnectionStateAsEndUserState(); case REJECTED: case REJECTED_RACED: @@ -1591,6 +2010,18 @@ public class JingleRtpConnection extends AbstractJingleConnection } } + public ContentAddition getPendingContentAddition() { + final RtpContentMap in = this.incomingContentAdd; + final RtpContentMap out = this.outgoingContentAdd; + if (out != null) { + return ContentAddition.of(ContentAddition.Direction.OUTGOING, out); + } else if (in != null) { + return ContentAddition.of(ContentAddition.Direction.INCOMING, in); + } else { + return null; + } + } + public Set getMedia() { final State current = getState(); if (current == State.NULL) { @@ -1604,14 +2035,16 @@ public class JingleRtpConnection extends AbstractJingleConnection return Preconditions.checkNotNull( this.proposedMedia, "RTP connection has not been initialized properly"); } + final RtpContentMap localContentMap = getLocalContentMap(); final RtpContentMap initiatorContentMap = initiatorRtpContentMap; - if (initiatorContentMap != null) { + if (localContentMap != null) { + return localContentMap.getMedia(); + } else if (initiatorContentMap != null) { return initiatorContentMap.getMedia(); } else if (isTerminated()) { - return Collections.emptySet(); // we might fail before we ever got a chance to set media + return Collections.emptySet(); //we might fail before we ever got a chance to set media } else { - return Preconditions.checkNotNull( - this.proposedMedia, "RTP connection has not been initialized properly"); + return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly"); } } @@ -1625,6 +2058,16 @@ public class JingleRtpConnection extends AbstractJingleConnection return status != null && status.isVerified(); } + public boolean addMedia(final Media media) { + final Set currentMedia = getMedia(); + if (currentMedia.contains(media)) { + throw new IllegalStateException(String.format("%s has already been proposed", media)); + } + // TODO add state protection - can only add while ACCEPTED or so + Log.d(Config.LOGTAG,"adding media: "+media); + return webRTCWrapper.addTrack(media); + } + public synchronized void acceptCall() { switch (this.state) { case PROPOSED: @@ -1743,17 +2186,9 @@ public class JingleRtpConnection extends AbstractJingleConnection finish(); } - private void setupWebRTC( - final Set media, final List iceServers) - throws WebRTCWrapper.InitializationException { + private void setupWebRTC(final Set media, final List iceServers) throws WebRTCWrapper.InitializationException { this.jingleConnectionManager.ensureConnectionIsRegistered(this); - final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference; - if (media.contains(Media.VIDEO)) { - speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER; - } else { - speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE; - } - this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference); + this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media)); this.webRTCWrapper.initializePeerConnection(media, iceServers); } @@ -1905,21 +2340,23 @@ public class JingleRtpConnection extends AbstractJingleConnection webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection); return; } else { - webRTCWrapper.restartIce(); + this.restartIce(); } } updateEndUserState(); } + private void restartIce() { + this.stateHistory.clear(); + this.webRTCWrapper.restartIce(); + } + @Override public void onRenegotiationNeeded() { this.webRTCWrapper.execute(this::renegotiate); } private void renegotiate() { - //TODO needs to be called only for ice restarts; maybe in the call to restartICe() - this.stateHistory.clear(); - this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false); final SessionDescription sessionDescription; try { sessionDescription = setLocalSessionDescription(); @@ -1945,19 +2382,26 @@ public class JingleRtpConnection extends AbstractJingleConnection if (diff.hasModifications() && iceRestart) { webRTCWrapper.close(); - sendSessionTerminate(Reason.FAILED_APPLICATION, "WebRTC unexpectedly tried to modify content and transport at once"); + sendSessionTerminate( + Reason.FAILED_APPLICATION, + "WebRTC unexpectedly tried to modify content and transport at once"); return; } if (iceRestart) { initiateIceRestart(rtpContentMap); return; + } else if (diff.isEmpty()) { + Log.d( + Config.LOGTAG, + "renegotiation. nothing to do. SignalingState=" + + this.webRTCWrapper.getSignalingState()); } if (diff.added.size() > 0) { - sendContentAdd(rtpContentMap); + modifyLocalContentMap(rtpContentMap); + sendContentAdd(rtpContentMap, diff.added); } - } private void initiateIceRestart(final RtpContentMap rtpContentMap) { @@ -1977,8 +2421,7 @@ public class JingleRtpConnection extends AbstractJingleConnection return; } if (response.getType() == IqPacket.TYPE.ERROR) { - final Element error = response.findChild("error"); - if (error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS)) { + if (isTieBreak(response)) { Log.d(Config.LOGTAG, "received tie-break as result of ice restart"); return; } @@ -1990,8 +2433,40 @@ public class JingleRtpConnection extends AbstractJingleConnection }); } - private void sendContentAdd(final RtpContentMap rtpContentMap) { + private boolean isTieBreak(final IqPacket response) { + final Element error = response.findChild("error"); + return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS); + } + private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection added) { + final RtpContentMap contentAdd = rtpContentMap.toContentModification(added); + this.outgoingContentAdd = contentAdd; + final JinglePacket jinglePacket = + contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId); + jinglePacket.setTo(id.with); + xmppConnectionService.sendIqPacket( + id.account, + jinglePacket, + (connection, response) -> { + if (response.getType() == IqPacket.TYPE.RESULT) { + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": received ACK to our content-add"); + return; + } + if (response.getType() == IqPacket.TYPE.ERROR) { + if (isTieBreak(response)) { + this.outgoingContentAdd = null; + Log.d(Config.LOGTAG, "received tie-break as result of our content-add"); + return; + } + handleIqErrorResponse(response); + } + if (response.getType() == IqPacket.TYPE.TIMEOUT) { + handleIqTimeoutResponse(response); + } + }); } private void setLocalContentMap(final RtpContentMap rtpContentMap) { @@ -2010,6 +2485,15 @@ public class JingleRtpConnection extends AbstractJingleConnection } } + // this method is to be used for content map modifications that modify media + private void modifyLocalContentMap(final RtpContentMap rtpContentMap) { + final RtpContentMap activeContents = rtpContentMap.activeContents(); + setLocalContentMap(activeContents); + this.webRTCWrapper.switchSpeakerPhonePreference( + AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia())); + updateEndUserState(); + } + private SessionDescription setLocalSessionDescription() throws ExecutionException, InterruptedException { final org.webrtc.SessionDescription sessionDescription = diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java index da25516ca..6a41c8906 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java @@ -1,11 +1,18 @@ package eu.siacs.conversations.xmpp.jingle; +import com.google.common.collect.ImmutableSet; + import java.util.Locale; +import java.util.Set; + +import javax.annotation.Nonnull; public enum Media { + VIDEO, AUDIO, UNKNOWN; @Override + @Nonnull public String toString() { return super.toString().toLowerCase(Locale.ROOT); } @@ -17,4 +24,12 @@ public enum Media { return UNKNOWN; } } + + public static boolean audioOnly(Set media) { + return ImmutableSet.of(AUDIO).equals(media); + } + + public static boolean videoOnly(Set media) { + return ImmutableSet.of(VIDEO).equals(media); + } } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java index 7af1469cf..994c3a233 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java @@ -14,6 +14,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -92,6 +93,10 @@ public class RtpContentMap { })); } + public Set getSenders() { + return ImmutableSet.copyOf(Collections2.transform(contents.values(),dt -> dt.senders)); + } + public List getNames() { return ImmutableList.copyOf(contents.keySet()); } @@ -281,6 +286,14 @@ public class RtpContentMap { return new RtpContentMap(this.group, contentMapBuilder.build()); } + public RtpContentMap modifiedSenders(final Content.Senders senders) { + return new RtpContentMap( + this.group, + Maps.transformValues( + contents, + dt -> new DescriptionTransport(senders, dt.description, dt.transport))); + } + public RtpContentMap toContentModification(final Collection modifications) { return new RtpContentMap( this.group, @@ -291,6 +304,22 @@ public class RtpContentMap { dt.senders, dt.description, IceUdpTransportInfo.STUB))); } + public RtpContentMap toStub() { + return new RtpContentMap( + null, + Maps.transformValues( + this.contents, + dt -> + new DescriptionTransport( + dt.senders, + RtpDescription.stub(dt.description.getMedia()), + IceUdpTransportInfo.STUB))); + } + + public RtpContentMap activeContents() { + return new RtpContentMap(group, Maps.filterValues(this.contents, dt -> dt.senders != Content.Senders.NONE)); + } + public Diff diff(final RtpContentMap rtpContentMap) { final Set existingContentIds = this.contents.keySet(); final Set newContentIds = rtpContentMap.contents.keySet(); @@ -307,24 +336,32 @@ public class RtpContentMap { } } - public RtpContentMap addContent(final RtpContentMap modification) { + public RtpContentMap addContent( + final RtpContentMap modification, final IceUdpTransportInfo.Setup setup) { final IceUdpTransportInfo.Credentials credentials = getDistinctCredentials(); final DTLS dtls = getDistinctDtls(); final IceUdpTransportInfo iceUdpTransportInfo = - IceUdpTransportInfo.of(credentials, dtls.setup, dtls.hash, dtls.fingerprint); - final Map combined = - new ImmutableMap.Builder() + IceUdpTransportInfo.of(credentials, setup, dtls.hash, dtls.fingerprint); + final Map combined = merge(contents, modification.contents); + /*new ImmutableMap.Builder() .putAll(contents) - .putAll( - Maps.transformValues( - modification.contents, - dt -> - new DescriptionTransport( - dt.senders, - dt.description, - iceUdpTransportInfo))) - .build(); - return new RtpContentMap(modification.group, combined); + .putAll(modification.contents) + .build();*/ + final Map combinedFixedTransport = + Maps.transformValues( + combined, + dt -> + new DescriptionTransport( + dt.senders, dt.description, iceUdpTransportInfo)); + return new RtpContentMap(modification.group, combinedFixedTransport); + } + + private static Map merge( + final Map a, final Map b) { + final Map combined = new HashMap<>(); + combined.putAll(a); + combined.putAll(b); + return ImmutableMap.copyOf(combined); } public static class DescriptionTransport { @@ -410,6 +447,10 @@ public class RtpContentMap { return !this.added.isEmpty() || !this.removed.isEmpty(); } + public boolean isEmpty() { + return this.added.isEmpty() && this.removed.isEmpty(); + } + @Override @Nonnull public String toString() { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java index 9a431bc01..24ed790dd 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java @@ -5,6 +5,7 @@ public enum RtpEndUserState { CONNECTING, //session-initiate or session-accepted but no webrtc peer connection yet CONNECTED, //session-accepted and webrtc peer connection is connected RECONNECTING, //session-accepted and webrtc peer connection was connected once but is currently disconnected or failed + INCOMING_CONTENT_ADD, //session-accepted with a pending, incoming content-add FINDING_DEVICE, //'propose' has been sent out; no 184 ack yet RINGING, //'propose' has been sent out and it has been 184 acked ACCEPTING_CALL, //'proceed' message has been sent; but no session-initiate has been received diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java index eef7ae0da..f0f98260b 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java @@ -298,7 +298,7 @@ public class SessionDescription { mediaAttributes.put("mid", name); mediaAttributes.put(descriptionTransport.senders.asMediaAttribute(isInitiatorContentMap), ""); - if (description.hasChild("rtcp-mux", Namespace.JINGLE_APPS_RTP)) { + if (description.hasChild("rtcp-mux", Namespace.JINGLE_APPS_RTP) || group != null) { mediaAttributes.put("rtcp-mux", ""); } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/ToneManager.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/ToneManager.java index e368d3b09..02c1f6fe1 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/ToneManager.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/ToneManager.java @@ -5,6 +5,7 @@ import android.media.AudioManager; import android.media.ToneGenerator; import android.util.Log; +import java.util.Arrays; import java.util.Set; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -19,6 +20,7 @@ class ToneManager { private final Context context; private ToneState state = null; + private RtpEndUserState endUserState = null; private ScheduledFuture currentTone; private ScheduledFuture currentResetFuture; private boolean appRtcAudioManagerHasControl = false; @@ -51,7 +53,11 @@ class ToneManager { return ToneState.ENDING_CALL; } } - if (state == RtpEndUserState.CONNECTED || state == RtpEndUserState.RECONNECTING) { + if (Arrays.asList( + RtpEndUserState.CONNECTED, + RtpEndUserState.RECONNECTING, + RtpEndUserState.INCOMING_CONTENT_ADD) + .contains(state)) { if (media.contains(Media.VIDEO)) { return ToneState.NULL; } else { @@ -62,14 +68,19 @@ class ToneManager { } void transition(final RtpEndUserState state, final Set media) { - transition(of(true, state, media), media); + transition(state, of(true, state, media), media); } void transition(final boolean isInitiator, final RtpEndUserState state, final Set media) { - transition(of(isInitiator, state, media), media); + transition(state, of(isInitiator, state, media), media); } - private synchronized void transition(ToneState state, final Set media) { + private synchronized void transition(final RtpEndUserState endUserState, final ToneState state, final Set media) { + final RtpEndUserState normalizeEndUserState = normalize(endUserState); + if (this.endUserState == normalizeEndUserState) { + return; + } + this.endUserState = normalizeEndUserState; if (this.state == state) { return; } @@ -105,6 +116,18 @@ class ToneManager { this.state = state; } + private static RtpEndUserState normalize(final RtpEndUserState endUserState) { + if (Arrays.asList( + RtpEndUserState.CONNECTED, + RtpEndUserState.RECONNECTING, + RtpEndUserState.INCOMING_CONTENT_ADD) + .contains(endUserState)) { + return RtpEndUserState.CONNECTED; + } else { + return endUserState; + } + } + void setAppRtcAudioManagerHasControl(final boolean appRtcAudioManagerHasControl) { this.appRtcAudioManagerHasControl = appRtcAudioManagerHasControl; } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/TrackWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/TrackWrapper.java index 4e2952127..31c3577ee 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/TrackWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/TrackWrapper.java @@ -1,15 +1,26 @@ package eu.siacs.conversations.xmpp.jingle; +import android.util.Log; + +import com.google.common.base.CaseFormat; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import org.webrtc.MediaStreamTrack; import org.webrtc.PeerConnection; import org.webrtc.RtpSender; +import org.webrtc.RtpTransceiver; + +import java.util.UUID; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import eu.siacs.conversations.Config; class TrackWrapper { - private final T track; - private final RtpSender rtpSender; + public final T track; + public final RtpSender rtpSender; private TrackWrapper(final T track, final RtpSender rtpSender) { Preconditions.checkNotNull(track); @@ -25,7 +36,41 @@ class TrackWrapper { } public static Optional get( - final TrackWrapper trackWrapper) { - return trackWrapper == null ? Optional.absent() : Optional.of(trackWrapper.track); + @Nullable final PeerConnection peerConnection, final TrackWrapper trackWrapper) { + if (trackWrapper == null) { + return Optional.absent(); + } + final RtpTransceiver transceiver = + peerConnection == null ? null : getTransceiver(peerConnection, trackWrapper); + if (transceiver == null) { + Log.w(Config.LOGTAG, "unable to detect transceiver for " + trackWrapper.rtpSender.id()); + return Optional.of(trackWrapper.track); + } + final RtpTransceiver.RtpTransceiverDirection direction = transceiver.getDirection(); + if (direction == RtpTransceiver.RtpTransceiverDirection.SEND_ONLY + || direction == RtpTransceiver.RtpTransceiverDirection.SEND_RECV) { + return Optional.of(trackWrapper.track); + } else { + Log.d(Config.LOGTAG, "withholding track because transceiver is " + direction); + return Optional.absent(); + } + } + + public static RtpTransceiver getTransceiver( + @Nonnull final PeerConnection peerConnection, final TrackWrapper trackWrapper) { + final RtpSender rtpSender = trackWrapper.rtpSender; + for (final RtpTransceiver transceiver : peerConnection.getTransceivers()) { + if (transceiver.getSender().id().equals(rtpSender.id())) { + return transceiver; + } + } + return null; + } + + public static String id(final Class clazz) { + return String.format( + "%s-%s", + CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_HYPHEN, clazz.getSimpleName()), + UUID.randomUUID().toString()); } } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java index 53b7de1e0..b5ccf5c41 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -225,7 +225,7 @@ public class WebRTCWrapper { public void setup( final XmppConnectionService service, - final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) + @Nonnull final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) throws InitializationException { try { PeerConnectionFactory.initialize( @@ -330,18 +330,35 @@ public class WebRTCWrapper { throw new IllegalStateException(String.format("Could not add track for %s", media)); } + public synchronized void removeTrack(final Media media) { + if (media == Media.VIDEO) { + removeVideoTrack(requirePeerConnection()); + } + } + private boolean addAudioTrack(final PeerConnection peerConnection) { final AudioSource audioSource = requirePeerConnectionFactory().createAudioSource(new MediaConstraints()); final AudioTrack audioTrack = - requirePeerConnectionFactory().createAudioTrack("my-audio-track", audioSource); + requirePeerConnectionFactory() + .createAudioTrack(TrackWrapper.id(AudioTrack.class), audioSource); this.localAudioTrack = TrackWrapper.addTrack(peerConnection, audioTrack); return true; } private boolean addVideoTrack(final PeerConnection peerConnection) { - Preconditions.checkState( - this.localVideoTrack == null, "A local video track already exists"); + final TrackWrapper existing = this.localVideoTrack; + if (existing != null) { + final RtpTransceiver transceiver = + TrackWrapper.getTransceiver(peerConnection, existing); + if (transceiver == null) { + Log.w(EXTENDED_LOGGING_TAG, "unable to restart video transceiver"); + return false; + } + transceiver.setDirection(RtpTransceiver.RtpTransceiverDirection.SEND_RECV); + this.videoSourceWrapper.startCapture(); + return true; + } final VideoSourceWrapper videoSourceWrapper; try { videoSourceWrapper = initializeVideoSourceWrapper(); @@ -351,11 +368,34 @@ public class WebRTCWrapper { } final VideoTrack videoTrack = requirePeerConnectionFactory() - .createVideoTrack("my-video-track", videoSourceWrapper.getVideoSource()); + .createVideoTrack( + TrackWrapper.id(VideoTrack.class), + videoSourceWrapper.getVideoSource()); this.localVideoTrack = TrackWrapper.addTrack(peerConnection, videoTrack); return true; } + private void removeVideoTrack(final PeerConnection peerConnection) { + final TrackWrapper localVideoTrack = this.localVideoTrack; + if (localVideoTrack != null) { + + final RtpTransceiver exactTransceiver = + TrackWrapper.getTransceiver(peerConnection, localVideoTrack); + if (exactTransceiver == null) { + throw new IllegalStateException(); + } + exactTransceiver.setDirection(RtpTransceiver.RtpTransceiverDirection.INACTIVE); + } + final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper; + if (videoSourceWrapper != null) { + try { + videoSourceWrapper.stopCapture(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + private static PeerConnection.RTCConfiguration buildConfiguration( final List iceServers) { final PeerConnection.RTCConfiguration rtcConfig = @@ -375,7 +415,12 @@ public class WebRTCWrapper { } void restartIce() { - executorService.execute(() -> requirePeerConnection().restartIce()); + executorService.execute(() -> { + final PeerConnection peerConnection = requirePeerConnection(); + setIsReadyToReceiveIceCandidates(false); + peerConnection.restartIce(); + requirePeerConnection().restartIce();} + ); } public void setIsReadyToReceiveIceCandidates(final boolean ready) { @@ -450,7 +495,8 @@ public class WebRTCWrapper { } boolean isMicrophoneEnabled() { - final Optional audioTrack = TrackWrapper.get(this.localAudioTrack); + final Optional audioTrack = + TrackWrapper.get(peerConnection, this.localAudioTrack); if (audioTrack.isPresent()) { try { return audioTrack.get().enabled(); @@ -465,7 +511,8 @@ public class WebRTCWrapper { } boolean setMicrophoneEnabled(final boolean enabled) { - final Optional audioTrack = TrackWrapper.get(this.localAudioTrack); + final Optional audioTrack = + TrackWrapper.get(peerConnection, this.localAudioTrack); if (audioTrack.isPresent()) { try { audioTrack.get().setEnabled(enabled); @@ -481,7 +528,8 @@ public class WebRTCWrapper { } boolean isVideoEnabled() { - final Optional videoTrack = TrackWrapper.get(this.localVideoTrack); + final Optional videoTrack = + TrackWrapper.get(peerConnection, this.localVideoTrack); if (videoTrack.isPresent()) { return videoTrack.get().enabled(); } @@ -489,7 +537,8 @@ public class WebRTCWrapper { } void setVideoEnabled(final boolean enabled) { - final Optional videoTrack = TrackWrapper.get(this.localVideoTrack); + final Optional videoTrack = + TrackWrapper.get(peerConnection, this.localVideoTrack); if (videoTrack.isPresent()) { videoTrack.get().setEnabled(enabled); return; @@ -528,7 +577,7 @@ public class WebRTCWrapper { MoreExecutors.directExecutor()); } - private static void logDescription(final SessionDescription sessionDescription) { + public static void logDescription(final SessionDescription sessionDescription) { for (final String line : sessionDescription.description.split( eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) { @@ -612,7 +661,7 @@ public class WebRTCWrapper { } Optional getLocalVideoTrack() { - return TrackWrapper.get(this.localVideoTrack); + return TrackWrapper.get(peerConnection, this.localVideoTrack); } Optional getRemoteVideoTrack() { @@ -635,6 +684,10 @@ public class WebRTCWrapper { executorService.execute(command); } + public void switchSpeakerPhonePreference(AppRTCAudioManager.SpeakerPhonePreference preference) { + mainHandler.post(() -> appRTCAudioManager.switchSpeakerPhonePreference(preference)); + } + public interface EventCallback { void onIceCandidate(IceCandidate iceCandidate); diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java index 650c26bef..a7b62363c 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java @@ -22,7 +22,6 @@ import eu.siacs.conversations.xmpp.jingle.SessionDescription; public class RtpDescription extends GenericDescription { - private RtpDescription(final String media) { super("description", Namespace.JINGLE_APPS_RTP); this.setAttribute("media", media); @@ -32,6 +31,10 @@ public class RtpDescription extends GenericDescription { super("description", Namespace.JINGLE_APPS_RTP); } + public static RtpDescription stub(final Media media) { + return new RtpDescription(media.toString()); + } + public Media getMedia() { return Media.of(this.getAttribute("media")); } @@ -57,7 +60,8 @@ public class RtpDescription extends GenericDescription { public List getHeaderExtensions() { final ImmutableList.Builder builder = new ImmutableList.Builder<>(); for (final Element child : getChildren()) { - if ("rtp-hdrext".equals(child.getName()) && Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(child.getNamespace())) { + if ("rtp-hdrext".equals(child.getName()) + && Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(child.getNamespace())) { builder.add(RtpHeaderExtension.upgrade(child)); } } @@ -67,7 +71,9 @@ public class RtpDescription extends GenericDescription { public List getSources() { final ImmutableList.Builder builder = new ImmutableList.Builder<>(); for (final Element child : this.children) { - if ("source".equals(child.getName()) && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(child.getNamespace())) { + if ("source".equals(child.getName()) + && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals( + child.getNamespace())) { builder.add(Source.upgrade(child)); } } @@ -77,7 +83,9 @@ public class RtpDescription extends GenericDescription { public List getSourceGroups() { final ImmutableList.Builder builder = new ImmutableList.Builder<>(); for (final Element child : this.children) { - if ("ssrc-group".equals(child.getName()) && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(child.getNamespace())) { + if ("ssrc-group".equals(child.getName()) + && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals( + child.getNamespace())) { builder.add(SourceGroup.upgrade(child)); } } @@ -85,8 +93,12 @@ public class RtpDescription extends GenericDescription { } public static RtpDescription upgrade(final Element element) { - Preconditions.checkArgument("description".equals(element.getName()), "Name of provided element is not description"); - Preconditions.checkArgument(Namespace.JINGLE_APPS_RTP.equals(element.getNamespace()), "Element does not match the jingle rtp namespace"); + Preconditions.checkArgument( + "description".equals(element.getName()), + "Name of provided element is not description"); + Preconditions.checkArgument( + Namespace.JINGLE_APPS_RTP.equals(element.getNamespace()), + "Element does not match the jingle rtp namespace"); final RtpDescription description = new RtpDescription(); description.setAttributes(element.getAttributes()); description.setChildren(element.getChildren()); @@ -116,7 +128,8 @@ public class RtpDescription extends GenericDescription { private static FeedbackNegotiation upgrade(final Element element) { Preconditions.checkArgument("rtcp-fb".equals(element.getName())); - Preconditions.checkArgument(Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace())); final FeedbackNegotiation feedback = new FeedbackNegotiation(); feedback.setAttributes(element.getAttributes()); feedback.setChildren(element.getChildren()); @@ -126,13 +139,13 @@ public class RtpDescription extends GenericDescription { public static List fromChildren(final List children) { ImmutableList.Builder builder = new ImmutableList.Builder<>(); for (final Element child : children) { - if ("rtcp-fb".equals(child.getName()) && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) { + if ("rtcp-fb".equals(child.getName()) + && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) { builder.add(upgrade(child)); } } return builder.build(); } - } public static class FeedbackNegotiationTrrInt extends Element { @@ -142,7 +155,6 @@ public class RtpDescription extends GenericDescription { this.setAttribute("value", value); } - private FeedbackNegotiationTrrInt() { super("rtcp-fb-trr-int", Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION); } @@ -150,12 +162,12 @@ public class RtpDescription extends GenericDescription { public int getValue() { final String value = getAttribute("value"); return Integer.parseInt(value); - } private static FeedbackNegotiationTrrInt upgrade(final Element element) { Preconditions.checkArgument("rtcp-fb-trr-int".equals(element.getName())); - Preconditions.checkArgument(Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace())); final FeedbackNegotiationTrrInt trr = new FeedbackNegotiationTrrInt(); trr.setAttributes(element.getAttributes()); trr.setChildren(element.getChildren()); @@ -163,9 +175,11 @@ public class RtpDescription extends GenericDescription { } public static List fromChildren(final List children) { - ImmutableList.Builder builder = new ImmutableList.Builder<>(); + ImmutableList.Builder builder = + new ImmutableList.Builder<>(); for (final Element child : children) { - if ("rtcp-fb-trr-int".equals(child.getName()) && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) { + if ("rtcp-fb-trr-int".equals(child.getName()) + && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) { builder.add(upgrade(child)); } } @@ -173,9 +187,8 @@ public class RtpDescription extends GenericDescription { } } - - //XEP-0294: Jingle RTP Header Extensions Negotiation - //maps to `extmap:$id $uri` + // XEP-0294: Jingle RTP Header Extensions Negotiation + // maps to `extmap:$id $uri` public static class RtpHeaderExtension extends Element { private RtpHeaderExtension() { @@ -198,7 +211,8 @@ public class RtpDescription extends GenericDescription { public static RtpHeaderExtension upgrade(final Element element) { Preconditions.checkArgument("rtp-hdrext".equals(element.getName())); - Preconditions.checkArgument(Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(element.getNamespace())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(element.getNamespace())); final RtpHeaderExtension extension = new RtpHeaderExtension(); extension.setAttributes(element.getAttributes()); extension.setChildren(element.getChildren()); @@ -217,7 +231,7 @@ public class RtpDescription extends GenericDescription { } } - //maps to `rtpmap:$id $name/$clockrate/$channels` + // maps to `rtpmap:$id $name/$clockrate/$channels` public static class PayloadType extends Element { private PayloadType() { @@ -238,8 +252,14 @@ public class RtpDescription extends GenericDescription { final int channels = getChannels(); final String name = getPayloadTypeName(); Preconditions.checkArgument(name != null, "Payload-type name must not be empty"); - SessionDescription.checkNoWhitespace(name, "payload-type name must not contain whitespaces"); - return getId() + " " + name + "/" + getClockRate() + (channels == 1 ? "" : "/" + channels); + SessionDescription.checkNoWhitespace( + name, "payload-type name must not contain whitespaces"); + return getId() + + " " + + name + + "/" + + getClockRate() + + (channels == 1 ? "" : "/" + channels); } public int getIntId() { @@ -251,7 +271,6 @@ public class RtpDescription extends GenericDescription { return this.getAttribute("id"); } - public String getPayloadTypeName() { return this.getAttribute("name"); } @@ -271,7 +290,8 @@ public class RtpDescription extends GenericDescription { public int getChannels() { final String channels = this.getAttribute("channels"); if (channels == null) { - return 1; // The number of channels; if omitted, it MUST be assumed to contain one channel + return 1; // The number of channels; if omitted, it MUST be assumed to contain one + // channel } try { return Integer.parseInt(channels); @@ -299,7 +319,9 @@ public class RtpDescription extends GenericDescription { } public static PayloadType of(final Element element) { - Preconditions.checkArgument("payload-type".equals(element.getName()), "element name must be called payload-type"); + Preconditions.checkArgument( + "payload-type".equals(element.getName()), + "element name must be called payload-type"); PayloadType payloadType = new PayloadType(); payloadType.setAttributes(element.getAttributes()); payloadType.setChildren(element.getChildren()); @@ -339,8 +361,8 @@ public class RtpDescription extends GenericDescription { } } - //map to `fmtp $id key=value;key=value - //where id is the id of the parent payload-type + // map to `fmtp $id key=value;key=value + // where id is the id of the parent payload-type public static class Parameter extends Element { private Parameter() { @@ -362,7 +384,8 @@ public class RtpDescription extends GenericDescription { } public static Parameter of(final Element element) { - Preconditions.checkArgument("parameter".equals(element.getName()), "element name must be called parameter"); + Preconditions.checkArgument( + "parameter".equals(element.getName()), "element name must be called parameter"); Parameter parameter = new Parameter(); parameter.setAttributes(element.getAttributes()); parameter.setChildren(element.getChildren()); @@ -375,12 +398,18 @@ public class RtpDescription extends GenericDescription { for (int i = 0; i < parameters.size(); ++i) { final Parameter p = parameters.get(i); final String name = p.getParameterName(); - Preconditions.checkArgument(name != null, String.format("parameter for %s must have a name", id)); - SessionDescription.checkNoWhitespace(name, String.format("parameter names for %s must not contain whitespaces", id)); + Preconditions.checkArgument( + name != null, String.format("parameter for %s must have a name", id)); + SessionDescription.checkNoWhitespace( + name, + String.format("parameter names for %s must not contain whitespaces", id)); final String value = p.getParameterValue(); - Preconditions.checkArgument(value != null, String.format("parameter for %s must have a value", id)); - SessionDescription.checkNoWhitespace(value, String.format("parameter values for %s must not contain whitespaces", id)); + Preconditions.checkArgument( + value != null, String.format("parameter for %s must have a value", id)); + SessionDescription.checkNoWhitespace( + value, + String.format("parameter values for %s must not contain whitespaces", id)); stringBuilder.append(name).append('=').append(value); if (i != parameters.size() - 1) { @@ -393,8 +422,11 @@ public class RtpDescription extends GenericDescription { public static String toSdpString(final String id, final Parameter parameter) { final String name = parameter.getParameterName(); final String value = parameter.getParameterValue(); - Preconditions.checkArgument(value != null, String.format("parameter for %s must have a value", id)); - SessionDescription.checkNoWhitespace(value, String.format("parameter values for %s must not contain whitespaces", id)); + Preconditions.checkArgument( + value != null, String.format("parameter for %s must have a value", id)); + SessionDescription.checkNoWhitespace( + value, + String.format("parameter values for %s must not contain whitespaces", id)); if (Strings.isNullOrEmpty(name)) { return String.format("%s %s", id, value); } else { @@ -420,8 +452,8 @@ public class RtpDescription extends GenericDescription { } } - //XEP-0339: Source-Specific Media Attributes in Jingle - //maps to `a=ssrc: :` + // XEP-0339: Source-Specific Media Attributes in Jingle + // maps to `a=ssrc: :` public static class Source extends Element { private Source() { @@ -452,7 +484,9 @@ public class RtpDescription extends GenericDescription { public static Source upgrade(final Element element) { Preconditions.checkArgument("source".equals(element.getName())); - Preconditions.checkArgument(Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(element.getNamespace())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals( + element.getNamespace())); final Source source = new Source(); source.setChildren(element.getChildren()); source.setAttributes(element.getAttributes()); @@ -489,7 +523,6 @@ public class RtpDescription extends GenericDescription { return parameter; } } - } public static class SourceGroup extends Element { @@ -525,7 +558,9 @@ public class RtpDescription extends GenericDescription { public static SourceGroup upgrade(final Element element) { Preconditions.checkArgument("ssrc-group".equals(element.getName())); - Preconditions.checkArgument(Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(element.getNamespace())); + Preconditions.checkArgument( + Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals( + element.getNamespace())); final SourceGroup group = new SourceGroup(); group.setChildren(element.getChildren()); group.setAttributes(element.getAttributes()); @@ -533,15 +568,18 @@ public class RtpDescription extends GenericDescription { } } - public static RtpDescription of(final SessionDescription sessionDescription, final SessionDescription.Media media) { + public static RtpDescription of( + final SessionDescription sessionDescription, final SessionDescription.Media media) { final RtpDescription rtpDescription = new RtpDescription(media.media); final Map> parameterMap = new HashMap<>(); - final ArrayListMultimap feedbackNegotiationMap = ArrayListMultimap.create(); - final ArrayListMultimap sourceParameterMap = ArrayListMultimap.create(); - final Set attributes = Sets.newHashSet(Iterables.concat( - sessionDescription.attributes.keySet(), - media.attributes.keySet() - )); + final ArrayListMultimap feedbackNegotiationMap = + ArrayListMultimap.create(); + final ArrayListMultimap sourceParameterMap = + ArrayListMultimap.create(); + final Set attributes = + Sets.newHashSet( + Iterables.concat( + sessionDescription.attributes.keySet(), media.attributes.keySet())); for (final String rtcpFb : media.attributes.get("rtcp-fb")) { final String[] parts = rtcpFb.split(" "); if (parts.length >= 2) { @@ -550,7 +588,10 @@ public class RtpDescription extends GenericDescription { final String subType = parts.length >= 3 ? parts[2] : null; if ("trr-int".equals(type)) { if (subType != null) { - feedbackNegotiationMap.put(id, new FeedbackNegotiationTrrInt(SessionDescription.ignorantIntParser(subType))); + feedbackNegotiationMap.put( + id, + new FeedbackNegotiationTrrInt( + SessionDescription.ignorantIntParser(subType))); } } else { feedbackNegotiationMap.put(id, new FeedbackNegotiation(type, subType)); @@ -602,7 +643,8 @@ public class RtpDescription extends GenericDescription { rtpDescription.addChild(new SourceGroup(semantics, builder.build())); } } - for (Map.Entry> source : sourceParameterMap.asMap().entrySet()) { + for (Map.Entry> source : + sourceParameterMap.asMap().entrySet()) { rtpDescription.addChild(new Source(source.getKey(), source.getValue())); } if (media.attributes.containsKey("rtcp-mux")) {