implement sending of session-accept

src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java

@@ -5,16 +5,7 @@ import android.util.Log;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
-import org.webrtc.AudioSource;
-import org.webrtc.AudioTrack;
-import org.webrtc.DataChannel;
 import org.webrtc.IceCandidate;
-import org.webrtc.MediaConstraints;
-import org.webrtc.MediaStream;
-import org.webrtc.PeerConnection;
-import org.webrtc.PeerConnectionFactory;
-import org.webrtc.RtpReceiver;
-import org.webrtc.SdpObserver;
 
 import java.util.ArrayDeque;
 import java.util.Arrays;
@@ -32,7 +23,7 @@ import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;
 import eu.siacs.conversations.xmpp.stanzas.MessagePacket;
 import rocks.xmpp.addr.Jid;
 
-public class JingleRtpConnection extends AbstractJingleConnection {
+public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback {
 
     private static final Map<State, Collection<State>> VALID_TRANSITIONS;
 
@@ -41,14 +32,15 @@ public class JingleRtpConnection extends AbstractJingleConnection {
         transitionBuilder.put(State.NULL, ImmutableList.of(State.PROPOSED, State.SESSION_INITIALIZED));
         transitionBuilder.put(State.PROPOSED, ImmutableList.of(State.ACCEPTED, State.PROCEED));
         transitionBuilder.put(State.PROCEED, ImmutableList.of(State.SESSION_INITIALIZED));
+        transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of(State.SESSION_ACCEPTED));
         VALID_TRANSITIONS = transitionBuilder.build();
     }
 
-    private State state = State.NULL;
+    private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
-    private RtpContentMap initialRtpContentMap;
-    private PeerConnection peerConnection;
-
     private final ArrayDeque<IceCandidate> pendingIceCandidates = new ArrayDeque<>();
+    private State state = State.NULL;
+    private RtpContentMap initiatorRtpContentMap;
+    private RtpContentMap responderRtpContentMap;
 
 
     public JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
@@ -80,21 +72,22 @@ public class JingleRtpConnection extends AbstractJingleConnection {
                 Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", e);
                 return;
             }
-            final Group originalGroup = this.initialRtpContentMap != null ? this.initialRtpContentMap.group : null;
+            //TODO pick proper rtpContentMap
+            final Group originalGroup = this.initiatorRtpContentMap != null ? this.initiatorRtpContentMap.group : null;
             final List<String> identificationTags = originalGroup == null ? Collections.emptyList() : originalGroup.getIdentificationTags();
             if (identificationTags.size() == 0) {
-                Log.w(Config.LOGTAG,id.account.getJid().asBareJid()+": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
+                Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
             }
-            for(final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contentMap.contents.entrySet()) {
+            for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contentMap.contents.entrySet()) {
                 final String ufrag = content.getValue().transport.getAttribute("ufrag");
-                for(final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
+                for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
                     final String sdp = candidate.toSdpAttribute(ufrag);
                     final String sdpMid = content.getKey();
                     final int mLineIndex = identificationTags.indexOf(sdpMid);
                     final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
-                    Log.d(Config.LOGTAG,"received candidate: "+iceCandidate);
+                    Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
                     if (isInState(State.SESSION_ACCEPTED)) {
-                        this.peerConnection.addIceCandidate(iceCandidate);
+                        this.webRTCWrapper.addIceCandidate(iceCandidate);
                     } else {
                         this.pendingIceCandidates.push(iceCandidate);
                     }
@@ -106,7 +99,6 @@ public class JingleRtpConnection extends AbstractJingleConnection {
     }
 
     private void receiveSessionInitiate(final JinglePacket jinglePacket) {
-        Log.d(Config.LOGTAG, jinglePacket.toString());
         if (isInitiator()) {
             Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid()));
             //TODO respond with out-of-order
@@ -123,11 +115,12 @@ public class JingleRtpConnection extends AbstractJingleConnection {
         Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents");
         final State oldState = this.state;
         if (transition(State.SESSION_INITIALIZED)) {
-            this.initialRtpContentMap = contentMap;
+            this.initiatorRtpContentMap = contentMap;
             if (oldState == State.PROCEED) {
-                processContents(contentMap);
+                Log.d(Config.LOGTAG, "automatically accepting");
                 sendSessionAccept();
             } else {
+                Log.d(Config.LOGTAG, "start ringing");
                 //TODO start ringing
             }
         } else {
@@ -135,32 +128,35 @@ public class JingleRtpConnection extends AbstractJingleConnection {
         }
     }
 
-    private void processContents(final RtpContentMap contentMap) {
+    private void sendSessionAccept() {
+        final RtpContentMap rtpContentMap = this.initiatorRtpContentMap;
+        if (rtpContentMap == null) {
+            throw new IllegalStateException("intital RTP Content Map has not been set");
+        }
         setupWebRTC();
-        org.webrtc.SessionDescription sessionDescription = new org.webrtc.SessionDescription(org.webrtc.SessionDescription.Type.OFFER, SessionDescription.of(contentMap).toString());
+        final org.webrtc.SessionDescription offer = new org.webrtc.SessionDescription(
-        Log.d(Config.LOGTAG, "debug print for sessionDescription:" + sessionDescription.description);
+                org.webrtc.SessionDescription.Type.OFFER,
-        this.peerConnection.setRemoteDescription(new SdpObserver() {
+                SessionDescription.of(rtpContentMap).toString()
-            @Override
+        );
-            public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
+        try {
+            this.webRTCWrapper.setRemoteDescription(offer).get();
+            org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get();
+            this.webRTCWrapper.setLocalDescription(webRTCSessionDescription);
+            final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
+            final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription);
+            sendSessionAccept(respondingRtpContentMap);
+        } catch (Exception e) {
+            Log.d(Config.LOGTAG, "unable to send session accept", e);
 
-            }
+        }
+    }
 
-            @Override
+    private void sendSessionAccept(final RtpContentMap rtpContentMap) {
-            public void onSetSuccess() {
+        this.responderRtpContentMap = rtpContentMap;
-                Log.d(Config.LOGTAG, "onSetSuccess() for setRemoteDescription");
+        this.transitionOrThrow(State.SESSION_ACCEPTED);
-            }
+        final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId);
-
+        Log.d(Config.LOGTAG, sessionAccept.toString());
-            @Override
+        send(sessionAccept);
-            public void onCreateFailure(String s) {
-
-            }
-
-            @Override
-            public void onSetFailure(String s) {
-                Log.d(Config.LOGTAG, "onSetFailure() for setRemoteDescription. " + s);
-
-            }
-        }, sessionDescription);
     }
 
     void deliveryMessage(final Jid from, final Element message) {
@@ -178,6 +174,7 @@ public class JingleRtpConnection extends AbstractJingleConnection {
 
     private void receivePropose(final Jid from, final Element propose) {
         final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid());
+        //TODO we can use initiator logic here
         if (originatedFromMyself) {
             Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from mysql. ignoring");
         } else if (transition(State.PROPOSED)) {
@@ -207,21 +204,31 @@ public class JingleRtpConnection extends AbstractJingleConnection {
     private void sendSessionInitiate() {
         Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": prepare session-initiate");
         setupWebRTC();
-        createOffer();
+        try {
+            org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get();
+            final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description);
+            Log.d(Config.LOGTAG, "description: " + webRTCSessionDescription.description);
+            final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
+            sendSessionInitiate(rtpContentMap);
+            this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get();
+        } catch (Exception e) {
+            Log.d(Config.LOGTAG, "unable to sendSessionInitiate", e);
+        }
     }
 
     private void sendSessionInitiate(RtpContentMap rtpContentMap) {
-        this.initialRtpContentMap = rtpContentMap;
+        this.initiatorRtpContentMap = rtpContentMap;
+        this.transitionOrThrow(State.SESSION_INITIALIZED);
         final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId);
         Log.d(Config.LOGTAG, sessionInitiate.toString());
-        Log.d(Config.LOGTAG, "here is what we think the sdp looks like" + SessionDescription.of(rtpContentMap).toString());
         send(sessionInitiate);
     }
 
     private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) {
         final RtpContentMap transportInfo;
         try {
-            transportInfo = this.initialRtpContentMap.transportInfo(contentName, candidate);
+            //TODO when responding use responderRtpContentMap
+            transportInfo = this.initiatorRtpContentMap.transportInfo(contentName, candidate);
         } catch (Exception e) {
             Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName);
             return;
@@ -238,10 +245,6 @@ public class JingleRtpConnection extends AbstractJingleConnection {
     }
 
 
-    private void sendSessionAccept() {
-        Log.d(Config.LOGTAG, "sending session-accept");
-    }
-
     public void pickUpCall() {
         switch (this.state) {
             case PROPOSED:
@@ -256,133 +259,8 @@ public class JingleRtpConnection extends AbstractJingleConnection {
     }
 
     private void setupWebRTC() {
-        PeerConnectionFactory.initialize(
+        this.webRTCWrapper.setup(this.xmppConnectionService);
-                PeerConnectionFactory.InitializationOptions.builder(xmppConnectionService).createInitializationOptions()
+        this.webRTCWrapper.initializePeerConnection();
-        );
-        final PeerConnectionFactory.Options options = new PeerConnectionFactory.Options();
-        PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder().createPeerConnectionFactory();
-
-        final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
-
-        final AudioTrack audioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
-        final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
-        stream.addTrack(audioTrack);
-
-
-        final List<PeerConnection.IceServer> iceServers = ImmutableList.of(
-                PeerConnection.IceServer.builder("stun:xmpp.conversations.im:3478").createIceServer()
-        );
-        this.peerConnection = peerConnectionFactory.createPeerConnection(iceServers, new PeerConnection.Observer() {
-            @Override
-            public void onSignalingChange(PeerConnection.SignalingState signalingState) {
-
-            }
-
-            @Override
-            public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
-
-            }
-
-            @Override
-            public void onIceConnectionReceivingChange(boolean b) {
-
-            }
-
-            @Override
-            public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
-                Log.d(Config.LOGTAG, "onIceGatheringChange() " + iceGatheringState);
-            }
-
-            @Override
-            public void onIceCandidate(IceCandidate iceCandidate) {
-                IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
-                Log.d(Config.LOGTAG, "onIceCandidate: " + iceCandidate.sdp + " mLineIndex=" + iceCandidate.sdpMLineIndex);
-                sendTransportInfo(iceCandidate.sdpMid, candidate);
-
-            }
-
-            @Override
-            public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
-
-            }
-
-            @Override
-            public void onAddStream(MediaStream mediaStream) {
-
-            }
-
-            @Override
-            public void onRemoveStream(MediaStream mediaStream) {
-
-            }
-
-            @Override
-            public void onDataChannel(DataChannel dataChannel) {
-
-            }
-
-            @Override
-            public void onRenegotiationNeeded() {
-
-            }
-
-            @Override
-            public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
-
-            }
-        });
-
-        peerConnection.addStream(stream);
-    }
-
-    private void createOffer() {
-        Log.d(Config.LOGTAG, "createOffer()");
-        peerConnection.createOffer(new SdpObserver() {
-
-            @Override
-            public void onCreateSuccess(org.webrtc.SessionDescription description) {
-                final SessionDescription sessionDescription = SessionDescription.parse(description.description);
-                Log.d(Config.LOGTAG, "description: " + description.description);
-                final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
-                sendSessionInitiate(rtpContentMap);
-                peerConnection.setLocalDescription(new SdpObserver() {
-                    @Override
-                    public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
-
-                    }
-
-                    @Override
-                    public void onSetSuccess() {
-                        Log.d(Config.LOGTAG, "onSetSuccess()");
-                    }
-
-                    @Override
-                    public void onCreateFailure(String s) {
-
-                    }
-
-                    @Override
-                    public void onSetFailure(String s) {
-
-                    }
-                }, description);
-            }
-
-            @Override
-            public void onSetSuccess() {
-
-            }
-
-            @Override
-            public void onCreateFailure(String s) {
-
-            }
-
-            @Override
-            public void onSetFailure(String s) {
-
-            }
-        }, new MediaConstraints());
     }
 
     private void pickupCallFromProposed() {
@@ -419,4 +297,11 @@ public class JingleRtpConnection extends AbstractJingleConnection {
             throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target));
         }
     }
+
+    @Override
+    public void onIceCandidate(final IceCandidate iceCandidate) {
+        final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
+        Log.d(Config.LOGTAG, "onIceCandidate: " + iceCandidate.sdp + " mLineIndex=" + iceCandidate.sdpMLineIndex);
+        sendTransportInfo(iceCandidate.sdpMid, candidate);
+    }
 }

src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java

@@ -0,0 +1,247 @@
+package eu.siacs.conversations.xmpp.jingle;
+
+import android.content.Context;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.SettableFuture;
+
+import org.webrtc.AudioSource;
+import org.webrtc.AudioTrack;
+import org.webrtc.DataChannel;
+import org.webrtc.IceCandidate;
+import org.webrtc.MediaConstraints;
+import org.webrtc.MediaStream;
+import org.webrtc.PeerConnection;
+import org.webrtc.PeerConnectionFactory;
+import org.webrtc.RtpReceiver;
+import org.webrtc.SdpObserver;
+import org.webrtc.SessionDescription;
+
+import java.util.List;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+public class WebRTCWrapper {
+
+    private final EventCallback eventCallback;
+
+    private final PeerConnection.Observer peerConnectionObserver = new PeerConnection.Observer() {
+        @Override
+        public void onSignalingChange(PeerConnection.SignalingState signalingState) {
+
+        }
+
+        @Override
+        public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
+
+        }
+
+        @Override
+        public void onIceConnectionReceivingChange(boolean b) {
+
+        }
+
+        @Override
+        public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
+
+        }
+
+        @Override
+        public void onIceCandidate(IceCandidate iceCandidate) {
+            eventCallback.onIceCandidate(iceCandidate);
+        }
+
+        @Override
+        public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) {
+
+        }
+
+        @Override
+        public void onAddStream(MediaStream mediaStream) {
+
+        }
+
+        @Override
+        public void onRemoveStream(MediaStream mediaStream) {
+
+        }
+
+        @Override
+        public void onDataChannel(DataChannel dataChannel) {
+
+        }
+
+        @Override
+        public void onRenegotiationNeeded() {
+
+        }
+
+        @Override
+        public void onAddTrack(RtpReceiver rtpReceiver, MediaStream[] mediaStreams) {
+
+        }
+    };
+    @Nullable
+    private PeerConnection peerConnection = null;
+
+    public WebRTCWrapper(final EventCallback eventCallback) {
+        this.eventCallback = eventCallback;
+    }
+
+    public void setup(final Context context) {
+        PeerConnectionFactory.initialize(
+                PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()
+        );
+    }
+
+    public void initializePeerConnection() {
+        final PeerConnectionFactory.Options options = new PeerConnectionFactory.Options();
+        PeerConnectionFactory peerConnectionFactory = PeerConnectionFactory.builder().createPeerConnectionFactory();
+
+        final AudioSource audioSource = peerConnectionFactory.createAudioSource(new MediaConstraints());
+
+        final AudioTrack audioTrack = peerConnectionFactory.createAudioTrack("my-audio-track", audioSource);
+        final MediaStream stream = peerConnectionFactory.createLocalMediaStream("my-media-stream");
+        stream.addTrack(audioTrack);
+
+
+        final List<PeerConnection.IceServer> iceServers = ImmutableList.of(
+                PeerConnection.IceServer.builder("stun:xmpp.conversations.im:3478").createIceServer()
+        );
+        final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(iceServers, peerConnectionObserver);
+        if (peerConnection == null) {
+            throw new IllegalStateException("Unable to create PeerConnection");
+        }
+        peerConnection.addStream(stream);
+        this.peerConnection = peerConnection;
+    }
+
+    public ListenableFuture<SessionDescription> createOffer() {
+        return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
+            final SettableFuture<SessionDescription> future = SettableFuture.create();
+            peerConnection.createOffer(new CreateSdpObserver() {
+                @Override
+                public void onCreateSuccess(SessionDescription sessionDescription) {
+                    future.set(sessionDescription);
+                }
+
+                @Override
+                public void onCreateFailure(String s) {
+                    future.setException(new IllegalStateException("Unable to create offer: " + s));
+                }
+            }, new MediaConstraints());
+            return future;
+        }, MoreExecutors.directExecutor());
+    }
+
+    public ListenableFuture<SessionDescription> createAnswer() {
+        return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
+            final SettableFuture<SessionDescription> future = SettableFuture.create();
+            peerConnection.createAnswer(new CreateSdpObserver() {
+                @Override
+                public void onCreateSuccess(SessionDescription sessionDescription) {
+                    future.set(sessionDescription);
+                }
+
+                @Override
+                public void onCreateFailure(String s) {
+                    future.setException(new IllegalStateException("Unable to create answer: " + s));
+                }
+            }, new MediaConstraints());
+            return future;
+        }, MoreExecutors.directExecutor());
+    }
+
+    public ListenableFuture<Void> setLocalDescription(final SessionDescription sessionDescription) {
+        return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
+            final SettableFuture<Void> future = SettableFuture.create();
+            peerConnection.setLocalDescription(new SetSdpObserver() {
+                @Override
+                public void onSetSuccess() {
+                    future.set(null);
+                }
+
+                @Override
+                public void onSetFailure(String s) {
+                    future.setException(new IllegalArgumentException("unable to set local session description: "+s));
+
+                }
+            }, sessionDescription);
+            return future;
+        }, MoreExecutors.directExecutor());
+    }
+
+    public ListenableFuture<Void> setRemoteDescription(final SessionDescription sessionDescription) {
+        return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> {
+            final SettableFuture<Void> future = SettableFuture.create();
+            peerConnection.setRemoteDescription(new SetSdpObserver() {
+                @Override
+                public void onSetSuccess() {
+                    future.set(null);
+                }
+
+                @Override
+                public void onSetFailure(String s) {
+                    future.setException(new IllegalArgumentException("unable to set remote session description: "+s));
+
+                }
+            }, sessionDescription);
+            return future;
+        }, MoreExecutors.directExecutor());
+    }
+
+    @Nonnull
+    private ListenableFuture<PeerConnection> getPeerConnectionFuture() {
+        final PeerConnection peerConnection = this.peerConnection;
+        if (peerConnection == null) {
+            return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first"));
+        } else {
+            return Futures.immediateFuture(peerConnection);
+        }
+    }
+
+    public void addIceCandidate(IceCandidate iceCandidate) {
+        final PeerConnection peerConnection = this.peerConnection;
+        if (peerConnection == null) {
+            throw new IllegalStateException("initialize PeerConnection first");
+        }
+        peerConnection.addIceCandidate(iceCandidate);
+    }
+
+    private static abstract class SetSdpObserver implements SdpObserver {
+
+        @Override
+        public void onCreateSuccess(org.webrtc.SessionDescription sessionDescription) {
+            throw new IllegalStateException("Not able to use SetSdpObserver");
+        }
+
+        @Override
+        public void onCreateFailure(String s) {
+            throw new IllegalStateException("Not able to use SetSdpObserver");
+        }
+
+    }
+
+    private static abstract class CreateSdpObserver implements SdpObserver {
+
+
+        @Override
+        public void onSetSuccess() {
+            throw new IllegalStateException("Not able to use CreateSdpObserver");
+        }
+
+
+        @Override
+        public void onSetFailure(String s) {
+            throw new IllegalStateException("Not able to use CreateSdpObserver");
+        }
+    }
+
+    public interface EventCallback {
+        void onIceCandidate(IceCandidate iceCandidate);
+    }
+}