From dbf71e5d54bae57f3c506248fb17f7c18dd98ec9 Mon Sep 17 00:00:00 2001 From: Daniel Gultsch Date: Tue, 10 Oct 2023 18:42:23 +0200 Subject: [PATCH] handle senders modification via content-modify Dino uses this to enable/disable video when a video content is already present --- .../xmpp/jingle/JingleConnectionManager.java | 4 +- .../xmpp/jingle/JingleRtpConnection.java | 88 ++++++++++++++++--- .../jingle/stanzas/IceUdpTransportInfo.java | 67 +++++++------- 3 files changed, 117 insertions(+), 42 deletions(-) diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java index 1872f6b32..1a55b9a96 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java @@ -251,7 +251,9 @@ public class JingleConnectionManager extends AbstractConnectionManager { final Element error = response.addChild("error"); error.setAttribute("type", conditionType); error.addChild(condition, "urn:ietf:params:xml:ns:xmpp-stanzas"); - error.addChild(jingleCondition, Namespace.JINGLE_ERRORS); + if (jingleCondition != null) { + error.addChild(jingleCondition, Namespace.JINGLE_ERRORS); + } account.getXmppConnection().sendIqPacket(response, null); } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index 38cba946f..fa54a9886 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -14,8 +14,11 @@ import com.google.common.base.Throwables; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.FutureCallback; @@ -551,15 +554,17 @@ public class JingleRtpConnection extends AbstractJingleConnection } private void receiveContentModify(final JinglePacket jinglePacket) { + // TODO check session accepted final Map modification = Maps.transformEntries( jinglePacket.getJingleContents(), (key, value) -> value.getSenders()); - respondOk(jinglePacket); + final boolean isInitiator = isInitiator(); final RtpContentMap currentOutgoing = this.outgoingContentAdd; + final RtpContentMap remoteContentMap = this.getRemoteContentMap(); final Set currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet(); Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")"); if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) { - final boolean isInitiator = isInitiator(); + respondOk(jinglePacket); final RtpContentMap modifiedContentMap; try { modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification); @@ -570,18 +575,72 @@ public class JingleRtpConnection extends AbstractJingleConnection } this.outgoingContentAdd = modifiedContentMap; Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add"); + } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) { + respondOk(jinglePacket); + final RtpContentMap modifiedRemoteContentMap; + try { + modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification); + } catch (final IllegalArgumentException e) { + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); + return; + } + final SessionDescription offer; + try { + offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator()); + } catch (final IllegalArgumentException | NullPointerException e) { + Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); + return; + } + Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification"); + this.autoAcceptContentModify(modifiedRemoteContentMap, offer); } else { - webRTCWrapper.close(); - sendSessionTerminate( - Reason.FAILED_APPLICATION, - String.format( - "%s only supports %s as a means to modify a not yet accepted %s", - BuildConfig.APP_NAME, - JinglePacket.Action.CONTENT_MODIFY, - JinglePacket.Action.CONTENT_ADD)); + Log.d(Config.LOGTAG,"received unsupported content modification "+modification); + respondWithItemNotFound(jinglePacket); } } + private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) { + this.setRemoteContentMap(modifiedRemoteContentMap); + final org.webrtc.SessionDescription sdp = + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.OFFER, offer.toString()); + try { + this.webRTCWrapper.setRemoteDescription(sdp).get(); + // auto accept is only done when we already have tracks + final SessionDescription answer = setLocalSessionDescription(); + final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator()); + modifyLocalContentMap(rtpContentMap); + // we do not need to send an answer but do we have to resend the candidates currently in SDP? + //resendCandidatesFromSdp(answer); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + } catch (final Exception e) { + Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e)); + webRTCWrapper.close(); + sendSessionTerminate(Reason.FAILED_APPLICATION); + } + } + + private void resendCandidatesFromSdp(final SessionDescription answer) { + final ImmutableMultimap.Builder candidateBuilder = new ImmutableMultimap.Builder<>(); + for(final SessionDescription.Media media : answer.media) { + final String mid = Iterables.getFirst(media.attributes.get("mid"), null); + if (Strings.isNullOrEmpty(mid)) { + continue; + } + for(final String sdpCandidate : media.attributes.get("candidate")) { + final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null); + if (candidate != null) { + candidateBuilder.put(mid,candidate); + } + } + } + final ImmutableMultimap candidates = candidateBuilder.build(); + sendTransportInfo(candidates); + } + private void receiveContentReject(final JinglePacket jinglePacket) { final RtpContentMap receivedContentReject; try { @@ -1942,6 +2001,11 @@ public class JingleRtpConnection extends AbstractJingleConnection send(jinglePacket); } + private void sendTransportInfo(final Multimap candidates) { + // TODO send all candidates in one transport-info + } + + private void send(final JinglePacket jinglePacket) { jinglePacket.setTo(id.with); xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse); @@ -2028,6 +2092,10 @@ public class JingleRtpConnection extends AbstractJingleConnection respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait"); } + private void respondWithItemNotFound(final JinglePacket jinglePacket) { + respondWithJingleError(jinglePacket, null, "item-not-found", "cancel"); + } + void respondWithJingleError( final IqPacket original, String jingleCondition, diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java index 8939ecb1b..026adbd02 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java @@ -202,41 +202,46 @@ public class IceUdpTransportInfo extends GenericTransportInfo { public static Candidate fromSdpAttribute(final String attribute, String currentUfrag) { final String[] pair = attribute.split(":", 2); if (pair.length == 2 && "candidate".equals(pair[0])) { - final String[] segments = pair[1].split(" "); - if (segments.length >= 6) { - final String id = UUID.randomUUID().toString(); - final String foundation = segments[0]; - final String component = segments[1]; - final String transport = segments[2].toLowerCase(Locale.ROOT); - final String priority = segments[3]; - final String connectionAddress = segments[4]; - final String port = segments[5]; - final HashMap additional = new HashMap<>(); - for (int i = 6; i < segments.length - 1; i = i + 2) { - additional.put(segments[i], segments[i + 1]); - } - final String ufrag = additional.get("ufrag"); - if (ufrag != null && !ufrag.equals(currentUfrag)) { - return null; - } - final Candidate candidate = new Candidate(); - candidate.setAttribute("component", component); - candidate.setAttribute("foundation", foundation); - candidate.setAttribute("generation", additional.get("generation")); - candidate.setAttribute("rel-addr", additional.get("raddr")); - candidate.setAttribute("rel-port", additional.get("rport")); - candidate.setAttribute("id", id); - candidate.setAttribute("ip", connectionAddress); - candidate.setAttribute("port", port); - candidate.setAttribute("priority", priority); - candidate.setAttribute("protocol", transport); - candidate.setAttribute("type", additional.get("typ")); - return candidate; - } + return fromSdpAttributeValue(pair[1], currentUfrag); } return null; } + public static Candidate fromSdpAttributeValue(final String value, final String currentUfrag) { + final String[] segments = value.split(" "); + if (segments.length < 6) { + return null; + } + final String id = UUID.randomUUID().toString(); + final String foundation = segments[0]; + final String component = segments[1]; + final String transport = segments[2].toLowerCase(Locale.ROOT); + final String priority = segments[3]; + final String connectionAddress = segments[4]; + final String port = segments[5]; + final HashMap additional = new HashMap<>(); + for (int i = 6; i < segments.length - 1; i = i + 2) { + additional.put(segments[i], segments[i + 1]); + } + final String ufrag = additional.get("ufrag"); + if (currentUfrag != null && ufrag != null && !ufrag.equals(currentUfrag)) { + return null; + } + final Candidate candidate = new Candidate(); + candidate.setAttribute("component", component); + candidate.setAttribute("foundation", foundation); + candidate.setAttribute("generation", additional.get("generation")); + candidate.setAttribute("rel-addr", additional.get("raddr")); + candidate.setAttribute("rel-port", additional.get("rport")); + candidate.setAttribute("id", id); + candidate.setAttribute("ip", connectionAddress); + candidate.setAttribute("port", port); + candidate.setAttribute("priority", priority); + candidate.setAttribute("protocol", transport); + candidate.setAttribute("type", additional.get("typ")); + return candidate; + } + public int getComponent() { return getAttributeAsInt("component"); }