From 728d15167c1eb91df760c8617a8062597e9e120c Mon Sep 17 00:00:00 2001 From: Lyubomir Marinov <lyubomir.marinov@jitsi.org> Date: Mon, 7 Jan 2013 07:01:37 +0000 Subject: [PATCH] Works on fixing issues with Jitsi VideoBridge telephony conferencing. --- .../impl/neomedia/AudioMediaStreamImpl.java | 416 +++++++++--------- .../device/AudioMediaDeviceSession.java | 88 ++-- .../device/AudioMixerMediaDevice.java | 2 + .../service/neomedia/AudioMediaStream.java | 47 +- 4 files changed, 277 insertions(+), 276 deletions(-) diff --git a/src/org/jitsi/impl/neomedia/AudioMediaStreamImpl.java b/src/org/jitsi/impl/neomedia/AudioMediaStreamImpl.java index b5bfc20c..26f6d179 100644 --- a/src/org/jitsi/impl/neomedia/AudioMediaStreamImpl.java +++ b/src/org/jitsi/impl/neomedia/AudioMediaStreamImpl.java @@ -38,23 +38,6 @@ public class AudioMediaStreamImpl PropertyChangeListener { - /** - * The <tt>Logger</tt> used by the <tt>AudioMediaStreamImpl</tt> class and - * its instances for logging output. - */ - private static final Logger logger - = Logger.getLogger(AudioMediaStreamImpl.class); - - /** - * The transformer that we use for sending and receiving DTMF packets. - */ - private DtmfTransformEngine dtmfTransfrmEngine ; - - /** - * List of DTMF listeners; - */ - private List<DTMFListener> dtmfListeners = new ArrayList<DTMFListener>(); - /** * List of RTP format strings which are supported by SIP Communicator in * addition to the JMF standard formats. @@ -83,10 +66,11 @@ public class AudioMediaStreamImpl }; /** - * The listener that gets notified of changes in the audio level of - * remote conference participants. + * The <tt>Logger</tt> used by the <tt>AudioMediaStreamImpl</tt> class and + * its instances for logging output. */ - private CsrcAudioLevelListener csrcAudioLevelListener = null; + private static final Logger logger + = Logger.getLogger(AudioMediaStreamImpl.class); /** * A property change notifier which will inform this stream if a selected @@ -96,6 +80,23 @@ public class AudioMediaStreamImpl */ private PropertyChangeNotifier audioSystemChangeNotifier; + /** + * The listener that gets notified of changes in the audio level of + * remote conference participants. + */ + private CsrcAudioLevelListener csrcAudioLevelListener = null; + + /** + * The list of DTMF listeners; + */ + private final List<DTMFListener> dtmfListeners + = new ArrayList<DTMFListener>(); + + /** + * The transformer that we use for sending and receiving DTMF packets. + */ + private DtmfTransformEngine dtmfTransfrmEngine ; + /** * Initializes a new <tt>AudioMediaStreamImpl</tt> instance which will use * the specified <tt>MediaDevice</tt> for both capture and playback of audio @@ -124,6 +125,67 @@ public AudioMediaStreamImpl(StreamConnector connector, this.audioSystemChangeNotifier.addPropertyChangeListener(this); } + /** + * Adds a <tt>DTMFListener</tt> to this <tt>AudioMediaStream</tt> which is + * to receive notifications when the remote party starts sending DTMF tones + * to us. + * + * @param listener the <tt>DTMFListener</tt> to register for notifications + * about the remote party starting sending of DTM tones to this + * <tt>AudioMediaStream</tt> + * @see AudioMediaStream#addDTMFListener(DTMFListener) + */ + public void addDTMFListener(DTMFListener listener) + { + if((listener != null) && !dtmfListeners.contains(listener)) + dtmfListeners.add(listener); + } + + /** + * In addition to calling + * {@link MediaStreamImpl#addRTPExtension(byte, RTPExtension)} + * this method enables sending of CSRC audio levels. The reason we are + * doing this here rather than in the super class is that CSRC levels only + * make sense for audio streams so we don't want them enabled in any other + * type. + * + * @param extensionID the ID assigned to <tt>rtpExtension</tt> for the + * lifetime of this stream. + * @param rtpExtension the RTPExtension that is being added to this stream. + */ + @Override + public void addRTPExtension(byte extensionID, RTPExtension rtpExtension) + { + super.addRTPExtension(extensionID, rtpExtension); + + if (RTPExtension.CSRC_AUDIO_LEVEL_URN.equals( + rtpExtension.getURI().toString())) + { + getCsrcEngine().setCsrcAudioLevelAudioLevelExtensionID( + extensionID, + rtpExtension.getDirection()); + } + } + + /** + * Releases the resources allocated by this instance in the course of its + * execution and prepares it to be garbage collected. + * + * @see MediaStream#close() + */ + @Override + public void close() + { + super.close(); + + if(dtmfTransfrmEngine != null) + { + dtmfTransfrmEngine = null; + } + + this.audioSystemChangeNotifier.removePropertyChangeListener(this); + } + /** * Performs any optional configuration on the <tt>BufferControl</tt> of the * specified <tt>RTPManager</tt> which is to be used as the @@ -198,53 +260,110 @@ protected DtmfTransformEngine createDtmfTransformEngine() { if(this.dtmfTransfrmEngine == null) this.dtmfTransfrmEngine = new DtmfTransformEngine(this); - return this.dtmfTransfrmEngine; } /** - * Adds a <tt>DTMFListener</tt> to this <tt>AudioMediaStream</tt> which is - * to receive notifications when the remote party starts sending DTMF tones - * to us. + * Delivers the <tt>audioLevels</tt> map to whoever's interested. This + * method is meant for use primarily by the transform engine handling + * incoming RTP packets (currently <tt>CsrcTransformEngine</tt>). * - * @param listener the <tt>DTMFListener</tt> to register for notifications - * about the remote party starting sending of DTM tones to this - * <tt>AudioMediaStream</tt> - * @see AudioMediaStream#addDTMFListener(DTMFListener) + * @param audioLevels a array mapping CSRC IDs to audio levels in + * consecutive elements. */ - public void addDTMFListener(DTMFListener listener) + public void fireConferenceAudioLevelEvent(long[] audioLevels) { - if(!dtmfListeners.contains(listener)) + CsrcAudioLevelListener csrcAudioLevelListener + = this.csrcAudioLevelListener; + + if (csrcAudioLevelListener != null) + csrcAudioLevelListener.audioLevelsReceived(audioLevels); + } + + /** + * Delivers the <tt>DTMF</tt> tones. The method is meant for use primarily + * by the transform engine handling incoming RTP packets (currently + * <tt>DtmfTransformEngine</tt>). + * + * @param tone the new tone + * @param end <tt>true</tt> if the tone is to be ended or <tt>false</tt> to + * be started + */ + public void fireDTMFEvent(DTMFRtpTone tone, boolean end) + { + DTMFToneEvent ev = new DTMFToneEvent(this, tone); + + for (DTMFListener listener : dtmfListeners) { - dtmfListeners.add(listener); + if(end) + listener.dtmfToneReceptionEnded(ev); + else + listener.dtmfToneReceptionStarted(ev); } } /** - * Sets <tt>listener</tt> as the <tt>SimpleAudioLevelListener</tt> - * registered to receive notifications from our device session for changes - * in the levels of the party that's at the other end of this stream. + * Returns the <tt>MediaDeviceSession</tt> associated with this stream + * after first casting it to <tt>AudioMediaDeviceSession</tt> since this is, + * after all, an <tt>AudioMediaStreamImpl</tt>. * - * @param listener the <tt>SimpleAudioLevelListener</tt> that we'd like to - * register or <tt>null</tt> if we want to stop stream audio level - * measurements. + * @return the <tt>AudioMediaDeviceSession</tt> associated with this stream. */ - public void setStreamAudioLevelListener(SimpleAudioLevelListener listener) + @Override + public AudioMediaDeviceSession getDeviceSession() { - getDeviceSession().setStreamAudioLevelListener(listener); + return (AudioMediaDeviceSession) super.getDeviceSession(); } /** - * Registers <tt>listener</tt> as the <tt>CsrcAudioLevelListener</tt> that - * will receive notifications for changes in the levels of conference - * participants that the remote party could be mixing. + * Returns the last audio level that was measured by the underlying device + * session for the specified <tt>ssrc</tt> (where <tt>ssrc</tt> could also + * correspond to our local sync source identifier). * - * @param listener the <tt>CsrcAudioLevelListener</tt> that we'd like to - * register or <tt>null</tt> if we'd like to stop receiving notifications. + * @param ssrc the SSRC ID whose last measured audio level we'd like to + * retrieve. + * + * @return the audio level that was last measured for the specified + * <tt>ssrc</tt> or <tt>-1</tt> if no level has been cached for that ID. */ - public void setCsrcAudioLevelListener(CsrcAudioLevelListener listener) + public int getLastMeasuredAudioLevel(long ssrc) { - this.csrcAudioLevelListener = listener; + AudioMediaDeviceSession devSession = getDeviceSession(); + + if (devSession == null) + return -1; + else if (ssrc == getLocalSourceID()) + return devSession.getLastMeasuredLocalUserAudioLevel(); + else + return devSession.getLastMeasuredAudioLevel(ssrc); + } + + /** + * The priority of the audio is 3, which is meant to be higher than + * other threads and higher than the video one. + * @return audio priority. + */ + @Override + protected int getPriority() + { + return 3; + } + + /** + * Receives and reacts to property change events: if the selected device + * (for capture, playback or notifications) has changed, then create or + * recreate the streams in order to use it. + * We want to listen to these events, especially for those generated after + * the audio system has changed. + * + * @param evt The event which may contain a audio system change event. + */ + public void propertyChange(PropertyChangeEvent evt) + { + if (sendStreamsAreCreated) + recreateSendStreams(); + else + start(); } /** @@ -292,6 +411,48 @@ public void removeDTMFListener(DTMFListener listener) dtmfListeners.remove(listener); } + /** + * Registers <tt>listener</tt> as the <tt>CsrcAudioLevelListener</tt> that + * will receive notifications for changes in the levels of conference + * participants that the remote party could be mixing. + * + * @param listener the <tt>CsrcAudioLevelListener</tt> that we'd like to + * register or <tt>null</tt> if we'd like to stop receiving notifications. + */ + public void setCsrcAudioLevelListener(CsrcAudioLevelListener listener) + { + this.csrcAudioLevelListener = listener; + } + + /** + * Sets <tt>listener</tt> as the <tt>SimpleAudioLevelListener</tt> + * registered to receive notifications from our device session for changes + * in the levels of the audio that this stream is sending out. + * + * @param listener the <tt>SimpleAudioLevelListener</tt> that we'd like to + * register or <tt>null</tt> if we want to stop local audio level + * measurements. + */ + public void setLocalUserAudioLevelListener( + SimpleAudioLevelListener listener) + { + getDeviceSession().setLocalUserAudioLevelListener(listener); + } + + /** + * Sets <tt>listener</tt> as the <tt>SimpleAudioLevelListener</tt> + * registered to receive notifications from our device session for changes + * in the levels of the party that's at the other end of this stream. + * + * @param listener the <tt>SimpleAudioLevelListener</tt> that we'd like to + * register or <tt>null</tt> if we want to stop stream audio level + * measurements. + */ + public void setStreamAudioLevelListener(SimpleAudioLevelListener listener) + { + getDeviceSession().setStreamAudioLevelListener(listener); + } + /** * Starts sending the specified <tt>DTMFTone</tt> until the * <tt>stopSendingDTMF()</tt> method is called (Excepts for INBAND DTMF, @@ -375,167 +536,4 @@ public void stopSendingDTMF(DTMFMethod dtmfMethod) throw new IllegalArgumentException("dtmfMethod"); } } - - /** - * In addition to calling - * {@link MediaStreamImpl#addRTPExtension(byte, RTPExtension)} - * this method enables sending of CSRC audio levels. The reason we are - * doing this here rather than in the super class is that CSRC levels only - * make sense for audio streams so we don't want them enabled in any other - * type. - * - * @param extensionID the ID assigned to <tt>rtpExtension</tt> for the - * lifetime of this stream. - * @param rtpExtension the RTPExtension that is being added to this stream. - */ - @Override - public void addRTPExtension(byte extensionID, RTPExtension rtpExtension) - { - super.addRTPExtension(extensionID, rtpExtension); - - if (RTPExtension.CSRC_AUDIO_LEVEL_URN.equals( - rtpExtension.getURI().toString())) - { - getCsrcEngine().setCsrcAudioLevelAudioLevelExtensionID( - extensionID, - rtpExtension.getDirection()); - } - } - - /** - * Sets <tt>listener</tt> as the <tt>SimpleAudioLevelListener</tt> - * registered to receive notifications from our device session for changes - * in the levels of the audio that this stream is sending out. - * - * @param listener the <tt>SimpleAudioLevelListener</tt> that we'd like to - * register or <tt>null</tt> if we want to stop local audio level - * measurements. - */ - public void setLocalUserAudioLevelListener( - SimpleAudioLevelListener listener) - { - getDeviceSession().setLocalUserAudioLevelListener(listener); - } - - /** - * Returns the <tt>MediaDeviceSession</tt> associated with this stream - * after first casting it to <tt>AudioMediaDeviceSession</tt> since this is, - * after all, an <tt>AudioMediaStreamImpl</tt>. - * - * @return the <tt>AudioMediaDeviceSession</tt> associated with this stream. - */ - @Override - public AudioMediaDeviceSession getDeviceSession() - { - return (AudioMediaDeviceSession) super.getDeviceSession(); - } - - /** - * Returns the last audio level that was measured by the underlying device - * session for the specified <tt>ssrc</tt> (where <tt>ssrc</tt> could also - * correspond to our local sync source identifier). - * - * @param ssrc the SSRC ID whose last measured audio level we'd like to - * retrieve. - * - * @return the audio level that was last measured for the specified - * <tt>ssrc</tt> or <tt>-1</tt> if no level has been cached for that ID. - */ - public int getLastMeasuredAudioLevel(long ssrc) - { - AudioMediaDeviceSession devSession = getDeviceSession(); - - if (devSession == null) - return -1; - else if (ssrc == getLocalSourceID()) - return devSession.getLastMeasuredLocalUserAudioLevel(); - else - return devSession.getLastMeasuredAudioLevel(ssrc); - } - - /** - * Delivers the <tt>audioLevels</tt> map to whoever's interested. This - * method is meant for use primarily by the transform engine handling - * incoming RTP packets (currently <tt>CsrcTransformEngine</tt>). - * - * @param audioLevels a array mapping CSRC IDs to audio levels in - * consecutive elements. - */ - public void fireConferenceAudioLevelEvent(long[] audioLevels) - { - CsrcAudioLevelListener csrcAudioLevelListener - = this.csrcAudioLevelListener; - - if (csrcAudioLevelListener != null) - csrcAudioLevelListener.audioLevelsReceived(audioLevels); - } - - /** - * Delivers the <tt>DTMF</tt> tones. This - * method is meant for use primarily by the transform engine handling - * incoming RTP packets (currently <tt>DtmfTransformEngine</tt>). - * - * @param tone the new tone - * @param end is end or start of tone. - */ - public void fireDTMFEvent(DTMFRtpTone tone, boolean end) - { - Iterator<DTMFListener> iter = dtmfListeners.iterator(); - DTMFToneEvent ev = new DTMFToneEvent(this, tone); - while (iter.hasNext()) - { - DTMFListener listener = iter.next(); - if(end) - listener.dtmfToneReceptionEnded(ev); - else - listener.dtmfToneReceptionStarted(ev); - } - } - - /** - * Releases the resources allocated by this instance in the course of its - * execution and prepares it to be garbage collected. - * - * @see MediaStream#close() - */ - @Override - public void close() - { - super.close(); - - if(dtmfTransfrmEngine != null) - { - dtmfTransfrmEngine = null; - } - - this.audioSystemChangeNotifier.removePropertyChangeListener(this); - } - - /** - * The priority of the audio is 3, which is meant to be higher than - * other threads and higher than the video one. - * @return audio priority. - */ - @Override - protected int getPriority() - { - return 3; - } - - /** - * Receives and reacts to property change events: if the selected device - * (for capture, playback or notifications) has changed, then create or - * recreate the streams in order to use it. - * We want to listen to these events, especially for those generated after - * the audio system has changed. - * - * @param evt The event which may contain a audio system change event. - */ - public void propertyChange(PropertyChangeEvent evt) - { - if (sendStreamsAreCreated) - recreateSendStreams(); - else - start(); - } } diff --git a/src/org/jitsi/impl/neomedia/device/AudioMediaDeviceSession.java b/src/org/jitsi/impl/neomedia/device/AudioMediaDeviceSession.java index 9bab25bb..df731e0c 100644 --- a/src/org/jitsi/impl/neomedia/device/AudioMediaDeviceSession.java +++ b/src/org/jitsi/impl/neomedia/device/AudioMediaDeviceSession.java @@ -57,6 +57,50 @@ protected AudioMediaDeviceSession(AbstractMediaDevice device) super(device); } + /** + * Copies the playback part of a specific <tt>MediaDeviceSession</tt> into + * this instance. + * + * @param deviceSession the <tt>MediaDeviceSession</tt> to copy the playback + * part of into this instance + */ + @Override + public void copyPlayback(MediaDeviceSession deviceSession) + { + AudioMediaDeviceSession amds = (AudioMediaDeviceSession) deviceSession; + + setStreamAudioLevelListener( + amds.streamAudioLevelEffect.getAudioLevelListener()); + setLocalUserAudioLevelListener( + amds.localUserAudioLevelEffect.getAudioLevelListener()); + } + + /** + * Returns the last audio level that was measured by this device session + * for the specified <tt>ssrc</tt>. + * + * @param ssrc the SSRC ID whose last measured audio level we'd like to + * retrieve. + * + * @return the audio level that was last measured for the specified + * <tt>ssrc</tt> or <tt>-1</tt> if no level has been cached for that ID. + */ + public int getLastMeasuredAudioLevel(long ssrc) + { + return -1; + } + + /** + * Returns the last audio level that was measured by the underlying + * mixer for local user. + * + * @return the audio level that was last measured for the local user. + */ + public int getLastMeasuredLocalUserAudioLevel() + { + return -1; + } + /** * Called by {@link MediaDeviceSession#playerControllerUpdate( * ControllerEvent event)} when the player associated with this session's @@ -219,48 +263,4 @@ public void setStreamAudioLevelListener(SimpleAudioLevelListener listener) { this.streamAudioLevelEffect.setAudioLevelListener(listener); } - - /** - * Returns the last audio level that was measured by this device session - * for the specified <tt>ssrc</tt>. - * - * @param ssrc the SSRC ID whose last measured audio level we'd like to - * retrieve. - * - * @return the audio level that was last measured for the specified - * <tt>ssrc</tt> or <tt>-1</tt> if no level has been cached for that ID. - */ - public int getLastMeasuredAudioLevel(long ssrc) - { - return -1; - } - - /** - * Returns the last audio level that was measured by the underlying - * mixer for local user. - * - * @return the audio level that was last measured for the local user. - */ - public int getLastMeasuredLocalUserAudioLevel() - { - return -1; - } - - /** - * Copies the playback part of a specific <tt>MediaDeviceSession</tt> into - * this instance. - * - * @param deviceSession the <tt>MediaDeviceSession</tt> to copy the playback - * part of into this instance - */ - @Override - public void copyPlayback(MediaDeviceSession deviceSession) - { - AudioMediaDeviceSession amds = (AudioMediaDeviceSession) deviceSession; - - this.setStreamAudioLevelListener( - amds.streamAudioLevelEffect.getAudioLevelListener()); - this.setLocalUserAudioLevelListener( - amds.localUserAudioLevelEffect.getAudioLevelListener()); - } } diff --git a/src/org/jitsi/impl/neomedia/device/AudioMixerMediaDevice.java b/src/org/jitsi/impl/neomedia/device/AudioMixerMediaDevice.java index d4d9a43a..697e8000 100644 --- a/src/org/jitsi/impl/neomedia/device/AudioMixerMediaDevice.java +++ b/src/org/jitsi/impl/neomedia/device/AudioMixerMediaDevice.java @@ -1155,8 +1155,10 @@ public void setLocalUserAudioLevelListener(SimpleAudioLevelListener l) // this happens when holding a conversation, stream is muted // and when recreated listener is again set if(!isMute()) + { audioMixerMediaDeviceSession.addLocalUserAudioLevelListener( l); + } } } diff --git a/src/org/jitsi/service/neomedia/AudioMediaStream.java b/src/org/jitsi/service/neomedia/AudioMediaStream.java index e624f531..28ad183b 100644 --- a/src/org/jitsi/service/neomedia/AudioMediaStream.java +++ b/src/org/jitsi/service/neomedia/AudioMediaStream.java @@ -14,20 +14,26 @@ * audio streaming. * * @author Emil Ivov + * @author Lyubomir Marinov */ public interface AudioMediaStream extends MediaStream { /** - * Sets <tt>listener</tt> as the <tt>SimpleAudioLevelListener</tt> - * registered to receive notifications for changes in the levels of the - * party that's at the other end of this stream. + * Registers a listener that would receive notification events if the + * remote party starts sending DTMF tones to us. * - * @param listener the <tt>SimpleAudioLevelListener</tt> that we'd like to - * register or <tt>null</tt> if we want to stop stream audio level - * measurements. + * @param listener the <tt>DTMFListener</tt> that we'd like to register. */ - public void setStreamAudioLevelListener(SimpleAudioLevelListener listener); + public void addDTMFListener(DTMFListener listener); + + /** + * Removes <tt>listener</tt> from the list of <tt>DTMFListener</tt>s + * registered to receive events for incoming DTMF tones. + * + * @param listener the listener that we'd like to unregister + */ + public void removeDTMFListener(DTMFListener listener); /** * Registers <tt>listener</tt> as the <tt>CsrcAudioLevelListener</tt> that @@ -51,6 +57,17 @@ public interface AudioMediaStream public void setLocalUserAudioLevelListener( SimpleAudioLevelListener listener); + /** + * Sets <tt>listener</tt> as the <tt>SimpleAudioLevelListener</tt> + * registered to receive notifications for changes in the levels of the + * party that's at the other end of this stream. + * + * @param listener the <tt>SimpleAudioLevelListener</tt> that we'd like to + * register or <tt>null</tt> if we want to stop stream audio level + * measurements. + */ + public void setStreamAudioLevelListener(SimpleAudioLevelListener listener); + /** * Starts sending the specified <tt>DTMFTone</tt> until the * <tt>stopSendingDTMF()</tt> method is called (Excepts for INBAND DTMF, @@ -75,20 +92,4 @@ public void setLocalUserAudioLevelListener( * @param dtmfMethod the <tt>DTMFMethod</tt> to stop sending. */ public void stopSendingDTMF(DTMFMethod dtmfMethod); - - /** - * Registers a listener that would receive notification events if the - * remote party starts sending DTMF tones to us. - * - * @param listener the <tt>DTMFListener</tt> that we'd like to register. - */ - public void addDTMFListener(DTMFListener listener); - - /** - * Removes <tt>listener</tt> from the list of <tt>DTMFListener</tt>s - * registered to receive events for incoming DTMF tones. - * - * @param listener the listener that we'd like to unregister - */ - public void removeDTMFListener(DTMFListener listener); } -- GitLab