From ae76b616b6422bb7f11467a3e7b954b93bd2860d Mon Sep 17 00:00:00 2001
From: Lyubomir Marinov <lyubomir.marinov@jitsi.org>
Date: Fri, 5 Jul 2013 15:21:15 +0300
Subject: [PATCH] Attempts to optimize and improve the acoustic echo
cancellation (AEC) with Windows Audio Session API (WASAPI).
---
.../impl/neomedia/device/WASAPISystem.java | 8 +
.../protocol/wasapi/AudioCaptureClient.java | 186 +++++--
.../media/protocol/wasapi/DataSource.java | 19 +
.../media/protocol/wasapi/IMediaBuffer.java | 36 ++
.../media/protocol/wasapi/PtrMediaBuffer.java | 110 +++++
.../media/protocol/wasapi/WASAPIStream.java | 458 +++++++++++++-----
6 files changed, 671 insertions(+), 146 deletions(-)
create mode 100644 src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/IMediaBuffer.java
create mode 100644 src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/PtrMediaBuffer.java
diff --git a/src/org/jitsi/impl/neomedia/device/WASAPISystem.java b/src/org/jitsi/impl/neomedia/device/WASAPISystem.java
index a3c35604..db257d45 100644
--- a/src/org/jitsi/impl/neomedia/device/WASAPISystem.java
+++ b/src/org/jitsi/impl/neomedia/device/WASAPISystem.java
@@ -893,6 +893,14 @@ protected String getRendererClassName()
return WASAPIRenderer.class.getName();
}
+ /**
+ * Initializes a new <tt>IMediaObject</tt> instance which represents a Voice
+ * Capture DSP implementing acoustic echo cancellation (AEC).
+ *
+ * @return a new <tt>IMediaObject</tt> instance which represents a Voice
+ * Capture DSP implementing acoustic echo cancellation (AEC)
+ * @throws Exception if initializing the new instance fails
+ */
public long initializeAEC()
throws Exception
{
diff --git a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/AudioCaptureClient.java b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/AudioCaptureClient.java
index 962fb9b2..979b9a40 100644
--- a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/AudioCaptureClient.java
+++ b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/AudioCaptureClient.java
@@ -19,6 +19,18 @@
import org.jitsi.impl.neomedia.jmfext.media.renderer.audio.*;
import org.jitsi.util.*;
+/**
+ * Abstracts the initialization of an <tt>IAudioCaptureClient</tt> instance from
+ * a <tt>MediaLocator</tt>, the input of data from that
+ * <tt>IAudioCaptureClient</tt>, the buffering of that data, the starting,
+ * stopping and closing of the <tt>IAudioCaptureClient</tt>. Allows
+ * {@link WASAPIStream} to simultaneously utilize multiple
+ * <tt>IAudioCaptureClient</tt> instances (e.g. in the case of acoustic echo
+ * cancellation in which audio is input from both the capture and the render
+ * endpoint devices).
+ *
+ * @author Lyubomir Marinov
+ */
public class AudioCaptureClient
{
/**
@@ -35,18 +47,17 @@ public class AudioCaptureClient
= Logger.getLogger(AudioCaptureClient.class);
/**
- * The number of frames to be filled in a <tt>Buffer</tt> in an invocation
- * of {@link #read(Buffer)}. If this instance implements the
- * <tt>PushBufferStream</tt> interface,
+ * The number of audio frames to be filled in a <tt>byte[]</tt> in an
+ * invocation of {@link #read(byte[], int, int)}. The method
* {@link #runInEventHandleCmd(Runnable)} will push via
* {@link BufferTransferHandler#transferData(PushBufferStream)} when
- * {@link #iAudioClient} has made at least that many frames available.
+ * {@link #iAudioClient} has made at least that many audio frames available.
*/
private int bufferFrames;
/**
- * The size/length in bytes of the <tt>Buffer</tt> to be filled in an
- * invocation of {@link #read(Buffer)}.
+ * The size/length in bytes of the <tt>byte[]</tt> to be filled in an
+ * invocation of {@link #read(byte[], int, int)}.
*/
final int bufferSize;
@@ -54,7 +65,8 @@ public class AudioCaptureClient
* The indicator which determines whether the audio stream represented by
* this instance, {@link #iAudioClient} and {@link #iAudioCaptureClient} is
* busy and, consequently, its state should not be modified. For example,
- * the audio stream is busy during the execution of {@link #read(Buffer)}.
+ * the audio stream is busy during the execution of
+ * {@link #read(byte[], int, int)}.
*/
private boolean busy;
@@ -65,21 +77,20 @@ public class AudioCaptureClient
final long devicePeriod;
/**
- * The number of channels which which this <tt>SourceStream</tt> has been
- * connected.
+ * The number of channels of the audio data made available by this instance.
*/
private int dstChannels;
/**
- * The frame size in bytes with which this <tt>SourceStream</tt> has been
- * connected. It is the product of {@link #dstSampleSize} and
+ * The frame size in bytes of the audio data made available by this
+ * instance. It is the product of {@link #dstSampleSize} and
* {@link #dstChannels}.
*/
private int dstFrameSize;
/**
- * The sample size in bytes with which this <tt>SourceStream</tt> has been
- * connected.
+ * The sample size in bytes of the audio data made available by this
+ * instance.
*/
private int dstSampleSize;
@@ -90,9 +101,9 @@ public class AudioCaptureClient
private long eventHandle;
/**
- * The <tt>Runnable</tt> which is scheduled by this <tt>WASAPIStream</tt>
- * and executed by {@link #eventHandleExecutor} and waits for
- * {@link #eventHandle} to be signaled.
+ * The <tt>Runnable</tt> which is scheduled by this instance and executed by
+ * {@link #eventHandleExecutor} and waits for {@link #eventHandle} to be
+ * signaled.
*/
private Runnable eventHandleCmd;
@@ -104,27 +115,37 @@ public class AudioCaptureClient
/**
* The WASAPI <tt>IAudioCaptureClient</tt> obtained from
- * {@link #iAudioClient} which enables this <tt>SourceStream</tt> to read
- * input data from the capture endpoint buffer.
+ * {@link #iAudioClient} which enables this instance to read input data from
+ * the capture endpoint buffer.
*/
private long iAudioCaptureClient;
/**
* The WASAPI <tt>IAudioClient</tt> instance which enables this
- * <tt>SourceStream</tt> to create and initialize an audio stream between
- * this <tt>SourceStream</tt> and the audio engine of the associated audio
+ * <tt>AudioCaptureClient</tt> to create and initialize an audio stream
+ * between this instance and the audio engine of the associated audio
* endpoint device.
*/
private long iAudioClient;
/**
- * The <tt>AudioFormat</tt> of the data output by this
+ * The <tt>AudioFormat</tt> of the data output/made available by this
* <tt>AudioCaptureClient</tt>.
*/
final AudioFormat outFormat;
+ /**
+ * The internal buffer of this instance in which audio data is read from the
+ * associated <tt>IAudioCaptureClient</tt> by the instance and awaits to be
+ * read out of this instance via {@link #read(byte[], int, int)}.
+ */
private byte[] remainder;
+ /**
+ * The number of bytes in {@link #remainder} which represent valid audio
+ * data read from the associated <tt>IAudioCaptureClient</tt> by this
+ * instance.
+ */
private int remainderLength;
/**
@@ -140,14 +161,50 @@ public class AudioCaptureClient
private int srcSampleSize;
/**
- * The indicator which determines whether this <tt>SourceStream</tt> is
- * started i.e. there has been a successful invocation of {@link #start()}
- * without an intervening invocation of {@link #stop()}.
+ * The indicator which determines whether this <tt>AudioCaptureClient</tt>
+ * is started i.e. there has been a successful invocation of
+ * {@link #start()} without an intervening invocation of {@link #stop()}.
*/
private boolean started;
+ /**
+ * The <tt>BufferTransferHandler</tt> which is to be invoked when this
+ * instance has made audio data available to be read via
+ * {@link #read(byte[], int, int)}.
+ * {@link BufferTransferHandler#transferData(PushBufferStream)} will be
+ * called with a <tt>null</tt> argument because <tt>AudioCaptureClient</tt>
+ * does not implement <tt>PushBufferStream</tt> and has rather been
+ * refactored out of a <tt>PushBufferStream</tt> implementation (i.e.
+ * <tt>WASAPIStream</tt>).
+ */
private final BufferTransferHandler transferHandler;
+ /**
+ * Initializes a new <tt>AudioCaptureClient</tt> instance.
+ *
+ * @param audioSystem the <tt>WASAPISystem</tt> instance which has
+ * contributed <tt>locator</tt>
+ * @param locator a <tt>MediaLocator</tt> which identifies the audio
+ * endpoint device to be opened and read by the new instance
+ * @param dataFlow the <tt>AudioSystem.DataFlow</tt> of the audio endpoint
+ * device identified by <tt>locator</tt>. If
+ * <tt>AudioSystem.DataFlow.PLAYBACK</tt> and <tt>streamFlags</tt> includes
+ * {@link WASAPI#AUDCLNT_STREAMFLAGS_LOOPBACK}, allows opening a render
+ * endpoint device in loopback mode and inputing the data that is being
+ * written on that render endpoint device
+ * @param streamFlags zero or more of the <tt>AUDCLNT_STREAMFLAGS_XXX</tt>
+ * flags defined by the <tt>WASAPI</tt> class
+ * @param outFormat the <tt>AudioFormat</tt> of the data to be made
+ * available by the new instance. Eventually, the
+ * <tt>IAudioCaptureClient</tt> to be represented by the new instance may be
+ * initialized with a different <tt>AudioFormat</tt> in which case the new
+ * instance will automatically transcode the data input from the
+ * <tt>IAudioCaptureClient</tt> into the specified <tt>outFormat</tt>.
+ * @param transferHandler the <tt>BufferTransferHandler</tt> to be invoked
+ * when the new instance has made data available to be read via
+ * {@link #read(byte[], int, int)}
+ * @throws Exception if the initialization of the new instance fails
+ */
public AudioCaptureClient(
WASAPISystem audioSystem,
MediaLocator locator,
@@ -294,6 +351,10 @@ public AudioCaptureClient(
}
}
+ /**
+ * Releases the resources acquired by this instance throughout its lifetime
+ * and prepares it to be garbage collected.
+ */
public void close()
{
if (iAudioCaptureClient != 0)
@@ -325,7 +386,26 @@ public void close()
started = false;
}
- private int doRead(byte[] buffer, int offset, int length)
+ /**
+ * Reads audio data from the internal buffer of this instance which has
+ * previously/already been read by this instance from the associated
+ * <tt>IAudioCaptureClient</tt>. Invoked by {@link #read(byte[], int, int)}.
+ *
+ * @param buffer the <tt>byte</tt> array into which the audio data read from
+ * the internal buffer of this instance is to be written
+ * @param offset the offset into <tt>buffer</tt> at which the writing of the
+ * audio data is to begin
+ * @param length the maximum number of bytes in <tt>buffer</tt> starting at
+ * <tt>offset</tt> to be written
+ * @return the number of bytes read from the internal buffer of this
+ * instance and written into the specified <tt>buffer</tt>
+ * @throws IOException if the reading from the internal buffer of this
+ * instance or writing into the specified <tt>buffer</tt> fails
+ */
+ private int doRead(
+ IMediaBuffer iMediaBuffer,
+ byte[] buffer, int offset,
+ int length)
throws IOException
{
int toRead = Math.min(length, remainderLength);
@@ -335,9 +415,14 @@ private int doRead(byte[] buffer, int offset, int length)
read = 0;
else
{
- System.arraycopy(remainder, 0, buffer, offset, toRead);
- popFromRemainder(toRead);
- read = toRead;
+ if (iMediaBuffer == null)
+ {
+ read = toRead;
+ System.arraycopy(remainder, 0, buffer, offset, toRead);
+ }
+ else
+ read = iMediaBuffer.push(remainder, 0, toRead);
+ popFromRemainder(read);
}
return read;
}
@@ -355,8 +440,31 @@ private void popFromRemainder(int length)
= WASAPIRenderer.pop(remainder, remainderLength, length);
}
+ /**
+ * Reads audio data from this instance into a spcific <tt>byte</tt> array.
+ *
+ * @param buffer the <tt>byte</tt> array into which the audio data read from
+ * this instance is to be written
+ * @param offset the offset in <tt>buffer</tt> at which the writing of the
+ * audio data is to start
+ * @param length the maximum number of bytes in <tt>buffer</tt> starting at
+ * <tt>offset</tt> to be written
+ * @return the number of bytes read from this instance and written into the
+ * specified <tt>buffer</tt>
+ * @throws IOException if the reading from this instance or the writing into
+ * the specified <tt>buffer</tt> fails
+ */
public int read(byte[] buffer, int offset, int length)
throws IOException
+ {
+ return read(/* iMediaBuffer */ null, buffer, offset, length);
+ }
+
+ private int read(
+ IMediaBuffer iMediaBuffer,
+ byte[] buffer, int offset,
+ int length)
+ throws IOException
{
String message;
@@ -380,7 +488,7 @@ else if (!started)
try
{
- read = doRead(buffer, offset, length);
+ read = doRead(iMediaBuffer, buffer, offset, length);
cause = null;
}
catch (Throwable t)
@@ -413,6 +521,12 @@ else if (cause instanceof IOException)
return read;
}
+ public int read(IMediaBuffer iMediaBuffer, int length)
+ throws IOException
+ {
+ return read(iMediaBuffer, /* buffer */ null, /* offset */ 0, length);
+ }
+
/**
* Reads from {@link #iAudioCaptureClient} into {@link #remainder} and
* returns a non-<tt>null</tt> <tt>BufferTransferHandler</tt> if this
@@ -593,6 +707,13 @@ private void runInEventHandleCmd(Runnable eventHandleCmd)
}
}
+ /**
+ * Starts the transfer of media from the <tt>IAudioCaptureClient</tt>
+ * identified by the <tt>MediaLocator</tt> with which this instance has
+ * been initialized.
+ *
+ * @throws IOException if the starting of the transfer of media fails
+ */
public synchronized void start()
throws IOException
{
@@ -652,6 +773,13 @@ public void run()
}
}
+ /**
+ * Stops the transfer of media from the <tt>IAudioCaptureClient</tt>
+ * identified by the <tt>MediaLocator</tt> with which this instance has
+ * been initialized.
+ *
+ * @throws IOException if the stopping of the transfer of media fails
+ */
public synchronized void stop()
throws IOException
{
diff --git a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/DataSource.java b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/DataSource.java
index e1ec07fe..098acb83 100644
--- a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/DataSource.java
+++ b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/DataSource.java
@@ -131,6 +131,13 @@ protected void doDisconnect()
}
}
+ /**
+ * Gets the <tt>Format</tt>s of media data supported by the audio endpoint
+ * device associated with this instance.
+ *
+ * @return the <tt>Format</tt>s of media data supported by the audio
+ * endpoint device associated with this instance
+ */
Format[] getIAudioClientSupportedFormats()
{
return
@@ -139,6 +146,18 @@ Format[] getIAudioClientSupportedFormats()
audioSystem.getAECSupportedFormats());
}
+ /**
+ * Gets the <tt>Format</tt>s of media data supported by the audio endpoint
+ * device associated with this instance.
+ *
+ * @param streamIndex the index of the <tt>SourceStream</tt> within the list
+ * of <tt>SourceStream</tt>s of this <tt>DataSource</tt> on behalf of which
+ * the query is being made
+ * @param aecSupportedFormats the list of <tt>AudioFormat</tt>s supported by
+ * the voice capture DMO implementing acoustic echo cancellation
+ * @return the <tt>Format</tt>s of media data supported by the audio
+ * endpoint device associated with this instance
+ */
private Format[] getIAudioClientSupportedFormats(
int streamIndex,
List<AudioFormat> aecSupportedFormats)
diff --git a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/IMediaBuffer.java b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/IMediaBuffer.java
new file mode 100644
index 00000000..56b0b1f7
--- /dev/null
+++ b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/IMediaBuffer.java
@@ -0,0 +1,36 @@
+/*
+ * Jitsi, the OpenSource Java VoIP and Instant Messaging client.
+ *
+ * Distributable under LGPL license.
+ * See terms of license at gnu.org.
+ */
+package org.jitsi.impl.neomedia.jmfext.media.protocol.wasapi;
+
+import java.io.*;
+
+/**
+ * Defines the API of Microsoft's <tt>IMediaBuffer</tt> interface (referred to
+ * as unmanaged) and allows implementing similar abstractions on the Java side
+ * (referred to as managed).
+ *
+ * @author Lyubomir Marinov
+ */
+public interface IMediaBuffer
+{
+ int GetLength()
+ throws IOException;
+
+ int GetMaxLength()
+ throws IOException;
+
+ int pop(byte[] buffer, int offset, int length)
+ throws IOException;
+
+ int push(byte[] buffer, int offset, int length)
+ throws IOException;
+
+ int Release();
+
+ void SetLength(int length)
+ throws IOException;
+}
diff --git a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/PtrMediaBuffer.java b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/PtrMediaBuffer.java
new file mode 100644
index 00000000..a3ea0905
--- /dev/null
+++ b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/PtrMediaBuffer.java
@@ -0,0 +1,110 @@
+/*
+ * Jitsi, the OpenSource Java VoIP and Instant Messaging client.
+ *
+ * Distributable under LGPL license.
+ * See terms of license at gnu.org.
+ */
+package org.jitsi.impl.neomedia.jmfext.media.protocol.wasapi;
+
+import static org.jitsi.impl.neomedia.jmfext.media.protocol.wasapi.VoiceCaptureDSP.*;
+
+import java.io.*;
+
+/**
+ * Implements a managed <tt>IMediaBuffer</tt> which wraps around an unmanaged
+ * <tt>IMediaBuffer</tt>.
+ *
+ * @author Lyubomir Marinov
+ */
+public class PtrMediaBuffer
+ implements IMediaBuffer
+{
+ /**
+ * The unmanaged <tt>IMediaBuffer</tt> represented by this instance.
+ */
+ final long ptr;
+
+ /**
+ * Initializes a new managed <tt>IMediaBuffer</tt> which is to represent a
+ * specific unmanaged <tt>IMediaBuffer</tt>.
+ *
+ * @param ptr the unmanaged <tt>IMediaBuffer</tt> to be represented by the
+ * new instance
+ */
+ public PtrMediaBuffer(long ptr)
+ {
+ if (ptr == 0)
+ throw new IllegalArgumentException("ptr");
+ this.ptr = ptr;
+ }
+
+ public int GetLength()
+ throws IOException
+ {
+ try
+ {
+ return IMediaBuffer_GetLength(ptr);
+ }
+ catch (HResultException hre)
+ {
+ throw new IOException(hre);
+ }
+ }
+
+ public int GetMaxLength()
+ throws IOException
+ {
+ try
+ {
+ return IMediaBuffer_GetMaxLength(ptr);
+ }
+ catch (HResultException hre)
+ {
+ throw new IOException(hre);
+ }
+ }
+
+ public int pop(byte[] buffer, int offset, int length)
+ throws IOException
+ {
+ try
+ {
+ return MediaBuffer_pop(ptr, buffer, offset, length);
+ }
+ catch (HResultException hre)
+ {
+ throw new IOException(hre);
+ }
+ }
+
+ public int push(byte[] buffer, int offset, int length)
+ throws IOException
+ {
+ try
+ {
+ return MediaBuffer_push(ptr, buffer, offset, length);
+ }
+ catch (HResultException hre)
+ {
+ throw new IOException(hre);
+ }
+ }
+
+ public int Release()
+ {
+ return IMediaBuffer_Release(ptr);
+ }
+
+ public void SetLength(int length)
+ throws IOException
+ {
+ try
+ {
+ IMediaBuffer_SetLength(ptr, length);
+ }
+ catch (HResultException hre)
+ {
+ throw new IOException(hre);
+ }
+ }
+}
diff --git a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/WASAPIStream.java b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/WASAPIStream.java
index 47c7b3f9..57dd2f0f 100644
--- a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/WASAPIStream.java
+++ b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/WASAPIStream.java
@@ -39,11 +39,23 @@ public class WASAPIStream
*/
private static Logger logger = Logger.getLogger(WASAPIStream.class);
+ /**
+ * Finds an <tt>AudioFormat</tt> in a specific list of <tt>Format</tt>s
+ * which is as similar to a specific <tt>AudioFormat</tt> as possible.
+ *
+ * @param formats the list of <tt>Format</tt>s into which an
+ * <tt>AudioFormat</tt> as similar to the specified <tt>format</tt> as
+ * possible is to be found
+ * @param format the <tt>AudioFormat</tt> for which a similar
+ * <tt>AudioFormat</tt> is to be found in <tt>formats</tt>
+ * @return an <tt>AudioFormat</tt> which is an element of <tt>formats</tt>
+ * and is as similar to the specified <tt>format</tt> as possible or
+ * <tt>null</tt> if no similarity could be established
+ */
private static AudioFormat findClosestMatch(
Format[] formats,
AudioFormat format)
{
-
// Try to find the very specified format.
AudioFormat match = findFirstMatch(formats, format);
@@ -91,6 +103,17 @@ private static AudioFormat findClosestMatch(
return match;
}
+ /**
+ * Finds the first element of a specific array of <tt>Format</tt>s which
+ * matches in the sense of {@link Format#matches(Format)} a specific
+ * <tt>AudioFormat</tt>.
+ *
+ * @param formats the array of <tt>Format</tt>s which si to be searched
+ * @param format the <tt>AudioFormat</tt> for which a match is to be found
+ * in the specified <tt>formats</tt>
+ * @return the first element of <tt>formats</tt> which matches the specified
+ * <tt>format</tt> or <tt>null</tt> if no match could be found
+ */
private static AudioFormat findFirstMatch(
Format[] formats,
AudioFormat format)
@@ -101,6 +124,29 @@ private static AudioFormat findFirstMatch(
return null;
}
+ /**
+ * Sets the media type of an input or output stream of a specific
+ * <tt>IMediaObject</tt>.
+ *
+ * @param iMediaObject the <tt>IMediaObject</tt> to set the media type of
+ * @param inOrOut <tt>true</tt> if the media type of an input stream of the
+ * specified <tt>iMediaObject</tt> is to be set or <tt>false</tt> if the
+ * media type of an output stream of the specified <tt>iMediaObject</tT> is
+ * to be set
+ * @param dwXXXputStreamIndex the zero-based index of the input or output
+ * stream on the specified <tt>iMediaObject</tt> of which the media type is
+ * to be set
+ * @param audioFormat the <tt>AudioFormat</tt> to be set on the specified
+ * stream of the DMO
+ * @param dwFlags bitwise combination of zero or more
+ * <tt>DMO_SET_TYPEF_XXX</tt> flags (defined by the <tt>VoiceCaptureDSP</tt>
+ * class
+ * @return an <tt>HRESULT</tt> value indicating whether the specified
+ * <tt>audioFormat</tt> is acceptable and/or whether it has been set
+ * successfully
+ * @throws HResultException if setting the media type of the specified
+ * stream of the specified <tt>iMediaObject</tt> fails
+ */
private static int IMediaObject_SetXXXputType(
long iMediaObject,
boolean inOrOut,
@@ -206,6 +252,101 @@ private static int IMediaObject_SetXXXputType(
return hresult;
}
+ /**
+ * Invokes {@link IMediaBuffer#GetLength()} and logs and swallows any
+ * <tt>IOException</tt>.
+ *
+ * @param iMediaBuffer the <tt>IMediaBuffer</tt> on which the method
+ * <tt>GetLength<tt> is to be invoked
+ * @return the length of the specified <tt>iMediaBuffer</tt>. If the method
+ * <tt>GetLength<tt> fails, returns <tt>0</tt>.
+ */
+ private static int maybeIMediaBufferGetLength(IMediaBuffer iMediaBuffer)
+ {
+ int length;
+
+ try
+ {
+ length = iMediaBuffer.GetLength();
+ }
+ catch (IOException ioe)
+ {
+ length = 0;
+ logger.error("IMediaBuffer::GetLength", ioe);
+ }
+ return length;
+ }
+
+ /**
+ * Invokes {@link VoiceCaptureDSP#IMediaBuffer_GetLength(long)} and logs and
+ * swallows any <tt>HResultException</tt>.
+ *
+ * @param iMediaBuffer the <tt>IMediaBuffer</tt> on which the function
+ * <tt>IMediaBuffer_GetLength<tt> is to be invoked
+ * @return the length of the specified <tt>iMediaBuffer</tt>. If the
+ * function <tt>IMediaBuffer_GetLength<tt> fails, returns <tt>0</tt>.
+ */
+ @SuppressWarnings("unused")
+ private static int maybeIMediaBufferGetLength(long iMediaBuffer)
+ {
+ int length;
+
+ try
+ {
+ length = IMediaBuffer_GetLength(iMediaBuffer);
+ }
+ catch (HResultException hre)
+ {
+ length = 0;
+ logger.error("IMediaBuffer_GetLength", hre);
+ }
+ return length;
+ }
+
+ /**
+ * Invokes {@link VoiceCaptureDSP#MediaBuffer_push(long, byte[], int, int)}
+ * on a specific <tt>IMediaBuffer</tt> and logs and swallows any
+ * <tt>HResultException</tT>.
+ *
+ * @param pBuffer the <tt>IMediaBuffer</tt> into which the specified bytes
+ * are to be pushed/written
+ * @param buffer the bytes to be pushed/written into the specified
+ * <tt>pBuffer</tt>
+ * @param offset the offset in <tt>buffer</tt> at which the bytes to be
+ * pushed/written into the specified <tt>pBuffer</tt> start
+ * @param length the number of bytes in <tt>buffer</tt> starting at
+ * <tt>offset</tt> to be pushed/written into the specified <tt>pBuffer</tt>
+ * @return the number of bytes from the specified <tt>buffer</tt>
+ * pushed/written into the specified <tt>pBuffer</tt>
+ */
+ private static int maybeMediaBufferPush(
+ long pBuffer,
+ byte[] buffer, int offset, int length)
+ {
+ int written;
+ Throwable exception;
+
+ try
+ {
+ written = MediaBuffer_push(pBuffer, buffer, offset, length);
+ exception = null;
+ }
+ catch (HResultException hre)
+ {
+ written = 0;
+ exception = hre;
+ }
+ if ((exception != null) || (written != length))
+ {
+ logger.error(
+ "Failed to push/write "
+ + ((written <= 0) ? length : (length - written))
+ + " bytes into an IMediaBuffer.",
+ exception);
+ }
+ return written;
+ }
+
/**
* Throws a new <tt>IOException</tt> instance initialized with a specific
* <tt>String</tt> message and a specific <tt>HResultException</tt> cause.
@@ -238,8 +379,14 @@ static void throwNewIOException(String message, HResultException hre)
private int captureBufferMaxLength;
- private long captureIMediaBuffer;
+ private PtrMediaBuffer captureIMediaBuffer;
+ /**
+ * The indicator which determines whether {@link #capture} and its
+ * associated resources/states are busy and, consequently, should not be
+ * modified. For example, <tt>capture</tt> is busy during the execution of
+ * {@link #read(Buffer)}.
+ */
private boolean captureIsBusy;
/**
@@ -257,6 +404,10 @@ static void throwNewIOException(String message, HResultException hre)
private long iMediaBuffer;
+ /**
+ * The <tt>IMediaObject</tt> reference to the Voice Capture DSP that
+ * implements the acoustic echo cancellation (AEC) feature.
+ */
private long iMediaObject;
/**
@@ -277,8 +428,14 @@ static void throwNewIOException(String message, HResultException hre)
private int renderBufferMaxLength;
- private long renderIMediaBuffer;
+ private PtrMediaBuffer renderIMediaBuffer;
+ /**
+ * The indicator which determines whether {@link #render} and its associated
+ * resources/states are busy and, consequently, should not be modified. For
+ * example, <tt>render</tt> is busy during the execution of
+ * {@link #read(Buffer)}.
+ */
private boolean renderIsBusy;
/**
@@ -304,7 +461,7 @@ public WASAPIStream(DataSource dataSource, FormatControl formatControl)
}
/**
- * Performs optional configuration of the Voice Capture DSP that implements
+ * Performs optional configuration on the Voice Capture DSP that implements
* acoustic echo cancellation (AEC).
*
* @param iPropertyStore a reference to the <tt>IPropertyStore</tt>
@@ -320,6 +477,35 @@ private void configureAEC(long iPropertyStore)
* property to true and override the default settings on the
* MFPKEY_WMAAECMA_FEATR_XXX properties of the Voice Capture DSP.
*/
+ try
+ {
+ if (MFPKEY_WMAAECMA_FEATURE_MODE != 0)
+ {
+ IPropertyStore_SetValue(
+ iPropertyStore,
+ MFPKEY_WMAAECMA_FEATURE_MODE, true);
+ if (MFPKEY_WMAAECMA_FEATR_AES != 0)
+ {
+ IPropertyStore_SetValue(
+ iPropertyStore,
+ MFPKEY_WMAAECMA_FEATR_AES, 2);
+ }
+ if (MFPKEY_WMAAECMA_FEATR_ECHO_LENGTH != 0)
+ {
+ IPropertyStore_SetValue(
+ iPropertyStore,
+ MFPKEY_WMAAECMA_FEATR_ECHO_LENGTH, 256);
+ }
+ }
+ }
+ catch (HResultException hre)
+ {
+ logger.error(
+ "Failed to perform optional configuration on the Voice"
+ + " Capture DSP that implements acoustic echo"
+ + " cancellation (AEC).",
+ hre);
+ }
}
/**
@@ -654,7 +840,8 @@ private void initializeAEC(
processed = new byte[bufferMaxLength * 3];
processedLength = 0;
- this.captureIMediaBuffer = captureIMediaBuffer;
+ this.captureIMediaBuffer
+ = new PtrMediaBuffer(captureIMediaBuffer);
captureIMediaBuffer = 0;
this.dmoOutputDataBuffer = dmoOutputDataBuffer;
dmoOutputDataBuffer = 0;
@@ -662,7 +849,8 @@ private void initializeAEC(
iMediaBuffer = 0;
this.iMediaObject = iMediaObject;
iMediaObject = 0;
- this.renderIMediaBuffer = renderIMediaBuffer;
+ this.renderIMediaBuffer
+ = new PtrMediaBuffer(renderIMediaBuffer);
renderIMediaBuffer = 0;
}
finally
@@ -757,21 +945,29 @@ private void popFromProcessed(int length)
= WASAPIRenderer.pop(processed, processedLength, length);
}
- private int processInput(int dwInputStreamIndex)
+ /**
+ * Inputs audio samples from {@link #capture} or {@link #render} and
+ * delivers them to {@link #iMediaObject} which implements the acoustic echo
+ * cancellation (AEC) feature.
+ *
+ * @param dwInputStreamIndex the zero-based index of the input stream on
+ * <tt>iMediaObject</tt> to which audio samples are to be delivered
+ */
+ private void processInput(int dwInputStreamIndex)
{
- long pBuffer;
+ PtrMediaBuffer oBuffer;
int maxLength;
AudioCaptureClient audioCaptureClient;
switch (dwInputStreamIndex)
{
case 0:
- pBuffer = captureIMediaBuffer;
+ oBuffer = captureIMediaBuffer;
maxLength = captureBufferMaxLength;
audioCaptureClient = capture;
break;
case 1:
- pBuffer = renderIMediaBuffer;
+ oBuffer = renderIMediaBuffer;
maxLength = renderBufferMaxLength;
audioCaptureClient = render;
break;
@@ -779,11 +975,15 @@ private int processInput(int dwInputStreamIndex)
throw new IllegalArgumentException("dwInputStreamIndex");
}
+ long pBuffer = oBuffer.ptr;
int hresult = S_OK;
- int processed = 0;
do
{
+ /*
+ * Attempt to deliver audio samples to the specified input stream
+ * only if it accepts input data at this time.
+ */
int dwFlags;
try
@@ -802,6 +1002,12 @@ private int processInput(int dwInputStreamIndex)
if ((dwFlags & DMO_INPUT_STATUSF_ACCEPT_DATA)
== DMO_INPUT_STATUSF_ACCEPT_DATA)
{
+ /*
+ * The specified input stream reports that it accepts input data
+ * at this time so read audio samples from the associated
+ * AudioCaptureClient and then deliver them to the specified
+ * input stream.
+ */
int toRead;
try
@@ -816,57 +1022,34 @@ private int processInput(int dwInputStreamIndex)
}
if (toRead > 0)
{
- if ((processInputBuffer == null)
- || (processInputBuffer.length < toRead))
- processInputBuffer = new byte[toRead];
-
- int read;
-
+ /*
+ * Read audio samples from the associated
+ * AudioCaptureClient.
+ */
try
{
- read
- = audioCaptureClient.read(
- processInputBuffer,
- 0,
- toRead);
+ audioCaptureClient.read(oBuffer, toRead);
}
catch (IOException ioe)
{
- read = 0;
logger.error(
"Failed to read from IAudioCaptureClient.",
ioe);
}
- if (read > 0)
- {
- int written;
-
- try
- {
- written
- = MediaBuffer_push(
- pBuffer,
- processInputBuffer, 0, read);
- }
- catch (HResultException hre)
- {
- written = 0;
- logger.error("MediaBuffer_push", hre);
- }
- if (written < read)
- {
- logger.error(
- "Failed to push/write "
- + ((written <= 0)
- ? read
- : (read - written))
- + " bytes into an IMediaBuffer.");
- }
- if (written > 0)
- processed += written;
- }
}
+ /*
+ * If the capture endpoint device has delivered audio samples,
+ * they have to go through the acoustic echo cancellation (AEC)
+ * regardless of whether the render endpoint device has
+ * delivered audio samples. Additionally, the duration of the
+ * audio samples delivered by the render has to be the same as
+ * the duration of the audio samples delivered by the capture in
+ * order to have the audio samples delivered by the capture pass
+ * through the voice capture DSO in entirety. To achieve the
+ * above, read from the render endpoint device as many audio
+ * samples as possible and pad with silence if necessary.
+ */
if (dwInputStreamIndex == 1)
{
int length;
@@ -890,19 +1073,16 @@ private int processInput(int dwInputStreamIndex)
|| (processInputBuffer.length < silence))
processInputBuffer = new byte[silence];
Arrays.fill(processInputBuffer, 0, silence, (byte) 0);
- try
- {
- MediaBuffer_push(
- pBuffer,
- processInputBuffer, 0, silence);
- }
- catch (HResultException hre)
- {
- logger.error("MediaBuffer_push", hre);
- }
+ maybeMediaBufferPush(
+ pBuffer,
+ processInputBuffer, 0, silence);
}
}
+ /*
+ * Deliver the audio samples read from the associated
+ * AudioCaptureClient to the specified input stream.
+ */
try
{
hresult
@@ -925,8 +1105,6 @@ private int processInput(int dwInputStreamIndex)
break; // The input stream cannot accept more input data.
}
while (SUCCEEDED(hresult));
-
- return processed;
}
/**
@@ -1047,65 +1225,86 @@ else if (cause instanceof IOException)
while (true);
}
+ /**
+ * Executed by {@link #processThread} and invoked by
+ * {@link #runInProcessThread(Thread)}, inputs audio samples from
+ * {@link #capture} and {@link #render}, delivers them to
+ * {@link #iMediaBuffer} which implements the acoustic echo cancellation
+ * (AEC) features and produces output and caches the output so that it can
+ * be read out of this instance via {@link #read(Buffer)}.
+ *
+ * @return a <tt>BufferTransferHandler</tt> to be invoked if the method has
+ * made available audio samples to be read out of this instance; otherwise,
+ * <tt>null</tt>
+ */
private BufferTransferHandler runInProcessThread()
{
// ProcessInput
processInput(/* capture */ 0);
- processInput(/* render */ 1);
+ /*
+ * If the capture endpoint device has not made any audio samples
+ * available, there is no input to be processed. Moreover, it is
+ * incorrect to input from the render endpoint device in such a case.
+ */
+ if (maybeIMediaBufferGetLength(captureIMediaBuffer) != 0)
+ {
+ processInput(/* render */ 1);
- // ProcessOutput
- int dwStatus = 0;
+ // ProcessOutput
+ int dwStatus = 0;
- do
- {
- try
- {
- IMediaObject_ProcessOutput(
- iMediaObject,
- /* dwFlags */ 0,
- 1,
- dmoOutputDataBuffer);
- }
- catch (HResultException hre)
- {
- dwStatus = 0;
- logger.error("IMediaObject_ProcessOutput", hre);
- }
- try
+ do
{
- int toRead = IMediaBuffer_GetLength(iMediaBuffer);
-
- if (toRead > 0)
+ try
{
- /*
- * Make sure there is enough room in processed to
- * accommodate toRead.
- */
- int toPop = toRead - (processed.length - processedLength);
+ IMediaObject_ProcessOutput(
+ iMediaObject,
+ /* dwFlags */ 0,
+ 1,
+ dmoOutputDataBuffer);
+ }
+ catch (HResultException hre)
+ {
+ dwStatus = 0;
+ logger.error("IMediaObject_ProcessOutput", hre);
+ }
+ try
+ {
+ int toRead = IMediaBuffer_GetLength(iMediaBuffer);
- if (toPop > 0)
- popFromProcessed(toPop);
+ if (toRead > 0)
+ {
+ /*
+ * Make sure there is enough room in processed to
+ * accommodate toRead.
+ */
+ int toPop
+ = toRead - (processed.length - processedLength);
- int read
- = MediaBuffer_pop(
- iMediaBuffer,
- processed, processedLength, toRead);
+ if (toPop > 0)
+ popFromProcessed(toPop);
- if (read > 0)
- processedLength += read;
+ int read
+ = MediaBuffer_pop(
+ iMediaBuffer,
+ processed, processedLength, toRead);
+
+ if (read > 0)
+ processedLength += read;
+ }
+ }
+ catch (HResultException hre)
+ {
+ logger.error(
+ "Failed to read from acoustic echo cancellation"
+ + " (AEC) output IMediaBuffer.",
+ hre);
+ break;
}
}
- catch (HResultException hre)
- {
- logger.error(
- "Failed to read from acoustic echo cancellation (AEC)"
- + " output IMediaBuffer.",
- hre);
- break;
- }
+ while ((dwStatus & DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE)
+ == DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE);
}
- while ((dwStatus & DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE)
- == DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE);
/*
* IMediaObject::ProcessOutput has completed which means that, as far as
@@ -1122,18 +1321,31 @@ private BufferTransferHandler runInProcessThread()
if (SUCCEEDED(hresult))
{
- IMediaBuffer_SetLength(captureIMediaBuffer, 0);
- IMediaBuffer_SetLength(renderIMediaBuffer, 0);
+ captureIMediaBuffer.SetLength(0);
+ renderIMediaBuffer.SetLength(0);
}
}
catch (HResultException hre)
{
- logger.error("IMediaBuffer_SetLength", hre);
+ logger.error("IMediaBuffer_Flush", hre);
+ }
+ catch (IOException ioe)
+ {
+ logger.error("IMediaBuffer::SetLength", ioe);
}
return (processedLength >= bufferMaxLength) ? transferHandler : null;
}
+ /**
+ * Executed by {@link #processThread}, inputs audio samples from
+ * {@link #capture} and {@link #render}, delivers them to
+ * {@link #iMediaBuffer} which implements the acoustic echo cancellation
+ * (AEC) features and produces output and caches the output so that it can
+ * be read out of this instance via {@link #read(Buffer)}.
+ *
+ * @param processThread the <tt>Thread</tt> which is executing the method
+ */
private void runInProcessThread(Thread processThread)
{
try
@@ -1293,6 +1505,10 @@ public synchronized void stop()
processedLength = 0;
}
+ /**
+ * Notifies this instance that audio data has been made available in
+ * {@link #capture}.
+ */
private void transferCaptureData()
{
if (dataSource.aec)
@@ -1311,6 +1527,10 @@ private void transferCaptureData()
}
}
+ /**
+ * Notifies this instance that audio data has been made available in
+ * {@link #render}.
+ */
private void transferRenderData()
{
synchronized (this)
@@ -1336,15 +1556,15 @@ private void uninitializeAEC()
IMediaBuffer_Release(iMediaBuffer);
iMediaBuffer = 0;
}
- if (renderIMediaBuffer != 0)
+ if (renderIMediaBuffer != null)
{
- IMediaBuffer_Release(renderIMediaBuffer);
- renderIMediaBuffer =0;
+ renderIMediaBuffer.Release();
+ renderIMediaBuffer = null;
}
- if (captureIMediaBuffer != 0)
+ if (captureIMediaBuffer != null)
{
- IMediaBuffer_Release(captureIMediaBuffer);
- captureIMediaBuffer = 0;
+ captureIMediaBuffer.Release();
+ captureIMediaBuffer = null;
}
}
@@ -1389,6 +1609,10 @@ private synchronized void waitWhileCaptureIsBusy()
Thread.currentThread().interrupt();
}
+ /**
+ * Waits on this instance while the value of {@link #precessThread} is not
+ * equal to <tt>null</tt>.
+ */
private synchronized void waitWhileProcessThread()
{
while (processThread != null)
--
GitLab