From 774c0c4cf5bbdf3488e4844262765cdbd8055980 Mon Sep 17 00:00:00 2001
From: Lyubomir Marinov <lyubomir.marinov@jitsi.org>
Date: Tue, 9 Jul 2013 15:01:42 +0300
Subject: [PATCH] Works on improving the acoustic echo cancellation (AEC) of
Windows Audio Session API (WASAPI).
---
.../impl/neomedia/device/WASAPISystem.java | 28 +-
.../protocol/AbstractBufferCaptureDevice.java | 1 -
.../protocol/wasapi/AudioCaptureClient.java | 110 +++--
.../media/protocol/wasapi/WASAPIStream.java | 432 ++++++++++++------
.../media/renderer/audio/WASAPIRenderer.java | 9 +-
5 files changed, 391 insertions(+), 189 deletions(-)
diff --git a/src/org/jitsi/impl/neomedia/device/WASAPISystem.java b/src/org/jitsi/impl/neomedia/device/WASAPISystem.java
index db257d45..e72023ca 100644
--- a/src/org/jitsi/impl/neomedia/device/WASAPISystem.java
+++ b/src/org/jitsi/impl/neomedia/device/WASAPISystem.java
@@ -33,6 +33,15 @@ public class WASAPISystem
*/
private static String audioSessionGuid;
+ /**
+ * The default duration of audio data in milliseconds to be read from
+ * <tt>WASAPIStream</tt> in an invocation of
+ * {@link WASAPIStream#read(Buffer)} or to be processed by
+ * <tt>WASAPIRenderer</tt> in an invocation of
+ * {@link WASAPIRenderer#process(Buffer)}.
+ */
+ public static final long DEFAULT_BUFFER_DURATION = 20;
+
/**
* The default interval in milliseconds between periodic processing passes
* by the audio engine.
@@ -971,7 +980,10 @@ public long initializeAEC()
* endpoint device identified by the specified <tt>locator</tt>
* @param streamFlags
* @param eventHandle
- * @param hnsBufferDuration
+ * @param hnsBufferDuration the base of the duration in milliseconds of the
+ * buffer that the audio application will share with the audio engine. If
+ * {@link Format#NOT_SPECIFIED}, the method uses the default interval
+ * between periodic passes by the audio engine.
* @param formats an array of alternative <tt>AudioFormat</tt>s with which
* initialization of a new <tt>IAudioClient</tt> instance is to be
* attempted. The first element of the <tt>formats</tt> array which is
@@ -1107,12 +1119,24 @@ public long initializeIAudioClient(
if (eventHandle != 0)
streamFlags |= AUDCLNT_STREAMFLAGS_EVENTCALLBACK;
+ if (hnsBufferDuration == Format.NOT_SPECIFIED)
+ {
+ hnsBufferDuration
+ = IAudioClient_GetDefaultDevicePeriod(iAudioClient)
+ / 10000;
+ if (hnsBufferDuration <= 1)
+ {
+ hnsBufferDuration
+ = WASAPISystem.DEFAULT_DEVICE_PERIOD;
+ }
+ }
+
int hresult
= IAudioClient_Initialize(
iAudioClient,
shareMode,
streamFlags,
- hnsBufferDuration,
+ 3 * hnsBufferDuration * 10000,
/* hnsPeriodicity */ 0,
waveformatex,
audioSessionGuid);
diff --git a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/AbstractBufferCaptureDevice.java b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/AbstractBufferCaptureDevice.java
index 3663dbb4..2151934d 100644
--- a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/AbstractBufferCaptureDevice.java
+++ b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/AbstractBufferCaptureDevice.java
@@ -19,7 +19,6 @@
import org.jitsi.impl.neomedia.control.*;
import org.jitsi.util.*;
-// disambiguation
/**
* Facilitates the implementations of the <tt>CaptureDevice</tt> and
diff --git a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/AudioCaptureClient.java b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/AudioCaptureClient.java
index 979b9a40..06722ef2 100644
--- a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/AudioCaptureClient.java
+++ b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/AudioCaptureClient.java
@@ -33,12 +33,6 @@
*/
public class AudioCaptureClient
{
- /**
- * The default duration of the audio data in milliseconds to be read from
- * <tt>WASAPIStream</tt> in an invocation of {@link #read(Buffer)}.
- */
- static final long DEFAULT_BUFFER_DURATION = 20;
-
/**
* The <tt>Logger</tt> used by the <tt>AudioCaptureClient</tt> class and its
* instances to log debug information.
@@ -46,6 +40,20 @@ public class AudioCaptureClient
private static final Logger logger
= Logger.getLogger(AudioCaptureClient.class);
+ /**
+ * The internal buffer of this instance in which audio data is read from the
+ * associated <tt>IAudioCaptureClient</tt> by the instance and awaits to be
+ * read out of this instance via {@link #read(byte[], int, int)}.
+ */
+ private byte[] available;
+
+ /**
+ * The number of bytes in {@link #available} which represent valid audio
+ * data read from the associated <tt>IAudioCaptureClient</tt> by this
+ * instance.
+ */
+ private int availableLength;
+
/**
* The number of audio frames to be filled in a <tt>byte[]</tt> in an
* invocation of {@link #read(byte[], int, int)}. The method
@@ -134,20 +142,6 @@ public class AudioCaptureClient
*/
final AudioFormat outFormat;
- /**
- * The internal buffer of this instance in which audio data is read from the
- * associated <tt>IAudioCaptureClient</tt> by the instance and awaits to be
- * read out of this instance via {@link #read(byte[], int, int)}.
- */
- private byte[] remainder;
-
- /**
- * The number of bytes in {@link #remainder} which represent valid audio
- * data read from the associated <tt>IAudioCaptureClient</tt> by this
- * instance.
- */
- private int remainderLength;
-
/**
* The number of channels with which {@link #iAudioClient} has been
* initialized.
@@ -194,6 +188,10 @@ public class AudioCaptureClient
* written on that render endpoint device
* @param streamFlags zero or more of the <tt>AUDCLNT_STREAMFLAGS_XXX</tt>
* flags defined by the <tt>WASAPI</tt> class
+ * @param hnsBufferDuration the base of the duration in milliseconds of the
+ * buffer that the audio application will share with the audio engine. If
+ * {@link Format#NOT_SPECIFIED}, the method uses the default interval
+ * between periodic passes by the audio engine.
* @param outFormat the <tt>AudioFormat</tt> of the data to be made
* available by the new instance. Eventually, the
* <tt>IAudioCaptureClient</tt> to be represented by the new instance may be
@@ -210,6 +208,7 @@ public AudioCaptureClient(
MediaLocator locator,
AudioSystem.DataFlow dataFlow,
int streamFlags,
+ long hnsBufferDuration,
AudioFormat outFormat,
BufferTransferHandler transferHandler)
throws Exception
@@ -227,7 +226,6 @@ public AudioCaptureClient(
* WASAPIRenderer and WASAPIStream. There is no particular
* reason/requirement to do so.
*/
- long hnsBufferDuration = 3 * DEFAULT_BUFFER_DURATION * 10000;
long iAudioClient
= audioSystem.initializeIAudioClient(
locator,
@@ -304,6 +302,8 @@ public AudioCaptureClient(
= WASAPISystem.DEFAULT_DEVICE_PERIOD;
}
this.devicePeriod = devicePeriod;
+ if (hnsBufferDuration == Format.NOT_SPECIFIED)
+ hnsBufferDuration = devicePeriod;
srcChannels = inFormat.getChannels();
srcSampleSize
@@ -315,12 +315,11 @@ public AudioCaptureClient(
dstFrameSize = dstSampleSize * dstChannels;
bufferFrames
- = (int)
- (DEFAULT_BUFFER_DURATION * sampleRate / 1000);
+ = (int) (hnsBufferDuration * sampleRate / 1000);
bufferSize = dstFrameSize * bufferFrames;
- remainder = new byte[numBufferFrames * dstFrameSize];
- remainderLength = 0;
+ available = new byte[numBufferFrames * dstFrameSize];
+ availableLength = 0;
this.eventHandle = eventHandle;
eventHandle = 0;
@@ -381,8 +380,8 @@ public void close()
eventHandle = 0;
}
- remainder = null;
- remainderLength = 0;
+ available = null;
+ availableLength = 0;
started = false;
}
@@ -408,7 +407,7 @@ private int doRead(
int length)
throws IOException
{
- int toRead = Math.min(length, remainderLength);
+ int toRead = Math.min(length, availableLength);
int read;
if (toRead == 0)
@@ -418,30 +417,46 @@ private int doRead(
if (iMediaBuffer == null)
{
read = toRead;
- System.arraycopy(remainder, 0, buffer, offset, toRead);
+ System.arraycopy(available, 0, buffer, offset, toRead);
}
else
- read = iMediaBuffer.push(remainder, 0, toRead);
- popFromRemainder(read);
+ read = iMediaBuffer.push(available, 0, toRead);
+ popFromAvailable(read);
}
return read;
}
/**
- * Pops a specific number of bytes from {@link #remainder}. For example,
- * because such a number of bytes have been read from <tt>remainder</tt> and
+ * Gets the number of bytes of audio samples which have been read from the
+ * associated <tt>IAudioCaptureClient</tt> by this instance and are
+ * available to be read out of this instance via
+ * {@link #read(byte[], int, int)}.
+ *
+ * @return the number of bytes of audio samples which have been read from
+ * the associated <tt>IAudioCaptureClient</tt> by this instance and are
+ * available to be read out of this instance via
+ * <tt>read(byte[], int, int)</tt>
+ */
+ int getAvailableLength()
+ {
+ return availableLength;
+ }
+
+ /**
+ * Pops a specific number of bytes from {@link #available}. For example,
+ * because such a number of bytes have been read from <tt>available</tt> and
* written into a <tt>Buffer</tt>.
*
- * @param length the number of bytes to pop from <tt>remainder</tt>
+ * @param length the number of bytes to pop from <tt>available</tt>
*/
- private void popFromRemainder(int length)
+ private void popFromAvailable(int length)
{
- remainderLength
- = WASAPIRenderer.pop(remainder, remainderLength, length);
+ availableLength
+ = WASAPIRenderer.pop(available, availableLength, length);
}
/**
- * Reads audio data from this instance into a spcific <tt>byte</tt> array.
+ * Reads audio data from this instance into a specific <tt>byte</tt> array.
*
* @param buffer the <tt>byte</tt> array into which the audio data read from
* this instance is to be written
@@ -528,7 +543,7 @@ public int read(IMediaBuffer iMediaBuffer, int length)
}
/**
- * Reads from {@link #iAudioCaptureClient} into {@link #remainder} and
+ * Reads from {@link #iAudioCaptureClient} into {@link #available} and
* returns a non-<tt>null</tt> <tt>BufferTransferHandler</tt> if this
* instance is to push audio data.
*
@@ -553,30 +568,29 @@ private BufferTransferHandler readInEventHandleCmd()
numFramesInNextPacket = 0; // Silence the compiler.
logger.error("IAudioCaptureClient_GetNextPacketSize", hre);
}
-
if (numFramesInNextPacket != 0)
{
int toRead = numFramesInNextPacket * dstFrameSize;
/*
- * Make sure there is enough room in remainder to accommodate
+ * Make sure there is enough room in available to accommodate
* toRead.
*/
- int toPop = toRead - (remainder.length - remainderLength);
+ int toPop = toRead - (available.length - availableLength);
if (toPop > 0)
- popFromRemainder(toPop);
+ popFromAvailable(toPop);
try
{
int read
= IAudioCaptureClient_Read(
iAudioCaptureClient,
- remainder, remainderLength, toRead,
+ available, availableLength, toRead,
srcSampleSize, srcChannels,
dstSampleSize, dstChannels);
- remainderLength += read;
+ availableLength += read;
}
catch (HResultException hre)
{
@@ -584,7 +598,7 @@ private BufferTransferHandler readInEventHandleCmd()
}
}
- return (remainderLength >= bufferSize) ? transferHandler : null;
+ return (availableLength >= bufferSize) ? transferHandler : null;
}
/**
@@ -727,7 +741,7 @@ public synchronized void start()
IAudioClient_Start(iAudioClient);
started = true;
- remainderLength = 0;
+ availableLength = 0;
if ((eventHandle != 0) && (this.eventHandleCmd == null))
{
Runnable eventHandleCmd
@@ -798,7 +812,7 @@ public synchronized void stop()
started = false;
waitWhileEventHandleCmd();
- remainderLength = 0;
+ availableLength = 0;
}
catch (HResultException hre)
{
diff --git a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/WASAPIStream.java b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/WASAPIStream.java
index 57dd2f0f..51b50143 100644
--- a/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/WASAPIStream.java
+++ b/src/org/jitsi/impl/neomedia/jmfext/media/protocol/wasapi/WASAPIStream.java
@@ -33,12 +33,28 @@
public class WASAPIStream
extends AbstractPushBufferStream<DataSource>
{
+ /**
+ * The zero-based index of the input stream of the <tt>IMediaObject</tt>
+ * that represents the Voice Capture DSP implementing the acoustic echo
+ * cancellation (AEC) feature that delivers the audio samples from the
+ * microphone.
+ */
+ private static final int CAPTURE_INPUT_STREAM_INDEX = 0;
+
/**
* The <tt>Logger</tt> used by the <tt>WASAPIStream</tt> class and its
* instances to log debug information.
*/
private static Logger logger = Logger.getLogger(WASAPIStream.class);
+ /**
+ * The zero-based index of the input stream of the <tt>IMediaObject</tt>
+ * that represents the Voice Capture DSP implementing the acoustic echo
+ * cancellation (AEC) feature that delivers the audio samples from the
+ * speaker (line).
+ */
+ private static final int RENDER_INPUT_STREAM_INDEX = 1;
+
/**
* Finds an <tt>AudioFormat</tt> in a specific list of <tt>Format</tt>s
* which is as similar to a specific <tt>AudioFormat</tt> as possible.
@@ -257,9 +273,9 @@ private static int IMediaObject_SetXXXputType(
* <tt>IOException</tt>.
*
* @param iMediaBuffer the <tt>IMediaBuffer</tt> on which the method
- * <tt>GetLength<tt> is to be invoked
+ * <tt>GetLength</tt> is to be invoked
* @return the length of the specified <tt>iMediaBuffer</tt>. If the method
- * <tt>GetLength<tt> fails, returns <tt>0</tt>.
+ * <tt>GetLength</tt> fails, returns <tt>0</tt>.
*/
private static int maybeIMediaBufferGetLength(IMediaBuffer iMediaBuffer)
{
@@ -272,7 +288,7 @@ private static int maybeIMediaBufferGetLength(IMediaBuffer iMediaBuffer)
catch (IOException ioe)
{
length = 0;
- logger.error("IMediaBuffer::GetLength", ioe);
+ logger.error("IMediaBuffer.GetLength", ioe);
}
return length;
}
@@ -282,9 +298,9 @@ private static int maybeIMediaBufferGetLength(IMediaBuffer iMediaBuffer)
* swallows any <tt>HResultException</tt>.
*
* @param iMediaBuffer the <tt>IMediaBuffer</tt> on which the function
- * <tt>IMediaBuffer_GetLength<tt> is to be invoked
+ * <tt>IMediaBuffer_GetLength</tt> is to be invoked
* @return the length of the specified <tt>iMediaBuffer</tt>. If the
- * function <tt>IMediaBuffer_GetLength<tt> fails, returns <tt>0</tt>.
+ * function <tt>IMediaBuffer_GetLength</tt> fails, returns <tt>0</tt>.
*/
@SuppressWarnings("unused")
private static int maybeIMediaBufferGetLength(long iMediaBuffer)
@@ -389,6 +405,8 @@ static void throwNewIOException(String message, HResultException hre)
*/
private boolean captureIsBusy;
+ private double captureNanosPerByte;
+
/**
* The length in milliseconds of the interval between successive, periodic
* processing passes by the audio engine on the data in the endpoint buffer.
@@ -428,6 +446,8 @@ static void throwNewIOException(String message, HResultException hre)
private int renderBufferMaxLength;
+ private double renderBytesPerNano;
+
private PtrMediaBuffer renderIMediaBuffer;
/**
@@ -438,6 +458,12 @@ static void throwNewIOException(String message, HResultException hre)
*/
private boolean renderIsBusy;
+ /**
+ * The indicator which determines whether no reading from {@link #render} is
+ * to be performed until it reaches a certain threshold of availability.
+ */
+ private boolean replenishRender;
+
/**
* The indicator which determines whether this <tt>SourceStream</tt> is
* started i.e. there has been a successful invocation of {@link #start()}
@@ -460,6 +486,27 @@ public WASAPIStream(DataSource dataSource, FormatControl formatControl)
super(dataSource, formatControl);
}
+ /**
+ * Computes/determines the duration in nanoseconds of audio samples which
+ * are represented by a specific number of bytes and which are in encoded in
+ * the <tt>outFormat</tt> of {@link #capture}.
+ *
+ * @param length the number of bytes comprising the audio samples of which
+ * the duration in nanoseconds is to be computed/determined
+ * @return the duration in nanoseconds of audio samples which are
+ * represented by the specified number of bytes and which are encoded in the
+ * <tt>outFormat</tt> of <tt>capture</tt>
+ */
+ private long computeCaptureDuration(int length)
+ {
+ return (long) (length * captureNanosPerByte);
+ }
+
+ private int computeRenderLength(long duration)
+ {
+ return (int) (duration * renderBytesPerNano);
+ }
+
/**
* Performs optional configuration on the Voice Capture DSP that implements
* acoustic echo cancellation (AEC).
@@ -716,11 +763,12 @@ private void initializeAEC(
}
try
{
+ int dwInputStreamIndex = CAPTURE_INPUT_STREAM_INDEX;
int hresult
= IMediaObject_SetXXXputType(
iMediaObject,
/* IMediaObject_SetInputType */ true,
- /* dwInputStreamIndex */ 0,
+ dwInputStreamIndex,
inFormat0,
/* dwFlags */ 0);
@@ -728,22 +776,23 @@ private void initializeAEC(
{
throw new HResultException(
hresult,
- "IMediaObject_SetInputType, dwOutputStreamIndex 0, "
- + inFormat0);
+ "IMediaObject_SetInputType, dwInputStreamIndex "
+ + dwInputStreamIndex + ", " + inFormat0);
}
+ dwInputStreamIndex = RENDER_INPUT_STREAM_INDEX;
hresult
= IMediaObject_SetXXXputType(
iMediaObject,
/* IMediaObject_SetInputType */ true,
- /* dwInputStreamIndex */ 1,
+ dwInputStreamIndex,
inFormat1,
/* dwFlags */ 0);
if (FAILED(hresult))
{
throw new HResultException(
hresult,
- "IMediaObject_SetInputType, dwOutputStreamIndex 1, "
- + inFormat1);
+ "IMediaObject_SetInputType, dwInputStreamIndex "
+ + dwInputStreamIndex + ", " + inFormat1);
}
hresult
= IMediaObject_SetXXXputType(
@@ -804,9 +853,8 @@ private void initializeAEC(
* outFormat.getChannels();
int outFrames
= (int)
- (AudioCaptureClient.DEFAULT_BUFFER_DURATION
- * ((int) outFormat.getSampleRate())
- / 1000);
+ (WASAPISystem.DEFAULT_BUFFER_DURATION
+ * ((int) outFormat.getSampleRate()) / 1000);
long iMediaBuffer
= MediaBuffer_alloc(outFrameSize * outFrames);
@@ -852,6 +900,45 @@ private void initializeAEC(
this.renderIMediaBuffer
= new PtrMediaBuffer(renderIMediaBuffer);
renderIMediaBuffer = 0;
+
+ /*
+ * Prepare to be ready to compute/determine the
+ * duration in nanoseconds of a specific number
+ * of bytes representing audio samples encoded
+ * in the outFormat of capture.
+ */
+ {
+ AudioFormat af = capture.outFormat;
+ double sampleRate = af.getSampleRate();
+ int sampleSizeInBits
+ = af.getSampleSizeInBits();
+ int channels = af.getChannels();
+
+ captureNanosPerByte
+ = (8d * 1000d * 1000d * 1000d)
+ / (sampleRate
+ * sampleSizeInBits
+ * channels);
+ }
+ /*
+ * Prepare to be ready to compute/determine the
+ * number of bytes representing a specific
+ * duration in nanoseconds of audio samples
+ * encoded in the outFormat of render.
+ */
+ {
+ AudioFormat af = render.outFormat;
+ double sampleRate = af.getSampleRate();
+ int sampleSizeInBits
+ = af.getSampleSizeInBits();
+ int channels = af.getChannels();
+
+ renderBytesPerNano
+ = (sampleRate
+ * sampleSizeInBits
+ * channels)
+ / (8d * 1000d * 1000d * 1000d);
+ }
}
finally
{
@@ -893,21 +980,28 @@ private void initializeAEC(
private void initializeCapture(MediaLocator locator, AudioFormat format)
throws Exception
{
+ long hnsBufferDuration
+ = dataSource.aec
+ ? Format.NOT_SPECIFIED
+ : WASAPISystem.DEFAULT_BUFFER_DURATION;
+ BufferTransferHandler transferHandler
+ = new BufferTransferHandler()
+ {
+ public void transferData(PushBufferStream stream)
+ {
+ transferCaptureData();
+ }
+ };
+
capture
= new AudioCaptureClient(
dataSource.audioSystem,
locator,
AudioSystem.DataFlow.CAPTURE,
/* streamFlags */ 0,
+ hnsBufferDuration,
format,
- new BufferTransferHandler()
- {
- public void transferData(
- PushBufferStream stream)
- {
- transferCaptureData();
- }
- });
+ transferHandler);
bufferSize = capture.bufferSize;
devicePeriod = capture.devicePeriod;
}
@@ -915,21 +1009,23 @@ public void transferData(
private void initializeRender(final MediaLocator locator, AudioFormat format)
throws Exception
{
+ /*
+ * XXX The method transferRenderData does not read any data from render
+ * at this time. If the transferHandler (which will normally invoke
+ * transferRenderData) was non-null, it would cause excessive CPU use.
+ */
+ BufferTransferHandler transferHandler = null;
+
render
= new AudioCaptureClient(
dataSource.audioSystem,
locator,
AudioSystem.DataFlow.PLAYBACK,
WASAPI.AUDCLNT_STREAMFLAGS_LOOPBACK,
+ /* hnsBufferDuration */ Format.NOT_SPECIFIED,
format,
- new BufferTransferHandler()
- {
- public void transferData(
- PushBufferStream stream)
- {
- transferRenderData();
- }
- });
+ transferHandler);
+ replenishRender = true;
}
/**
@@ -952,28 +1048,34 @@ private void popFromProcessed(int length)
*
* @param dwInputStreamIndex the zero-based index of the input stream on
* <tt>iMediaObject</tt> to which audio samples are to be delivered
+ * @param maxLength the maximum number of bytes to the delivered through the
+ * specified input stream. Ignored if negative or greater than the actual
+ * capacity/maximum length of the <tt>IMediaBuffer</tt> associated with the specified
+ * <tt>dwInputStreamIndex</tt>.
*/
- private void processInput(int dwInputStreamIndex)
+ private void processInput(int dwInputStreamIndex, int maxLength)
{
PtrMediaBuffer oBuffer;
- int maxLength;
+ int bufferMaxLength;
AudioCaptureClient audioCaptureClient;
switch (dwInputStreamIndex)
{
- case 0:
+ case CAPTURE_INPUT_STREAM_INDEX:
oBuffer = captureIMediaBuffer;
- maxLength = captureBufferMaxLength;
+ bufferMaxLength = captureBufferMaxLength;
audioCaptureClient = capture;
break;
- case 1:
+ case RENDER_INPUT_STREAM_INDEX:
oBuffer = renderIMediaBuffer;
- maxLength = renderBufferMaxLength;
+ bufferMaxLength = renderBufferMaxLength;
audioCaptureClient = render;
break;
default:
throw new IllegalArgumentException("dwInputStreamIndex");
}
+ if ((maxLength < 0) || (maxLength > bufferMaxLength))
+ maxLength = bufferMaxLength;
long pBuffer = oBuffer.ptr;
int hresult = S_OK;
@@ -1008,17 +1110,31 @@ private void processInput(int dwInputStreamIndex)
* AudioCaptureClient and then deliver them to the specified
* input stream.
*/
- int toRead;
+ int toRead = Format.NOT_SPECIFIED;
- try
+ if ((dwInputStreamIndex == RENDER_INPUT_STREAM_INDEX)
+ && replenishRender)
{
- toRead = maxLength - IMediaBuffer_GetLength(pBuffer);
+ int replenishThreshold = renderBufferMaxLength;
+
+ if (audioCaptureClient.getAvailableLength()
+ < replenishThreshold)
+ toRead = 0;
+ else
+ replenishRender = false;
}
- catch (HResultException hre)
+ if (toRead == Format.NOT_SPECIFIED)
{
- hresult = hre.getHResult();
- toRead = 0;
- logger.error("IMediaBuffer_GetLength", hre);
+ try
+ {
+ toRead = maxLength - IMediaBuffer_GetLength(pBuffer);
+ }
+ catch (HResultException hre)
+ {
+ hresult = hre.getHResult();
+ toRead = 0;
+ logger.error("IMediaBuffer_GetLength", hre);
+ }
}
if (toRead > 0)
{
@@ -1028,7 +1144,11 @@ private void processInput(int dwInputStreamIndex)
*/
try
{
- audioCaptureClient.read(oBuffer, toRead);
+ int read = audioCaptureClient.read(oBuffer, toRead);
+
+ if ((dwInputStreamIndex == RENDER_INPUT_STREAM_INDEX)
+ && (read == 0))
+ replenishRender = true;
}
catch (IOException ioe)
{
@@ -1050,7 +1170,7 @@ private void processInput(int dwInputStreamIndex)
* above, read from the render endpoint device as many audio
* samples as possible and pad with silence if necessary.
*/
- if (dwInputStreamIndex == 1)
+ if (dwInputStreamIndex == RENDER_INPUT_STREAM_INDEX)
{
int length;
@@ -1100,6 +1220,7 @@ private void processInput(int dwInputStreamIndex)
if (hresult != DMO_E_NOTACCEPTING)
logger.error("IMediaObject_ProcessInput", hre);
}
+ break; // XXX We risk a busy wait unless we break here.
}
else
break; // The input stream cannot accept more input data.
@@ -1107,6 +1228,67 @@ private void processInput(int dwInputStreamIndex)
while (SUCCEEDED(hresult));
}
+ /**
+ * Invokes <tt>IMediaObject::ProcessOutput</tt> on {@link #iMediaObject}
+ * that represents the Voice Capture DSP implementing the acoustic echo
+ * cancellation (AEC) feature.
+ */
+ private void processOutput()
+ {
+ int dwStatus = 0;
+
+ do
+ {
+ try
+ {
+ IMediaObject_ProcessOutput(
+ iMediaObject,
+ /* dwFlags */ 0,
+ 1,
+ dmoOutputDataBuffer);
+ }
+ catch (HResultException hre)
+ {
+ dwStatus = 0;
+ logger.error("IMediaObject_ProcessOutput", hre);
+ }
+ try
+ {
+ int toRead = IMediaBuffer_GetLength(iMediaBuffer);
+
+ if (toRead > 0)
+ {
+ /*
+ * Make sure there is enough room in processed to
+ * accommodate toRead.
+ */
+ int toPop = toRead - (processed.length - processedLength);
+
+ if (toPop > 0)
+ popFromProcessed(toPop);
+
+ int read
+ = MediaBuffer_pop(
+ iMediaBuffer,
+ processed, processedLength, toRead);
+
+ if (read > 0)
+ processedLength += read;
+ }
+ }
+ catch (HResultException hre)
+ {
+ logger.error(
+ "Failed to read from acoustic echo cancellation (AEC)"
+ + " output IMediaBuffer.",
+ hre);
+ break;
+ }
+ }
+ while ((dwStatus & DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE)
+ == DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE);
+ }
+
/**
* {@inheritDoc}
*/
@@ -1206,6 +1388,15 @@ public void read(Buffer buffer)
buffer.setLength(length);
break;
}
+ else
+ {
+ /*
+ * TODO The implementation of PushBufferStream.read(Buffer)
+ * should not block, it should return with whatever is
+ * available.
+ */
+ yield();
+ }
}
else
{
@@ -1239,102 +1430,77 @@ else if (cause instanceof IOException)
*/
private BufferTransferHandler runInProcessThread()
{
- // ProcessInput
- processInput(/* capture */ 0);
- /*
- * If the capture endpoint device has not made any audio samples
- * available, there is no input to be processed. Moreover, it is
- * incorrect to input from the render endpoint device in such a case.
- */
- if (maybeIMediaBufferGetLength(captureIMediaBuffer) != 0)
+ int captureMaxLength = this.captureBufferMaxLength;
+
+ do
{
- processInput(/* render */ 1);
+ processInput(CAPTURE_INPUT_STREAM_INDEX, captureMaxLength);
- // ProcessOutput
- int dwStatus = 0;
+ /*
+ * If the capture endpoint device has not made any audio samples
+ * available, there is no input to be processed. Moreover, inputting
+ * from the render endpoint device in such a case will be
+ * inappropriate because it will (repeatedly) introduce random skew
+ * in the audio delivered by the render endpoint device.
+ */
+ int captureLength = maybeIMediaBufferGetLength(captureIMediaBuffer);
+ boolean flush;
- do
+ if (captureLength < captureMaxLength)
+ flush = false;
+ else
{
- try
- {
- IMediaObject_ProcessOutput(
- iMediaObject,
- /* dwFlags */ 0,
- 1,
- dmoOutputDataBuffer);
- }
- catch (HResultException hre)
- {
- dwStatus = 0;
- logger.error("IMediaObject_ProcessOutput", hre);
- }
- try
- {
- int toRead = IMediaBuffer_GetLength(iMediaBuffer);
+ int renderMaxLength
+ = computeRenderLength(
+ computeCaptureDuration(captureLength));
- if (toRead > 0)
- {
- /*
- * Make sure there is enough room in processed to
- * accommodate toRead.
- */
- int toPop
- = toRead - (processed.length - processedLength);
-
- if (toPop > 0)
- popFromProcessed(toPop);
+ processInput(RENDER_INPUT_STREAM_INDEX, renderMaxLength);
- int read
- = MediaBuffer_pop(
- iMediaBuffer,
- processed, processedLength, toRead);
-
- if (read > 0)
- processedLength += read;
- }
- }
- catch (HResultException hre)
- {
- logger.error(
- "Failed to read from acoustic echo cancellation"
- + " (AEC) output IMediaBuffer.",
- hre);
- break;
- }
+ processOutput();
+ flush = true;
}
- while ((dwStatus & DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE)
- == DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE);
- }
- /*
- * IMediaObject::ProcessOutput has completed which means that, as far as
- * it is concerned, it does not have any input data to process. Make
- * sure that the states of the IMediaBuffer instances are in accord.
- */
- try
- {
/*
- * XXX Make sure that the IMediaObject releases any IMediaBuffer
- * references it holds.
+ * IMediaObject::ProcessOutput has completed which means that, as
+ * far as it is concerned, it does not have any input data to
+ * process. Make sure that the states of the IMediaBuffer instances
+ * are in accord.
*/
- int hresult = IMediaObject_Flush(iMediaObject);
+ try
+ {
+ /*
+ * XXX Make sure that the IMediaObject releases any IMediaBuffer
+ * references it holds.
+ */
+ if (SUCCEEDED(IMediaObject_Flush(iMediaObject)) && flush)
+ {
+ captureIMediaBuffer.SetLength(0);
+ renderIMediaBuffer.SetLength(0);
+ }
+ }
+ catch (HResultException hre)
+ {
+ logger.error("IMediaBuffer_Flush", hre);
+ }
+ catch (IOException ioe)
+ {
+ logger.error("IMediaBuffer.SetLength", ioe);
+ }
- if (SUCCEEDED(hresult))
+ if (!flush)
{
- captureIMediaBuffer.SetLength(0);
- renderIMediaBuffer.SetLength(0);
+ BufferTransferHandler transferHandler = this.transferHandler;
+
+ if ((transferHandler != null)
+ && (processedLength >= bufferMaxLength))
+ return transferHandler;
+ else
+ break;
}
}
- catch (HResultException hre)
- {
- logger.error("IMediaBuffer_Flush", hre);
- }
- catch (IOException ioe)
- {
- logger.error("IMediaBuffer::SetLength", ioe);
- }
+ while (true);
- return (processedLength >= bufferMaxLength) ? transferHandler : null;
+ return null;
}
/**
@@ -1498,6 +1664,7 @@ public synchronized void stop()
{
waitWhileRenderIsBusy();
render.stop();
+ replenishRender = true;
}
started = false;
@@ -1531,12 +1698,17 @@ private void transferCaptureData()
* Notifies this instance that audio data has been made available in
* {@link #render}.
*/
+ @SuppressWarnings("unused")
private void transferRenderData()
{
- synchronized (this)
- {
- notifyAll();
- }
+ /*
+ * This is a CaptureDevice and its goal is to push the audio samples
+ * delivered by the capture endpoint device out in their entirety. When
+ * the render endpoint device pushes and whether it pushes frequently
+ * and sufficiently enough to stay in sync with the capture endpoint
+ * device for the purposes of the acoustic echo cancellation (AEC) is a
+ * separate question.
+ */
}
private void uninitializeAEC()
diff --git a/src/org/jitsi/impl/neomedia/jmfext/media/renderer/audio/WASAPIRenderer.java b/src/org/jitsi/impl/neomedia/jmfext/media/renderer/audio/WASAPIRenderer.java
index b795421f..64543bcd 100644
--- a/src/org/jitsi/impl/neomedia/jmfext/media/renderer/audio/WASAPIRenderer.java
+++ b/src/org/jitsi/impl/neomedia/jmfext/media/renderer/audio/WASAPIRenderer.java
@@ -32,12 +32,6 @@
public class WASAPIRenderer
extends AbstractAudioRenderer<WASAPISystem>
{
- /**
- * The default base of the endpoint buffer capacity in milliseconds to
- * initialize new <tt>IAudioClient</tt> instances with.
- */
- private static final long DEFAULT_BUFFER_DURATION = 60;
-
/**
* The <tt>Logger</tt> used by the <tt>WASAPIRenderer</tt> class and its
* instances to log debug information.
@@ -316,14 +310,13 @@ public synchronized void open()
try
{
- long hnsBufferDuration = DEFAULT_BUFFER_DURATION * 10000;
long iAudioClient
= audioSystem.initializeIAudioClient(
locator,
dataFlow,
/* streamFlags */ 0,
eventHandle,
- hnsBufferDuration,
+ WASAPISystem.DEFAULT_BUFFER_DURATION,
formats);
if (iAudioClient == 0)
--
GitLab