Skip to content
Snippets Groups Projects
Commit ae76b616 authored by Lyubomir Marinov's avatar Lyubomir Marinov
Browse files

Attempts to optimize and improve the acoustic echo cancellation (AEC) with...

Attempts to optimize and improve the acoustic echo cancellation (AEC) with Windows Audio Session API (WASAPI).
parent 6105d636
No related branches found
No related tags found
No related merge requests found
...@@ -893,6 +893,14 @@ protected String getRendererClassName() ...@@ -893,6 +893,14 @@ protected String getRendererClassName()
return WASAPIRenderer.class.getName(); return WASAPIRenderer.class.getName();
} }
/**
* Initializes a new <tt>IMediaObject</tt> instance which represents a Voice
* Capture DSP implementing acoustic echo cancellation (AEC).
*
* @return a new <tt>IMediaObject</tt> instance which represents a Voice
* Capture DSP implementing acoustic echo cancellation (AEC)
* @throws Exception if initializing the new instance fails
*/
public long initializeAEC() public long initializeAEC()
throws Exception throws Exception
{ {
......
...@@ -19,6 +19,18 @@ ...@@ -19,6 +19,18 @@
import org.jitsi.impl.neomedia.jmfext.media.renderer.audio.*; import org.jitsi.impl.neomedia.jmfext.media.renderer.audio.*;
import org.jitsi.util.*; import org.jitsi.util.*;
/**
* Abstracts the initialization of an <tt>IAudioCaptureClient</tt> instance from
* a <tt>MediaLocator</tt>, the input of data from that
* <tt>IAudioCaptureClient</tt>, the buffering of that data, the starting,
* stopping and closing of the <tt>IAudioCaptureClient</tt>. Allows
* {@link WASAPIStream} to simultaneously utilize multiple
* <tt>IAudioCaptureClient</tt> instances (e.g. in the case of acoustic echo
* cancellation in which audio is input from both the capture and the render
* endpoint devices).
*
* @author Lyubomir Marinov
*/
public class AudioCaptureClient public class AudioCaptureClient
{ {
/** /**
...@@ -35,18 +47,17 @@ public class AudioCaptureClient ...@@ -35,18 +47,17 @@ public class AudioCaptureClient
= Logger.getLogger(AudioCaptureClient.class); = Logger.getLogger(AudioCaptureClient.class);
/** /**
* The number of frames to be filled in a <tt>Buffer</tt> in an invocation * The number of audio frames to be filled in a <tt>byte[]</tt> in an
* of {@link #read(Buffer)}. If this instance implements the * invocation of {@link #read(byte[], int, int)}. The method
* <tt>PushBufferStream</tt> interface,
* {@link #runInEventHandleCmd(Runnable)} will push via * {@link #runInEventHandleCmd(Runnable)} will push via
* {@link BufferTransferHandler#transferData(PushBufferStream)} when * {@link BufferTransferHandler#transferData(PushBufferStream)} when
* {@link #iAudioClient} has made at least that many frames available. * {@link #iAudioClient} has made at least that many audio frames available.
*/ */
private int bufferFrames; private int bufferFrames;
/** /**
* The size/length in bytes of the <tt>Buffer</tt> to be filled in an * The size/length in bytes of the <tt>byte[]</tt> to be filled in an
* invocation of {@link #read(Buffer)}. * invocation of {@link #read(byte[], int, int)}.
*/ */
final int bufferSize; final int bufferSize;
...@@ -54,7 +65,8 @@ public class AudioCaptureClient ...@@ -54,7 +65,8 @@ public class AudioCaptureClient
* The indicator which determines whether the audio stream represented by * The indicator which determines whether the audio stream represented by
* this instance, {@link #iAudioClient} and {@link #iAudioCaptureClient} is * this instance, {@link #iAudioClient} and {@link #iAudioCaptureClient} is
* busy and, consequently, its state should not be modified. For example, * busy and, consequently, its state should not be modified. For example,
* the audio stream is busy during the execution of {@link #read(Buffer)}. * the audio stream is busy during the execution of
* {@link #read(byte[], int, int)}.
*/ */
private boolean busy; private boolean busy;
...@@ -65,21 +77,20 @@ public class AudioCaptureClient ...@@ -65,21 +77,20 @@ public class AudioCaptureClient
final long devicePeriod; final long devicePeriod;
/** /**
* The number of channels which which this <tt>SourceStream</tt> has been * The number of channels of the audio data made available by this instance.
* connected.
*/ */
private int dstChannels; private int dstChannels;
/** /**
* The frame size in bytes with which this <tt>SourceStream</tt> has been * The frame size in bytes of the audio data made available by this
* connected. It is the product of {@link #dstSampleSize} and * instance. It is the product of {@link #dstSampleSize} and
* {@link #dstChannels}. * {@link #dstChannels}.
*/ */
private int dstFrameSize; private int dstFrameSize;
/** /**
* The sample size in bytes with which this <tt>SourceStream</tt> has been * The sample size in bytes of the audio data made available by this
* connected. * instance.
*/ */
private int dstSampleSize; private int dstSampleSize;
...@@ -90,9 +101,9 @@ public class AudioCaptureClient ...@@ -90,9 +101,9 @@ public class AudioCaptureClient
private long eventHandle; private long eventHandle;
/** /**
* The <tt>Runnable</tt> which is scheduled by this <tt>WASAPIStream</tt> * The <tt>Runnable</tt> which is scheduled by this instance and executed by
* and executed by {@link #eventHandleExecutor} and waits for * {@link #eventHandleExecutor} and waits for {@link #eventHandle} to be
* {@link #eventHandle} to be signaled. * signaled.
*/ */
private Runnable eventHandleCmd; private Runnable eventHandleCmd;
...@@ -104,27 +115,37 @@ public class AudioCaptureClient ...@@ -104,27 +115,37 @@ public class AudioCaptureClient
/** /**
* The WASAPI <tt>IAudioCaptureClient</tt> obtained from * The WASAPI <tt>IAudioCaptureClient</tt> obtained from
* {@link #iAudioClient} which enables this <tt>SourceStream</tt> to read * {@link #iAudioClient} which enables this instance to read input data from
* input data from the capture endpoint buffer. * the capture endpoint buffer.
*/ */
private long iAudioCaptureClient; private long iAudioCaptureClient;
/** /**
* The WASAPI <tt>IAudioClient</tt> instance which enables this * The WASAPI <tt>IAudioClient</tt> instance which enables this
* <tt>SourceStream</tt> to create and initialize an audio stream between * <tt>AudioCaptureClient</tt> to create and initialize an audio stream
* this <tt>SourceStream</tt> and the audio engine of the associated audio * between this instance and the audio engine of the associated audio
* endpoint device. * endpoint device.
*/ */
private long iAudioClient; private long iAudioClient;
/** /**
* The <tt>AudioFormat</tt> of the data output by this * The <tt>AudioFormat</tt> of the data output/made available by this
* <tt>AudioCaptureClient</tt>. * <tt>AudioCaptureClient</tt>.
*/ */
final AudioFormat outFormat; final AudioFormat outFormat;
/**
* The internal buffer of this instance in which audio data is read from the
* associated <tt>IAudioCaptureClient</tt> by the instance and awaits to be
* read out of this instance via {@link #read(byte[], int, int)}.
*/
private byte[] remainder; private byte[] remainder;
/**
* The number of bytes in {@link #remainder} which represent valid audio
* data read from the associated <tt>IAudioCaptureClient</tt> by this
* instance.
*/
private int remainderLength; private int remainderLength;
/** /**
...@@ -140,14 +161,50 @@ public class AudioCaptureClient ...@@ -140,14 +161,50 @@ public class AudioCaptureClient
private int srcSampleSize; private int srcSampleSize;
/** /**
* The indicator which determines whether this <tt>SourceStream</tt> is * The indicator which determines whether this <tt>AudioCaptureClient</tt>
* started i.e. there has been a successful invocation of {@link #start()} * is started i.e. there has been a successful invocation of
* without an intervening invocation of {@link #stop()}. * {@link #start()} without an intervening invocation of {@link #stop()}.
*/ */
private boolean started; private boolean started;
/**
* The <tt>BufferTransferHandler</tt> which is to be invoked when this
* instance has made audio data available to be read via
* {@link #read(byte[], int, int)}.
* {@link BufferTransferHandler#transferData(PushBufferStream)} will be
* called with a <tt>null</tt> argument because <tt>AudioCaptureClient</tt>
* does not implement <tt>PushBufferStream</tt> and has rather been
* refactored out of a <tt>PushBufferStream</tt> implementation (i.e.
* <tt>WASAPIStream</tt>).
*/
private final BufferTransferHandler transferHandler; private final BufferTransferHandler transferHandler;
/**
* Initializes a new <tt>AudioCaptureClient</tt> instance.
*
* @param audioSystem the <tt>WASAPISystem</tt> instance which has
* contributed <tt>locator</tt>
* @param locator a <tt>MediaLocator</tt> which identifies the audio
* endpoint device to be opened and read by the new instance
* @param dataFlow the <tt>AudioSystem.DataFlow</tt> of the audio endpoint
* device identified by <tt>locator</tt>. If
* <tt>AudioSystem.DataFlow.PLAYBACK</tt> and <tt>streamFlags</tt> includes
* {@link WASAPI#AUDCLNT_STREAMFLAGS_LOOPBACK}, allows opening a render
* endpoint device in loopback mode and inputing the data that is being
* written on that render endpoint device
* @param streamFlags zero or more of the <tt>AUDCLNT_STREAMFLAGS_XXX</tt>
* flags defined by the <tt>WASAPI</tt> class
* @param outFormat the <tt>AudioFormat</tt> of the data to be made
* available by the new instance. Eventually, the
* <tt>IAudioCaptureClient</tt> to be represented by the new instance may be
* initialized with a different <tt>AudioFormat</tt> in which case the new
* instance will automatically transcode the data input from the
* <tt>IAudioCaptureClient</tt> into the specified <tt>outFormat</tt>.
* @param transferHandler the <tt>BufferTransferHandler</tt> to be invoked
* when the new instance has made data available to be read via
* {@link #read(byte[], int, int)}
* @throws Exception if the initialization of the new instance fails
*/
public AudioCaptureClient( public AudioCaptureClient(
WASAPISystem audioSystem, WASAPISystem audioSystem,
MediaLocator locator, MediaLocator locator,
...@@ -294,6 +351,10 @@ public AudioCaptureClient( ...@@ -294,6 +351,10 @@ public AudioCaptureClient(
} }
} }
/**
* Releases the resources acquired by this instance throughout its lifetime
* and prepares it to be garbage collected.
*/
public void close() public void close()
{ {
if (iAudioCaptureClient != 0) if (iAudioCaptureClient != 0)
...@@ -325,7 +386,26 @@ public void close() ...@@ -325,7 +386,26 @@ public void close()
started = false; started = false;
} }
private int doRead(byte[] buffer, int offset, int length) /**
* Reads audio data from the internal buffer of this instance which has
* previously/already been read by this instance from the associated
* <tt>IAudioCaptureClient</tt>. Invoked by {@link #read(byte[], int, int)}.
*
* @param buffer the <tt>byte</tt> array into which the audio data read from
* the internal buffer of this instance is to be written
* @param offset the offset into <tt>buffer</tt> at which the writing of the
* audio data is to begin
* @param length the maximum number of bytes in <tt>buffer</tt> starting at
* <tt>offset</tt> to be written
* @return the number of bytes read from the internal buffer of this
* instance and written into the specified <tt>buffer</tt>
* @throws IOException if the reading from the internal buffer of this
* instance or writing into the specified <tt>buffer</tt> fails
*/
private int doRead(
IMediaBuffer iMediaBuffer,
byte[] buffer, int offset,
int length)
throws IOException throws IOException
{ {
int toRead = Math.min(length, remainderLength); int toRead = Math.min(length, remainderLength);
...@@ -335,9 +415,14 @@ private int doRead(byte[] buffer, int offset, int length) ...@@ -335,9 +415,14 @@ private int doRead(byte[] buffer, int offset, int length)
read = 0; read = 0;
else else
{ {
System.arraycopy(remainder, 0, buffer, offset, toRead); if (iMediaBuffer == null)
popFromRemainder(toRead); {
read = toRead; read = toRead;
System.arraycopy(remainder, 0, buffer, offset, toRead);
}
else
read = iMediaBuffer.push(remainder, 0, toRead);
popFromRemainder(read);
} }
return read; return read;
} }
...@@ -355,8 +440,31 @@ private void popFromRemainder(int length) ...@@ -355,8 +440,31 @@ private void popFromRemainder(int length)
= WASAPIRenderer.pop(remainder, remainderLength, length); = WASAPIRenderer.pop(remainder, remainderLength, length);
} }
/**
* Reads audio data from this instance into a spcific <tt>byte</tt> array.
*
* @param buffer the <tt>byte</tt> array into which the audio data read from
* this instance is to be written
* @param offset the offset in <tt>buffer</tt> at which the writing of the
* audio data is to start
* @param length the maximum number of bytes in <tt>buffer</tt> starting at
* <tt>offset</tt> to be written
* @return the number of bytes read from this instance and written into the
* specified <tt>buffer</tt>
* @throws IOException if the reading from this instance or the writing into
* the specified <tt>buffer</tt> fails
*/
public int read(byte[] buffer, int offset, int length) public int read(byte[] buffer, int offset, int length)
throws IOException throws IOException
{
return read(/* iMediaBuffer */ null, buffer, offset, length);
}
private int read(
IMediaBuffer iMediaBuffer,
byte[] buffer, int offset,
int length)
throws IOException
{ {
String message; String message;
...@@ -380,7 +488,7 @@ else if (!started) ...@@ -380,7 +488,7 @@ else if (!started)
try try
{ {
read = doRead(buffer, offset, length); read = doRead(iMediaBuffer, buffer, offset, length);
cause = null; cause = null;
} }
catch (Throwable t) catch (Throwable t)
...@@ -413,6 +521,12 @@ else if (cause instanceof IOException) ...@@ -413,6 +521,12 @@ else if (cause instanceof IOException)
return read; return read;
} }
public int read(IMediaBuffer iMediaBuffer, int length)
throws IOException
{
return read(iMediaBuffer, /* buffer */ null, /* offset */ 0, length);
}
/** /**
* Reads from {@link #iAudioCaptureClient} into {@link #remainder} and * Reads from {@link #iAudioCaptureClient} into {@link #remainder} and
* returns a non-<tt>null</tt> <tt>BufferTransferHandler</tt> if this * returns a non-<tt>null</tt> <tt>BufferTransferHandler</tt> if this
...@@ -593,6 +707,13 @@ private void runInEventHandleCmd(Runnable eventHandleCmd) ...@@ -593,6 +707,13 @@ private void runInEventHandleCmd(Runnable eventHandleCmd)
} }
} }
/**
* Starts the transfer of media from the <tt>IAudioCaptureClient</tt>
* identified by the <tt>MediaLocator</tt> with which this instance has
* been initialized.
*
* @throws IOException if the starting of the transfer of media fails
*/
public synchronized void start() public synchronized void start()
throws IOException throws IOException
{ {
...@@ -652,6 +773,13 @@ public void run() ...@@ -652,6 +773,13 @@ public void run()
} }
} }
/**
* Stops the transfer of media from the <tt>IAudioCaptureClient</tt>
* identified by the <tt>MediaLocator</tt> with which this instance has
* been initialized.
*
* @throws IOException if the stopping of the transfer of media fails
*/
public synchronized void stop() public synchronized void stop()
throws IOException throws IOException
{ {
......
...@@ -131,6 +131,13 @@ protected void doDisconnect() ...@@ -131,6 +131,13 @@ protected void doDisconnect()
} }
} }
/**
* Gets the <tt>Format</tt>s of media data supported by the audio endpoint
* device associated with this instance.
*
* @return the <tt>Format</tt>s of media data supported by the audio
* endpoint device associated with this instance
*/
Format[] getIAudioClientSupportedFormats() Format[] getIAudioClientSupportedFormats()
{ {
return return
...@@ -139,6 +146,18 @@ Format[] getIAudioClientSupportedFormats() ...@@ -139,6 +146,18 @@ Format[] getIAudioClientSupportedFormats()
audioSystem.getAECSupportedFormats()); audioSystem.getAECSupportedFormats());
} }
/**
* Gets the <tt>Format</tt>s of media data supported by the audio endpoint
* device associated with this instance.
*
* @param streamIndex the index of the <tt>SourceStream</tt> within the list
* of <tt>SourceStream</tt>s of this <tt>DataSource</tt> on behalf of which
* the query is being made
* @param aecSupportedFormats the list of <tt>AudioFormat</tt>s supported by
* the voice capture DMO implementing acoustic echo cancellation
* @return the <tt>Format</tt>s of media data supported by the audio
* endpoint device associated with this instance
*/
private Format[] getIAudioClientSupportedFormats( private Format[] getIAudioClientSupportedFormats(
int streamIndex, int streamIndex,
List<AudioFormat> aecSupportedFormats) List<AudioFormat> aecSupportedFormats)
......
/*
* Jitsi, the OpenSource Java VoIP and Instant Messaging client.
*
* Distributable under LGPL license.
* See terms of license at gnu.org.
*/
package org.jitsi.impl.neomedia.jmfext.media.protocol.wasapi;
import java.io.*;
/**
* Defines the API of Microsoft's <tt>IMediaBuffer</tt> interface (referred to
* as unmanaged) and allows implementing similar abstractions on the Java side
* (referred to as managed).
*
* @author Lyubomir Marinov
*/
public interface IMediaBuffer
{
int GetLength()
throws IOException;
int GetMaxLength()
throws IOException;
int pop(byte[] buffer, int offset, int length)
throws IOException;
int push(byte[] buffer, int offset, int length)
throws IOException;
int Release();
void SetLength(int length)
throws IOException;
}
/*
* Jitsi, the OpenSource Java VoIP and Instant Messaging client.
*
* Distributable under LGPL license.
* See terms of license at gnu.org.
*/
package org.jitsi.impl.neomedia.jmfext.media.protocol.wasapi;
import static org.jitsi.impl.neomedia.jmfext.media.protocol.wasapi.VoiceCaptureDSP.*;
import java.io.*;
/**
* Implements a managed <tt>IMediaBuffer</tt> which wraps around an unmanaged
* <tt>IMediaBuffer</tt>.
*
* @author Lyubomir Marinov
*/
public class PtrMediaBuffer
implements IMediaBuffer
{
/**
* The unmanaged <tt>IMediaBuffer</tt> represented by this instance.
*/
final long ptr;
/**
* Initializes a new managed <tt>IMediaBuffer</tt> which is to represent a
* specific unmanaged <tt>IMediaBuffer</tt>.
*
* @param ptr the unmanaged <tt>IMediaBuffer</tt> to be represented by the
* new instance
*/
public PtrMediaBuffer(long ptr)
{
if (ptr == 0)
throw new IllegalArgumentException("ptr");
this.ptr = ptr;
}
public int GetLength()
throws IOException
{
try
{
return IMediaBuffer_GetLength(ptr);
}
catch (HResultException hre)
{
throw new IOException(hre);
}
}
public int GetMaxLength()
throws IOException
{
try
{
return IMediaBuffer_GetMaxLength(ptr);
}
catch (HResultException hre)
{
throw new IOException(hre);
}
}
public int pop(byte[] buffer, int offset, int length)
throws IOException
{
try
{
return MediaBuffer_pop(ptr, buffer, offset, length);
}
catch (HResultException hre)
{
throw new IOException(hre);
}
}
public int push(byte[] buffer, int offset, int length)
throws IOException
{
try
{
return MediaBuffer_push(ptr, buffer, offset, length);
}
catch (HResultException hre)
{
throw new IOException(hre);
}
}
public int Release()
{
return IMediaBuffer_Release(ptr);
}
public void SetLength(int length)
throws IOException
{
try
{
IMediaBuffer_SetLength(ptr, length);
}
catch (HResultException hre)
{
throw new IOException(hre);
}
}
}
...@@ -39,11 +39,23 @@ public class WASAPIStream ...@@ -39,11 +39,23 @@ public class WASAPIStream
*/ */
private static Logger logger = Logger.getLogger(WASAPIStream.class); private static Logger logger = Logger.getLogger(WASAPIStream.class);
/**
* Finds an <tt>AudioFormat</tt> in a specific list of <tt>Format</tt>s
* which is as similar to a specific <tt>AudioFormat</tt> as possible.
*
* @param formats the list of <tt>Format</tt>s into which an
* <tt>AudioFormat</tt> as similar to the specified <tt>format</tt> as
* possible is to be found
* @param format the <tt>AudioFormat</tt> for which a similar
* <tt>AudioFormat</tt> is to be found in <tt>formats</tt>
* @return an <tt>AudioFormat</tt> which is an element of <tt>formats</tt>
* and is as similar to the specified <tt>format</tt> as possible or
* <tt>null</tt> if no similarity could be established
*/
private static AudioFormat findClosestMatch( private static AudioFormat findClosestMatch(
Format[] formats, Format[] formats,
AudioFormat format) AudioFormat format)
{ {
// Try to find the very specified format. // Try to find the very specified format.
AudioFormat match = findFirstMatch(formats, format); AudioFormat match = findFirstMatch(formats, format);
...@@ -91,6 +103,17 @@ private static AudioFormat findClosestMatch( ...@@ -91,6 +103,17 @@ private static AudioFormat findClosestMatch(
return match; return match;
} }
/**
* Finds the first element of a specific array of <tt>Format</tt>s which
* matches in the sense of {@link Format#matches(Format)} a specific
* <tt>AudioFormat</tt>.
*
* @param formats the array of <tt>Format</tt>s which si to be searched
* @param format the <tt>AudioFormat</tt> for which a match is to be found
* in the specified <tt>formats</tt>
* @return the first element of <tt>formats</tt> which matches the specified
* <tt>format</tt> or <tt>null</tt> if no match could be found
*/
private static AudioFormat findFirstMatch( private static AudioFormat findFirstMatch(
Format[] formats, Format[] formats,
AudioFormat format) AudioFormat format)
...@@ -101,6 +124,29 @@ private static AudioFormat findFirstMatch( ...@@ -101,6 +124,29 @@ private static AudioFormat findFirstMatch(
return null; return null;
} }
/**
* Sets the media type of an input or output stream of a specific
* <tt>IMediaObject</tt>.
*
* @param iMediaObject the <tt>IMediaObject</tt> to set the media type of
* @param inOrOut <tt>true</tt> if the media type of an input stream of the
* specified <tt>iMediaObject</tt> is to be set or <tt>false</tt> if the
* media type of an output stream of the specified <tt>iMediaObject</tT> is
* to be set
* @param dwXXXputStreamIndex the zero-based index of the input or output
* stream on the specified <tt>iMediaObject</tt> of which the media type is
* to be set
* @param audioFormat the <tt>AudioFormat</tt> to be set on the specified
* stream of the DMO
* @param dwFlags bitwise combination of zero or more
* <tt>DMO_SET_TYPEF_XXX</tt> flags (defined by the <tt>VoiceCaptureDSP</tt>
* class
* @return an <tt>HRESULT</tt> value indicating whether the specified
* <tt>audioFormat</tt> is acceptable and/or whether it has been set
* successfully
* @throws HResultException if setting the media type of the specified
* stream of the specified <tt>iMediaObject</tt> fails
*/
private static int IMediaObject_SetXXXputType( private static int IMediaObject_SetXXXputType(
long iMediaObject, long iMediaObject,
boolean inOrOut, boolean inOrOut,
...@@ -206,6 +252,101 @@ private static int IMediaObject_SetXXXputType( ...@@ -206,6 +252,101 @@ private static int IMediaObject_SetXXXputType(
return hresult; return hresult;
} }
/**
* Invokes {@link IMediaBuffer#GetLength()} and logs and swallows any
* <tt>IOException</tt>.
*
* @param iMediaBuffer the <tt>IMediaBuffer</tt> on which the method
* <tt>GetLength<tt> is to be invoked
* @return the length of the specified <tt>iMediaBuffer</tt>. If the method
* <tt>GetLength<tt> fails, returns <tt>0</tt>.
*/
private static int maybeIMediaBufferGetLength(IMediaBuffer iMediaBuffer)
{
int length;
try
{
length = iMediaBuffer.GetLength();
}
catch (IOException ioe)
{
length = 0;
logger.error("IMediaBuffer::GetLength", ioe);
}
return length;
}
/**
* Invokes {@link VoiceCaptureDSP#IMediaBuffer_GetLength(long)} and logs and
* swallows any <tt>HResultException</tt>.
*
* @param iMediaBuffer the <tt>IMediaBuffer</tt> on which the function
* <tt>IMediaBuffer_GetLength<tt> is to be invoked
* @return the length of the specified <tt>iMediaBuffer</tt>. If the
* function <tt>IMediaBuffer_GetLength<tt> fails, returns <tt>0</tt>.
*/
@SuppressWarnings("unused")
private static int maybeIMediaBufferGetLength(long iMediaBuffer)
{
int length;
try
{
length = IMediaBuffer_GetLength(iMediaBuffer);
}
catch (HResultException hre)
{
length = 0;
logger.error("IMediaBuffer_GetLength", hre);
}
return length;
}
/**
* Invokes {@link VoiceCaptureDSP#MediaBuffer_push(long, byte[], int, int)}
* on a specific <tt>IMediaBuffer</tt> and logs and swallows any
* <tt>HResultException</tT>.
*
* @param pBuffer the <tt>IMediaBuffer</tt> into which the specified bytes
* are to be pushed/written
* @param buffer the bytes to be pushed/written into the specified
* <tt>pBuffer</tt>
* @param offset the offset in <tt>buffer</tt> at which the bytes to be
* pushed/written into the specified <tt>pBuffer</tt> start
* @param length the number of bytes in <tt>buffer</tt> starting at
* <tt>offset</tt> to be pushed/written into the specified <tt>pBuffer</tt>
* @return the number of bytes from the specified <tt>buffer</tt>
* pushed/written into the specified <tt>pBuffer</tt>
*/
private static int maybeMediaBufferPush(
long pBuffer,
byte[] buffer, int offset, int length)
{
int written;
Throwable exception;
try
{
written = MediaBuffer_push(pBuffer, buffer, offset, length);
exception = null;
}
catch (HResultException hre)
{
written = 0;
exception = hre;
}
if ((exception != null) || (written != length))
{
logger.error(
"Failed to push/write "
+ ((written <= 0) ? length : (length - written))
+ " bytes into an IMediaBuffer.",
exception);
}
return written;
}
/** /**
* Throws a new <tt>IOException</tt> instance initialized with a specific * Throws a new <tt>IOException</tt> instance initialized with a specific
* <tt>String</tt> message and a specific <tt>HResultException</tt> cause. * <tt>String</tt> message and a specific <tt>HResultException</tt> cause.
...@@ -238,8 +379,14 @@ static void throwNewIOException(String message, HResultException hre) ...@@ -238,8 +379,14 @@ static void throwNewIOException(String message, HResultException hre)
private int captureBufferMaxLength; private int captureBufferMaxLength;
private long captureIMediaBuffer; private PtrMediaBuffer captureIMediaBuffer;
/**
* The indicator which determines whether {@link #capture} and its
* associated resources/states are busy and, consequently, should not be
* modified. For example, <tt>capture</tt> is busy during the execution of
* {@link #read(Buffer)}.
*/
private boolean captureIsBusy; private boolean captureIsBusy;
/** /**
...@@ -257,6 +404,10 @@ static void throwNewIOException(String message, HResultException hre) ...@@ -257,6 +404,10 @@ static void throwNewIOException(String message, HResultException hre)
private long iMediaBuffer; private long iMediaBuffer;
/**
* The <tt>IMediaObject</tt> reference to the Voice Capture DSP that
* implements the acoustic echo cancellation (AEC) feature.
*/
private long iMediaObject; private long iMediaObject;
/** /**
...@@ -277,8 +428,14 @@ static void throwNewIOException(String message, HResultException hre) ...@@ -277,8 +428,14 @@ static void throwNewIOException(String message, HResultException hre)
private int renderBufferMaxLength; private int renderBufferMaxLength;
private long renderIMediaBuffer; private PtrMediaBuffer renderIMediaBuffer;
/**
* The indicator which determines whether {@link #render} and its associated
* resources/states are busy and, consequently, should not be modified. For
* example, <tt>render</tt> is busy during the execution of
* {@link #read(Buffer)}.
*/
private boolean renderIsBusy; private boolean renderIsBusy;
/** /**
...@@ -304,7 +461,7 @@ public WASAPIStream(DataSource dataSource, FormatControl formatControl) ...@@ -304,7 +461,7 @@ public WASAPIStream(DataSource dataSource, FormatControl formatControl)
} }
/** /**
* Performs optional configuration of the Voice Capture DSP that implements * Performs optional configuration on the Voice Capture DSP that implements
* acoustic echo cancellation (AEC). * acoustic echo cancellation (AEC).
* *
* @param iPropertyStore a reference to the <tt>IPropertyStore</tt> * @param iPropertyStore a reference to the <tt>IPropertyStore</tt>
...@@ -320,6 +477,35 @@ private void configureAEC(long iPropertyStore) ...@@ -320,6 +477,35 @@ private void configureAEC(long iPropertyStore)
* property to true and override the default settings on the * property to true and override the default settings on the
* MFPKEY_WMAAECMA_FEATR_XXX properties of the Voice Capture DSP. * MFPKEY_WMAAECMA_FEATR_XXX properties of the Voice Capture DSP.
*/ */
try
{
if (MFPKEY_WMAAECMA_FEATURE_MODE != 0)
{
IPropertyStore_SetValue(
iPropertyStore,
MFPKEY_WMAAECMA_FEATURE_MODE, true);
if (MFPKEY_WMAAECMA_FEATR_AES != 0)
{
IPropertyStore_SetValue(
iPropertyStore,
MFPKEY_WMAAECMA_FEATR_AES, 2);
}
if (MFPKEY_WMAAECMA_FEATR_ECHO_LENGTH != 0)
{
IPropertyStore_SetValue(
iPropertyStore,
MFPKEY_WMAAECMA_FEATR_ECHO_LENGTH, 256);
}
}
}
catch (HResultException hre)
{
logger.error(
"Failed to perform optional configuration on the Voice"
+ " Capture DSP that implements acoustic echo"
+ " cancellation (AEC).",
hre);
}
} }
/** /**
...@@ -654,7 +840,8 @@ private void initializeAEC( ...@@ -654,7 +840,8 @@ private void initializeAEC(
processed = new byte[bufferMaxLength * 3]; processed = new byte[bufferMaxLength * 3];
processedLength = 0; processedLength = 0;
this.captureIMediaBuffer = captureIMediaBuffer; this.captureIMediaBuffer
= new PtrMediaBuffer(captureIMediaBuffer);
captureIMediaBuffer = 0; captureIMediaBuffer = 0;
this.dmoOutputDataBuffer = dmoOutputDataBuffer; this.dmoOutputDataBuffer = dmoOutputDataBuffer;
dmoOutputDataBuffer = 0; dmoOutputDataBuffer = 0;
...@@ -662,7 +849,8 @@ private void initializeAEC( ...@@ -662,7 +849,8 @@ private void initializeAEC(
iMediaBuffer = 0; iMediaBuffer = 0;
this.iMediaObject = iMediaObject; this.iMediaObject = iMediaObject;
iMediaObject = 0; iMediaObject = 0;
this.renderIMediaBuffer = renderIMediaBuffer; this.renderIMediaBuffer
= new PtrMediaBuffer(renderIMediaBuffer);
renderIMediaBuffer = 0; renderIMediaBuffer = 0;
} }
finally finally
...@@ -757,21 +945,29 @@ private void popFromProcessed(int length) ...@@ -757,21 +945,29 @@ private void popFromProcessed(int length)
= WASAPIRenderer.pop(processed, processedLength, length); = WASAPIRenderer.pop(processed, processedLength, length);
} }
private int processInput(int dwInputStreamIndex) /**
* Inputs audio samples from {@link #capture} or {@link #render} and
* delivers them to {@link #iMediaObject} which implements the acoustic echo
* cancellation (AEC) feature.
*
* @param dwInputStreamIndex the zero-based index of the input stream on
* <tt>iMediaObject</tt> to which audio samples are to be delivered
*/
private void processInput(int dwInputStreamIndex)
{ {
long pBuffer; PtrMediaBuffer oBuffer;
int maxLength; int maxLength;
AudioCaptureClient audioCaptureClient; AudioCaptureClient audioCaptureClient;
switch (dwInputStreamIndex) switch (dwInputStreamIndex)
{ {
case 0: case 0:
pBuffer = captureIMediaBuffer; oBuffer = captureIMediaBuffer;
maxLength = captureBufferMaxLength; maxLength = captureBufferMaxLength;
audioCaptureClient = capture; audioCaptureClient = capture;
break; break;
case 1: case 1:
pBuffer = renderIMediaBuffer; oBuffer = renderIMediaBuffer;
maxLength = renderBufferMaxLength; maxLength = renderBufferMaxLength;
audioCaptureClient = render; audioCaptureClient = render;
break; break;
...@@ -779,11 +975,15 @@ private int processInput(int dwInputStreamIndex) ...@@ -779,11 +975,15 @@ private int processInput(int dwInputStreamIndex)
throw new IllegalArgumentException("dwInputStreamIndex"); throw new IllegalArgumentException("dwInputStreamIndex");
} }
long pBuffer = oBuffer.ptr;
int hresult = S_OK; int hresult = S_OK;
int processed = 0;
do do
{ {
/*
* Attempt to deliver audio samples to the specified input stream
* only if it accepts input data at this time.
*/
int dwFlags; int dwFlags;
try try
...@@ -802,6 +1002,12 @@ private int processInput(int dwInputStreamIndex) ...@@ -802,6 +1002,12 @@ private int processInput(int dwInputStreamIndex)
if ((dwFlags & DMO_INPUT_STATUSF_ACCEPT_DATA) if ((dwFlags & DMO_INPUT_STATUSF_ACCEPT_DATA)
== DMO_INPUT_STATUSF_ACCEPT_DATA) == DMO_INPUT_STATUSF_ACCEPT_DATA)
{ {
/*
* The specified input stream reports that it accepts input data
* at this time so read audio samples from the associated
* AudioCaptureClient and then deliver them to the specified
* input stream.
*/
int toRead; int toRead;
try try
...@@ -816,57 +1022,34 @@ private int processInput(int dwInputStreamIndex) ...@@ -816,57 +1022,34 @@ private int processInput(int dwInputStreamIndex)
} }
if (toRead > 0) if (toRead > 0)
{ {
if ((processInputBuffer == null) /*
|| (processInputBuffer.length < toRead)) * Read audio samples from the associated
processInputBuffer = new byte[toRead]; * AudioCaptureClient.
*/
int read;
try try
{ {
read audioCaptureClient.read(oBuffer, toRead);
= audioCaptureClient.read(
processInputBuffer,
0,
toRead);
} }
catch (IOException ioe) catch (IOException ioe)
{ {
read = 0;
logger.error( logger.error(
"Failed to read from IAudioCaptureClient.", "Failed to read from IAudioCaptureClient.",
ioe); ioe);
} }
if (read > 0)
{
int written;
try
{
written
= MediaBuffer_push(
pBuffer,
processInputBuffer, 0, read);
}
catch (HResultException hre)
{
written = 0;
logger.error("MediaBuffer_push", hre);
}
if (written < read)
{
logger.error(
"Failed to push/write "
+ ((written <= 0)
? read
: (read - written))
+ " bytes into an IMediaBuffer.");
}
if (written > 0)
processed += written;
}
} }
/*
* If the capture endpoint device has delivered audio samples,
* they have to go through the acoustic echo cancellation (AEC)
* regardless of whether the render endpoint device has
* delivered audio samples. Additionally, the duration of the
* audio samples delivered by the render has to be the same as
* the duration of the audio samples delivered by the capture in
* order to have the audio samples delivered by the capture pass
* through the voice capture DSO in entirety. To achieve the
* above, read from the render endpoint device as many audio
* samples as possible and pad with silence if necessary.
*/
if (dwInputStreamIndex == 1) if (dwInputStreamIndex == 1)
{ {
int length; int length;
...@@ -890,19 +1073,16 @@ private int processInput(int dwInputStreamIndex) ...@@ -890,19 +1073,16 @@ private int processInput(int dwInputStreamIndex)
|| (processInputBuffer.length < silence)) || (processInputBuffer.length < silence))
processInputBuffer = new byte[silence]; processInputBuffer = new byte[silence];
Arrays.fill(processInputBuffer, 0, silence, (byte) 0); Arrays.fill(processInputBuffer, 0, silence, (byte) 0);
try maybeMediaBufferPush(
{ pBuffer,
MediaBuffer_push( processInputBuffer, 0, silence);
pBuffer,
processInputBuffer, 0, silence);
}
catch (HResultException hre)
{
logger.error("MediaBuffer_push", hre);
}
} }
} }
/*
* Deliver the audio samples read from the associated
* AudioCaptureClient to the specified input stream.
*/
try try
{ {
hresult hresult
...@@ -925,8 +1105,6 @@ private int processInput(int dwInputStreamIndex) ...@@ -925,8 +1105,6 @@ private int processInput(int dwInputStreamIndex)
break; // The input stream cannot accept more input data. break; // The input stream cannot accept more input data.
} }
while (SUCCEEDED(hresult)); while (SUCCEEDED(hresult));
return processed;
} }
/** /**
...@@ -1047,65 +1225,86 @@ else if (cause instanceof IOException) ...@@ -1047,65 +1225,86 @@ else if (cause instanceof IOException)
while (true); while (true);
} }
/**
* Executed by {@link #processThread} and invoked by
* {@link #runInProcessThread(Thread)}, inputs audio samples from
* {@link #capture} and {@link #render}, delivers them to
* {@link #iMediaBuffer} which implements the acoustic echo cancellation
* (AEC) features and produces output and caches the output so that it can
* be read out of this instance via {@link #read(Buffer)}.
*
* @return a <tt>BufferTransferHandler</tt> to be invoked if the method has
* made available audio samples to be read out of this instance; otherwise,
* <tt>null</tt>
*/
private BufferTransferHandler runInProcessThread() private BufferTransferHandler runInProcessThread()
{ {
// ProcessInput // ProcessInput
processInput(/* capture */ 0); processInput(/* capture */ 0);
processInput(/* render */ 1); /*
* If the capture endpoint device has not made any audio samples
* available, there is no input to be processed. Moreover, it is
* incorrect to input from the render endpoint device in such a case.
*/
if (maybeIMediaBufferGetLength(captureIMediaBuffer) != 0)
{
processInput(/* render */ 1);
// ProcessOutput // ProcessOutput
int dwStatus = 0; int dwStatus = 0;
do do
{
try
{
IMediaObject_ProcessOutput(
iMediaObject,
/* dwFlags */ 0,
1,
dmoOutputDataBuffer);
}
catch (HResultException hre)
{
dwStatus = 0;
logger.error("IMediaObject_ProcessOutput", hre);
}
try
{ {
int toRead = IMediaBuffer_GetLength(iMediaBuffer); try
if (toRead > 0)
{ {
/* IMediaObject_ProcessOutput(
* Make sure there is enough room in processed to iMediaObject,
* accommodate toRead. /* dwFlags */ 0,
*/ 1,
int toPop = toRead - (processed.length - processedLength); dmoOutputDataBuffer);
}
catch (HResultException hre)
{
dwStatus = 0;
logger.error("IMediaObject_ProcessOutput", hre);
}
try
{
int toRead = IMediaBuffer_GetLength(iMediaBuffer);
if (toPop > 0) if (toRead > 0)
popFromProcessed(toPop); {
/*
* Make sure there is enough room in processed to
* accommodate toRead.
*/
int toPop
= toRead - (processed.length - processedLength);
int read if (toPop > 0)
= MediaBuffer_pop( popFromProcessed(toPop);
iMediaBuffer,
processed, processedLength, toRead);
if (read > 0) int read
processedLength += read; = MediaBuffer_pop(
iMediaBuffer,
processed, processedLength, toRead);
if (read > 0)
processedLength += read;
}
}
catch (HResultException hre)
{
logger.error(
"Failed to read from acoustic echo cancellation"
+ " (AEC) output IMediaBuffer.",
hre);
break;
} }
} }
catch (HResultException hre) while ((dwStatus & DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE)
{ == DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE);
logger.error(
"Failed to read from acoustic echo cancellation (AEC)"
+ " output IMediaBuffer.",
hre);
break;
}
} }
while ((dwStatus & DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE)
== DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE);
/* /*
* IMediaObject::ProcessOutput has completed which means that, as far as * IMediaObject::ProcessOutput has completed which means that, as far as
...@@ -1122,18 +1321,31 @@ private BufferTransferHandler runInProcessThread() ...@@ -1122,18 +1321,31 @@ private BufferTransferHandler runInProcessThread()
if (SUCCEEDED(hresult)) if (SUCCEEDED(hresult))
{ {
IMediaBuffer_SetLength(captureIMediaBuffer, 0); captureIMediaBuffer.SetLength(0);
IMediaBuffer_SetLength(renderIMediaBuffer, 0); renderIMediaBuffer.SetLength(0);
} }
} }
catch (HResultException hre) catch (HResultException hre)
{ {
logger.error("IMediaBuffer_SetLength", hre); logger.error("IMediaBuffer_Flush", hre);
}
catch (IOException ioe)
{
logger.error("IMediaBuffer::SetLength", ioe);
} }
return (processedLength >= bufferMaxLength) ? transferHandler : null; return (processedLength >= bufferMaxLength) ? transferHandler : null;
} }
/**
* Executed by {@link #processThread}, inputs audio samples from
* {@link #capture} and {@link #render}, delivers them to
* {@link #iMediaBuffer} which implements the acoustic echo cancellation
* (AEC) features and produces output and caches the output so that it can
* be read out of this instance via {@link #read(Buffer)}.
*
* @param processThread the <tt>Thread</tt> which is executing the method
*/
private void runInProcessThread(Thread processThread) private void runInProcessThread(Thread processThread)
{ {
try try
...@@ -1293,6 +1505,10 @@ public synchronized void stop() ...@@ -1293,6 +1505,10 @@ public synchronized void stop()
processedLength = 0; processedLength = 0;
} }
/**
* Notifies this instance that audio data has been made available in
* {@link #capture}.
*/
private void transferCaptureData() private void transferCaptureData()
{ {
if (dataSource.aec) if (dataSource.aec)
...@@ -1311,6 +1527,10 @@ private void transferCaptureData() ...@@ -1311,6 +1527,10 @@ private void transferCaptureData()
} }
} }
/**
* Notifies this instance that audio data has been made available in
* {@link #render}.
*/
private void transferRenderData() private void transferRenderData()
{ {
synchronized (this) synchronized (this)
...@@ -1336,15 +1556,15 @@ private void uninitializeAEC() ...@@ -1336,15 +1556,15 @@ private void uninitializeAEC()
IMediaBuffer_Release(iMediaBuffer); IMediaBuffer_Release(iMediaBuffer);
iMediaBuffer = 0; iMediaBuffer = 0;
} }
if (renderIMediaBuffer != 0) if (renderIMediaBuffer != null)
{ {
IMediaBuffer_Release(renderIMediaBuffer); renderIMediaBuffer.Release();
renderIMediaBuffer =0; renderIMediaBuffer = null;
} }
if (captureIMediaBuffer != 0) if (captureIMediaBuffer != null)
{ {
IMediaBuffer_Release(captureIMediaBuffer); captureIMediaBuffer.Release();
captureIMediaBuffer = 0; captureIMediaBuffer = null;
} }
} }
...@@ -1389,6 +1609,10 @@ private synchronized void waitWhileCaptureIsBusy() ...@@ -1389,6 +1609,10 @@ private synchronized void waitWhileCaptureIsBusy()
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} }
/**
* Waits on this instance while the value of {@link #precessThread} is not
* equal to <tt>null</tt>.
*/
private synchronized void waitWhileProcessThread() private synchronized void waitWhileProcessThread()
{ {
while (processThread != null) while (processThread != null)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment