From a420170c28d0aca90c57c9a483f3c71563c5c16a Mon Sep 17 00:00:00 2001
From: Boris Grozev <boris@jitsi.org>
Date: Fri, 5 Oct 2012 11:58:37 +0000
Subject: [PATCH] Adds FEC support for the SILK codec. Minor clean-ups in
EncodingConfiguration and MediaConfigurationImpl.
---
.../EncodingConfigurationConfigImpl.java | 8 ++
.../codec/EncodingConfigurationImpl.java | 8 ++
.../codec/audio/silk/JavaDecoder.java | 136 +++++++++++++++---
.../codec/audio/silk/JavaEncoder.java | 37 ++++-
.../codec/audio/silk/Silk_define_FLP.java | 2 +-
.../neomedia/codec/EncodingConfiguration.java | 2 +-
6 files changed, 161 insertions(+), 32 deletions(-)
diff --git a/src/org/jitsi/impl/neomedia/codec/EncodingConfigurationConfigImpl.java b/src/org/jitsi/impl/neomedia/codec/EncodingConfigurationConfigImpl.java
index 881afbda..df366dab 100644
--- a/src/org/jitsi/impl/neomedia/codec/EncodingConfigurationConfigImpl.java
+++ b/src/org/jitsi/impl/neomedia/codec/EncodingConfigurationConfigImpl.java
@@ -9,6 +9,7 @@
import org.jitsi.service.configuration.*;
import org.jitsi.service.libjitsi.*;
import org.jitsi.service.neomedia.format.MediaFormat;
+import org.jitsi.util.*;
import java.util.*;
@@ -22,6 +23,13 @@
public class EncodingConfigurationConfigImpl
extends EncodingConfigurationImpl
{
+ /**
+ * The <tt>Logger</tt> used by this <tt>EncodingConfigurationConfigImpl</tt>
+ * instance for logging output.
+ */
+ private final Logger logger
+ = Logger.getLogger(EncodingConfigurationConfigImpl.class);
+
/**
* Holds the prefix that will be used to store properties
*/
diff --git a/src/org/jitsi/impl/neomedia/codec/EncodingConfigurationImpl.java b/src/org/jitsi/impl/neomedia/codec/EncodingConfigurationImpl.java
index f3f5ad7a..627b9972 100644
--- a/src/org/jitsi/impl/neomedia/codec/EncodingConfigurationImpl.java
+++ b/src/org/jitsi/impl/neomedia/codec/EncodingConfigurationImpl.java
@@ -8,6 +8,7 @@
import org.jitsi.impl.neomedia.format.*;
import org.jitsi.service.neomedia.codec.*;
+import org.jitsi.util.*;
/**
* Configuration of encoding priorities.
@@ -27,6 +28,13 @@ public class EncodingConfigurationImpl extends EncodingConfiguration
*/
public static final boolean G729 = false;
+ /**
+ * The <tt>Logger</tt> used by this <tt>EncodingConfigurationImpl</tt>
+ * instance for logging output.
+ */
+ private final Logger logger
+ = Logger.getLogger(EncodingConfigurationImpl.class);
+
/**
* Constructor. Loads the default preferences.
*/
diff --git a/src/org/jitsi/impl/neomedia/codec/audio/silk/JavaDecoder.java b/src/org/jitsi/impl/neomedia/codec/audio/silk/JavaDecoder.java
index 77147a67..b74f9983 100644
--- a/src/org/jitsi/impl/neomedia/codec/audio/silk/JavaDecoder.java
+++ b/src/org/jitsi/impl/neomedia/codec/audio/silk/JavaDecoder.java
@@ -10,15 +10,24 @@
import javax.media.format.*;
import org.jitsi.impl.neomedia.codec.*;
+import org.jitsi.util.*;
/**
* Implements the SILK decoder as an FMJ/JMF <tt>Codec</tt>.
*
* @author Dingxin Xu
+ * @author Boris Grozev
*/
public class JavaDecoder
extends AbstractCodecExt
{
+ /**
+ * The <tt>Logger</tt> used by this <tt>JavaDecoder</tt> instance
+ * for logging output.
+ */
+ private final Logger logger
+ = Logger.getLogger(JavaDecoder.class);
+
/**
* The duration of a frame in milliseconds as defined by the SILK standard.
*/
@@ -71,6 +80,30 @@ public class JavaDecoder
*/
private final short[] outputLength = new short[1];
+ /**
+ * Previous packet RTP sequence number
+ */
+ private long lastPacketSeq;
+
+ /**
+ * Whether at least one packet has already been processed. Use this to
+ * prevent FEC data from trying to be decoded from the first packet in a
+ * session.
+ */
+ private boolean firstPacketProcessed = false;
+
+ /**
+ * Temporary buffer used to hold the lbrr data when decoding FEC. Defined
+ * here to avoid using <tt>new</tt> in <tt>doProcess</tt>.
+ */
+ private byte[] lbrrData = new byte[JavaEncoder.MAX_BYTES_PER_FRAME];
+
+ /**
+ * Temporary buffer used when decoding FEC. Defined here to
+ * avoid using <tt>new</tt> in <tt>doProcess</tt>.
+ */
+ private short[] lbrrBytes = new short[1];
+
/**
* Initializes a new <tt>JavaDecoder</tt> instance.
*/
@@ -115,38 +148,95 @@ protected int doProcess(Buffer inputBuffer, Buffer outputBuffer)
short[] outputData = validateShortArraySize(outputBuffer, frameLength);
int outputOffset = 0;
+ boolean decodeFEC = false;
+
+ /* Check whether a packet has been lost.
+ * If a packet has more than one frame, we go through each frame in a
+ * new call to <tt>process</tt>, so having the same sequence number as
+ * on the previous pass is fine. */
+ long sequenceNumber = inputBuffer.getSequenceNumber();
+ if(firstPacketProcessed &&
+ sequenceNumber != lastPacketSeq &&
+ sequenceNumber != lastPacketSeq+1 &&
+ /* RTP sequence number is a 16bit field */
+ !(lastPacketSeq == 65535 && sequenceNumber == 0))
+ decodeFEC = true;
+
int processed;
- outputLength[0] = frameLength;
- if (Silk_dec_API.SKP_Silk_SDK_Decode(
- decState, decControl,
- 0,
- inputData, inputOffset, inputLength,
- outputData, outputOffset, outputLength)
- == 0)
+ /* Decode packet normally */
+ if(!decodeFEC)
{
- outputBuffer.setDuration(FRAME_DURATION * 1000000);
- outputBuffer.setLength(outputLength[0]);
- outputBuffer.setOffset(outputOffset);
+ outputLength[0] = frameLength;
+ if (Silk_dec_API.SKP_Silk_SDK_Decode(
+ decState, decControl,
+ 0,
+ inputData, inputOffset, inputLength,
+ outputData, outputOffset, outputLength)
+ == 0)
+ {
+ outputBuffer.setDuration(FRAME_DURATION * 1000000);
+ outputBuffer.setLength(outputLength[0]);
+ outputBuffer.setOffset(outputOffset);
- if (decControl.moreInternalDecoderFrames == 0)
- processed = BUFFER_PROCESSED_OK;
+ if (decControl.moreInternalDecoderFrames == 0)
+ processed = BUFFER_PROCESSED_OK;
+ else
+ {
+ framesPerPayload++;
+ processed
+ = (framesPerPayload >= MAX_FRAMES_PER_PAYLOAD)
+ ? BUFFER_PROCESSED_OK
+ : INPUT_BUFFER_NOT_CONSUMED;
+ }
+ }
else
+ processed = BUFFER_PROCESSED_FAILED;
+
+ if ((processed & INPUT_BUFFER_NOT_CONSUMED)
+ != INPUT_BUFFER_NOT_CONSUMED)
+ framesPerPayload = 0;
+ }
+ else /* Decode the packet's FEC data */
+ {
+ outputLength[0] = frameLength;
+
+ lbrrBytes[0] = 0;
+ Silk_dec_API.SKP_Silk_SDK_search_for_LBRR(
+ inputData, inputOffset, (short)inputLength,
+ 1 /* previous packet */,
+ lbrrData, 0, lbrrBytes);
+
+ if(logger.isTraceEnabled())
{
- framesPerPayload++;
- processed
- = (framesPerPayload >= MAX_FRAMES_PER_PAYLOAD)
- ? BUFFER_PROCESSED_OK
- : INPUT_BUFFER_NOT_CONSUMED;
+ logger.trace("Packet loss detected. Last seen " + lastPacketSeq
+ + ", current " + sequenceNumber);
+ logger.trace("Looking for LBRR info, got " + lbrrBytes[0] + "bytes");
}
- }
- else
- processed = BUFFER_PROCESSED_FAILED;
- if ((processed & INPUT_BUFFER_NOT_CONSUMED)
- != INPUT_BUFFER_NOT_CONSUMED)
- framesPerPayload = 0;
+ if(lbrrBytes[0] == 0)
+ //No FEC data found, process the normal data in the packet next
+ processed = INPUT_BUFFER_NOT_CONSUMED;
+ else if(Silk_dec_API.SKP_Silk_SDK_Decode(
+ decState, decControl, 0,
+ lbrrData, 0, lbrrBytes[0],
+ outputData, outputOffset, outputLength)
+ == 0)
+ {
+ //Found FEC data, decode it
+ outputBuffer.setDuration(FRAME_DURATION * 1000000);
+ outputBuffer.setLength(outputLength[0]);
+ outputBuffer.setOffset(outputOffset);
+
+ //Go on and process the normal data in the packet next
+ processed = INPUT_BUFFER_NOT_CONSUMED;
+ }
+ else
+ processed = BUFFER_PROCESSED_FAILED;
+ }
+ lastPacketSeq = sequenceNumber;
+ firstPacketProcessed = true;
return processed;
}
diff --git a/src/org/jitsi/impl/neomedia/codec/audio/silk/JavaEncoder.java b/src/org/jitsi/impl/neomedia/codec/audio/silk/JavaEncoder.java
index 5bfd82eb..e7d4f496 100644
--- a/src/org/jitsi/impl/neomedia/codec/audio/silk/JavaEncoder.java
+++ b/src/org/jitsi/impl/neomedia/codec/audio/silk/JavaEncoder.java
@@ -10,6 +10,8 @@
import javax.media.format.*;
import org.jitsi.impl.neomedia.codec.*;
+import org.jitsi.service.configuration.*;
+import org.jitsi.service.libjitsi.*;
/**
* Implements the SILK encoder as an FMJ/JMF <tt>Codec</tt>.
@@ -27,9 +29,7 @@ public class JavaEncoder
* The maximum number of output payload bytes per input frame. Equals peak
* bitrate of 100 kbps.
*/
- private static final int MAX_BYTES_PER_FRAME = 250;
-
- private static final int PACKET_LOSS_PERCENTAGE = 0;
+ static final int MAX_BYTES_PER_FRAME = 250;
/**
* The list of <tt>Format</tt>s of audio data supported as input by
@@ -52,8 +52,6 @@ public class JavaEncoder
private static final boolean USE_DTX = false;
- private static final boolean USE_IN_BAND_FEC = false;
-
/**
* The duration an output <tt>Buffer</tt> produced by this <tt>Codec</tt>
* in nanosecond.
@@ -141,16 +139,41 @@ protected void doOpen()
double sampleRate = inputFormat.getSampleRate();
int channels = inputFormat.getChannels();
+
+ ConfigurationService cfg = LibJitsi.getConfigurationService();
+ //TODO: we should have a default value dependent on the SDP parameters
+ //here.
+ boolean useFEC = cfg.getBoolean("net.java.sip.communicator.impl."
+ +"neomedia.codec.audio.silk.encoder.usefec", true);
+ boolean forcePacketLoss = cfg.getBoolean("net.java.sip.communicator." +
+ "impl.neomedia.codec.audio.silk.encoder." +
+ "forcepacketloss", true);
+
+ //Update the statically defined value for "speech activity threshold"
+ //according to our configuration
+ String satStr = cfg.getString("net.java.sip." +
+ "communicator.impl.neomedia.codec.audio.silk.encoder.sat", "0.5");
+ float sat = Silk_define_FLP.LBRR_SPEECH_ACTIVITY_THRES;
+ try
+ {
+ sat = Float.parseFloat(satStr);
+ }
+ catch (Exception e){}
+ Silk_define_FLP.LBRR_SPEECH_ACTIVITY_THRES = sat;
+
encControl.API_sampleRate = (int) sampleRate;
encControl.bitRate = BITRATE;
encControl.complexity = COMPLEXITY;
encControl.maxInternalSampleRate = encControl.API_sampleRate;
- encControl.packetLossPercentage = PACKET_LOSS_PERCENTAGE;
+ //At the moment we do not support dynamically setting the expected
+ //packet loss. Therefore to force the encoder to always expect packet
+ //loss we set this higher than all thresholds.
+ encControl.packetLossPercentage = forcePacketLoss ? 100 : 0;
encControl.packetSize
= (int)
((JavaDecoder.FRAME_DURATION * sampleRate * channels) / 1000);
encControl.useDTX = USE_DTX ? 1 : 0;
- encControl.useInBandFEC = USE_IN_BAND_FEC ? 1 : 0;
+ encControl.useInBandFEC = useFEC ? 1 : 0;
}
protected int doProcess(Buffer inputBuffer, Buffer outputBuffer)
diff --git a/src/org/jitsi/impl/neomedia/codec/audio/silk/Silk_define_FLP.java b/src/org/jitsi/impl/neomedia/codec/audio/silk/Silk_define_FLP.java
index 34f411e9..de5ed003 100644
--- a/src/org/jitsi/impl/neomedia/codec/audio/silk/Silk_define_FLP.java
+++ b/src/org/jitsi/impl/neomedia/codec/audio/silk/Silk_define_FLP.java
@@ -71,7 +71,7 @@ public class Silk_define_FLP
static final float SPEECH_ACTIVITY_DTX_THRES = 0.1f;
/* Speech Activity LBRR enable threshold (needs tuning) */
- static final float LBRR_SPEECH_ACTIVITY_THRES = 0.5f;
+ static float LBRR_SPEECH_ACTIVITY_THRES = 0.5f;
static final float Q14_CONVERSION_FAC = 6.1035e-005f; // 1 / 2^14
}
diff --git a/src/org/jitsi/service/neomedia/codec/EncodingConfiguration.java b/src/org/jitsi/service/neomedia/codec/EncodingConfiguration.java
index 936e9a63..f90d3d83 100644
--- a/src/org/jitsi/service/neomedia/codec/EncodingConfiguration.java
+++ b/src/org/jitsi/service/neomedia/codec/EncodingConfiguration.java
@@ -27,7 +27,7 @@ public class EncodingConfiguration
* The <tt>Logger</tt> used by this <tt>EncodingConfiguration</tt> instance
* for logging output.
*/
- protected final Logger logger
+ private final Logger logger
= Logger.getLogger(EncodingConfiguration.class);
/**
--
GitLab