From 1b968f72c9a5db683b2ac2a4bd411988fbf7c0d1 Mon Sep 17 00:00:00 2001
From: Lyubomir Marinov <lyubomir.marinov@jitsi.org>
Date: Wed, 19 Mar 2014 08:15:00 +0200
Subject: [PATCH] Enhances the support for received H.264 SDP format parameters
such as sprop-parameter-sets.
---
src/org/jitsi/impl/neomedia/MediaUtils.java | 3 +-
.../codec/video/h264/DePacketizer.java | 47 ++++++-
.../neomedia/codec/video/h264/JNIDecoder.java | 121 +++++++++++++++++-
.../neomedia/codec/video/h264/JNIEncoder.java | 19 ++-
.../neomedia/codec/video/h264/Packetizer.java | 14 +-
.../device/VideoMediaDeviceSession.java | 6 +-
.../neomedia/format/VideoMediaFormatImpl.java | 14 +-
7 files changed, 196 insertions(+), 28 deletions(-)
diff --git a/src/org/jitsi/impl/neomedia/MediaUtils.java b/src/org/jitsi/impl/neomedia/MediaUtils.java
index cab8124d..25a9e0a5 100644
--- a/src/org/jitsi/impl/neomedia/MediaUtils.java
+++ b/src/org/jitsi/impl/neomedia/MediaUtils.java
@@ -211,7 +211,8 @@ public class MediaUtils
/* H264 */
Map<String, String> h264FormatParams
= new HashMap<String, String>();
- String packetizationMode = "packetization-mode";
+ String packetizationMode
+ = VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP;
Map<String, String> h264AdvancedAttributes
= new HashMap<String, String>();
diff --git a/src/org/jitsi/impl/neomedia/codec/video/h264/DePacketizer.java b/src/org/jitsi/impl/neomedia/codec/video/h264/DePacketizer.java
index 40663ff4..acdc5206 100644
--- a/src/org/jitsi/impl/neomedia/codec/video/h264/DePacketizer.java
+++ b/src/org/jitsi/impl/neomedia/codec/video/h264/DePacketizer.java
@@ -14,6 +14,7 @@
import net.sf.fmj.media.*;
import org.jitsi.impl.neomedia.codec.*;
+import org.jitsi.impl.neomedia.format.*;
import org.jitsi.service.neomedia.codec.*;
import org.jitsi.service.neomedia.control.*;
import org.jitsi.util.*;
@@ -156,9 +157,9 @@ public class DePacketizer
public DePacketizer()
{
super(
- "H264 DePacketizer",
- VideoFormat.class,
- new VideoFormat[] { new VideoFormat(Constants.H264) });
+ "H264 DePacketizer",
+ VideoFormat.class,
+ new VideoFormat[] { new VideoFormat(Constants.H264) });
List<Format> inputFormats = new ArrayList<Format>();
@@ -508,6 +509,46 @@ else if (nal_unit_type == 28) // FU-A Fragmentation unit (FU)
return ret;
}
+ /**
+ * {@inheritDoc}
+ *
+ * Makes sure that the format parameters of a
+ * <tt>ParameterizedVideoFormat</tt> input which are of no concern to this
+ * <tt>DePacketizer</tt> get passed on through the output to the next
+ * <tt>Codec</tt> in the codec chain (i.e. <tt>JNIDecoder</tt>).
+ */
+ @Override
+ protected Format[] getMatchingOutputFormats(Format inputFormat)
+ {
+ Format[] matchingOutputFormats
+ = super.getMatchingOutputFormats(inputFormat);
+
+ if ((matchingOutputFormats != null)
+ && (matchingOutputFormats.length != 0)
+ && (inputFormat instanceof ParameterizedVideoFormat))
+ {
+ Map<String,String> fmtps
+ = ((ParameterizedVideoFormat) inputFormat)
+ .getFormatParameters();
+
+ if (fmtps != null)
+ {
+ fmtps.remove(VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP);
+ if (!fmtps.isEmpty())
+ {
+ for (int i = 0; i < matchingOutputFormats.length; i++)
+ {
+ matchingOutputFormats[i]
+ = new ParameterizedVideoFormat(
+ Constants.H264,
+ fmtps);
+ }
+ }
+ }
+ }
+ return matchingOutputFormats;
+ }
+
/**
* Appends {@link #outputPaddingSize} number of bytes to <tt>out</tt>
* beginning at index <tt>outOffset</tt>. The specified <tt>out</tt> is
diff --git a/src/org/jitsi/impl/neomedia/codec/video/h264/JNIDecoder.java b/src/org/jitsi/impl/neomedia/codec/video/h264/JNIDecoder.java
index 11fcb16e..160b8b29 100644
--- a/src/org/jitsi/impl/neomedia/codec/video/h264/JNIDecoder.java
+++ b/src/org/jitsi/impl/neomedia/codec/video/h264/JNIDecoder.java
@@ -7,16 +7,20 @@
package org.jitsi.impl.neomedia.codec.video.h264;
import java.awt.*;
+import java.io.*;
import javax.media.*;
import javax.media.format.*;
+import net.iharder.*;
import net.sf.fmj.media.*;
import org.jitsi.impl.neomedia.codec.*;
import org.jitsi.impl.neomedia.codec.video.*;
+import org.jitsi.impl.neomedia.format.*;
import org.jitsi.service.neomedia.codec.*;
import org.jitsi.service.neomedia.control.*;
+import org.jitsi.util.*;
/**
* Decodes H.264 NAL units and returns the resulting frames as FFmpeg
@@ -35,6 +39,12 @@ public class JNIDecoder
private static final VideoFormat[] DEFAULT_OUTPUT_FORMATS
= new VideoFormat[] { new AVFrameFormat(FFmpeg.PIX_FMT_YUV420P) };
+ /**
+ * The <tt>Logger</tt> used by the <tt>JNIDecoder</tt> class and its
+ * instances to print debug-related information.
+ */
+ private static final Logger logger = Logger.getLogger(JNIDecoder.class);
+
/**
* Plugin name.
*/
@@ -87,7 +97,19 @@ public class JNIDecoder
*/
public JNIDecoder()
{
- inputFormats = new VideoFormat[] { new VideoFormat(Constants.H264) };
+ inputFormats
+ = new VideoFormat[]
+ {
+ /*
+ * Explicitly state both ParameterizedVideoFormat (to
+ * receive any format parameters which may be of concern
+ * to this JNIDecoder) and VideoFormat (to make sure
+ * that nothing breaks because of equality and/or
+ * matching tests involving ParameterizedVideoFormat).
+ */
+ new ParameterizedVideoFormat(Constants.H264),
+ new VideoFormat(Constants.H264)
+ };
outputFormats = DEFAULT_OUTPUT_FORMATS;
}
@@ -183,14 +205,18 @@ public Format[] getSupportedOutputFormats(Format inputFormat)
Format[] supportedOutputFormats;
if (inputFormat == null)
+ {
supportedOutputFormats = outputFormats;
+ }
else
{
// mismatch input format
if (!(inputFormat instanceof VideoFormat)
|| (AbstractCodec2.matches(inputFormat, inputFormats)
== null))
- supportedOutputFormats = new Format[0];
+ {
+ supportedOutputFormats = AbstractCodec2.EMPTY_FORMATS;
+ }
else
{
// match input format
@@ -200,6 +226,86 @@ public Format[] getSupportedOutputFormats(Format inputFormat)
return supportedOutputFormats;
}
+ /**
+ * Handles any format parameters of the input and/or output <tt>Format</tt>s
+ * with which this <tt>JNIDecoder</tt> has been configured. For example,
+ * takes into account the format parameter <tt>sprop-parameter-sets</tt> if
+ * it is specified by the input <tt>Format</tt>.
+ */
+ private void handleFmtps()
+ {
+ try
+ {
+
+ Format f = getInputFormat();
+
+ if (f instanceof ParameterizedVideoFormat)
+ {
+ ParameterizedVideoFormat pvf = (ParameterizedVideoFormat) f;
+ String spropParameterSets
+ = pvf.getFormatParameter(
+ VideoMediaFormatImpl.H264_SPROP_PARAMETER_SETS_FMTP);
+
+ if (spropParameterSets != null)
+ {
+ ByteArrayOutputStream nals = new ByteArrayOutputStream();
+
+ for (String s : spropParameterSets.split(","))
+ {
+ if ((s != null) && (s.length() != 0))
+ {
+ byte[] nal = Base64.decode(s);
+
+ if ((nal != null) && (nal.length != 0))
+ {
+ nals.write(DePacketizer.NAL_PREFIX);
+ nals.write(nal);
+ }
+ }
+ }
+ if (nals.size() != 0)
+ {
+ // Add padding because it seems to be required by FFmpeg.
+ for (int i = 0;
+ i < FFmpeg.FF_INPUT_BUFFER_PADDING_SIZE;
+ i++)
+ {
+ nals.write(0);
+ }
+
+ /*
+ * In accord with RFC 6184 "RTP Payload Format for H.264
+ * Video", place the NAL units conveyed by
+ * sprop-parameter-sets in the NAL unit stream to precede
+ * any other NAL units in decoding order.
+ */
+ FFmpeg.avcodec_decode_video(
+ avctx,
+ avframe.getPtr(),
+ got_picture,
+ nals.toByteArray(), nals.size());
+ }
+ }
+ }
+
+ /*
+ * Because the handling of format parameter is new at the time of this
+ * writing and it currently handles only the format parameter
+ * sprop-parameter-sets the failed handling of which will be made
+ * visible later on anyway, do not let it kill this JNIDecoder.
+ */
+ }
+ catch (Throwable t)
+ {
+ if (t instanceof InterruptedException)
+ Thread.currentThread().interrupt();
+ else if (t instanceof ThreadDeath)
+ throw (ThreadDeath) t;
+ else
+ logger.error("Failed to handle format parameters", t);
+ }
+ }
+
/**
* Inits the codec instances.
*
@@ -236,6 +342,13 @@ public synchronized void open()
opened = true;
super.open();
+
+ /*
+ * After this JNIDecoder has been opened, handle format parameters such
+ * as sprop-parameter-sets which require this JNIDecoder to be in the
+ * opened state.
+ */
+ handleFmtps();
}
/**
@@ -265,7 +378,7 @@ public synchronized int process(Buffer in, Buffer out)
// Ask FFmpeg to decode.
got_picture[0] = false;
- // TODO Take into account the offset of inputBuffer.
+ // TODO Take into account the offset of the input Buffer.
FFmpeg.avcodec_decode_video(
avctx,
avframe.getPtr(),
@@ -317,7 +430,9 @@ public synchronized int process(Buffer in, Buffer out)
long pts = FFmpeg.AV_NOPTS_VALUE; // TODO avframe_get_pts(avframe);
if (pts == FFmpeg.AV_NOPTS_VALUE)
+ {
out.setTimeStamp(Buffer.TIME_UNKNOWN);
+ }
else
{
out.setTimeStamp(pts);
diff --git a/src/org/jitsi/impl/neomedia/codec/video/h264/JNIEncoder.java b/src/org/jitsi/impl/neomedia/codec/video/h264/JNIEncoder.java
index ea39d84e..5c598723 100644
--- a/src/org/jitsi/impl/neomedia/codec/video/h264/JNIEncoder.java
+++ b/src/org/jitsi/impl/neomedia/codec/video/h264/JNIEncoder.java
@@ -132,12 +132,6 @@ public class JNIEncoder
*/
private static final Logger logger = Logger.getLogger(JNIEncoder.class);
- /**
- * The name of the format parameter which specifies the packetization mode
- * of H.264 RTP payload.
- */
- public static final String PACKETIZATION_MODE_FMTP = "packetization-mode";
-
/**
* Minimum interval between two PLI request processing (in milliseconds).
*/
@@ -166,10 +160,10 @@ public class JNIEncoder
= {
new ParameterizedVideoFormat(
Constants.H264,
- PACKETIZATION_MODE_FMTP, "0"),
+ VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP, "0"),
new ParameterizedVideoFormat(
Constants.H264,
- PACKETIZATION_MODE_FMTP, "1")
+ VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP, "1")
};
public static final int X264_KEYINT_MAX_INFINITE = 1 << 30;
@@ -364,7 +358,8 @@ private Format[] getMatchingOutputFormats(Format inputFormat)
Format.byteArray,
frameRate,
ParameterizedVideoFormat.toMap(
- PACKETIZATION_MODE_FMTP,
+ VideoMediaFormatImpl
+ .H264_PACKETIZATION_MODE_FMTP,
packetizationModes[index]));
}
return matchingOutputFormats;
@@ -916,7 +911,11 @@ public Format setOutputFormat(Format format)
if (fmtps == null)
fmtps = new HashMap<String, String>();
if (packetizationMode != null)
- fmtps.put(PACKETIZATION_MODE_FMTP, packetizationMode);
+ {
+ fmtps.put(
+ VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP,
+ packetizationMode);
+ }
outputFormat
= new ParameterizedVideoFormat(
diff --git a/src/org/jitsi/impl/neomedia/codec/video/h264/Packetizer.java b/src/org/jitsi/impl/neomedia/codec/video/h264/Packetizer.java
index 3aa682ba..59d41984 100644
--- a/src/org/jitsi/impl/neomedia/codec/video/h264/Packetizer.java
+++ b/src/org/jitsi/impl/neomedia/codec/video/h264/Packetizer.java
@@ -47,10 +47,10 @@ public class Packetizer
= {
new ParameterizedVideoFormat(
Constants.H264_RTP,
- JNIEncoder.PACKETIZATION_MODE_FMTP, "0"),
+ VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP, "0"),
new ParameterizedVideoFormat(
Constants.H264_RTP,
- JNIEncoder.PACKETIZATION_MODE_FMTP, "1")
+ VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP, "1")
};
/**
@@ -150,7 +150,8 @@ private Format[] getMatchingOutputFormats(Format input)
Format.byteArray,
frameRate,
ParameterizedVideoFormat.toMap(
- JNIEncoder.PACKETIZATION_MODE_FMTP,
+ VideoMediaFormatImpl
+ .H264_PACKETIZATION_MODE_FMTP,
packetizationMode))
};
}
@@ -182,7 +183,7 @@ private String getPacketizationMode(Format format)
if (format instanceof ParameterizedVideoFormat)
packetizationMode
= ((ParameterizedVideoFormat) format).getFormatParameter(
- JNIEncoder.PACKETIZATION_MODE_FMTP);
+ VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP);
if (packetizationMode == null)
packetizationMode = "0";
@@ -535,10 +536,11 @@ public Format setOutputFormat(Format format)
fmtps = ((ParameterizedVideoFormat) format).getFormatParameters();
if (fmtps == null)
fmtps = new HashMap<String, String>();
- if (fmtps.get(JNIEncoder.PACKETIZATION_MODE_FMTP) == null)
+ if (fmtps.get(VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP)
+ == null)
{
fmtps.put(
- JNIEncoder.PACKETIZATION_MODE_FMTP,
+ VideoMediaFormatImpl.H264_PACKETIZATION_MODE_FMTP,
getPacketizationMode(inputFormat));
}
diff --git a/src/org/jitsi/impl/neomedia/device/VideoMediaDeviceSession.java b/src/org/jitsi/impl/neomedia/device/VideoMediaDeviceSession.java
index 0fa92239..85eda9b0 100644
--- a/src/org/jitsi/impl/neomedia/device/VideoMediaDeviceSession.java
+++ b/src/org/jitsi/impl/neomedia/device/VideoMediaDeviceSession.java
@@ -1110,8 +1110,7 @@ protected void playerConfigureComplete(Processor player)
SwScale playerScaler = null;
if ((trackControls != null) && (trackControls.length != 0)
- /* We do not add scaler nor key frames control on Android
- for now */
+ /* We don't add SwScale, KeyFrameControl on Android. */
&& !OSUtils.IS_ANDROID)
{
String fmjEncoding = getFormat().getJMFEncoding();
@@ -1715,7 +1714,8 @@ protected Format setProcessorFormat(
= (formatParameters == null)
? null
: formatParameters.get(
- JNIEncoder.PACKETIZATION_MODE_FMTP);
+ VideoMediaFormatImpl
+ .H264_PACKETIZATION_MODE_FMTP);
encoder.setPacketizationMode(packetizationMode);
}
diff --git a/src/org/jitsi/impl/neomedia/format/VideoMediaFormatImpl.java b/src/org/jitsi/impl/neomedia/format/VideoMediaFormatImpl.java
index fcb19dd5..15af476f 100644
--- a/src/org/jitsi/impl/neomedia/format/VideoMediaFormatImpl.java
+++ b/src/org/jitsi/impl/neomedia/format/VideoMediaFormatImpl.java
@@ -30,6 +30,16 @@ public class VideoMediaFormatImpl
*/
public static final double DEFAULT_CLOCK_RATE = 90000;
+ /**
+ * The name of the format parameter which specifies the packetization mode
+ * of H.264 RTP payload.
+ */
+ public static final String H264_PACKETIZATION_MODE_FMTP
+ = "packetization-mode";
+
+ public static final String H264_SPROP_PARAMETER_SETS_FMTP
+ = "sprop-parameter-sets";
+
/**
* The clock rate of this <tt>VideoMediaFormat</tt>.
*/
@@ -218,7 +228,7 @@ public static boolean formatParametersAreEqual(
if ("H264".equalsIgnoreCase(encoding)
|| "h264/rtp".equalsIgnoreCase(encoding))
{
- String packetizationMode = "packetization-mode";
+ String packetizationMode = H264_PACKETIZATION_MODE_FMTP;
String pm1 = null;
String pm2 = null;
@@ -287,7 +297,7 @@ public static boolean formatParametersMatch(
if ("H264".equalsIgnoreCase(encoding)
|| "h264/rtp".equalsIgnoreCase(encoding))
{
- String packetizationMode = "packetization-mode";
+ String packetizationMode = H264_PACKETIZATION_MODE_FMTP;
String pm1
= (fmtps1 == null) ? null : fmtps1.get(packetizationMode);
String pm2
--
GitLab