Skip to content

Commit

Permalink
AAC streaming with the MediaCodec API working. H264Packetizer now add…
Browse files Browse the repository at this point in the history 
…s NALU type 7 and 8 in the stream.
  • Loading branch information
fyhertz committed Oct 6, 2013
1 parent 819481b commit c051416
Show file tree
Hide file tree
Showing 11 changed files with 228 additions and 121 deletions.
130 changes: 71 additions & 59 deletions src/net/majorkernelpanic/streaming/audio/AACStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.nio.Buffer;
import java.nio.ByteBuffer;

import net.majorkernelpanic.streaming.rtp.AACADTSPacketizer;
Expand All @@ -51,8 +51,6 @@ public class AACStream extends AudioStream {

public final static String TAG = "AACStream";

protected AudioQuality mQuality = new AudioQuality(32000,32000);

/** MPEG-4 Audio Object Types supported by ADTS. **/
private static final String[] AUDIO_OBJECT_TYPES = {
"NULL", // 0
Expand All @@ -63,7 +61,7 @@ public class AACStream extends AudioStream {
};

/** There are 13 supported frequencies by ADTS. **/
public static final int[] ADTS_SAMPLING_RATES = {
public static final int[] AUDIO_SAMPLING_RATES = {
96000, // 0
88200, // 1
64000, // 2
Expand All @@ -86,16 +84,21 @@ public class AACStream extends AudioStream {
private int mProfile, mSamplingRateIndex, mChannel, mConfig;
private SharedPreferences mSettings = null;
private AudioRecord mAudioRecord = null;
private Thread mThread = null, mThread2 = null;
private Thread mThread = null;

public AACStream() throws IOException {
super();

if (!AACStreamingSupported()) {
Log.e(TAG,"AAC not supported on this phone");
throw new AACNotSupportedException();
}

if (mMode == MODE_MEDIARECORDER_API)
mPacketizer = new AACADTSPacketizer();
else
mPacketizer = new AACLATMPacketizer();

}

private static boolean AACStreamingSupported() {
Expand Down Expand Up @@ -125,9 +128,7 @@ public void start() throws IllegalStateException, IOException {
@SuppressLint("InlinedApi")
protected void encodeWithMediaRecorder() throws IOException {
testADTS();
if (mPacketizer == null || mPacketizer.getClass() != AACADTSPacketizer.class) {
mPacketizer = new AACADTSPacketizer();
}
((AACADTSPacketizer)mPacketizer).setSamplingRate(mActualSamplingRate);
setAudioEncoder(MediaRecorder.AudioEncoder.AAC);
try {
Field name = MediaRecorder.OutputFormat.class.getField("AAC_ADTS");
Expand All @@ -140,25 +141,35 @@ protected void encodeWithMediaRecorder() throws IOException {
@Override
@SuppressLint({ "InlinedApi", "NewApi" })
protected void encodeWithMediaCodec() throws IOException {
if (mPacketizer == null || mPacketizer.getClass() != AACLATMPacketizer.class) {
mPacketizer = new AACLATMPacketizer();

// Checks if the user has supplied an exotic sampling rate
int i=0;
for (;i<AUDIO_SAMPLING_RATES.length;i++) {
if (AUDIO_SAMPLING_RATES[i] == mQuality.samplingRate) {
break;
}
}
// If he did, we force a reasonable one: 24 kHz
if (i>12) mQuality.samplingRate = 24000;

final int bufferSize = AudioRecord.getMinBufferSize(mQuality.samplingRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT);
mAudioRecord = new AudioRecord(MediaRecorder.AudioSource.MIC, mQuality.samplingRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_DEFAULT, bufferSize);

final int bufferSize = AudioRecord.getMinBufferSize(mQuality.samplingRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)*2;

((AACLATMPacketizer)mPacketizer).setSamplingRate(mQuality.samplingRate);

mAudioRecord = new AudioRecord(MediaRecorder.AudioSource.MIC, mQuality.samplingRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSize);
mMediaCodec = MediaCodec.createEncoderByType("audio/mp4a-latm");
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, "audio/mp4a-latm");
format.setInteger(MediaFormat.KEY_BIT_RATE, mQuality.bitRate);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, mQuality.samplingRate);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectHE);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, bufferSize);
mMediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mAudioRecord.startRecording();
mMediaCodec.start();

final InputStream inputStream = new MediaCodecInputStream(mMediaCodec);
final MediaCodecInputStream inputStream = new MediaCodecInputStream(mMediaCodec);
final ByteBuffer[] inputBuffers = mMediaCodec.getInputBuffers();

mThread = new Thread(new Runnable() {
Expand All @@ -167,48 +178,30 @@ public void run() {
int len = 0, bufferIndex = 0;
try {
while (!Thread.interrupted()) {
bufferIndex = mMediaCodec.dequeueInputBuffer(-1);
inputBuffers[bufferIndex].clear();
len = mAudioRecord.read(inputBuffers[bufferIndex], bufferSize);
if (len == AudioRecord.ERROR_INVALID_OPERATION || len == AudioRecord.ERROR_BAD_VALUE) {
Log.e(TAG,"An error occured with the AudioRecord API !");
} else {
mMediaCodec.queueInputBuffer(bufferIndex, 0, len, System.nanoTime()/1000, 0);
bufferIndex = mMediaCodec.dequeueInputBuffer(10000);
if (bufferIndex>=0) {
inputBuffers[bufferIndex].clear();
len = mAudioRecord.read(inputBuffers[bufferIndex], bufferSize);
if (len == AudioRecord.ERROR_INVALID_OPERATION || len == AudioRecord.ERROR_BAD_VALUE) {
Log.e(TAG,"An error occured with the AudioRecord API !");
} else {
//Log.v(TAG,"Pushing raw audio to the decoder: len="+len+" bs: "+inputBuffers[bufferIndex].capacity());
mMediaCodec.queueInputBuffer(bufferIndex, 0, len, System.nanoTime()/1000, 0);
}
}
}
} catch (RuntimeException e) {}
Log.e(TAG,"Thread 1 over");
}
});

mThread2 = new Thread(new Runnable() {
@Override
public void run() {

byte[] buffer = new byte[128*1024];
try {
while (!Thread.interrupted()) {
int len = inputStream.read(buffer, 0, 128*1024);
Log.d(TAG,"len: "+len);
}
} catch (IOException e) {
} catch (RuntimeException e) {
e.printStackTrace();
}
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
Log.e(TAG,"Thread 2 over");
//Log.e(TAG,"Thread 1 over");
}
});

mThread2.start();
mThread.start();

try {
// mReceiver.getInputStream contains the data from the camera
// the mPacketizer encapsulates this stream in an RTP stream and send it over the network
// the packetizer encapsulates this stream in an RTP stream and send it over the network
mPacketizer.setDestination(mDestination, mRtpPort, mRtcpPort);
mPacketizer.setInputStream(inputStream);
mPacketizer.start();
Expand All @@ -217,8 +210,6 @@ public void run() {
stop();
throw new IOException("Something happened with the local sockets :/ Start failed !");
}

mStreaming = true;

}

Expand All @@ -228,7 +219,6 @@ public synchronized void stop() {
if (mMode == MODE_MEDIACODEC_API) {
Log.d(TAG, "Interrupting threads...");
mThread.interrupt();
mThread2.interrupt();
mAudioRecord.stop();
mAudioRecord.release();
mAudioRecord = null;
Expand All @@ -242,17 +232,39 @@ public synchronized void stop() {
* Will fail if called when streaming.
*/
public String generateSessionDescription() throws IllegalStateException, IOException {
testADTS();

// All the MIME types parameters used here are described in RFC 3640
// SizeLength: 13 bits will be enough because ADTS uses 13 bits for frame length
// config: contains the object type + the sampling rate + the channel number
if (mMode == MODE_MEDIARECORDER_API) {

// TODO: streamType always 5 ? profile-level-id always 15 ?
testADTS();

// All the MIME types parameters used here are described in RFC 3640
// SizeLength: 13 bits will be enough because ADTS uses 13 bits for frame length
// config: contains the object type + the sampling rate + the channel number

// TODO: streamType always 5 ? profile-level-id always 15 ?

return "m=audio "+String.valueOf(getDestinationPorts()[0])+" RTP/AVP 96\r\n" +
"a=rtpmap:96 mpeg4-generic/"+mActualSamplingRate+"\r\n"+
"a=fmtp:96 streamtype=5; profile-level-id=15; mode=AAC-hbr; config="+Integer.toHexString(mConfig)+"; SizeLength=13; IndexLength=3; IndexDeltaLength=3;\r\n";

} else {

for (int i=0;i<AUDIO_SAMPLING_RATES.length;i++) {
if (AUDIO_SAMPLING_RATES[i] == mQuality.samplingRate) {
mSamplingRateIndex = i;
break;
}
}
mProfile = 2; // AAC LC
mChannel = 1;
mConfig = mProfile<<11 | mSamplingRateIndex<<7 | mChannel<<3;

return "m=audio "+String.valueOf(getDestinationPorts()[0])+" RTP/AVP 96\r\n" +
"a=rtpmap:96 mpeg4-generic/"+mQuality.samplingRate+"\r\n"+
"a=fmtp:96 streamtype=5; profile-level-id=15; mode=AAC-hbr; config="+Integer.toHexString(mConfig)+"; SizeLength=13; IndexLength=3; IndexDeltaLength=3;\r\n";

}

return "m=audio "+String.valueOf(getDestinationPorts()[0])+" RTP/AVP 96\r\n" +
"a=rtpmap:96 mpeg4-generic/"+mActualSamplingRate+"\r\n"+
"a=fmtp:96 streamtype=5; profile-level-id=15; mode=AAC-hbr; config="+Integer.toHexString(mConfig)+"; SizeLength=13; IndexLength=3; IndexDeltaLength=3;\r\n";
}

/**
Expand Down Expand Up @@ -322,7 +334,7 @@ private void testADTS() throws IllegalStateException, IOException {
mSamplingRateIndex = (buffer[1]&0x3C) >> 2;
mProfile = ( (buffer[1]&0xC0) >> 6 ) + 1 ;
mChannel = (buffer[1]&0x01) << 2 | (buffer[2]&0xC0) >> 6 ;
mActualSamplingRate = ADTS_SAMPLING_RATES[mSamplingRateIndex];
mActualSamplingRate = AUDIO_SAMPLING_RATES[mSamplingRateIndex];

// 5 bits for the object type / 4 bits for the sampling rate / 4 bits for the channel / padding
mConfig = mProfile<<11 | mSamplingRateIndex<<7 | mChannel<<3;
Expand Down
29 changes: 18 additions & 11 deletions src/net/majorkernelpanic/streaming/mp4/MP4Parser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@
* along with this source code; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/

package net.majorkernelpanic.streaming.mp4;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.HashMap;

import android.util.Base64;
Expand All @@ -40,7 +40,6 @@ class MP4Parser {
private HashMap<String, Long> boxes = new HashMap<String, Long>();
private final RandomAccessFile file;
private long pos = 0;
private byte[] buffer = new byte[8];

public MP4Parser(final String path) throws IOException, FileNotFoundException {
this.file = new RandomAccessFile(new File(path), "r");
Expand Down Expand Up @@ -85,20 +84,26 @@ public StsdBox getStsdBox() throws IOException {
}

private void parse(String path, long len) throws IOException {
byte[] buffer = new byte[8];
String name="";
long sum = 0, newlen = 0;

boxes.put(path, pos-8);
if(!path.equals(""))
boxes.put(path, pos-8);


while (sum<len) {

file.read(buffer,0,8); sum += 8; pos += 8;
if (validBoxName()) {
file.read(buffer,0,8);
sum += 8;
pos += 8;
if (validBoxName(buffer)) {

ByteBuffer byteBuffer = ByteBuffer.wrap(buffer,0,4);
newlen = byteBuffer.getInt()-8;

newlen = ( buffer[3]&0xFF | (buffer[2]&0xFF)<<8 | (buffer[1]&0xFF)<<16 | (buffer[0]&0xFF)<<24 ) - 8;
// 1061109559+8 correspond to "????" in ASCII the HTC Desire S seems to write that sometimes, maybe other phones do
if (newlen<=0 || newlen==1061109559) throw new IOException();
// "wide" atom would produce a newlen == 0, and we shouldn't throw an exception because of that
if (newlen < 0 || newlen == 1061109559) throw new IOException();
name = new String(buffer,4,4);
Log.d(TAG,"Atom -> name: "+name+" newlen: "+newlen+" pos: "+pos);
sum += newlen;
Expand All @@ -110,7 +115,8 @@ private void parse(String path, long len) throws IOException {
file.seek(file.getFilePointer() - 8 + len);
sum += len-8;
} else {
if (file.skipBytes((int) (len-8))<len-8) {
int skipped = file.skipBytes((int)(len-8));
if (skipped < ((int)(len-8))) {
throw new IOException();
}
pos += len-8;
Expand All @@ -120,9 +126,10 @@ private void parse(String path, long len) throws IOException {
}
}

private boolean validBoxName() {
private boolean validBoxName(byte[] buffer) {
for (int i=0;i<4;i++) {
if ((buffer[i+4]<97 || buffer[i+4]>122) && (buffer[i+4]<48 || buffer[i+4]>57) ) return false;
// If the next 4 bytes are neither lowercase letters nor numbers
if ((buffer[i+4]< 'a' || buffer[i+4]>'z') && (buffer[i+4]<'0'|| buffer[i+4]>'9') ) return false;
}
return true;
}
Expand Down
6 changes: 3 additions & 3 deletions src/net/majorkernelpanic/streaming/rtp/AACADTSPacketizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public void setSamplingRate(int samplingRate) {

public void run() {

Log.d(TAG,"AAC packetizer started !");
Log.d(TAG,"AAC ADTS packetizer started !");

// "A packet SHALL carry either one or more complete Access Units, or a
// single fragment of an Access Unit. Fragments of the same Access Unit
Expand Down Expand Up @@ -121,7 +121,7 @@ public void run() {
// Read CRS if any
if (!protection) is.read(header,0,2);

samplingRate = AACStream.ADTS_SAMPLING_RATES[(header[2]&0x3C) >> 2];
samplingRate = AACStream.AUDIO_SAMPLING_RATES[(header[2]&0x3C) >> 2];
profile = ( (header[2]&0xC0) >> 6 ) + 1 ;

// We update the RTP timestamp
Expand Down Expand Up @@ -181,7 +181,7 @@ public void run() {
e.printStackTrace();
} catch (InterruptedException ignore) {}

Log.d(TAG,"AAC packetizer stopped !");
Log.d(TAG,"AAC ADTS packetizer stopped !");

}

Expand Down
Loading

0 comments on commit c051416

Please sign in to comment.