一. MediaMuxer API 简介
MediaExtractor 是什么？
顾名思义，MediaExtractor 可以从数据源中提取经过编码的媒体数据。MediaExtractor 不仅可以解析本地媒体文件，还可以解析网络媒体资源。
MediaMuxer 是什么？
同样，名字已经说明了一切。MediaMuxer 可以将多个流混合封装起来，支持 MP4、Webm 和 3GP 文件作为输出，而且从 Android N 开始，已经支持在 MP4 中混合 B 帧了。
这次的任务是什么？
这次的任务是从一个 MP4 文件中只提取视频数据，并封装为一个新的 MP4 文件。外在表现就是将一个有声视频，转换为一个无声视频。
二. MediaExtractor
MediaExtractor 的作用就是将音频和视频分离。
主要是以下几个步骤：
2.1 创建实例
MediaExtractor mediaExtractor = new MediaExtractor();
2.2 设置数据源
mediaExtractor.setDataSource(path);
2.3 获取数据源的轨道数，切换到想要的轨道
// 轨道索引 int videoIndex = -1;
// 视频轨道格式信息
MediaFormat mediaFormat = null;
// 数据源的轨道数
int trackCount = mediaExtractor.getTrackCount();
for (int i = 0; i < trackCount; i++) {
MediaFormat format = mediaExtractor.getTrackFormat(i);
String mimeType = format.getString(MediaFormat.KEY_MIME);
if (mimeType.startsWith("video/")) {
videoIndex = i; mediaFormat = format; break;
}
}// 切换到想要的轨道 mediaExtractor.selectTrack(videoIndex);
2.4 对所需轨道数据循环读取读取每帧，进行处理
while (true) { // 将样本数据存储到字节缓存区
int readSampleSize = mediaExtractor.readSampleData(byteBuffer, 0);
// 如果没有可获取的样本，退出循环
if (readSampleSize < 0) {
mediaExtractor.unselectTrack(videoIndex);
break;
}
... ...
// 读取下一帧数据 mediaExtractor.advance(); }
2.5 完成后释放资源
mediaExtractor.release();
三. MediaMuxer
MediaMuxer 的作用是生成音频或视频文件；还可以把音频与视频混合成一个音视频文件。
主要是以下几个步骤：
3.1 创建实例
MediaMuxermediaMuxer = new MediaMuxer(path, format);
3.2 将音频轨或视频轨添加到 MediaMuxer，返回新的轨道
int trackIndex = mediaMuxer.addTrack(videoFormat);
3.3 开始合成
mediaMuxer.start();
3.4 循环将音频轨或视频轨的数据写到文件
while (true) {
// 将样本数据存储到字节缓存区
int readSampleSize = mediaExtractor.readSampleData(byteBuffer, 0);
// 如果没有可获取的样本，退出循环
if (readSampleSize < 0) {
mediaExtractor.unselectTrack(videoIndex);
break;
}
bufferInfo.size = readSampleSize;
bufferInfo.flags = mediaExtractor.getSampleFlags();
bufferInfo.offset = 0;
bufferInfo.presentationTimeUs = mediaExtractor.getSampleTime();
mediaMuxer.writeSampleData(trackIndex, byteBuffer, bufferInfo);
// 读取下一帧数据
mediaExtractor.advance();
mediaMuxer.writeSampleData(trackIndex, byteBuffer, bufferInfo);
// 读取下一帧数据
mediaExtractor.advance();
}
3.5 完成后释放资源
mediaMuxer.stop();
mediaMuxer.release();
四. 实例
/*** 分离视频的视频轨，输入视频 input.mp4，输出 output_video.mp4 */
private void extractVideo() {
MediaExtractor mediaExtractor = new MediaExtractor();
MediaMuxer mediaMuxer = null;
File fileDir = FileUtil.getExternalAssetsDir(this);
try {
// 设置视频源
mediaExtractor.setDataSource(new File(fileDir, VIDEO_SOURCE).getAbsolutePath());
// 轨道索引
int videoIndex = -1;
// 视频轨道格式信息
MediaFormat mediaFormat = null;
// 数据源的轨道数
int trackCount = mediaExtractor.getTrackCount();
for (int i = 0; i < trackCount; i++) {
MediaFormat format = mediaExtractor.getTrackFormat(i);
String mimeType = format.getString(MediaFormat.KEY_MIME);
if (mimeType.startsWith("video/")) {
videoIndex = i;
mediaFormat = format;
break;
}
}
// 切换到想要的轨道
mediaExtractor.selectTrack(videoIndex);
File outFile = new File(FileUtil.getMuxerAndExtractorDir(this), OUTPUT_VIDEO);
if (outFile.exists()) {
outFile.delete();
}
mediaMuxer = new MediaMuxer(outFile.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
// 将视频轨添加到 MediaMuxer，返回新的轨道
int trackIndex = mediaMuxer.addTrack(mediaFormat);
ByteBuffer byteBuffer = ByteBuffer.allocate(mediaFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE));
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
mediaMuxer.start();
while (true) {
// 将样本数据存储到字节缓存区
int readSampleSize = mediaExtractor.readSampleData(byteBuffer, 0);
// 如果没有可获取的样本，退出循环
if (readSampleSize < 0) {
mediaExtractor.unselectTrack(videoIndex);
break;
}
bufferInfo.size = readSampleSize;
bufferInfo.flags = mediaExtractor.getSampleFlags();
bufferInfo.offset = 0;
bufferInfo.presentationTimeUs = mediaExtractor.getSampleTime();
mediaMuxer.writeSampleData(trackIndex, byteBuffer, bufferInfo);
// 读取下一帧数据
mediaExtractor.advance();
}
Toast.makeText(this, "分离视频完成", Toast.LENGTH_SHORT).show();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (mediaMuxer != null) {
mediaMuxer.stop();
mediaMuxer.release();
}
mediaExtractor.release();
}
}
五. 核心代码
public class MusicProcess {
private static int TIMEOUT = 1000;
public static void mixAudioTrack(Context context,
final String videoInput,
final String audioInput,
final String output,
final Integer startTimeUs, final Integer endTimeUs,
int videoVolume,
int aacVolume
) throws Exception {
File cacheDir = Environment.getExternalStorageDirectory();
// 音乐转换城pcm
File aacPcmFile = new File(cacheDir, "audio" + ".pcm");
// 视频自带的音乐转换城pcm
final File videoPcmFile = new File(cacheDir, "video" + ".pcm");
MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
mediaMetadataRetriever.setDataSource(audioInput);
// 读取音乐时间
final int aacDurationMs = Integer.parseInt(mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION));
mediaMetadataRetriever.release();
MediaExtractor audioExtractor = new MediaExtractor();
audioExtractor.setDataSource(audioInput);
decodeToPCM(videoInput, videoPcmFile.getAbsolutePath(), startTimeUs, endTimeUs);
decodeToPCM(audioInput, aacPcmFile.getAbsolutePath(), startTimeUs, endTimeUs );
File adjustedPcm = new File(cacheDir, "混合后的" + ".pcm");
mixPcm(videoPcmFile.getAbsolutePath(), aacPcmFile.getAbsolutePath(), adjustedPcm.getAbsolutePath() , videoVolume, aacVolume);
File wavFile = new File(cacheDir, adjustedPcm.getName() + ".wav");
new PcmToWavUtil(44100, AudioFormat.CHANNEL_IN_STEREO,
2, AudioFormat.ENCODING_PCM_16BIT).pcmToWav(adjustedPcm.getAbsolutePath()
, wavFile.getAbsolutePath());
mixVideoAndMusic(videoInput, output, startTimeUs, endTimeUs, wavFile);
}
private static void mixVideoAndMusic(String videoInput, String output, Integer startTimeUs, Integer endTimeUs, File wavFile) throws IOException {
MediaMuxer mediaMuxer = new MediaMuxer(output,MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
//先取视频
MediaExtractor mediaExtractor = new MediaExtractor();
mediaExtractor.setDataSource(videoInput);
int videoIndex = selectTrack(mediaExtractor, false);
int audioIndex = selectTrack(mediaExtractor, true);
MediaFormat videoFormat= mediaExtractor.getTrackFormat(videoIndex);
mediaMuxer.addTrack(videoFormat);
MediaFormat audioFormat = mediaExtractor.getTrackFormat(audioIndex);
int audioBitrate = audioFormat.getInteger(MediaFormat.KEY_BIT_RATE);
audioFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_AAC);
int muxerAudioIndex = mediaMuxer.addTrack(audioFormat);
mediaMuxer.start();
//音频的wav
MediaExtractor pcmExtrator = new MediaExtractor();
pcmExtrator.setDataSource(wavFile.getAbsolutePath());
int audioTrack = selectTrack(pcmExtrator, true);
pcmExtrator.selectTrack(audioTrack);
MediaFormat pcmTrackFormat = pcmExtrator.getTrackFormat(audioTrack);
int maxBufferSize = 0;
try {
if (audioFormat.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE)) {
maxBufferSize = pcmTrackFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE);
}
} catch (Exception e) {
maxBufferSize = 100 * 1000;
}
MediaFormat encodeFormat = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC,
44100, 2);//参数对应-> mime type、采样率、声道数
encodeFormat.setInteger(MediaFormat.KEY_BIT_RATE, audioBitrate);//比特率
encodeFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);//音质等级
encodeFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, maxBufferSize);
MediaCodec encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
encoder.configure(encodeFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
encoder.start();
ByteBuffer buffer = ByteBuffer.allocateDirect(maxBufferSize);
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean encodeDone = false;
while (!encodeDone) {
int inputBufferIndex = encoder.dequeueInputBuffer(10000);
if (inputBufferIndex >= 0) {
long sampleTime = pcmExtrator.getSampleTime();
if (sampleTime < 0) {
//文件末尾
encoder.queueInputBuffer(inputBufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
int flags = pcmExtrator.getSampleFlags();
int size = pcmExtrator.readSampleData(buffer, 0);
ByteBuffer inputBuffer = encoder.getInputBuffer(inputBufferIndex);
inputBuffer.clear();
inputBuffer.put(buffer);
inputBuffer.position(0);
encoder.queueInputBuffer(inputBufferIndex, 0, size, sampleTime, flags);
pcmExtrator.advance();
}
}
//获取编码完的数据
int outputBufferIndex = encoder.dequeueOutputBuffer(info, TIMEOUT);
while (outputBufferIndex>=0) {
if (info.flags == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
encodeDone = true;
break;
}
ByteBuffer encodeOutputBuffer = encoder.getOutputBuffer(outputBufferIndex);
//将编码好的数据 压缩 aac
mediaMuxer.writeSampleData(muxerAudioIndex, encodeOutputBuffer , info);
encodeOutputBuffer.clear();
encoder.releaseOutputBuffer(outputBufferIndex, false);
outputBufferIndex = encoder.dequeueOutputBuffer(info, TIMEOUT);
}
}
if (audioTrack >= 0) {
mediaExtractor.unselectTrack(audioTrack);
}
//视频
mediaExtractor.selectTrack(videoIndex);
mediaExtractor.seekTo(startTimeUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
maxBufferSize = videoFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE);
buffer = ByteBuffer.allocateDirect(maxBufferSize);
while (true) {
long sampleTimeUs = mediaExtractor.getSampleTime();
if (sampleTimeUs == -1) {
break;
}
if (sampleTimeUs < startTimeUs) {
mediaExtractor.advance();
continue;
}
if (endTimeUs != null && sampleTimeUs > endTimeUs) {
break;
}
info.presentationTimeUs = sampleTimeUs - startTimeUs+600;
info.flags = mediaExtractor.getSampleFlags();
info.size = mediaExtractor.readSampleData(buffer, 0);
if (info.size < 0) {
break;
}
mediaMuxer.writeSampleData(videoIndex, buffer, info);
mediaExtractor.advance();
}
try {
pcmExtrator.release();
mediaExtractor.release();
encoder.stop();
encoder.release();
mediaMuxer.release();
} catch (Exception e) {
}
}
public static void mixPcm(String pcm1Path, String pcm2Path, String toPath
, int vol1, int vol2) throws IOException {
float volume1 = normalizeVolume(vol1);
float volume2 = normalizeVolume(vol2);
byte[] buffer1 = new byte[2048];
byte[] buffer2 = new byte[2048];
byte[] buffer3 = new byte[2048];
FileInputStream is1 = new FileInputStream(pcm1Path);
FileInputStream is2 = new FileInputStream(pcm2Path);
FileOutputStream fileOutputStream = new FileOutputStream(toPath);
boolean end1 = false, end2 = false;
short temp2, temp1;
int temp;
try {
while (!end1 || !end2) {
if (!end1) {
end1 = (is1.read(buffer1) == -1);
System.arraycopy(buffer1, 0, buffer3, 0, buffer1.length);
}
if (!end2) {
end2 = (is2.read(buffer2) == -1);
int voice = 0;
for (int i = 0; i < buffer2.length; i += 2) {
temp1 = (short) ((buffer1[i] & 0xff) | (buffer1[i + 1] & 0xff) << 8);
temp2 = (short) ((buffer2[i] & 0xff) | (buffer2[i + 1] & 0xff) << 8);
temp = (int) (temp2 * volume2 + temp1 * volume1);
if (temp > 32767) {
temp = 32767;
} else if (temp < -32768) {
temp = -32768;
}
buffer3[i] = (byte) (temp & 0xFF);
buffer3[i + 1] = (byte) ((temp >>> 8) & 0xFF);
}
}
fileOutputStream.write(buffer3);
}
} finally {
is1.close();
is2.close();
fileOutputStream.close();
}
}
private static float normalizeVolume(int volume) {
return volume / 100f * 1;
}
//MP3截取并且输出pcm
public static void decodeToPCM(String musicPath, String outPath, int startTime, int endTime) throws Exception {
if (endTime < startTime) {
return;
}
MediaExtractor mediaExtractor = new MediaExtractor();
mediaExtractor.setDataSource(musicPath);
int audioTrack = selectTrack(mediaExtractor,true );
mediaExtractor.selectTrack(audioTrack);
mediaExtractor.seekTo(startTime, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
MediaFormat audioFormat = mediaExtractor.getTrackFormat(audioTrack);
int maxBufferSize = 100 * 1000;
if (audioFormat.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE)) {
maxBufferSize = audioFormat.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE);
} else {
maxBufferSize = 100 * 1000;
}
ByteBuffer buffer = ByteBuffer.allocateDirect(maxBufferSize);
MediaCodec mediaCodec = MediaCodec.createDecoderByType(audioFormat.getString((MediaFormat.KEY_MIME)));
mediaCodec.configure(audioFormat, null, null, 0);
File pcmFile = new File(outPath);
FileChannel writeChannel = new FileOutputStream(pcmFile).getChannel();
mediaCodec.start();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int outputBufferIndex = -1;
while (true) {
int decodeInputIndex = mediaCodec.dequeueInputBuffer(1000);
if (decodeInputIndex >= 0) {
long sampleTimeUs = mediaExtractor.getSampleTime();
if (sampleTimeUs == -1) {
break;
} else if (sampleTimeUs < startTime) {
mediaExtractor.advance();
continue;
} else if (sampleTimeUs > endTime) {
break;
}
info.size = mediaExtractor.readSampleData(buffer, 0);
info.presentationTimeUs = sampleTimeUs;
info.flags = mediaExtractor.getSampleFlags();
byte[] content = new byte[buffer.remaining()];
buffer.get(content);
ByteBuffer inputBuffer = mediaCodec.getInputBuffer(decodeInputIndex);
inputBuffer.put(content);
mediaCodec.queueInputBuffer(decodeInputIndex, 0, info.size, info.presentationTimeUs, info.flags);
mediaExtractor.advance();
}
outputBufferIndex = mediaCodec.dequeueOutputBuffer(info, 1_000);
while (outputBufferIndex>=0) {
ByteBuffer decodeOutputBuffer = mediaCodec.getOutputBuffer(outputBufferIndex);
writeChannel.write(decodeOutputBuffer);//MP3 1 pcm2
mediaCodec.releaseOutputBuffer(outputBufferIndex, false);
outputBufferIndex = mediaCodec.dequeueOutputBuffer(info, 1_000);
}
}
writeChannel.close();
mediaExtractor.release();
mediaCodec.stop();
mediaCodec.release();
Log.i("MusicProcess", "decodeToPCM: 转换完毕");
}
public static int selectTrack(MediaExtractor extractor, boolean audio) {
int numTracks = extractor.getTrackCount();
for (int i = 0; i < numTracks; i++) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (audio) {
if (mime.startsWith("audio/")) {
return i;
}
} else {
if (mime.startsWith("video/")) {
return i;
}
}
}
return -5;
}
}
六. 代码地址
https://gitee.com/luisliuyi/android-video-audio-mix.git
网友评论