- 从设备(手机)的摄像头、MIC中采集音频、视频的原始数据
ios的音视频采集可以从AVFoundation框架里采集
视频采集
这里我们选取GPUImage来采集视频，因为这个框架集成了很多视频滤镜，例如美颜
采集流程：
image.png
摄像头采集视频代码
GPUImageVideoCamera.m
// 从前摄像头或后摄像头获取视频
NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
for (AVCaptureDevice *device in devices)
{
if ([device position] == cameraPosition)
{
_inputCamera = device;
}
}
if (!_inputCamera) {
return nil;
}
// 创建采集回话session，session可以控制视频采集开始/暂停
_captureSession = [[AVCaptureSession alloc] init];
// 开始配置session
[_captureSession beginConfiguration];
// 视频输入加入到session
NSError *error = nil;
videoInput = [[AVCaptureDeviceInput alloc] initWithDevice:_inputCamera error:&error];
if ([_captureSession canAddInput:videoInput])
{
[_captureSession addInput:videoInput];
}
// 创建视频输出
videoOutput = [[AVCaptureVideoDataOutput alloc] init];
[videoOutput setAlwaysDiscardsLateVideoFrames:NO];
// if (captureAsYUV && [GPUImageContext deviceSupportsRedTextures])
if (captureAsYUV && [GPUImageContext supportsFastTextureUpload])
{
BOOL supportsFullYUVRange = NO;
NSArray *supportedPixelFormats = videoOutput.availableVideoCVPixelFormatTypes;
for (NSNumber *currentPixelFormat in supportedPixelFormats)
{
if ([currentPixelFormat intValue] == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
{
supportsFullYUVRange = YES;
}
}
if (supportsFullYUVRange)
{
[videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];
isFullYUVRange = YES;
}
else
{
[videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];
isFullYUVRange = NO;
}
}
else
{
[videoOutput setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];
}
runSynchronouslyOnVideoProcessingQueue(^{
if (captureAsYUV)
{
[GPUImageContext useImageProcessingContext];
// if ([GPUImageContext deviceSupportsRedTextures])
// {
// yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVVideoRangeConversionForRGFragmentShaderString];
// }
// else
// {
if (isFullYUVRange)
{
yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVFullRangeConversionForLAFragmentShaderString];
}
else
{
yuvConversionProgram = [[GPUImageContext sharedImageProcessingContext] programForVertexShaderString:kGPUImageVertexShaderString fragmentShaderString:kGPUImageYUVVideoRangeConversionForLAFragmentShaderString];
}
// }
if (!yuvConversionProgram.initialized)
{
[yuvConversionProgram addAttribute:@"position"];
[yuvConversionProgram addAttribute:@"inputTextureCoordinate"];
if (![yuvConversionProgram link])
{
NSString *progLog = [yuvConversionProgram programLog];
NSLog(@"Program link log: %@", progLog);
NSString *fragLog = [yuvConversionProgram fragmentShaderLog];
NSLog(@"Fragment shader compile log: %@", fragLog);
NSString *vertLog = [yuvConversionProgram vertexShaderLog];
NSLog(@"Vertex shader compile log: %@", vertLog);
yuvConversionProgram = nil;
NSAssert(NO, @"Filter shader link failed");
}
}
yuvConversionPositionAttribute = [yuvConversionProgram attributeIndex:@"position"];
yuvConversionTextureCoordinateAttribute = [yuvConversionProgram attributeIndex:@"inputTextureCoordinate"];
yuvConversionLuminanceTextureUniform = [yuvConversionProgram uniformIndex:@"luminanceTexture"];
yuvConversionChrominanceTextureUniform = [yuvConversionProgram uniformIndex:@"chrominanceTexture"];
yuvConversionMatrixUniform = [yuvConversionProgram uniformIndex:@"colorConversionMatrix"];
[GPUImageContext setActiveShaderProgram:yuvConversionProgram];
glEnableVertexAttribArray(yuvConversionPositionAttribute);
glEnableVertexAttribArray(yuvConversionTextureCoordinateAttribute);
}
});
[videoOutput setSampleBufferDelegate:self queue:cameraProcessingQueue];
// session增加视频输出
if ([_captureSession canAddOutput:videoOutput])
{
[_captureSession addOutput:videoOutput];
}
else
{
NSLog(@"Couldn't add video output");
return nil;
}
_captureSessionPreset = sessionPreset;
[_captureSession setSessionPreset:_captureSessionPreset];
// This will let you get 60 FPS video from the 720p preset on an iPhone 4S, but only that device and that preset
// AVCaptureConnection *conn = [videoOutput connectionWithMediaType:AVMediaTypeVideo];
//
// if (conn.supportsVideoMinFrameDuration)
// conn.videoMinFrameDuration = CMTimeMake(1,60);
// if (conn.supportsVideoMaxFrameDuration)
// conn.videoMaxFrameDuration = CMTimeMake(1,60);
// session提交配置
[_captureSession commitConfiguration];
上层API调用：
self.videoCamera = [[GPUImageVideoCamera alloc] initWithSessionPreset:_configuration.avSessionPreset cameraPosition:AVCaptureDevicePositionFront];
self.videoCamera.outputImageOrientation = _configuration.outputImageOrientation; // 设置视频输出方向，不然看到的视频会反向
self.videoCamera.horizontallyMirrorFrontFacingCamera = NO;
self.videoCamera.horizontallyMirrorRearFacingCamera = NO;
self.videoCamera.frameRate = (int32_t)_configuration.videoFrameRate; // 设置帧率
名词介绍
帧率
- 帧率表示图形处理器处理场时每秒钟能够更新的次数，即:每秒视频播放的图片数
- 人眼舒适放松时可视帧数是每秒24帧，集中精神时不超过30帧。眨眼时睁开眼瞬间可以捕捉到的帧数是30帧以上。
- 帧率过小会造成视频卡顿，帧率过大会造成视频过大。
- iOS默认输出的视频帧率为30帧/秒，普通用途的话设置成24～30就够用了。
音频采集
采集流程：
image.png
主要代码：
AVAudioSession *session = [AVAudioSession sharedInstance];
// 监听声音路线改变
[[NSNotificationCenter defaultCenter] addObserver: self
selector: @selector(handleRouteChange:)
name: AVAudioSessionRouteChangeNotification
object: session];
// 监听声音被打断
[[NSNotificationCenter defaultCenter] addObserver: self
selector: @selector(handleInterruption:)
name: AVAudioSessionInterruptionNotification
object: session];
/// 创建AudioComponent
AudioComponentDescription acd;
acd.componentType = kAudioUnitType_Output;
//acd.componentSubType = kAudioUnitSubType_VoiceProcessingIO;
acd.componentSubType = kAudioUnitSubType_RemoteIO;
acd.componentManufacturer = kAudioUnitManufacturer_Apple; // 厂商 直接写kAudioUnitManufacturer_Apple
acd.componentFlags = 0; // 没有明确值时必须设为0
acd.componentFlagsMask = 0; // 没有明确值时必须设为0
self.component = AudioComponentFindNext(NULL, &acd);
OSStatus status = noErr;
status = AudioComponentInstanceNew(self.component, &_componetInstance);
if (noErr != status) {
[self handleAudioComponentCreationFailure];
}
/// 连接麦克风
UInt32 flagOne = 1;
AudioUnitSetProperty(self.componetInstance, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, 1, &flagOne, sizeof(flagOne));
AudioStreamBasicDescription desc = {0};
desc.mSampleRate = _configuration.audioSampleRate; // 采样率
desc.mFormatID = kAudioFormatLinearPCM;
desc.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsPacked;
desc.mChannelsPerFrame = (UInt32)_configuration.numberOfChannels;
desc.mFramesPerPacket = 1;
desc.mBitsPerChannel = 16; // 表示每个声道的音频数据要多少位，一个字节是8位，所以用8 * 每个采样的字节数
desc.mBytesPerFrame = desc.mBitsPerChannel / 8 * desc.mChannelsPerFrame;
desc.mBytesPerPacket = desc.mBytesPerFrame * desc.mFramesPerPacket; // 根据mFormatFlags指定的Float类型非交错存储，就设置为bytesPerSample表示每个采样的字节数。但如果是Interleaved交错存储的，就应该设置为bytesPerSample * mChannelsPerFrame 因为左右声道数据是交错存在一起的。
// 连接扬声器
AudioUnitSetProperty(self.componetInstance, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 1, &desc, sizeof(desc));
// 设置声音回调
AURenderCallbackStruct cb;
cb.inputProcRefCon = (__bridge void *)(self);
cb.inputProc = handleInputBuffer;
AudioUnitSetProperty(self.componetInstance, kAudioOutputUnitProperty_SetInputCallback, kAudioUnitScope_Global, 1, &cb, sizeof(cb));
// 初始化AudioComponentInstance
status = AudioUnitInitialize(self.componetInstance);
if (noErr != status) {
[self handleAudioComponentCreationFailure];
}
[session setPreferredSampleRate:_configuration.audioSampleRate error:nil];
/// AVAudioSessionCategoryPlayAndRecord 支持音频播放和录音、打断其他不支持混音APP、不会被静音键或锁屏键静音
/// AVAudioSessionCategoryOptionDefaultToSpeaker 系统会自动选择最佳的内置麦克风组合支持视频聊天。
/// AVAudioSessionCategoryOptionInterruptSpokenAudioAndMixWithOthers 支持和其他APP音频混合
[session setCategory:AVAudioSessionCategoryPlayAndRecord withOptions:AVAudioSessionCategoryOptionDefaultToSpeaker | AVAudioSessionCategoryOptionInterruptSpokenAudioAndMixWithOthers error:nil];
[session setActive:YES withOptions:kAudioSessionSetActiveFlag_NotifyOthersOnDeactivation error:nil];
[session setActive:YES error:nil];
音频回调：
static OSStatus handleInputBuffer(void *inRefCon,
AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *inTimeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList *ioData) {
@autoreleasepool {
WSAudioCapture *source = (__bridge WSAudioCapture *)inRefCon;
if (!source) return -1;
AudioBuffer buffer;
buffer.mData = NULL;
buffer.mDataByteSize = 0;
buffer.mNumberChannels = 1;
AudioBufferList buffers;
buffers.mNumberBuffers = 1;
buffers.mBuffers[0] = buffer;
OSStatus status = AudioUnitRender(source.componetInstance,
ioActionFlags,
inTimeStamp,
inBusNumber,
inNumberFrames,
&buffers);
if (source.muted) {
for (int i = 0; i < buffers.mNumberBuffers; i++) {
AudioBuffer ab = buffers.mBuffers[i];
memset(ab.mData, 0, ab.mDataByteSize);
}
}
if (!status) {
if (source.delegate && [source.delegate respondsToSelector:@selector(captureOutput:audioData:)]) {
[source.delegate captureOutput:source audioData:[NSData dataWithBytes:buffers.mBuffers[0].mData length:buffers.mBuffers[0].mDataByteSize]];
}
}
return status;
}
}
名词介绍
音频采样率
- 音频采样率是指录音设备在一秒钟内对声音信号的采样次数，采样频率越高声音的还原就越真实越自然。
- 在当今的主流采集卡上，采样频率一般共分为11025Hz、22050Hz、24000Hz、44100Hz、48000Hz五个等级，11025Hz能达到AM调幅广播的声音品质，而22050Hz和24000HZ能达到FM调频广播的声音品质，44100Hz则是理论上的CD音质界限，48000Hz则更加精确一些。
- 一般音频质量采用44100Hz, 高等音频质量采用48000Hz
网友评论