版本记录
| 版本号 | 时间 |
|---|---|
| V1.0 | 2022.02.26 星期六 |
前言
iOS 11+和macOS 10.13+新出了Vision框架，提供了人脸识别、物体检测、物体跟踪等技术，它是基于Core ML的。可以说是人工智能的一部分，接下来几篇我们就详细的解析一下Vision框架。感兴趣的看下面几篇文章。
1. Vision框架详细解析（一） —— 基本概览（一）
2. Vision框架详细解析（二） —— 基于Vision的人脸识别（一）
3. Vision框架详细解析（三） —— 基于Vision的人脸识别（二）
4. Vision框架详细解析（四） —— 在iOS中使用Vision和Metal进行照片堆叠（一）
5. Vision框架详细解析（五） —— 在iOS中使用Vision和Metal进行照片堆叠（二）
6. Vision框架详细解析（六） —— 基于Vision的显著性分析（一）
7. Vision框架详细解析（七） —— 基于Vision的显著性分析（二）
8. Vision框架详细解析（八） —— 基于Vision的QR扫描（一）
9. Vision框架详细解析（九） —— 基于Vision的QR扫描（二）
10. Vision框架详细解析（十） —— 基于Vision的Body Detect和Hand Pose（一）
11. Vision框架详细解析（十一） —— 基于Vision的Body Detect和Hand Pose（二)
12. Vision框架详细解析（十二） —— 基于Vision的Face Detection新特性（一)
源码
首先看下工程组织结构
下面就是源码了
1. AppMain.swift
import SwiftUI
@main
struct AppMain: App {
var body: some Scene {
WindowGroup {
PassportPhotosAppView(model: CameraViewModel())
}
}
}
2. CameraControlsFooterView.swift
import SwiftUI
struct CameraControlsFooterView: View {
@ObservedObject var model: CameraViewModel
var body: some View {
ZStack {
Rectangle()
.fill(Color.black)
CameraControlsView(model: model)
}
}
struct CameraControlsView: View {
@ObservedObject var model: CameraViewModel
var body: some View {
HStack(spacing: 20) {
Spacer()
VStack(spacing: 20) {
HideBackgroundButton(isHideBackgroundEnabled: model.hideBackgroundModeEnabled) {
model.perform(action: .toggleHideBackgroundMode)
}
DebugButton(isDebugEnabled: model.debugModeEnabled) {
model.perform(action: .toggleDebugMode)
}
}
Spacer()
ShutterButton(isDisabled: !model.hasDetectedValidFace) {
model.perform(action: .takePhoto)
}
Spacer()
ThumbnailView(passportPhoto: model.passportPhoto)
Spacer()
}
}
}
struct HideBackgroundButton: View {
let isHideBackgroundEnabled: Bool
let action: (() -> Void)
var body: some View {
Button(action: {
action()
}, label: {
FooterIconView(imageName: "photo.fill")
})
.tint(isHideBackgroundEnabled ? .green : .gray)
}
}
struct DebugButton: View {
let isDebugEnabled: Bool
let action: (() -> Void)
var body: some View {
Button(action: {
action()
}, label: {
FooterIconView(imageName: "ladybug.fill")
})
.tint(isDebugEnabled ? .green : .gray)
}
}
struct ShutterButton: View {
let isDisabled: Bool
let action: (() -> Void)
var body: some View {
Button(action: {
action()
}, label: {
Image(systemName: "camera.aperture")
.font(.system(size: 72))
})
.disabled(isDisabled)
.tint(.white)
}
}
struct ThumbnailView: View {
let passportPhoto: UIImage?
@State private var isShowingPassportPhoto = false
var body: some View {
if let photo = passportPhoto {
VStack {
NavigationLink(
destination: PassportPhotoView(passportPhoto: photo),
isActive: $isShowingPassportPhoto
) {
EmptyView()
}
Button(action: {
isShowingPassportPhoto = true
}, label: {
Image(uiImage: photo)
.resizable()
.frame(width: 45.0, height: 60.0)
})
}
} else {
FooterIconView(imageName: "photo.fill.on.rectangle.fill")
.foregroundColor(.gray)
}
}
}
struct FooterIconView: View {
var imageName: String
var body: some View {
return Image(systemName: imageName)
.font(.system(size: 36))
}
}
}
struct CameraControlsFooterView_Previews: PreviewProvider {
static var previews: some View {
CameraControlsFooterView(model: CameraViewModel())
}
}
3. CameraControlsHeaderView.swift
import SwiftUI
struct CameraControlsHeaderView: View {
@ObservedObject var model: CameraViewModel
var body: some View {
ZStack {
Rectangle()
.fill(Color.black)
UserInstructionsView(model: model)
}
}
}
struct CameraControlsHeaderView_Previews: PreviewProvider {
static var previews: some View {
CameraControlsHeaderView(model: CameraViewModel())
}
}
4. CameraOverlayView.swift
import SwiftUI
struct CameraOverlayView: View {
@ObservedObject private(set) var model: CameraViewModel
var body: some View {
GeometryReader { geometry in
VStack {
CameraControlsHeaderView(model: model)
Spacer()
.frame(height: geometry.size.width * 4 / 3)
CameraControlsFooterView(model: model)
}
}
}
}
struct CameraControlsView_Previews: PreviewProvider {
static var previews: some View {
CameraOverlayView(model: CameraViewModel())
}
}
5. CameraView.swift
import SwiftUI
struct CameraView: UIViewControllerRepresentable {
typealias UIViewControllerType = CameraViewController
private(set) var model: CameraViewModel
func makeUIViewController(context: Context) -> CameraViewController {
let faceDetector = FaceDetector()
faceDetector.model = model
let viewController = CameraViewController()
viewController.faceDetector = faceDetector
return viewController
}
func updateUIViewController(_ uiViewController: CameraViewController, context: Context) { }
}
6. CameraViewController.swift
import AVFoundation
import CoreImage
import MetalKit
class CameraViewController: UIViewController {
var faceDetector: FaceDetector?
var previewLayer: AVCaptureVideoPreviewLayer?
let session = AVCaptureSession()
var isUsingMetal = false
var metalDevice: MTLDevice?
var metalCommandQueue: MTLCommandQueue?
var metalView: MTKView?
var ciContext: CIContext?
var currentCIImage: CIImage? {
didSet {
metalView?.draw()
}
}
let videoOutputQueue = DispatchQueue(
label: "Video Output Queue",
qos: .userInitiated,
attributes: [],
autoreleaseFrequency: .workItem
)
override func viewDidLoad() {
super.viewDidLoad()
faceDetector?.viewDelegate = self
configureMetal()
configureCaptureSession()
session.startRunning()
}
}
// MARK: - Setup video capture
extension CameraViewController {
func configureCaptureSession() {
// Define the capture device we want to use
guard let camera = AVCaptureDevice.default(
.builtInWideAngleCamera,
for: .video,
position: .front
) else {
fatalError("No front video camera available")
}
// Connect the camera to the capture session input
do {
let cameraInput = try AVCaptureDeviceInput(device: camera)
session.addInput(cameraInput)
} catch {
fatalError(error.localizedDescription)
}
// Create the video data output
let videoOutput = AVCaptureVideoDataOutput()
videoOutput.alwaysDiscardsLateVideoFrames = true
videoOutput.setSampleBufferDelegate(faceDetector, queue: videoOutputQueue)
videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
// Add the video output to the capture session
session.addOutput(videoOutput)
let videoConnection = videoOutput.connection(with: .video)
videoConnection?.videoOrientation = .portrait
// Configure the preview layer
previewLayer = AVCaptureVideoPreviewLayer(session: session)
previewLayer?.videoGravity = .resizeAspect
previewLayer?.frame = view.bounds
if !isUsingMetal, let previewLayer = previewLayer {
view.layer.insertSublayer(previewLayer, at: 0)
}
}
}
// MARK: Setup Metal
extension CameraViewController {
func configureMetal() {
guard let metalDevice = MTLCreateSystemDefaultDevice() else {
fatalError("Could not instantiate required metal properties")
}
isUsingMetal = true
metalCommandQueue = metalDevice.makeCommandQueue()
metalView = MTKView()
if let metalView = metalView {
metalView.device = metalDevice
metalView.isPaused = true
metalView.enableSetNeedsDisplay = false
metalView.delegate = self
metalView.framebufferOnly = false
metalView.frame = view.bounds
metalView.layer.contentsGravity = .resizeAspect
view.layer.insertSublayer(metalView.layer, at: 0)
}
ciContext = CIContext(mtlDevice: metalDevice)
}
}
// MARK: - Metal view delegate methods
extension CameraViewController: MTKViewDelegate {
func draw(in view: MTKView) {
guard
let metalView = metalView,
let metalCommandQueue = metalCommandQueue
else {
return
}
// Grab command buffer so we can encode instructions to GPU
guard let commandBuffer = metalCommandQueue.makeCommandBuffer() else {
return
}
guard let ciImage = currentCIImage else {
return
}
// Ensure drawable is free and not tied in the preivous drawing cycle
guard let currentDrawable = view.currentDrawable else {
return
}
// Make sure the image is full width, and scaled in height appropriately
let drawSize = metalView.drawableSize
let scaleX = drawSize.width / ciImage.extent.width
let newImage = ciImage.transformed(by: .init(scaleX: scaleX, y: scaleX))
let originY = (newImage.extent.height - drawSize.height) / 2
// Render into the metal texture
ciContext?.render(
newImage,
to: currentDrawable.texture,
commandBuffer: commandBuffer,
bounds: CGRect(x: 0, y: originY, width: newImage.extent.width, height: newImage.extent.height),
colorSpace: CGColorSpaceCreateDeviceRGB()
)
// Register drawwable to command buffer
commandBuffer.present(currentDrawable)
commandBuffer.commit()
}
func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) { }
}
// MARK: FaceDetectorDelegate methods
extension CameraViewController: FaceDetectorDelegate {
func convertFromMetadataToPreviewRect(rect: CGRect) -> CGRect {
guard let previewLayer = previewLayer else {
return CGRect.zero
}
return previewLayer.layerRectConverted(fromMetadataOutputRect: rect)
}
func draw(image: CIImage) {
currentCIImage = image
}
}
7. CameraViewModel.swift
import Combine
import CoreGraphics
import UIKit
import Vision
enum CameraViewModelAction {
// View setup and configuration actions
case windowSizeDetected(CGRect)
// Face detection actions
case noFaceDetected
case faceObservationDetected(FaceGeometryModel)
case faceQualityObservationDetected(FaceQualityModel)
// Other
case toggleDebugMode
case toggleHideBackgroundMode
case takePhoto
case savePhoto(UIImage)
}
enum FaceDetectedState {
case faceDetected
case noFaceDetected
case faceDetectionErrored
}
enum FaceBoundsState {
case unknown
case detectedFaceTooSmall
case detectedFaceTooLarge
case detectedFaceOffCentre
case detectedFaceAppropriateSizeAndPosition
}
struct FaceGeometryModel {
let boundingBox: CGRect
let roll: NSNumber
let pitch: NSNumber
let yaw: NSNumber
}
struct FaceQualityModel {
let quality: Float
}
final class CameraViewModel: ObservableObject {
// MARK: - Publishers
@Published var debugModeEnabled: Bool
@Published var hideBackgroundModeEnabled: Bool
// MARK: - Publishers of derived state
@Published private(set) var hasDetectedValidFace: Bool
@Published private(set) var isAcceptableRoll: Bool {
didSet {
calculateDetectedFaceValidity()
}
}
@Published private(set) var isAcceptablePitch: Bool {
didSet {
calculateDetectedFaceValidity()
}
}
@Published private(set) var isAcceptableYaw: Bool {
didSet {
calculateDetectedFaceValidity()
}
}
@Published private(set) var isAcceptableBounds: FaceBoundsState {
didSet {
calculateDetectedFaceValidity()
}
}
@Published private(set) var isAcceptableQuality: Bool {
didSet {
calculateDetectedFaceValidity()
}
}
@Published private(set) var passportPhoto: UIImage?
// MARK: - Publishers of Vision data directly
@Published private(set) var faceDetectedState: FaceDetectedState
@Published private(set) var faceGeometryState: FaceObservation<FaceGeometryModel> {
didSet {
processUpdatedFaceGeometry()
}
}
@Published private(set) var faceQualityState: FaceObservation<FaceQualityModel> {
didSet {
processUpdatedFaceQuality()
}
}
// MARK: - Public properties
let shutterReleased = PassthroughSubject<Void, Never>()
// MARK: - Private variables
var faceLayoutGuideFrame = CGRect(x: 0, y: 0, width: 200, height: 300)
init() {
faceDetectedState = .noFaceDetected
isAcceptableRoll = false
isAcceptablePitch = false
isAcceptableYaw = false
isAcceptableBounds = .unknown
isAcceptableQuality = false
hasDetectedValidFace = false
faceGeometryState = .faceNotFound
faceQualityState = .faceNotFound
#if DEBUG
debugModeEnabled = true
#else
debugModeEnabled = false
#endif
hideBackgroundModeEnabled = false
}
// MARK: Actions
func perform(action: CameraViewModelAction) {
switch action {
case .windowSizeDetected(let windowRect):
handleWindowSizeChanged(toRect: windowRect)
case .noFaceDetected:
publishNoFaceObserved()
case .faceObservationDetected(let faceObservation):
publishFaceObservation(faceObservation)
case .faceQualityObservationDetected(let faceQualityObservation):
publishFaceQualityObservation(faceQualityObservation)
case .toggleDebugMode:
toggleDebugMode()
case .toggleHideBackgroundMode:
toggleHideBackgroundMode()
case .takePhoto:
takePhoto()
case .savePhoto(let image):
savePhoto(image)
}
}
// MARK: Action handlers
private func handleWindowSizeChanged(toRect: CGRect) {
faceLayoutGuideFrame = CGRect(
x: toRect.midX - faceLayoutGuideFrame.width / 2,
y: toRect.midY - faceLayoutGuideFrame.height / 2,
width: faceLayoutGuideFrame.width,
height: faceLayoutGuideFrame.height
)
}
private func publishNoFaceObserved() {
DispatchQueue.main.async { [self] in
faceDetectedState = .noFaceDetected
faceGeometryState = .faceNotFound
faceQualityState = .faceNotFound
}
}
private func publishFaceObservation(_ faceGeometryModel: FaceGeometryModel) {
DispatchQueue.main.async { [self] in
faceDetectedState = .faceDetected
faceGeometryState = .faceFound(faceGeometryModel)
}
}
private func publishFaceQualityObservation(_ faceQualityModel: FaceQualityModel) {
DispatchQueue.main.async { [self] in
faceDetectedState = .faceDetected
faceQualityState = .faceFound(faceQualityModel)
}
}
private func toggleDebugMode() {
debugModeEnabled.toggle()
}
private func toggleHideBackgroundMode() {
hideBackgroundModeEnabled.toggle()
}
private func takePhoto() {
shutterReleased.send()
}
private func savePhoto(_ photo: UIImage) {
UIImageWriteToSavedPhotosAlbum(photo, nil, nil, nil)
DispatchQueue.main.async { [self] in
passportPhoto = photo
}
}
}
// MARK: Private instance methods
extension CameraViewModel {
func invalidateFaceGeometryState() {
isAcceptableRoll = false
isAcceptablePitch = false
isAcceptableYaw = false
isAcceptableBounds = .unknown
}
func processUpdatedFaceGeometry() {
switch faceGeometryState {
case .faceNotFound:
invalidateFaceGeometryState()
case .errored(let error):
print(error.localizedDescription)
invalidateFaceGeometryState()
case .faceFound(let faceGeometryModel):
let boundingBox = faceGeometryModel.boundingBox
let roll = faceGeometryModel.roll.doubleValue
let pitch = faceGeometryModel.pitch.doubleValue
let yaw = faceGeometryModel.yaw.doubleValue
updateAcceptableBounds(using: boundingBox)
updateAcceptableRollPitchYaw(using: roll, pitch: pitch, yaw: yaw)
}
}
func updateAcceptableBounds(using boundingBox: CGRect) {
// First, check face is roughly the same size as the layout guide
if boundingBox.width > 1.2 * faceLayoutGuideFrame.width {
isAcceptableBounds = .detectedFaceTooLarge
} else if boundingBox.width * 1.2 < faceLayoutGuideFrame.width {
isAcceptableBounds = .detectedFaceTooSmall
} else {
// Next, check face is roughly centered in the frame
if abs(boundingBox.midX - faceLayoutGuideFrame.midX) > 50 {
isAcceptableBounds = .detectedFaceOffCentre
} else if abs(boundingBox.midY - faceLayoutGuideFrame.midY) > 50 {
isAcceptableBounds = .detectedFaceOffCentre
} else {
isAcceptableBounds = .detectedFaceAppropriateSizeAndPosition
}
}
}
func updateAcceptableRollPitchYaw(using roll: Double, pitch: Double, yaw: Double) {
isAcceptableRoll = (roll > 1.2 && roll < 1.6)
isAcceptablePitch = abs(CGFloat(pitch)) < 0.2
isAcceptableYaw = abs(CGFloat(yaw)) < 0.15
}
func processUpdatedFaceQuality() {
switch faceQualityState {
case .faceNotFound:
isAcceptableQuality = false
case .errored(let error):
print(error.localizedDescription)
isAcceptableQuality = false
case .faceFound(let faceQualityModel):
if faceQualityModel.quality < 0.2 {
isAcceptableQuality = false
}
isAcceptableQuality = true
}
}
func calculateDetectedFaceValidity() {
hasDetectedValidFace =
isAcceptableBounds == .detectedFaceAppropriateSizeAndPosition &&
isAcceptableRoll &&
isAcceptablePitch &&
isAcceptableYaw &&
isAcceptableQuality
}
}
8. DebugView.swift
import SwiftUI
struct DebugView: View {
@ObservedObject var model: CameraViewModel
var body: some View {
ZStack {
FaceBoundingBoxView(model: model)
FaceLayoutGuideView(model: model)
VStack(alignment: .leading, spacing: 5) {
DebugSection(observation: model.faceGeometryState) { geometryModel in
DebugText("R: \(geometryModel.roll)")
.debugTextStatus(status: model.isAcceptableRoll ? .passing : .failing)
DebugText("P: \(geometryModel.pitch)")
.debugTextStatus(status: model.isAcceptablePitch ? .passing : .failing)
DebugText("Y: \(geometryModel.yaw)")
.debugTextStatus(status: model.isAcceptableYaw ? .passing : .failing)
}
DebugSection(observation: model.faceQualityState) { qualityModel in
DebugText("Q: \(qualityModel.quality)")
.debugTextStatus(status: model.isAcceptableQuality ? .passing : .failing)
}
}
.frame(maxWidth: .infinity, alignment: .leading)
}
}
}
struct DebugSection<Model, Content: View>: View {
let observation: FaceObservation<Model>
let content: (Model) -> Content
public init(
observation: FaceObservation<Model>,
@ViewBuilder content: @escaping (Model) -> Content
) {
self.observation = observation
self.content = content
}
var body: some View {
switch observation {
case .faceNotFound:
AnyView(Spacer())
case .faceFound(let model):
AnyView(content(model))
case .errored(let error):
AnyView(
DebugText("ERROR: \(error.localizedDescription)")
)
}
}
}
enum DebugTextStatus {
case neutral
case failing
case passing
}
struct DebugText: View {
let content: String
@inlinable
public init(_ content: String) {
self.content = content
}
var body: some View {
Text(content)
.frame(maxWidth: .infinity, alignment: .leading)
}
}
struct Status: ViewModifier {
let foregroundColor: Color
func body(content: Content) -> some View {
content
.foregroundColor(foregroundColor)
}
}
extension DebugText {
func colorForStatus(status: DebugTextStatus) -> Color {
switch status {
case .neutral:
return .white
case .failing:
return .red
case .passing:
return .green
}
}
func debugTextStatus(status: DebugTextStatus) -> some View {
self.modifier(Status(foregroundColor: colorForStatus(status: status)))
}
}
struct DebugView_Previews: PreviewProvider {
static var previews: some View {
DebugView(model: CameraViewModel())
}
}
9. FaceBoundingBoxView.swift
import SwiftUI
struct FaceBoundingBoxView: View {
@ObservedObject private(set) var model: CameraViewModel
var body: some View {
switch model.faceGeometryState {
case .faceNotFound:
Rectangle().fill(Color.clear)
case .faceFound(let faceGeometryModel):
Rectangle()
.path(in: CGRect(
x: faceGeometryModel.boundingBox.origin.x,
y: faceGeometryModel.boundingBox.origin.y,
width: faceGeometryModel.boundingBox.width,
height: faceGeometryModel.boundingBox.height
))
.stroke(Color.yellow, lineWidth: 2.0)
case .errored:
Rectangle().fill(Color.clear)
}
}
}
struct FaceBoundingBoxView_Previews: PreviewProvider {
static var previews: some View {
FaceBoundingBoxView(model: CameraViewModel())
}
}`
10. FaceDetector.swift
import AVFoundation
import Combine
import CoreImage.CIFilterBuiltins
import UIKit
import Vision
protocol FaceDetectorDelegate: NSObjectProtocol {
func convertFromMetadataToPreviewRect(rect: CGRect) -> CGRect
func draw(image: CIImage)
}
class FaceDetector: NSObject {
weak var viewDelegate: FaceDetectorDelegate?
weak var model: CameraViewModel? {
didSet {
model?.shutterReleased.sink { completion in
switch completion {
case .finished:
return
case .failure(let error):
print("Received error: \(error)")
}
} receiveValue: { _ in
self.isCapturingPhoto = true
}
.store(in: &subscriptions)
}
}
var sequenceHandler = VNSequenceRequestHandler()
var isCapturingPhoto = false
var currentFrameBuffer: CVImageBuffer?
var subscriptions = Set<AnyCancellable>()
let imageProcessingQueue = DispatchQueue(
label: "Image Processing Queue",
qos: .userInitiated,
attributes: [],
autoreleaseFrequency: .workItem
)
}
// MARK: - AVCaptureVideoDataOutputSampleBufferDelegate methods
extension FaceDetector: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return
}
if isCapturingPhoto {
isCapturingPhoto = false
savePassportPhoto(from: imageBuffer)
}
let detectFaceRectanglesRequest = VNDetectFaceRectanglesRequest(completionHandler: detectedFaceRectangles)
detectFaceRectanglesRequest.revision = VNDetectFaceRectanglesRequestRevision3
let detectCaptureQualityRequest = VNDetectFaceCaptureQualityRequest(completionHandler: detectedFaceQualityRequest)
detectCaptureQualityRequest.revision = VNDetectFaceCaptureQualityRequestRevision2
let detectSegmentationRequest = VNGeneratePersonSegmentationRequest(completionHandler: detectedSegmentationRequest)
detectSegmentationRequest.qualityLevel = .balanced
currentFrameBuffer = imageBuffer
do {
try sequenceHandler.perform(
[detectFaceRectanglesRequest, detectCaptureQualityRequest, detectSegmentationRequest],
on: imageBuffer,
orientation: .leftMirrored)
} catch {
print(error.localizedDescription)
}
}
}
// MARK: - Private methods
extension FaceDetector {
func detectedFaceRectangles(request: VNRequest, error: Error?) {
guard let model = model, let viewDelegate = viewDelegate else {
return
}
guard
let results = request.results as? [VNFaceObservation],
let result = results.first
else {
model.perform(action: .noFaceDetected)
return
}
let convertedBoundingBox =
viewDelegate.convertFromMetadataToPreviewRect(rect: result.boundingBox)
let faceObservationModel = FaceGeometryModel(
boundingBox: convertedBoundingBox,
roll: result.roll ?? 0,
pitch: result.pitch ?? 0,
yaw: result.yaw ?? 0
)
model.perform(action: .faceObservationDetected(faceObservationModel))
}
func detectedFaceQualityRequest(request: VNRequest, error: Error?) {
guard let model = model else {
return
}
guard
let results = request.results as? [VNFaceObservation],
let result = results.first
else {
model.perform(action: .noFaceDetected)
return
}
let faceQualityModel = FaceQualityModel(
quality: result.faceCaptureQuality ?? 0
)
model.perform(action: .faceQualityObservationDetected(faceQualityModel))
}
func detectedSegmentationRequest(request: VNRequest, error: Error?) {
guard
let model = model,
let results = request.results as? [VNPixelBufferObservation],
let result = results.first,
let currentFrameBuffer = currentFrameBuffer
else {
return
}
if model.hideBackgroundModeEnabled {
let originalImage = CIImage(cvImageBuffer: currentFrameBuffer)
let maskPixelBuffer = result.pixelBuffer
let outputImage = removeBackgroundFrom(image: originalImage, using: maskPixelBuffer)
viewDelegate?.draw(image: outputImage.oriented(.upMirrored))
} else {
let originalImage = CIImage(cvImageBuffer: currentFrameBuffer).oriented(.upMirrored)
viewDelegate?.draw(image: originalImage)
}
}
func savePassportPhoto(from pixelBuffer: CVPixelBuffer) {
guard let model = model else {
return
}
imageProcessingQueue.async { [self] in
let originalImage = CIImage(cvPixelBuffer: pixelBuffer)
var outputImage = originalImage
if model.hideBackgroundModeEnabled {
let detectSegmentationRequest = VNGeneratePersonSegmentationRequest()
detectSegmentationRequest.qualityLevel = .accurate
try? sequenceHandler.perform(
[detectSegmentationRequest],
on: pixelBuffer,
orientation: .leftMirrored
)
if let maskPixelBuffer = detectSegmentationRequest.results?.first?.pixelBuffer {
outputImage = removeBackgroundFrom(image: originalImage, using: maskPixelBuffer)
}
}
let coreImageWidth = outputImage.extent.width
let coreImageHeight = outputImage.extent.height
let desiredImageHeight = coreImageWidth * 4 / 3
// Calculate frame of photo
let yOrigin = (coreImageHeight - desiredImageHeight) / 2
let photoRect = CGRect(x: 0, y: yOrigin, width: coreImageWidth, height: desiredImageHeight)
let context = CIContext()
if let cgImage = context.createCGImage(outputImage, from: photoRect) {
let passportPhoto = UIImage(cgImage: cgImage, scale: 1, orientation: .upMirrored)
DispatchQueue.main.async {
model.perform(action: .savePhoto(passportPhoto))
}
}
}
}
func removeBackgroundFrom(image: CIImage, using maskPixelBuffer: CVPixelBuffer) -> CIImage {
var maskImage = CIImage(cvPixelBuffer: maskPixelBuffer)
let originalImage = image.oriented(.right)
// Scale the mask image to fit the bounds of the video frame.
let scaleX = originalImage.extent.width / maskImage.extent.width
let scaleY = originalImage.extent.height / maskImage.extent.height
maskImage = maskImage.transformed(by: .init(scaleX: scaleX, y: scaleY)).oriented(.upMirrored)
let backgroundImage = CIImage(color: .white).clampedToExtent().cropped(to: originalImage.extent)
let blendFilter = CIFilter.blendWithRedMask()
blendFilter.inputImage = originalImage
blendFilter.backgroundImage = backgroundImage
blendFilter.maskImage = maskImage
if let outputImage = blendFilter.outputImage?.oriented(.left) {
return outputImage
}
return originalImage
}
}
11. FaceLayoutGuideView.swift
import SwiftUI
struct FaceLayoutGuideView: View {
@ObservedObject private(set) var model: CameraViewModel
var body: some View {
Rectangle()
.path(in: CGRect(
x: model.faceLayoutGuideFrame.minX,
y: model.faceLayoutGuideFrame.minY,
width: model.faceLayoutGuideFrame.width,
height: model.faceLayoutGuideFrame.height
))
.stroke(Color.red)
}
}
struct FaceLayoutGuideView_Previews: PreviewProvider {
static var previews: some View {
FaceLayoutGuideView(model: CameraViewModel())
}
}
12. FaceObservation.swift
import Foundation
enum FaceObservation<T> {
case faceFound(T)
case faceNotFound
case errored(Error)
}
13. LayoutGuideView.swift
import SwiftUI
struct LayoutGuideView: View {
let layoutGuideFrame: CGRect
let hasDetectedValidFace: Bool
var body: some View {
VStack {
Ellipse()
.stroke(hasDetectedValidFace ? Color.green : Color.red)
.frame(width: layoutGuideFrame.width, height: layoutGuideFrame.height)
}
}
}
struct LayoutGuideView_Previews: PreviewProvider {
static var previews: some View {
LayoutGuideView(
layoutGuideFrame: CGRect(x: 0, y: 0, width: 200, height: 300),
hasDetectedValidFace: true
)
}
}
14. PassportPhotosAppView.swift
import SwiftUI
struct PassportPhotosAppView: View {
@ObservedObject private(set) var model: CameraViewModel
init(model: CameraViewModel) {
self.model = model
}
var body: some View {
GeometryReader { geo in
NavigationView {
ZStack {
CameraView(model: model)
LayoutGuideView(
layoutGuideFrame: model.faceLayoutGuideFrame,
hasDetectedValidFace: model.hasDetectedValidFace
)
if model.debugModeEnabled {
DebugView(model: model)
}
CameraOverlayView(model: model)
}
.ignoresSafeArea()
.onAppear {
model.perform(action: .windowSizeDetected(geo.frame(in: .global)))
}
}
}
}
}
struct ContentView_Previews: PreviewProvider {
static var previews: some View {
PassportPhotosAppView(model: CameraViewModel())
}
}
15. PassportPhotoView.swift
import SwiftUI
import UIKit
struct PassportPhotoView: View {
let passportPhoto: UIImage
var body: some View {
VStack {
Spacer()
Image(uiImage: passportPhoto)
.resizable()
.aspectRatio(contentMode: .fit)
Spacer()
}
.ignoresSafeArea()
.background(.black)
}
}
struct PassportPhotoView_Previews: PreviewProvider {
static var previews: some View {
if let image = UIImage(named: "rw-logo") {
PassportPhotoView(passportPhoto: image)
}
}
}
16. UserInstructionsView.swift
import SwiftUI
struct UserInstructionsView: View {
@ObservedObject var model: CameraViewModel
var body: some View {
Text(faceDetectionStateLabel())
.font(.title)
}
}
// MARK: Private instance methods
extension UserInstructionsView {
func faceDetectionStateLabel() -> String {
switch model.faceDetectedState {
case .faceDetectionErrored:
return "An unexpected error occurred"
case .noFaceDetected:
return "Please look at the camera"
case .faceDetected:
if model.hasDetectedValidFace {
return "Please take your photo :]"
} else if model.isAcceptableBounds == .detectedFaceTooSmall {
return "Please bring your face closer to the camera"
} else if model.isAcceptableBounds == .detectedFaceTooLarge {
return "Please hold the camera further from your face"
} else if model.isAcceptableBounds == .detectedFaceOffCentre {
return "Please move your face to the centre of the frame"
} else if !model.isAcceptableRoll || !model.isAcceptablePitch || !model.isAcceptableYaw {
return "Please look straight at the camera"
} else if !model.isAcceptableQuality {
return "Image quality too low"
} else {
return "We cannot take your photo right now"
}
}
}
}
struct UserInstructionsView_Previews: PreviewProvider {
static var previews: some View {
UserInstructionsView(
model: CameraViewModel()
)
}
}
后记
本篇主要讲述了基于
Vision的Face Detection新特性，感兴趣的给个赞或者关注~~~
网友评论