在iOS11的时候苹果引入了Vision框架，可以人脸监测识别、图片分析、条形码识别、文本识别。是人工智能（Core ML）的一部分，今天写的是iOS14新增加的识别手指，直接上代码
- CameraView.swift
import UIKit
import AVFoundation
import Vision
enum AppError: Error {
case captureSessionSetup(reason: String)
case visionError(error: Error)
case otherError(error: Error)
static func display(_ error: Error, inViewController viewController: UIViewController) {
if let appError = error as? AppError {
appError.displayInViewController(viewController)
} else {
AppError.otherError(error: error).displayInViewController(viewController)
}
}
func displayInViewController(_ viewController: UIViewController) {
let title: String?
let message: String?
switch self {
case .captureSessionSetup(reason: let reason):
title = "AVSession Setup Error"
message = reason
case .visionError(error: let error):
title = "Vision Error"
message = error.localizedDescription
case .otherError(error: let error):
title = "Error"
message = error.localizedDescription
}
print("错误：\(title!) \(message!)")
}
}
class CameraView: UIView {
private var cameraSession: AVCaptureSession?
private var videoDataOutputQueue = DispatchQueue(label: "CamerDataOutput", qos: .userInteractive)
private var handPoseRequest = VNDetectHumanHandPoseRequest()
private var lastObservationTimestamp = Date()
private var gestureProcessor = HandGestureProcessor()
private var pointsPath = UIBezierPath()
private var overlayLayer = CAShapeLayer()
private var evidenceBuffer = [HandGestureProcessor.PointsPair]()
var previewLayer: AVCaptureVideoPreviewLayer {
return self.layer as! AVCaptureVideoPreviewLayer
}
override class var layerClass: AnyClass {
return AVCaptureVideoPreviewLayer.self
}
override func layoutSublayers(of layer: CALayer) {
super.layoutSublayers(of: layer)
if layer == previewLayer {
overlayLayer.frame = layer.bounds
}
}
override init(frame: CGRect) {
super.init(frame: frame)
handPoseRequest.maximumHandCount = 1
gestureProcessor.didChangeStateClosure = {[weak self] state in
self?.handleGestureStateChange(state: state)
}
setupOverlay()
do {
if cameraSession == nil {
previewLayer.videoGravity = .resizeAspectFill
try setupAVSession()
previewLayer.session = cameraSession
}
cameraSession?.startRunning()
} catch {
AppError.display(error, inViewController: UIApplication.shared.windows.first!.rootViewController!)
}
}
private func setupOverlay() {
previewLayer.addSublayer(overlayLayer)
}
fileprivate func setupAVSession() throws {
guard let videoDevice = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back) else {
throw AppError.captureSessionSetup(reason: "没有发现摄像头")
}
guard let deviceInput = try? AVCaptureDeviceInput(device: videoDevice) else {
throw AppError.captureSessionSetup(reason: "不能创建摄像输入端")
}
let session = AVCaptureSession()
session.beginConfiguration()
session.sessionPreset = AVCaptureSession.Preset.high
guard session.canAddInput(deviceInput) else {
throw AppError.captureSessionSetup(reason: "不能添加输入的Session")
}
session.addInput(deviceInput)
let dataOutput = AVCaptureVideoDataOutput()
if session.canAddOutput(dataOutput) {
session.addOutput(dataOutput)
dataOutput.alwaysDiscardsLateVideoFrames = true
dataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange]
dataOutput.setSampleBufferDelegate(self, queue: videoDataOutputQueue)
} else {
throw AppError.captureSessionSetup(reason: "不能添加输出的Session")
}
session.commitConfiguration()
cameraSession = session
}
private func handleGestureStateChange(state: HandGestureProcessor.State) {
let pointsPair = gestureProcessor.lastProcessedPointsPair
var tipsColor: UIColor
switch state {
case .possiblePinch, .possibleApart:
evidenceBuffer.append(pointsPair)
tipsColor = .orange
case .pinched:
evidenceBuffer.removeAll()
tipsColor = .green
case .apart, .unknown:
evidenceBuffer.removeAll()
tipsColor = .red
}
showPoints([pointsPair.thumbTip, pointsPair.indexTip, pointsPair.middleTip, pointsPair.ringTip, pointsPair.littleTip], color: tipsColor)
}
func processPoints(thumbTip: CGPoint?, indexTip: CGPoint?, middleTip: CGPoint?, ringTip: CGPoint?, littleTip: CGPoint?) {
guard let thumbPoint = thumbTip, let indexPoint = indexTip, let middlePoint = middleTip, let ringPoint = ringTip, let littlePoint = littleTip else {
// 如果没有超过2s复位手势处理器
if Date().timeIntervalSince(lastObservationTimestamp) > 2 {
gestureProcessor.reset()
}
showPoints([], color: .clear)
return
}
let thumbPointConverted = previewLayer.layerPointConverted(fromCaptureDevicePoint: thumbPoint)
let indexPointConverted = previewLayer.layerPointConverted(fromCaptureDevicePoint: indexPoint)
let middlePointConverted = previewLayer.layerPointConverted(fromCaptureDevicePoint: middlePoint)
let ringPointConverted = previewLayer.layerPointConverted(fromCaptureDevicePoint: ringPoint)
let littlePointConverted = previewLayer.layerPointConverted(fromCaptureDevicePoint: littlePoint)
gestureProcessor.processPointsPair((thumbPointConverted, indexPointConverted, middlePointConverted, ringPointConverted, littlePointConverted))
}
func showPoints(_ points: [CGPoint], color: UIColor) {
self.subviews.forEach { v in
v.removeFromSuperview()
}
let titles = ["大拇指", "食指", "中指", "无名指", "小手指"]
for (index,point) in points.enumerated() {
let label = UILabel(frame: CGRect(x: point.x, y: point.y, width: 70, height: 20))
label.text = titles[index]
label.font = UIFont.systemFont(ofSize: 12)
label.textAlignment = .center
label.textColor = color
self.addSubview(label)
}
}
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
/*
// Only override draw() if you perform custom drawing.
// An empty implementation adversely affects performance during animation.
override func draw(_ rect: CGRect) {
// Drawing code
}
*/
}
extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
var thumbTip: CGPoint? //大拇指
var indexTip: CGPoint? //食指
var middleTip: CGPoint? //中指
var ringTip: CGPoint? //无名指
var littleTip: CGPoint? //小手指
defer {
DispatchQueue.main.sync {
self.processPoints(thumbTip: thumbTip, indexTip: indexTip, middleTip: middleTip, ringTip: ringTip, littleTip: littleTip)
}
}
let handler = VNImageRequestHandler(cmSampleBuffer: sampleBuffer, orientation: .up, options: [ : ])
do {
try handler.perform([handPoseRequest])
guard let observation = handPoseRequest.results?.first else {
return
}
// 获取大拇指和食指的点
let thumbPoints = try observation.recognizedPoints(.thumb)
let indexFingerPoints = try observation.recognizedPoints(.indexFinger)
let middlePoints = try observation.recognizedPoints(.middleFinger)
let ringPoints = try observation.recognizedPoints(.ringFinger)
let littlePoints = try observation.recognizedPoints(.littleFinger)
guard let thumbTipPoint = thumbPoints[.thumbTip], let indexTipPoint = indexFingerPoints[.indexTip], let midleTipPoint = middlePoints[.middleTip], let ringTipPoint = ringPoints[.ringTip], let littleTipPoint = littlePoints[.littleTip] else {
return
}
// 忽略掉的点
guard thumbTipPoint.confidence > 0.3 && indexTipPoint.confidence > 0.3 && midleTipPoint.confidence > 0.3 && ringTipPoint.confidence > 0.3 && littleTipPoint.confidence > 0.3 else {
return
}
// 视觉坐标转成AVFoundation坐标
thumbTip = CGPoint(x: thumbTipPoint.location.x, y: 1 - thumbTipPoint.location.y)
indexTip = CGPoint(x: indexTipPoint.location.x, y: 1 - indexTipPoint.location.y)
middleTip = CGPoint(x: midleTipPoint.location.x, y: 1 - midleTipPoint.location.y)
ringTip = CGPoint(x: ringTipPoint.location.x, y: 1 - ringTipPoint.location.y)
littleTip = CGPoint(x: littleTipPoint.location.x, y: 1 - littleTipPoint.location.y)
} catch {
cameraSession?.stopRunning()
let error = AppError.visionError(error: error)
DispatchQueue.main.async {
error.displayInViewController(UIApplication.shared.windows.first!.rootViewController!)
}
}
}
}
- HandGestureProcessor.swift
import CoreGraphics
class HandGestureProcessor {
enum State {
case possiblePinch
case pinched
case possibleApart
case apart
case unknown
}
typealias PointsPair = (thumbTip: CGPoint, indexTip: CGPoint, middleTip: CGPoint, ringTip: CGPoint, littleTip: CGPoint)
private var state = State.unknown {
didSet {
didChangeStateClosure?(state)
}
}
private var pinchEvidenceCounter = 0
private var apartEvidenceCounter = 0
private var pinchMaxDistance: CGFloat
private var evidenceCounterStateTrigger: Int
var didChangeStateClosure: ((State) -> Void)?
private (set) var lastProcessedPointsPair = PointsPair(.zero, .zero, .zero, .zero, .zero)
init(pinchMaxDistance: CGFloat = 40, evidenceCounterStateTrigger: Int = 3) {
self.pinchMaxDistance = pinchMaxDistance
self.evidenceCounterStateTrigger = evidenceCounterStateTrigger
}
func reset() {
state = .unknown
pinchEvidenceCounter = 0
apartEvidenceCounter = 0
}
func processPointsPair(_ pointsPair: PointsPair) {
lastProcessedPointsPair = pointsPair
let distance = pointsPair.indexTip.distance(from: pointsPair.thumbTip)
if distance < pinchMaxDistance {
pinchEvidenceCounter += 1
apartEvidenceCounter = 0
state = (pinchEvidenceCounter >= evidenceCounterStateTrigger) ? .pinched : .possiblePinch
} else {
apartEvidenceCounter += 1
pinchEvidenceCounter = 0
state = (apartEvidenceCounter >= evidenceCounterStateTrigger) ? .apart : .possibleApart
}
}
}
extension CGPoint {
static func midPoint(p1: CGPoint, p2: CGPoint) -> CGPoint {
return CGPoint(x: (p1.x + p2.x) / 2, y: (p1.y + p2.y) / 2)
}
func distance(from point: CGPoint) -> CGFloat {
return hypot(point.x - self.x, point.y - self.y)
}
}
-
只是简单的实现了识别每根手指的位置，如果感兴趣的可以自己来实现判断手指的动作来控制视图的效果
-
最简单的效果
IMG_0022.PNG
网友评论