javacv(openCV)matchTemplate多目标匹配

1. 应用背景

之前使用adb的指令
adb shell uiautomator dump --compressed /data/local/tmp/uidump.xml
来获取布局文件，然后识别控件的坐标位置，但发现会报
ERROR: could not get idle stateorcould not get idle state的错误，效率很低。因此后来采用了先截屏，然后通过图片匹配识别控件位置，返回控件的坐标，即是本文要介绍的内容，由于开发用java，顺其自然的使用了javaCV，但目前这方面的资料较少。

2. 先来看具体效果

截屏得到的原图
需要识别点赞按钮图标
匹配结果效果图

重要：一定要保证原图与目标图的分辨率一致，不能压缩，简单的办法是使用电脑自带的画图工具来抠去目标图。

3. 引入maven

<dependency>
      <groupId>org.bytedeco</groupId>
      <artifactId>javacv-platform</artifactId>
      <version>1.5.3</version>
</dependency>

用的是最新版，只引入这个包即可，但下载需要好久，后来更换为阿里云的仓库地址，快了很多，后期考虑精简依赖。

4. 具体实现

package com.hilbp.web.controller;

import static org.bytedeco.opencv.global.opencv_imgproc.cvtColor;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;

import org.bytedeco.javacpp.DoublePointer;
import org.bytedeco.javacpp.indexer.FloatIndexer;
import org.bytedeco.opencv.global.opencv_core;
import org.bytedeco.opencv.global.opencv_highgui;
import org.bytedeco.opencv.global.opencv_imgcodecs;
import org.bytedeco.opencv.global.opencv_imgproc;
import org.bytedeco.opencv.opencv_core.Mat;
import org.bytedeco.opencv.opencv_core.Point;
import org.bytedeco.opencv.opencv_core.Rect;
import org.bytedeco.opencv.opencv_core.Scalar;
import org.bytedeco.opencv.opencv_core.Size;

import lombok.extern.slf4j.Slf4j;

@Slf4j
public class ImageTest {
    
    
    
    public void test() {
        String[] args = new String[2];
        args[0] = "log/screen.png"; //截屏图片
        args[1] = "log/1.png"; //点赞的图标
        newStyle(args);
    }
    
    public void newStyle(String[] args){
        
        //read in image default colors
        Mat sourceColor = opencv_imgcodecs.imread(args[0]);
        Mat sourceGrey = new Mat(sourceColor.size(), opencv_core.CV_8UC1);
       cvtColor(sourceColor, sourceGrey, opencv_imgproc.COLOR_BGR2GRAY);
       
       //load in template in grey 
       Mat template = opencv_imgcodecs.imread(args[1], opencv_imgcodecs.IMREAD_GRAYSCALE);//int = 0
       
       //Size for the result image
       Size size = new Size(sourceGrey.cols()-template.cols()+1, sourceGrey.rows()-template.rows()+1);
       Mat result = new Mat(size, opencv_core.CV_32FC1);
       opencv_imgproc.matchTemplate(sourceGrey, template, result, opencv_imgproc.TM_CCORR_NORMED);
//       opencv_imgproc.threshold(src, dst, thresh, maxval, ThresholdTypes.Tozero);
//       opencv_imgproc.floodFill(image, seedPoint, newVal)
       
       DoublePointer minVal= new DoublePointer();
       DoublePointer maxVal= new DoublePointer();
       Point min = new Point();
       Point max = new Point();
       opencv_core.minMaxLoc(result, minVal, maxVal, min, max, null);
//       log.info("[{}, {}]", max.x(), max.y());
//       opencv_imgproc.rectangle(sourceColor,new Rect(max.x(),max.y(),template.cols(),template.rows()), randColor(), 2, 0, 0);
       
       int centerWith = template.cols() / 2;
       int centerHeight = template.rows() / 2;
       getPointsFromMatAboveThreshold(result, 0.9999f).stream().forEach((point) -> {
           log.info("[{}, {}]", point.x(), point.y());
           log.info("[{}, {}]", point.x() + centerWith, point.y() + centerHeight);
           opencv_imgproc.rectangle(sourceColor, new Rect(point.x(), point.y(), template.cols(), template.rows()), randColor(), 2, 0, 0);
       });

//       List<Point> points = this.getPointsFromMatAboveThreshold(result, 0.99f);
//       for(Point point : points) {
//         opencv_imgproc.rectangle(sourceColor,new Rect(point.x(), point.y(), 30, 30), randColor(), 2, 0, 0);
//         
//       }
       opencv_highgui.imshow("Original marked", sourceColor);
//       imshow("Ttemplate", template);
//       imshow("Results matrix", result);
       
       opencv_imgcodecs.imwrite("log/res.png", sourceColor);
       
       opencv_highgui.waitKey(0);
       
       
       opencv_highgui.destroyAllWindows();
        
    }

    // some usefull things.
    public Scalar randColor(){
       int b,g,r;
       b= ThreadLocalRandom.current().nextInt(0, 255 + 1);
       g= ThreadLocalRandom.current().nextInt(0, 255 + 1);
       r= ThreadLocalRandom.current().nextInt(0, 255 + 1);
       return new Scalar (b,g,r,0);
    }
    
    public List<Point> getPointsFromMatAboveThreshold(Mat m, float t){
        
        List<Point> matches = new ArrayList<Point>();
        FloatIndexer indexer = m.createIndexer();
        for (int y = 0; y < m.rows(); y++) {
            for (int x = 0; x < m.cols(); x++) {
                if (indexer.get(y,x) > t) {
                    //System.out.println("(" + x + "," + y +") = "+ indexer.get(y,x));
                    matches.add(new Point(x, y));
       
                }
            }
        }
        return matches;
    }
    
}

代码把最佳匹配的代码的注释了，很重要的一点是
getPointsFromMatAboveThreshold(result, 0.9999f)
中的0.9999f的阈值的设置，这个很重要，多试几次。调低了的话结果可能不准。
代码的一下语句：打印目标图左上角的坐标和计算后的目标图的中心点坐标

log.info("[{}, {}]", point.x(), point.y());
log.info("[{}, {}]", point.x() + centerWith, point.y() + centerHeight);

5. 后记

本文重在功能逻辑的实现，关于javacv的学习，由于篇幅限制不予展开。