除去重复文件

一句话：使用md5

package main

import (
    "crypto/md5"
    "fmt"
    "io/ioutil"
    "os"
    "path/filepath"
)

var fileHash = map[[16]byte]struct{}{}

func main() {
    if len(os.Args) == 1 {
        log.Println("no input path")
        return
    }
    Rm(os.Args[1])
}

func Rm(path string) {
    err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
        if info.IsDir() {
            //忽略目录
            return nil
        }

        data, _ := ioutil.ReadFile(path)
        checksum := md5.Sum(data)
        if _, ok := fileHash[checksum]; ok {
            fmt.Println("find dup image:", info.Name())
            os.Remove(path)
            return nil
        }

        fileHash[checksum] = struct{}{}
        return nil
    })
}