Golang 实现自动清理日志，可用于生产环境 （2）

新建：go.mod

新建:clear-dick.go ，代码如下：

package main

import (
    "auto-clear-log/utils"
    "auto-clear-log/utils/color"
    "bytes"
    "flag"
    "fmt"
    "github.com/shirou/gopsutil/disk"
    "gopkg.in/yaml.v2"
    "io/ioutil"
    "log"
    "os"
    "path"
    "path/filepath"
    "regexp"
    "sort"
    "strings"
    "syscall"
    "time"
)

type Conf struct {
    Dir        string
    Prefix     string
    Suffix     string
    BeforeTime int64 `yaml:"beforeTime"`
    Mode       string
    Retain     int
    MinSize    int `yaml:"minSize"`
    Debug      bool
}
type FileType struct {
    Path    string
    Name    string
    Size    int64
    ModTime time.Time
}

//邮件内容
var EmailContent bytes.Buffer

//清理大小总计
var ClearSize int64

//预估清理大小
var EstimateSize int64

const TimeFormat = "2006-01-02 15:04:05"
//重新计算文件大小的最小值
var MinRecountFileSize int64
//安全的扩展名
var safeExt = []string{".log", ".json", ".gz", ".tgz", ".tar", ".zip", ".rar", ".mp4", ".out", ".bz2", ".txt"}

func init()  {
    MinRecountFileSize = 10 * 1024 * 1024
}
func main() {
    var minSize float64
    var configDir string
    var configFile string
    var email string
    var help bool
    flag.BoolVar(&help, "help", false, "显示帮助")
    flag.Float64Var(&minSize, "min-size", 90, "阈值比例，例如：磁盘使用率大于90%后执行")
    flag.StringVar(&configDir, "config-dir", "./rules/", "配置目录")
    flag.StringVar(&configFile, "config-file", "", "指定配置文件")
    flag.StringVar(&email, "email", "", "指定邮箱收件人")
    flag.Parse()

    if help {
        flag.Usage()
        os.Exit(0)
    }

    parts, _ := disk.Partitions(true)
    var minDiskSize uint64

    //小于1Gb的磁盘，不做处理
    minDiskSize = 1024 * 1024 * 1024
    isCheck := false
    diskData := ""
    for _, diskinfo := range parts {
        data, _ := disk.Usage(diskinfo.Mountpoint)
        if data.Total < minDiskSize {
            continue
        }

        if data.UsedPercent > minSize-1 {
            isCheck = true
            UsedPercent := fmt.Sprintf("%.1f", data.UsedPercent)
            DiskTotalSize := int64(data.Total)
            LeftSize := int64(data.Free)
            diskData = "挂载点: "+data.Path+", 使用比例: " + UsedPercent + "%, 磁盘容量:" + utils.FormatFileSize(DiskTotalSize) + ", 剩余容量: " + utils.FormatFileSize(LeftSize) + ", 磁盘类型:" + data.Fstype
            fmt.Println(data)
            break
        }

    }

    if isCheck == false {
        fmt.Println("磁盘容量没有达到设定的阈值:", minSize, "% , 正常退出")
        os.Exit(0)
    }

    nowTime := time.Now().Format(TimeFormat)
    fmt.Println("[开始时间]", nowTime)
    fmt.Println("+-----------------------------------------------------------------------+")
    fmt.Println("[磁盘信息]", diskData)
    fmt.Println("+-----------------------------------------------------------------------+")
    EmailContent.WriteString("[开始时间] " + nowTime + "\n")
    EmailContent.WriteString("+-----------------------------------------------------------------------+\n")
    hostname, _ := os.Hostname()
    ip := utils.GetIp()
    EmailContent.WriteString("[主机名] " + hostname + "\n")
    EmailContent.WriteString("[IP] " + ip + "\n")
    EmailContent.WriteString("[磁盘信息] " + diskData + "\n")
    EmailContent.WriteString("+-----------------------------------------------------------------------+\n")

    files, err := ioutil.ReadDir(configDir)
    if err != nil {
        fmt.Println("配置目录不存在", err)
        os.Exit(101)
    }
    if configFile == "" {
        for _, cFile := range files {
            ext := path.Ext(cFile.Name())
            if ext != ".yml" {
                continue
            }
            err = findConf(configDir + cFile.Name())
            if err != nil {
                continue
            }
        }
    } else {
        _ = findConf(configDir + configFile)
    }

    endTime := time.Now().Format(TimeFormat)
    fmt.Println("+-----------------------------------------------------------------------+")
    successClearSize := utils.FormatFileSize(ClearSize)
    estimateSize := utils.FormatFileSize(EstimateSize)
    fmt.Println("[成功清理]", successClearSize)
    fmt.Println("[预估清理]", estimateSize)

    EmailContent.WriteString("+-----------------------------------------------------------------------+\n")
    EmailContent.WriteString("[成功清理] " + successClearSize + "\n")
    EmailContent.WriteString("[预估清理] " + estimateSize + "\n")
    EmailContent.WriteString("[完成时间]" + endTime + "\n")
    if email != "" {

        isCheck = false
        for _, diskinfo := range parts {
            data, _ := disk.Usage(diskinfo.Mountpoint)
            if data.Total < minDiskSize {
                continue
            }
            if data.UsedPercent > minSize {
                isCheck = true
                break
            }
        }
        if isCheck == true {
            EmailContent.WriteString("\n 清理任务已执行，但清理规则似乎未完全覆盖，需要人工处理处理 \n")
        } else {
            EmailContent.WriteString("\n 磁盘容量已下降，可忽略本邮件 \n")
        }

        content := "<pre>" + EmailContent.String() + "</pre>"
        if isCheck == true {
            utils.SendEmailApi(email, "【磁盘容量不足警告】🔥", content, "日志清理")
        } else {
            utils.SendEmailApi(email, "磁盘清理完成通知", content, "日志清理")
        }

    }
    fmt.Println("[完成时间]", endTime)
}
func findConf(filename string) error {


    var confList []Conf
    yamlFile, err := ioutil.ReadFile(filename)
    //log.Println("yamlFile:", string(yamlFile))
    if err != nil {
        log.Printf("yamlFile.Get err #%v ", err)
        return err
    }

    err = yaml.Unmarshal(yamlFile, &confList)
    if err != nil {
        log.Fatalf("配置文件格式错误:%s , %v", filename, err)
        return err
    }
    //打印配置
    //fmt.Println(confList)
    if len(confList) > 0 {
        fmt.Println("[配置文件]", filename)
        fmt.Println("+-----------------------------------------------------------------------+")

        EmailContent.WriteString("[配置文件] ➜ " + filename + "\n")
        EmailContent.WriteString("+-----------------------------------------------------------------------+\n")

        for _, conf := range confList {
            tmpdir := strings.TrimRight(conf.Dir, "/")
            stringCount := strings.Count(tmpdir, "/")
            //获取前缀是否配置并满足规则，必须包含 .
            stringCountPrefix := 1
            if conf.Prefix != "" {
                stringCountPrefix = strings.Count(conf.Prefix, ".")
            }

            if stringCount > 1 && stringCountPrefix > 0 {
                ListDir(conf, conf.Dir)
            } else if stringCountPrefix == 0 {
                fmt.Println(color.Magenta("[异常]"), "文件前缀不符合规范,必须包含 \".\" ➜ 当前配置为:", conf.Prefix)
                EmailContent.WriteString("[异常] 文件前缀不符合规范,必须包含 \".\" ➜ 当前配置为:" + conf.Prefix + "\n")
            } else {
                fmt.Println(color.Magenta("[异常]"), "目录规范不符合规范 ➜ 禁止根目录和一级目录 ➜", conf.Dir)
                EmailContent.WriteString("[异常] 目录规范不符合规范 ➜ 禁止根目录和一级目录 ➜ " + conf.Dir + "\n")
            }

        }
    } else {
        fmt.Println("+-----------------------------------------------------------------------+")
        fmt.Println("[配置为空]", filename)
    }
    return nil
}
func ListDir(config Conf, folder string) error {
    files, errDir := ioutil.ReadDir(folder)

    if errDir != nil {
        fmt.Println("[提示]", errDir)
        return nil
    }
    files = sortByTime(files)
    var filelist []FileType
    for _, file := range files {
        if file.IsDir() {
            ListDir(config, folder+"/"+file.Name())
        } else {
            // 输出绝对路径
            strAbsPath, errPath := filepath.Abs(folder + "/" + file.Name())
            if errPath != nil {
                fmt.Println(errPath)
            }
            //error.log.20200705
            //%2Fworkspace%2Flogs%2Fservice%2Ftest%2Faccesslog.log.20200721
            tmpfile := strings.ReplaceAll(file.Name(), "%2F", "")
            match, _ := regexp.MatchString("^[A-Za-z0-9-_]+\\.log[.-][0-9-]+$", tmpfile)
            if match == false {
                //86_70_20200715160311.cache.172.16.49.19
                match, _ = regexp.MatchString("^[A-Za-z0-9_-]+\\.cache\\.[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$", file.Name())
            }
            //后缀优先
            if config.Suffix != "" {
                //获取后缀
                ext := path.Ext(strAbsPath)
                if ext == ".properties" {
                    continue
                }
                if ext == "" {
                    continue
                }

                if match == false {
                    //验证后缀有没有在安全范围
                    if utils.InArray(ext, safeExt) == false {
                        continue
                    }
                    if ext != config.Suffix {
                        continue
                    }
                }
            } else if config.Prefix != "" {
                ext := path.Ext(strAbsPath)
                //后缀长度必须大于8
                if len(ext) < 8 {
                    continue
                }
                //判断前缀是否匹配,0表示，出现在最前面
                if strings.Index(file.Name(), config.Prefix) != 0 {
                    continue
                }
            } else if match == false {
                continue
            }
            //大于10M的文件，重新计算真实的大小
            //fmt.Println(file.Size(),file.Name())
            if file.Size() > MinRecountFileSize {
                nf := file.Sys().(*syscall.Stat_t)
                nfsize := nf.Blocks * 512
                //如果大小 4096 * 2 小于 2 个扇区大小，忽略文件
                if nfsize < 8192 {
                    continue
                }
            }
            var fileInfo FileType
            fileInfo.Path = strAbsPath
            fileInfo.Size = file.Size()
            fileInfo.Name = file.Name()
            fileInfo.ModTime = file.ModTime()
            //fmt.Println(strAbsPath,file.Size())
            filelist = append(filelist, fileInfo)
        }
    }
    //fmt.Println(filelist)
    //如果配置了保留数量 > 0
    minFileSize := config.MinSize
    if minFileSize > 0 {
        minFileSize = minFileSize * 1024 * 1024
    }
    beforeTime := config.BeforeTime
    if beforeTime > 0 {
        beforeTime = time.Now().Unix() - beforeTime*60
    }
    //fmt.Println(config.MinSize,config)
    if config.Retain > 0 {
        if len(filelist) > config.Retain {
            for i, file := range filelist {
                //fmt.Println(file.ModTime.Format(TimeFormat),file.Path)
                if i >= config.Retain {
                    RunRule(config, file, int64(minFileSize), beforeTime)
                }
            }
        } else {
            return nil
        }
    } else {
        for _, file := range filelist {
            RunRule(config, file, int64(minFileSize), beforeTime)
        }
    }
    return nil
}
func RunRule(config Conf, file FileType, minFileSize int64, beforeTime int64) error {
    filesize := utils.FormatFileSize(file.Size)
    colorSize := color.Cyan(filesize)

    if minFileSize > 0 && file.Size < minFileSize {
        fmt.Println(color.Green("[忽略]"), file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize)
        return nil
    }
    if beforeTime > 0 && file.ModTime.Unix() > beforeTime {
        fmt.Println(color.Green("[忽略]"), file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize)
        return nil
    }
    var result string
    if config.Mode == "rm" {
        if config.Debug == false {
            //执行删除
            err := os.Remove(file.Path)
            if err != nil {
                result = "[✖]"
            } else {
                result = "[✔]"
                ClearSize += file.Size
            }
            fmt.Println("[删除]", file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize, result)
            EmailContent.WriteString("[删除]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + " " + result + "\n")

        } else {
            fmt.Println("[删除]", file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize, "[Debug]")
            EmailContent.WriteString("[删除]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + " [Debug]\n")
        }
        EstimateSize += file.Size
    } else if config.Mode == "waring" {
        fmt.Println(color.Red("[告警]"), color.Red(file.ModTime.Format(TimeFormat)), "➜", color.Red(file.Path), colorSize)
        EmailContent.WriteString("[告警]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + "\n")
        return nil
    } else {
        //文件大小为0的，不做清空处理
        if file.Size==0 {
            return nil
        }
        if config.Debug == false {
            err := os.Truncate(file.Path, 0)
            //f, err := os.OpenFile(file.Path, os.O_WRONLY|os.O_TRUNC, 0600)
            if err != nil {
                result = "[✖]"
            } else {
                result = "[✔]"
                ClearSize += file.Size
            }
            //f.WriteString("")
            //f.Close()
            fmt.Println("[清空]", file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize, result)
            EmailContent.WriteString("[清空]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + " " + result + "\n")
        } else {
            fmt.Println("[清空]", file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize, "[Debug]")
            EmailContent.WriteString("[清空]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + " [Debug]\n")
        }
        EstimateSize += file.Size
    }

    return nil
}

func sortByTime(pl []os.FileInfo) []os.FileInfo {
    sort.Slice(pl, func(i, j int) bool {
        flag := false
        if pl[i].ModTime().After(pl[j].ModTime()) {
            flag = true
        } else if pl[i].ModTime().Equal(pl[j].ModTime()) {
            if pl[i].Name() < pl[j].Name() {
                flag = true
            }
        }
        return flag
    })
    return pl
}

喜欢就给个赞吧

未完 … 请看下一篇