美文网首页
python3 log文件处理获取某天需要的数据

python3 log文件处理获取某天需要的数据

作者: 宁静消失何如 | 来源:发表于2017-08-23 11:04 被阅读141次

    问题描述:需要读取log文件,并获取出两天的日志做对比,查询增加或者减少量,规则化输出
    完成心得:python的字典,很好用,读写时间快,用法自由

    #!/usr/bin/env python
    # -*- coding: utf-8 -*-
    # @Time    : 17/8/18 下午5:08
    # @Author  : lee
    # @File    : read.py
    # @Software: PyCharm
    # 说明: code后有'#'的时测试时加的或者需要修改的code
    # 用法 传入两个值 day1是前一天的日期 day2是后一天的内容
    import sys
    def log_data(address, day1, day2):
        try:
            # 打开文件
            log = open("%s" % address)
    
            # 定义:
            dict_yesterday = {}  # 昨天的数据
            dict_the_day_before_yesterday = {}  # 前天的数据
            dict_yesterday_difference = {}  # 记录昨天中有的数据,前天没有
            dict_the_day_before_yesterday_difference = {}  # 和上一条相反
            dict_identical = {}  # 记录两天内相同的数据
            dict_result = {}  # 记录结果,输出时读取
            ii = jj = z = x = 0
            #遍历 将两个日期的日志筛选出并装入对应的字典
    
            for line in log:
                log_data = ','.join(filter(lambda x: x,line.split(' ')))
                date = (log_data.split(',')[0])[-10:].replace('.','-')
                if date == day2:
                    dict_yesterday[ii] = log_data
                    dict_yesterday_difference[ii] = log_data
                    ii += 1
                elif date == day1:
                    dict_the_day_before_yesterday[jj] = log_data
                    dict_the_day_before_yesterday_difference[jj] = log_data
                    jj += 1
    
    
    
            if len(dict_yesterday) >= len(dict_the_day_before_yesterday):
                for i in dict_yesterday:
                    object_list1 = (dict_yesterday[i]).split(',')
                    entry_name_yesterday = ((object_list1[0]).split(',')[0])[:-10]+object_list1[1]+'-'+object_list1[2]
                    for j in dict_the_day_before_yesterday:
                        object_list2 = (dict_the_day_before_yesterday[j]).split(',')
                        entry_name_tdat = ((object_list2[0]).split(',')[0])[:-10]+object_list2[1]+'-'+object_list2[2]
                        if entry_name_yesterday == entry_name_tdat:
                            dict_identical[x] = entry_name_yesterday
                            docs_result = int(((dict_yesterday[i]).split(','))[4]) - int(((dict_the_day_before_yesterday[j]).split(','))[4])
                            capacity1 = (dict_yesterday[i].split(',')[5])[-2:]
                            capacity2 = (dict_the_day_before_yesterday[j].split(',')[5])[-2:]
                            if capacity1 == capacity2:
                                store_result = str(round(float((dict_yesterday[i].split(',')[5])[:-2]) - float((dict_the_day_before_yesterday[j].split(',')[5])[:-2]),2)) +(dict_yesterday[i].split(',')[5])[-2:]
                            # 1gb = 1024mb = 1024kb
                            elif capacity1 != capacity2:
                                n_y = round(float((dict_yesterday[i].split(',')[5])[:-2]), 2)
                                n_t = float((dict_the_day_before_yesterday[j].split(',')[5])[:-2])
    
                                if capacity1 == 'gb'and capacity2 == 'mb':
                                    if n_y * 1024  > n_t:
                                        store_result  = str(abs(round(n_y * 1024 - n_t), 2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
                                    else:
                                        store_result  = str(round(n_y * 1024 - n_t),) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
                                elif capacity1 == 'gb'and capacity2 == 'kb':
                                    if n_y * 1024 * 1024 > n_t:
                                        # round(,2)
                                        store_result = str(abs(round(n_y * 1024 * 1024 - n_t,2))) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
                                    else:
                                        store_result = str(round(n_y * 1024 * 1024 - n_t,2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
                                elif capacity1 == 'mb'and capacity2 == 'gb':
                                    if n_y / 1024  > n_t:
                                        store_result = str(abs(round(n_y / 1024 - n_t,2))) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
                                    else:
                                        store_result = str(round(n_y / 1024 - n_t,2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
                                elif capacity1 == 'mb'and capacity2 == 'kb':
                                    if n_y * 1024 > n_t:
                                        store_result = str(abs(round(n_y * 1024 - n_t,2))) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
                                    else:
                                        store_result = str(round(n_y * 1024 - n_t,2)) +(dict_the_day_before_yesterday[j].split(',')[5])[-2:]
                                elif capacity1 == 'kb'and capacity2 == 'gb':
                                    store_result = str(round(n_t - n_t * 1024 * 1024,2)) +(dict_yesterday[i].split(',')[5])[-2:]
                                elif capacity1 == 'kb'and capacity2 == 'mb':
                                    if n_y / 1024  > n_t:
                                        store_result = str(abs(round(n_t - n_t * 1024,2))) +(dict_yesterday[i].split(',')[5])[-2:]
                                    else:
                                        store_result = str(round(n_t - n_t * 1024,2)) +(dict_yesterday[i].split(',')[5])[-2:]
    
                            dict_result[z] = entry_name_tdat + ' docs: ' + str(docs_result) + ' store: ' +store_result
                            z += 1
                            x += 1
    
           # else:
              #  print("当前日期下没有相关日志")
    
    
            # 筛选出昨天出现 前天没有的数据
            if len(dict_yesterday) >= len(dict_identical):
                for i in dict_yesterday:
                    object_list_1 = (dict_yesterday[i]).split(',')
                    entry_name_yesterday_1 = ((object_list_1[0]).split(',')[0])[:-10]+object_list_1[1]+'-'+object_list_1[2]
                    for j in dict_identical:
                        entry_name_identical = dict_identical[j]
                        if entry_name_yesterday_1 == entry_name_identical:
                            dict_yesterday_difference.pop(i)
    
            # 筛选出前天出现 昨天没有的数据
            if len(dict_the_day_before_yesterday) >= len(dict_identical):
                for i in dict_the_day_before_yesterday:
                    object_list_1 = (dict_the_day_before_yesterday[i]).split(',')
                    entry_name_yesterday_1 = ((object_list_1[0]).split(',')[0])[:-10]+object_list_1[1]+'-'+object_list_1[2]
                    for j in dict_identical:
                        entry_name_identical = dict_identical[j]
                        if entry_name_yesterday_1 == entry_name_identical:
                            dict_the_day_before_yesterday_difference.pop(i)
    
            # 输出两天都有的数据
            print("下面是 %s 和 %s 内%s中 docs、store差值"%(day1,day2,address))
            print('')
            print("++++++++++++++++两天都有的数据++++++++++++++++")
            for i in dict_result:
                print(dict_result[i])
    
            # 输出两天中前者中有的数据而在后者没有出现,取正值(+)区别
            print('')
            if len(dict_yesterday) != 0 :
                print("++++++++++++++++只有%s有的数据取正值(+)予以区别++++++++++++++++"%day2)
                for i in dict_yesterday_difference:
                    object = (dict_yesterday_difference[i]).split(',')
                    name =  ((object[0]).split(',')[0])[:-10]+object[1]+'-'+object[2]
                    docs = object[4]
                    store = object[5]
                    print(name + ' docs: +' + str(docs) + ' store: ' + store)
            else:
               print("%s日期下没有相关日志"%day2)
    
    
    
            # 输出两天中后者中有的数据而在前者没有出现,取负值(-)区别
            print('')
            if len(dict_the_day_before_yesterday) != 0:
                print("++++++++++++++++只有%s有的数据,都取负值(-)予以区别++++++++++++++++" % day1)
                for i in dict_the_day_before_yesterday_difference:
                    object = (dict_the_day_before_yesterday_difference[i]).split(',')
                    name =  ((object[0]).split(',')[0])[:-10]+object[1]+'-'+object[2]
                    docs = object[4]
                    store = object[5]
                    print(name+ ' docs: -' + str(docs) + ' store: -' + store)
            else:
               print("%s日期下没有相关日志"%day1)
    
        except:
    
            print("输入的参数可能不对,三个参数,地址,day1 day2")
    
    
    # log_data('/Users/lee/Desktop/py交易/es.log','2017-08-11','2017-08-12')
    
    log_data(sys.argv[1],sys.argv[2],sys.argv[3])
    
    # /Users/lee/Desktop/py交易/test.log
    
    
    

    相关文章

      网友评论

          本文标题:python3 log文件处理获取某天需要的数据

          本文链接:https://www.haomeiwen.com/subject/awlddxtx.html