采集下拉框多线程-正则-多进程

成果

和之前的多进程相比，这个实用点，原来那个并没有处理好循环中的重复采集问题，实际上是不能用于多进程的
改动如下：
所有的读写操作都放在最后一步执行，这样能避免一些复杂的情况，减少涉及的参数
读取的东西化作数列或者会比较好
def read(kw,zm1s):
所有的读写操作只发生在final（kw）
记住return只能返回一次，然后函数就结束了
稍微引入了json
其他变化不大

import requests
import re
import time
import json
from multiprocessing import Pool
time1 = time.strftime("%H:%M:%S").replace(':','')
def get_baidu_dasou(keyword ='剑圣'):
    try:
        url1 = 'http://suggestion.baidu.com/su?json=1&p=3?wd=' + keyword
        url2 = 'http://suggestion.baidu.com/su?sugmode=2&json=1&wd=' + keyword
        reslt_baidu_dasou1 = requests.get(url1).text
        reslt_baidu_dasou2 = requests.get(url2).text
        reslt_baidu_dasou1 = reslt_baidu_dasou1.replace('window.baidu.sug(', '').replace(')', '').replace(';', '')
        reslt_baidu_dasou2 = reslt_baidu_dasou2.replace('window.baidu.sug(', '').replace(')', '').replace(';', '')
        jcont_reslt_baidu_dasou1 = json.loads(reslt_baidu_dasou1)
        jcont_reslt_baidu_dasou2 = json.loads(reslt_baidu_dasou2)
        final_longtaikeywords = set(jcont_reslt_baidu_dasou1['s']) | set(jcont_reslt_baidu_dasou2['s'])  # 取集合交集
        print(str(keyword)+':'+str(final_longtaikeywords))
        return list(final_longtaikeywords) #返回的是数列
    except Exception as e:
        print(e)
        time.sleep(5)

def get_sougou(keyword ='剑圣'):
    final_longtaikeywords=[]
    try:
        urlsougou = 'https://www.sogou.com/suggnew/ajajjson?key='+ keyword+'&type=web&ori=yes&pr=web'
        reslt_sougou_dasou1 = requests.get(urlsougou).text
        reslt_sougou_dasou1=reslt_sougou_dasou1.split('[')[2] #获得中间的长尾词结果
        kws = re.compile(r'"(.*?)",')
        kws_list_dasou1 = kws.findall(reslt_sougou_dasou1)
        for kws1 in kws_list_dasou1:
            final_longtaikeywords.append(kws1)
        print(str(keyword)+':'+str(final_longtaikeywords))
        return (final_longtaikeywords) #返回的是数列
    except Exception as e:
        print(e)
        time.sleep(5)

def get_360(keyword ='剑圣'):
    final_longtaikeywords=[]
    try:
        url360 = 'http://sug.so.360.cn/suggest?encodein=utf-8&encodeout=utf-8&format=json&fields=word,obdata&word=' + keyword
        reslt_360_dasou1 = requests.get(url360).text
        json_reslt_360 = json.loads(reslt_360_dasou1)
        for i in range(0, len(json_reslt_360['result'])):
            final_longtaikeywords.append(json_reslt_360['result'][i]['word'])
        print(str(keyword) + ':' + str(final_longtaikeywords))
        return (final_longtaikeywords)  # 返回的是数列
    except Exception as e:
        print(e)
        time.sleep(5)

def read(kw,zm1s):
    final_longtais=[]
    last_longtais=[]
    for zm1 in zm1s:
        print('百度搜索：'+kw+zm1)#是否还要判断
        final_longtais.append(get_baidu_dasou(kw+zm1))
        print('百度搜索：'  + zm1+ kw)
        final_longtais.append(get_baidu_dasou(zm1+kw))#其他搜索引擎
        print('搜狗搜索：' + kw + zm1)
        final_longtais.append(get_sougou(kw + zm1))#搜狗不做
        print('360搜索：' + kw + zm1)
        final_longtais.append(get_360(kw + zm1))
        print('360搜索：' + zm1+ kw)
        final_longtais.append(get_360(zm1+ kw))
        if final_longtais!=None:
            for final_longtai in final_longtais: #第一层数列
                if final_longtai != None:
                    for each_word in final_longtai:  #数列中的数列
                        last_longtais.append(each_word)
        last_longtais2=quchong(last_longtais)
    print(last_longtais2)
    print(len(last_longtais2)) #执行完return就结束了，所以这里只能返回一个数列
    return last_longtais2

def quchong(firstlist): # 去重
    last_firstlist = list(set(firstlist))
    last_firstlist.sort(key=firstlist.index)
    return last_firstlist

def final(kw):
    resultpath= r'C:/Users/Administrator/Desktop/result/xiala/'+time1+'resultkeywords.txt'
    finalpath = r'C:/Users/Administrator/Desktop/result/xiala/'+time1+'finalkeywords.txt'
    lastpath = r'C:/Users/Administrator/Desktop/result/xiala/' +time1+'lastpath.txt'
    zm1s=['',' ','a','b','c','d','e','f','g','h','j','k','l','m','n','o','p','q','r','s','t','w','x','y','z','0','1','2','3','4','5','6','7','8','9']
    #zm1s = ['','a']
    zm2s=['',' ']
    #zm2s = ['']
    finalist =[]   # 扩展第二次,存放2层列表
    finalist2= [] #存放最终的关键词
    lastlist=[]     #扩展第三次
    firstlist=read(kw,zm1s)#返回第一次排重后的数列
    with open(resultpath,'a+',encoding='utf-8')as text:
        for keywords in firstlist:
            text.write(keywords+'\n')
    lastlist=read(kw,zm1s)
    if firstlist != None:
        for kw in firstlist:
            finalist.append(read(kw, zm2s)) #二层数列
            if finalist != None:
                for final_longtai in finalist:  # 第一层数列
                    if final_longtai != None:
                        for each_word in final_longtai:  # 数列中的数列
                            finalist2.append(each_word)
                            lastlist.append(each_word)
    finalist2=quchong(finalist2)
    with open(finalpath,'a+',encoding='utf-8')as text:
        for keywords in finalist2:
            text.write(keywords+'\n')
    lastlist=quchong(lastlist)
    with open(lastpath,'a+',encoding='utf-8')as text:
        for keywords in lastlist:
            text.write(keywords+'\n')

if __name__ == '__main__':#为什么会重复了
    start = time.clock()
    pool = Pool(processes=6)
    keywords_list=[]
    startpath1 = r'C:\Users\Administrator\Desktop/result\xiala\testforkeywords.txt'  # 关键词存放的地方
    with open(startpath1, 'r', encoding='utf-8') as text:  # 打开关键词文件，做列表list
        kwsline = text.readlines()
        for kw in kwsline:
            kw = kw.strip()  # 去除空行
            kw = kw.replace('\n', '')  # 替换最后一行
            keywords_list.append(kw)
    pool.map(final,keywords_list)#这里似乎需要一次排重
    pool.close()
    pool.join()
    end = time.clock()
    print ("read: %f s" %(end - start))