美文网首页学习提升Python
python爬虫爬取特定网站的全部网页

python爬虫爬取特定网站的全部网页

作者: 腰果大妖怪 | 来源:发表于2019-01-01 11:46 被阅读0次

    话不多说直接上代码:

    获取目标网站的大量相关网页链接

    if name == 'main':
    import ClimbHealthSkyCom

    ClimbHealthSkyCom.main()
    

    对链接进行进一步处理,去重

    def LinkDeduplication():
    print('开始链接去重')
    try:
    fp = open('secondList.txt', 'r')
    linkList = []
    line = fp.readline()
    while line:
    linkList.append(line)
    line = fp.readline()
    except:
    print('未找到相关文件')
    # 去重
    dealLinkListResult = list(set(linkList))
    dealLinkListResult.sort()
    # 保存去重后的链接
    try:
    fp = open('dealLinkListResult.txt', 'a')
    except:
    print('创建dealLinkListResult.txt失败')
    for temp in dealLinkListResult:
    fp.write(temp)
    print('链接去重结束')

    LinkDeduplication()

    爬取中国糖尿病网站中糖尿病相关信息链接

    author 王耀国

    time 2019年12月29日

    '''
    经过分析,该网站的首页是动态加载内容,并且加载网页的url为http://www.health-sky.com/zhishi/page/
    目前为止不清楚网页的规模大小,也就是不知道?最大时多少
    经过分析,首页右侧有几大分类,可以依据其相似格式对网站进行爬取
    现在构造一个链接盒子listold[]存放已经爬取过的链接,目前时单线程爬取,以后考虑多线程,一边更新自建的目标网站链接库,一边爬取并解析网页
    构造一个二级链接库
    '''

    import urllib.request
    from bs4 import BeautifulSoup
    import re
    import os

    第一步获取原网页 因为网页位动态加载所以需要分析抓取动态网页 经过分析找到了动态加载的链接。

    def gethtml(url):
    print('正在获取网页')
    req = urllib.request.Request(url)
    response = urllib.request.urlopen(req)
    res = response.read().decode('utf-8')
    print('获取网页结束')
    return res

    利用Beautiful Soup解析网页

    def ParsingWebpage(res):
    print('正在解析网页')
    # 暂存新的链接
    secondList = []
    # 将新获得的链接保存到文件中
    if (os.path.exists('secondList.txt')):
    fp2 = open('secondList.txt', 'a')
    print("打开已存在的'secondList.txt'")
    else:
    fp2 = open('secondList.txt', 'w')
    print("创建新的'secondList.txt'")
    soup = BeautifulSoup(res, "lxml")
    # 获取网站的全部指定链接
    for linkList in soup.find_all('a',href = re.compile(r"http://www.health-sky.com/.*?.html")):
    # print(linkList['href'])
    secondList.append(linkList['href'] + "\n")
    for temp in secondList:
    fp2.write(temp)
    fp2.close()
    print('解析网页结束')

    从此处开始爬取

    def main():
    print('进入main')
    # 参照首页右侧的链接大类分类爬取并保存
    firstLink = [] # 原始的链接库
    firstLink.append('http://www.health-sky.com/zhishi')
    firstLink.append('http://www.health-sky.com/bingyin')
    firstLink.append('http://www.health-sky.com/zhengzhuang')
    firstLink.append('http://www.health-sky.com/zhiliao')
    firstLink.append('http://www.health-sky.com/jiancha')
    firstLink.append('http://www.health-sky.com/zhenduan')
    firstLink.append('http://www.health-sky.com/bingfazheng')
    firstLink.append('http://www.health-sky.com/huli')
    firstLink.append('http://www.health-sky.com/yufang')
    firstLink.append('http://www.health-sky.com/yiyuan')
    firstLink.append('http://www.health-sky.com/yaowu')

    # #与之相匹配的正则表达式解析页面获取网页链接
    # ListOfRe = []
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/zhishi/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/bingyin/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/zhengzhuang/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/zhiliao/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/jiancha/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/zhenduan/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/bingfazheng/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/huli/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/yufang/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/yiyuan/.*?.html")')
    # ListOfRe.append(' href = re.compile(r"http://www.health-sky.com/yaowu/.*?.html")')
    
    listOld = []  # 存放已经爬过的链接库
    # 判断自建的目标网站链接库是否存在
    if (os.path.exists('oldList.txt')):
        fp = open("oldList.txt", 'a')  # 追加新的链接
        print("打开已存在的oldList.txt")
    else:
        fp = open("oldList.txt", 'w')
        print("创建新的oldList.txt")
    # 根据原始链接库循环爬取
    for temp in firstLink:
        print(temp)
        i = 1
        res = gethtml(temp)
        listOld.append(temp)
        while res:
            ParsingWebpage(res)  # 解析网页
            i += 1
            urlTemp = temp + '/page/%d' % i
            print(urlTemp)
            try:
                res = gethtml(urlTemp)
                listOld.append(urlTemp)
            except:
                print('爬取%s结束!共爬取网页%d页' % (urlTemp, i - 1))
                break
    for temp in listOld:
        fp.write(temp + "\n")
    fp.close()
    print('离开main')
    

    爬取根据dealLinkListResult链接库爬取最终结果

    from bs4 import BeautifulSoup
    import os

    if name == 'main':
    import ClimbHealthSkyCom

    def ClimbFinalResult():
    print('进入ClimbFinalResult')
    #最终的链接库
    linkList = []
    linkList.append('Link/LinkOfCauseOfDiabetes.txt')
    linkList.append('Link/LinkOfDiabetesCare.txt')
    linkList.append('Link/LinkOfDiabetesCheck.txt')
    linkList.append('Link/LinkOfDiabetesComplications.txt')
    linkList.append('Link/LinkOfDiabetesDiagnosis.txt')
    linkList.append('Link/LinkOfDiabetesHospital.txt')
    linkList.append('Link/LinkOfDiabetesKnowledge.txt')
    linkList.append('Link/LinkOfDiabetesMedication.txt')
    linkList.append('Link/LinkOfDiabetesPrevention.txt')
    linkList.append('Link/LinkOfDiabetesSymptoms.txt')
    linkList.append('Link/LinkOfDiabetesTreatment.txt')
    #最终的文档库
    contentList = []
    contentList.append('content/CententOfCauseOfDiabetes.txt')
    contentList.append('content/CententOfDiabetesCare.txt')
    contentList.append('content/CententOfDiabetesCheck.txt')
    contentList.append('content/CententOfDiabetesComplications.txt')
    contentList.append('content/CententOfDiabetesDiagnosis.txt')
    contentList.append('content/CententOfDiabetesHospital.txt')
    contentList.append('content/CententOfDiabetesKnowledge.txt')
    contentList.append('content/CententOfDiabetesMedication.txt')
    contentList.append('content/CententOfDiabetesPrevention.txt')
    contentList.append('content/CententOfDiabetesSymptoms.txt')
    contentList.append('content/CententOfDiabetesTreatment.txt')

    # 打开链接库Link
    contenNumber = 0
    for temp in linkList:
        print(temp)
        print(contenNumber)
        try:
            fp = open(temp, 'r')
            print('打开%s成功'%temp)
        except:
            print('打开%s失败'%temp)
    
        # 保存网页数据
        if os.path.exists(contentList[contenNumber]):
            print(contentList[contenNumber])
            fpContent = open(contentList[contenNumber], 'a', encoding="utf8")  # 追加新的内容
            print("打开已存在的%s" % contentList[contenNumber])
        else:
            fpContent = open(contentList[contenNumber], 'w', encoding="utf8")
            print("创建新的%s" % contentList[contenNumber])
    
        #获取网页内容
        temp = fp.readline()
        while temp:
            print(temp)
            try:
                res = ClimbHealthSkyCom.gethtml(temp)
                #print(res)
                # 解析返回结果
                soup = BeautifulSoup(res,'lxml')
                # 获取网页内容
                for content in soup.find('h1'):
                    fpContent.write(content.text)
                for content in soup.find_all('article'):
                    fpContent.write(content.text)
            except:
                print('获取网页内容失败!')
            temp = fp.readline()
        fp.close()
        fpContent.close()
        contenNumber += 1  # 保持文档与链接的一致性
    

    ClimbFinalResult()
    注意最后的链接需要自己分类存放到相应的txt中,然后进行相应的网页最终内容提取

    相关文章

      网友评论

        本文标题:python爬虫爬取特定网站的全部网页

        本文链接:https://www.haomeiwen.com/subject/bvlnlqtx.html