【Python爬虫】第十六次 xpath整站抓取阳光电影网电影资

# 一、解析电影url
# 请求15题构造出的每个电影菜单的分页url，解析出每个电影url
# 二、xpath解析电影资源
# 对第一题解析出来的电影url进行请求,解析出最终的电影名称 电影下载地址
# 三、对代码进行分装成函数

#整个网站的层次： main_menu => page => item => source
import requests
from lxml import etree
import os
def get_html(url):
    headers=''
    req=requests.get(url)
    # print(type(req))
    ret=req.status_code
    if ret!=200 :
        print('获取主菜单失败，返回错误：'+str(ret))
        return ('')
    req.encoding='gb2312'
    html=req.text
    return(html)

def get_main_menus(url):
    html=get_html(url)
    # print(html)
    selector=etree.HTML(html)
    main_menus=selector.xpath('//div[@class="contain"]/ul/li[position()<=9]/a')
    main_menus_dict={}
    for main_menu in main_menus :
        text=main_menu.xpath('text()')[0]
        if text == '国内电影' or text == '最新影片' or text =='华语电视' or text=='欧美电影' or text=='欧美电视' or text=='日韩电影': #网络中断造成执行中断，只要略去之前完成的内容
            pass
        else:
            if len(text)>=1 :
                href = url + main_menu.xpath('@href')[0]
                # print(text,href)
                main_menus_dict.update({text:href})
            else:
                pass
    return(main_menus_dict)

def get_pages(menus):
    url_list=[]
    # main_menus_dict={}

    menu_name=menu[0]
    menu_url=menu[1]
    html_one_menu = get_html(menu_url)
    # print(html_one_menu)
    selector_one_menu=etree.HTML(html_one_menu)
    # // *[ @ id = "header"] / div / div[3] / div[3] / div[2] / div[2] / div[2] / div / select / option[2]
    pages=selector_one_menu.xpath('//select/option[starts-with(@value,"list")]')
    print(menu_name,'小计'+str(len(pages))+'页:')
    if len(pages)>=1 :
        for page in pages :
            page_url=menu_url.split('index.html')[0] + page.xpath('@value')[0]
            # print(page_url)
            url_list.append(page_url)
    return(url_list)
def get_items(menu_name,page_url):
    html_one_page=get_html(page_url)
    selector_one_page = etree.HTML(html_one_page)
    #// *[ @ id = "header"] / div / div[3] / div[3] / div[2] / div[2] / div[2] / ul / table[23] / tbody / tr[2] / td[2] / b / a[2]
    items=selector_one_page.xpath('//a[@class="ulink" and not(contains(@href,"index.html")) ]')
    i=0
    for item in items :
        i = i+1
        if len(item.xpath('text()')) >=1 :
            item_name=item.xpath('text()')[0]
            item_url=url+item.xpath('@href')[0]
            # print(menu_name,item_name)
            get_source(menu_name, item_name, item_url)
    return(i)

def get_source(menu_name,item_name,item_url):
    print(item_name,item_url)
    html_one_item=get_html(item_url)
    selector_one_item = etree.HTML(html_one_item)
    sources=selector_one_item.xpath('//td[@style="WORD-WRAP: break-word"]/a')
    if len(sources)>=1 :
        dirname = 'G:\\阳光电影网\\' + menu_name
        if not (os.path.exists(dirname)):
            os.mkdir(dirname)
        filename = dirname + '\\' + item_name.replace('/', '-').replace('?','？').replace(':', '：').replace('|','_').replace('"', '_') + '.txt'
        fp=open(filename,'w',encoding='utf-8')
        for source in sources:
            print(menu_name,item_name)
            if len(source.xpath('text()'))>=1 :
                source_text=source.xpath('text()')[0]
                source_url=source.xpath('@href')[0]
                print(source_text,source_url)
                fp.write(source_url+'\n')
                fp.write(source_text+'\n')
        fp.close()




if __name__ == '__main__' :
    url = 'http://www.ygdy8.com'
    main_menu_dict=get_main_menus(url)
    # print(main_menu_dict)
    total_page=0
    item_count = 0
    for menu in main_menu_dict.items():
        print('菜单',menu)
        menu_name = menu[0]
        page_url_list=[] #当前菜单下的全部电影页面的列表
        page_url_list=get_pages(menu)
        total_page = total_page + len(page_url_list)

        for page_url in page_url_list :
            item_count=item_count+ get_items(menu_name,page_url)
        print('累计' + str(total_page) + '页'+str(item_count)+'条')