爬取豆瓣top250电影

import requests
from lxml import etree
import pandas as pd
def parse():
    """豆瓣网top250爬虫"""
    # 1、获取url地址
    # for i in range(0, 226, 25):
    #     url = 'https://movie.douban.com/top250?start={}&filter='.format(i)
    #     print(url)
    #     # 获取 byte的类型的响应
    #     resp = requests.get(url)
    #     data = resp.content
    headers = {"User-Agent": "ozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"}
    #定义列表存储top250信息[{},{},{}]
    movie_info_list=[]
    # 1、获取url地址
    # 循环每一页
    for i in range(0, 226, 25):
        url = 'https://movie.douban.com/top250?start={}&filter='.format(i)
        # print(url)
        # 获取 byte的类型的响应
        # resp = requests.get(url)
        # data = resp.content
        # url = 'https://movie.douban.com/top250?start=0&filter='
        # 获取 byte的类型的响应
        resp = requests.get(url,headers=headers)
        data = resp.content
        # 调用etree.HTML获取html对象，然后调用html的xpath语法
        html = etree.HTML(data)

        movie_list = html.xpath('//div[@id="content"]//ol/li')
        # print(len(movie_list))
        for movie in movie_list:
            # 获取电影序号
            serial_number = movie.xpath('./div[@class="item"]/div[@class="pic"]/em/text()')
            serial_number = '' if len(serial_number)==0 else serial_number[0]
            # print(serial_number)

            movie_name = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="hd"]/a/span[1]/text()')
            movie_name = '' if len(movie_name) == 0 else movie_name[0]
            # print(movie_name)


    # serial_number = html.xpath('//*[@id="content"]/div/div[1]/ol/li[1]/div/div[1]/em/text()')
    # print(serial_number)
    # 电影名称
    # movie_name =

    # 电影介绍
    # introduce =
            introduce = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/p/text()')
            introduce = '' if len(introduce) == 0 else introduce[0]
            #去两端空格操作
            introduce=introduce.strip()
            # print(introduce)

    # 电影星级
    # star =
            star = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[2]/text()')
            star = '' if len(star) == 0 else star[0]
            # print(star)

    # 电影的评价
    # evalute =
            evalute = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[4]/text()')
            evalute = '人评价' if len(evalute) == 0 else evalute[0]
            evalute=evalute.replace('人评价','')
            # print(evalute)
    # 电影的描述
    # describe =
            describe = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/p[@class="quote"]/span[1]/text()')
            describe = '' if len(describe) == 0 else describe[0]
            # print(describe)
    #详情链接的地址
            detail_link = movie.xpath('./div[@class="item"]/div[@class="pic"]/a/@href')
            detail_link  = '' if len(detail_link ) == 0 else detail_link [0]
            # print(detail_link )
    #图片的地址
            img_url=movie.xpath('./div[@class="item"]/div[@class="pic"]/a/img/@src')
            img_url  = '' if len(img_url ) == 0 else img_url [0]
            # print(img_url )

            movie_info_list.append({
                'serial_number':serial_number,
                'movie_name':movie_name,
                'introduce':introduce,
                'star':star,
                'evalute':evalute,
                'describe':describe,
                'detail_link':detail_link,
                'img_url':img_url,



            })
    # for movie_info in movie_info_list:
    #     print(movie_info)
    #     resp=requests.get(movie_info['img_url'])
    #     if resp.status_code==200:
    #         #执行图片写入操作
    #         #参考图片命名方式  0000001.jpg
    #         img_name='000000{}.jpg'.format(movie_info['serial_number'])
    #         with open('./imgs/{}'.format(img_name),'wb') as f:
    #             f.write(resp.content)

    for movie_info in movie_info_list:
        print(movie_info)
        resp = requests.get(movie_info['img_url'])
        if resp.status_code == 200:
            # 执行图片写入操作
            # 参考图片命名方式   001.jpg
            img_name = '000000{}.jpg'.format(movie_info['serial_number'])
            with open('./imgs/{}'.format(img_name), 'wb')as f:
                f.write(resp.content)

    # 存储成csv

    df = pd.DataFrame(movie_info_list)
    df.to_csv('douban_top250_info.csv')



parse()

输出结果：

image.png

爬取猫眼top100

import requests
from lxml import etree
import pandas as pd
def parse():
    """豆瓣网top250爬虫"""
    # 1、获取url地址
    # for i in range(0, 226, 25):
    #     url = 'https://movie.douban.com/top250?start={}'.format(i)
    #     print(url)
    #     # 获取 byte的类型的响应
    #     resp = requests.get(url)
    #     data = resp.content
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"}
    # 定义列表存储top250信息 [{},{},{}]
    movie_info_list = []
    #
    # 循环每一页
    for i in range(0, 91, 10):
        url = 'https://maoyan.com/board/4?offset={}'.format(i)

        # 获取 byte的类型的响应
        req = requests.get(url , headers = headers)
        req.encoding = 'utf-8'
        data = req.text
        # 调用etree.HTML获取html对象，然后调用html的xpath语法
        html = etree.HTML(data)

        movie_list = html.xpath('//div[@class="main"]/dl/dd')
        # print(len(movie_list))
        for movie in movie_list:

            movie_name = movie.xpath('./div/div/div[1]/p[1]/a/text()')
            movie_name = '' if len(movie_name) == 0 else movie_name[0]
            # print(movie_name)


            # 演员介绍
            # role =
            role = movie.xpath('./div/div/div[1]/p[2]/text()')
            role = '' if len(role) == 0 else role[0]
            #去两端空格操作
            role=role.strip()
            # print(role)

            # 电影时间
            # # time =
            time = movie.xpath('./div/div/div[1]/p[3]/text()')
            time = '' if len(time) == 0 else time[0]
            # print(time)

            movie_info_list.append({

                'movie_name':movie_name,
                'role':role,
                'time':time,

            })
    for movie_info in movie_info_list:
        print(movie_info)
parse()

输出结果：

image.png