爬取豆瓣电影top250全部的页面信息

不在是单一页面的爬取,但是这个页面很有规律

"""
@author:Rudy

@time : 12月4日

@message:爬取豆瓣电影排行榜top250上的全部电影排名,电影名字,评价人数,以及电影星数

"""

import requests
from lxml import etree


class DouBan():
    def __init__(self):
        self.temp_url = "https://movie.douban.com/top250?start={}&filter="
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"}

    def get_url_list(self):
        return [self.temp_url.format((i-1)*25) for i in range(1, 11)]

    def parse_url(self, url):
        response = requests.get(url, headers=self.headers)
        return response.content.decode()

    def get_content_list(self, html_str):
        html = etree.HTML(html_str)
        li_list = html.xpath("//*[@id='content']/div/div[1]/ol/li")
        content_list = []
        for li in li_list:
            item = {}
            item["rank"] = li.xpath("div/div[1]/em/text()")[0]
            item["名字"] = li.xpath("div/div[2]/div[1]/a/span[1]/text()")[0]
            item["评价人数"] = li.xpath("div/div[2]/div[2]/div/span[4]/text()")[0]
            item["电影星数"] = li.xpath("div/div[2]/div[2]/div/span[2]/text()")[0]
            content_list.append(item)
        return content_list

    def save_content_list(self, content_list):
        for content in content_list:
            print(content)

    def run(self):
        # 1 获取URL列表
        url_list = self.get_url_list()
        # 2 遍历发送请求 获取响应
        for url in url_list:
            html_str = self.parse_url(url)
            # 3 提取数据
            content_list = self.get_content_list(html_str)

            # 保存
            self.save_content_list(content_list)

if __name__ == '__main__':
    douban = DouBan()
    douban.run()