Creating a project

scrapy startproject top_250

Source

in spiders/top_250.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request, Spider
from top_250.items import Top250Item

class MovieSpider(Spider):
    name = 'top_250'
    allowed_domains = ['movie.douban.com'] # don't add any `http` or `https`
    headers = {
        'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' 
    }
    def start_requests(self):
        url = 'https://movie.douban.com/top250'
        yield Request(url, headers=self.headers)
    

    def parse(self, response):
        item = Top250Item()
        for movie in response.xpath("//div[@class='item']"):
            item['movie_name'] = movie.xpath(".//a/span[@class='title']/text()").extract_first()
            item['movie_url'] = movie.xpath(".//a/@href").extract_first()
            item['movie_rank'] = movie.xpath(".//div[@class='star']/span[@class='rating_num']/text()").extract_first()
            yield item

        next_page_url = response.xpath("//div[@class='paginator']/span[@class='next']/a/@href").extract_first()
        if next_page_url:
            next_page_url = 'https://movie.douban.com/top250' + next_page_url
            yield Request(next_page_url, headers=self.headers)

in items.py

import scrapy

class Top250Item(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    movie_name = scrapy.Field()
    movie_url = scrapy.Field()
    movie_rank = scrapy.Field()

in pipelines.py

# -*- coding: utf-8 -*-
import codecs
import json
import os
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html


class Top250Pipeline(object):
    def __init__(self):
        self.file = codecs.open('top_250.json', 'w', encoding='utf-8')
        self.file.write('[')

    def process_item(self, item, spider):
        line = json.dumps(dict(item), ensure_ascii=False) + '\n'
        self.file.write(line+',')
        return item

    def close_spider(self, spider):
        self.file.seek(-1, os.SEEK_END)
        self.file.truncate()
        self.file.write(']')
        self.file.close()

in setting.py

...
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
...
...
# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'top_250.pipelines.Top250Pipeline': 300,
}
...
FEED_EXPORT_ENCODING = 'utf-8' # use utf-8 to store Chinese