Python 抓取猫眼TOP100的电影相关信息

#! -*- coding:utf-8 -*-
# http://maoyan.com/board/4
# 抓取猫眼TOP100的电影相关信息

import re
import requests
from requests.exceptions import RequestException
import json
from multiprocessing import Pool

# 获取一个页面内容
def get_one_page(url):
    try:
        headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36"
                   ,"accept-language":"zh-CN,zh;q=0.9"
                   ,"accept-encoding":"gzip"}
        request = requests.get(url,headers=headers)
        if request.status_code == 200:
            return request.text
        return None
    except RequestException:
        return None

# 解析一个页面内容
def parse_noe_page(html):
    pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name">.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(\d+)</i>.*?</dd>',re.S)
    try:
        dds = re.findall(pattern,html)
        for i in dds:
            yield {
                'index':i[0],
                'image':i[1],
                'title':i[2],
                'author':i[3].strip()[3:],
                'time':i[4][5:],
                'score':i[5]+i[6]
            }
    except Exception:
        print('解析页面异常')
        return None

# 数据保存到文件
def save_file(content):
    with open('Maoyan.txt','a', encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False) + '\n')
        f.close()

def main(offset):
    url = 'http://maoyan.com/board/4?offset=' + str(offset)
    html = get_one_page(url)
    for it in parse_noe_page(html):
        save_file(it)

if __name__ == "__main__":
    # 多线程序抓取
    pool = Pool()
    pool.map(main,[i*10 for i in range(10)])

效果如下：

QQ截图20181226122521.png