Python3 requests多线程抓取猫眼电影Top100保

#coding=utf-8
import json
import re
from multiprocessing import Pool
import requests
from requests.exceptions import RequestException
from fake_useragent import UserAgent


ua=UserAgent()

def get_page(url):
    try:
        headers={'User-Agent':ua.chrome}
        response=requests.get(url,headers=headers)
        if response.status_code==200:
            return response.text
        return None
    except RequestException:
        return None

'''
<dd>
                        <i class="board-index board-index-1">1</i>
    <a href="/films/1203" title="霸王别姬" class="image-link" data-act="boarditem-click" data-val="{movieId:1203}">
      <img src="//s3plus.meituan.net/v1/mss_e2821d7f0cfe4ac1bf9202ecf9590e67/cdn-prod/file:5788b470/image/loading_2.e3d934bf.png" alt="" class="poster-default">
      <img alt="霸王别姬" class="board-img" src="https://p1.meituan.net/movie/20803f59291c47e1e116c11963ce019e68711.jpg@160w_220h_1e_1c">
    </a>
    <div class="board-item-main">
      <div class="board-item-content">
              <div class="movie-item-info">
        <p class="name"><a href="/films/1203" title="霸王别姬" data-act="boarditem-click" data-val="{movieId:1203}">霸王别姬</a></p>
        <p class="star">
                主演：张国荣,张丰毅,巩俐
        </p>
<p class="releasetime">上映时间：1993-01-01</p>    </div>
    <div class="movie-item-number score-num">
<p class="score"><i class="integer">9.</i><i class="fraction">5</i></p>        
    </div>

      </div>
    </div>

                </dd>
'''

def parse_page(html):
    pattern=re.compile(r'<dd>.*?board-index-\d+">(.*?)</i>.*?<img src="(.*?)"'
                       +'.*?data-val="{movieId:\d+}">(.*?)</a>'
                       +'.*?<p class="star">(.*?)</p>.*?class="releasetime">(.*?)</p>'
                       +'.*?<i class="integer">(.*?)</i>'
                       +'.*?<i class="fraction">(.*?)</i>.*?</dd>',re.S)
    items=pattern.findall(html)

    for item in items:
        yield{
            'index':item[0],
            'image':item[1],
            'title':item[2],
            'star': item[3].strip()[4:],
            'time':item[4][5:],
            'score':item[5]+item[6]
        }
def write_to_file(content):
    with open("res.txt","a",encoding="utf-8") as f:
        f.write(content)

def main(offset):
    url="https://maoyan.com/board/4?offset="+str(offset)
    html=get_page(url)
    if html!=None:
        for item in parse_page(html):
            print(item)
            write_to_file(json.dumps(item,ensure_ascii=False)+"\n")

if __name__=="__main__":
    pool=Pool()
    pool.map(main,[i*10 for i in range(10)])
    pool.close()
    pool.join()

User-Agentx 必须要加,否则403
ensure_ascii=False 保证中文是友好显示

原文: https://rumenz.com/rumenbiji/python3-requests-multiprocessing.html