python 抓取网页

import json

import urllib.request

import urllib.parse

#获取豆瓣json数据

url = "https://movie.douban.com/j/search_subjects"

params = {

    'type': 'movie',

    'tag': '豆瓣高分',

    'sort': 'recommend',

    'page_limit': 20,

    'page_start': 0

}

headers = {

    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36'

}

params_encode = urllib.parse.urlencode(params).encode('utf-8')

# print(params_encode)

# url = url + '?' + params_encode

req = urllib.request.Request(url, params_encode, headers)

with urllib.request.urlopen(req) as response:

    data = response.read().decode('utf-8')

print(data)

# with open('f:/douban.json', 'w') as file:

#    json.dump(data, file, ensure_ascii=False