python爬虫爬取王者荣耀英雄图片
python爬取数据四步走
1、确定目标
2、分析目标
3、编写代码
4、执行爬虫
1、确定目标
爬取目标:
url = 'https://pvp.qq.com/web201605/herolist.shtml'
注意:有时候有些页面是异步加载，直接请求url地址是获得不了数据的。而王者荣耀英雄页面就是异步加载的，英雄列表数据是动态请求的，F12或者selenium分析找到一个json文件:
# json文件地址
https://pvp.qq.com/web201605/js/herolist.json
2、分析目标
思路：
先在英雄列表得到所有的英雄信息，然后依次循环爬取单个英雄的信息，得到单个英雄的所有皮肤。
选择几个英雄的路径如下:
<a href="herodetail/105.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/105/105.jpg" alt="廉颇" width="91" height="91">廉颇</a>
# 对应的json对象
{
"0": {
"ename": 105,
"cname": "廉颇",
"title": "正义爆轰",
"new_type": 0,
"hero_type": 3,
"skin_name": "正义爆轰|地狱岩魂"
}
}
<a href="herodetail/106.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/106/106.jpg" alt="小乔" width="91" height="91">小乔</a>
# 对应的json对象
{
"1": {
"ename": 106,
"cname": "小乔",
"title": "恋之微风",
"new_type": 0,
"hero_type": 2,
"skin_name": "恋之微风|万圣前夜|天鹅之梦|纯白花嫁|缤纷独角兽"
}
}
<a href="herodetail/107.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/107/107.jpg" alt="赵云" width="91" height="91">赵云</a>
# 对应的json对象
{
"2": {
"ename": 107,
"cname": "赵云",
"title": "苍天翔龙",
"new_type": 0,
"hero_type": 1,
"hero_type2": 4,
"skin_name": "苍天翔龙|忍●炎影|未来纪元|皇家上将|嘻哈天王|白执事|引擎之心"
}
}
<a href="herodetail/108.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/108/108.jpg" alt="墨子" width="91" height="91">墨子</a>
# 对应的json对象
{
"3": {
"ename": 108,
"cname": "墨子",
"title": "和平守望",
"new_type": 0,
"hero_type": 2,
"hero_type2": 1,
"skin_name": "和平守望|金属风暴|龙骑士|进击墨子号"
}
}
<a href="herodetail/109.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/109/109.jpg" alt="妲己" width="91" height="91">妲己</a>
# 对应的json对象
{
"4": {
"ename": 109,
"cname": "妲己",
"title": "魅力之狐",
"pay_type": 11,
"new_type": 0,
"hero_type": 2,
"skin_name": "魅惑之狐|女仆咖啡|魅力维加斯|仙境爱丽丝|少女阿狸|热情桑巴"
}
}
根据上述，可以推测出英雄图片地址:
# [http://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/(英雄编号)/(英雄编号)-bigskin-(第几个皮肤).jpg](http://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/137/137-bigskin-1.jpg)
https//game.gtimg.cn/images/yxzj/img201606/heroimg + ename + ename + .jpg
3、编写代码
封装请求方法、
# 封装请求方法
def send_get(self, url):
try:
response = requests.get(url, headers=self.headers)
# 断言测试
assert response.status_code == 200, '{}请求失败'.format(url)
return response
except Exception as e:
print(e)
return None
获得首页英雄列表、
# 获得所有的英雄列表
def hero_list(self):
# 获取英雄列表
response = self.send_get(self.hero_list_url)
if response:
hero_list_text = response.text # 获得响应数据
hero_list_dict = json.loads(hero_list_text) # 响应数据封装json
self.process_heroes(**hero_list_dict) # 调用处理方法
else:
print('英雄列表为空,请检查获取URL:{}'.format(self.hero_list_url))
处理英雄列表数据、
# 对英雄列表进行处理
def process_heroes(self, **hero_list_dict):
for hero in hero_list_dict['hero']:
hero_info_url = self.hero_url.format(hero['heroId'])
resp = self.send_get(hero_info_url)
if resp:
hero_info_dict = json.loads(resp.text)
self.process_hero(**hero_info_dict)
else:
print('获取英雄:{}失败,请检查获取URL:{}'.format(hero['name'], self.hero_list_url))
单个英雄数据获取
# 单个英雄处理
def process_hero(self, **hero_info_dict):
hero = hero_info_dict['hero']
skins = hero_info_dict['skins']
for skin in skins:
if skin['mainImg']:
skin_content = self.send_get(skin['mainImg']).content
hero_image_name = '{}.jpg'.format(skin['name'])
hero_image_dir = os.path.join(self.base_path, hero['name'] + hero['title'])
self.save_image(hero_image_dir, hero_image_name, skin_content)
print('hero:{},skins:{}张,处理完成'.format(hero['name'], len(skins)))
time.sleep(1)
图片保存、
def save_image(self,image_dir, image_name, image_content):
if not os.path.exists(image_dir):
os.makedirs(image_dir)
try:
hero_image_path = os.path.join(image_dir, re.sub(r'[/|?]', '', image_name))
with open(hero_image_path, 'wb') as image:
image.write(image_content)
except Exception as e:
print('{}保存失败,错误原因:{}'.format(hero_image_path, e))
4、爬虫执行
if __name__ == '__main__':
LOLHeroSpider().hero_list()
5、完整代码
import requests
import random
import time
import re
import json
import os
# url = https://lol.qq.com/data/info-heros.shtml
# url = 'https://lol.qq.com/data/info-heros.shtml'
# 需要设置USER_AGENT,假装自己是浏览器访问网页
user_agent_list = [
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11'
]
USER_AGENT = random.choice(user_agent_list)
header = {
'User-Agent':USER_AGENT
}
class LOLHeroSpider():
def __init__(self):
self.header = header
self.hero_list_url = 'https://game.gtimg.cn/images/lol/act/img/js/heroList/hero_list.js'
self.hero_url = 'https://game.gtimg.cn/images/lol/act/img/js/hero/{}.js'
self.base_path = os.path.join('d:' + os.path.sep, '英雄联盟')
# 封装请求方法
def send_get(self, url):
try:
response = requests.get(url, headers=self.header)
# 断言测试
assert response.status_code == 200, '{}请求失败'.format(url)
return response
except Exception as e:
print(e)
return None
# 获得所有的英雄列表
def hero_list(self):
# 获取英雄列表
response = self.send_get(self.hero_list_url)
if response:
hero_list_text = response.text # 获得响应数据
hero_list_dict = json.loads(hero_list_text) # 响应数据封装json
self.process_heroes(**hero_list_dict) # 调用处理方法
else:
print('英雄列表为空,请检查获取URL:{}'.format(self.hero_list_url))
# 对英雄列表进行处理
def process_heroes(self, **hero_list_dict):
for hero in hero_list_dict['hero']:
hero_info_url = self.hero_url.format(hero['heroId'])
resp = self.send_get(hero_info_url)
if resp:
hero_info_dict = json.loads(resp.text)
self.process_hero(**hero_info_dict)
else:
print('获取英雄:{}失败,请检查获取URL:{}'.format(hero['name'], self.hero_list_url))
# 单个英雄处理
def process_hero(self, **hero_info_dict):
hero = hero_info_dict['hero']
skins = hero_info_dict['skins']
for skin in skins:
if skin['mainImg']:
skin_content = self.send_get(skin['mainImg']).content
hero_image_name = '{}.jpg'.format(skin['name'])
hero_image_dir = os.path.join(self.base_path, hero['name'] + hero['title'])
self.save_image(hero_image_dir, hero_image_name, skin_content)
print('hero:{},skins:{}张,处理完成'.format(hero['name'], len(skins)))
time.sleep(1)
# 图片保存
def save_image(self,image_dir, image_name, image_content):
if not os.path.exists(image_dir):
os.makedirs(image_dir)
try:
hero_image_path = os.path.join(image_dir, re.sub(r'[/|?]', '', image_name))
with open(hero_image_path, 'wb') as image:
image.write(image_content)
except Exception as e:
print('{}保存失败,错误原因:{}'.format(hero_image_path, e))
if __name__ == '__main__':
LOLHeroSpider().hero_list()
网友评论