美文网首页
python爬虫爬取王者荣耀英雄列表图片

python爬虫爬取王者荣耀英雄列表图片

作者: 逛逛_堆栈 | 来源:发表于2021-06-01 12:35 被阅读0次

    python爬虫爬取王者荣耀英雄图片

    python爬取数据四步走
    1、确定目标
    2、分析目标
    3、编写代码
    4、执行爬虫

    1、确定目标

    爬取目标:

    url = 'https://pvp.qq.com/web201605/herolist.shtml'
    

    注意:有时候有些页面是异步加载,直接请求url地址是获得不了数据的。而王者荣耀英雄页面就是异步加载的,英雄列表数据是动态请求的,F12或者selenium分析找到一个json文件:


    # json文件地址
    https://pvp.qq.com/web201605/js/herolist.json
    
    

    2、分析目标

    思路:
    先在英雄列表得到所有的英雄信息,然后依次循环爬取单个英雄的信息,得到单个英雄的所有皮肤。
    选择几个英雄的路径如下:

    <a href="herodetail/105.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/105/105.jpg" alt="廉颇" width="91" height="91">廉颇</a>
    # 对应的json对象
    {
        "0": {
            "ename": 105,
            "cname": "廉颇",
            "title": "正义爆轰",
            "new_type": 0,
            "hero_type": 3,
            "skin_name": "正义爆轰|地狱岩魂"
        }
    }
    <a href="herodetail/106.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/106/106.jpg" alt="小乔" width="91" height="91">小乔</a>
    # 对应的json对象
    {
        "1": {
            "ename": 106,
            "cname": "小乔",
            "title": "恋之微风",
            "new_type": 0,
            "hero_type": 2,
            "skin_name": "恋之微风|万圣前夜|天鹅之梦|纯白花嫁|缤纷独角兽"
        }
    }
    <a href="herodetail/107.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/107/107.jpg" alt="赵云" width="91" height="91">赵云</a>
    # 对应的json对象
    {
        "2": {
            "ename": 107,
            "cname": "赵云",
            "title": "苍天翔龙",
            "new_type": 0,
            "hero_type": 1,
            "hero_type2": 4,
            "skin_name": "苍天翔龙|忍●炎影|未来纪元|皇家上将|嘻哈天王|白执事|引擎之心"
        }
    }
    <a href="herodetail/108.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/108/108.jpg" alt="墨子" width="91" height="91">墨子</a>
    # 对应的json对象
    {
        "3": {
            "ename": 108,
            "cname": "墨子",
            "title": "和平守望",
            "new_type": 0,
            "hero_type": 2,
            "hero_type2": 1,
            "skin_name": "和平守望|金属风暴|龙骑士|进击墨子号"
        }
    }
    <a href="herodetail/109.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/109/109.jpg" alt="妲己" width="91" height="91">妲己</a>
    # 对应的json对象
    {
        "4": {
            "ename": 109,
            "cname": "妲己",
            "title": "魅力之狐",
            "pay_type": 11,
            "new_type": 0,
            "hero_type": 2,
            "skin_name": "魅惑之狐|女仆咖啡|魅力维加斯|仙境爱丽丝|少女阿狸|热情桑巴"
        }
    }
    

    根据上述,可以推测出英雄图片地址:

    # [http://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/(英雄编号)/(英雄编号)-bigskin-(第几个皮肤).jpg](http://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/137/137-bigskin-1.jpg)
    
    
    https//game.gtimg.cn/images/yxzj/img201606/heroimg + ename + ename + .jpg
    
    

    3、编写代码

    封装请求方法、

    # 封装请求方法
      def send_get(self, url):
          try:
              response = requests.get(url, headers=self.headers)
              # 断言测试
              assert response.status_code == 200, '{}请求失败'.format(url)
              return response
          except Exception as e:
              print(e)
              return None
    

    获得首页英雄列表、

    # 获得所有的英雄列表
       def hero_list(self):
           # 获取英雄列表
           response = self.send_get(self.hero_list_url)
           if response:
               hero_list_text = response.text  # 获得响应数据
               hero_list_dict = json.loads(hero_list_text)  # 响应数据封装json
               self.process_heroes(**hero_list_dict)  # 调用处理方法
           else:
               print('英雄列表为空,请检查获取URL:{}'.format(self.hero_list_url))
    

    处理英雄列表数据、

    # 对英雄列表进行处理
       def process_heroes(self, **hero_list_dict):
           for hero in hero_list_dict['hero']:
               hero_info_url = self.hero_url.format(hero['heroId'])
               resp = self.send_get(hero_info_url)
               if resp:
                   hero_info_dict = json.loads(resp.text)
                   self.process_hero(**hero_info_dict)
               else:
                   print('获取英雄:{}失败,请检查获取URL:{}'.format(hero['name'], self.hero_list_url))
    

    单个英雄数据获取

    # 单个英雄处理
       def process_hero(self, **hero_info_dict):
           hero = hero_info_dict['hero']
           skins = hero_info_dict['skins']
           for skin in skins:
               if skin['mainImg']:
                   skin_content = self.send_get(skin['mainImg']).content
                   hero_image_name = '{}.jpg'.format(skin['name'])
                   hero_image_dir = os.path.join(self.base_path, hero['name'] + hero['title'])
                   self.save_image(hero_image_dir, hero_image_name, skin_content)
           print('hero:{},skins:{}张,处理完成'.format(hero['name'], len(skins)))
           time.sleep(1)
    

    图片保存、

       def save_image(self,image_dir, image_name, image_content):
           if not os.path.exists(image_dir):
               os.makedirs(image_dir)
           try:
               hero_image_path = os.path.join(image_dir, re.sub(r'[/|?]', '', image_name))
               with open(hero_image_path, 'wb') as image:
                   image.write(image_content)
           except Exception as e:
               print('{}保存失败,错误原因:{}'.format(hero_image_path, e))
    

    4、爬虫执行

    if __name__ == '__main__':
        LOLHeroSpider().hero_list()
    

    5、完整代码

    import requests
    import random
    import time
    import re
    import json
    import os
    
    # url = https://lol.qq.com/data/info-heros.shtml
    # url = 'https://lol.qq.com/data/info-heros.shtml'
    
    # 需要设置USER_AGENT,假装自己是浏览器访问网页
    user_agent_list = [
     'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
     'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',
     'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
     'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
     'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',
     'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
     'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
     'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
     'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11'
     ]
    USER_AGENT = random.choice(user_agent_list)
    header = {
     'User-Agent':USER_AGENT
    }
    
    class LOLHeroSpider():
       def __init__(self):
           self.header = header
           self.hero_list_url = 'https://game.gtimg.cn/images/lol/act/img/js/heroList/hero_list.js'
           self.hero_url = 'https://game.gtimg.cn/images/lol/act/img/js/hero/{}.js'
           self.base_path = os.path.join('d:' + os.path.sep, '英雄联盟')
       # 封装请求方法
       def send_get(self, url):
           try:
               response = requests.get(url, headers=self.header)
               # 断言测试
               assert response.status_code == 200, '{}请求失败'.format(url)
               return response
           except Exception as e:
               print(e)
               return None
    
       # 获得所有的英雄列表
       def hero_list(self):
           # 获取英雄列表
           response = self.send_get(self.hero_list_url)
           if response:
               hero_list_text = response.text  # 获得响应数据
               hero_list_dict = json.loads(hero_list_text)  # 响应数据封装json
               self.process_heroes(**hero_list_dict)  # 调用处理方法
           else:
               print('英雄列表为空,请检查获取URL:{}'.format(self.hero_list_url))
    
       # 对英雄列表进行处理
       def process_heroes(self, **hero_list_dict):
           for hero in hero_list_dict['hero']:
               hero_info_url = self.hero_url.format(hero['heroId'])
               resp = self.send_get(hero_info_url)
               if resp:
                   hero_info_dict = json.loads(resp.text)
                   self.process_hero(**hero_info_dict)
               else:
                   print('获取英雄:{}失败,请检查获取URL:{}'.format(hero['name'], self.hero_list_url))
       # 单个英雄处理
       def process_hero(self, **hero_info_dict):
           hero = hero_info_dict['hero']
           skins = hero_info_dict['skins']
           for skin in skins:
               if skin['mainImg']:
                   skin_content = self.send_get(skin['mainImg']).content
                   hero_image_name = '{}.jpg'.format(skin['name'])
                   hero_image_dir = os.path.join(self.base_path, hero['name'] + hero['title'])
                   self.save_image(hero_image_dir, hero_image_name, skin_content)
           print('hero:{},skins:{}张,处理完成'.format(hero['name'], len(skins)))
           time.sleep(1)
       # 图片保存
       def save_image(self,image_dir, image_name, image_content):
           if not os.path.exists(image_dir):
               os.makedirs(image_dir)
           try:
               hero_image_path = os.path.join(image_dir, re.sub(r'[/|?]', '', image_name))
               with open(hero_image_path, 'wb') as image:
                   image.write(image_content)
           except Exception as e:
               print('{}保存失败,错误原因:{}'.format(hero_image_path, e))
    
    if __name__ == '__main__':
        LOLHeroSpider().hero_list()
    
    

    相关文章

      网友评论

          本文标题:python爬虫爬取王者荣耀英雄列表图片

          本文链接:https://www.haomeiwen.com/subject/qwnojltx.html