Python爬虫 --- Scrapy爬取IT桔子网

目标：

此次爬取主要是针对IT桔子网的事件信息模块，然后把爬取的数据存储到mysql数据库中。

目标分析：

通过浏览器浏览发现事件模块需要登录才能访问，因此我们需要先登录，抓取登录接口：

可以看到桔子网的登录接口是：https://www.itjuzi.com/api/authorizations，请求方式是post请求，数据的提交方式是payload，提交的数据格式是json(payload方式提交的数据一般都是json)，再看一下响应：

发现没有响应数据，其实是有响应数据的，只是F12调试看不到，我们可以用postman来查看响应体：

可以发现响应体是json数据，我们先把它放到一边，我们再来分析事件模块，通过F12抓包调试发现事件模块的数据其实是一个ajax请求得到的：

ajax请求得到的是json数据，我们再看看headers：



可以发现headers里有一个Authorization参数，参数的值恰好是我们登录时登录接口返回的json数据的token部分，所以这个参数很有可能是判断我们是否登录的凭证，我们可以用postman模拟请求一下：

通过postman的模拟请求发现如我们所料，我们只要在请求头里加上这个参数我们就可以获得对应的数据了。
解决了如何获得数据的问题，再来分析一下如何翻页，通过对比第一页和第二页提交的json数据可以发现几个关键参数，page、pagetotal、per_page分别代表当前请求页、记录总数、每页显示条数，因此根据pagetotal和per_page我们可以算出总的页数，到此整个项目分析结束，可以开始写程序了。

scrapy代码的编写

1.创建scrapy项目和爬虫：

E:\>scrapy startproject ITjuzi
E:\>scrapy genspider juzi itjuzi.com

2.编写items.py:

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class ItjuziItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    invse_des = scrapy.Field()
    invse_title = scrapy.Field()
    money = scrapy.Field()
    name = scrapy.Field()
    prov = scrapy.Field()
    round = scrapy.Field()
    invse_time = scrapy.Field()
    city = scrapy.Field()
    com_registered_name = scrapy.Field()
    com_scope = scrapy.Field()
    invse_company = scrapy.Field()

3.编写Spider:

import scrapy
from itjuzi.settings import JUZI_PWD, JUZI_USER
import json


class JuziSpider(scrapy.Spider):
    name = 'juzi'
    allowed_domains = ['itjuzi.com']

    def start_requests(self):
        """
        先登录桔子网
        """
        url = "https://www.itjuzi.com/api/authorizations"
        payload = {"account": JUZI_USER, "password": JUZI_PWD}
        # 提交json数据不能用scrapy.FormRequest，需要使用scrapy.Request，然后需要method、headers参数
        yield scrapy.Request(url=url,
                             method="POST",
                             body=json.dumps(payload),
                             headers={'Content-Type': 'application/json'},
                             callback=self.parse
                             )

    def parse(self, response):
        # 获取Authorization参数的值
        token = json.loads(response.text)
        url = "https://www.itjuzi.com/api/investevents"
        payload = {
                    "pagetotal": 0, "total": 0, "per_page": 20, "page": 1, "type": 1, "scope": "", "sub_scope": "",
                    "round": [], "valuation": [], "valuations": "", "ipo_platform": "", "equity_ratio": [""],
                    "status": "", "prov": "", "city": [], "time": [], "selected": "", "location": "", "currency": [],
                    "keyword": ""
                }
        yield scrapy.Request(url=url,
                             method="POST",
                             body=json.dumps(payload),
                             meta={'token': token},
                             # 把Authorization参数放到headers中
                             headers={'Content-Type': 'application/json', 'Authorization': token['data']['token']},
                             callback=self.parse_info
                             )
    def parse_info(self, response):
        # 获取传递的Authorization参数的值
        token = response.meta["token"]
        # 获取总记录数
        total = json.loads(response.text)["data"]["page"]["total"]
        # 因为每页20条数据，所以可以算出一共有多少页
        if type(int(total)/20) is not int:
            page = int(int(total)/20)+1
        else:
            page = int(total)/20

        url = "https://www.itjuzi.com/api/investevents"
        for i in range(1,page+1):
            payload = {
                "pagetotal": total, "total": 0, "per_page": 20, "page":i  , "type": 1, "scope": "", "sub_scope": "",
                "round": [], "valuation": [], "valuations": "", "ipo_platform": "", "equity_ratio": [""],
                "status": "", "prov": "", "city": [], "time": [], "selected": "", "location": "", "currency": [],
                "keyword": ""
            }
            yield scrapy.Request(url=url,
                                 method="POST",
                                 body=json.dumps(payload),
                                 headers={'Content-Type': 'application/json', 'Authorization': token['data']['token']},
                                 callback=self.parse_detail
                                 )
       

    def parse_detail(self, response):
        infos = json.loads(response.text)["data"]["data"]      
        for i in infos:
            item = ItjuziItem()
            item["invse_des"] = i["invse_des"]
            item["com_des"] = i["com_des"]
            item["invse_title"] = i["invse_title"]
            item["money"] = i["money"]
            item["com_name"] = i["name"]
            item["prov"] = i["prov"]
            item["round"] = i["round"]
            item["invse_time"] = str(i["year"])+"-"+str(i["year"])+"-"+str(i["day"])
            item["city"] = i["city"]
            item["com_registered_name"] = i["com_registered_name"]
            item["com_scope"] = i["com_scope"]
            invse_company = []
            for j in i["investor"]:
                invse_company.append(j["name"])
            item["invse_company"] = ",".join(invse_company)
            yield item

4.编写PIPELINE：

from itjuzi.settings import DATABASE_DB, DATABASE_HOST, DATABASE_PORT, DATABASE_PWD, DATABASE_USER
import pymysql

class ItjuziPipeline(object):
    def __init__(self):
        host = DATABASE_HOST
        port = DATABASE_PORT
        user = DATABASE_USER
        passwd = DATABASE_PWD
        db = DATABASE_DB
        try:
            self.conn = pymysql.Connect(host=host, port=port, user=user, passwd=passwd, db=db, charset='utf8')
        except Exception as e:
            print("连接数据库出错,错误原因%s"%e)
        self.cur = self.conn.cursor()

    def process_item(self, item, spider):
        params = [item['com_name'], item['com_registered_name'], item['com_des'], item['com_scope'],
                  item['prov'], item['city'], item['round'], item['money'], item['invse_company'],item['invse_des'],item['invse_time'],item['invse_title']]
        try:
            com = self.cur.execute(
                'insert into juzi(com_name, com_registered_name, com_des, com_scope, prov, city, round, money, invse_company, invse_des, invse_time, invse_title)values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',params)
            self.conn.commit()
        except Exception as e:
            print("插入数据出错,错误原因%s" % e)
        return item

    def close_spider(self, spider):
        self.cur.close()
        self.conn.close()

5.编写settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for itjuzi project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'itjuzi'

SPIDER_MODULES = ['itjuzi.spiders']
NEWSPIDER_MODULE = 'itjuzi.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'itjuzi (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 0.25
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'itjuzi.middlewares.ItjuziSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   # 'itjuzi.middlewares.ItjuziDownloaderMiddleware': 543,
   'itjuzi.middlewares.RandomUserAgent': 102,
   'itjuzi.middlewares.RandomProxy': 103,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'itjuzi.pipelines.ItjuziPipeline': 100,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
JUZI_USER = "1871111111111"
JUZI_PWD = "123456789"

DATABASE_HOST = '数据库ip'
DATABASE_PORT = 3306
DATABASE_USER = '数据库用户名'
DATABASE_PWD = '数据库密码'
DATABASE_DB = '数据表'

USER_AGENTS = [
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.29 Safari/537.36"
    ]

PROXIES = [
    {'ip_port': '代理ip:代理IP端口', 'user_passwd': '代理ip用户名:代理ip密码'},
    {'ip_port': '代理ip:代理IP端口', 'user_passwd': '代理ip用户名:代理ip密码'},
    {'ip_port': '代理ip:代理IP端口', 'user_passwd': '代理ip用户名:代理ip密码'},
]

6.让项目跑起来：

E:\>scrapy crawl juzi

7.结果展示:

PS：详情信息这里没有爬取，详情信息主要是根据上面列表页返回的json数据中每个公司的id来爬取，详情页的数据可以不用登录就能拿到如：https://www.itjuzi.com/api/investevents/10262327，https://www.itjuzi.com/api/get_investevent_down/10262327，还有最重要的一点是如果你的账号不是vip会员的话只能爬取前3页数据这个有点坑，其他的信息模块也是一样的分析方法，需要的可以自己去分析爬取。