Scrapy --异步插入数据库

• 在settings中激活管道、设置数据库参数
• 导入twisted的异步插入数据库的模块

from twisted.enterprise import adbapi
from chufang.items import ChufangUserItem, ChufangMenuItem, ChufangTagsItem
import scrapy, pymysql


class ChufangPipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_crawler(cls, cralwer):
        db_parmars = {
            'host': cralwer.settings['MYSQL_HOST'],
            'user': cralwer.settings['MYSQL_USER'],
            'passwd': cralwer.settings['MYSQL_PWD'],
            'db': cralwer.settings['MYSQL_DB'],
            'port': cralwer.settings['MYSQL_PORT'],
            'charset': cralwer.settings['MYSQL_CHARSET']
        }
        dbpool = adbapi.ConnectionPool('pymysql', **db_parmars)
        return cls(dbpool)

    def process_item(self, item, spider):
        query = self.dbpool.runInteraction(
            self.insert_data_to_mysql,
            item
        )
        query.addErrback(
            self.insert_err,
            item
        )

        return item

    def insert_data_to_mysql(self, cursor, item):
        data_dict = dict(item)
        sql, data = item.get_insert_sql_data(data_dict)
        cursor.execute(sql, data)

    def insert_err(self, failure, item):
        print(failure, '失败', item)