02 ——管道模块(Pipeline.py)

管道模块

# -*- coding: utf-8 -*-

# Define your item pipelines here

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymysql




class MyspiderPipeline(object):

    def __int__(self):

        print('my pipelines is ready....__init__ invoking....')

    def open_spider(self,spider):

        '''

        建立数据库连接

        :param spider:

        :return:

        '''

        self.my_conn = pymysql.connect(

            host ="localhost",
            port =3306,
            database ="jobs",
            user ='root',
            password ='123',
            charset ='utf8'

        )

        #获取游标对象

        self.my_cursor =self.my_conn.cursor()

    def process_item(self,item,spider):

        print('----------------------process_item is invoking----------------------')

        # print item['name']

        # print item['salary']

        # print item['company']

        # print item['day']

        # print item['experience']

        # print item['area']

        # print item['number']

        # print item['nature']

        # print item['education']

        # print item['description']

        insert_sql ="insert into wuxi(name, salary, company, day,experience,area,number,nature,education,description) values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"

        self.my_cursor.execute(insert_sql,[item['name'],item['salary'],item['company'],item['day'],item['experience'],item['area'],item['number'],item['nature'],item['education'],item['description']])

    def close_spider(self, spider):

        #提交MySQL语句

        self.my_conn.commit()

        #关闭游标对象与数据库连接

        self.my_cursor.close()

        self.my_conn.close()

作者：陳CHEN陈
链接：https://www.jianshu.com/p/0bafcf4aaae7
來源：简书
简书著作权归作者所有，任何形式的转载都请联系作者获得授权并注明出处。