爬虫模块

# -*- coding: utf-8 -*-

'''深度爬虫：采集工作信息及分页深度采集'''

# 引入爬虫模块

from scrapy.spiderimport CrawlSpider

# 引入链接提取模块

from scrapy.linkextractorsimport LinkExtractor

# 引入规则模块

from scrapy.spiderimport Rule

#导入item模块

from ..itemsimport MyspiderItem

class ZhilianJobSpider(CrawlSpider):

name ="zhilian1"

    allowed_domains = ["zhaopin.com"]

start_urls = ["http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E6%97%A0%E9%94%A1&kw=python&isadv=0&isfilter=1&p=1&pd=30",]

# 提取信息1：提取工作链接

    link_job = LinkExtractor(restrict_xpaths=("//td[@class='zwmc']/div/a[1]"))

# 提起信息2：提取下一页链接

    link_next_page = LinkExtractor(restrict_xpaths=("//a[@class='next-page']"))

# 定义规则和指定的处理函数

    # 注意：千万千万千万不要重写parse函数

    rules = [

Rule(link_job,callback="parse_job"),

Rule(link_next_page,follow=True),

]

def parse_job(self, response):

'''response中包含的就是详细工作信息'''

        print("---------------------job_name---------------------")

item =MyspiderItem()

#职位

        job_name = response.xpath("//div[@class='top-fixed-box'][1]/div[@class='fixed-inner-box']/div[@class='inner-left fl']/h1/text()").extract()[0]

if len(job_name) >0:

item['name'] = job_name

# 薪资  .strip去图片

        job_salary = response.xpath("//ul[@class='terminal-ul clearfix']/li[1]/strong/text()").extract()[0].strip()

if len(job_salary) >0:

item['salary'] = job_salary

#公司

        job_company = response.xpath("//div[@class='top-fixed-box'][1]/div[@class='fixed-inner-box']/div[@class='inner-left fl']/h2/a/text()").extract()[0]

if len(job_company) >0:

item['company'] = job_company

#招聘日期

        job_day = response.xpath("//strong/span[@id='span4freshdate']/text()").extract()[0]

if len(job_day) >0:

item['day'] = job_day

#工作经验

        job_experience= response.xpath("//ul[@class='terminal-ul clearfix']/li[5]/strong/text()").extract()[0]

if len(job_experience) >0:

item['experience'] = job_experience

#工作地点

        job_area = response.xpath("//ul[@class='terminal-ul clearfix']/li[2]/strong/a/text()").extract()[0]

if len(job_area) >0:

item['area'] = job_area

#招聘人数

        job_number = response.xpath("//ul[@class='terminal-ul clearfix']/li[7]/strong/text()").extract()[0]

if len(job_number) >0:

item['number'] = job_number

#工作性质

        job_nature = response.xpath("//ul[@class='terminal-ul clearfix']/li[4]/strong/text()").extract()[0]

if len(job_nature) >0:

item['nature'] = job_nature

#学历

        education_background = response.xpath("//ul[@class='terminal-ul clearfix']/li[6]/strong/text()").extract()[0]

if len(education_background) >0 :

item['education'] = education_background

#职位描述

        # job_description = response.xpath("//div[@class='terminalpage-main clearfix']/div[@class='tab-cont-box']/div[@class='tab-inner-cont'][1]")

# if len(job_description) > 0:

#    item['description'] = job_description.xpath("string(.)").extract()[0].strip()

        job_description = response.xpath("//div[@class='terminalpage-main clearfix']/div[@class='tab-cont-box']/div[@class='tab-inner-cont'][1]")

if len(job_description) >0:

item['description'] = job_description.xpath("string(.)").extract()[0].strip()

# 交给 pipelines管道模块进行处理

        yield item

# print (job_name)

# print (job_salary)

# print (job_company)

# print (job_day)

# print (job_experience)

# print (job_area)

# print (job_number)

# print (job_nature)

# print (education_background)

# print (item['description'])