自动爬取58的二手商品信息

实现58二手商品的标题、价格、类别、浏览量、区域等信息的自动爬取

新增：导出数据到excel表格内

# -*- coding: utf-8 -*-
#!/use/bin/env python

import requests,time
import xlsxwriter
from bs4 import BeautifulSoup

headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',
'Cookie':'myfeet_tooltip=end; bj58_id58s="VVk4K1FxVk80dUI5MDk5Nw=="; id58=c5/ns1hqLtNLk1KnJbZHAg==; als=0; 58home=sh; myfeet_tooltip=end; final_history=26088204291258; bj58_new_session=1; bj58_init_refer=""; bj58_new_uv=5; sessionid=ca5902c9-e44c-44ea-a087-bbcceaf1074b; 58tj_uuid=49f57f42-3481-42f2-9640-cb8d85a4af80; new_session=0; new_uv=6; utm_source=; spm=; init_refer='

}   #防爬

workbook = xlsxwriter.Workbook('58data_1.xlsx')  #新建表格
worksheet= workbook.add_worksheet('工作表1')    #在表格内新建工作簿
row = 0    #从第0行开始
col = 0    #从第0列开始

def main_link(url):  #从列表页中获取非广告商品的详情页链接
    wb_data = requests.get(url,headers=headers)

    Soup = BeautifulSoup(wb_data.text,'lxml')
    page_links = Soup.select('div.infocon > table > tbody > tr > td.t > a')
    for page_link in page_links:   #取出每一个商品的详情页链接，并将其设为函数page_urls的参数
        page_urls(page_link.get('href'))


def page_urls(url):   #从商品详情页获取所需要的信息
    wb_data = requests.get(url,headers=headers)
    Soup = BeautifulSoup(wb_data.text,'lxml')
    categorys = Soup.select('#nav > div > span:nth-of-type(4) > a')   #商品类别
    titles = Soup.select('div.info_lubotu.clearfix > div.box_left_top '
                         '> h1')   #商品标题
    prises = Soup.select('div.info_lubotu.clearfix > div.info_massege.left > '
                         'div.price_li > span.price_now > i')  #价格
    areas = Soup.select('div.info_lubotu.clearfix > div.info_massege.left >'
                        ' div.palce_li > span > i')   #区域
    pageviews = Soup.select('div.info_lubotu.clearfix > div.box_left_top > '
                            'p > span.look_time')   #页面浏览量

    for category, title, prise, area, pageview in zip(categorys,  #将取出的信息放入字典data
                                                      titles, prises, areas, pageviews):
        data = {                       #剔除多余信息
            'category': category.get_text(),
            'title': title.get_text(),
            'prise': prise.get_text(),
            'area': area.get_text(),
            'pageview': pageview.get_text()
        }
        global row   #引入变量row和col
        global col
        worksheet.write(row,col,data['category'])   #表格内写入类目
        worksheet.write(row,col+1,data['title'])    #表格内写入标题
        worksheet.write(row,col+2,data['prise'])    #表格内写入价格
        worksheet.write(row,col+3,data['area'])     #表格内写入区域
        worksheet.write(row,col+4,data['pageview'])     #表格内写入PV
        row += 1   #每次循环+1行
        print('Down')
        time.sleep(1)            #一秒爬取一次的保护机制

#取到每一页列表的链接
main_urls = ['http://bj.58.com/pbdn/0/pn{}/?PGTID=0d305a36-0000-1980-da0a-4541effe84e2&ClickID=2'.format(number)    for number in range(1, 21)]     #第1-20页的列表页所有链接
for single_url in main_urls:   #将20页的列表页链接每一个都取出来赋值给sing_url
    main_link(single_url)   #单个列表页的链接设为函数main_link的参数

workbook.close()