python爬取小猪短租网信息

用python的requests，bs4等库爬取了小猪短租网的杭州租房信息。
注意

1. 有些房子居然没有写地址，这房还怎么租。
2. python的UnboundLocalError： local variable 'xxx' referenced b
   参考 博客和博客
   其他一些应注意的点写在注释里

这里总共获取了300多条信息，下一步应该考虑把这些信息存储在mongdb里面，并且采用多进程在加速爬取速度。

import requests,time
from bs4 import BeautifulSoup

start_urls = ['http://hz.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i)) for i in range(1,14)] #摘取13页的信息
count = 0
urls = []

def get_info(url):
    try:
        r = requests.get(url,timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
    except:
        print('error')
    soup = BeautifulSoup(r.text,'lxml')
    title = soup.select('div.pho_info > h4')
    address = soup.select('span.pr5')
    price = soup.select('div#pricePart > div.day_l > span')
    name = soup.select('a.lorder_name')
    sex = 'boy' if soup.findAll('span',class_='member_boy_ico') else 'girl' #简单的判断男女性别
    houseImage = soup.select('img#curBigImage')
    menberImage = soup.select('div.member_pic > a > img')
    info ={
        'title':title[0].get_text().strip(),
        'address':address[0].get_text().strip() if address  else '杭州',  #为防止地址不存在，特地加以判断
        'price':price[0].get_text(),
        'name':name[0].get_text().strip(),
        'sex':sex,
        'houseImage':houseImage[0].get('src'),
        'menberImage':menberImage[0].get('src')
    }
    global count  #全局变量在局部变量使用时需要指定global
    print(count)
    count = count + 1
    print(info)

def get_urls(url,urls):
    r = requests.get(url,timeout = 30)
    soup = BeautifulSoup(r.text,'lxml')
    links = soup.select('a.resule_img_a')
    for link in links:
        urls.append(link.get('href'))


def main():
    for url in start_urls:  # 获取所有url
        get_urls(url,urls)
    for url in urls:   #对每个url进行信息获取
        get_info(url)

if __name__ == '__main__':
    main()

更新：
把数据存入数据库mongdb

import requests,time,pymongo
from bs4 import BeautifulSoup

start_urls = ['http://hz.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i)) for i in range(1,14)] #摘取13页的信息
count = 0
urls = []

client  = pymongo.MongoClient('localhost',27017)    #打开数据库客户端
xiaozhu = client.xiaozhu
hz = xiaozhu.hz

def get_info(url):
    try:
        r = requests.get(url,timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
    except:
        print('error')
    soup = BeautifulSoup(r.text,'lxml')
    title = soup.select('div.pho_info > h4')
    address = soup.select('span.pr5')
    price = soup.select('div#pricePart > div.day_l > span')
    name = soup.select('a.lorder_name')
    sex = 'boy' if soup.findAll('span',class_='member_boy_ico') else 'girl' #简单的判断男女性别
    houseImage = soup.select('img#curBigImage')
    menberImage = soup.select('div.member_pic > a > img')
    info ={
        'title':title[0].get_text().strip(),
        'address':address[0].get_text().strip() if address  else '杭州',  #为防止地址不存在，特地加以判断
        'price':price[0].get_text(),
        'name':name[0].get_text().strip(),
        'sex':sex,
        'houseImage':houseImage[0].get('src'),
        'menberImage':menberImage[0].get('src')
    }
    global count  #全局变量在局部变量使用时需要指定global
    #print(count)
    count = count + 1
    #print(info)
    hz.insert_one(info)    #插入数据库
    print('\r当前进度为:{:.2f}%'.format(((count/299)*100)),end="")    #在命令行运行可以显示动态窗口,end=""是关键

def get_urls(url,urls):
    r = requests.get(url,timeout = 30)
    soup = BeautifulSoup(r.text,'lxml')
    links = soup.select('a.resule_img_a')
    for link in links:
        urls.append(link.get('href'))


def main():
    for url in start_urls:  # 获取所有url
        get_urls(url,urls)
    for url in urls:   #对每个url进行信息获取
        get_info(url)

if __name__ == '__main__':
    main()