四. 爬虫实战 - 小猪短租网广州信息

爬取网址：http://gz.xiaozhu.com
网页共有13页，规律为：

http://gz.xiaozhu.com/search-duanzufang-p1-0/
http://gz.xiaozhu.com/search-duanzufang-p2-0/
...
http://gz.xiaozhu.com/search-duanzufang-p13-0/

爬取内容：进入每个租房的详细页面，爬取标题、地址、价格、房东名称、房东名称和房东头像的链接。

• 代码：

import requests
from bs4 import BeautifulSoup
import time

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3294.6 Safari/537.36'}

url_list = ["http://gz.xiaozhu.com/search-duanzufang-p{}-0/".format(i) for i in range(1,14)]
#print(url_list)

i = 0
for url in url_list:   
    i += 1
    r = requests.get(url, headers=headers)
    print(r.status_code)
    soup = BeautifulSoup(r.text,"lxml")
    detail_urls = soup.select("#page_list > ul > li > a")
    
    j = 0
    for detail_url in detail_urls:
        j += 1
        detail_url = detail_url.get("href")
        #print(detail_url)
        
        r_detail = requests.get(detail_url,headers = headers)
        soup1 = BeautifulSoup(r_detail.text,"lxml")
        title = soup1.select("div.pho_info h4")[0].text
        address = soup1.select("div.pho_info p span")[0].text.strip()
        price = soup1.select("div.day_l span")[0].text
        host_name = soup1.select("div.w_240 h6 a")[0].text
        sex_info = soup1.select("div.w_240 h6 span")[0].get("class")
        if sex_info == ["member_girl_ico"]:
            sex = "女"
        if sex_info == ["member_boy_ico"]:
            sex = "男"
        img_url = soup1.select("div.member_pic > a > img")[0].get("src")
        
        print(i,"_",j,":",title,address,price,host_name,sex,img_url)
        time.sleep(1)

• 上述代码层次不够鲜明，改成下述框架：

def get_url(url):
    pass

def get_info(url):
    male_or_female()
    pass

def male_or_female():
    pass

if "__name__" == "__main__":
    pass

• 改框架后的代码：

import requests
from bs4 import BeautifulSoup
import time

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3294.6 Safari/537.36'}

#获取详细页面的URL
def get_url(url):
    r = requests.get(url, headers = headers)
    print(r.status_code)
    soup = BeautifulSoup(r.text,"lxml")
    detail_urls = soup.select("#page_list > ul > li > a")
    return detail_urls

#获取网页信息
def get_info(detail_url):
    detail_url = detail_url.get("href")
        
    r_detail = requests.get(detail_url,headers = headers)
    soup1 = BeautifulSoup(r_detail.text,"lxml")
    title = soup1.select("div.pho_info h4")[0].text
    address = soup1.select("div.pho_info p span")[0].text.strip()
    price = soup1.select("div.day_l span")[0].text
    host_name = soup1.select("div.w_240 h6 a")[0].text
    sex_info = soup1.select("div.w_240 h6 span")[0].get("class")
    sex = male_or_female(sex_info)
    img_url = soup1.select("div.member_pic > a > img")[0].get("src")
    #print(title,address,price,host_name,sex,img_url)
    print(i,"_",j,":",title,address,price,host_name,sex,img_url)

#判断用户性别
def male_or_female(sex_info):    
    if sex_info == ["member_girl_ico"]:
        sex = "女"
    if sex_info == ["member_boy_ico"]:
        sex = "男"
    return sex

#程序主入口
if __name__ == "__main__":
    url_list = ["http://gz.xiaozhu.com/search-duanzufang-p{}-0/".format(i) for i in range(1,14)]
    i = 0   #记录页数
    for url in url_list:
        i += 1
        detail_urls = get_url(url)

        j = 0   #记录页数中的编号
        for detail_url in detail_urls:
            j += 1
            get_info(detail_url)
            time.sleep(3)

• 使用xpath方法的代码实现：

import requests
from lxml import etree
import time

headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3294.6 Safari/537.36'}

#获取详细页面的URL
def get_url(url):
    r = requests.get(url, headers = headers)
    print(r.status_code)
    selector = etree.HTML(r.text)
    detail_urls = selector.xpath('//*[@id="page_list"]/ul/li/a')
    time.sleep(1)
    return detail_urls

#获取网页信息
def get_info(detail_url):
    detail_url = detail_url.get("href")
        
    r_detail = requests.get(detail_url,headers = headers)
    selector1 = etree.HTML(r_detail.text)
    
    title = selector1.xpath('//div[@class="pho_info"]/h4/em/text()')[0]
    address = selector1.xpath('//div[@class="pho_info"]/p/span/text()')[0].strip()
    price = selector1.xpath('//div[@class="day_l"]/span/text()')[0]
    host_name = selector1.xpath('//div[@class="w_240"]/h6/a/text()')[0]
    sex_info = selector1.xpath('//div[@class="w_240"]/h6/span/@class')[0]
    sex = male_or_female(sex_info)
    img_url = selector1.xpath('//div[@class="member_pic"]/a/img/@src')[0]
    
    print(i,"_",j,":",title,address,price,host_name,sex,img_url)

#判断用户性别
def male_or_female(sex_info):    
    if sex_info == "member_girl_ico":
        return "女"
    if sex_info == "member_boy_ico":
        return "男"

#程序主入口
if __name__ == "__main__":
    url_list = ["http://gz.xiaozhu.com/search-duanzufang-p{}-0/".format(i) for i in range(1,14)]
    i = 0   #记录页数
    for url in url_list:
        i += 1
        detail_urls = get_url(url)

        j = 0   #记录页数中的编号
        for detail_url in detail_urls:
            j += 1
            get_info(detail_url)
            time.sleep(3)

部分结果为：

1 _ 1 : 
琶洲&广州塔&财经大学客村赤岗地铁口整套房源
 广东省广州市海珠区新港中路456号纵横公寓 330 麦西西雪糕 女 https://image.xiaozhustatic1.com/21/8,0,56,9231,260,260,47556991.jpg
1 _ 2 : 
【逆旅·北欧】新房特惠！越秀公园纪念堂北京路
 广东省广州市越秀区解放北路861号 458 小羽和小果爸爸 男 https://image.xiaozhustatic1.com/21/12,0,53,22858,267,267,216342cc.jpg
...
1 _ 15 : 
a金融城&会展中心/地铁四，五号线&两房
 广东省广州市天河区车陂路骏景花园骏雅轩A2座 328 谢彩蝶 女 https://image.xiaozhustatic1.com/21/12,0,53,23389,329,329,672ca75e.jpg
...

当然，上述代码还爬不了大批量数据，因为短租网的反爬虫还是挺给力的，随时封IP。
这个课题先留着，以后解决。

最后，简单总结一下selector()和xpath()方法的一些不同点：