1、抓取本地网页解析其中的图片、标题、价格、星级和浏览量

经过查看和分析，每一项都是由一个div包裹

<div class="col-sm-4 col-lg-4 col-md-4">
  <div class="thumbnail">
    <img src="img/pic_0000_073a9256d9624c92a05dc680fc28865f.jpg" alt="">
    <div class="caption">
      <h4 class="pull-right">$24.99</h4>
      <h4>
        <a href="#">EarPod</a></h4>
      <p>See more snippets like this online store item at web store</p>
    </div>
    <div class="ratings">
      <p class="pull-right">65 reviews</p>
      <p>
        <span class="glyphicon glyphicon-star"></span>
        <span class="glyphicon glyphicon-star"></span>
        <span class="glyphicon glyphicon-star"></span>
        <span class="glyphicon glyphicon-star"></span>
        <span class="glyphicon glyphicon-star"></span>
      </p>
    </div>
  </div>
</div>

抓取数据的Python代码#

from bs4 import BeautifulSoup

path = r'G:/1_2_homework_required/index.html'
with open(path,'r') as wb_data:
    soup = BeautifulSoup(wb_data,'lxml')
    imgs = soup.select('div.col-sm-4 > div.thumbnail > img')
    titles = soup.select('div.col-sm-4 > div.thumbnail > div.caption > h4:nth-of-type(2) > a')
    prices = soup.select('div.col-sm-4 > div.thumbnail > div.caption > h4:nth-of-type(1)')
    stars = soup.select('div.col-sm-4 > div.thumbnail > div.ratings > p:nth-of-type(2)')
    views = soup.select('div.col-sm-4 > div.thumbnail > div.ratings > p.pull-right')
    for img,title,price,star,view in zip(imgs,titles,prices,stars,views):
        data = {
            'title' : title.get_text(),
            'img' : img.get('src'),
            'price' : price.get_text(),
            'star' : len(star.find_all('span',class_='glyphicon glyphicon-star')),
            'view' : view.get_text()
        }
        print(data)

这题的难点在于星星数的抓取， 观察发现,每一个星星会有一次
<span class="glyphicon glyphicon-star"></span>
所以统计有多少次,就知道有多少个星星了;

使用find_all 统计有几处是星星的样式,第一个参数定位标签名,第二个参数定位css 样式由于find_all()返回的结果是列表,我们再使用len()方法去计算列表中的元素个数

2、抓取小猪短租网的列表页和详情页数据

1. 列表页的抓取

def item_link_list(page):
    for i in range(1,page+1):
        ti = random.randrange(1,4)
        time.sleep(ti)
        url = 'http://sh.xiaozhu.com/search-duanzufang-p{}-0/'.format(i)
        print(url)
        wb_data = requests.get(url)
        soup = BeautifulSoup(wb_data.text,'lxml')
        urls = imgs = soup.select('ul.pic_list.clearfix > li > a')
        prices = soup.select('div.result_btm_con.lodgeunitname > span > i')
        titles = soup.select('div.result_btm_con.lodgeunitname > div.result_intro > a > span')
        for title,url,price,img in zip(titles,urls,prices,imgs):
            da = {
                'title' : title.get_text(),
                'url' : url.get('href'),
                'price' : price.get_text(),
            }
            print(da)

结果为

1.jpg

2.根据url抓取详情页数据

def returnSex(sexclass):
    if sexclass == 'member_ico':
        return '男'
    if sexclass == 'member_ico1':
        return '女'

def item_detail(url):
    wd_data = requests.get(url)
    soup = BeautifulSoup(wd_data.text,'lxml')
    title = soup.select('div.pho_info > h4 > em')[0].get_text()
    address = soup.select('div.pho_info > p > span.pr5')[0].get_text()
    price = soup.select('div.day_l > span')[0].get_text()
    img = soup.select('#curBigImage')[0].get('src')
    host_img = soup.select('div.member_pic > a > img')[0].get('src')
    host_sex = soup.select('div.member_pic > div')[0].get('class')[0]
    host_name = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')[0].get_text()
    data = {
        'title': title,
        'address': address.strip().lstrip().rstrip(','),
        'price': price,
        'img': img,
        'host_img': host_img,
        'ownersex': returnSex(host_sex),
        'ownername': host_name
    }
    print(data)

结果为#

1.jpg

3.总结

这次的作业基本无太大的问题，最难的是判断房东的性别，需要通过类名来判断

3.抓取Weheartit前20页数据

根据传入页数抓取1到页数的所有图片链接

def get_list_imgs(page):
    for index in range(1,page+1):
        time.sleep(3)
        url = 'http://weheartit.com/recent?scrolling=true&page={0}'.format(index)
        wb_data = requests.get(url)
        soup = BeautifulSoup(wb_data.text,'lxml')
        imgs = soup.select('img.entry_thumbnail')
        for img in imgs:
            img = img.get('src')
            down_url.append(img)
        print(url)

根据url下载图片

def down_img(urls):
    for item in urls:
        time.sleep(1)
        name = item[-24:-15]
        urlsd = path + name + '.jpg'
        print(urlsd)
        urllib.request.urlretrieve(item, urlsd)
        print('Done')

4.抓取58同城的列表页和详情页

首先是根据用户的类别和要抓取的页数来获得物品的链接地址

def get_item_list(who_sells,page):
    links = []
    for index in range(1,page+1):
        url = 'http://bj.58.com/pbdn/{}/pn{}'.format(str(who_sells),index)
        wb_data = requests.get(url)
        soup = BeautifulSoup(wb_data.text,'lxml')
        for item_url in soup.select('td.t > a.t'):
            if  'bj.58.com' in str(item_url):
                link = item_url.get('href').split('?')[0]
                links.append(link)
            else:
                pass
    return links

再根据链接地址抓取物品的信息

# 获得物品成色
def get_quality(qu):
    if qu == '-':
        return '不明'
    else:
        return qu

def get_detail_item(url):
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text,'lxml')
    category = soup.select('div.breadCrumb.f12 > span:nth-of-type(3) > a')[0].get_text()
    title = soup.select('div.col_sub.mainTitle > h1')[0].get_text()
    date = soup.select('li.time')[0].get_text()
    price = soup.select('span.price.c_f50')[0].get_text()
    quality = soup.select('div.su_con > span')[1].get_text().strip().lstrip().rstrip(',')
    quality = get_quality(quality)
    area = list(soup.select('.c_25d')[0].stripped_strings) if soup.find_all('span','c_25d') else None
    date = {
        'category' : category,
        'title' : title,
        'date' : date,
        'price' : price,
        'quality' : quality,
        'area' : area
    }
    print(date)

通过这儿一周的练习，我了解了有关于爬虫的基本信息。也在课外爬了一些网页作为练习。深感python语言的精妙之处，希望在第二周的学习中更近一步