第一周作业

作业一：动手做网页

动手做网页

总结：了解了网页的结构，先写结构代码，再填充样式

作业二：爬取本地网页

from bs4 import BeautifulSoup

path='F:/百度云/Plan-for-combating-master/week1/1_2/1_2answer_of_homework/index.html'
with open(path,'r') as web_data:
   soup=BeautifulSoup(web_data.read(),'lxml')
   titles=soup.select('body > div > div > div.col-md-9 > div > div > div > div.caption > h4 > a')
   images=soup.select('body > div > div > div.col-md-9 > div > div > div > img')
   prices=soup.select('body > div > div > div.col-md-9 > div > div > div > div.caption > h4.pull-right')
   stars=soup.select('body > div > div > div.col-md-9 > div > div > div > div.ratings ')
   reviews=soup.select('body > div > div > div.col-md-9 > div > div > div > div.ratings > p.pull-right')
   #print(stars)
   for title,image,price,star,review in zip (titles,images,prices,stars,reviews):
      data={
          'title':title.get_text(),
          'image':image.get('src'),
          'price':price.get_text()[1:],
          'star':len(star.find_all('span','glyphicon glyphicon-star')),
          'review':review.get_text()[:-8],
      }
      print(data)

结果：

爬取结果

总结：了解了BeautifulSoup的基本用法，CSS Select的基本用法，用之前学过的[1:]截取需要的信息，结果是可显示的，但是不知道用法对不对，

作业三：爬取租房信息

作业代码：

from bs4 import BeautifulSoup
import requests,time,random

url='http://bj.xiaozhu.com/search-duanzufang-p1-0/'

def get_urls (url):
    wb_data = requests.get(url)
    soup=BeautifulSoup(wb_data.text,'lxml')
    urls=soup.select('#page_list > ul > li > a')
    url_list = []
    for url in urls:
        url_list.append(url.get('href'))
    return url_list

def get_member_sex (class_name):
    if class_name ==['member_boy_ico']:
        return  u'男'
    else:
        return  u'女'

def get_date (url):
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    title=soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
    address=soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5')
    price=soup.select('#pricePart > div.day_l > span')
    image=soup.select('#curBigImage')
    member_pic=soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
    member_name=soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
    member_sex=soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span')
    title_data=title[0].get_text()
    address_data=address[0].get_text()
    price_data=price[0].get_text()
    image_data=image[0].get('src')
    member_pic_data=member_pic[0].get('src')
    member_name_data=member_name[0].get_text()
    member_sex_data=get_member_sex(member_sex[0].get('class'))
    data={
        'title':title_data,
        'address':address_data,
        'price':price_data,
        'image':image_data,
        'member_pic':member_pic_data,
        'member_name': member_name_data,
        'member_sex': member_sex_data,
    }
    print(data)

def  spider (url):
    for i in get_urls(url):
        get_date(i)

#spider(url)

def spider2 (page=1):
    for j in range(1,page+1):
        url=('http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(j))
        for i in get_urls(url):
            get_date(i)
        t=random.randrange(1,5)
        print('爬完一页，休息{}秒---------------'.format(t))
        time.sleep(t)

spider2(2)

爬取结果：

爬取结果

总结：熟悉了requests的用法，member_boy_ico，get class之后变成一个列表，不知为何，所以判断的时候只能以列表去对比。

作业四：爬取58

from bs4 import BeautifulSoup
import requests,time,random

def  get_urls (url):
    web_data=requests.get(url)
    soup=BeautifulSoup(web_data.text,'lxml')
    urls=soup.select('td.t a.t')
    url_list = []
    for url in urls:
        url_list.append(url.get('href'))
    return url_list

def get_views(url):
    id = url.split('/')[-1].strip('x.shtml')
    headers = {
        'User-Agent':r'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36',
        'Cookie':r'id58=c5/ns1ct99sKkWWeFSQCAg==; city=bj; 58home=bj; ipcity=yiwu%7C%u4E49%u4E4C%7C0; als=0; myfeet_tooltip=end; bj58_id58s="NTZBZ1Mrd3JmSDdENzQ4NA=="; sessionid=021b1d13-b32e-407d-a76f-924ec040579e; bangbigtip2=1; 58tj_uuid=0ed4f4ba-f709-4c42-8972-77708fcfc553; new_session=0; new_uv=1; utm_source=; spm=; init_refer=; final_history={}; bj58_new_session=0; bj58_init_refer=""; bj58_new_uv=1'.format(str(id)),
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'Host':'jst1.58.com',
        'Referer':r'http://bj.58.com/pingbandiannao/{}x.shtml'.format(id)
    }
    api = 'http://jst1.58.com/counter?infoid={}'.format(id)
    js = requests.get(api,headers = headers)
    view = js.text.split('=')[-1]
    return view

def  get_info(url):
    web_data=requests.get(url)
    soup=BeautifulSoup(web_data.text,'lxml')
    data = {
        'title' : (soup.title.text).split('-')[0],
        'price' : soup.select('.price.c_f50')[0].get_text() if soup.find('span','price c_f50') else '0',
        'date': soup.select('li.time')[0].text,
        'area': list(soup.select('.c_25d')[0].stripped_strings) if soup.find_all('span','c_25d') else '[]',
        'fenlei':soup.select('.crb_i')[1].text,
        'views': get_views(url),
    }
    print(data)

def spider (page=1):
    for j in range(1,page+1):
        url=('http://bj.58.com/pbdn/0/pn{}/'.format(j))
        for i in get_urls(url):
            get_info(i)
        t=random.randrange(1,5)
        print('爬完一页，休息{}秒---------------'.format(t))
        time.sleep(t)

spider(2)

总结：根据老师和同学的代码，学习了split,回顾了stripped_strings，和requests的加载headers，前面讲过的基础知识还是要多看几遍，不然一下就忘了