python爬虫，第二天作业抓取猫途鹰

frombs4importBeautifulSoup

importrequests

importtime

info=[]

headers={

'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1'

}

urls=['http://www.tripadvisor.cn/Attractions-g60763-Activities-oa{}-New_York_City_New_York.html#ATTRACTION_LIST'.format(i)foriinrange(0,976,20)]

deftripying(url):

wb_data=requests.get(url,headers=headers)

time.sleep(2)

soup=BeautifulSoup(wb_data.text,'lxml')

titles=soup.select(' div.title.titleLLR > div')

images=soup.find_all('div','missing lazyMiss')

rates=soup.select('div.overflowEllipsis.popIndexLLR > div')

cates=soup.select('div.attraction_types > span')

comments=soup.find_all(class_='reviewCountLLR overflowEllipsis')

stars=soup.select(' div.rating.ratingLLR.overflowEllipsis > div.list_item_rating > div > div > div.center')

defcommentstar(values):

data_in=values

countclass=0

countstar=0

suanstar=[]

foriindata_in:

countclass=countclass+1

job=i.get('class')

ifjob[1]=='':

countstar=countstar+1

elifjob[1]=='semi_full':

countstar=countstar+0.5

else:

countstar=countstar+0

ifcountclass==5:

data_in=data_in[:5]

countclass=0

suanstar.append(countstar)

countstar=0

return(suanstar)

stars1=commentstar(stars)

fortitle, image, rate, cate, comment, starinzip(titles, images, rates, cates, comments, stars1):

data={

'title':title.get_text(),

'image':image.get('data-thumburl'),

'rate':rate.get_text(),

'cate':cate.get_text(),

'comment':comment.get_text(),

'star':star

}

info.append(data)

foriininfo:

print(i['title'], i['image'], i['rate'], i['cate'], i['comment'],'评级:'+str(i['star'])+'星')

forubinurls:

tripying(ub)

离我改变世界又近了一步

通过这个作业，我学习了如何将浏览器伪装成为手机，如何全面的爬取一些数据

但是也还有一些问题，就是爬出来的数据会重复很多次，希望有老师能够解答这个问题及原因

又学习了format和rang的用法。

有些数据爬出来还是很困难