
untitled.png
一. 使用beautifulsoup解析爬取街拍美女图片.

image.png
上代码
import requests
from bs4 import BeautifulSoup
class MzSpider(object):
def __init__(self,):
self.base_url = "https://www.mzitu.com/jiepai/"
self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
+" AppleWebKit/537.36 (KHTML, like Gecko)"
+" Chrome/80.0.3987.149 Safari/537.36"}
def get_page(self):
return requests.get(url = self.base_url, headers = self.headers)
def run(self):
response = self.get_page()
soup = BeautifulSoup(response.text,'lxml')
result = soup.find_all('img',attrs = {'class':'lazy'})
for value in result:
response = requests.get(url=value.attrs['data-original'], headers=self.headers)
path_name = './'+str(value.attrs['data-original'])[-5:-8:-1]+'.png'
with open(path_name,'wb') as f:
f.write(response.content)
if __name__ == '__main__':
mzspider = MzSpider()
mzspider.run()
最终GET(放两张出来给大家看看)

4ow.png

nkw.png
二. 使用Xpath爬取当当网关于python书籍的信息.
代码
#爬取当当网图书
import requests
import csv
import time
from lxml import etree
class DDSpider(object):
def __init__(self,key):
self.key = key
self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
+" AppleWebKit/537.36 (KHTML, like Gecko)"
+" Chrome/80.0.3987.149 Safari/537.36"}
self.base_url ="http://search.dangdang.com/?key="+key+"&act=input&page_index={}"
pass
#获取界面的内容
def get_pageInfo(self):
num_page = 0
while True:
num_page += 1
page_url=self.base_url.format(num_page)
response = requests.get(url=page_url,headers=self.headers)
return self.parse_pageInfo(response.content.decode("GBK"))
time.sleep(1)
if num_page == 1:
break
def parse_pageInfo(self,html):
doc = etree.HTML(html)
#图片
img_ul1 = doc.xpath('//*[@id="component_59"]/li/a/img/@src')
img_ul2 = doc.xpath('//*[@id="component_59"]/li/a/img/@data-original')
img_ul1.extend(img_ul2)
img_ul = []
for i in img_ul1:
if i != 'images/model/guan/url_none.png':
img_ul.append(i)
else:
continue
#标题
title_ul = doc.xpath('//*[@class="name"]/a/@title')
#价格
price_ul = doc.xpath('//*[@class="search_now_price"]/text()')
print(price_ul)
#评价数
comments_ul = doc.xpath('//*[@class="search_star_line"]/a/text()')
#星星
stars_ul = doc.xpath('//*[@class="search_star_black"]/span/@style')
my_list = []
final_list = []
for num in range(len(img_ul)):
my_list.append(img_ul[num])
my_list.append(title_ul[num])
my_list.append(price_ul[num])
my_list.append(comments_ul[num])
my_list.append(str(float(stars_ul[num].split(':')[1].strip('%;'))/20))
final_list.append(tuple(my_list))
my_list = []
return final_list
def save_to_csv(self,data):
list_name =["picture_url","book_name","price","comments","star_level"]
with open('./dangd.csv', 'a', newline='', encoding="utf-8-sig") as f:
writer = csv.writer(f)
writer.writerow(list_name)
for turple_value in data:
with open('./dangd.csv','a',newline='',encoding="utf-8-sig") as f:
writer = csv.writer(f)
writer.writerow(list(turple_value))
if __name__ == "__main__":
dds= DDSpider("python")
data = dds.get_pageInfo()
dds.save_to_csv(data)
几个注意的地方:
代码中key代表你要搜索的关键字

image.png
翻页功能需要你翻页观察网页地址的变化

image.png
使用GBK不使用常用的UTF8的缘由

image.png
以下是爬虫得到的csv文件在pycharm的中的图片.

image.png

image.png
可以看到,我们爬取了书籍的图片,名称,价格,评论数,评分.
总结:以上是基本的爬虫练习成果.还有许多问题需要我们去解决.
网友评论