untitled.png

一. 使用beautifulsoup解析爬取街拍美女图片.

image.png

上代码

import requests
from bs4 import BeautifulSoup
class MzSpider(object):
    def __init__(self,):
        self.base_url = "https://www.mzitu.com/jiepai/"
        self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
                                    +" AppleWebKit/537.36 (KHTML, like Gecko)"
                                    +" Chrome/80.0.3987.149 Safari/537.36"}


    def get_page(self):
        return requests.get(url = self.base_url, headers = self.headers)

    def run(self):
        response = self.get_page()
        soup = BeautifulSoup(response.text,'lxml')
        result = soup.find_all('img',attrs = {'class':'lazy'})
        for value in result:
            response = requests.get(url=value.attrs['data-original'], headers=self.headers)
            path_name = './'+str(value.attrs['data-original'])[-5:-8:-1]+'.png'
            with open(path_name,'wb') as f:
                f.write(response.content)


if __name__ == '__main__':

    mzspider = MzSpider()
    mzspider.run()

最终GET(放两张出来给大家看看)

4ow.png nkw.png

二. 使用Xpath爬取当当网关于python书籍的信息.

代码

#爬取当当网图书
import requests
import csv
import time
from lxml import etree

class  DDSpider(object):
    def __init__(self,key):
        self.key = key
        self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
                                     +" AppleWebKit/537.36 (KHTML, like Gecko)"
                                     +" Chrome/80.0.3987.149 Safari/537.36"}
        self.base_url ="http://search.dangdang.com/?key="+key+"&act=input&page_index={}"
        pass
    #获取界面的内容
    def get_pageInfo(self):
        num_page = 0
        while True:
            num_page += 1
            page_url=self.base_url.format(num_page)
            response = requests.get(url=page_url,headers=self.headers)
            return self.parse_pageInfo(response.content.decode("GBK"))
            time.sleep(1)
            if num_page == 1:
                break
    def parse_pageInfo(self,html):
        doc = etree.HTML(html)
        #图片
        img_ul1 = doc.xpath('//*[@id="component_59"]/li/a/img/@src')
        img_ul2 = doc.xpath('//*[@id="component_59"]/li/a/img/@data-original')
        img_ul1.extend(img_ul2)
        img_ul = []
        for i in img_ul1:
            if i != 'images/model/guan/url_none.png':
                img_ul.append(i)
            else:
                continue
        #标题
        title_ul = doc.xpath('//*[@class="name"]/a/@title')
        #价格
        price_ul = doc.xpath('//*[@class="search_now_price"]/text()')
        print(price_ul)
        #评价数
        comments_ul = doc.xpath('//*[@class="search_star_line"]/a/text()')
        #星星
        stars_ul = doc.xpath('//*[@class="search_star_black"]/span/@style')
        my_list = []
        final_list = []
        for num in range(len(img_ul)):
            my_list.append(img_ul[num])
            my_list.append(title_ul[num])
            my_list.append(price_ul[num])
            my_list.append(comments_ul[num])
            my_list.append(str(float(stars_ul[num].split(':')[1].strip('%;'))/20))
            final_list.append(tuple(my_list))
            my_list = []
        return final_list
    def save_to_csv(self,data):
        list_name =["picture_url","book_name","price","comments","star_level"]
        with open('./dangd.csv', 'a', newline='', encoding="utf-8-sig") as f:
            writer = csv.writer(f)
            writer.writerow(list_name)
        for turple_value in data:
            with open('./dangd.csv','a',newline='',encoding="utf-8-sig") as f:
                writer = csv.writer(f)
                writer.writerow(list(turple_value))
if __name__ == "__main__":
    dds= DDSpider("python")
    data = dds.get_pageInfo()
    dds.save_to_csv(data)

几个注意的地方:

代码中key代表你要搜索的关键字

image.png

翻页功能需要你翻页观察网页地址的变化

image.png

使用GBK不使用常用的UTF8的缘由

image.png

以下是爬虫得到的csv文件在pycharm的中的图片.

image.png
image.png

可以看到,我们爬取了书籍的图片,名称,价格,评论数,评分.

总结:以上是基本的爬虫练习成果.还有许多问题需要我们去解决.