简单的python爬取图片

之前有一次网页上图片不能复制，就自己写了一个下载图片程序，只能针对例子中网页获取图片，若想下载其他网页中图片，需对程序进行改造。

#coding=utf-8
from bs4 import BeautifulSoup
import aiohttp
import asyncio
from urllib import request

#回调函数，打印进度
def callbackFunc(blocknum, blocksize, totalsize):
    '''回调函数
    @blocknum: 已经下载的数据块
    @blocksize: 数据块的大小
    @totalsize: 远程文件的大小
    '''
    percent = 100.0*blocknum*blocksize/totalsize
    if percent > 100:
        percent = 100
    print("%.2f%%" % percent)

#获取页面源代码
async def getPage(url, res_list):
    headers = {'User-Agent':'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}
    async with aiohttp.ClientSession() as session:
        async with session.get(url, headers = headers) as resp:
            assert resp.status == 200
            res_list.append(await resp.text())

#分析html源代码，找到所需要的标记
class parsePage():
    def __init__(self, page_str):
        self.page_str = page_str
    #__enter__和__exit__配合，主要实现with...as...功能
    def __enter__(self):
        page_str = self.page_str
        #这里只是例子中获取图片地址的规则，具体html页面需要具体分析
        page = BeautifulSoup(page_str, 'lxml')
        articles = page.findAll('img')
        art_urls = []
        for a in articles:
            art_urls.append(a['src'])
        return art_urls
    def __exit__(self, exc_type, exc_val, exc_tb):
        pass

class jianshuParsePage(parsePage):
    def __init__(self, page_str):
        super(jianshuParsePage, self).__init__(page_str)
    
'''
获取网页中所有的图片网址，并下载图片
'''
if __name__ == '__main__':
    page_url_base = 'http://www.baiyinghui.com/gallery/show/id/MDAwMDAwMDAwMLGIgtuFtKF2?spm=a220o.1000855.0.0.iuLTnS'
    page_urls = [page_url_base]
    loop = asyncio.get_event_loop()
    ret_list = []
    tasks = [getPage(host,ret_list) for host in page_urls]
    loop.run_until_complete(asyncio.wait(tasks))

    articles_url = []
    x = 0
    for ret in ret_list:
        with jianshuParsePage(ret) as tmp:
            for jpg in tmp:
                #这里是过滤掉包含特殊字符和汉字的url，因为
                if jpg.find('(') != -1 or jpg.find('副本') != -1 or jpg.find(')') != -1 or jpg.find(' ') != -1:
                    pass
                elif jpg.find('http') != -1:
                    print(jpg)
                    #根据获取到的图片的url下载图片
                    request.urlretrieve(jpg, '/home/pycode/tupian/%s.jpg' % x, callbackFunc)
                    x += 1
                    #这里为了测试方便，就直接跳出循环了
                    break
    ret_list = []

    loop.close()