python 多进程下载并显示进度条

一 最简单的操作方法

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import requests
def main():
    r = requests.get(url, headers=headers, stream=True)
    f = open("movie.flv", "wb")
    print(r.__dict__)  # 查看响应信息
    size = 31693740.0
    lenth = 0
    for chunk in r.iter_content(chunk_size=1024):
        lenth += len(chunk)
        if chunk:
            f.write(chunk)
        print(str(int(lenth/size*100))+'%', end='\r')
    f.close()

企业微信截图_15905794341677.png

二 多进程下载

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import os
import requests
import progressbar
import multiprocessing

def multi_process_download(download_url, fileName, process_num=1, file_size=0, **kwargs):
    '''
     数据流多进程写入本地文件
    :param process_num:
    :param file_size:
    :return:
    '''
    if process_num >= 100:
        process_num = 99
    bsize = file_size / process_num
    plist = []
    for i in range(0, process_num):
        begin = bsize * i
        # 最后一个进程的下载数据大小
        if (begin + bsize) > file_size or ((i + 1) == process_num and (begin + bsize) < file_size):
            bsize = file_size - begin
        p = multiprocessing.Process(target=_process_fetch_file,
                                    args=(i, fileName, begin, bsize, download_url))
        plist.append(p)
    for p in plist:
        p.start()
    for p in plist:
        p.join(600)
    is_timeout = False
    for p in plist:
        # 结束超时的子进程
        if p.is_alive():
            is_timeout = True
            os.kill(p.pid, signal.SIGKILL)
    if is_timeout:
        # 下载超时重新下载
        return multi_process_download(download_url, fileName, process_num=1, file_size=0, **kwargs)
    # 合并文件
    download_file_path = kwargs.get('download_file_path')
    fp = open(download_file_path, 'wb')
    for i in range(0, process_num):
        # 分别将每个部分文件按顺序写入新文件
        fname = 'part_%02d_%s' % (i + 1, fileName)
        print '合并文件 %s 中...' % fname
        fpath = download_dir_path + fname
        if not os.path.isfile(fpath):
            print '-' * 100
            continue
        fp1 = open(fpath, 'rb')
        fp.write(fp1.read())
        fp1.close()
        try:
            # 清理碎片文件
            os.remove(fpath)
        except:
            pass
    fp.close()    

def _process_fetch_file(self, inx, fileName, begin=0, bsize=0, download_url_=''):
    """
    多进程获取文件
    """
    end = begin + bsize
    __headers = {'Range':'bytes=%s-%s'%(begin, end)}
    headers.update(__headers)
    r = requests.get(url=download_url_, headers=headers, stream=True, timeout=self.timeout, verify=False)
    fname = 'part_%02d_%s' % (inx + 1, fileName)
    widgets = ['文件 %s 下载中:' % fname, progressbar.Bar(marker='=', left='[', right=']'), '大小:',
               progressbar.FileSize(), ' ,已下载: ', progressbar.Percentage(),
               ' ,速度:', progressbar.FileTransferSpeed()]
    pbar = progressbar.ProgressBar(widgets=widgets, maxval=bsize).start()
    file_path = download_dir_path + fname
    file_size__ = 0
    with open(file_path, 'wb') as f:
        for chunk in r.iter_content(chunk_size=2048):
            if not chunk:
                break
            f.write(chunk)
            f.flush()
            file_size__ += len(chunk)
            if file_size__ % 20480 == 0:
                print 'fname: {0} 已下载字符长度: {1}, 预计长度: {2}'.format(fname, file_size__, bsize)
            # pbar.update(file_size__)
    pbar.finish()