美文网首页工作生活
Using multiple processing to dow

Using multiple processing to dow

作者: 狼无雨雪 | 来源:发表于2019-07-05 12:56 被阅读0次
#coding=utf-8
from time import sleep, ctime 
import threading
import urllib.request
import os
import time

print("program start %s"%ctime())

path = "huaban_tem_images"
if not os.path.exists(path):
    os.makedirs(path)

url_set = []
filename_set = []
temp_url_set = []
temp_filename_set = []
with open("images_urls.txt",'r',encoding="utf8") as read_file:
    all_lines = read_file.readlines()
    print("number: ",len(all_lines))
    len_all_lines = len(all_lines)
    for index, line in enumerate(all_lines):
        url = line.strip()
        temp_url_set.append(url)
        file_name = path + "/" +str(url.split("/")[-1]) + ".png"
        temp_filename_set.append(file_name)
        if (index + 1) % 10 == 0 or index == (len_all_lines - 1):
            url_set.append(temp_url_set)
            filename_set.append(temp_filename_set)
            temp_url_set = []
            temp_filename_set = []



def downloading_images(url, filename):
#         print("start downloading", url)
        urllib.request.urlretrieve(url, filename = filename)
        sleep(1)
        print("end downloading", url)



if __name__ == '__main__': 
    epoch = 1
    for (urls, filenames) in zip(url_set, filename_set):
        threads = []
        for (url, filename) in zip(urls, filenames):
            t = threading.Thread(target=downloading_images, args=(url, filename))
            threads.append(t)
        for i in range(len(threads)):
            threads[i].start()
        
        for i in range(len(threads)):
            threads[i].join()
        print("epoch %d finished in %s"%(epoch, ctime()))
        epoch += 1
        
    print('program end:%s' %ctime())

相关文章

网友评论

    本文标题:Using multiple processing to dow

    本文链接:https://www.haomeiwen.com/subject/bzzrhctx.html