使用生产者与消费者模式多线程下载表情包

import requests
from lxml import etree
import re
from urllib import request
import os
import threading
from queue import Queue

class Producer(threading.Thread):
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
    def __init__(self,page_queue,img_queue): # (self,page_queue,img_queue,*args,**kwargs)
        # super(Producer, self).__init__(*args,**kwargs)
        threading.Thread.__init__(self) # 没有该语句的话则会出现Python RuntimeError: thread.__init__() not called异常
        self.page_queue = page_queue
        self.img_queue = img_queue
    def run(self):
        while True:
            if self.page_queue.empty():
                break
            url = self.page_queue.get()
            self.parse_page(url)

    def parse_page(self,url):
        response = requests.get(url,headers = self.headers)
        text = response.content.decode('utf-8')
        html = etree.HTML(text)
        imgs = html.xpath('//*[@id="pic-detail"]/div/div[3]/div[2]/ul/li/div//img[@class!="gif"]') 
        for img in imgs:
            # print(etree.tostring(img)) # 可以查看筛选出来的img中的是否符合想要的标签
            img_urls = img.xpath('@data-original')
            for img_url in img_urls:
                img_url = re.sub(r'!dta','',img_url)
            img_name = img.xpath('@alt')[0]
            suffix = os.path.splitext(img_url)[1]
            filename = img_name + suffix
            self.img_queue.put((img_url,filename))

class Consumer(threading.Thread):
    def __init__(self,page_queue,img_queue):
        # super(Consumer, self).__init__(*args,**kwargs)
        threading.Thread.__init__(self) # 没有该语句的话则会出现Python RuntimeError: thread.__init__() not called异常
        self.page_queue = page_queue
        self.img_queue = img_queue
    
    def run(self):
        while True:
            if self.img_queue.empty() and self.page_queue.empty():
                break
            img_url,filename = self.img_queue.get()
            request.urlretrieve(img_url,'images/'+filename)

def main():
    page_queue = Queue(100) # 图片url存储空间
    img_queue = Queue(1000) # 图片下载器存储空间
    for x in range(1,21):
        url = "http://www.doutula.com/photo/list/?page=%d" % x
        page_queue.put(url)
    
    for x in range(5):
        t = Producer(page_queue,img_queue)
        t.start()
    
    for x in range(5):
        t = Consumer(page_queue,img_queue)
        t.start()


if __name__ == '__main__':
    main()