python爬虫之进程池的创建及使用

from concurrent.futures import ProcessPoolExecutor
import os
  """
def download_page_data(page):
print(page,os.getpid())
return '下载完成'+str(page),page
def download_done(futures):
result = futures.result()
print(result)
next_page = int(result[1])+1
handler = pool.submit(download_page_data,next_page)
handler.add_done_callback(download_done)
if __name__ == '__main__':
#创建进程池
pool = ProcessPoolExecutor(4)
for page in range(0,200):
    hanlder = pool.submit(download_page_data,page)
    #回调函数的设置,看自己是否需要
    hanlder.add_done_callback(download_done)
#cannot schedule new futures after shutdown
# pool.shutdown()
"""
#方式二
from multiprocessing import Pool
def download_page_data(page):
  print(page,os.getpid())
  return '下载完成'+str(page),page
def done(futures):
print(futures)
if __name__ == '__main__':
#创建进程池
pool = Pool(4)
for page in range(0,200):
    # pool.apply_async() 异步非阻塞添加任务
    # pool.apply() 同步的方式添加任务
    # func, 要执行的方法(函数)
    # args=(),给函数传递的参数
    #callback = None,成功的回调
    #error_callback = None,执行错误的回调
    pool.apply_async(download_page_data,args=(page,),callback=done)
pool.close() #执行close后不可以再添加任务了
pool.join()