python并发处理库--concurrent.futures

concurrent提供了两种并发模型，一个是多线程ThreadPoolExecutor，一个是多进程ProcessPoolExecutor。对于IO密集型任务宜使用多线程模型。对于计算密集型任务应该使用多进程模型。

一、多线程

# coding: utf8

import time
import sys
import fire
import threading
from concurrent.futures import ThreadPoolExecutor, wait


# 分割子任务
def each_task(index):
    time.sleep(2)  # 睡1s，模拟IO
    sys.stdout.write("thread %s square %d\n" % (threading.current_thread().ident, index))
    return index * index  # 返回结果

# 基础的调用过程
# def run(thread_num, task_num):
#     # 实例化线程池，thread_num个线程
#     executor = ThreadPoolExecutor(thread_num)
#     start = time.time()
#     fs = []  # future列表
#     for i in range(task_num):
#         fs.append(executor.submit(each_task, i))  # 提交任务
#     wait(fs)  # 等待计算结束
#     end = time.time()
#     duration = end - start
#     s = sum([f.result() for f in fs])  # 求和
#     print("normal process total result=%s cost: %.2fs" % (s, duration))
#     executor.shutdown()  # 销毁线程池

# map函数的调用
def run(thread_num, task_num):
    # 实例化线程池，thread_num个线程
    task = [i for i in range(task_num)]
    start = time.time()
    with ThreadPoolExecutor(thread_num) as e:
        fs = e.map(each_task, task)

    s = sum([f for f in fs])  # 求和
    end = time.time()
    duration = end - start
    print(s)
    print("total result=%s cost: %.2fs" % (s, duration))


if __name__ == '__main__':
    run(10, 10)

二、多进程

代码类似，风格一样，具体参考
https://juejin.im/post/5b1e36476fb9a01e4a6e02e4