concurrent.futures.ThreadPoolExecutor.map()：超时不起作用

Question

import concurrent.futures
import time 

def process_one(i):
    try:                                                                             
        print("dealing with {}".format(i))                                           
        time.sleep(50)
        print("{} Done.".format(i))                                                  
    except Exception as e:                                                           
        print(e)

def process_many():
    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: 
        executor.map(process_one,
                range(100),                                                          
                timeout=3)                                                           


if __name__ == '__main__':                                                           
    MAX_WORKERS = 10
    try:
        process_many()
    except Exception as e:                                                           
        print(e)      

docs 说：

如果调用 __next__() 并且在从最初调用 Executor.map() 开始的 timeout 秒后结果不可用，则返回的迭代器会引发 concurrent.futures.TimeoutError

但是这里的脚本没有引发任何异常并一直在等待。有什么建议吗？

Answer 1

正如我们在source（对于python 3.7）中看到的那样，映射返回一个函数：

def map(self, fn, *iterables, timeout=None, chunksize=1):
    ...
    if timeout is not None:
        end_time = timeout + time.time()
    fs = [self.submit(fn, *args) for args in zip(*iterables)]
    # Yield must be hidden in closure so that the futures are submitted
    # before the first iterator value is required.
    def result_iterator():
        try:
            # reverse to keep finishing order
            fs.reverse()
            while fs:
                # Careful not to keep a reference to the popped future
                if timeout is None:
                    yield fs.pop().result()
                else:
                    yield fs.pop().result(end_time - time.time())
        finally:
            for future in fs:
                future.cancel()
    return result_iterator()

TimeoutError 是从 yield fs.pop().result(end_time - time.time()) 调用引发的，但您必须请求结果才能到达该调用。

理由是您不关心提交任务。任务被提交并开始在后台线程中运行。您关心的是请求结果时的超时 - 这是一个常见的用例，您提交任务并在有限的时间内向他们请求结果，而不仅仅是提交它们并期望它们在有限的时间内终止。

如果你是后者，你可以使用wait，例如Individual timeouts for concurrent.futures中所示

Answer 2

正如文档指定的那样，只有在地图上调用 __next__() 方法时才会引发超时错误。要调用此方法，您可以将输出转换为列表：

from concurrent import futures
import threading
import time


def task(n):
    print("Launching task {}".format(n))
    time.sleep(n)
    print('{}: done with {}'.format(threading.current_thread().name, n))
    return n / 10


with futures.ThreadPoolExecutor(max_workers=5) as ex:
    results = ex.map(task, range(1, 6), timeout=3)
    print('main: starting')
    try:
        # without this conversion to a list, the timeout error is not raised
        real_results = list(results) 
    except futures._base.TimeoutError:
        print("TIMEOUT")

输出：

Launching task 1
Launching task 2
Launching task 3
Launching task 4
Launching task 5
ThreadPoolExecutor-9_0: done with 1
ThreadPoolExecutor-9_1: done with 2
TIMEOUT
ThreadPoolExecutor-9_2: done with 3
ThreadPoolExecutor-9_3: done with 4
ThreadPoolExecutor-9_4: done with 5

这里，第n个任务休眠n秒，所以在任务2完成后超时。

---

编辑：如果您想终止未完成的任务，您可以尝试this 问题中的答案（虽然他们不使用ThreadPoolExecutor.map()），或者您可以忽略其他任务的返回值，让它们完成：

from concurrent import futures
import threading
import time


def task(n):
    print("Launching task {}".format(n))
    time.sleep(n)
    print('{}: done with {}'.format(threading.current_thread().name, n))
    return n


with futures.ThreadPoolExecutor(max_workers=5) as ex:
    results = ex.map(task, range(1, 6), timeout=3)
    outputs = []
    try:
        for i in results:
            outputs.append(i)
    except futures._base.TimeoutError:
        print("TIMEOUT")
    print(outputs)

输出：

Launching task 1
Launching task 2
Launching task 3
Launching task 4
Launching task 5
ThreadPoolExecutor-5_0: done with 1
ThreadPoolExecutor-5_1: done with 2
TIMEOUT
[1, 2]
ThreadPoolExecutor-5_2: done with 3
ThreadPoolExecutor-5_3: done with 4
ThreadPoolExecutor-5_4: done with 5