如何在使用 ProcessPoolExecutor 的 Python 进程之间安全地访问变量？

Question

我有一个使用多处理来执行许多 io 绑定任务的脚本。我想以安全的方式访问进程之间的变量。有没有一种简单的方法可以做到这一点，而不涉及诸如操作锁之类的低级逻辑？

import concurrent.futures
import time
import random

def do_book_task(books, year):
    books.append(year)

    print(f'Doing task with books from {year}.')
    time.sleep(random.random() + 0.5)

    books.remove(year)

    return f'Result for {year}'

def main():
    years = ['1996', '1997', '1998', '1999', '2000', '2001']

    books = [] # I want this variable to be process-safe

    with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor:
        futures = []
        for year in years:
            futures.append(executor.submit(do_book_task, books, year))
            print(f'Submitted {year} to process queue')
        
        for future in concurrent.futures.as_completed(futures):
            try:
                year = years[futures.index(future)]
                print(f'Done {year}')
                print(future.result())
            except Exception as e:
                print(f'Error with year: {year}')
                print(e)

Answer 1

是的，您可以使用

Manager

中的

multiprocessing.managers

类。请参阅https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Manager。

from multiprocessing import Manager

def main():
    years = ['1996', '1997', '1998', '1999', '2000', '2001']

    with Manager() as manager:
        books = manager.list() # creates a proxy object which is process-safe
        # the rest of the code is the same

        with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor:
            futures = []
            for year in years:
                futures.append(executor.submit(do_book_task, books, year))
                print(f'Submitted {year} to process queue')
            
            for future in concurrent.futures.as_completed(futures):
                try:
                    year = years[futures.index(future)]
                    print(f'Done {year}')
                    print(future.result())
                except Exception as e:
                    print(f'Error with year: {year}')
                    print(e)