如何同时访问多个python/shelve数据库？

Question

我正在构建一个简单的程序，通过多线程将数据同时保存到不同的货架数据库，但是当 2 个线程调用时发生错误

shelve.open()

（对于不同的文件）：

import threading
import shelve
import time


def parallel_shelve(idx):
    print("thread {}: start".format(idx))
    with shelve.open("cache_shelve_{}".format(idx)) as db:
        time.sleep(4)
        db["0"] = 0
        db.close()
    print("thread {}: done".format(idx))


if __name__ == "__main__":
    threads = []
    for idx in range(2):
        threads += [threading.Thread(target=parallel_shelve, args=(idx,))]

    for idx in range(len(threads)):
        threads[idx].start()

    for idx in range(len(threads)):
        threads[idx].join()

完整日志：

$ python multi_database.py
thread 0: start
thread 1: start
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/home/blahblah/anaconda3/lib/python3.9/threading.py", line 980, in _bootstrap_inner
Exception in thread Thread-2:
Traceback (most recent call last):
  File "/home/blahblah/anaconda3/lib/python3.9/threading.py", line 980, in _bootstrap_inner
    self.run()
  File "/home/blahblah/anaconda3/lib/python3.9/threading.py", line 917, in run
    self._target(*self._args, **self._kwargs)
  File "/home/blahblah/Desktop/multi_database.py", line 8, in parallel_shelve
    with shelve.open("cache_shelve_{}".format(idx)) as db:
  File "/home/blahblah/anaconda3/lib/python3.9/shelve.py", line 243, in open
    self.run()
  File "/home/blahblah/anaconda3/lib/python3.9/threading.py", line 917, in run
    return DbfilenameShelf(filename, flag, protocol, writeback)
  File "/home/blahblah/anaconda3/lib/python3.9/shelve.py", line 227, in __init__
    Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback)
  File "/home/blahblah/anaconda3/lib/python3.9/dbm/__init__.py", line 95, in open
    return mod.open(file, flag, mode)
AttributeError: module 'dbm.gnu' has no attribute 'open'
    self._target(*self._args, **self._kwargs)
  File "/home/blahblah/Desktop/multi_database.py", line 8, in parallel_shelve
    with shelve.open("cache_shelve_{}".format(idx)) as db:
  File "/home/blahblah/anaconda3/lib/python3.9/shelve.py", line 243, in open
    return DbfilenameShelf(filename, flag, protocol, writeback)
  File "/home/blahblah/anaconda3/lib/python3.9/shelve.py", line 227, in __init__
    Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback)
  File "/home/blahblah/anaconda3/lib/python3.9/dbm/__init__.py", line 95, in open
    return mod.open(file, flag, mode)
AttributeError: module 'dbm.gnu' has no attribute 'open'

$ python --version
Python 3.9.13

如何修复它以同时访问不同的搁置文件？

Answer 1

我只能发现

dbm.open

在第一次调用时试图找到一个可用的 dbm 实现。如果这同时发生两次，神秘的事情就会发生（我无法解释）导致错误。

解决方法是在启动线程之前触发实现搜索，例如

import dbm
try:
    dbm.open(None)
except TypeError:
    pass

Answer 2

通过实验，我发现该错误仅在您创建新数据库时触发，并且在读取/写入现有数据库时工作正常。对我来说最简单的解决方案似乎是在创建数据库时添加一个锁：

import threading
import shelve
import time
from threading import Lock


def parallel_shelve(lock, idx):
    print("thread {}: start".format(idx))

    file_name = "cache_shelve_{}".format(idx)

    lock.acquire()
    with shelve.open(file_name) as db:
        db.clear()
    lock.release()

    with shelve.open(file_name) as db:
        time.sleep(1)
        db["0"] = 0

    print("thread {}: done".format(idx))


if __name__ == "__main__":
    threads = []
    lock = Lock()
    for idx in range(10):
        threads += [
            threading.Thread(target=parallel_shelve, args=(
                lock,
                idx,
            ))
        ]

    for idx in range(len(threads)):
        threads[idx].start()

    for idx in range(len(threads)):
        threads[idx].join()