我在使用 aiosqlite 和 SQLAlchemy 的 FastAPI 应用程序遇到大量并发请求时遇到问题，特别是在使用 Locust 进行 100 个并行查询的渗透测试期间。该应用程序在处理一些并行请求时运行良好，但大约 20% 的查询在重负载下会失败。
import uuid
import asyncio
from fastapi import FastAPI, Depends
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, AsyncEngine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import SingletonThreadPool
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String
Base = declarative_base()
class testdb(Base):
__tablename__ = "testdb"
id = Column(Integer, primary_key=True, index=True, nullable=False)
data = Column(String, index=True, nullable=False)
async_engine:AsyncEngine = create_async_engine("sqlite+aiosqlite:///./test.db", echo=True, future=True, poolclass=SingletonThreadPool)
async def getsession() -> AsyncSession:
async_session = sessionmaker(bind=async_engine, class_=AsyncSession, expire_on_commit=False)
async with async_session() as newsession:
yield newsession
app = FastAPI()
lock = asyncio.Lock()
@app.post('/save')
async def save(session:AsyncSession=Depends(getsession)):
new = testdb(data=uuid.uuid4().hex)
session.add(new)
await session.commit()
await session.refresh(new)
return new
@app.on_event('startup')
async def onstartup():
async with async_engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
详情：
该问题发生在执行
/saveawait session.refresh(new)我使用
AsyncSessionAsyncEngineSingletonThreadPool我收到的错误回溯如下：
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py", line 419, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py", line 84, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/fastapi/applications.py", line 1054, in __call__
await super().__call__(scope, receive, send)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/applications.py", line 123, in __call__
await self.middleware_stack(scope, receive, send)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 186, in __call__
raise exc
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 164, in __call__
await self.app(scope, receive, _send)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/routing.py", line 758, in __call__
await self.middleware_stack(scope, receive, send)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/routing.py", line 778, in app
await route.handle(scope, receive, send)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/routing.py", line 299, in handle
await self.app(scope, receive, send)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/routing.py", line 79, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/starlette/routing.py", line 74, in app
response = await func(request)
^^^^^^^^^^^^^^^^^^^
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 299, in app
raise e
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 294, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 191, in run_endpoint_function
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/xxxx/testdb.nosync/test_sqlalchemy.py", line 34, in save
await session.refresh(new)
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/sqlalchemy/ext/asyncio/session.py", line 327, in refresh
await greenlet_spawn(
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 202, in greenlet_spawn
result = context.switch(value)
^^^^^^^^^^^^^^^^^^^^^
File "/Users/xxxx/testdb.nosync/.venv/lib/python3.11/site-packages/sqlalchemy/orm/session.py", line 3140, in refresh
raise sa_exc.InvalidRequestError(
sqlalchemy.exc.InvalidRequestError: Could not refresh instance '<testdb at 0x108ed9410>'
如何解决此问题？
我使用 Locust 对 /save 端点进行了 100 个并行用户会话的渗透测试，并预期无缝执行，没有任何回溯。测试使用的模块版本如下：
- uvicorn[标准]==0.27.0
- fastapi==0.109.2
- aiosqlite==0.19.0
- sqlalchemy[asyncio]==2.0.25
Python 版本 3.11
尽管做出了这些努力，我仍然遇到同样的错误。我还尝试了使用 SQLModel 的版本，以及旧版本 FastAPI 和 SQLAlchemy 的各种组合，所有这些都导致了问题的持续存在。
我使用 Linux Ubuntu 22.04 进行了测试。和 MacBook Air M2 上的 Mac 操作系统。
我测试并使用了
async_engine:AsyncEngine=create_async_engine("sqlite+aiosqlite:///./test.db",echo=True,future=True,poolclass=QueuePool,max_overflow=-1,pool_size=1)持续到
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) database is locked重负载下。
不使用参数pool_size时同样