执行多个 PyODBC sql 查询时 Python 脚本执行停止

Question

我正在尝试编写一个在数据库中的多个表上运行的循环，将结果存储在数据框中，然后将该数据框写入 excel 工作簿。但是，即使我的列表中有 29 个表，我的代码执行也会在少量后停止（并且不一致，有时会在 1 个查询后停止，但通常会在 5-6 个查询中停止）。关于为什么会这样的任何想法？

conn = pyodbc.connect("DSN=XXXX")
conn.setdecoding(pyodbc.SQL_CHAR, encoding='utf-16-le')
conn.setdecoding(pyodbc.SQL_WCHAR, encoding='utf-16-le')
conn.setencoding(encoding='utf-16-le')
TABLES = ['Table1', 'Table2', ..., 'Table29']

def query_multiple_tables(tables: list[str]) -> list[pd.DataFrame]:
    dataframes = []
    for table in tables:
        QUERY = f"SELECT * FROM [db].[dbo].[{table}]"
        col_crsr = conn.cursor()
        data_crsr = conn.cursor()
        
        cols = [row.column_name for row in col_crsr.columns(table=f"{table}")]
        data = pd.DataFrame.from_records(data_crsr.execute(QUERY).fetchmany(20), columns=cols)
        
        print(f"Extracting table: {table}")
        dataframes.append(data)

        col_crsr.close()
        data_crsr.close()
    
    return dataframes

def write_to_excel(dataframes: list[pd.DataFrame]) -> None:
    
    try:
        with pd.ExcelWriter("Data.xlsx", mode='a', if_sheet_exists='replace') as writer:
            for i, dataframe in enumerate(dataframes):
                dataframe.to_excel(writer, sheet_name=f'{i}')
                print(f"Append successful for: {i}")
    except FileNotFoundError:
        with pd.ExcelWriter("Data.xlsx", mode='w') as writer:
            for i, dataframe in enumerate(dataframes):
                dataframe.to_excel(writer, sheet_name=f'{i}')
                print(f"Initial write successful for: {i}")

data = query_multiple_tables(FIA_TABLES)
write_to_excel(data)

Answer 1

结果将放入 df“查询”并循环并加入主 df，然后您可以将其放入 csv/xlsx

import pyodbc
import pandas as pd
connection_string = "DRIVER={SQL SERVER};Server=server;Database=db;Trusted_Connection=yes;"
connection = URL.create("mssql+pyodbc", query={"odbc_connect": connection_string})
engine = create_engine(connection)
connection = engine.connect() 
df = pd.DataFrame()
Tables = ['1', '2'....]
for table in Tables:
   query = f'''
   select * from {table}
'''
   query = pd.read_sql_query(query, connection)
   df = pd.concat(df, query)