SQLAlchemy 核心插入

问题描述 投票：0回答：1

当我这样生成几十万条数据时，速度很慢，所以想用CORE来插入数据。我该怎么做？

# USE ORM
def create_data(sess, engine, count=50,
                username_list=None, product_list=None, project_list=None, dept_list=None,
                organization_tree=None):
    table_data_list = []
    for counter in range(1, count):
        init_time = datetime.strptime(
            (datetime.now() - timedelta(days=random.randrange(365))).strftime("%Y-%m-%d %H:%M:%S"), "%Y-%m-%d %H:%M:%S")
        LEVEL_1_DEVOPS_DEPT_CODE = get_random_node_code(organization_tree)
        LEVEL_2_DEVOPS_DEPT_CODE = get_random_node_code(organization_tree[LEVEL_1_DEVOPS_DEPT_CODE])
        LEVEL_3_DEVOPS_DEPT_CODE = get_random_node_code(
            organization_tree[LEVEL_1_DEVOPS_DEPT_CODE][LEVEL_2_DEVOPS_DEPT_CODE])
        LEVEL_4_DEVOPS_DEPT_CODE = get_random_node_code(
            organization_tree[LEVEL_1_DEVOPS_DEPT_CODE][LEVEL_2_DEVOPS_DEPT_CODE][LEVEL_3_DEVOPS_DEPT_CODE])

        table_data = TableData(
            pkey=random.choice(product_list),
            user_name=random.choice(username_list),
            qa_version=init_time.strftime("%Y-%m-%d"),
            updated_time=init_time,
            created_time=init_time,
            build_exec_time=init_time,
            tests_failed=0,
            tests_skipped=0,
            tests_total=0,
            is_incoco=random.choice(['true', 'false']),
            jacoco_branch_coverage_rate=round(random.uniform(0, 48), 4),
            jacoco_line_coverage_rate=round(random.uniform(0, 48), 4),
            blocker_issues=random.randint(0, 6),
            major_issues=random.randint(0, 6),
            critical_issues=random.randint(0, 6),
            project_key=f"project_key_{counter}",
            applicant_sn=random.choice(username_list),
            jira_version=random.choice(['2.10.24', '7.5.13', '4.15.37', '9.8.5', '3.19.42']),
            git_urls=f"https://gitlab.demo.com/git-number{counter}/git",
            quality_scope=random.choice(['overall', 'newcode']),
            dev_lang=random.choice(['java', 'python', 'golang']),
            source=random.choice([0, 1]),
            unit_username=random.choice(username_list),
            pipeline_type='cd_pipeline',
            pname=random.choice(product_list),
            jenkins_template=random.choice(['dailyCiJava', 'dailyCiPython', 'dailyCiGolang']),
            jacoco_lines_covered=random.randint(100, 1000),
            LEVEL_1_DEVOPS_DEPT_CODE=LEVEL_1_DEVOPS_DEPT_CODE,
            LEVEL_2_DEVOPS_DEPT_CODE=LEVEL_2_DEVOPS_DEPT_CODE,
            LEVEL_3_DEVOPS_DEPT_CODE=LEVEL_3_DEVOPS_DEPT_CODE,
            LEVEL_4_DEVOPS_DEPT_CODE=LEVEL_4_DEVOPS_DEPT_CODE

        )
        table_data_list.append(table_data)

        if len(table_data_list) % 10000 == 0:
            sess.bulk_save_objects(table_data_list)
            sess.commit()
            table_data_list = []

    if table_data_list:
        sess.bulk_save_objects(table_data_list)
        sess.commit()

如何调用get_random_node_code函数并生成对应的日期 当我使用 __table__insert() 生成数据时，确实快了好几倍，但我不知道如何生成所需的变量并在 for 循环中调用函数。

# USE CORE
variable in the for loop?
def create_data(sess, engine, count=50,
                username_list=None, product_list=None, project_list=None, dept_list=None,
                organization_tree=None):
    core.table_action.create_table_if_not_exists(DwdDeployBranchData, engine)
    sess.execute(
        TableData.__table__.insert(),
        [
            {'pkey': random.choice(product_list),
             'user_name': random.choice(username_list),
             'qa_version': init_time.strftime("%Y-%m-%d"),
             'updated_time': init_time,
             'created_time': init_time,
             'build_exec_time': init_time,
             'tests_failed': 0,
             'tests_skipped': 0,
             'tests_total': 0,
             'is_incoco': random.choice(['true', 'false']),
             'jacoco_branch_coverage_rate': round(random.uniform(0, 48), 4),
             'jacoco_line_coverage_rate': round(random.uniform(0, 48), 4),
             'blocker_issues': random.randint(0, 6),
             'major_issues': random.randint(0, 6),
             'critical_issues': random.randint(0, 6),
             'project_key': f"project_key_{counter}",
             'applicant_sn': random.choice(username_list),
             'jira_version': random.choice(['2.10.24', '7.5.13', '4.15.37', '9.8.5', '3.19.42']),
             'git_urls': f"https://gitlab.demo.com/git-number{counter}/git",
             'quality_scope': random.choice(['overall', 'newcode']),
             'dev_lang': random.choice(['java', 'python', 'golang']),
             'source': random.choice([0, 1]),
             'unit_username': random.choice(username_list),
             'pipeline_type': 'cd_pipeline',
             'pname': random.choice(product_list),
             'jenkins_template': random.choice(['dailyCiJava', 'dailyCiPython', 'dailyCiGolang']),
             'jacoco_lines_covered': random.randint(100, 1000),
             'LEVEL_1_DEVOPS_DEPT_CODE': LEVEL_1_DEVOPS_DEPT_CODE,
             'LEVEL_2_DEVOPS_DEPT_CODE': LEVEL_2_DEVOPS_DEPT_CODE,
             'LEVEL_3_DEVOPS_DEPT_CODE': LEVEL_3_DEVOPS_DEPT_CODE,
             'LEVEL_4_DEVOPS_DEPT_CODE': LEVEL_4_DEVOPS_DEPT_CODE
             }
            for counter in range(1, count)]
        )
    sess.commit()
sqlalchemy
1个回答
0
投票

您可以像在示例中使用 ORM 所做的那样实现 

for loop
。在循环中创建数据列表，然后可以将此列表传递给 
sess.execute()
:

def create_data(sess, engine, count=50,
                username_list=None, product_list=None, project_list=None, dept_list=None,
                organization_tree=None):
    core.table_action.create_table_if_not_exists(DwdDeployBranchData, engine)

    table_data_list = []

    for counter in range(1, count):
        init_time = datetime.strptime(
            (datetime.now() - timedelta(days=random.randrange(365))).strftime("%Y-%m-%d %H:%M:%S"), "%Y-%m-%d %H:%M:%S")
        LEVEL_1_DEVOPS_DEPT_CODE = get_random_node_code(organization_tree)
        LEVEL_2_DEVOPS_DEPT_CODE = get_random_node_code(organization_tree[LEVEL_1_DEVOPS_DEPT_CODE])
        LEVEL_3_DEVOPS_DEPT_CODE = get_random_node_code(
            organization_tree[LEVEL_1_DEVOPS_DEPT_CODE][LEVEL_2_DEVOPS_DEPT_CODE])
        LEVEL_4_DEVOPS_DEPT_CODE = get_random_node_code(
            organization_tree[LEVEL_1_DEVOPS_DEPT_CODE][LEVEL_2_DEVOPS_DEPT_CODE][LEVEL_3_DEVOPS_DEPT_CODE])


        table_data_list.append(
            {
                'pkey': random.choice(product_list),
                'user_name': random.choice(username_list),
                'qa_version': init_time.strftime("%Y-%m-%d"),
                'updated_time': init_time,
                'created_time': init_time,
                'build_exec_time': init_time,
                'tests_failed': 0,
                'tests_skipped': 0,
                'tests_total': 0,
                'is_incoco': random.choice(['true', 'false']),
                'jacoco_branch_coverage_rate': round(random.uniform(0, 48), 4),
                'jacoco_line_coverage_rate': round(random.uniform(0, 48), 4),
                'blocker_issues': random.randint(0, 6),
                'major_issues': random.randint(0, 6),
                'critical_issues': random.randint(0, 6),
                'project_key': f"project_key_{counter}",
                'applicant_sn': random.choice(username_list),
                'jira_version': random.choice(['2.10.24', '7.5.13', '4.15.37', '9.8.5', '3.19.42']),
                'git_urls': f"https://gitlab.demo.com/git-number{counter}/git",
                'quality_scope': random.choice(['overall', 'newcode']),
                'dev_lang': random.choice(['java', 'python', 'golang']),
                'source': random.choice([0, 1]),
                'unit_username': random.choice(username_list),
                'pipeline_type': 'cd_pipeline',
                'pname': random.choice(product_list),
                'jenkins_template': random.choice(['dailyCiJava', 'dailyCiPython', 'dailyCiGolang']),
                'jacoco_lines_covered': random.randint(100, 1000),
                'LEVEL_1_DEVOPS_DEPT_CODE': LEVEL_1_DEVOPS_DEPT_CODE,
                'LEVEL_2_DEVOPS_DEPT_CODE': LEVEL_2_DEVOPS_DEPT_CODE,
                'LEVEL_3_DEVOPS_DEPT_CODE': LEVEL_3_DEVOPS_DEPT_CODE,
                'LEVEL_4_DEVOPS_DEPT_CODE': LEVEL_4_DEVOPS_DEPT_CODE
             }
        )

    sess.execute(
        TableData.__table__.insert(),
        table_data_list
    )
    sess.commit()
最新问题
© www.soinside.com 2019 - 2024. All rights reserved.