在 Postgresql 中使用 Node.js 进行数据插入避免重复

Question

const { app } = require('@azure/functions');
const {Client} = require('pg');
const XLSX = require('xlsx');

app.storageBlob('storageBlobTrigger', {
    path: 'attachments/{name}',
    connection: 'MyStorageConnectionString',
    handler: async (blob,context) => {
        const connection = new Client( 
            {
                host: "++++++",
                user: "+++++",
                password: "++++",
                database: "++",
                port: 5432,
                ssl:{
                    rejectUnauthorized: false
                }
            });
            try{
                await connection.connect();
    
                context.log(`Storage blob function processed blob "${context.triggerMetadata.name}" with size ${blob.length} bytes`);
                console.log("excel data: ",blob)
    
                const workbook = XLSX.read(blob, { type: 'buffer' });
                const sheetName = workbook.SheetNames[0];
                const sheet = workbook.Sheets[sheetName];
    
                const range = XLSX.utils.decode_range(sheet['!ref']);
                range.s.r = 1; 
                sheet['!ref'] = XLSX.utils.encode_range(range);
    
                const data = XLSX.utils.sheet_to_json(sheet, { raw: false })
                console.log(data);


                for (let row of data) {
                    await connection.query(`
                    INSERT INTO stock2.stock (
                        project_prefix,
                        material_code,
                        producer_item_code,
                        description,
                        serial_numbers,
                        total_stock,
                        unit,
                        warehouse
                    ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
                    [
                    row['Project Prefix'],
                    row['Material Code'], 
                    row['Producer Item Code'],
                    row['Description'],
                    row['Serial Numbers'],
                    row['Total Stock'],
                    row['Unit'],
                    row['Warehouse']
                ]);
                }
                const result = await connection.query('Select * stock2.stock')
    
                console.log("Result: ",result.rows);
    
            }
            catch(error){
                console.log("Error: ",error);
            }
            finally{
                connection.end()
            }   
        }
    });    

我正在读取每日更新的 Excel 文件中的数据。我需要相应地更新数据库中的表。代码每次更新时都会添加行，但问题是最后有无限行，因为 Excel 文件有大约 10000 行。我做了一些研究，并结束了使用合并或插入......冲突，但我不确定这是否是最好的解决方案。有人可以帮忙看一下代码吗？

Answer 1

是的，您可以使用

MERGE

语句将数据插入到 Azure Postgresql DB 中，避免重复。下面是将数据从源表插入到 Azure Postgresql 数据库到目标表的示例。下面的源是

employees

表，目标是

newEmployees

表。

数据存储在源

employees

表中：

postgres=> SELECT * FROM Employees;

输出：

id |        email        |     name      
----+---------------------+---------------
  1 | [email protected]    | John Doe
  2 | [email protected]   | Alice Smith
  3 | [email protected]     | Bob Johnson
  4 | [email protected]    | John Smith
  5 | [email protected] | Charlie Brown
  6 | [email protected]   | Alice Cooper
(6 rows)

如上表所示，数据在

email

列中有重复值。下面的查询使用

MERGE

语句和

DISTINCT ON (email)

来避免重复。它从

employees

表中选择不同的电子邮件和姓名对，确保为每个唯一电子邮件值仅选择一行，并将新行插入到

newEmployees

表中以获取表中尚不存在的电子邮件值。

MERGE INTO newEmployees AS n
USING (
    SELECT DISTINCT ON (email) email, name
    FROM employees
    ORDER BY email
) AS e
ON n.email = e.email
WHEN NOT MATCHED THEN
    INSERT (email, name)
    VALUES (e.email, e.name);

postgres=> SELECT * FROM newEmployees;

输出：

postgres=> SELECT * FROM newEmployees;
 id |        email        |     name      
----+---------------------+---------------
  1 | [email protected]     | Bob Johnson
  2 | [email protected]   | Alice Smith
  3 | [email protected] | Charlie Brown
  4 | [email protected]    | John Doe
(4 rows)