如何批处理.commit() 来存储大量文档？

Question

我有大约 31,000 个文档发送给 batch.commit()。

我正在使用 Blaze 计划。

一个批次最多可以携带 500 个文档。所以，我用 490 个文档分批。我有 65 个批次。

这是我的firebase函数代码：

'use strict';

const express = require('express');
const cors = require('cors');
const axios = require('axios');

// Firebase init
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
const firestore = admin.firestore();


const echo = express().use(cors()).post("/", (request, response) => {
    axios.get('https://example.com/api').then(result => {

        const data = result.data.items;
        let batchArray = [];
        let batchIndex = 0;
        let operationCounter = 0;

        //initiate batch;
        batchArray.push(firestore.batch());

        data.forEach(item => {

            const collectionRef = firestore.collection('items').doc();

            const row = {
                itemName: item.name,
                // ... and so on...
            };


            batchArray[batchIndex].set(collectionRef, row);
            operationCounter++;

            if (operationCounter === 490) {
                batchArray.push(firestore.batch());
                functions.logger.info(`Batch index added.`, batchIndex);
                batchIndex++;
                operationCounter = 0;
            }

        });

        /*  
        This code wrote only 140 documents.
        Throws Error: 4 DEADLINE_EXCEEDED: Deadline exceeded

        
        batchArray.forEach(batch => {
            batch.commit()
                .then(result=> functions.logger.info("batch.commit() succeeded:", result) )
                .catch(error=>functions.logger.info("batch.commit() failed:", error));
        })

        */

        /* 
        This code wrote only 630 documents 
        Throws Error: 4 DEADLINE_EXCEEDED: Deadline exceeded

        Promise.all([
            batchArray.forEach(batch => {
                setTimeout(
                    ()=>batch.commit().then(result=> functions.logger.info("batch.commit() succeeded:", result) ).catch(error=>functions.logger.info("batch.commit() failed:", error)),
                    1000);
            })
        ]).catch(error => functions.logger.error("batch.commit() error:", error));

        */
        // This code wrote 2100 documents.
        return Promise.all([
            batchArray.forEach(batch => {
                batch.commit()
                    .then(result => functions.logger.info("batch.commit() succeeded:", result))
                    .catch(error => functions.logger.warn("batch.commit() failed:", error))
            })
        ]).then(result => {
            functions.logger.info("all batches succeeded:", result);
            return response.status(200).json({ "status": "success", "data": `success` });
        })
        .catch(error => {
            functions.logger.warn("all batches failed:", error);
            return response.status(200).json({ "status": "error", "data": `${error}` });
        });


    }).catch(error => {
        functions.logger.error("HTTPS Response Error", error);
        return response.status(204).json({ "status": "error", "data": `${error}` });

    });
});


exports.echo = functions.runWith({
    timeoutSeconds: 60 * 9,
}).https.onRequest(echo);

几秒钟后我收到了“成功”的回复。但是插入的firestore数据在7分钟后才出现，并且在云函数日志中显示了错误日志，65批次中有5批次成功。

抛出的错误是：

batch.commit() failed: { Error: 4 DEADLINE_EXCEEDED: Deadline exceeded 
at Object.callErrorFromStatus (/workspace/node_modules/@grpc/grpc-js/build/src/call.js:31:26) 
at Object.onReceiveStatus (/workspace/node_modules/@grpc/grpc-js/build/src/client.js:176:52) 
at Object.onReceiveStatus (/workspace/node_modules/@grpc/grpc-js/build/src/client-interceptors.js:342:141) 
at Object.onReceiveStatus (/workspace/node_modules/@grpc/grpc-js/build/src/client-interceptors.js:305:181) 
at process.nextTick (/workspace/node_modules/@grpc/grpc-js/build/src/call-stream.js:124:78) 
at process._tickCallback (internal/process/next_tick.js:61:11) 
Caused by: Error at WriteBatch.commit (/workspace/node_modules/@google-cloud/firestore/build/src/write-batch.js:419:23) 
at Promise.all.batchArray.forEach.batch (/workspace/index.js:100:23) 
at Array.forEach (<anonymous>) at axios.get.then.result (/workspace/index.js:99:24) 
at process._tickCallback (internal/process/next_tick.js:68:7) code: 4, details: 'Deadline exceeded', metadata: Metadata { internalRepr: Map {}, options: {} }, note: 'Exception occurred in retry method that was not classified as transient' }

错误Error: 4 DEADLINE_EXCEEDED 可能与firestore 配额有关。但我不知道哪个限制与这个问题有关。

Answer 1

这很可能是因为您正在执行commit 的forEach 没有按预期工作。一次又一次，awaits 和 forEach 函数会导致这样的问题。很久以前，我认为既然await 在forEach 中，它会等到它完成，然后转到数组中的下一项，但事实并非如此。它将一次运行它们。我建议使用传统的 for 循环。

另外，我建议不要使用 .then 语法。在这个原因中，它仍然会同时运行它们。尝试将 await 与传统的 for 循环一起使用。这将解决您的问题。

另一件事，你的 Promise.all 在这里没有帮助。 Promise.all 用于同时运行多个命令，但是由于超出了错误，您需要一次运行它们（我知道，因为您有这么多，这很糟糕）。

for (const batch of batchArray) {
  await batch.commit()
}

我不确定在您获得超出数量之前需要多少次提交（使用上述方法），但我很好奇您是否一次执行 2-3 次提交或其他什么。但是，通常最好一次做一个。