在 Google Cloud Bucket Upload 之前解决 Promise

Question

我正在编写一些循环遍历 CSV 并创建 JSON 文件 based on the CSV 的代码。 JSON 中包含一个名为photos 的array，它包含函数内上传到Google Cloud Storage 的图像的返回url。但是，承诺等待上传完成让我很困惑，因为一切都是异步运行的，并且在完成存储桶上传并返回 url 之前完成了 promise 和 JSON 编译。在检索到 URL 并将其添加到 currentJSON.photos 后，如何使 promise 解析？

const csv=require('csvtojson')
const fs = require('fs');
const {Storage} = require('@google-cloud/storage');
var serviceAccount = require("./my-firebase-storage-spot.json");
const testFolder = './Images/';
var csvFilePath = './Inventory.csv';

var dirArr = ['./Images/Subdirectory-A','./Images/Subdirectory-B','./Images/Subdirectory-C'];
var allData = [];

csv()
.fromFile(csvFilePath)
.subscribe((json)=>{
  return new Promise((resolve,reject)=>{
    for (var i in dirArr ) {
      if (json['Name'] == dirArr[i]) {

        var currentJSON = {
          "photos" : [],
        };         

        fs.readdir(testFolder+json['Name'], (err, files) => {
          files.forEach(file => {
            if (file.match(/.(jpg|jpeg|png|gif)$/i)){
              var imgName = testFolder + json['Name'] + '/' + file;
              bucket.upload(imgName, function (err, file) {
                if (err) throw new Error(err);
                //returned uploaded img address is found at file.metadata.mediaLink
                currentJSON.photos.push(file.metadata.mediaLink);
              });              
            }else {
              //do nothing
            }
          });
        });
        allData.push(currentJSON);
      }
    }

    resolve(); 
  })
},onError,onComplete);

function onError() {
  // console.log(err)
}
function onComplete() {
  console.log('finito');
}

我尝试过移动resolve()，还尝试将上传器部分放入onComplete() 函数中（这会产生新的基于promise 的问题）。

Answer 1

你正在寻找这个图书馆 ELT。

您可以从 CSV 并行读取行并并行处理它们，而不是一个接一个地处理。

我试图解释下面代码中的行。希望这是有道理的。

const etl = require("etl");
const fs = require("fs");
const csvFilePath = `${__dirname }/Inventory.csv`;
const testFolder = "./Images/";

const dirArr = [
  "./Images/Subdirectory-A",
  "./Images/Subdirectory-B",
  "./Images/Subdirectory-C"
];

fs.createReadStream(csvFilePath)
  .pipe(etl.csv()) // parse the csv file
  .pipe(etl.collect(10)) // this could be any value depending on how many you want to do in parallel.
  .pipe(etl.map(async items => {
    return Promise.all(items.map(async item => { // Iterate through 10 items
      const finalResult = await Promise.all(dirArr.filter(i => i === item.Name).map(async () => { // filter the matching one and iterate
        const files = await fs.promises.readdir(testFolder + item.Name); // read all files
        const filteredFiles = files.filter(file => file.match(/\.(jpg|jpeg|png|gif)$/i)); // filter out only images
        const result = await Promise.all(filteredFiles).map(async file => {
          const imgName = `${testFolder}${item.Name}/${file}`;
          const bucketUploadResult = await bucket.upload(imgName); // upload image
          return bucketUploadResult.metadata.mediaLink;
        });
        return result; // This contains all the media link for matching files
      }));
      // eslint-disable-next-line no-console
      console.log(finalResult); // Return arrays of media links for files
      return finalResult;
    }));
  }))
  .promise()
  .then(() => console.log("finsihed"))
  .catch(err => console.error(err));

Answer 2

这是一种方法，我们将一些功能提取到一些单独的辅助方法中，并精简一些代码。我不得不推断您的一些要求，但这似乎与我对您原始代码意图的理解非常吻合：

const csv=require('csvtojson')
const fs = require('fs');
const {Storage} = require('@google-cloud/storage');
var serviceAccount = require("./my-firebase-storage-spot.json");
const testFolder = './Images/';
var csvFilePath = './Inventory.csv';

var dirArr = ['./Images/Subdirectory-A','./Images/Subdirectory-B','./Images/Subdirectory-C'];
var allData = [];

// Using nodejs 'path' module ensures more reliable construction of file paths than string manipulation:
const path = require('path');

// Helper function to convert bucket.upload into a Promise
// From other responses, it looks like if you just omit the callback then it will be a Promise
const bucketUpload_p = fileName => new Promise((resolve, reject) => {
  bucket.upload(fileName, function (err, file) {
    if (err) reject(err);

    resolve(file);
  });
});

// Helper function to convert readdir into a Promise
// Again, there are other APIs out there to do this, but this is a rl simple solution too:
const readdir_p = dirName => new Promise((resolve, reject) => {
  fs.readdir(dirName, function (err, files) {
    if (err) reject(err);

    resolve(files);
  });
});

// Here we're expecting the string that we found in the "Name" property of our JSON from "subscribe".
// It should match one of the strings in `dirArr`, but this function's job ISN'T to check for that,
// we just trust that the code already found the right one.
const getImageFilesFromJson_p = jsonName => new Promise((resolve, reject) => {
  const filePath = path.join(testFolder, jsonName);

  try {
    const files = await readdir_p(filePath);

    resolve(files.filter(fileName => fileName.match(/\.(jpg|jpeg|png|gif)$/i)));
  } catch (err) {
    reject(err);
  }
});

csv()
.fromFile(csvFilePath)
.subscribe(async json => {
  // Here we appear to be validating that the "Name" prop from the received JSON matches one of the paths that
  // we're expecting...?  If that's the case, this is a slightly more semantic way to do it.
  const nameFromJson = dirArr.find(dirName => json['Name'] === dirName);

  // If we don't find that it matches one of our expecteds, we'll reject the promise.
  if (!nameFromJson) {
    // We can do whatever we want though in this case, I think it's maybe not necessarily an error:
    // return Promise.resolve([]);
    return Promise.reject('Did not receive a matching value in the Name property from \'.subscribe\'');
  }

  // We can use `await` here since `getImageFilesFromJson_p` returns a Promise
  const imageFiles = await getImageFilesFromJson_p(nameFromJson);
  // We're getting just the filenames; map them to build the full path
  const fullPathArray = imageFiles.map(fileName => path.join(testFolder, nameFromJson, fileName));

  // Here we Promise.all, using `.map` to convert the array of strings into an array of Promises;
  // if they all resolve, we'll get the array of file objects returned from each invocation of `bucket.upload`
  return Promise.all(fullPathArray.map(filePath => bucketUpload_p(filePath)))
    .then(fileResults => {
      // So, now we've finished our two asynchronous functions; now that that's done let's do all our data
      // manipulation and resolve this promise

      // Here we just extract the metadata property we want
      const fileResultsMediaLinks = fileResults.map(file => file.metadata.mediaLink);

      // Before we return anything, we'll add it to the global array in the format from the original code
      allData.push({ photos: fileResultsMediaLinks });

      // Returning this array, which is the `mediaLink` value from the metadata of each of the uploaded files.
      return fileResultsMediaLinks;
    })
}, onError, onComplete);

Answer 3

问题是您的代码没有在您的forEach 中等待。我强烈建议寻找流并尝试尽可能并行地做事情。有一个库非常强大，可以为您完成这项工作。图书馆是etl。

您可以从 csv 中并行读取行并并行处理它们，而不是一一处理。

我试图解释下面代码中的行。希望这是有道理的。

const etl = require("etl");
const fs = require("fs");
const csvFilePath = `${__dirname }/Inventory.csv`;
const testFolder = "./Images/";

const dirArr = [
  "./Images/Subdirectory-A",
  "./Images/Subdirectory-B",
  "./Images/Subdirectory-C"
];

fs.createReadStream(csvFilePath)
  .pipe(etl.csv()) // parse the csv file
  .pipe(etl.collect(10)) // this could be any value depending on how many you want to do in parallel.
  .pipe(etl.map(async items => {
    return Promise.all(items.map(async item => { // Iterate through 10 items
      const finalResult = await Promise.all(dirArr.filter(i => i === item.Name).map(async () => { // filter the matching one and iterate
        const files = await fs.promises.readdir(testFolder + item.Name); // read all files
        const filteredFiles = files.filter(file => file.match(/\.(jpg|jpeg|png|gif)$/i)); // filter out only images
        const result = await Promise.all(filteredFiles).map(async file => {
          const imgName = `${testFolder}${item.Name}/${file}`;
          const bucketUploadResult = await bucket.upload(imgName); // upload image
          return bucketUploadResult.metadata.mediaLink;
        });
        return result; // This contains all the media link for matching files
      }));
      // eslint-disable-next-line no-console
      console.log(finalResult); // Return arrays of media links for files
      return finalResult;
    }));
  }))
  .promise()
  .then(() => console.log("finsihed"))
  .catch(err => console.error(err));

Answer 4

确实，您的代码没有等待异步调用 readdir 回调函数，也没有等待 bucket.upload 回调函数。

当您使用这些函数的 promise 版本时，异步编码变得更容易。

bucket.upload在省略回调函数时会返回一个promise，所以很简单。

要让readdir返回一个promise，你需要使用fs Promise API：然后你可以使用 基于promise的readdir方法和使用 整个代码中的承诺。

所以使用fs = require('fs').promises 而不是fs = require('fs')

通过这些准备，您的代码可以转换为：

const testFolder = './Images/';
var csvFilePath = './Inventory.csv';
var dirArr = ['./Images/Subdirectory-A','./Images/Subdirectory-B','./Images/Subdirectory-C'];

(async function () {
    let arr = await csv().fromFile(csvFilePath);
    arr = arr.filter(obj => dirArr.includes(obj.Name));
    let allData = await Promise.all(arr.map(async obj => {
        let files = await fs.readdir(testFolder + obj.Name);
        files = files.filter(file => file.match(/\.(jpg|jpeg|png|gif)$/i));
        let photos = await Promise.all(
            files.map(async file => {
                var imgName = testFolder + obj.Name + '/' + file;
                let result = await bucket.upload(imgName);
                return result.metadata.mediaLink;
            })
        );
        return {photos};
    }));
    console.log('finito', allData);
})().catch(err => {  // <-- The above async function runs immediately and returns a promise
    console.log(err);
});

一些备注：

• 您的正则表达式存在缺陷。你打算匹配一个文字点，但你没有转义它（在上面的代码中修复）。

• allData 将包含一组{ photos: [......] } 对象，我想知道为什么您不希望所有照片元素都成为一个数组的一部分。但是，我保留了您的逻辑，因此上述内容仍将在这些块中生成它们。可能您还打算拥有其他属性（在photos 旁边），这使得拥有这些单独的对象实际上很有用。