node.js 使用 fs writestream 异步下载多个文件

Question

在第一个问题没有回答 Node.js Javascript don't download images from array 之后（你可以在评论中看到我的第一个代码）我已经解决了我的代码并搜索了其他有我同样问题的人（在这里找到了没有答案 Node.js Downloading multiples files asynchronously）所以我''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''ve seen that he has引入了一个下载队列以尝试进行异步回调，但试图让''''首先下载''''''''''''''''''''''''''''ve seen that he has引入了一个下载队列以尝试执行异步回调，因此我试图找出可能是什么问题，也许程序没有先完成writestream before 开始另一个，所以我尝试过file.on("close", function() { 之类的东西，如果不是"close" 至少是"end" 但这只是阻止尝试下载第一个错误的文件。所以我的问题是我在回调中缺少什么？为什么只下载最后一个文件？

        const FILEPATH = "mypath/myfolder/" 
        const HREFSFILENAME = "urls.txt"
        var NodeHelper = require("node_helper")
        var fs = require("fs");
        var http = require('http');
        var urls;/*[] = new Array();*/
        var url = "";
        filename = "";
        var regex    = /(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?/;

module.exports = NodeHelper.create({        

      start: function() {
        fs.readFile(FILEPATH + HREFSFILENAME, "utf8", function (err,data) {
          if (err) return console.log(err);
          urls = data.split("\n")
          for (i=0; i < urls.length; i++) {
            console.log("ROW" + i + ": " + urls[i])
            url = urls[i]
            if (!url.match(regex)) continue;
            filename = FILEPATH + url.substring(url.lastIndexOf('/') + 1)
            downloadQueue.addItem(url, filename);
          }

        /*
          for (let i=0; i < urls.length; i++) {
              var request = http.get(urls[i], function(response) {
                console.log("GET" + i)
                var filename = urls[i]
                filename = filename.substr(filename.lastIndexOf("/") + 1)
                console.log("FILENAME: " + filename)
                filename = FILEPATH + filename
                var file = fs.createWriteStream(filename);
                response.on("end", function() {
                  file.end();
                });
                response.pipe(file);
              });
          }*/
        });
      },
    })
            var downloadQueue = {
                queue: [],
                addItem: function(p_sSrc, p_sDest) {
                    this.queue.push({
                        src: p_sSrc,
                        dest: p_sDest
                    });
                    if (this.queue.length === 1) {
                        this.getNext();
                    }
                },
                getNext: function() {
                    var l_oItem = this.queue[0];
                    http.get(l_oItem.src, function(response) {
                        console.log("Downloading: " + l_oItem.dest);
                        var file = fs.createWriteStream(l_oItem.dest);
                        response.on("end", function() {
                            file.end()
                            console.log("Download complete.");
                            //file.on("close", function() {
                              downloadQueue.removeItem();
                            //});
                        }).on("error", function(error) {
                            console.log("Error: " + error.message);
                            fs.unlink(l_oItem.dest);
                        });
                        response.pipe(file);
                    });
                },
                removeItem: function() {
                    this.queue.splice(0, 1);
                    if (this.queue.length != 0) {
                        this.getNext();
                    } else {
                        console.log("All items downloaded");
                    }
                }
            };

我的日志实际上是这个，你可以看到它只下载最后一张图片

    [16:45:51.266] [LOG]    ROW0: http://link1/image1.jpg
    [16:45:51.271] [LOG]    ROW1: http://link2/image2.png
    [16:45:51.271] [LOG]    ROW2: http://link3/image3.png
    [16:45:52.336] [LOG]    Downloading: myPath/myfolder/image1.jpg
[18:10:58.456] [LOG]    Download complete.
[18:10:58.459] [LOG]    Whoops! There was an uncaught exception...
[18:10:58.459] [ERROR]  { [Error: ENOENT: no such file or directory, open 'C:\Users\kenzo\myPath\myfolder\image1.jpg
  errno: -4058,
  code: 'ENOENT',
  syscall: 'open',
  path:
   'C:\\Users\\kenzo\\mypath\\myfolder\\image1.jpg\r' }
[18:10:58.736] [LOG]    Downloading: myPath/myfolder/image2.png
[18:10:58.736] [LOG]    Download complete.
[18:10:58.737] [LOG]    Whoops! There was an uncaught exception...
']8:10:58.737] [ERROR]  { [Error: ENOENT: no such file or directory, open 'C:\Users\kenzo\myPath\myfolder\image2.png
  errno: -4058,
  code: 'ENOENT',
  syscall: 'open',
  path:
   'C:\\Users\\kenzo\\myPath\\myfolder\\image2.png\r' }
[18:10:58.994] [LOG]    Downloading: mmyPath/myfolder/image3.png
[18:10:59.026] [LOG]    Download complete.
[18:10:59.026] [LOG]    All items downloaded

Answer 1

看起来像getNext 函数的问题，你总是使用queue[0]

getNext: function() {
           var l_oItem = this.queue[0];

您可以使用承诺的基础链来解决异步问题。这里readFile 代表异步读取操作。 ELse 使用 async-await 来使这个更干净。

// Solution 3.1: Using promise
const promiseReadFile = (name) => {
    return new Promise((resolve, _) => {
        readFile('promise', name, resolve)
    })
}
const p1 = promiseReadFile(files[0])
const p2 = promiseReadFile(files[1])
const p3 = promiseReadFile(files[2])
console.time("PROMISE: TOTAL_TIME")
p1
    .then(console.log)
    .then(() => p2)
    .then(console.log)
    .then(() => p3)
    .then(console.log)
    .then(console.timeEnd.bind(null, "PROMISE: TOTAL_TIME"))
// Solution 3.2: Using promise accumulator
console.time("PROMISE-V2: TOTAL_TIME")
files
    .map(promiseReadFile)
    .reduce((chain, pr) => {
        return chain.then(() => pr)
            .then(console.log)
    }, Promise.resolve())
    .then(console.timeEnd.bind(null, "PROMISE-V2: TOTAL_TIME"))

更多：https://gist.github.com/deepakshrma/1a899bf49ccb77d3b5be771907b32e4c