迭代 node.js 请求函数

Question

这个问题是关于 node.js 中的爬虫的。 一个start_url 在他爬取URL 的地方给出，并将它们“推送”到一个.json-文件(output.json)。 目前，他只用start_url运行请求函数，并将收集到的URL保存在output.json中。我希望他通过将 start_url 替换为第一个收集的 URL 并再次收集链接来使用保存的 URL……等等……

var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');

var start_url = ["http://stackoverflow.com/"]

var req = function(url){
    request(url, function(error, response, html){
        var $ = cheerio.load(html);

        var data = [];

        $("a").each(function() {
            var link = $(this);
                var exurls = {exurl: new Array(link.attr("href"))}

                data.push(exurls);

                // Queue "exurls" for "start_url" and call the same function with the new URL (endless loop)
                // save to "output.json" from time to time, so you can stop it anytime
        });

        fs.writeFile("output.json", JSON.stringify(data, null, 4), function(err){
            if(err){
                console.log(err);
            } else {
                console.log("File successfully written!");
            }
        });
    });
}
for (var i = 0; i < start_url.length; i++){
    req(start_url[i]);
}

Answer 1

所以你可以做的是递归调用函数。下面的例子应该可以工作：

var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');

var start_url = ["http://stackoverflow.com/"]

var req = function(url){
  var count = 0;
  
  request(url, function(error, response, html){
    var $ = cheerio.load(html);

    $("a").each(function() {
      var link = $(this);
      var exurls = {exurl: new Array(link.attr("href"))}

      start_url.push(exurls);

      // Queue "exurls" for "start_url" and call the same function with the new URL (endless loop)
      // save to "output.json" from time to time, so you can stop it anytime
    });

    try {
      fs.writeFileSync("output.json");
      console.log("File successfully written!");
     }catch(err){
       console.log(err);
     }
      
      ++count;
      
      if(start_url.length > count) {
        req(start_url[count]);
      }
  });
}

return req(start_url[0]);

这样做的问题是您每次都在完全重写文件。如果这种情况持续一段时间，您将耗尽内存。另一种选择是创建一个写入流

var fs = require('fs');
    var request = require('request');
    var cheerio = require('cheerio');

    var start_url = ["http://stackoverflow.com/"]
    
    var wstream = fs.createWriteStream("output.json");

    var req = function(url){
      
      request(url, function(error, response, html){
        var $ = cheerio.load(html);

        $("a").each(function() {
          var link = $(this);
          var exurls = {exurl: new Array(link.attr("href"))}

          start_url.push(exurls);

          // Queue "exurls" for "start_url" and call the same function with the new URL (endless loop)
          // save to "output.json" from time to time, so you can stop it anytime
          wstream.write('"'+ exurls + '",');
        });
          
        start_url.shift();
        if(start_url.length > 0) {
          return req(start_url[0]);
        }
          
          wstream.end();
      });
    }

    req(start_url[0]);

编辑：切换到基本队列以解决内存问题

---

---