Node.js 网络抓取与 URL 数组上的循环

Question

我正在尝试构建一个小脚本来废弃一些数据。我对 javascript 有一些基础知识，但是我对所有异步回调或承诺的东西有点迷失了。这是我现在拥有的：

url = "http://Blablablabla.com";

var shares = function(req, res) {
    request(url, function (error, response, body) {
  if (!error) {
    var $ = cheerio.load(body),
      share = $(".theitemIwant").html();

    return res.send(url + ":" + share);
  } else {
    console.log("We've encountered an error: " + error);
  }
})

}

所以这段代码一切正常。我想做的是：

1. 使用 url 数组 var urls = [url1,url2,url3,etc...]
2. 将我的报废数据存储到另一个数组中，类似于 data = [{url: url1, shares: share},{url: url2, shares: share},etc...]

我知道我需要使用这样的东西data.push({ urls: url, shares: share})})

并且我知道我需要遍历我的第一个 url 数组以将数据推送到我的第二个数据数组中。

但是我有点迷失了request 方法以及在我的情况下我应该处理异步问题的方式。

谢谢！

编辑#1：

我试过这个来使用承诺：

var url = "www.blablabla.com"
var geturl = request(url, function (error, response, body) {
  if (!error) { return $ = cheerio.load(body) } else 
  { console.log("We've encountered an error: " + error); }
});

var shares = geturl.then( function() {
    return $(".nb-shares").html();
})

但出现以下错误geturl.then is not a function

Answer 1

我认为你应该使用async:

var async = require('async');

var urls = ["http://example.com", "http://example.com", "http://example.com"];
var data = [];
var calls = urls.map((url) => (cb) => {
    request(url, (error, response, body) => {
        if (error) {
            console.error("We've encountered an error:", error);
            return cb();
        }
        var $ = cheerio.load(body), 
            share = $(".theitemIwant").html();
        data.push({ url, share })
    })
})

async.parallel(calls, () => { /* YOUR CODE HERE */ })

你可以对 Promise 做同样的事情，但我不明白为什么。

Answer 2

我试了一下。你需要安装q库并要求它来

var Q = require('q');

//... where ever your function is
//start with an array of string urls
var urls = [ "http://Blablablabla.com", '...', '...'];

//store results in this array in the form:
//  { 
//       url: url, 
//       promise: <will be resolved when its done>, 
//       share:'code that you wanted'
//    }
var results = [];

//loop over each url and perform the request
urls.forEach(processUrl);

function processUrl(url) {
  //we use deferred object so we can know when the request is done
  var deferred = Q.defer();

  //create a new result object and add it to results
  var result = {
    url: url,
    promise: deferred.promise
  };
  results.push(result);


  //perform the request
  request(url, function (error, response, body) {
      if (!error) {
        var $ = cheerio.load(body),
          share = $(".theitemIwant").html();
        //resolve the promise so we know this request is done.
        //  no one is using the resolve, but if they were they would get the result of share
        deferred.resolve(share);
        //set the value we extracted to the results object
        result.share = share;
      } else {

        //request failed, reject the promise to abort the chain and fall into the "catch" block
        deferred.reject(error)
        console.log("We've encountered an error: " + error);
      }
  });
}

//results.map, converts the "array" to just promises
//Q.all takes in an array of promises
//when they are all done it rull call your then/catch block.
Q.all(results.map(function(i){i.promise}))
    .then(sendResponse) //when all promises are done it calls this
    .catch(sendError);  //if any promise fails it calls this

 function sendError(error){
   res.status(500).json({failed: error});
 }
 function sendResponse(data){ //data = response from every resolve call
  //process results and convert to your response
  return res.send(results);
}

Answer 3

这是我非常喜欢的另一个解决方案：

const requestPromise = require('request-promise');
const Promise = require('bluebird');
const cheerio = require('cheerio');

const urls = ['http://google.be', 'http://biiinge.konbini.com/series/au-dela-des-murs-serie-herve-hadmar-marc-herpoux-critique/?src=konbini_home']

Promise.map(urls, requestPromise)
  .map((htmlOnePage, index) => {
    const $ = cheerio.load(htmlOnePage);
    const share = $('.nb-shares').html();
    let shareTuple = {};
    shareTuple[urls[index]] = share;
    return shareTuple;
  })
  .then(console.log)
  .catch((e) => console.log('We encountered an error' + e));