nodejs中http请求的迭代

Question

我正在尝试在 nodeJs 中发出多个 http get 请求。我需要继续获取记录，直到没有更多可用记录为止。我在尝试对一大段代码进行迭代时遇到问题。我一直在阅读有关异步模块的信息，但我不明白如何应用它。请帮忙。

这是新代码11/25/2014： 我要重复的步骤是 GetUrl(newBlock) 是我要重复的步骤，直到没有更多块为止。当请求响应的 result.lenght 小于 5,000 时，我确定没有更多的块。

如果我使用 while 异步模块，有没有办法让我从 newBlock 函数中获取这个参数包，然后停止循环？

// JS Script:        GPSAPIClient
// Code Description: API client code to retrieve GPS data for customer with account number: 47631
//                   This script requires Node.Js environment installed with Request() and Node-crontab modules (through NPM).
//                   It will run (as a backend script) every minute and retrieve the GPS available.
//
// Author :          Vanessa Torres, Professional Services
var request = require("request");
var fs = require('fs');
var async = require('async');
var crontab = require('node-crontab');
var csvFile = "";


var debug = false;
process.argv.forEach(function(val, idx, array) {
    if (val.toLowerCase() === 'debug') {
        debug = true;
        console.log('\n\n\n******************DEBUG MODE ACTIVE(Do not run in prod)******************\n\n\n')
    }
});

console.log('waiting to start running');


var jobId = crontab.scheduleJob('* * * * *', function() {
    //Gettting  system date and adding leading zeros when are single digits.  I need this to build the get request with date filters.


    var d = new Date();
    var nday = d.getDate();
    var nmonth = d.getMonth();
    var nhour = d.getHours();
    var nmin = d.getMinutes();

    var nfullyear = d.getFullYear();
    if (nday < 10) {
        nday = '0' + nday;
    };
    //1 minute substraction except for when is zero (the value will negative).  
    if (nmin != 0) {
        var nmin = nmin - 1;
    };
    if (nmin < 10) {
        nmin = '0' + nmin;
    };
    // added because TLC Plumbing is 2 hours behind us. (MST)
    nhour = nhour - 2;
    if (nhour < 10) {
        nhour = '0' + nhour;
    };

    var nmonth = nmonth + 1;
    if (nmonth < 10) {
        nmonth = '0' + nmonth;
    };

    var options = {
        //url: 'https://credentials@api.comettracker.com/v1/gpsdata' + '?fromdate=' + nfullyear + '-' + nmonth + '-' + nday + 'T' + nhour + '%3a' + nmin + '%3a' + '00',
        url: 'https://credentials@api.comettracker.com/v1/gpsdata?fromdate=2015-11-23T17%3a00%3a00&todate=2015-11-24T10%3a00%3a00',
        method: 'GET',
        rejectUnauthorized: !debug
    };

    function GetUrl(callback) {

        return request(options, function(error, response, body) {
            console.log('request for links');
            if (error) throw new Error(error);
            var result = JSON.parse(body)['links'];
            var urlnext = result[2].href;
            console.log('UrlNext: ' + urlnext);
            console.log(result[2].rel);
            //moreBlocks = newBlock(urlnext);

            moreBlocks = callback(urlnext);
        });
        // request to get the next block of records (maximun 5,000)
    };


    // HTTP get request - 1st request
    request(options, function(error, response, body) {
        if (error) throw new Error(error);
        var result = JSON.parse(body)['gps-recs'];
        console.log(result.length);
        //console.log(result);


        //create .csv file if result.length is > = 1 (If we received GPS records)
        if (result.length >= 1 && result.length < 5000) {
            console.log('entered first if, result.length: ' + result.length);
            buildCSV(result);
        };

        //add the subsequent request when result.length = 5000

        if (result.length == 5000) {
            console.log('entered second if, result.length: ' + result.length);
            buildCSV(result);
            console.log('I came back from buildCSV.  result.length is : ' + result.length);
            **GetUrl(newBlock)**;
            //console.log('moreblocks back from newblock: ' + moreBlocks);
        };
    });
});

function newBlock(urlnext) {
    console.log('URL passed to newBlock: ' + urlnext);
    // option 2 - variables needed to built the url (like the first request)
    var n = urlnext.length;
    var rest = urlnext.substr(40, n);
    console.log('Rest: ' + rest);

    var options = {
        url: 'https://credentials@api.comettracker.com/v1/gpsdata/' + rest,
        method: 'GET',
        rejectUnauthorized: !debug
    };

    request(options, function(error, response, body) {
        console.log('nextblock request');
        if (error) throw new Error(error);
        var result = JSON.parse(body)['gps-recs'];
        if (result.length == 0) {
            //no records for filter
            console.log('first if' + result.length);
            moreBlocks = false;
        };

        if (result.length < 5000 && result.length > 0) {
            //last block appends record and ends
            appendCSV(result);
            moreBlocks = false;
            console.log('second if' + result.length);
        };
        if (result.length == 5000) {
            //appends records but has to keep running
            appendCSV(result);
            console.log('third if- moreBlocks' + result.length);
        };
        console.log('moreBlocks: ' + moreBlocks);
    });

};

function buildCSV(result) {

    var csvFile = " ";
    console.log('before csvFile: ' + csvFile);
    //adding headers
    csvFile = csvFile.concat('UserNumber' + ',' + 'UserTimeTag' + ',' + 'Latitude' + ',' + 'Longitude' + ',' + 'SpeedMph' + ',' + 'Heading' + ',' + 'Status' + '\r\n');
    // loop runs result.length times
    for (var i = 0; i < result.length; i++) {
        csvFile = csvFile.concat(result[i].UserInfo.UserNumber + ',' + result[i].UserTimeTag + ',' + result[i].Latitude + ',' + result[i].Longitude + ',' + result[i].SpeedMph + ',' + result[i].Heading + ',' + result[i].Status + '\r\n');

    };
    //console.log(csvFile);  
    fs.writeFile('file.csv', csvFile, function(err) {

        if (err) throw err;
        console.log('file saved');

    });
};



//appending additional blocks 
function appendCSV(result) {
    var csvString = " ";
    // loop runs result.length times
    for (var i = 0; i < result.length; i++) {
        csvString = csvString.concat(result[i].UserInfo.UserNumber + ',' + result[i].UserTimeTag + ',' + result[i].Latitude + ',' + result[i].Longitude + ',' + result[i].SpeedMph + ',' + result[i].Heading + ',' + result[i].Status + '\r\n');

    };
    // console.log(csvString);  
    fs.appendFile('file.csv', csvString, function(err) {

        if (err) throw err;
        console.log('Data appended');

    });
};

Answer 1

很难看出moreBlocks 是否与while 在同一范围内。另一方面，while 语句将开始无限期地调用，直到标志发生变化，这可能为时已晚。您需要确保一次完成每个后续请求。

您可以使用 async 的“while”功能：

var moreBlocks = true;

whilst(function(){
  return moreBlocks === true;
},function(callback){
  return request(options, function(err, res, body){
    console.log('request for links');
    if (err) throw new Error(err);
    var result = JSON.parse(body)['links'];
    var urlnext = result[2].href;
    console.log('UrlNext: ' + urlnext);
    console.log(result[2].rel);

    //You NEED to make sure you update moreBlocks
    moreBlocks = newBlock(urlnext);

    // Go to next step.
    return callback(res);

  })
},{
  //You are done!
  //Call any function you'd like.
});

查看文档here！

编辑，如果 newBlock 是异步的：

var moreBlocks = true;

whilst(function(){
  return moreBlocks === true;
},function(callback){
  return request(options, function(err, res, body){
    console.log('request for links');
    if (err) throw new Error(err);
    var result = JSON.parse(body)['links'];
    var urlnext = result[2].href;
    console.log('UrlNext: ' + urlnext);
    console.log(result[2].rel);

    //Now status has the info about moreBlocks.
    return newBlock(urlnext, function(status){
      moreBlocks = status;
      // Go to next step.
      return callback(res);
    });       

  })
},{
  //You are done!
  //Call any function you'd like.
});

newBlock 上的某处：

function newBlock(urlnext, callback) {
      // some code..
      return request(options, function(){
        //here status decides if we are done.
        return callback(status);
      });
    }