任务太多时谁nodejs http请求失败？

Question

我应该通过nodejs登录并调用http一些api点，

var mins = 150;
var url = require('url');
const domainim= 'example.com';
const request = require('requestretry').defaults({
        jar : true,
        timeout : 120000
    });
const req = require('request');
var fs = require('fs');
var linkler = [];
var timeoutsure = 120 * 1000;
var separateReqPool = {
    maxSockets : 20
};
let success = 0, fail = 0, toplam = 0, plusplus = 0;
let adet = 0;
let jardata = {};
let baslama = Math.round(new Date().getTime() / 1000);

function texts(l) {
    var t = "";
    var p = "abcdefghijklmnopqrstuvwxyz0123456789";
    for (var i = 0; i < l; i++) {
        t += p.charAt(Math.floor(Math.random() * p.length));
    }
    return t;
}

fs.readFile("data.txt", function (err, data) {
    if (err)
        throw err;
    data = data.toString().replace("\r", "");
    data = data.split(/\r\n|\r|\n/g);
    console.log("toplam data : " + data.length);
    toplam = data.length;
    data.forEach(
        function (line) {
        isle(line);
    });
});

function onExit() {
    console.log('\n%d sec,\n%d total,\n%d suc,\n%d fai,\n%d plusplus,\n---------------------\n',
        (Math.round(new Date().getTime() / 1000)) - baslama, toplam, success, fail, plusplus);
}
process.on('exit', onExit);

function dataok(id, pass, bakiye) {
    console.log(id + " " + pass + " " + bakiye);
    var fd = fs.openSync("hardblock.txt", 'a+');
    fs.writeSync(fd, id + " " + pass + " " + bakiye + "\r\n");
    fs.closeSync(fd);

}

function extradata(id, pass, bakiye) {
    console.log(id + " " + pass + " " + bakiye);
    var fd = fs.openSync("harddata.txt", 'a+');
    fs.writeSync(fd, id + " " + pass + " " + bakiye + "\r\n");
    fs.closeSync(fd);

}

function isle(line) {
    jardata[line.split(" ")[0]] = req.jar();
    request({
        url : 'http://'+domainim+'/',
        jar : jardata[line.split(" ")[0]],
        headers : {
            'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
        },
        maxAttempts : 15,
        retryDelay : 2000,
        followRedirect : true,
        //  proxy :"http://127.0.0.1:38936",
        timeout : timeoutsure,
        pool : separateReqPool
    }, function optionalCallback(err, httpResponse, body) {
        if (httpResponse && httpResponse.attempts) {
            adet += httpResponse.attempts;
        }
        if (err) {
            return console.error('failed: 49', err);
        } else {

            data = {
                username : line.split(" ")[0],
                password : line.split(" ")[1],
                submit : "Giriş"
            };
            request.post({
                url : 'http://'+domainim+'/dologin/',
                jar : jardata[line.split(" ")[0]],
                header : httpResponse.headers,
                headers : {
                    'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
                },
                maxAttempts : 15,
                retryDelay : 2000,
                followRedirect : false,
                followAllRedirects : false,
                //  proxy :"http://127.0.0.1:38936",
                timeout : timeoutsure,
                pool : separateReqPool,
                form : data
            }, function optionalCallback(err, httpResponse, body) {
                if (httpResponse && httpResponse.attempts) {
                    adet += httpResponse.attempts;
                }
                if (err) {
                    return console.error('failed: 49', err);
                } else {

                    request({
                        url : 'http://'+domainim+'/',
                        jar : jardata[line.split(" ")[0]],
                        headers : {
                            'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
                        },
                        maxAttempts : 15,
                        retryDelay : 2000,
                        followRedirect : true,
                        //  proxy :"http://127.0.0.1:38936",
                        timeout : timeoutsure,
                        pool : separateReqPool
                    }, function optionalCallback(err, httpResponse, body) {
                        if (httpResponse && httpResponse.attempts) {
                            adet += httpResponse.attempts;
                        }
                        if (err) {}
                        else {
                            if (body.indexOf('class="success"') > -1) {
                                dataok(line.split(" ")[0], line.split(" ")[1], body.split('class="succes" title="')[1].split('"')[0]);
                                success++;
                                if (parseFloat(body.split('class="succes" title="')[1].split('"')[0]) >= mins) {
                                    plusplus++;
                                    extradata(line.split(" ")[0], line.split(" ")[1], body.split('class="succes" title="')[1].split('"')[0]);
                                } else {

                                    request({
                                        url : 'http://'+domainim+'/refresh/',
                                        jar : jardata[line.split(" ")[0]],
                                        headers : {
                                            'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
                                        },
                                        maxAttempts : 15,
                                        retryDelay : 2000,
                                        followRedirect : true,
                                        //  proxy :"http://127.0.0.1:38936",
                                        timeout : timeoutsure,
                                        pool : separateReqPool
                                    }, function optionalCallback(err, httpResponse, body) {
                                        if (httpResponse && httpResponse.attempts) {
                                            adet += httpResponse.attempts;
                                        }
                                        if (err) {}
                                        else {
                                        if (parseFloat(body)>= mins){
                                        plusplus++;
                                        extradata(line.split(" ")[0], line.split(" ")[1], parseFloat(body));
                                        }
                                        }
                                    });
                                }
                            } else {
                                fail++;
                            }
                        }
                    });

                }
            });

        }
    });

}

当我在 data.txt 中的数据是 5,10,1000 时，代码可以正常工作。

但是当数据变得更大，例如 15,000 或 50,000 或更大时

它只能以 %10 的成功率进行 http 调用。和其他人失败并给我这样的错误：

failed: 49  Error: connect EADDRINUSE 103.253.186.202:80
    at Object._errnoException (util.js:1031:13)
    at _exceptionWithHostPort (util.js:1052:20)
    at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1195:14)
  errno: 'EADDRINUSE',
  code: 'EADDRINUSE',
  syscall: 'connect',
  address: '1.2.3.4',
  port: 80 

我可能做错了什么？ nodejs不是有能力做到这一点吗？

jar 不能按预期工作，它有时会使用其他会话作为错误

Answer 1

您不应该尝试一次进行 15,000 个 http 调用。就同​​时使用的资源而言，这对于您的服务器来说实在是太多了，如果请求都发往同一台主机，那么这可能比单个主机一次可以处理的要多。我猜想 EADDRINUSE 正是你得到的，因为当你尝试太多同时请求时，一些资源已经耗尽。

相反，您应该一次运行数量少得多的请求（您可以试验 10 到 50 之间的最佳性能）。还有许多其他答案显示了如何做到这一点。一些使用 Bluebird 或 Async 等外部库，而另一些则手动对其进行编码。这里有几个例子：

Run 1000 requests so that only 10 runs at a time

How to make millions of parallel http requests from nodejs app?

Node js socket explanation

Error: connect ETIMEDOUT when scraping

Make several requests to an API that can only handle 20 request a minute

Promise.all consumes all my RAM