因为for 循环同步运行并且您对needle() 的调用是异步的，因此不会阻塞，您最终会尝试同时启动超过100,000 个网络请求。这会使您的本地计算机或目标服务器不堪重负，并且您开始收到套接字错误。
对于这么多请求，您需要一次运行 X 个请求，因此同时运行的请求不超过 X 个。为了最大限度地提高性能，您必须弄清楚要使用的 X 值，因为它取决于目标服务器以及它如何处理大量同时请求。通常从值 5 开始是安全的，然后从那里增加它以测试更高的值。
如果您正在处理一个数组，则有许多预先构建的选项可以同时运行 X 请求。最简单的是使用预先构建的并发管理操作，例如 Bluebird。或者你可以自己写。您可以在此处查看两者的示例：Make several requests to an API that can only handle 20 request a minute
但是，由于您没有处理数组，而只是为每个连续的请求增加一个数字，所以我找不到执行此操作的预构建选项。所以，我写了一个通用的，你可以在其中填写将增加索引的函数：
// fn gets called on each iteration - must return a promise
// limit is max number of requests to be in flight at once
// cnt is number of times to call fn
// options is optional and can be {continueOnError: true}
// runN returns a promise that resolves with results array.
// If continueOnError is set, then results array
// contains error values too (presumed to be instanceof Error so caller can discern
// them from regular values)
function runN(fn, limit, cnt, options = {}) {
return new Promise((resolve, reject) => {
let inFlightCntr = 0;
let results = [];
let cntr = 0;
let doneCnt = 0;
function run() {
while (inFlightCntr < limit && cntr < cnt) {
let resultIndex = cntr++;
++inFlightCntr;
fn().then(result => {
--inFlightCntr;
++doneCnt;
results[resultIndex] = result;
run(); // run any more that still need to be run
}).catch(err => {
--inFlightCntr;
++doneCnt;
if (options.continueOnError) {
// assumes error is instanceof Error so caller can tell the
// difference between a genuine result and an error
results[resultIndex] = err;
run(); // run any more that still need to be run
} else {
reject(err);
}
});
}
if (doneCnt === cnt) {
resolve(results);
}
}
run();
});
}
然后，你可以这样使用：
const needle = require("needle");
const startIdx = 11059000;
const stopIdx = 11211109;
const numConcurrent = 5;
let idxCntr = startIdx;
runN(function() {
let idx = idxCntr++;
return needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
});
}, numConcurrent, stopIdx - startIdx + 1, {continueOnError: true}).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
为了最大程度地减少内存使用，您可以在调用 needle() 时使用 .then() 处理程序，并将响应缩减为仅在最终数组中需要的内容：
const needle = require("needle");
const startIdx = 11059000;
const stopIdx = 11211109;
const numConcurrent = 5;
let idxCntr = startIdx;
runN(function() {
let idx = idxCntr++;
return needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
}).then(response => {
// construct the smallest possible response here and then return it
// to minimize memory use for your 100,000+ requests
return response.someProperty;
});
}, numConcurrent, stopIdx - startIdx + 1, {continueOnError: true}).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});