如何在 Node 中限制 HTTP 请求的带宽？

Question

我正在尝试限制 HTTP 请求使用的带宽（下载/上传速度）。我正在使用 NPM 包stream-throttle。我创建了一个自定义的HTTP agent 来通过 Throttle 实例来管道套接字，并计算了 5MB 文件的下载速度。

const http = require("http");
const net = require("net");
const {Throttle, ThrottleGroup} = require("stream-throttle");

const maxBandwidth = 100;
// an example 5MB file of random data
const str = "http://212.183.159.230/5MB.zip";


// this pipes an instance of Throttle
class SlowAgent extends http.Agent {
    createConnection(options, callback){
        const socket = new net.Socket(options);
        socket.pipe(new Throttle({rate: maxBandwidth}));
        socket.connect(options);
        return socket;
    }
}

const options = {
    // this should slow down the request
    agent: new SlowAgent()
};

const time = Date.now();
const req = http.request(str, options, (res) => {
    res.on("data", () => {
    });
    res.on('end', () => {
        console.log("Done! Elapsed time: " + (Date.now() - time) + "ms");
    });
});

req.on('error', (e) => {
    console.error(`problem with request: ${e.message}`);
});

req.on("end", () => {
    console.log("done");
});

console.log("Request started");
req.end();

不管maxBandwidth 的值是多少，或者是否使用了SlowAgent（我已经尝试注释掉agent: new SlowAgent()），我注意到经过的时间（大约4000 毫秒）没有区别。如何修复我的 SlowAgent 类？ socket.pipe我听不懂吗？或者我还有什么需要做的吗？

怪异的pointed out 将SlowAgent 更改为：

// this pipes an instance of Throttle
class SlowAgent extends http.Agent {
    createConnection(options, callback){
        const socket = new net.Socket(options);
        socket.connect(options);
        return socket.pipe(new Throttle({rate: 10}));
    }
}

但这会导致这个问题：

problem with request: Parse Error: Expected HTTP/ Error: Parse Error: Expected HTTP/
    at Throttle.socketOnData (_http_client.js:456:22)
    at Throttle.emit (events.js:209:13)
    at addChunk (_stream_readable.js:305:12)
    at readableAddChunk (_stream_readable.js:286:11)
    at Throttle.Readable.push (_stream_readable.js:220:10)
    at Throttle.Transform.push (_stream_transform.js:150:32)
    at /home/max/Documents/Personal/node-projects/proxy/node_modules/stream-throttle/src/throttle.js:37:14
    at processTicksAndRejections (internal/process/task_queues.js:75:11) {
  bytesParsed: 0,
  code: 'HPE_INVALID_CONSTANT',
  reason: 'Expected HTTP/',
  rawPacket: <Buffer 47>
}

Answer 1

我设法通过取消自定义agent 并在http.request 选项中使用createConnection 来使其工作：

const options = {
    createConnection(options) {
        const socket = new net.Socket();
        return socket.connect({host: options.host, port: options.port});
    },
    hostname: "212.183.159.230",
    path: "/5MB.zip"
};

const time = Date.now();

const req = http.request(options, (res) => {

    res.pipe(new Throttle({rate: 200 * 1024}))
        .on("data", (chunk) => {
            console.log(chunk.length);
        })

    res.on("end", () => {
        console.log("Done! Elapsed time: " + (Date.now() - time) + "ms");
    });
});

Answer 2

流带宽控制必须在端、服务器和客户端实现。

从客户的角度，

上传速度可以通过节流来管理

客户端应用程序或 客户端网络层或 服务器网络层

下载速率可以通过节流来管理

服务器应用程序或 服务器网络层 客户端网络层

请看一下这个测试代码。 您可以在两侧更改速率变量。

环境

Windows 10 中的节点 v10.16.3。

server.js

var fs = require('fs');  // file system
var http = require('http');
const {ThrottleGroup} = require("stream-throttle");

/**
 * Change to various rate to test
 */
var tg = new ThrottleGroup({rate: 1024*1024}); //1 MiB per sec

/**
 * please copy your own file
 * my file is 4.73 MB (4,961,271 bytes) ,it takes 4~5 sec to send data chunk
 */
var source = "source.jpg"; //

var server = http.createServer((req, res) => {

    var rstream = fs.createReadStream(source);
    rstream
        .pipe(tg.throttle()) //throttle here
        .pipe(res);

    //define event 
    rstream
        .on('open', ()=>{
            console.log('open', new Date())
        })        
        .on('data', (chunk)=>{
            console.log(new Date(), chunk.length) // 65536 bytes
        })
        .on('close', () => {
            console.log('close', new Date())
        });       
});
server.listen(80, '127.0.0.1');  // start

//OUTPUT when client request, max chunk 65536 bytes
>node server.js

open 2019-09-13T05:27:40.724Z
2019-09-13T05:27:40.730Z 65536
2019-09-13T05:27:40.732Z 65536
...
2019-09-13T05:27:44.355Z 65536
2019-09-13T05:27:44.419Z 46071
close 2019-09-13T05:27:44.421Z

client.js

const fs = require('fs');
const http = require("http");
const {ThrottleGroup} = require("stream-throttle");

var tg = new ThrottleGroup({rate: 1024*1024*2}); //2 MiB /sec

/**
 receiving 4.73 MB (4,961,271 bytes) ,
 it takes 2~3 sec to receive 4.73MB, but server side throttle is 1Mib
 Thus, it still takes 4~5 sec to download as server has been throttled
 */
var wstream = fs.createWriteStream("ouput.jpg");
wstream
    .on('open', () => {
        console.log('open', new Date())        
    })
    .on('finish', () => {
        console.log('finish', new Date())        
    });  

var dataLength = 0;
http.get('http://127.0.0.1/', (res) => {    
    res
    .pipe(tg.throttle())
    .pipe(wstream);

    res
    .on('open', ()=>{
        console.log('res open', new Date())
    })        
    .on('data', (chunk)=>{
        dataLength += chunk.length
        console.log(new Date(), `data length: ${dataLength}`)
    })
    .on('close', () => {
        console.log('res close', new Date())        
    })          

  });

//OUTPUT
>node client.js

open 2019-09-13T05:27:40.718Z
2019-09-13T05:27:40.736Z 'data length: 65426'
2019-09-13T05:27:40.741Z 'data length: 65536' 
2019-09-13T05:27:40.742Z 'data length: 130953'
...
2019-09-13T05:27:44.463Z 'data length: 4961271'
finish 2019-09-13T05:27:44.474Z
res close 2019-09-13T05:27:44.476Z

对于现实世界的例子， 更改 client.js 节流速率和下一行

http.get('http://127.0.0.1/', (res) => {

类似

http.get('http://i.ytimg.com/vi/ZYifkcmIb-4/maxresdefault.jpg', (res) => {    

在现实世界中，参与的参与者越多，网络就越复杂。

服务器端 OSI 模型 网络 客户端 OSI 模型

由于互联网提供商或运营商会限制他们的端口，这会影响您的上传和下载速率。