【发布时间】：2016-03-09 20:20:19
【问题描述】：
我有一个关于 recv() 和 send() 缓冲区的大小如何影响 TCP 性能的问题。考虑以下完整的 C++ 示例，该示例通过 TCP 从客户端向服务器传输 1 GB（任意）数据。
#include <unistd.h>
#include <netdb.h>
#include <errno.h>
#include <netinet/tcp.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/ioctl.h>
#include <iostream>
#include <memory>
#include <cstring>
#include <cstdlib>
#include <stdexcept>
#include <algorithm>
#include <string>
#include <sstream>
typedef unsigned long long TimePoint;
typedef unsigned long long Duration;
inline TimePoint getTimePoint() {
struct ::timeval tv;
::gettimeofday(&tv, nullptr);
return tv.tv_sec * 1000000ULL + tv.tv_usec;
}
const size_t totalSize = 1024 * 1024 * 1024;
const int one = 1;
void server(const size_t blockSize, const std::string& serviceName) {
std::unique_ptr<char[]> block(new char[blockSize]);
const size_t atLeastReads = totalSize / blockSize;
std::cout << "Starting server. Receiving block size is " << blockSize << ", which requires at least " << atLeastReads << " reads." << std::endl;
addrinfo hints;
memset(&hints, 0, sizeof(addrinfo));
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = AI_PASSIVE;
hints.ai_protocol = 0;
addrinfo* firstAddress;
int result = getaddrinfo(nullptr, serviceName.c_str(), &hints, &firstAddress);
if (result != 0) return;
int listener = socket(firstAddress->ai_family, firstAddress->ai_socktype, firstAddress->ai_protocol);
if (listener == -1) return;
if (setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)) != 0) return;
if (bind(listener, firstAddress->ai_addr, firstAddress->ai_addrlen) != 0) return;
freeaddrinfo(firstAddress);
if (listen(listener, 1) != 0) return;
while (true) {
int server = accept(listener, nullptr, nullptr);
if (server == -1) return;
u_long mode = 1;
if (::ioctl(server, FIONBIO, &mode) != 0) return;
// if (setsockopt(server, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)) != 0) return;
// int size = 64000;
// if (setsockopt(server, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)) != 0) return;
// if (setsockopt(server, SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)) != 0) return;
std::cout << "Server accepted connection." << std::endl;
size_t leftToRead = totalSize;
size_t numberOfReads = 0;
size_t numberOfIncompleteReads = 0;
const TimePoint totalStart = ::getTimePoint();
Duration selectDuration = 0;
Duration readDuration = 0;
while (leftToRead > 0) {
fd_set readSet;
FD_ZERO(&readSet);
FD_SET(server, &readSet);
TimePoint selectStart = ::getTimePoint();
if (select(server + 1, &readSet, nullptr, nullptr, nullptr) == -1) return;
selectDuration += ::getTimePoint() - selectStart;
if (FD_ISSET(server, &readSet) != 0) {
const size_t toRead = std::min(leftToRead, blockSize);
TimePoint readStart = ::getTimePoint();
const ssize_t actuallyRead = recv(server, block.get(), toRead, 0);
readDuration += ::getTimePoint() - readStart;
if (actuallyRead == -1)
return;
else if (actuallyRead == 0) {
std::cout << "Got 0 bytes, which signals that the client closed the socket." << std::endl;
break;
}
else if (toRead != actuallyRead)
++numberOfIncompleteReads;
++numberOfReads;
leftToRead -= actuallyRead;
}
}
const Duration totalDuration = ::getTimePoint() - totalStart;
std::cout << "Receiving took " << totalDuration << " us, transfer rate was " << totalSize / (totalDuration / 1000000.0) << " bytes/s." << std::endl;
std::cout << "Selects took " << selectDuration << " us, while reads took " << readDuration << " us." << std::endl;
std::cout << "There were " << numberOfReads << " reads (factor " << numberOfReads / ((double)atLeastReads) << "), of which " << numberOfIncompleteReads << " (" << (numberOfIncompleteReads / ((double)numberOfReads)) * 100.0 << "%) were incomplete." << std::endl << std::endl;
close(server);
}
}
bool client(const size_t blockSize, const std::string& hostName, const std::string& serviceName) {
std::unique_ptr<char[]> block(new char[blockSize]);
const size_t atLeastWrites = totalSize / blockSize;
std::cout << "Starting client... " << std::endl;
addrinfo hints;
memset(&hints, 0, sizeof(addrinfo));
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = 0;
hints.ai_protocol = 0;
addrinfo* firstAddress;
if (getaddrinfo(hostName.c_str(), serviceName.c_str(), &hints, &firstAddress) != 0) return false;
int client = socket(firstAddress->ai_family, firstAddress->ai_socktype, firstAddress->ai_protocol);
if (client == -1) return false;
if (connect(client, firstAddress->ai_addr, firstAddress->ai_addrlen) != 0) return false;
freeaddrinfo(firstAddress);
u_long mode = 1;
if (::ioctl(client, FIONBIO, &mode) != 0) return false;
// if (setsockopt(client, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)) != 0) return false;
// int size = 64000;
// if (setsockopt(client, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)) != 0) return false;
// if (setsockopt(client, SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)) != 0) return false;
std::cout << "Client connected. Sending block size is " << blockSize << ", which requires at least " << atLeastWrites << " writes." << std::endl;
size_t leftToWrite = totalSize;
size_t numberOfWrites = 0;
size_t numberOfIncompleteWrites = 0;
const TimePoint totalStart = ::getTimePoint();
Duration selectDuration = 0;
Duration writeDuration = 0;
while (leftToWrite > 0) {
fd_set writeSet;
FD_ZERO(&writeSet);
FD_SET(client, &writeSet);
TimePoint selectStart = ::getTimePoint();
if (select(client + 1, nullptr, &writeSet, nullptr, nullptr) == -1) return false;
selectDuration += ::getTimePoint() - selectStart;
if (FD_ISSET(client, &writeSet) != 0) {
const size_t toWrite = std::min(leftToWrite, blockSize);
TimePoint writeStart = ::getTimePoint();
const ssize_t actuallyWritten = send(client, block.get(), toWrite, 0);
writeDuration += ::getTimePoint() - writeStart;
if (actuallyWritten == -1)
return false;
else if (actuallyWritten == 0) {
std::cout << "Got 0 bytes, which shouldn't happen!" << std::endl;
break;
}
else if (toWrite != actuallyWritten)
++numberOfIncompleteWrites;
++numberOfWrites;
leftToWrite -= actuallyWritten;
}
}
const Duration totalDuration = ::getTimePoint() - totalStart;
std::cout << "Writing took " << totalDuration << " us, transfer rate was " << totalSize / (totalDuration / 1000000.0) << " bytes/s." << std::endl;
std::cout << "Selects took " << selectDuration << " us, while writes took " << writeDuration << " us." << std::endl;
std::cout << "There were " << numberOfWrites << " writes (factor " << numberOfWrites / ((double)atLeastWrites) << "), of which " << numberOfIncompleteWrites << " (" << (numberOfIncompleteWrites / ((double)numberOfWrites)) * 100.0 << "%) were incomplete." << std::endl << std::endl;
if (shutdown(client, SHUT_WR) != 0) return false;
if (close(client) != 0) return false;
return true;
}
int main(int argc, char* argv[]) {
if (argc < 2)
std::cout << "Block size is missing." << std::endl;
else {
const size_t blockSize = static_cast<size_t>(std::atoll(argv[argc - 1]));
if (blockSize > 1024 * 1024)
std::cout << "Block size " << blockSize << " is suspicious." << std::endl;
else {
if (argc >= 3) {
if (!client(blockSize, argv[1], "12000"))
std::cout << "The client encountered an error." << std::endl;
}
else {
server(blockSize, "12000");
std::cout << "The server encountered an error." << std::endl;
}
}
}
return 0;
}
我在通过 1 Gbit/s LAN 连接的两台 Linux（内核版本 4.1.10-200.fc22.x86_64）机器上运行示例，在这些机器上我得到以下行为：如果 recv() 和 send( ) 系统调用使用 40 字节或更多的缓冲区，然后我使用所有可用带宽；但是，如果我在服务器或客户端上使用较小的缓冲区，则吞吐量会下降。此行为似乎不受注释掉的套接字选项（Nagle 的算法和/或发送/接收缓冲区大小）的影响。
我可以理解以小块发送数据可能效率低下：如果关闭 Nagle 算法并且块很小，那么 TCP 和 IP 的标头大小可能会主导有用的有效负载。但是，我不认为接收缓冲区的大小会影响传输速率：我希望 recv() 系统调用的成本与通过 LAN 实际发送数据的成本相比要便宜。因此，如果我以 5000 字节块的形式发送数据，我希望传输速率在很大程度上独立于接收缓冲区的大小，因为我调用 recv() 的速率仍应大于 LAN 传输速率.唉，事实并非如此！
如果有人能向我解释导致速度变慢的原因，我将不胜感激：这仅仅是系统调用的成本，还是在协议级别发生了什么？
我在编写基于消息的云应用程序时遇到了这个问题，如果有人能告诉我这个问题在他们看来应该如何影响系统的架构，我将不胜感激。由于种种原因，我没有使用 ZeroMQ 之类的消息传递库，而是自己编写消息传递接口。云中的计算使得服务器之间的消息流不是对称的（即，根据工作负载，服务器 A 可以向服务器 B 发送比反之多得多的数据），消息是异步的（即消息之间的时间是不可预测的，但许多消息可以突发发送），消息大小可变，通常很小（10 到 20 字节）。此外，原则上消息可以乱序传递，但重要的是不要丢弃消息，并且还需要一些流量/拥塞控制；因此，我使用的是 TCP 而不是 UDP。由于消息的大小不同，因此每条消息都以指定消息大小的整数开头，然后是消息有效负载。要从套接字读取消息，我首先读取消息大小，然后读取有效负载；因此，读取一条消息至少需要两次 recv() 调用（可能更多，因为 recv() 可以返回比请求更少的数据）。现在，因为消息大小和消息负载都很小，我最终会收到许多小的 recv() 请求，正如我的示例所示，这并不能让我充分利用可用带宽。有没有人对在这种情况下构建消息传递的“正确”方式有任何建议？
非常感谢您的帮助！
【问题讨论】：
-
“仅仅是系统调用的成本吗”。似乎很有可能。你不能用探查器运行你的 2 个版本来看看吗？祝你好运。
-
每个 recv() 调用处理的字节数不会影响通过网络发送的字节数。 recv() 调用仅将内核接收数据缓冲区中该套接字的下 N 个字节复制到用户空间。但是每个系统调用都会产生开销（上下文切换等），因此减少每秒需要调用 recv() 的次数确实可以提高 CPU 效率。 （不过，我认为每条传入消息调用两次 recv() 是在易于编码和效率之间的合理权衡）
-
我不确定我是否理解如何将系统调用的成本与底层工作的成本隔离开来：在分析器中，我会看到系统调用所花费的总时间，但是我不知道其中有多少在内核中用完，有多少是由于协议问题（例如，尚未收到数据）。有没有办法衡量内核切换的成本？
-
如果将套接字设置为非阻塞，则可以保证 recv() 调用将始终立即返回。 （当然，您需要在其他地方阻塞，例如在 select() 中，以避免 CPU 旋转，但是通过这样做，您将能够将等待传入数据的时间与在 recv() 中花费的时间分开）
-
这是个好主意——我已经按照 Jeremy 的建议更新了测试代码。经过一些实验，结果发现 select() 所花费的时间大约是 recv()/send() 所花费的时间的十倍。在我看来，这表明问题不是由于系统调用的开销而出现的。而是在协议级别发生了一些事情。
标签： linux performance sockets tcp