C++低吞吐量winsock TCP测试应用

Question

我正在尝试构建一个在 localhost 中工作的快速服务器和客户端。这个想法是从另一个程序发送数据块，并快速处理它。一次只有一个客户端连接到服务器。

我首先尝试使用 boost::asio 库来实现它。一切正常，除了吞吐量非常慢，415 兆字节/秒。

然后我开始使用 winsock 创建一个测试用例，它的吞吐量也非常相似，都是 434 兆字节/秒。非常慢。

我期望在 40GB 或至少几GB/秒的范围内有更多。

感谢任何建议，因为我已经脱离了网络编程的范畴。

我当前的客户端函数：

bool sendDataWin(size_t size, const size_t blocksize, size_t port)
{
    int result;

    struct addrinfo *addressinfo = nullptr, hints;
    auto sport = std::to_string(port);

    ZeroMemory(&hints, sizeof(hints));
    hints.ai_family = AF_INET;
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_protocol = IPPROTO_TCP;

    // Resolve the local address and port to be used by the server
    result = getaddrinfo("localhost", sport.c_str(), &hints, &addressinfo);
    if (result != 0) {
        printf("Error at client socket(): %ld\n", WSAGetLastError());
        return false;
    }

    SOCKET connection = socket(
        addressinfo->ai_family,
        addressinfo->ai_socktype,
        addressinfo->ai_protocol);

    if (connection == INVALID_SOCKET) {
        printf("Error at client socket(): %ld\n", WSAGetLastError());
        freeaddrinfo(addressinfo);
        return false;
    }

    // Try to put loopback fast path on.
    bool value;
    DWORD dwBytesRet;
    int status =
        WSAIoctl(
            connection,
            SIO_LOOPBACK_FAST_PATH,
            &value,
            sizeof(bool),
            NULL,
            0,
            &dwBytesRet,
            0,
            0);

    if (status == SOCKET_ERROR) {
        DWORD LastError = ::GetLastError();
        if (LastError == WSAEOPNOTSUPP) {
            printf("client call is not supported :: SIO_LOOPBACK_FAST_PATH\n");
        }
    }

    // Connect to server.
    result = connect(connection, addressinfo->ai_addr, (int)addressinfo->ai_addrlen);
    if (result == SOCKET_ERROR) {
        printf("Error at client socket(): %ld\n", WSAGetLastError());
        closesocket(connection);
        return false;
    }
    freeaddrinfo(addressinfo);

    std::vector<uint8_t> buffer;
    buffer.resize(blocksize);

    size_t total = 0;

    do {
        size_t sendSize = blocksize;
        size_t next = total + sendSize;
        if (next > size)
        {
            sendSize -= next - size;
        }

        // Echo the buffer back to the sender
        result = send(connection, (const char*)buffer.data(), (int)sendSize, 0);
        if (result == SOCKET_ERROR) {
            printf("client send failed: %d\n", WSAGetLastError());
            closesocket(connection);
            return false;
        }

        total += sendSize;
    } while (total < size);

    result = shutdown(connection, SD_SEND);
    if (result == SOCKET_ERROR) {
        printf("client shutdown failed: %d\n", WSAGetLastError());
        closesocket(connection);
        return false;
    }
    // cleanup
    closesocket(connection);

    return true;
}

服务器功能：

bool serverReceiveDataWin(size_t size, const size_t blocksize, size_t port)
{
    int result;

    struct addrinfo *addressinfo = nullptr, *ptr = nullptr, hints;
    auto sport = std::to_string(port);

    ZeroMemory(&hints, sizeof(hints));
    hints.ai_family = AF_INET;
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_protocol = IPPROTO_TCP;
    hints.ai_flags = AI_PASSIVE;

    // Resolve the local address and port to be used by the server
    result = getaddrinfo(nullptr, sport.c_str(), &hints, &addressinfo);
    if (result != 0) {
        printf("Error at server socket(): %ld\n", WSAGetLastError());
        return false;
    }

    SOCKET ListenSocket = socket(addressinfo->ai_family, addressinfo->ai_socktype, addressinfo->ai_protocol);
    if (ListenSocket == INVALID_SOCKET) {
        printf("Error at server socket(): %ld\n", WSAGetLastError());
        freeaddrinfo(addressinfo);
        return false;
    }

    // Try to put loopback fast path on.
    bool value;
    DWORD dwBytesRet;
    int status =
        WSAIoctl(
            ListenSocket,
            SIO_LOOPBACK_FAST_PATH,
            &value,
            sizeof(bool),
            NULL,
            0,
            &dwBytesRet,
            0,
            0);

    if (status == SOCKET_ERROR) {
        DWORD LastError = ::GetLastError();
        if (LastError == WSAEOPNOTSUPP) {
            printf("server call is not supported :: SIO_LOOPBACK_FAST_PATH\n");
        }
    }

    // Setup the TCP listening socket
    result = bind(ListenSocket, addressinfo->ai_addr, (int)addressinfo->ai_addrlen);
    if (result == SOCKET_ERROR) {
        printf("server bind failed with error: %d\n", WSAGetLastError());
        freeaddrinfo(addressinfo);
        closesocket(ListenSocket);
        return false;
    }
    if (listen(ListenSocket, SOMAXCONN) == SOCKET_ERROR) {
        printf("Listen failed with error: %ld\n", WSAGetLastError());
        closesocket(ListenSocket);
        return false;
    }

    // Accept a client socket
    SOCKET ClientSocket = accept(ListenSocket, nullptr, nullptr);
    if (ClientSocket == INVALID_SOCKET) {
        printf("server accept failed: %d\n", WSAGetLastError());
        closesocket(ListenSocket);
        return false;
    }

    std::vector<uint8_t> buffer;
    buffer.resize(blocksize);

    size_t total = 0;

    // Receive until the peer shuts down the connection
    do {
        size_t currentSize = blocksize;
        size_t next = total + currentSize;
        if (next > size)
        {
            currentSize -= next - size;
        }

        result = recv(ClientSocket, (char*)buffer.data(), (int)currentSize, 0);
        if (result > 0)
        {
            total += result;
        }
        else if (result == 0)
        {
            printf("server Connection closing...\n");
            return false;
        }
        else {
            printf("server recv failed: %d\n", WSAGetLastError());
            closesocket(ClientSocket);
            return false;
        }
    } while (total < size);

    result = shutdown(ClientSocket, SD_SEND);
    if (result == SOCKET_ERROR) {
        printf("server shutdown failed: %d\n", WSAGetLastError());
        closesocket(ClientSocket);
        return false;
    }
    // cleanup
    closesocket(ClientSocket);

    return true;
}

而测试程序本身是：

int main()
{
    int width = 1920;
    int height = 1080;
    const size_t totalBpp = 3;
    const size_t totalSize = width * height * totalBpp;
    size_t port = 27140;

    size_t times = 1000;
    size_t expectedData = totalSize * times;

    // Initialize Winsock
    int iResult;
    WSADATA wsaData;
    iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
    if (iResult != 0) {
        std::cout << "WSAStartup failed: " << iResult << std::endl;
        return EXIT_FAILURE;
    }

    std::atomic_bool serverOk{ false };
    std::atomic_bool clientOk{ false };

    auto start = std::chrono::high_resolution_clock::now();

    std::thread server = std::thread([&] {
        serverOk = serverReceiveDataWin(expectedData, totalSize, port);
    });

    std::thread client = std::thread([&] {
        clientOk = sendDataWin(expectedData, totalSize, port);
    });
    client.join();
    server.join();
    auto end = std::chrono::high_resolution_clock::now();
    WSACleanup();

    if (!(clientOk && serverOk))
    {
        if (!serverOk) std::cout << "Server was not OK." << std::endl;
        if (!clientOk) std::cout << "Client was not OK." << std::endl;
        return EXIT_FAILURE;
    }

    std::chrono::duration<double> diff = end - start;

    double frameTime = diff.count() / times;
    double fps = 1.0 / frameTime;

    std::cout << "Sent: " << width << "x" << height << "_" << totalBpp << "(" << totalSize << "). times: " << times << std::endl;
    std::cout << "frameTime: " << frameTime << "s." << std::endl;
    std::cout << "fps: " << fps << "." << std::endl;
    std::cout << "transfer rate : " << ((expectedData / diff.count()) / 1048576) << " mebibytes/s." << std::endl;
    std::cout << "transfer rate : " << ((expectedData / diff.count()) / 1000000) << " megabytes/s." << std::endl;
    std::this_thread::sleep_for(std::chrono::seconds{ 60 });

    return EXIT_SUCCESS;
}

在我的机器上我得到了这些结果：

Sent: 1920x1080_3(6220800).times : 1000
frameTime : 0.0138217s.
fps : 72.35.
transfer rate : 429.225 mebibytes / s.
transfer rate : 450.075 megabytes / s.

---

看来问题出在我的防火墙/杀毒软件上，我先卸载了F-secure，速度提高到500MB/s..卸载重启后，速度提高到4000MB/s。

Answer 1

几点。

1. IOCTL 调用不正确。

   bool value;
   DWORD dwBytesRet;
   int status =
       WSAIoctl(
           ListenSocket,
           SIO_LOOPBACK_FAST_PATH,
           &value,
           sizeof(bool),
           NULL,
           0,
           &dwBytesRet,
           0,
           0);
   

这应该传入一个设置为 1 的 DWORD（32 位整数）。上面的代码传递了一个从未初始化为任何内容的 C++ bool（可能只有 1 个字节）。

1. 由于这只是一个单一的套接字，这将受到 CPU 的限制，因为它实际上只是从用户/内核开始并在单个线程上进行内存复制。这不太可能通过单个连接达到 40 Gbps。特别是因为这只会发送 6GB 的数据。

Answer 2

像这样发送大量数据时，您应该关闭Nagle's Algorithm。在发送套接字上设置TCP_NODELAY。