c++ socket accept() 使用 epoll 获取资源死锁避免 errno [关闭]

Question

我是一名 C++ 初学者，正在学习 I/O 多路复用。

这是我的代码：

test.cpp（不使用epoll()并且效果很好）：

#include <iostream>
#include <cstdio>
#include <netinet/in.h>
#include <unistd.h>
#include <string.h>

int main() {
    std::cout << "Hello" << std::endl;

    char buffer[1024];
    buffer[0] = 'f';
    fprintf(stdout, "%s", buffer);
    std::cout << "Hello" << std::endl;

    int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    // bind & listen
    struct sockaddr_in serverAddr;
    memset(&serverAddr, 0, sizeof(serverAddr));
    serverAddr.sin_family = AF_INET;
    serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    serverAddr.sin_port = htons(80);
    int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
    if (bindResult < 0) {
        fprintf(stderr, "Fail to bind\n");
        return 1;
    }
    int listenResult = listen(serverFd, 1024);
    if (listenResult < 0) {
        fprintf(stderr, "Fail to listen\n");
        return 1;
    }
    struct sockaddr clientAddr;
    unsigned int clientlen = sizeof(clientAddr);
    int acceptFd = accept(serverFd, &clientAddr, &clientlen);
    if (acceptFd < 0) {
        fprintf(stderr, "Fail to create client connection file descriptor\n");
        return 1;
    }
    int fd = acceptFd;
    ssize_t received = recv(fd, &buffer, 1024, 0);
    if (received < 0) {
        fprintf(stderr, "Fail to received bytess from client\n");
        if (errno == EINTR) {
            fprintf(stderr, "Reason: EINTR\n");
        } else if (errno == EAGAIN || errno == EWOULDBLOCK) {
            fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
        } else {
            fprintf(stderr, "Reason: %d\n", errno);
            close(fd);
            return 1;
        }
    } else if (received == 0) {
        close(fd);
    } else {
        buffer[received] = '\0';
        fprintf(stdout, "%s", buffer);
    }
}

test_2.cpp（确实使用epoll()但效果不佳）：

#include <iostream>
#include <sys/epoll.h>
#include <unistd.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <signal.h>
#include <string.h>

int main() {
    // TODO: too much error message to handle, so it's necessary to deal with it (maybe macros can)
    std::cout << "Hello" << std::endl;

    // process ignore SIGPIPE which is caused by send(), or process will exit, which is hard to find out
    signal(SIGPIPE, SIG_IGN);

    // here needs a socket fd or other fd
    // well, AF_INET is okay;socket(PF_INET, SOCK_SEQPACKET, 0) is sctp, tcp cannot use SOCK_SEQPACKET :(
    // when using tcp, watch out **record boundaries**
    int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    if (serverFd < 0) {
        fprintf(stderr, "Fail to create socket file descriptor\n");
        return 1;
    }
    // nonblock
    // nonblock
    int flags = fcntl(serverFd, F_GETFL, 0);
    if (flags < 0) {
        fprintf(stderr, "Fail to get flags\n");
        return 1;
    }
    int setFlagResult = fcntl(serverFd, F_SETFL, flags | O_NONBLOCK);
    if (setFlagResult < 0) {
        fprintf(stderr, "Fail to set flags\n");
        return 1;
    }

    // bind & listen
    struct sockaddr_in serverAddr;
    memset(&serverAddr, 0, sizeof(serverAddr));
    serverAddr.sin_family = AF_INET;
    serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    serverAddr.sin_port = htons(80);
    int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
    if (bindResult < 0) {
        fprintf(stderr, "Fail to bind\n");
        return 1;
    }
    int listenResult = listen(serverFd, 1024);
    if (listenResult < 0) {
        fprintf(stderr, "Fail to listen\n");
        return 1;
    }

    // epoll fd
    int epollFd = epoll_create(1);
    if (epollFd < 0) {
        fprintf(stderr, "Fail to create epoll file descriptor\n");
        return 1;
    }
    // event
    struct epoll_event event, events[1024];
    event.events = EPOLLIN;
    event.data.fd = serverFd;
    // ctl
    int ctlResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, serverFd, &event);
    if (ctlResult < 0) {
        fprintf(stderr, "Fail to run epoll_ctl\n");
        return 1;
    }
    // wait
    while (1) {
        int event_count = epoll_wait(epollFd, events, 1024, -1);

        for (int i = 0; i < event_count; i++) {
            struct epoll_event event = events[i];
            // accept
            if (event.data.fd == serverFd) {
                unsigned int clientlen = sizeof(clientAddr);
                int acceptFd = accept(serverFd, (struct sockaddr *) &clientAddr, &clientlen);
                if (acceptFd < 0) {
                    fprintf(stderr, "Fail to create client connection file descriptor\n");
                    fprintf(stderr, "Fail Reason: %d\n", errno);
                    return 1;
                }

                // nonblock
                int flags = fcntl(acceptFd, F_GETFL, 0);
                if (flags < 0) {
                    fprintf(stderr, "Fail to get flags\n");
                    return 1;
                }
                int setFlagResult = fcntl(acceptFd, F_SETFL, flags | O_NONBLOCK);
                if (setFlagResult < 0) {
                    fprintf(stderr, "Fail to set flags\n");
                    return 1;
                }

                struct epoll_event event;
                event.events = EPOLLIN;
                event.data.fd = serverFd;
                int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);
                if (ctlClientResult < 0) {
                    fprintf(stderr, "Fail to run epoll_ctl\n");
                    return 1;
                }
                // client recv
            } else if (event.events & EPOLLIN) {
                int fd = event.data.fd;
                char buffer[1024+1];
                ssize_t received = recv(fd, &buffer, 1024, 0);
                if (received < 0) {
                    fprintf(stderr, "Fail to received bytess from client\n");
                    if (errno == EINTR) {
                        fprintf(stderr, "Reason: EINTR\n");
                    } else if (errno == EAGAIN || errno == EWOULDBLOCK) {
                        fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
                    } else {
                        fprintf(stderr, "Reason: %d\n", errno);
                        close(fd);
                        int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
                        if (ctlClientResult < 0) {
                            fprintf(stderr, "Fail to run epoll_ctl\n");
                            return 1;
                        }
                        return 1;
                    }
                } else if (received == 0) {
                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
                    if (ctlClientResult < 0) {
                        fprintf(stderr, "Fail to run epoll_ctl\n");
                        return 1;
                    }
                    close(fd);
                } else {
                    buffer[received] = '\0';
                    fprintf(stdout, "%s", buffer);
                    // if you want to send something...
                    event.events |= EPOLLOUT;
                    // here is some data that event can hold
                    event.data.u32 = (uint32_t) 1;
                    // you can now send data or just put event in epoll, which is maybe easier
                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
                    if (ctlClientResult < 0) {
                        fprintf(stderr, "Fail to run epoll_ctl\n");
                        return 1;
                    }
                }
                // client send
            }  else if (event.events & EPOLLOUT) {
                int fd = event.data.fd;
                char buffer[] = "I see you";
                ssize_t sendResult = send(fd, &buffer, 1024, 0);
                if (sendResult < 0) {
                    fprintf(stderr, "Fail to received bytess from client\n");
                    if (errno == EINTR) {
                        fprintf(stderr, "Reason: EINTR\n");
                    } else if (errno == EAGAIN || errno == EWOULDBLOCK) {
                        fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
                    } else {
                        if (errno == EPIPE) {
                            fprintf(stderr, "Reason: EPIPE\n");
                        } else {
                            fprintf(stderr, "Reason: %d\n", errno);
                        }
                        close(fd);
                        int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
                        if (ctlClientResult < 0) {
                            fprintf(stderr, "Fail to run epoll_ctl\n");
                            return 1;
                        }
                        return 1;
                    }
                } else if (sendResult == 0) {
                    event.events = EPOLLIN;
                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
                    if (ctlClientResult < 0) {
                        fprintf(stderr, "Fail to run epoll_ctl\n");
                        return 1;
                    }
                } else {

                    // if you want to recv something...
//                    event.events |= EPOLLIN;
//                    // you can now send data or just put event in epoll, which is maybe easier
//                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
//                    if (ctlClientResult < 0) {
//                        fprintf(stderr, "Fail to run epoll_ctl\n");
//                        return 1;
//                    }
                }
            }


        }
    }

    return 0;
}

当我尝试建立 TCP 套接字连接时（例如 curl -v "http://host:80/"，它可以使 test2.cpp 在 第 81 行 运行），acceptFd 是 < 0 并且 errno 是11 根据第 84 行，意思是“避免了资源死锁”。

为什么？没有线程相关的代码吧？

Answer 1

            struct epoll_event event;
            event.events = EPOLLIN;
            event.data.fd = serverFd;
            int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);

当您将新接受的连接添加到 epoll 集时，您告诉它将其报告为serverFd 上的命中。因此，当客户端向您发送数据时，您会尝试接受新的连接。

将event.data.fd = serverFd 更改为event.data.fd = acceptFd。

然后你可以继续你的下一个错误：

            char buffer[] = "I see you";
            ssize_t sendResult = send(fd, &buffer, 1024, 0);

1024?!

此外，无论何时使用非阻塞套接字，都应添加代码以将 EAGAIN 或 EWOULDBLOCK 错误视为非致命错误。

Answer 2

错误代码 11 是EAGAIN，这是在处理非阻塞套接字 I/O 时遇到的非常常见的错误代码。表示请求的操作无关，请稍后再试。

对于accept()，表示：

EAGAIN 或 EWOULDBLOCK

套接字被标记为非阻塞，并且不存在可以接受的连接。 POSIX.1-2001 和 POSIX.1-2008 允许在这种情况下返回任一错误，并且不要求这些常量具有相同的值，因此可移植应用程序应检查这两种可能性。

这意味着您在错误的时间调用accept()，此时侦听套接字没有任何客户端可以接受。仔细检查您的 epoll() 用法，它可能存在逻辑错误，导致您过早调用 accept()。

例如，accept() 成功接受客户端后，您在调用 epoll_ctl(EPOLL_CTL_ADD) 时将 listening 套接字而不是 client 套接字分配给 event.data.fd ：

struct epoll_event event;
event.events = EPOLLIN;
event.data.fd = serverFd; // <-- HERE, SHOULD BE acceptFd INSTEAD!
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);

因此，当 client 有数据等待读取时，您的循环最终会在 listening 套接字上调用 accept() 而不是在 on 上调用 recv() 客户端套接字。

此外，您没有检查报告的event.events 字段是否存在套接字错误。如果客户端套接字上发生错误，报告的events 可能包含EPOLLERR 和/或EPOLLHUP 标志。您应该检查这些标志，如果存在则关闭客户端套接字，然后再检查EPOLLIN 标志以调用recv()。

另外，请注意，在您的示例中，it is not necessary to call epoll_ctl(EPOLL_CTL_DEL) 在套接字上调用 close() 之前。关闭一个套接字文件描述符会自动从正在监视它的 epoll 实例中删除它。