分析 XDP eBPF 数据包丢失和性能不佳

Question

我创建了一个小项目 (https://github.com/NHAS/wag)，它使用 XDP 和 eBPF 允许基于时间通过 wireguard VPN 进行连接。

我已将 XDP eBPF 程序附加到 wireguard TUN 设备，但吞吐量很差（速度测试下降 ~20 Mbps wireguard + eBPF，对比 wireguard - eBPF ~100 Mbps）。此外，对 wireguard 服务器本身的 ping 具有不一致的延迟，并且以 1 个 ICMP 数据包/~600 ping 的速率被丢弃。

请注意，这发生在卸载期间。总流量将低于 100 Mbps。

下面的代码是用 cilium 加载到内核中的。

// Kernel load
...
    xdpLink, err = link.AttachXDP(link.XDPOptions{
        Program:   xdpObjects.XdpProgFunc,
        Interface: iface.Index,
    })
...

eBPF 内核：

// +build ignore

#include "bpf_endian.h"
#include "common.h"

char __license[] SEC("license") = "Dual MIT/GPL";

// One /24
#define MAX_MAP_ENTRIES 256

// Inner map is a LPM tri, so we use this as the key
struct ip4_trie_key
{
    __u32 prefixlen; // first member must be u32
    __u32 addr;      // rest can are arbitrary
};

// Map of users (ipv4) to BOOTTIME uint64 timestamp denoting authorization status
struct bpf_map_def SEC("maps") sessions = {
    .type = BPF_MAP_TYPE_HASH,
    .max_entries = MAX_MAP_ENTRIES,
    .key_size = sizeof(__u32),
    .value_size = sizeof(__u64),
    .map_flags = 0,
};

// Map of users (ipv4) to BOOTTIME uint64 timestamp denoting when the last packet was recieved
struct bpf_map_def SEC("maps") last_packet_time = {
    .type = BPF_MAP_TYPE_HASH,
    .max_entries = MAX_MAP_ENTRIES,
    .key_size = sizeof(__u32),
    .value_size = sizeof(__u64),
    .map_flags = 0,
};

// A single variable in nano seconds
struct bpf_map_def SEC("maps") inactivity_timeout_minutes = {
    .type = BPF_MAP_TYPE_ARRAY,
    .max_entries = 1,
    .key_size = sizeof(__u32),
    .value_size = sizeof(__u64),
    .map_flags = 0,
};

// Two tables of the same construction
// IP to LPM trie
struct bpf_map_def SEC("maps") mfa_table = {
    .type = BPF_MAP_TYPE_HASH_OF_MAPS,
    .max_entries = MAX_MAP_ENTRIES,
    .key_size = sizeof(__u32),
    .value_size = sizeof(__u32),
    .map_flags = 0,
};

struct bpf_map_def SEC("maps") public_table = {
    .type = BPF_MAP_TYPE_HASH_OF_MAPS,
    .max_entries = MAX_MAP_ENTRIES,
    .key_size = sizeof(__u32),
    .value_size = sizeof(__u32),
    .map_flags = 0,
};

/*
Attempt to parse the IPv4 source address from the packet.
Returns 0 if there is no IPv4 header field; otherwise returns non-zero.
*/
static int parse_ip_src_dst_addr(struct xdp_md *ctx, __u32 *ip_src_addr, __u32 *ip_dst_addr)
{
    void *data_end = (void *)(long)ctx->data_end;
    void *data = (void *)(long)ctx->data;

    // As this is being attached to a wireguard interface (tun device), we dont get layer 2 frames
    // Just happy little ip packets

    // Then parse the IP header.
    struct iphdr *ip = data;
    if ((void *)(ip + 1) > data_end)
    {
        return 0;
    }

    // We dont support ipv6
    if (ip->version != 4)
    {
        return 0;
    }

    // Return the source IP address in network byte order.
    *ip_src_addr = (__u32)(ip->saddr);
    *ip_dst_addr = (__u32)(ip->daddr);

    return 1;
}

static int conntrack(__u32 *src_ip, __u32 *dst_ip)
{

    // Max lifetime of the session.
    __u64 *session_expiry = bpf_map_lookup_elem(&sessions, src_ip);
    if (!session_expiry)
    {
        return 0;
    }

    // The most recent time a valid packet was received from our a user src_ip
    __u64 *lastpacket = bpf_map_lookup_elem(&last_packet_time, src_ip);
    if (!lastpacket)
    {
        return 0;
    }

    // Our userland defined inactivity timeout
    u32 index = 0;
    __u64 *inactivity_timeout = bpf_map_lookup_elem(&inactivity_timeout_minutes, &index);
    if (!inactivity_timeout)
    {
        return 0;
    }

    __u64 currentTime = bpf_ktime_get_boot_ns();

    // The inner map must be a LPM trie
    struct ip4_trie_key key = {
        .prefixlen = 32,
        .addr = *dst_ip,
    };

    // If the inactivity timeout is not disabled and users session has timed out
    u8 isTimedOut = (*inactivity_timeout != __UINT64_MAX__ && ((currentTime - *lastpacket) >= *inactivity_timeout));

    if (isTimedOut)
    {
        u64 locked = 0;
        bpf_map_update_elem(&sessions, src_ip, &locked, BPF_EXIST);
    }

    // Order of preference is MFA -> Public, just in case someone adds multiple entries for the same route to make sure accidental exposure is less likely
    // If the key is a match for the LPM in the public table
    void *user_restricted_routes = bpf_map_lookup_elem(&mfa_table, src_ip);
    if (user_restricted_routes)
    {

        if (bpf_map_lookup_elem(user_restricted_routes, &key) &&
            // 0 indicates invalid session
            *session_expiry != 0 &&
            // If max session lifetime is disabled, or we are before the max lifetime of the session
            (*session_expiry == __UINT64_MAX__ || *session_expiry > currentTime) &&
            !isTimedOut)
        {

            // Doesnt matter if the value is not atomically set
            *lastpacket = currentTime;

            return 1;
        }
    }

    void *user_public_routes = bpf_map_lookup_elem(&public_table, src_ip);
    if (user_public_routes && bpf_map_lookup_elem(user_public_routes, &key))
    {
        // Only update the lastpacket time if we're not expired
        if (!isTimedOut)
        {
            *lastpacket = currentTime;
        }
        return 1;
    }

    return 0;
}

SEC("xdp")
int xdp_prog_func(struct xdp_md *ctx)
{
    __u32 src_ip, dst_ip;
    if (!parse_ip_src_dst_addr(ctx, &src_ip, &dst_ip))
    {
        return XDP_DROP;
    }

    if (conntrack(&src_ip, &dst_ip) || conntrack(&dst_ip, &src_ip))
    {

        return XDP_PASS;
    }

    return XDP_DROP;
}

我要回答的问题是：

• 我如何描述 eBPF 程序的哪些区域（如果有）是密集的？
• 这是 XDP 的处理时间限制，还是要记住的最佳时间？
• 我的 eBPF 程序正常吗？

谢谢。

Answer 1

对于 BPF XDP 钩子，每个数据包开销的最常见来源是：

• JIT 编译器被禁用。您可以检查 /proc/sys/net/core/bpf_jit_enable 的值。
• 驱动程序不支持 XDP。您需要检查您正在使用的特定驱动程序和内核版本。

正如 cmets 中所讨论的，您属于第二种情况。您的程序连接到不支持 XDP 驱动程序模式的 TUN 设备。这意味着你的 BPF 程序在 skb 分配之后运行，性能不会比在 tc 钩子上好多少。