Skip to content

ixgbe: convert Rx to libeth and page_pool #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Magda-Pytel wants to merge 15 commits into
base: ixgbevf-xdp-public
Choose a base branch
from
Open

ixgbe: convert Rx to libeth and page_pool #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
ad388e4
ixgbevf: implement AF_XDP ZC initialization
walking-machine Oct 27, 2025
a0ee7f6
ixgbevf: implement AF_XDP zero-copy Tx
walking-machine Oct 27, 2025
bcdbbdd
ixgbevf: implement AF_XDP zero-copy Rx
walking-machine Oct 27, 2025
e828d3a
ixgbevf: implement .ndo_xsk_wakeup() and set features
walking-machine Oct 27, 2025
3176c41
ixgbevf: multi-buffer AF_XDP Tx
walking-machine Dec 19, 2025
6dc49d1
ixgbe: remove legacy rx
Magda-Pytel Apr 13, 2026
3536c94
ixgbe: do not share pages between packets
Magda-Pytel May 7, 2026
4d7562c
ixgbe: use libeth in Rx processing
Magda-Pytel May 7, 2026
97d1245
ixgbe: branch prediction and cleanup
Magda-Pytel May 28, 2026
d3a0b9d
ixgbe: support XDP multi-buffer on Rx path
Magda-Pytel Jun 1, 2026
ee84bfa
ixgbe: XDP_TX in multi-buffer through libeth
Magda-Pytel Jun 1, 2026
8b4bb7a
ixgbe: support XDP_REDIRECT and .ndo_xdp_xmit through libeth
Magda-Pytel Jun 1, 2026
fcb4918
ixgbe: add pseudo header split
Magda-Pytel Jun 1, 2026
6313904
ixgbe: reconfigure page pool when reallocating buffers
Magda-Pytel Jun 1, 2026
e0b9442
ixgbe: allow changing MTU when XDP program is attached
Magda-Pytel Jun 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions drivers/net/ethernet/intel/Kconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ config IXGBE
tristate "Intel(R) 10GbE PCI Express adapters support"
depends on PCI
depends on PTP_1588_CLOCK_OPTIONAL
select LIBETH_XDP
select LIBIE_FWLOG if DEBUG_FS
select MDIO
select NET_DEVLINK
Expand Down
156 changes: 37 additions & 119 deletions drivers/net/ethernet/intel/ixgbe/ixgbe.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
#include "ixgbe_ipsec.h"

#include <net/xdp.h>
#include <net/libeth/rx.h>
#include <net/libeth/types.h>

/* common prefix used by pr_<> macros */
#undef pr_fmt
Expand Down Expand Up @@ -69,65 +71,14 @@

/* Supported Rx Buffer Sizes */
#define IXGBE_RXBUFFER_256 256 /* Used for skb receive header */
#define IXGBE_RXBUFFER_1536 1536
#define IXGBE_RXBUFFER_2K 2048
#define IXGBE_RXBUFFER_3K 3072
#define IXGBE_RXBUFFER_4K 4096
#define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */

#define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2))

/* Attempt to maximize the headroom available for incoming frames. We
* use a 2K buffer for receives and need 1536/1534 to store the data for
* the frame. This leaves us with 512 bytes of room. From that we need
* to deduct the space needed for the shared info and the padding needed
* to IP align the frame.
*
* Note: For cache line sizes 256 or larger this value is going to end
* up negative. In these cases we should fall back to the 3K
* buffers.
*/
#if (PAGE_SIZE < 8192)
#define IXGBE_MAX_2K_FRAME_BUILD_SKB (IXGBE_RXBUFFER_1536 - NET_IP_ALIGN)
#define IXGBE_2K_TOO_SMALL_WITH_PADDING \
((NET_SKB_PAD + IXGBE_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IXGBE_RXBUFFER_2K))

static inline int ixgbe_compute_pad(int rx_buf_len)
{
int page_size, pad_size;

page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2);
pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len;

return pad_size;
}

static inline int ixgbe_skb_pad(void)
{
int rx_buf_len;

/* If a 2K buffer cannot handle a standard Ethernet frame then
* optimize padding for a 3K buffer instead of a 1.5K buffer.
*
* For a 3K buffer we need to add enough padding to allow for
* tailroom due to NET_IP_ALIGN possibly shifting us out of
* cache-line alignment.
*/
if (IXGBE_2K_TOO_SMALL_WITH_PADDING)
rx_buf_len = IXGBE_RXBUFFER_3K + SKB_DATA_ALIGN(NET_IP_ALIGN);
else
rx_buf_len = IXGBE_RXBUFFER_1536;

/* if needed make room for NET_IP_ALIGN */
rx_buf_len -= NET_IP_ALIGN;

return ixgbe_compute_pad(rx_buf_len);
}

#define IXGBE_SKB_PAD ixgbe_skb_pad()
#else
#define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN)
#endif
#define IXGBE_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \
IXGBE_SRRCTL_BSIZEPKT_STEP))
#define IXGBE_RX_SRRCTL_BUF_SIZE(mtu) (ALIGN((mtu) + LIBETH_RX_LL_LEN, \
IXGBE_SRRCTL_BSIZEPKT_STEP))

/*
* NOTE: netdev_alloc_skb reserves up to 64 bytes, NET_IP_ALIGN means we
Expand Down Expand Up @@ -271,20 +222,9 @@ struct ixgbe_tx_buffer {
u32 tx_flags;
};

struct ixgbe_rx_buffer {
union {
struct {
struct sk_buff *skb;
dma_addr_t dma;
struct page *page;
__u32 page_offset;
__u16 pagecnt_bias;
};
struct {
bool discard;
struct xdp_buff *xdp;
};
};
struct ixgbe_xsk_rx_buffer {
bool discard;
struct xdp_buff *xdp;
};

struct ixgbe_queue_stats {
Expand All @@ -311,8 +251,6 @@ struct ixgbe_rx_queue_stats {
#define IXGBE_TS_HDR_LEN 8

enum ixgbe_ring_state_t {
__IXGBE_RX_3K_BUFFER,
__IXGBE_RX_BUILD_SKB_ENABLED,
__IXGBE_RX_RSC_ENABLED,
__IXGBE_RX_CSUM_UDP_ZERO_ERR,
__IXGBE_RX_FCOE,
Expand All @@ -321,12 +259,10 @@ enum ixgbe_ring_state_t {
__IXGBE_TX_DETECT_HANG,
__IXGBE_HANG_CHECK_ARMED,
__IXGBE_TX_XDP_RING,
__IXGBE_TX_XDP_RING_PRIMED,
__IXGBE_TX_DISABLED,
};

#define ring_uses_build_skb(ring) \
test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &(ring)->state)

struct ixgbe_fwd_adapter {
unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
struct net_device *netdev;
Expand Down Expand Up @@ -357,12 +293,22 @@ struct ixgbe_ring {
struct ixgbe_ring *next; /* pointer to next ring in q_vector */
struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */
struct net_device *netdev; /* netdev ring belongs to */
struct bpf_prog *xdp_prog;
struct device *dev; /* device for DMA mapping */
struct bpf_prog __rcu *xdp_prog;

@walking-machine walking-machine Jul 21, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ring->xdp_prog is still accessed though xdp_prog = READ_ONCE(rx_ring->xdp_prog);, but rcu_dereference() should be used.
rx_ring->xdp_prog = NULL; could also utilize rcu_assign_pointer() for consistency, although AFAIK sparse does not compain about that. Neither it complains about accesses like !!rx_ring->xdp_prog, so those can stay as-is, although I think rcu_access_pointer() can be used there (rcu_dereference() is only for NAPI and other high priority contexts).

union {
struct page_pool *pp; /* Rx and XDP rings */
struct device *dev; /* Tx ring */
};
void *desc; /* descriptor ring memory */
union {
u32 truesize; /* Rx buffer full size */
u32 pending; /* Sent-not-completed descriptors */
};
u32 hdr_truesize; /* Rx header buffer full size */
union {
struct libeth_fqe *rx_fqes;
struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info;
struct ixgbe_tx_buffer *tx_buffer_info;
struct ixgbe_rx_buffer *rx_buffer_info;
struct libeth_sqe *xdp_sqes;
};
unsigned long state;
u8 __iomem *tail;
Expand All @@ -377,17 +323,14 @@ struct ixgbe_ring {
* associated with this ring, which is
* different for DCB and RSS modes
*/
u16 next_to_use;
u16 next_to_clean;
u32 next_to_use;

unsigned long last_rx_timestamp;

union {
u16 next_to_alloc;
struct {
u8 atr_sample_rate;
u8 atr_count;
};
struct {
u8 atr_sample_rate;
u8 atr_count;
};

u8 dcb_tc;
Expand All @@ -397,12 +340,17 @@ struct ixgbe_ring {
struct ixgbe_tx_queue_stats tx_stats;
struct ixgbe_rx_queue_stats rx_stats;
};
u16 rx_offset;
struct libeth_fqe *hdr_fqes;
struct page_pool *hdr_pp;
struct xdp_rxq_info xdp_rxq;
struct libeth_xdpsq_lock xdpq_lock;
u32 cached_ntu;
spinlock_t tx_lock; /* used in XDP mode */
struct xsk_buff_pool *xsk_pool;
u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */
u16 rx_buf_len;
u32 hdr_buf_len;
u32 rx_buf_len;
struct libeth_xdp_buff_stash xdp_stash;
} ____cacheline_internodealigned_in_smp;

enum ixgbe_ring_f_enum {
Expand Down Expand Up @@ -433,8 +381,6 @@ enum ixgbe_ring_f_enum {
#define IXGBE_MAX_TX_DESCRIPTORS 40
#define IXGBE_MAX_TX_VF_HANGS 4

DECLARE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key);

struct ixgbe_ring_feature {
u16 limit; /* upper limit on feature indices */
u16 indices; /* current value of indices */
Expand All @@ -446,31 +392,6 @@ struct ixgbe_ring_feature {
#define IXGBE_82599_VMDQ_4Q_MASK 0x7C
#define IXGBE_82599_VMDQ_2Q_MASK 0x7E

/*
* FCoE requires that all Rx buffers be over 2200 bytes in length. Since
* this is twice the size of a half page we need to double the page order
* for FCoE enabled Rx queues.
*/
static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring)

@walking-machine walking-machine Jul 21, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deletion of this function leads to compilation errors

drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c: In function ‘ixgbe_clean_test_rings’:
drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c:2122:41: error: implicit declaration of function ‘ixgbe_rx_bufsz’ [-Wimplicit-function-declaration]
 2122 |                                         ixgbe_rx_bufsz(rx_ring),
      |                                         ^~~~~~~~~~~~~~
make[7]: *** [scripts/Makefile.build:289: drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.o] Error 1

diff to fix:

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
index eaf50ce47fe5..c5ad98ca1a66 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
@@ -2119,7 +2119,7 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring,
                /* sync Rx buffer for CPU read */
                dma_sync_single_for_cpu(rx_ring->dev,
                                        rx_buffer->dma,
-                                       ixgbe_rx_bufsz(rx_ring),
+                                       IXGBE_RXBUFFER_3K,
                                        DMA_FROM_DEVICE);

                /* verify contents of skb */
@@ -2131,7 +2131,7 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring,
                /* sync Rx buffer for device write */
                dma_sync_single_for_device(rx_ring->dev,
                                           rx_buffer->dma,
-                                          ixgbe_rx_bufsz(rx_ring),
+                                          IXGBE_RXBUFFER_3K,
                                           DMA_FROM_DEVICE);

                /* increment Rx next to clean counter */

{
if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
return IXGBE_RXBUFFER_3K;
#if (PAGE_SIZE < 8192)
if (ring_uses_build_skb(ring))
return IXGBE_MAX_2K_FRAME_BUILD_SKB;
#endif
return IXGBE_RXBUFFER_2K;
}

static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring)
{
#if (PAGE_SIZE < 8192)
if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
return 1;
#endif
return 0;
}
#define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring))

Comment thread
Magda-Pytel marked this conversation as resolved.
#define IXGBE_ITR_ADAPTIVE_MIN_INC 2
#define IXGBE_ITR_ADAPTIVE_MIN_USECS 10
Expand Down Expand Up @@ -672,7 +593,7 @@ struct ixgbe_adapter {
#define IXGBE_FLAG2_VLAN_PROMISC BIT(13)
#define IXGBE_FLAG2_EEE_CAPABLE BIT(14)
#define IXGBE_FLAG2_EEE_ENABLED BIT(15)
#define IXGBE_FLAG2_RX_LEGACY BIT(16)
/* BIT16 used to be reserved for legacy RX flag */
#define IXGBE_FLAG2_IPSEC_ENABLED BIT(17)
#define IXGBE_FLAG2_VF_IPSEC_ENABLED BIT(18)
#define IXGBE_FLAG2_AUTO_DISABLE_VF BIT(19)
Expand All @@ -681,6 +602,7 @@ struct ixgbe_adapter {
#define IXGBE_FLAG2_MOD_POWER_UNSUPPORTED BIT(22)
#define IXGBE_FLAG2_API_MISMATCH BIT(23)
#define IXGBE_FLAG2_FW_ROLLBACK BIT(24)
#define IXGBE_FLAG2_HSPLIT BIT(25)

/* Tx fast path data */
int num_tx_queues;
Expand Down Expand Up @@ -859,10 +781,7 @@ static inline struct ixgbe_adapter *ixgbe_from_netdev(struct net_device *netdev)

static inline int ixgbe_determine_xdp_q_idx(int cpu)
{
if (static_key_enabled(&ixgbe_xdp_locking_key))
return cpu % IXGBE_MAX_XDP_QS;
else
return cpu;
return cpu % IXGBE_MAX_XDP_QS;
}

static inline
Expand Down Expand Up @@ -919,7 +838,6 @@ struct ixgbe_cb {
};
dma_addr_t dma;
u16 append_cnt;
bool page_released;
};
#define IXGBE_CB(skb) ((struct ixgbe_cb *)(skb)->cb)

Expand Down
36 changes: 11 additions & 25 deletions drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,11 +132,9 @@ static const char ixgbe_gstrings_test[][ETH_GSTRING_LEN] = {
#define IXGBE_TEST_LEN sizeof(ixgbe_gstrings_test) / ETH_GSTRING_LEN

static const char ixgbe_priv_flags_strings[][ETH_GSTRING_LEN] = {
#define IXGBE_PRIV_FLAGS_LEGACY_RX BIT(0)
"legacy-rx",
#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN BIT(1)
#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN BIT(0)
"vf-ipsec",
#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF BIT(2)
#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF BIT(1)
"mdd-disable-vf",
};

Expand Down Expand Up @@ -1951,7 +1949,6 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter)
/* Setup Rx Descriptor ring and Rx buffers */
rx_ring->count = IXGBE_DEFAULT_RXD;
rx_ring->queue_index = 0;
rx_ring->dev = &adapter->pdev->dev;

@walking-machine walking-machine Jul 21, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is still ring->dev = ... for an Rx ring in ixgbe_alloc_q_vector(). This is a mistake that you have admittedly inherited from me ;)

rx_ring->netdev = adapter->netdev;
rx_ring->reg_idx = adapter->rx_ring[0]->reg_idx;

Expand Down Expand Up @@ -2058,14 +2055,16 @@ static void ixgbe_create_lbtest_frame(struct sk_buff *skb,
skb->data[frame_size + 12] = 0xAF;
}

static bool ixgbe_check_lbtest_frame(struct ixgbe_rx_buffer *rx_buffer,
static bool ixgbe_check_lbtest_frame(const struct libeth_fqe *rx_buffer,
unsigned int frame_size)
{
u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset;
unsigned char *data;

frame_size >>= 1;

data = page_address(rx_buffer->page) + rx_buffer->page_offset;
data = page_address(__netmem_to_page(rx_buffer->netmem)) +
rx_buffer->offset + hr;

return data[3] == 0xFF && data[frame_size + 10] == 0xBE &&
data[frame_size + 12] == 0xAF;
Expand Down Expand Up @@ -2113,28 +2112,22 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring,
}

while (rx_desc->wb.upper.length) {
struct ixgbe_rx_buffer *rx_buffer;
struct libeth_fqe *rx_buffer;

/* check Rx buffer */
rx_buffer = &rx_ring->rx_buffer_info[rx_ntc];
rx_buffer = &rx_ring->rx_fqes[rx_ntc];

/* sync Rx buffer for CPU read */
dma_sync_single_for_cpu(rx_ring->dev,
rx_buffer->dma,
ixgbe_rx_bufsz(rx_ring),
DMA_FROM_DEVICE);
libeth_rx_sync_for_cpu(rx_buffer, rx_ring->rx_buf_len);

/* verify contents of skb */
if (ixgbe_check_lbtest_frame(rx_buffer, size))
count++;
else
break;

/* sync Rx buffer for device write */
dma_sync_single_for_device(rx_ring->dev,
rx_buffer->dma,
ixgbe_rx_bufsz(rx_ring),
DMA_FROM_DEVICE);
/* recycle the page back to the pool */
libeth_rx_recycle_slow(rx_buffer->netmem);

/* increment Rx next to clean counter */
rx_ntc++;
Expand Down Expand Up @@ -3667,9 +3660,6 @@ static u32 ixgbe_get_priv_flags(struct net_device *netdev)
struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
u32 priv_flags = 0;

if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY)
priv_flags |= IXGBE_PRIV_FLAGS_LEGACY_RX;

if (adapter->flags2 & IXGBE_FLAG2_VF_IPSEC_ENABLED)
priv_flags |= IXGBE_PRIV_FLAGS_VF_IPSEC_EN;

Expand All @@ -3685,10 +3675,6 @@ static int ixgbe_set_priv_flags(struct net_device *netdev, u32 priv_flags)
unsigned int flags2 = adapter->flags2;
unsigned int i;

flags2 &= ~IXGBE_FLAG2_RX_LEGACY;
if (priv_flags & IXGBE_PRIV_FLAGS_LEGACY_RX)
flags2 |= IXGBE_FLAG2_RX_LEGACY;

flags2 &= ~IXGBE_FLAG2_VF_IPSEC_ENABLED;
if (priv_flags & IXGBE_PRIV_FLAGS_VF_IPSEC_EN)
flags2 |= IXGBE_FLAG2_VF_IPSEC_ENABLED;
Expand Down
Loading