Merge branch 'fec-XDP_TX' (78c53eaa) · Commits · jan.koester / Linux

drivers/net/ethernet/freescale/fec.h

+2 −4

Original line number	Original line	Diff line number	Diff line
	@@ -548,13 +548,11 @@ enum {
	enum fec_txbuf_type {		enum fec_txbuf_type {
	FEC_TXBUF_T_SKB,		FEC_TXBUF_T_SKB,
	FEC_TXBUF_T_XDP_NDO,		FEC_TXBUF_T_XDP_NDO,
			FEC_TXBUF_T_XDP_TX,
	};		};

	struct fec_tx_buffer {		struct fec_tx_buffer {
	union {		void *buf_p;
	struct sk_buff *skb;
	struct xdp_frame *xdp;
	};
	enum fec_txbuf_type type;		enum fec_txbuf_type type;
	};		};

drivers/net/ethernet/freescale/fec_main.c

+130 −57

Original line number	Original line	Diff line number	Diff line
	@@ -69,6 +69,7 @@
	#include <soc/imx/cpuidle.h>		#include <soc/imx/cpuidle.h>
	#include <linux/filter.h>		#include <linux/filter.h>
	#include <linux/bpf.h>		#include <linux/bpf.h>
			#include <linux/bpf_trace.h>

	#include <asm/cacheflush.h>		#include <asm/cacheflush.h>

	@@ -76,6 +77,9 @@

	static void set_multicast_list(struct net_device *ndev);		static void set_multicast_list(struct net_device *ndev);
	static void fec_enet_itr_coal_set(struct net_device *ndev);		static void fec_enet_itr_coal_set(struct net_device *ndev);
			static int fec_enet_xdp_tx_xmit(struct fec_enet_private *fep,
			int cpu, struct xdp_buff *xdp,
			u32 dma_sync_len);

	#define DRIVER_NAME "fec"		#define DRIVER_NAME "fec"

	@@ -396,7 +400,7 @@ static void fec_dump(struct net_device *ndev)
	fec16_to_cpu(bdp->cbd_sc),		fec16_to_cpu(bdp->cbd_sc),
	fec32_to_cpu(bdp->cbd_bufaddr),		fec32_to_cpu(bdp->cbd_bufaddr),
	fec16_to_cpu(bdp->cbd_datlen),		fec16_to_cpu(bdp->cbd_datlen),
	txq->tx_buf[index].skb);		txq->tx_buf[index].buf_p);
	bdp = fec_enet_get_nextdesc(bdp, &txq->bd);		bdp = fec_enet_get_nextdesc(bdp, &txq->bd);
	index++;		index++;
	} while (bdp != txq->bd.base);		} while (bdp != txq->bd.base);
	@@ -653,7 +657,7 @@ static int fec_enet_txq_submit_skb(struct fec_enet_priv_tx_q *txq,

	index = fec_enet_get_bd_index(last_bdp, &txq->bd);		index = fec_enet_get_bd_index(last_bdp, &txq->bd);
	/* Save skb pointer */		/* Save skb pointer */
	txq->tx_buf[index].skb = skb;		txq->tx_buf[index].buf_p = skb;

	/* Make sure the updates to rest of the descriptor are performed before		/* Make sure the updates to rest of the descriptor are performed before
	* transferring ownership.		* transferring ownership.
	@@ -859,7 +863,7 @@ static int fec_enet_txq_submit_tso(struct fec_enet_priv_tx_q *txq,
	}		}

	/* Save skb pointer */		/* Save skb pointer */
	txq->tx_buf[index].skb = skb;		txq->tx_buf[index].buf_p = skb;

	skb_tx_timestamp(skb);		skb_tx_timestamp(skb);
	txq->bd.cur = bdp;		txq->bd.cur = bdp;
	@@ -956,26 +960,27 @@ static void fec_enet_bd_init(struct net_device *dev)
	fec32_to_cpu(bdp->cbd_bufaddr),		fec32_to_cpu(bdp->cbd_bufaddr),
	fec16_to_cpu(bdp->cbd_datlen),		fec16_to_cpu(bdp->cbd_datlen),
	DMA_TO_DEVICE);		DMA_TO_DEVICE);
	if (txq->tx_buf[i].skb) {		if (txq->tx_buf[i].buf_p)
	dev_kfree_skb_any(txq->tx_buf[i].skb);		dev_kfree_skb_any(txq->tx_buf[i].buf_p);
	txq->tx_buf[i].skb = NULL;		} else if (txq->tx_buf[i].type == FEC_TXBUF_T_XDP_NDO) {
	}
	} else {
	if (bdp->cbd_bufaddr)		if (bdp->cbd_bufaddr)
	dma_unmap_single(&fep->pdev->dev,		dma_unmap_single(&fep->pdev->dev,
	fec32_to_cpu(bdp->cbd_bufaddr),		fec32_to_cpu(bdp->cbd_bufaddr),
	fec16_to_cpu(bdp->cbd_datlen),		fec16_to_cpu(bdp->cbd_datlen),
	DMA_TO_DEVICE);		DMA_TO_DEVICE);

	if (txq->tx_buf[i].xdp) {		if (txq->tx_buf[i].buf_p)
	xdp_return_frame(txq->tx_buf[i].xdp);		xdp_return_frame(txq->tx_buf[i].buf_p);
	txq->tx_buf[i].xdp = NULL;		} else {
			struct page *page = txq->tx_buf[i].buf_p;

			if (page)
			page_pool_put_page(page->pp, page, 0, false);
	}		}

			txq->tx_buf[i].buf_p = NULL;
	/* restore default tx buffer type: FEC_TXBUF_T_SKB */		/* restore default tx buffer type: FEC_TXBUF_T_SKB */
	txq->tx_buf[i].type = FEC_TXBUF_T_SKB;		txq->tx_buf[i].type = FEC_TXBUF_T_SKB;
	}

	bdp->cbd_bufaddr = cpu_to_fec32(0);		bdp->cbd_bufaddr = cpu_to_fec32(0);
	bdp = fec_enet_get_nextdesc(bdp, &txq->bd);		bdp = fec_enet_get_nextdesc(bdp, &txq->bd);
	}		}
	@@ -1382,6 +1387,8 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id, int budget)
	struct netdev_queue *nq;		struct netdev_queue *nq;
	int index = 0;		int index = 0;
	int entries_free;		int entries_free;
			struct page *page;
			int frame_len;

	fep = netdev_priv(ndev);		fep = netdev_priv(ndev);

	@@ -1403,8 +1410,7 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id, int budget)
	index = fec_enet_get_bd_index(bdp, &txq->bd);		index = fec_enet_get_bd_index(bdp, &txq->bd);

	if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB) {		if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB) {
	skb = txq->tx_buf[index].skb;		skb = txq->tx_buf[index].buf_p;
	txq->tx_buf[index].skb = NULL;
	if (bdp->cbd_bufaddr &&		if (bdp->cbd_bufaddr &&
	!IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr)))		!IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr)))
	dma_unmap_single(&fep->pdev->dev,		dma_unmap_single(&fep->pdev->dev,
	@@ -1423,17 +1429,24 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id, int budget)
	if (unlikely(!budget))		if (unlikely(!budget))
	break;		break;

	xdpf = txq->tx_buf[index].xdp;		if (txq->tx_buf[index].type == FEC_TXBUF_T_XDP_NDO) {
			xdpf = txq->tx_buf[index].buf_p;
	if (bdp->cbd_bufaddr)		if (bdp->cbd_bufaddr)
	dma_unmap_single(&fep->pdev->dev,		dma_unmap_single(&fep->pdev->dev,
	fec32_to_cpu(bdp->cbd_bufaddr),		fec32_to_cpu(bdp->cbd_bufaddr),
	fec16_to_cpu(bdp->cbd_datlen),		fec16_to_cpu(bdp->cbd_datlen),
	DMA_TO_DEVICE);		DMA_TO_DEVICE);
			} else {
			page = txq->tx_buf[index].buf_p;
			}

	bdp->cbd_bufaddr = cpu_to_fec32(0);		bdp->cbd_bufaddr = cpu_to_fec32(0);
	if (!xdpf) {		if (unlikely(!txq->tx_buf[index].buf_p)) {
	txq->tx_buf[index].type = FEC_TXBUF_T_SKB;		txq->tx_buf[index].type = FEC_TXBUF_T_SKB;
	goto tx_buf_done;		goto tx_buf_done;
	}		}

			frame_len = fec16_to_cpu(bdp->cbd_datlen);
	}		}

	/* Check for errors. */		/* Check for errors. */
	@@ -1457,7 +1470,7 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id, int budget)
	if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB)		if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB)
	ndev->stats.tx_bytes += skb->len;		ndev->stats.tx_bytes += skb->len;
	else		else
	ndev->stats.tx_bytes += xdpf->len;		ndev->stats.tx_bytes += frame_len;
	}		}

	/* Deferred means some collisions occurred during transmit,		/* Deferred means some collisions occurred during transmit,
	@@ -1482,13 +1495,16 @@ fec_enet_tx_queue(struct net_device *ndev, u16 queue_id, int budget)

	/* Free the sk buffer associated with this last transmit */		/* Free the sk buffer associated with this last transmit */
	dev_kfree_skb_any(skb);		dev_kfree_skb_any(skb);
	} else {		} else if (txq->tx_buf[index].type == FEC_TXBUF_T_XDP_NDO) {
	xdp_return_frame(xdpf);		xdp_return_frame_rx_napi(xdpf);
			} else { /* recycle pages of XDP_TX frames */
			/* The dma_sync_size = 0 as XDP_TX has already synced DMA for_device */
			page_pool_put_page(page->pp, page, 0, true);
			}

	txq->tx_buf[index].xdp = NULL;		txq->tx_buf[index].buf_p = NULL;
	/* restore default tx buffer type: FEC_TXBUF_T_SKB */		/* restore default tx buffer type: FEC_TXBUF_T_SKB */
	txq->tx_buf[index].type = FEC_TXBUF_T_SKB;		txq->tx_buf[index].type = FEC_TXBUF_T_SKB;
	}

	tx_buf_done:		tx_buf_done:
	/* Make sure the update to bdp and tx_buf are performed		/* Make sure the update to bdp and tx_buf are performed
	@@ -1542,7 +1558,7 @@ static void fec_enet_update_cbd(struct fec_enet_priv_rx_q *rxq,

	static u32		static u32
	fec_enet_run_xdp(struct fec_enet_private fep, struct bpf_prog prog,		fec_enet_run_xdp(struct fec_enet_private fep, struct bpf_prog prog,
	struct xdp_buff xdp, struct fec_enet_priv_rx_q rxq, int index)		struct xdp_buff xdp, struct fec_enet_priv_rx_q rxq, int cpu)
	{		{
	unsigned int sync, len = xdp->data_end - xdp->data;		unsigned int sync, len = xdp->data_end - xdp->data;
	u32 ret = FEC_ENET_XDP_PASS;		u32 ret = FEC_ENET_XDP_PASS;
	@@ -1552,8 +1568,10 @@ fec_enet_run_xdp(struct fec_enet_private fep, struct bpf_prog prog,

	act = bpf_prog_run_xdp(prog, xdp);		act = bpf_prog_run_xdp(prog, xdp);

	/* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */		/* Due xdp_adjust_tail and xdp_adjust_head: DMA sync for_device cover
	sync = xdp->data_end - xdp->data_hard_start - FEC_ENET_XDP_HEADROOM;		* max len CPU touch
			*/
			sync = xdp->data_end - xdp->data;
	sync = max(sync, len);		sync = max(sync, len);

	switch (act) {		switch (act) {
	@@ -1574,11 +1592,19 @@ fec_enet_run_xdp(struct fec_enet_private fep, struct bpf_prog prog,
	}		}
	break;		break;

	default:
	bpf_warn_invalid_xdp_action(fep->netdev, prog, act);
	fallthrough;

	case XDP_TX:		case XDP_TX:
			err = fec_enet_xdp_tx_xmit(fep, cpu, xdp, sync);
			if (unlikely(err)) {
			ret = FEC_ENET_XDP_CONSUMED;
			page = virt_to_head_page(xdp->data);
			page_pool_put_page(rxq->page_pool, page, sync, true);
			trace_xdp_exception(fep->netdev, prog, act);
			} else {
			ret = FEC_ENET_XDP_TX;
			}
			break;

			default:
	bpf_warn_invalid_xdp_action(fep->netdev, prog, act);		bpf_warn_invalid_xdp_action(fep->netdev, prog, act);
	fallthrough;		fallthrough;

	@@ -1620,6 +1646,7 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
	struct bpf_prog *xdp_prog = READ_ONCE(fep->xdp_prog);		struct bpf_prog *xdp_prog = READ_ONCE(fep->xdp_prog);
	u32 ret, xdp_result = FEC_ENET_XDP_PASS;		u32 ret, xdp_result = FEC_ENET_XDP_PASS;
	u32 data_start = FEC_ENET_XDP_HEADROOM;		u32 data_start = FEC_ENET_XDP_HEADROOM;
			int cpu = smp_processor_id();
	struct xdp_buff xdp;		struct xdp_buff xdp;
	struct page *page;		struct page *page;
	u32 sub_len = 4;		u32 sub_len = 4;
	@@ -1698,7 +1725,7 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
	/* subtract 16bit shift and FCS */		/* subtract 16bit shift and FCS */
	xdp_prepare_buff(&xdp, page_address(page),		xdp_prepare_buff(&xdp, page_address(page),
	data_start, pkt_len - sub_len, false);		data_start, pkt_len - sub_len, false);
	ret = fec_enet_run_xdp(fep, xdp_prog, &xdp, rxq, index);		ret = fec_enet_run_xdp(fep, xdp_prog, &xdp, rxq, cpu);
	xdp_result \|= ret;		xdp_result \|= ret;
	if (ret != FEC_ENET_XDP_PASS)		if (ret != FEC_ENET_XDP_PASS)
	goto rx_processing_done;		goto rx_processing_done;
	@@ -3208,7 +3235,6 @@ static void fec_enet_free_buffers(struct net_device *ndev)
	{		{
	struct fec_enet_private *fep = netdev_priv(ndev);		struct fec_enet_private *fep = netdev_priv(ndev);
	unsigned int i;		unsigned int i;
	struct sk_buff *skb;
	struct fec_enet_priv_tx_q *txq;		struct fec_enet_priv_tx_q *txq;
	struct fec_enet_priv_rx_q *rxq;		struct fec_enet_priv_rx_q *rxq;
	unsigned int q;		unsigned int q;
	@@ -3233,21 +3259,26 @@ static void fec_enet_free_buffers(struct net_device *ndev)
	kfree(txq->tx_bounce[i]);		kfree(txq->tx_bounce[i]);
	txq->tx_bounce[i] = NULL;		txq->tx_bounce[i] = NULL;

			if (!txq->tx_buf[i].buf_p) {
			txq->tx_buf[i].type = FEC_TXBUF_T_SKB;
			continue;
			}

	if (txq->tx_buf[i].type == FEC_TXBUF_T_SKB) {		if (txq->tx_buf[i].type == FEC_TXBUF_T_SKB) {
	skb = txq->tx_buf[i].skb;		dev_kfree_skb(txq->tx_buf[i].buf_p);
	txq->tx_buf[i].skb = NULL;		} else if (txq->tx_buf[i].type == FEC_TXBUF_T_XDP_NDO) {
	dev_kfree_skb(skb);		xdp_return_frame(txq->tx_buf[i].buf_p);
	} else {		} else {
	if (txq->tx_buf[i].xdp) {		struct page *page = txq->tx_buf[i].buf_p;
	xdp_return_frame(txq->tx_buf[i].xdp);
	txq->tx_buf[i].xdp = NULL;		page_pool_put_page(page->pp, page, 0, false);
	}		}

			txq->tx_buf[i].buf_p = NULL;
	txq->tx_buf[i].type = FEC_TXBUF_T_SKB;		txq->tx_buf[i].type = FEC_TXBUF_T_SKB;
	}		}
	}		}
	}		}
	}

	static void fec_enet_free_queue(struct net_device *ndev)		static void fec_enet_free_queue(struct net_device *ndev)
	{		{
	@@ -3767,12 +3798,14 @@ fec_enet_xdp_get_tx_queue(struct fec_enet_private *fep, int index)

	static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep,		static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep,
	struct fec_enet_priv_tx_q *txq,		struct fec_enet_priv_tx_q *txq,
	struct xdp_frame *frame)		void *frame, u32 dma_sync_len,
			bool ndo_xmit)
	{		{
	unsigned int index, status, estatus;		unsigned int index, status, estatus;
	struct bufdesc *bdp;		struct bufdesc *bdp;
	dma_addr_t dma_addr;		dma_addr_t dma_addr;
	int entries_free;		int entries_free;
			u16 frame_len;

	entries_free = fec_enet_get_free_txdesc_num(txq);		entries_free = fec_enet_get_free_txdesc_num(txq);
	if (entries_free < MAX_SKB_FRAGS + 1) {		if (entries_free < MAX_SKB_FRAGS + 1) {
	@@ -3787,17 +3820,37 @@ static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep,

	index = fec_enet_get_bd_index(bdp, &txq->bd);		index = fec_enet_get_bd_index(bdp, &txq->bd);

	dma_addr = dma_map_single(&fep->pdev->dev, frame->data,		if (ndo_xmit) {
	frame->len, DMA_TO_DEVICE);		struct xdp_frame *xdpf = frame;

			dma_addr = dma_map_single(&fep->pdev->dev, xdpf->data,
			xdpf->len, DMA_TO_DEVICE);
	if (dma_mapping_error(&fep->pdev->dev, dma_addr))		if (dma_mapping_error(&fep->pdev->dev, dma_addr))
	return -ENOMEM;		return -ENOMEM;

			frame_len = xdpf->len;
			txq->tx_buf[index].buf_p = xdpf;
			txq->tx_buf[index].type = FEC_TXBUF_T_XDP_NDO;
			} else {
			struct xdp_buff *xdpb = frame;
			struct page *page;

			page = virt_to_page(xdpb->data);
			dma_addr = page_pool_get_dma_addr(page) +
			(xdpb->data - xdpb->data_hard_start);
			dma_sync_single_for_device(&fep->pdev->dev, dma_addr,
			dma_sync_len, DMA_BIDIRECTIONAL);
			frame_len = xdpb->data_end - xdpb->data;
			txq->tx_buf[index].buf_p = page;
			txq->tx_buf[index].type = FEC_TXBUF_T_XDP_TX;
			}

	status \|= (BD_ENET_TX_INTR \| BD_ENET_TX_LAST);		status \|= (BD_ENET_TX_INTR \| BD_ENET_TX_LAST);
	if (fep->bufdesc_ex)		if (fep->bufdesc_ex)
	estatus = BD_ENET_TX_INT;		estatus = BD_ENET_TX_INT;

	bdp->cbd_bufaddr = cpu_to_fec32(dma_addr);		bdp->cbd_bufaddr = cpu_to_fec32(dma_addr);
	bdp->cbd_datlen = cpu_to_fec16(frame->len);		bdp->cbd_datlen = cpu_to_fec16(frame_len);

	if (fep->bufdesc_ex) {		if (fep->bufdesc_ex) {
	struct bufdesc_ex ebdp = (struct bufdesc_ex )bdp;		struct bufdesc_ex ebdp = (struct bufdesc_ex )bdp;
	@@ -3809,9 +3862,6 @@ static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep,
	ebdp->cbd_esc = cpu_to_fec32(estatus);		ebdp->cbd_esc = cpu_to_fec32(estatus);
	}		}

	txq->tx_buf[index].type = FEC_TXBUF_T_XDP_NDO;
	txq->tx_buf[index].xdp = frame;

	/* Make sure the updates to rest of the descriptor are performed before		/* Make sure the updates to rest of the descriptor are performed before
	* transferring ownership.		* transferring ownership.
	*/		*/
	@@ -3837,6 +3887,29 @@ static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep,
	return 0;		return 0;
	}		}

			static int fec_enet_xdp_tx_xmit(struct fec_enet_private *fep,
			int cpu, struct xdp_buff *xdp,
			u32 dma_sync_len)
			{
			struct fec_enet_priv_tx_q *txq;
			struct netdev_queue *nq;
			int queue, ret;

			queue = fec_enet_xdp_get_tx_queue(fep, cpu);
			txq = fep->tx_queue[queue];
			nq = netdev_get_tx_queue(fep->netdev, queue);

			__netif_tx_lock(nq, cpu);

			/* Avoid tx timeout as XDP shares the queue with kernel stack */
			txq_trans_cond_update(nq);
			ret = fec_enet_txq_xmit_frame(fep, txq, xdp, dma_sync_len, false);

			__netif_tx_unlock(nq);

			return ret;
			}

	static int fec_enet_xdp_xmit(struct net_device *dev,		static int fec_enet_xdp_xmit(struct net_device *dev,
	int num_frames,		int num_frames,
	struct xdp_frame **frames,		struct xdp_frame **frames,
	@@ -3859,7 +3932,7 @@ static int fec_enet_xdp_xmit(struct net_device *dev,
	/* Avoid tx timeout as XDP shares the queue with kernel stack */		/* Avoid tx timeout as XDP shares the queue with kernel stack */
	txq_trans_cond_update(nq);		txq_trans_cond_update(nq);
	for (i = 0; i < num_frames; i++) {		for (i = 0; i < num_frames; i++) {
	if (fec_enet_txq_xmit_frame(fep, txq, frames[i]) < 0)		if (fec_enet_txq_xmit_frame(fep, txq, frames[i], 0, true) < 0)
	break;		break;
	sent_frames++;		sent_frames++;
	}		}