From 9d77db3c730780c8ef5ccd4b66403ff5675cfe4e Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Mon, 13 May 2024 10:30:14 +0000
Subject: [PATCH] modify sin led gpio
---
kernel/drivers/net/ethernet/marvell/mvneta.c | 1233 ++++++++++++++++++++++++++++++++++++++++++++--------------
1 files changed, 936 insertions(+), 297 deletions(-)
diff --git a/kernel/drivers/net/ethernet/marvell/mvneta.c b/kernel/drivers/net/ethernet/marvell/mvneta.c
index 382d010..3656a39 100644
--- a/kernel/drivers/net/ethernet/marvell/mvneta.c
+++ b/kernel/drivers/net/ethernet/marvell/mvneta.c
@@ -27,6 +27,7 @@
#include <linux/of_irq.h>
#include <linux/of_mdio.h>
#include <linux/of_net.h>
+#include <linux/phy/phy.h>
#include <linux/phy.h>
#include <linux/phylink.h>
#include <linux/platform_device.h>
@@ -36,6 +37,8 @@
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/tso.h>
+#include <net/page_pool.h>
+#include <linux/bpf_trace.h>
/* Registers */
#define MVNETA_RXQ_CONFIG_REG(q) (0x1400 + ((q) << 2))
@@ -103,9 +106,11 @@
#define MVNETA_TX_IN_PRGRS BIT(0)
#define MVNETA_TX_FIFO_EMPTY BIT(8)
#define MVNETA_RX_MIN_FRAME_SIZE 0x247c
+/* Only exists on Armada XP and Armada 370 */
#define MVNETA_SERDES_CFG 0x24A0
#define MVNETA_SGMII_SERDES_PROTO 0x0cc7
#define MVNETA_QSGMII_SERDES_PROTO 0x0667
+#define MVNETA_HSGMII_SERDES_PROTO 0x1107
#define MVNETA_TYPE_PRIO 0x24bc
#define MVNETA_FORCE_UNI BIT(21)
#define MVNETA_TXQ_CMD_1 0x24e4
@@ -221,6 +226,8 @@
#define MVNETA_GMAC_AN_FLOW_CTRL_EN BIT(11)
#define MVNETA_GMAC_CONFIG_FULL_DUPLEX BIT(12)
#define MVNETA_GMAC_AN_DUPLEX_EN BIT(13)
+#define MVNETA_GMAC_CTRL_4 0x2c90
+#define MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE BIT(1)
#define MVNETA_MIB_COUNTERS_BASE 0x3000
#define MVNETA_MIB_LATE_COLLISION 0x7c
#define MVNETA_DA_FILT_SPEC_MCAST 0x3400
@@ -319,6 +326,12 @@
ETH_HLEN + ETH_FCS_LEN, \
cache_line_size())
+/* Driver assumes that the last 3 bits are 0 */
+#define MVNETA_SKB_HEADROOM ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8)
+#define MVNETA_SKB_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info) + \
+ MVNETA_SKB_HEADROOM))
+#define MVNETA_MAX_RX_BUF_SIZE (PAGE_SIZE - MVNETA_SKB_PAD)
+
#define IS_TSO_HEADER(txq, addr) \
((addr >= txq->tso_hdrs_phys) && \
(addr < txq->tso_hdrs_phys + txq->size * TSO_HEADER_SIZE))
@@ -330,6 +343,13 @@
ETHTOOL_STAT_EEE_WAKEUP,
ETHTOOL_STAT_SKB_ALLOC_ERR,
ETHTOOL_STAT_REFILL_ERR,
+ ETHTOOL_XDP_REDIRECT,
+ ETHTOOL_XDP_PASS,
+ ETHTOOL_XDP_DROP,
+ ETHTOOL_XDP_TX,
+ ETHTOOL_XDP_TX_ERR,
+ ETHTOOL_XDP_XMIT,
+ ETHTOOL_XDP_XMIT_ERR,
ETHTOOL_MAX_STATS,
};
@@ -342,6 +362,11 @@
#define T_REG_32 32
#define T_REG_64 64
#define T_SW 1
+
+#define MVNETA_XDP_PASS 0
+#define MVNETA_XDP_DROPPED BIT(0)
+#define MVNETA_XDP_TX BIT(1)
+#define MVNETA_XDP_REDIR BIT(2)
static const struct mvneta_statistic mvneta_statistics[] = {
{ 0x3000, T_REG_64, "good_octets_received", },
@@ -379,16 +404,42 @@
{ ETHTOOL_STAT_EEE_WAKEUP, T_SW, "eee_wakeup_errors", },
{ ETHTOOL_STAT_SKB_ALLOC_ERR, T_SW, "skb_alloc_errors", },
{ ETHTOOL_STAT_REFILL_ERR, T_SW, "refill_errors", },
+ { ETHTOOL_XDP_REDIRECT, T_SW, "rx_xdp_redirect", },
+ { ETHTOOL_XDP_PASS, T_SW, "rx_xdp_pass", },
+ { ETHTOOL_XDP_DROP, T_SW, "rx_xdp_drop", },
+ { ETHTOOL_XDP_TX, T_SW, "rx_xdp_tx", },
+ { ETHTOOL_XDP_TX_ERR, T_SW, "rx_xdp_tx_errors", },
+ { ETHTOOL_XDP_XMIT, T_SW, "tx_xdp_xmit", },
+ { ETHTOOL_XDP_XMIT_ERR, T_SW, "tx_xdp_xmit_errors", },
+};
+
+struct mvneta_stats {
+ u64 rx_packets;
+ u64 rx_bytes;
+ u64 tx_packets;
+ u64 tx_bytes;
+ /* xdp */
+ u64 xdp_redirect;
+ u64 xdp_pass;
+ u64 xdp_drop;
+ u64 xdp_xmit;
+ u64 xdp_xmit_err;
+ u64 xdp_tx;
+ u64 xdp_tx_err;
+};
+
+struct mvneta_ethtool_stats {
+ struct mvneta_stats ps;
+ u64 skb_alloc_error;
+ u64 refill_error;
};
struct mvneta_pcpu_stats {
- struct u64_stats_sync syncp;
- u64 rx_packets;
- u64 rx_bytes;
+ struct u64_stats_sync syncp;
+
+ struct mvneta_ethtool_stats es;
u64 rx_dropped;
u64 rx_errors;
- u64 tx_packets;
- u64 tx_bytes;
};
struct mvneta_pcpu_port {
@@ -402,10 +453,16 @@
u32 cause_rx_tx;
};
+enum {
+ __MVNETA_DOWN,
+};
+
struct mvneta_port {
u8 id;
struct mvneta_pcpu_port __percpu *ports;
struct mvneta_pcpu_stats __percpu *stats;
+
+ unsigned long state;
int pkt_size;
void __iomem *base;
@@ -424,6 +481,8 @@
u32 cause_rx_tx;
struct napi_struct napi;
+ struct bpf_prog *xdp_prog;
+
/* Core clock */
struct clk *clk;
/* AXI clock */
@@ -436,6 +495,8 @@
struct device_node *dn;
unsigned int tx_csum_limit;
struct phylink *phylink;
+ struct phylink_config phylink_config;
+ struct phy *comphy;
struct mvneta_bm *bm_priv;
struct mvneta_bm_pool *pool_long;
@@ -493,7 +554,7 @@
#if defined(__LITTLE_ENDIAN)
struct mvneta_tx_desc {
u32 command; /* Options used by HW for packet transmitting.*/
- u16 reserverd1; /* csum_l4 (for future use) */
+ u16 reserved1; /* csum_l4 (for future use) */
u16 data_size; /* Data size of transmitted packet in bytes */
u32 buf_phys_addr; /* Physical addr of transmitted buffer */
u32 reserved2; /* hw_cmd - (for future use, PMT) */
@@ -518,7 +579,7 @@
#else
struct mvneta_tx_desc {
u16 data_size; /* Data size of transmitted packet in bytes */
- u16 reserverd1; /* csum_l4 (for future use) */
+ u16 reserved1; /* csum_l4 (for future use) */
u32 command; /* Options used by HW for packet transmitting.*/
u32 reserved2; /* hw_cmd - (for future use, PMT) */
u32 buf_phys_addr; /* Physical addr of transmitted buffer */
@@ -542,6 +603,20 @@
};
#endif
+enum mvneta_tx_buf_type {
+ MVNETA_TYPE_SKB,
+ MVNETA_TYPE_XDP_TX,
+ MVNETA_TYPE_XDP_NDO,
+};
+
+struct mvneta_tx_buf {
+ enum mvneta_tx_buf_type type;
+ union {
+ struct xdp_frame *xdpf;
+ struct sk_buff *skb;
+ };
+};
+
struct mvneta_tx_queue {
/* Number of this TX queue, in the range 0-7 */
u8 id;
@@ -557,8 +632,8 @@
int tx_stop_threshold;
int tx_wake_threshold;
- /* Array of transmitted skb */
- struct sk_buff **tx_skb;
+ /* Array of transmitted buffers */
+ struct mvneta_tx_buf *buf;
/* Index of last TX DMA descriptor that was inserted */
int txq_put_index;
@@ -600,6 +675,10 @@
u32 pkts_coal;
u32 time_coal;
+ /* page_pool */
+ struct page_pool *page_pool;
+ struct xdp_rxq_info xdp_rxq;
+
/* Virtual address of the RX buffer */
void **buf_virt_addr;
@@ -618,14 +697,6 @@
/* Index of first RX DMA descriptor to refill */
int first_to_refill;
u32 refill_num;
-
- /* pointer to uncomplete skb buffer */
- struct sk_buff *skb;
- int left_size;
-
- /* error counters */
- u32 skb_alloc_err;
- u32 refill_err;
};
static enum cpuhp_state online_hpstate;
@@ -638,7 +709,6 @@
static int rxq_def;
static int rx_copybreak __read_mostly = 256;
-static int rx_header_size __read_mostly = 128;
/* HW BM need that each port be identify by a unique ID */
static int global_port_id;
@@ -681,13 +751,12 @@
static void mvneta_mib_counters_clear(struct mvneta_port *pp)
{
int i;
- u32 dummy;
/* Perform dummy reads from MIB counters */
for (i = 0; i < MVNETA_MIB_LATE_COLLISION; i += 4)
- dummy = mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i));
- dummy = mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT);
- dummy = mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT);
+ mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i));
+ mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT);
+ mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT);
}
/* Get System Network Statistics */
@@ -711,12 +780,12 @@
cpu_stats = per_cpu_ptr(pp->stats, cpu);
do {
start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
- rx_packets = cpu_stats->rx_packets;
- rx_bytes = cpu_stats->rx_bytes;
+ rx_packets = cpu_stats->es.ps.rx_packets;
+ rx_bytes = cpu_stats->es.ps.rx_bytes;
rx_dropped = cpu_stats->rx_dropped;
rx_errors = cpu_stats->rx_errors;
- tx_packets = cpu_stats->tx_packets;
- tx_bytes = cpu_stats->tx_bytes;
+ tx_packets = cpu_stats->es.ps.tx_packets;
+ tx_bytes = cpu_stats->es.ps.tx_bytes;
} while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
stats->rx_packets += rx_packets;
@@ -1119,7 +1188,7 @@
SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(bm_pool->pkt_size));
/* Fill entire long pool */
- num = hwbm_pool_add(hwbm_pool, hwbm_pool->size, GFP_ATOMIC);
+ num = hwbm_pool_add(hwbm_pool, hwbm_pool->size);
if (num != hwbm_pool->size) {
WARN(1, "pool %d: %d of %d allocated\n",
bm_pool->id, num, hwbm_pool->size);
@@ -1134,6 +1203,7 @@
mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 1 << pp->id);
pp->bm_priv = NULL;
+ pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
mvreg_write(pp, MVNETA_ACC_MODE, MVNETA_ACC_MODE_EXT1);
netdev_info(pp->dev, "fail to update MTU, fall back to software BM\n");
}
@@ -1401,7 +1471,7 @@
*/
if (txq_number == 1)
txq_map = (cpu == pp->rxq_def) ?
- MVNETA_CPU_TXQ_ACCESS(1) : 0;
+ MVNETA_CPU_TXQ_ACCESS(0) : 0;
} else {
txq_map = MVNETA_CPU_TXQ_ACCESS_ALL_MASK;
@@ -1761,30 +1831,34 @@
/* Free tx queue skbuffs */
static void mvneta_txq_bufs_free(struct mvneta_port *pp,
struct mvneta_tx_queue *txq, int num,
- struct netdev_queue *nq)
+ struct netdev_queue *nq, bool napi)
{
unsigned int bytes_compl = 0, pkts_compl = 0;
int i;
for (i = 0; i < num; i++) {
+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_get_index];
struct mvneta_tx_desc *tx_desc = txq->descs +
txq->txq_get_index;
- struct sk_buff *skb = txq->tx_skb[txq->txq_get_index];
-
- if (skb) {
- bytes_compl += skb->len;
- pkts_compl++;
- }
mvneta_txq_inc_get(txq);
- if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr))
+ if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr) &&
+ buf->type != MVNETA_TYPE_XDP_TX)
dma_unmap_single(pp->dev->dev.parent,
tx_desc->buf_phys_addr,
tx_desc->data_size, DMA_TO_DEVICE);
- if (!skb)
- continue;
- dev_kfree_skb_any(skb);
+ if (buf->type == MVNETA_TYPE_SKB && buf->skb) {
+ bytes_compl += buf->skb->len;
+ pkts_compl++;
+ dev_kfree_skb_any(buf->skb);
+ } else if (buf->type == MVNETA_TYPE_XDP_TX ||
+ buf->type == MVNETA_TYPE_XDP_NDO) {
+ if (napi && buf->type == MVNETA_TYPE_XDP_TX)
+ xdp_return_frame_rx_napi(buf->xdpf);
+ else
+ xdp_return_frame(buf->xdpf);
+ }
}
netdev_tx_completed_queue(nq, pkts_compl, bytes_compl);
@@ -1801,7 +1875,7 @@
if (!tx_done)
return;
- mvneta_txq_bufs_free(pp, txq, tx_done, nq);
+ mvneta_txq_bufs_free(pp, txq, tx_done, nq, true);
txq->count -= tx_done;
@@ -1821,20 +1895,14 @@
dma_addr_t phys_addr;
struct page *page;
- page = __dev_alloc_page(gfp_mask);
+ page = page_pool_alloc_pages(rxq->page_pool,
+ gfp_mask | __GFP_NOWARN);
if (!page)
return -ENOMEM;
- /* map page for use */
- phys_addr = dma_map_page(pp->dev->dev.parent, page, 0, PAGE_SIZE,
- DMA_FROM_DEVICE);
- if (unlikely(dma_mapping_error(pp->dev->dev.parent, phys_addr))) {
- __free_page(page);
- return -ENOMEM;
- }
-
- phys_addr += pp->rx_offset_correction;
+ phys_addr = page_pool_get_dma_addr(page) + pp->rx_offset_correction;
mvneta_rx_desc_fill(rx_desc, phys_addr, page, rxq);
+
return 0;
}
@@ -1900,10 +1968,28 @@
if (!data || !(rx_desc->buf_phys_addr))
continue;
- dma_unmap_page(pp->dev->dev.parent, rx_desc->buf_phys_addr,
- PAGE_SIZE, DMA_FROM_DEVICE);
- __free_page(data);
+ page_pool_put_full_page(rxq->page_pool, data, false);
}
+ if (xdp_rxq_info_is_reg(&rxq->xdp_rxq))
+ xdp_rxq_info_unreg(&rxq->xdp_rxq);
+ page_pool_destroy(rxq->page_pool);
+ rxq->page_pool = NULL;
+}
+
+static void
+mvneta_update_stats(struct mvneta_port *pp,
+ struct mvneta_stats *ps)
+{
+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
+
+ u64_stats_update_begin(&stats->syncp);
+ stats->es.ps.rx_packets += ps->rx_packets;
+ stats->es.ps.rx_bytes += ps->rx_bytes;
+ /* xdp */
+ stats->es.ps.xdp_redirect += ps->xdp_redirect;
+ stats->es.ps.xdp_pass += ps->xdp_pass;
+ stats->es.ps.xdp_drop += ps->xdp_drop;
+ u64_stats_update_end(&stats->syncp);
}
static inline
@@ -1917,9 +2003,15 @@
rx_desc = rxq->descs + curr_desc;
if (!(rx_desc->buf_phys_addr)) {
if (mvneta_rx_refill(pp, rx_desc, rxq, GFP_ATOMIC)) {
+ struct mvneta_pcpu_stats *stats;
+
pr_err("Can't refill queue %d. Done %d from %d\n",
rxq->id, i, rxq->refill_num);
- rxq->refill_err++;
+
+ stats = this_cpu_ptr(pp->stats);
+ u64_stats_update_begin(&stats->syncp);
+ stats->es.refill_error++;
+ u64_stats_update_end(&stats->syncp);
break;
}
}
@@ -1931,38 +2023,346 @@
return i;
}
+static void
+mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
+ struct xdp_buff *xdp, int sync_len, bool napi)
+{
+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
+ int i;
+
+ for (i = 0; i < sinfo->nr_frags; i++)
+ page_pool_put_full_page(rxq->page_pool,
+ skb_frag_page(&sinfo->frags[i]), napi);
+ page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data),
+ sync_len, napi);
+}
+
+static int
+mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq,
+ struct xdp_frame *xdpf, bool dma_map)
+{
+ struct mvneta_tx_desc *tx_desc;
+ struct mvneta_tx_buf *buf;
+ dma_addr_t dma_addr;
+
+ if (txq->count >= txq->tx_stop_threshold)
+ return MVNETA_XDP_DROPPED;
+
+ tx_desc = mvneta_txq_next_desc_get(txq);
+
+ buf = &txq->buf[txq->txq_put_index];
+ if (dma_map) {
+ /* ndo_xdp_xmit */
+ dma_addr = dma_map_single(pp->dev->dev.parent, xdpf->data,
+ xdpf->len, DMA_TO_DEVICE);
+ if (dma_mapping_error(pp->dev->dev.parent, dma_addr)) {
+ mvneta_txq_desc_put(txq);
+ return MVNETA_XDP_DROPPED;
+ }
+ buf->type = MVNETA_TYPE_XDP_NDO;
+ } else {
+ struct page *page = virt_to_page(xdpf->data);
+
+ dma_addr = page_pool_get_dma_addr(page) +
+ sizeof(*xdpf) + xdpf->headroom;
+ dma_sync_single_for_device(pp->dev->dev.parent, dma_addr,
+ xdpf->len, DMA_BIDIRECTIONAL);
+ buf->type = MVNETA_TYPE_XDP_TX;
+ }
+ buf->xdpf = xdpf;
+
+ tx_desc->command = MVNETA_TXD_FLZ_DESC;
+ tx_desc->buf_phys_addr = dma_addr;
+ tx_desc->data_size = xdpf->len;
+
+ mvneta_txq_inc_put(txq);
+ txq->pending++;
+ txq->count++;
+
+ return MVNETA_XDP_TX;
+}
+
+static int
+mvneta_xdp_xmit_back(struct mvneta_port *pp, struct xdp_buff *xdp)
+{
+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
+ struct mvneta_tx_queue *txq;
+ struct netdev_queue *nq;
+ struct xdp_frame *xdpf;
+ int cpu;
+ u32 ret;
+
+ xdpf = xdp_convert_buff_to_frame(xdp);
+ if (unlikely(!xdpf))
+ return MVNETA_XDP_DROPPED;
+
+ cpu = smp_processor_id();
+ txq = &pp->txqs[cpu % txq_number];
+ nq = netdev_get_tx_queue(pp->dev, txq->id);
+
+ __netif_tx_lock(nq, cpu);
+ ret = mvneta_xdp_submit_frame(pp, txq, xdpf, false);
+ if (ret == MVNETA_XDP_TX) {
+ u64_stats_update_begin(&stats->syncp);
+ stats->es.ps.tx_bytes += xdpf->len;
+ stats->es.ps.tx_packets++;
+ stats->es.ps.xdp_tx++;
+ u64_stats_update_end(&stats->syncp);
+
+ mvneta_txq_pend_desc_add(pp, txq, 0);
+ } else {
+ u64_stats_update_begin(&stats->syncp);
+ stats->es.ps.xdp_tx_err++;
+ u64_stats_update_end(&stats->syncp);
+ }
+ __netif_tx_unlock(nq);
+
+ return ret;
+}
+
+static int
+mvneta_xdp_xmit(struct net_device *dev, int num_frame,
+ struct xdp_frame **frames, u32 flags)
+{
+ struct mvneta_port *pp = netdev_priv(dev);
+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
+ int i, nxmit_byte = 0, nxmit = num_frame;
+ int cpu = smp_processor_id();
+ struct mvneta_tx_queue *txq;
+ struct netdev_queue *nq;
+ u32 ret;
+
+ if (unlikely(test_bit(__MVNETA_DOWN, &pp->state)))
+ return -ENETDOWN;
+
+ if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
+ return -EINVAL;
+
+ txq = &pp->txqs[cpu % txq_number];
+ nq = netdev_get_tx_queue(pp->dev, txq->id);
+
+ __netif_tx_lock(nq, cpu);
+ for (i = 0; i < num_frame; i++) {
+ ret = mvneta_xdp_submit_frame(pp, txq, frames[i], true);
+ if (ret == MVNETA_XDP_TX) {
+ nxmit_byte += frames[i]->len;
+ } else {
+ xdp_return_frame_rx_napi(frames[i]);
+ nxmit--;
+ }
+ }
+
+ if (unlikely(flags & XDP_XMIT_FLUSH))
+ mvneta_txq_pend_desc_add(pp, txq, 0);
+ __netif_tx_unlock(nq);
+
+ u64_stats_update_begin(&stats->syncp);
+ stats->es.ps.tx_bytes += nxmit_byte;
+ stats->es.ps.tx_packets += nxmit;
+ stats->es.ps.xdp_xmit += nxmit;
+ stats->es.ps.xdp_xmit_err += num_frame - nxmit;
+ u64_stats_update_end(&stats->syncp);
+
+ return nxmit;
+}
+
+static int
+mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
+ struct bpf_prog *prog, struct xdp_buff *xdp,
+ u32 frame_sz, struct mvneta_stats *stats)
+{
+ unsigned int len, data_len, sync;
+ u32 ret, act;
+
+ len = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction;
+ data_len = xdp->data_end - xdp->data;
+ act = bpf_prog_run_xdp(prog, xdp);
+
+ /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */
+ sync = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction;
+ sync = max(sync, len);
+
+ switch (act) {
+ case XDP_PASS:
+ stats->xdp_pass++;
+ return MVNETA_XDP_PASS;
+ case XDP_REDIRECT: {
+ int err;
+
+ err = xdp_do_redirect(pp->dev, xdp, prog);
+ if (unlikely(err)) {
+ mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
+ ret = MVNETA_XDP_DROPPED;
+ } else {
+ ret = MVNETA_XDP_REDIR;
+ stats->xdp_redirect++;
+ }
+ break;
+ }
+ case XDP_TX:
+ ret = mvneta_xdp_xmit_back(pp, xdp);
+ if (ret != MVNETA_XDP_TX)
+ mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
+ break;
+ default:
+ bpf_warn_invalid_xdp_action(act);
+ fallthrough;
+ case XDP_ABORTED:
+ trace_xdp_exception(pp->dev, prog, act);
+ fallthrough;
+ case XDP_DROP:
+ mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
+ ret = MVNETA_XDP_DROPPED;
+ stats->xdp_drop++;
+ break;
+ }
+
+ stats->rx_bytes += frame_sz + xdp->data_end - xdp->data - data_len;
+ stats->rx_packets++;
+
+ return ret;
+}
+
+static void
+mvneta_swbm_rx_frame(struct mvneta_port *pp,
+ struct mvneta_rx_desc *rx_desc,
+ struct mvneta_rx_queue *rxq,
+ struct xdp_buff *xdp, int *size,
+ struct page *page)
+{
+ unsigned char *data = page_address(page);
+ int data_len = -MVNETA_MH_SIZE, len;
+ struct net_device *dev = pp->dev;
+ enum dma_data_direction dma_dir;
+ struct skb_shared_info *sinfo;
+
+ if (*size > MVNETA_MAX_RX_BUF_SIZE) {
+ len = MVNETA_MAX_RX_BUF_SIZE;
+ data_len += len;
+ } else {
+ len = *size;
+ data_len += len - ETH_FCS_LEN;
+ }
+ *size = *size - len;
+
+ dma_dir = page_pool_get_dma_dir(rxq->page_pool);
+ dma_sync_single_for_cpu(dev->dev.parent,
+ rx_desc->buf_phys_addr,
+ len, dma_dir);
+
+ rx_desc->buf_phys_addr = 0;
+
+ /* Prefetch header */
+ prefetch(data);
+
+ xdp->data_hard_start = data;
+ xdp->data = data + pp->rx_offset_correction + MVNETA_MH_SIZE;
+ xdp->data_end = xdp->data + data_len;
+ xdp_set_data_meta_invalid(xdp);
+
+ sinfo = xdp_get_shared_info_from_buff(xdp);
+ sinfo->nr_frags = 0;
+}
+
+static void
+mvneta_swbm_add_rx_fragment(struct mvneta_port *pp,
+ struct mvneta_rx_desc *rx_desc,
+ struct mvneta_rx_queue *rxq,
+ struct xdp_buff *xdp, int *size,
+ struct page *page)
+{
+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
+ struct net_device *dev = pp->dev;
+ enum dma_data_direction dma_dir;
+ int data_len, len;
+
+ if (*size > MVNETA_MAX_RX_BUF_SIZE) {
+ len = MVNETA_MAX_RX_BUF_SIZE;
+ data_len = len;
+ } else {
+ len = *size;
+ data_len = len - ETH_FCS_LEN;
+ }
+ dma_dir = page_pool_get_dma_dir(rxq->page_pool);
+ dma_sync_single_for_cpu(dev->dev.parent,
+ rx_desc->buf_phys_addr,
+ len, dma_dir);
+ rx_desc->buf_phys_addr = 0;
+
+ if (data_len > 0 && sinfo->nr_frags < MAX_SKB_FRAGS) {
+ skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags];
+
+ skb_frag_off_set(frag, pp->rx_offset_correction);
+ skb_frag_size_set(frag, data_len);
+ __skb_frag_set_page(frag, page);
+ sinfo->nr_frags++;
+ } else {
+ page_pool_put_full_page(rxq->page_pool, page, true);
+ }
+ *size -= len;
+}
+
+static struct sk_buff *
+mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
+ struct xdp_buff *xdp, u32 desc_status)
+{
+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
+ int i, num_frags = sinfo->nr_frags;
+ struct sk_buff *skb;
+
+ skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
+ if (!skb)
+ return ERR_PTR(-ENOMEM);
+
+ page_pool_release_page(rxq->page_pool, virt_to_page(xdp->data));
+
+ skb_reserve(skb, xdp->data - xdp->data_hard_start);
+ skb_put(skb, xdp->data_end - xdp->data);
+ mvneta_rx_csum(pp, desc_status, skb);
+
+ for (i = 0; i < num_frags; i++) {
+ skb_frag_t *frag = &sinfo->frags[i];
+
+ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+ skb_frag_page(frag), skb_frag_off(frag),
+ skb_frag_size(frag), PAGE_SIZE);
+ page_pool_release_page(rxq->page_pool, skb_frag_page(frag));
+ }
+
+ return skb;
+}
+
/* Main rx processing when using software buffer management */
static int mvneta_rx_swbm(struct napi_struct *napi,
struct mvneta_port *pp, int budget,
struct mvneta_rx_queue *rxq)
{
+ int rx_proc = 0, rx_todo, refill, size = 0;
struct net_device *dev = pp->dev;
- int rx_todo, rx_proc;
- int refill = 0;
- u32 rcvd_pkts = 0;
- u32 rcvd_bytes = 0;
+ struct xdp_buff xdp_buf = {
+ .frame_sz = PAGE_SIZE,
+ .rxq = &rxq->xdp_rxq,
+ };
+ struct mvneta_stats ps = {};
+ struct bpf_prog *xdp_prog;
+ u32 desc_status, frame_sz;
/* Get number of received packets */
rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);
- rx_proc = 0;
+
+ rcu_read_lock();
+ xdp_prog = READ_ONCE(pp->xdp_prog);
/* Fairness NAPI loop */
- while ((rcvd_pkts < budget) && (rx_proc < rx_todo)) {
+ while (rx_proc < budget && rx_proc < rx_todo) {
struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
- unsigned char *data;
- struct page *page;
- dma_addr_t phys_addr;
u32 rx_status, index;
- int rx_bytes, skb_size, copy_size;
- int frag_num, frag_size, frag_offset;
+ struct sk_buff *skb;
+ struct page *page;
index = rx_desc - rxq->descs;
page = (struct page *)rxq->buf_virt_addr[index];
- data = page_address(page);
- /* Prefetch header */
- prefetch(data);
- phys_addr = rx_desc->buf_phys_addr;
rx_status = rx_desc->status;
rx_proc++;
rxq->refill_num++;
@@ -1971,132 +2371,72 @@
/* Check errors only for FIRST descriptor */
if (rx_status & MVNETA_RXD_ERR_SUMMARY) {
mvneta_rx_error(pp, rx_desc);
- /* leave the descriptor untouched */
- continue;
+ goto next;
}
- rx_bytes = rx_desc->data_size -
- (ETH_FCS_LEN + MVNETA_MH_SIZE);
- /* Allocate small skb for each new packet */
- skb_size = max(rx_copybreak, rx_header_size);
- rxq->skb = netdev_alloc_skb_ip_align(dev, skb_size);
- if (unlikely(!rxq->skb)) {
- struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
+ size = rx_desc->data_size;
+ frame_sz = size - ETH_FCS_LEN;
+ desc_status = rx_status;
- netdev_err(dev,
- "Can't allocate skb on queue %d\n",
- rxq->id);
-
- rxq->skb_alloc_err++;
-
- u64_stats_update_begin(&stats->syncp);
- stats->rx_dropped++;
- u64_stats_update_end(&stats->syncp);
- continue;
- }
- copy_size = min(skb_size, rx_bytes);
-
- /* Copy data from buffer to SKB, skip Marvell header */
- memcpy(rxq->skb->data, data + MVNETA_MH_SIZE,
- copy_size);
- skb_put(rxq->skb, copy_size);
- rxq->left_size = rx_bytes - copy_size;
-
- mvneta_rx_csum(pp, rx_status, rxq->skb);
- if (rxq->left_size == 0) {
- int size = copy_size + MVNETA_MH_SIZE;
-
- dma_sync_single_range_for_cpu(dev->dev.parent,
- phys_addr, 0,
- size,
- DMA_FROM_DEVICE);
-
- /* leave the descriptor and buffer untouched */
- } else {
- /* refill descriptor with new buffer later */
- rx_desc->buf_phys_addr = 0;
-
- frag_num = 0;
- frag_offset = copy_size + MVNETA_MH_SIZE;
- frag_size = min(rxq->left_size,
- (int)(PAGE_SIZE - frag_offset));
- skb_add_rx_frag(rxq->skb, frag_num, page,
- frag_offset, frag_size,
- PAGE_SIZE);
- dma_unmap_page(dev->dev.parent, phys_addr,
- PAGE_SIZE, DMA_FROM_DEVICE);
- rxq->left_size -= frag_size;
- }
+ mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,
+ &size, page);
} else {
- /* Middle or Last descriptor */
- if (unlikely(!rxq->skb)) {
- pr_debug("no skb for rx_status 0x%x\n",
- rx_status);
+ if (unlikely(!xdp_buf.data_hard_start)) {
+ rx_desc->buf_phys_addr = 0;
+ page_pool_put_full_page(rxq->page_pool, page,
+ true);
continue;
}
- if (!rxq->left_size) {
- /* last descriptor has only FCS */
- /* and can be discarded */
- dma_sync_single_range_for_cpu(dev->dev.parent,
- phys_addr, 0,
- ETH_FCS_LEN,
- DMA_FROM_DEVICE);
- /* leave the descriptor and buffer untouched */
- } else {
- /* refill descriptor with new buffer later */
- rx_desc->buf_phys_addr = 0;
- frag_num = skb_shinfo(rxq->skb)->nr_frags;
- frag_offset = 0;
- frag_size = min(rxq->left_size,
- (int)(PAGE_SIZE - frag_offset));
- skb_add_rx_frag(rxq->skb, frag_num, page,
- frag_offset, frag_size,
- PAGE_SIZE);
-
- dma_unmap_page(dev->dev.parent, phys_addr,
- PAGE_SIZE, DMA_FROM_DEVICE);
-
- rxq->left_size -= frag_size;
- }
+ mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf,
+ &size, page);
} /* Middle or Last descriptor */
if (!(rx_status & MVNETA_RXD_LAST_DESC))
/* no last descriptor this time */
continue;
- if (rxq->left_size) {
- pr_err("get last desc, but left_size (%d) != 0\n",
- rxq->left_size);
- dev_kfree_skb_any(rxq->skb);
- rxq->left_size = 0;
- rxq->skb = NULL;
- continue;
+ if (size) {
+ mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
+ goto next;
}
- rcvd_pkts++;
- rcvd_bytes += rxq->skb->len;
- /* Linux processing */
- rxq->skb->protocol = eth_type_trans(rxq->skb, dev);
+ if (xdp_prog &&
+ mvneta_run_xdp(pp, rxq, xdp_prog, &xdp_buf, frame_sz, &ps))
+ goto next;
- if (dev->features & NETIF_F_GRO)
- napi_gro_receive(napi, rxq->skb);
- else
- netif_receive_skb(rxq->skb);
+ skb = mvneta_swbm_build_skb(pp, rxq, &xdp_buf, desc_status);
+ if (IS_ERR(skb)) {
+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
- /* clean uncomplete skb pointer in queue */
- rxq->skb = NULL;
- rxq->left_size = 0;
+ mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
+
+ u64_stats_update_begin(&stats->syncp);
+ stats->es.skb_alloc_error++;
+ stats->rx_dropped++;
+ u64_stats_update_end(&stats->syncp);
+
+ goto next;
+ }
+
+ ps.rx_bytes += skb->len;
+ ps.rx_packets++;
+
+ skb->protocol = eth_type_trans(skb, dev);
+ napi_gro_receive(napi, skb);
+next:
+ xdp_buf.data_hard_start = NULL;
}
+ rcu_read_unlock();
- if (rcvd_pkts) {
- struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
+ if (xdp_buf.data_hard_start)
+ mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
- u64_stats_update_begin(&stats->syncp);
- stats->rx_packets += rcvd_pkts;
- stats->rx_bytes += rcvd_bytes;
- u64_stats_update_end(&stats->syncp);
- }
+ if (ps.xdp_redirect)
+ xdp_do_flush_map();
+
+ if (ps.rx_packets)
+ mvneta_update_stats(pp, &ps);
/* return some buffers to hardware queue, one at a time is too slow */
refill = mvneta_rx_refill_queue(pp, rxq);
@@ -2104,7 +2444,7 @@
/* Update rxq management counters */
mvneta_rxq_desc_num_update(pp, rxq, rx_proc, refill);
- return rcvd_pkts;
+ return ps.rx_packets;
}
/* Main rx processing when using hardware buffer management */
@@ -2188,8 +2528,15 @@
/* Refill processing */
err = hwbm_pool_refill(&bm_pool->hwbm_pool, GFP_ATOMIC);
if (err) {
+ struct mvneta_pcpu_stats *stats;
+
netdev_err(dev, "Linux processing - Can't refill\n");
- rxq->refill_err++;
+
+ stats = this_cpu_ptr(pp->stats);
+ u64_stats_update_begin(&stats->syncp);
+ stats->es.refill_error++;
+ u64_stats_update_end(&stats->syncp);
+
goto err_drop_frame_ret_pool;
}
@@ -2223,8 +2570,8 @@
struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
u64_stats_update_begin(&stats->syncp);
- stats->rx_packets += rcvd_pkts;
- stats->rx_bytes += rcvd_bytes;
+ stats->es.ps.rx_packets += rcvd_pkts;
+ stats->es.ps.rx_bytes += rcvd_bytes;
u64_stats_update_end(&stats->syncp);
}
@@ -2238,16 +2585,19 @@
mvneta_tso_put_hdr(struct sk_buff *skb,
struct mvneta_port *pp, struct mvneta_tx_queue *txq)
{
- struct mvneta_tx_desc *tx_desc;
int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
+ struct mvneta_tx_desc *tx_desc;
- txq->tx_skb[txq->txq_put_index] = NULL;
tx_desc = mvneta_txq_next_desc_get(txq);
tx_desc->data_size = hdr_len;
tx_desc->command = mvneta_skb_tx_csum(pp, skb);
tx_desc->command |= MVNETA_TXD_F_DESC;
tx_desc->buf_phys_addr = txq->tso_hdrs_phys +
txq->txq_put_index * TSO_HEADER_SIZE;
+ buf->type = MVNETA_TYPE_SKB;
+ buf->skb = NULL;
+
mvneta_txq_inc_put(txq);
}
@@ -2256,6 +2606,7 @@
struct sk_buff *skb, char *data, int size,
bool last_tcp, bool is_last)
{
+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
struct mvneta_tx_desc *tx_desc;
tx_desc = mvneta_txq_next_desc_get(txq);
@@ -2269,7 +2620,8 @@
}
tx_desc->command = 0;
- txq->tx_skb[txq->txq_put_index] = NULL;
+ buf->type = MVNETA_TYPE_SKB;
+ buf->skb = NULL;
if (last_tcp) {
/* last descriptor in the TCP packet */
@@ -2277,7 +2629,7 @@
/* last descriptor in SKB */
if (is_last)
- txq->tx_skb[txq->txq_put_index] = skb;
+ buf->skb = skb;
}
mvneta_txq_inc_put(txq);
return 0;
@@ -2286,11 +2638,10 @@
static int mvneta_tx_tso(struct sk_buff *skb, struct net_device *dev,
struct mvneta_tx_queue *txq)
{
- int total_len, data_left;
+ int hdr_len, total_len, data_left;
int desc_count = 0;
struct mvneta_port *pp = netdev_priv(dev);
struct tso_t tso;
- int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
int i;
/* Count needed descriptors */
@@ -2303,7 +2654,7 @@
}
/* Initialize the TSO handler, and prepare the first payload */
- tso_start(skb, &tso);
+ hdr_len = tso_start(skb, &tso);
total_len = skb->len - hdr_len;
while (total_len > 0) {
@@ -2362,11 +2713,12 @@
int i, nr_frags = skb_shinfo(skb)->nr_frags;
for (i = 0; i < nr_frags; i++) {
+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
- void *addr = page_address(frag->page.p) + frag->page_offset;
+ void *addr = skb_frag_address(frag);
tx_desc = mvneta_txq_next_desc_get(txq);
- tx_desc->data_size = frag->size;
+ tx_desc->data_size = skb_frag_size(frag);
tx_desc->buf_phys_addr =
dma_map_single(pp->dev->dev.parent, addr,
@@ -2381,12 +2733,13 @@
if (i == nr_frags - 1) {
/* Last descriptor */
tx_desc->command = MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD;
- txq->tx_skb[txq->txq_put_index] = skb;
+ buf->skb = skb;
} else {
/* Descriptor in the middle: Not First, Not Last */
tx_desc->command = 0;
- txq->tx_skb[txq->txq_put_index] = NULL;
+ buf->skb = NULL;
}
+ buf->type = MVNETA_TYPE_SKB;
mvneta_txq_inc_put(txq);
}
@@ -2414,6 +2767,7 @@
struct mvneta_port *pp = netdev_priv(dev);
u16 txq_id = skb_get_queue_mapping(skb);
struct mvneta_tx_queue *txq = &pp->txqs[txq_id];
+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
struct mvneta_tx_desc *tx_desc;
int len = skb->len;
int frags = 0;
@@ -2446,16 +2800,17 @@
goto out;
}
+ buf->type = MVNETA_TYPE_SKB;
if (frags == 1) {
/* First and Last descriptor */
tx_cmd |= MVNETA_TXD_FLZ_DESC;
tx_desc->command = tx_cmd;
- txq->tx_skb[txq->txq_put_index] = skb;
+ buf->skb = skb;
mvneta_txq_inc_put(txq);
} else {
/* First but not Last */
tx_cmd |= MVNETA_TXD_F_DESC;
- txq->tx_skb[txq->txq_put_index] = NULL;
+ buf->skb = NULL;
mvneta_txq_inc_put(txq);
tx_desc->command = tx_cmd;
/* Continue with other skb fragments */
@@ -2472,8 +2827,8 @@
out:
if (frags > 0) {
- struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id);
+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
netdev_tx_sent_queue(nq, len);
@@ -2481,15 +2836,15 @@
if (txq->count >= txq->tx_stop_threshold)
netif_tx_stop_queue(nq);
- if (!skb->xmit_more || netif_xmit_stopped(nq) ||
+ if (!netdev_xmit_more() || netif_xmit_stopped(nq) ||
txq->pending + frags > MVNETA_TXQ_DEC_SENT_MASK)
mvneta_txq_pend_desc_add(pp, txq, frags);
else
txq->pending += frags;
u64_stats_update_begin(&stats->syncp);
- stats->tx_packets++;
- stats->tx_bytes += len;
+ stats->es.ps.tx_bytes += len;
+ stats->es.ps.tx_packets++;
u64_stats_update_end(&stats->syncp);
} else {
dev->stats.tx_dropped++;
@@ -2508,7 +2863,7 @@
struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id);
int tx_done = txq->count;
- mvneta_txq_bufs_free(pp, txq, tx_done, nq);
+ mvneta_txq_bufs_free(pp, txq, tx_done, nq, false);
/* reset txq */
txq->count = 0;
@@ -2523,12 +2878,13 @@
{
struct mvneta_tx_queue *txq;
struct netdev_queue *nq;
+ int cpu = smp_processor_id();
while (cause_tx_done) {
txq = mvneta_tx_done_policy(pp, cause_tx_done);
nq = netdev_get_tx_queue(pp->dev, txq->id);
- __netif_tx_lock(nq, smp_processor_id());
+ __netif_tx_lock(nq, cpu);
if (txq->count)
mvneta_txq_done(pp, txq);
@@ -2841,11 +3197,57 @@
return rx_done;
}
+static int mvneta_create_page_pool(struct mvneta_port *pp,
+ struct mvneta_rx_queue *rxq, int size)
+{
+ struct bpf_prog *xdp_prog = READ_ONCE(pp->xdp_prog);
+ struct page_pool_params pp_params = {
+ .order = 0,
+ .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+ .pool_size = size,
+ .nid = NUMA_NO_NODE,
+ .dev = pp->dev->dev.parent,
+ .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE,
+ .offset = pp->rx_offset_correction,
+ .max_len = MVNETA_MAX_RX_BUF_SIZE,
+ };
+ int err;
+
+ rxq->page_pool = page_pool_create(&pp_params);
+ if (IS_ERR(rxq->page_pool)) {
+ err = PTR_ERR(rxq->page_pool);
+ rxq->page_pool = NULL;
+ return err;
+ }
+
+ err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id);
+ if (err < 0)
+ goto err_free_pp;
+
+ err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL,
+ rxq->page_pool);
+ if (err)
+ goto err_unregister_rxq;
+
+ return 0;
+
+err_unregister_rxq:
+ xdp_rxq_info_unreg(&rxq->xdp_rxq);
+err_free_pp:
+ page_pool_destroy(rxq->page_pool);
+ rxq->page_pool = NULL;
+ return err;
+}
+
/* Handle rxq fill: allocates rxq skbs; called when initializing a port */
static int mvneta_rxq_fill(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
int num)
{
- int i;
+ int i, err;
+
+ err = mvneta_create_page_pool(pp, rxq, num);
+ if (err < 0)
+ return err;
for (i = 0; i < num; i++) {
memset(rxq->descs + i, 0, sizeof(struct mvneta_rx_desc));
@@ -2919,7 +3321,7 @@
/* Set Offset */
mvneta_rxq_offset_set(pp, rxq, 0);
mvneta_rxq_buf_size_set(pp, rxq, PAGE_SIZE < SZ_64K ?
- PAGE_SIZE :
+ MVNETA_MAX_RX_BUF_SIZE :
MVNETA_RX_BUF_SIZE(pp->pkt_size));
mvneta_rxq_bm_disable(pp, rxq);
mvneta_rxq_fill(pp, rxq, rxq->size);
@@ -2958,9 +3360,6 @@
{
mvneta_rxq_drop_pkts(pp, rxq);
- if (rxq->skb)
- dev_kfree_skb_any(rxq->skb);
-
if (rxq->descs)
dma_free_coherent(pp->dev->dev.parent,
rxq->size * MVNETA_DESC_ALIGNED_SIZE,
@@ -2973,8 +3372,6 @@
rxq->descs_phys = 0;
rxq->first_to_refill = 0;
rxq->refill_num = 0;
- rxq->skb = NULL;
- rxq->left_size = 0;
}
static int mvneta_txq_sw_init(struct mvneta_port *pp,
@@ -3000,26 +3397,16 @@
txq->last_desc = txq->size - 1;
- txq->tx_skb = kmalloc_array(txq->size, sizeof(*txq->tx_skb),
- GFP_KERNEL);
- if (!txq->tx_skb) {
- dma_free_coherent(pp->dev->dev.parent,
- txq->size * MVNETA_DESC_ALIGNED_SIZE,
- txq->descs, txq->descs_phys);
+ txq->buf = kmalloc_array(txq->size, sizeof(*txq->buf), GFP_KERNEL);
+ if (!txq->buf)
return -ENOMEM;
- }
/* Allocate DMA buffers for TSO MAC/IP/TCP headers */
txq->tso_hdrs = dma_alloc_coherent(pp->dev->dev.parent,
txq->size * TSO_HEADER_SIZE,
&txq->tso_hdrs_phys, GFP_KERNEL);
- if (!txq->tso_hdrs) {
- kfree(txq->tx_skb);
- dma_free_coherent(pp->dev->dev.parent,
- txq->size * MVNETA_DESC_ALIGNED_SIZE,
- txq->descs, txq->descs_phys);
+ if (!txq->tso_hdrs)
return -ENOMEM;
- }
/* Setup XPS mapping */
if (pp->neta_armada3700)
@@ -3069,7 +3456,7 @@
{
struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id);
- kfree(txq->tx_skb);
+ kfree(txq->buf);
if (txq->tso_hdrs)
dma_free_coherent(pp->dev->dev.parent,
@@ -3163,9 +3550,60 @@
return 0;
}
+static int mvneta_comphy_init(struct mvneta_port *pp, phy_interface_t interface)
+{
+ int ret;
+
+ ret = phy_set_mode_ext(pp->comphy, PHY_MODE_ETHERNET, interface);
+ if (ret)
+ return ret;
+
+ return phy_power_on(pp->comphy);
+}
+
+static int mvneta_config_interface(struct mvneta_port *pp,
+ phy_interface_t interface)
+{
+ int ret = 0;
+
+ if (pp->comphy) {
+ if (interface == PHY_INTERFACE_MODE_SGMII ||
+ interface == PHY_INTERFACE_MODE_1000BASEX ||
+ interface == PHY_INTERFACE_MODE_2500BASEX) {
+ ret = mvneta_comphy_init(pp, interface);
+ }
+ } else {
+ switch (interface) {
+ case PHY_INTERFACE_MODE_QSGMII:
+ mvreg_write(pp, MVNETA_SERDES_CFG,
+ MVNETA_QSGMII_SERDES_PROTO);
+ break;
+
+ case PHY_INTERFACE_MODE_SGMII:
+ case PHY_INTERFACE_MODE_1000BASEX:
+ mvreg_write(pp, MVNETA_SERDES_CFG,
+ MVNETA_SGMII_SERDES_PROTO);
+ break;
+
+ case PHY_INTERFACE_MODE_2500BASEX:
+ mvreg_write(pp, MVNETA_SERDES_CFG,
+ MVNETA_HSGMII_SERDES_PROTO);
+ break;
+ default:
+ break;
+ }
+ }
+
+ pp->phy_interface = interface;
+
+ return ret;
+}
+
static void mvneta_start_dev(struct mvneta_port *pp)
{
int cpu;
+
+ WARN_ON(mvneta_config_interface(pp, pp->phy_interface));
mvneta_max_rx_size_set(pp, pp->pkt_size);
mvneta_txq_max_tx_size_set(pp, pp->pkt_size);
@@ -3193,12 +3631,23 @@
MVNETA_CAUSE_LINK_CHANGE);
phylink_start(pp->phylink);
+
+ /* We may have called phylink_speed_down before */
+ phylink_speed_up(pp->phylink);
+
netif_tx_start_all_queues(pp->dev);
+
+ clear_bit(__MVNETA_DOWN, &pp->state);
}
static void mvneta_stop_dev(struct mvneta_port *pp)
{
unsigned int cpu;
+
+ set_bit(__MVNETA_DOWN, &pp->state);
+
+ if (device_may_wakeup(&pp->dev->dev))
+ phylink_speed_down(pp->phylink, false);
phylink_stop(pp->phylink);
@@ -3229,6 +3678,8 @@
mvneta_tx_reset(pp);
mvneta_rx_reset(pp);
+
+ WARN_ON(phy_power_off(pp->comphy));
}
static void mvneta_percpu_enable(void *arg)
@@ -3255,6 +3706,11 @@
netdev_info(dev, "Illegal MTU value %d, rounding to %d\n",
mtu, ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8));
mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8);
+ }
+
+ if (pp->xdp_prog && mtu > MVNETA_MAX_RX_BUF_SIZE) {
+ netdev_info(dev, "Illegal MTU value %d for XDP mode\n", mtu);
+ return -EINVAL;
}
dev->mtu = mtu;
@@ -3351,9 +3807,12 @@
return 0;
}
-static void mvneta_validate(struct net_device *ndev, unsigned long *supported,
+static void mvneta_validate(struct phylink_config *config,
+ unsigned long *supported,
struct phylink_link_state *state)
{
+ struct net_device *ndev = to_net_dev(config->dev);
+ struct mvneta_port *pp = netdev_priv(ndev);
__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
/* We only support QSGMII, SGMII, 802.3z and RGMII modes */
@@ -3372,9 +3831,16 @@
/* Asymmetric pause is unsupported */
phylink_set(mask, Pause);
+
/* Half-duplex at speeds higher than 100Mbit is unsupported */
- phylink_set(mask, 1000baseT_Full);
- phylink_set(mask, 1000baseX_Full);
+ if (pp->comphy || state->interface != PHY_INTERFACE_MODE_2500BASEX) {
+ phylink_set(mask, 1000baseT_Full);
+ phylink_set(mask, 1000baseX_Full);
+ }
+ if (pp->comphy || state->interface == PHY_INTERFACE_MODE_2500BASEX) {
+ phylink_set(mask, 2500baseT_Full);
+ phylink_set(mask, 2500baseX_Full);
+ }
if (!phy_interface_mode_is_8023z(state->interface)) {
/* 10M and 100M are only supported in non-802.3z mode */
@@ -3388,18 +3854,26 @@
__ETHTOOL_LINK_MODE_MASK_NBITS);
bitmap_and(state->advertising, state->advertising, mask,
__ETHTOOL_LINK_MODE_MASK_NBITS);
+
+ /* We can only operate at 2500BaseX or 1000BaseX. If requested
+ * to advertise both, only report advertising at 2500BaseX.
+ */
+ phylink_helper_basex_speed(state);
}
-static int mvneta_mac_link_state(struct net_device *ndev,
- struct phylink_link_state *state)
+static void mvneta_mac_pcs_get_state(struct phylink_config *config,
+ struct phylink_link_state *state)
{
+ struct net_device *ndev = to_net_dev(config->dev);
struct mvneta_port *pp = netdev_priv(ndev);
u32 gmac_stat;
gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS);
if (gmac_stat & MVNETA_GMAC_SPEED_1000)
- state->speed = SPEED_1000;
+ state->speed =
+ state->interface == PHY_INTERFACE_MODE_2500BASEX ?
+ SPEED_2500 : SPEED_1000;
else if (gmac_stat & MVNETA_GMAC_SPEED_100)
state->speed = SPEED_100;
else
@@ -3414,12 +3888,11 @@
state->pause |= MLO_PAUSE_RX;
if (gmac_stat & MVNETA_GMAC_TX_FLOW_CTRL_ENABLE)
state->pause |= MLO_PAUSE_TX;
-
- return 1;
}
-static void mvneta_mac_an_restart(struct net_device *ndev)
+static void mvneta_mac_an_restart(struct phylink_config *config)
{
+ struct net_device *ndev = to_net_dev(config->dev);
struct mvneta_port *pp = netdev_priv(ndev);
u32 gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
@@ -3429,28 +3902,27 @@
gmac_an & ~MVNETA_GMAC_INBAND_RESTART_AN);
}
-static void mvneta_mac_config(struct net_device *ndev, unsigned int mode,
- const struct phylink_link_state *state)
+static void mvneta_mac_config(struct phylink_config *config, unsigned int mode,
+ const struct phylink_link_state *state)
{
+ struct net_device *ndev = to_net_dev(config->dev);
struct mvneta_port *pp = netdev_priv(ndev);
u32 new_ctrl0, gmac_ctrl0 = mvreg_read(pp, MVNETA_GMAC_CTRL_0);
u32 new_ctrl2, gmac_ctrl2 = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
+ u32 new_ctrl4, gmac_ctrl4 = mvreg_read(pp, MVNETA_GMAC_CTRL_4);
u32 new_clk, gmac_clk = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER);
u32 new_an, gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
new_ctrl0 = gmac_ctrl0 & ~MVNETA_GMAC0_PORT_1000BASE_X;
new_ctrl2 = gmac_ctrl2 & ~(MVNETA_GMAC2_INBAND_AN_ENABLE |
MVNETA_GMAC2_PORT_RESET);
+ new_ctrl4 = gmac_ctrl4 & ~(MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE);
new_clk = gmac_clk & ~MVNETA_GMAC_1MS_CLOCK_ENABLE;
new_an = gmac_an & ~(MVNETA_GMAC_INBAND_AN_ENABLE |
MVNETA_GMAC_INBAND_RESTART_AN |
- MVNETA_GMAC_CONFIG_MII_SPEED |
- MVNETA_GMAC_CONFIG_GMII_SPEED |
MVNETA_GMAC_AN_SPEED_EN |
MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL |
- MVNETA_GMAC_CONFIG_FLOW_CTRL |
MVNETA_GMAC_AN_FLOW_CTRL_EN |
- MVNETA_GMAC_CONFIG_FULL_DUPLEX |
MVNETA_GMAC_AN_DUPLEX_EN);
/* Even though it might look weird, when we're configured in
@@ -3465,24 +3937,20 @@
if (phylink_test(state->advertising, Pause))
new_an |= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL;
- if (state->pause & MLO_PAUSE_TXRX_MASK)
- new_an |= MVNETA_GMAC_CONFIG_FLOW_CTRL;
if (!phylink_autoneg_inband(mode)) {
- /* Phy or fixed speed */
- if (state->duplex)
- new_an |= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
-
- if (state->speed == SPEED_1000)
- new_an |= MVNETA_GMAC_CONFIG_GMII_SPEED;
- else if (state->speed == SPEED_100)
- new_an |= MVNETA_GMAC_CONFIG_MII_SPEED;
+ /* Phy or fixed speed - nothing to do, leave the
+ * configured speed, duplex and flow control as-is.
+ */
} else if (state->interface == PHY_INTERFACE_MODE_SGMII) {
/* SGMII mode receives the state from the PHY */
new_ctrl2 |= MVNETA_GMAC2_INBAND_AN_ENABLE;
new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE;
new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN |
- MVNETA_GMAC_FORCE_LINK_PASS)) |
+ MVNETA_GMAC_FORCE_LINK_PASS |
+ MVNETA_GMAC_CONFIG_MII_SPEED |
+ MVNETA_GMAC_CONFIG_GMII_SPEED |
+ MVNETA_GMAC_CONFIG_FULL_DUPLEX)) |
MVNETA_GMAC_INBAND_AN_ENABLE |
MVNETA_GMAC_AN_SPEED_EN |
MVNETA_GMAC_AN_DUPLEX_EN;
@@ -3491,7 +3959,8 @@
new_ctrl0 |= MVNETA_GMAC0_PORT_1000BASE_X;
new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE;
new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN |
- MVNETA_GMAC_FORCE_LINK_PASS)) |
+ MVNETA_GMAC_FORCE_LINK_PASS |
+ MVNETA_GMAC_CONFIG_MII_SPEED)) |
MVNETA_GMAC_INBAND_AN_ENABLE |
MVNETA_GMAC_CONFIG_GMII_SPEED |
/* The MAC only supports FD mode */
@@ -3512,10 +3981,25 @@
MVNETA_GMAC_FORCE_LINK_DOWN);
}
+
+ /* When at 2.5G, the link partner can send frames with shortened
+ * preambles.
+ */
+ if (state->interface == PHY_INTERFACE_MODE_2500BASEX)
+ new_ctrl4 |= MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE;
+
+ if (pp->phy_interface != state->interface) {
+ if (pp->comphy)
+ WARN_ON(phy_power_off(pp->comphy));
+ WARN_ON(mvneta_config_interface(pp, state->interface));
+ }
+
if (new_ctrl0 != gmac_ctrl0)
mvreg_write(pp, MVNETA_GMAC_CTRL_0, new_ctrl0);
if (new_ctrl2 != gmac_ctrl2)
mvreg_write(pp, MVNETA_GMAC_CTRL_2, new_ctrl2);
+ if (new_ctrl4 != gmac_ctrl4)
+ mvreg_write(pp, MVNETA_GMAC_CTRL_4, new_ctrl4);
if (new_clk != gmac_clk)
mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, new_clk);
if (new_an != gmac_an)
@@ -3540,9 +4024,10 @@
mvreg_write(pp, MVNETA_LPI_CTRL_1, lpi_ctl1);
}
-static void mvneta_mac_link_down(struct net_device *ndev, unsigned int mode,
- phy_interface_t interface)
+static void mvneta_mac_link_down(struct phylink_config *config,
+ unsigned int mode, phy_interface_t interface)
{
+ struct net_device *ndev = to_net_dev(config->dev);
struct mvneta_port *pp = netdev_priv(ndev);
u32 val;
@@ -3559,17 +4044,48 @@
mvneta_set_eee(pp, false);
}
-static void mvneta_mac_link_up(struct net_device *ndev, unsigned int mode,
- phy_interface_t interface,
- struct phy_device *phy)
+static void mvneta_mac_link_up(struct phylink_config *config,
+ struct phy_device *phy,
+ unsigned int mode, phy_interface_t interface,
+ int speed, int duplex,
+ bool tx_pause, bool rx_pause)
{
+ struct net_device *ndev = to_net_dev(config->dev);
struct mvneta_port *pp = netdev_priv(ndev);
u32 val;
if (!phylink_autoneg_inband(mode)) {
val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
- val &= ~MVNETA_GMAC_FORCE_LINK_DOWN;
+ val &= ~(MVNETA_GMAC_FORCE_LINK_DOWN |
+ MVNETA_GMAC_CONFIG_MII_SPEED |
+ MVNETA_GMAC_CONFIG_GMII_SPEED |
+ MVNETA_GMAC_CONFIG_FLOW_CTRL |
+ MVNETA_GMAC_CONFIG_FULL_DUPLEX);
val |= MVNETA_GMAC_FORCE_LINK_PASS;
+
+ if (speed == SPEED_1000 || speed == SPEED_2500)
+ val |= MVNETA_GMAC_CONFIG_GMII_SPEED;
+ else if (speed == SPEED_100)
+ val |= MVNETA_GMAC_CONFIG_MII_SPEED;
+
+ if (duplex == DUPLEX_FULL)
+ val |= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
+
+ if (tx_pause || rx_pause)
+ val |= MVNETA_GMAC_CONFIG_FLOW_CTRL;
+
+ mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
+ } else {
+ /* When inband doesn't cover flow control or flow control is
+ * disabled, we need to manually configure it. This bit will
+ * only have effect if MVNETA_GMAC_AN_FLOW_CTRL_EN is unset.
+ */
+ val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
+ val &= ~MVNETA_GMAC_CONFIG_FLOW_CTRL;
+
+ if (tx_pause || rx_pause)
+ val |= MVNETA_GMAC_CONFIG_FLOW_CTRL;
+
mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
}
@@ -3583,7 +4099,7 @@
static const struct phylink_mac_ops mvneta_phylink_ops = {
.validate = mvneta_validate,
- .mac_link_state = mvneta_mac_link_state,
+ .mac_pcs_get_state = mvneta_mac_pcs_get_state,
.mac_an_restart = mvneta_mac_an_restart,
.mac_config = mvneta_mac_config,
.mac_link_down = mvneta_mac_link_down,
@@ -3600,6 +4116,10 @@
phylink_ethtool_get_wol(pp->phylink, &wol);
device_set_wakeup_capable(&pp->dev->dev, !!wol.supported);
+
+ /* PHY WoL may be enabled but device wakeup disabled */
+ if (wol.supported)
+ device_set_wakeup_enable(&pp->dev->dev, !!wol.wolopts);
return err;
}
@@ -3620,7 +4140,7 @@
/* Use the cpu associated to the rxq when it is online, in all
* the other cases, use the cpu 0 which can't be offline.
*/
- if (cpu_online(pp->rxq_def))
+ if (pp->rxq_def < nr_cpu_ids && cpu_online(pp->rxq_def))
elected_cpu = pp->rxq_def;
max_cpu = num_present_cpus();
@@ -3645,7 +4165,7 @@
*/
if (txq_number == 1)
txq_map = (cpu == elected_cpu) ?
- MVNETA_CPU_TXQ_ACCESS(1) : 0;
+ MVNETA_CPU_TXQ_ACCESS(0) : 0;
else
txq_map = mvreg_read(pp, MVNETA_CPU_MAP(cpu)) &
MVNETA_CPU_TXQ_ACCESS_ALL_MASK;
@@ -3813,9 +4333,6 @@
goto err_free_online_hp;
}
- /* In default link is down */
- netif_carrier_off(pp->dev);
-
ret = mvneta_mdio_probe(pp);
if (ret < 0) {
netdev_err(dev, "cannot probe MDIO bus\n");
@@ -3889,6 +4406,48 @@
struct mvneta_port *pp = netdev_priv(dev);
return phylink_mii_ioctl(pp->phylink, ifr, cmd);
+}
+
+static int mvneta_xdp_setup(struct net_device *dev, struct bpf_prog *prog,
+ struct netlink_ext_ack *extack)
+{
+ bool need_update, running = netif_running(dev);
+ struct mvneta_port *pp = netdev_priv(dev);
+ struct bpf_prog *old_prog;
+
+ if (prog && dev->mtu > MVNETA_MAX_RX_BUF_SIZE) {
+ NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP");
+ return -EOPNOTSUPP;
+ }
+
+ if (pp->bm_priv) {
+ NL_SET_ERR_MSG_MOD(extack,
+ "Hardware Buffer Management not supported on XDP");
+ return -EOPNOTSUPP;
+ }
+
+ need_update = !!pp->xdp_prog != !!prog;
+ if (running && need_update)
+ mvneta_stop(dev);
+
+ old_prog = xchg(&pp->xdp_prog, prog);
+ if (old_prog)
+ bpf_prog_put(old_prog);
+
+ if (running && need_update)
+ return mvneta_open(dev);
+
+ return 0;
+}
+
+static int mvneta_xdp(struct net_device *dev, struct netdev_bpf *xdp)
+{
+ switch (xdp->command) {
+ case XDP_SETUP_PROG:
+ return mvneta_xdp_setup(dev, xdp->prog, xdp->extack);
+ default:
+ return -EINVAL;
+ }
}
/* Ethtool methods */
@@ -4037,45 +4596,112 @@
}
}
+static void
+mvneta_ethtool_update_pcpu_stats(struct mvneta_port *pp,
+ struct mvneta_ethtool_stats *es)
+{
+ unsigned int start;
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ struct mvneta_pcpu_stats *stats;
+ u64 skb_alloc_error;
+ u64 refill_error;
+ u64 xdp_redirect;
+ u64 xdp_xmit_err;
+ u64 xdp_tx_err;
+ u64 xdp_pass;
+ u64 xdp_drop;
+ u64 xdp_xmit;
+ u64 xdp_tx;
+
+ stats = per_cpu_ptr(pp->stats, cpu);
+ do {
+ start = u64_stats_fetch_begin_irq(&stats->syncp);
+ skb_alloc_error = stats->es.skb_alloc_error;
+ refill_error = stats->es.refill_error;
+ xdp_redirect = stats->es.ps.xdp_redirect;
+ xdp_pass = stats->es.ps.xdp_pass;
+ xdp_drop = stats->es.ps.xdp_drop;
+ xdp_xmit = stats->es.ps.xdp_xmit;
+ xdp_xmit_err = stats->es.ps.xdp_xmit_err;
+ xdp_tx = stats->es.ps.xdp_tx;
+ xdp_tx_err = stats->es.ps.xdp_tx_err;
+ } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
+
+ es->skb_alloc_error += skb_alloc_error;
+ es->refill_error += refill_error;
+ es->ps.xdp_redirect += xdp_redirect;
+ es->ps.xdp_pass += xdp_pass;
+ es->ps.xdp_drop += xdp_drop;
+ es->ps.xdp_xmit += xdp_xmit;
+ es->ps.xdp_xmit_err += xdp_xmit_err;
+ es->ps.xdp_tx += xdp_tx;
+ es->ps.xdp_tx_err += xdp_tx_err;
+ }
+}
+
static void mvneta_ethtool_update_stats(struct mvneta_port *pp)
{
+ struct mvneta_ethtool_stats stats = {};
const struct mvneta_statistic *s;
void __iomem *base = pp->base;
u32 high, low;
u64 val;
int i;
+ mvneta_ethtool_update_pcpu_stats(pp, &stats);
for (i = 0, s = mvneta_statistics;
s < mvneta_statistics + ARRAY_SIZE(mvneta_statistics);
s++, i++) {
- val = 0;
-
switch (s->type) {
case T_REG_32:
val = readl_relaxed(base + s->offset);
+ pp->ethtool_stats[i] += val;
break;
case T_REG_64:
/* Docs say to read low 32-bit then high */
low = readl_relaxed(base + s->offset);
high = readl_relaxed(base + s->offset + 4);
val = (u64)high << 32 | low;
+ pp->ethtool_stats[i] += val;
break;
case T_SW:
switch (s->offset) {
case ETHTOOL_STAT_EEE_WAKEUP:
val = phylink_get_eee_err(pp->phylink);
+ pp->ethtool_stats[i] += val;
break;
case ETHTOOL_STAT_SKB_ALLOC_ERR:
- val = pp->rxqs[0].skb_alloc_err;
+ pp->ethtool_stats[i] = stats.skb_alloc_error;
break;
case ETHTOOL_STAT_REFILL_ERR:
- val = pp->rxqs[0].refill_err;
+ pp->ethtool_stats[i] = stats.refill_error;
+ break;
+ case ETHTOOL_XDP_REDIRECT:
+ pp->ethtool_stats[i] = stats.ps.xdp_redirect;
+ break;
+ case ETHTOOL_XDP_PASS:
+ pp->ethtool_stats[i] = stats.ps.xdp_pass;
+ break;
+ case ETHTOOL_XDP_DROP:
+ pp->ethtool_stats[i] = stats.ps.xdp_drop;
+ break;
+ case ETHTOOL_XDP_TX:
+ pp->ethtool_stats[i] = stats.ps.xdp_tx;
+ break;
+ case ETHTOOL_XDP_TX_ERR:
+ pp->ethtool_stats[i] = stats.ps.xdp_tx_err;
+ break;
+ case ETHTOOL_XDP_XMIT:
+ pp->ethtool_stats[i] = stats.ps.xdp_xmit;
+ break;
+ case ETHTOOL_XDP_XMIT_ERR:
+ pp->ethtool_stats[i] = stats.ps.xdp_xmit_err;
break;
}
break;
}
-
- pp->ethtool_stats[i] += val;
}
}
@@ -4261,8 +4887,7 @@
/* The Armada 37x documents do not give limits for this other than
* it being an 8-bit register. */
- if (eee->tx_lpi_enabled &&
- (eee->tx_lpi_timer < 0 || eee->tx_lpi_timer > 255))
+ if (eee->tx_lpi_enabled && eee->tx_lpi_timer > 255)
return -EINVAL;
lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0);
@@ -4288,9 +4913,13 @@
.ndo_fix_features = mvneta_fix_features,
.ndo_get_stats64 = mvneta_get_stats64,
.ndo_do_ioctl = mvneta_ioctl,
+ .ndo_bpf = mvneta_xdp,
+ .ndo_xdp_xmit = mvneta_xdp_xmit,
};
static const struct ethtool_ops mvneta_eth_tool_ops = {
+ .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS |
+ ETHTOOL_COALESCE_MAX_FRAMES,
.nway_reset = mvneta_ethtool_nway_reset,
.get_link = ethtool_op_get_link,
.set_coalesce = mvneta_ethtool_set_coalesce,
@@ -4415,12 +5044,10 @@
/* MAC Cause register should be cleared */
mvreg_write(pp, MVNETA_UNIT_INTR_CAUSE, 0);
- if (phy_mode == PHY_INTERFACE_MODE_QSGMII)
- mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_QSGMII_SERDES_PROTO);
- else if (phy_mode == PHY_INTERFACE_MODE_SGMII ||
- phy_mode == PHY_INTERFACE_MODE_1000BASEX)
- mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_SGMII_SERDES_PROTO);
- else if (!phy_interface_mode_is_rgmii(phy_mode))
+ if (phy_mode != PHY_INTERFACE_MODE_QSGMII &&
+ phy_mode != PHY_INTERFACE_MODE_SGMII &&
+ !phy_interface_mode_is_8023z(phy_mode) &&
+ !phy_interface_mode_is_rgmii(phy_mode))
return -EINVAL;
return 0;
@@ -4429,39 +5056,51 @@
/* Device initialization routine */
static int mvneta_probe(struct platform_device *pdev)
{
- struct resource *res;
struct device_node *dn = pdev->dev.of_node;
struct device_node *bm_node;
struct mvneta_port *pp;
struct net_device *dev;
struct phylink *phylink;
+ struct phy *comphy;
const char *dt_mac_addr;
char hw_mac_addr[ETH_ALEN];
+ phy_interface_t phy_mode;
const char *mac_from;
int tx_csum_limit;
- int phy_mode;
int err;
int cpu;
- dev = alloc_etherdev_mqs(sizeof(struct mvneta_port), txq_number, rxq_number);
+ dev = devm_alloc_etherdev_mqs(&pdev->dev, sizeof(struct mvneta_port),
+ txq_number, rxq_number);
if (!dev)
return -ENOMEM;
dev->irq = irq_of_parse_and_map(dn, 0);
- if (dev->irq == 0) {
- err = -EINVAL;
- goto err_free_netdev;
- }
+ if (dev->irq == 0)
+ return -EINVAL;
- phy_mode = of_get_phy_mode(dn);
- if (phy_mode < 0) {
+ err = of_get_phy_mode(dn, &phy_mode);
+ if (err) {
dev_err(&pdev->dev, "incorrect phy-mode\n");
- err = -EINVAL;
goto err_free_irq;
}
- phylink = phylink_create(dev, pdev->dev.fwnode, phy_mode,
- &mvneta_phylink_ops);
+ comphy = devm_of_phy_get(&pdev->dev, dn, NULL);
+ if (comphy == ERR_PTR(-EPROBE_DEFER)) {
+ err = -EPROBE_DEFER;
+ goto err_free_irq;
+ } else if (IS_ERR(comphy)) {
+ comphy = NULL;
+ }
+
+ pp = netdev_priv(dev);
+ spin_lock_init(&pp->lock);
+
+ pp->phylink_config.dev = &dev->dev;
+ pp->phylink_config.type = PHYLINK_NETDEV;
+
+ phylink = phylink_create(&pp->phylink_config, pdev->dev.fwnode,
+ phy_mode, &mvneta_phylink_ops);
if (IS_ERR(phylink)) {
err = PTR_ERR(phylink);
goto err_free_irq;
@@ -4473,9 +5112,8 @@
dev->ethtool_ops = &mvneta_eth_tool_ops;
- pp = netdev_priv(dev);
- spin_lock_init(&pp->lock);
pp->phylink = phylink;
+ pp->comphy = comphy;
pp->phy_interface = phy_mode;
pp->dn = dn;
@@ -4500,8 +5138,7 @@
if (!IS_ERR(pp->clk_bus))
clk_prepare_enable(pp->clk_bus);
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- pp->base = devm_ioremap_resource(&pdev->dev, res);
+ pp->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(pp->base)) {
err = PTR_ERR(pp->base);
goto err_clk;
@@ -4522,9 +5159,9 @@
}
dt_mac_addr = of_get_mac_address(dn);
- if (dt_mac_addr) {
+ if (!IS_ERR(dt_mac_addr)) {
mac_from = "device tree";
- memcpy(dev->dev_addr, dt_mac_addr, ETH_ALEN);
+ ether_addr_copy(dev->dev_addr, dt_mac_addr);
} else {
mvneta_get_mac_addr(pp, hw_mac_addr);
if (is_valid_ether_addr(hw_mac_addr)) {
@@ -4567,7 +5204,6 @@
SET_NETDEV_DEV(dev, &pdev->dev);
pp->id = global_port_id++;
- pp->rx_offset_correction = 0; /* not relevant for SW BM */
/* Obtain access to BM resources if enabled and already initialized */
bm_node = of_parse_phandle(dn, "buffer-manager", 0);
@@ -4592,11 +5228,15 @@
}
of_node_put(bm_node);
+ /* sw buffer management */
+ if (!pp->bm_priv)
+ pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
+
err = mvneta_init(&pdev->dev, pp);
if (err < 0)
goto err_netdev;
- err = mvneta_port_power_up(pp, phy_mode);
+ err = mvneta_port_power_up(pp, pp->phy_interface);
if (err < 0) {
dev_err(&pdev->dev, "can't power up port\n");
goto err_netdev;
@@ -4618,7 +5258,8 @@
}
}
- dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_TSO;
+ dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
+ NETIF_F_TSO | NETIF_F_RXCSUM;
dev->hw_features |= dev->features;
dev->vlan_features |= dev->features;
dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
@@ -4660,8 +5301,6 @@
phylink_destroy(pp->phylink);
err_free_irq:
irq_dispose_mapping(dev->irq);
-err_free_netdev:
- free_netdev(dev);
return err;
}
@@ -4678,7 +5317,6 @@
free_percpu(pp->stats);
irq_dispose_mapping(dev->irq);
phylink_destroy(pp->phylink);
- free_netdev(dev);
if (pp->bm_priv) {
mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
@@ -4751,6 +5389,7 @@
err = mvneta_bm_port_init(pdev, pp);
if (err < 0) {
dev_info(&pdev->dev, "use SW buffer management\n");
+ pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
pp->bm_priv = NULL;
}
}
@@ -4823,7 +5462,7 @@
{
int ret;
- ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/mvmeta:online",
+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/mvneta:online",
mvneta_cpu_online,
mvneta_cpu_down_prepare);
if (ret < 0)
--
Gitblit v1.6.2