~hc/RK356X_SDK_RELEASE.git

..	..	@@ -27,6 +27,7 @@
27	27	#include <linux/of_irq.h>
28	28	#include <linux/of_mdio.h>
29	29	#include <linux/of_net.h>
	30	+#include <linux/phy/phy.h>
30	31	#include <linux/phy.h>
31	32	#include <linux/phylink.h>
32	33	#include <linux/platform_device.h>
..	..	@@ -36,6 +37,8 @@
36	37	#include <net/ip.h>
37	38	#include <net/ipv6.h>
38	39	#include <net/tso.h>
	40	+#include <net/page_pool.h>
	41	+#include <linux/bpf_trace.h>
39	42
40	43	/* Registers */
41	44	#define MVNETA_RXQ_CONFIG_REG(q) (0x1400 + ((q) << 2))
..	..	@@ -103,9 +106,11 @@
103	106	#define MVNETA_TX_IN_PRGRS BIT(0)
104	107	#define MVNETA_TX_FIFO_EMPTY BIT(8)
105	108	#define MVNETA_RX_MIN_FRAME_SIZE 0x247c
	109	+/* Only exists on Armada XP and Armada 370 */
106	110	#define MVNETA_SERDES_CFG 0x24A0
107	111	#define MVNETA_SGMII_SERDES_PROTO 0x0cc7
108	112	#define MVNETA_QSGMII_SERDES_PROTO 0x0667
	113	+#define MVNETA_HSGMII_SERDES_PROTO 0x1107
109	114	#define MVNETA_TYPE_PRIO 0x24bc
110	115	#define MVNETA_FORCE_UNI BIT(21)
111	116	#define MVNETA_TXQ_CMD_1 0x24e4
..	..	@@ -221,6 +226,8 @@
221	226	#define MVNETA_GMAC_AN_FLOW_CTRL_EN BIT(11)
222	227	#define MVNETA_GMAC_CONFIG_FULL_DUPLEX BIT(12)
223	228	#define MVNETA_GMAC_AN_DUPLEX_EN BIT(13)
	229	+#define MVNETA_GMAC_CTRL_4 0x2c90
	230	+#define MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE BIT(1)
224	231	#define MVNETA_MIB_COUNTERS_BASE 0x3000
225	232	#define MVNETA_MIB_LATE_COLLISION 0x7c
226	233	#define MVNETA_DA_FILT_SPEC_MCAST 0x3400
..	..	@@ -319,6 +326,12 @@
319	326	ETH_HLEN + ETH_FCS_LEN, \
320	327	cache_line_size())
321	328
	329	+/* Driver assumes that the last 3 bits are 0 */
	330	+#define MVNETA_SKB_HEADROOM ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8)
	331	+#define MVNETA_SKB_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info) + \
	332	+ MVNETA_SKB_HEADROOM))
	333	+#define MVNETA_MAX_RX_BUF_SIZE (PAGE_SIZE - MVNETA_SKB_PAD)
	334	+
322	335	#define IS_TSO_HEADER(txq, addr) \
323	336	((addr >= txq->tso_hdrs_phys) && \
324	337	(addr < txq->tso_hdrs_phys + txq->size * TSO_HEADER_SIZE))
..	..	@@ -330,6 +343,13 @@
330	343	ETHTOOL_STAT_EEE_WAKEUP,
331	344	ETHTOOL_STAT_SKB_ALLOC_ERR,
332	345	ETHTOOL_STAT_REFILL_ERR,
	346	+ ETHTOOL_XDP_REDIRECT,
	347	+ ETHTOOL_XDP_PASS,
	348	+ ETHTOOL_XDP_DROP,
	349	+ ETHTOOL_XDP_TX,
	350	+ ETHTOOL_XDP_TX_ERR,
	351	+ ETHTOOL_XDP_XMIT,
	352	+ ETHTOOL_XDP_XMIT_ERR,
333	353	ETHTOOL_MAX_STATS,
334	354	};
335	355
..	..	@@ -342,6 +362,11 @@
342	362	#define T_REG_32 32
343	363	#define T_REG_64 64
344	364	#define T_SW 1
	365	+
	366	+#define MVNETA_XDP_PASS 0
	367	+#define MVNETA_XDP_DROPPED BIT(0)
	368	+#define MVNETA_XDP_TX BIT(1)
	369	+#define MVNETA_XDP_REDIR BIT(2)
345	370
346	371	static const struct mvneta_statistic mvneta_statistics[] = {
347	372	{ 0x3000, T_REG_64, "good_octets_received", },
..	..	@@ -379,16 +404,42 @@
379	404	{ ETHTOOL_STAT_EEE_WAKEUP, T_SW, "eee_wakeup_errors", },
380	405	{ ETHTOOL_STAT_SKB_ALLOC_ERR, T_SW, "skb_alloc_errors", },
381	406	{ ETHTOOL_STAT_REFILL_ERR, T_SW, "refill_errors", },
	407	+ { ETHTOOL_XDP_REDIRECT, T_SW, "rx_xdp_redirect", },
	408	+ { ETHTOOL_XDP_PASS, T_SW, "rx_xdp_pass", },
	409	+ { ETHTOOL_XDP_DROP, T_SW, "rx_xdp_drop", },
	410	+ { ETHTOOL_XDP_TX, T_SW, "rx_xdp_tx", },
	411	+ { ETHTOOL_XDP_TX_ERR, T_SW, "rx_xdp_tx_errors", },
	412	+ { ETHTOOL_XDP_XMIT, T_SW, "tx_xdp_xmit", },
	413	+ { ETHTOOL_XDP_XMIT_ERR, T_SW, "tx_xdp_xmit_errors", },
	414	+};
	415	+
	416	+struct mvneta_stats {
	417	+ u64 rx_packets;
	418	+ u64 rx_bytes;
	419	+ u64 tx_packets;
	420	+ u64 tx_bytes;
	421	+ /* xdp */
	422	+ u64 xdp_redirect;
	423	+ u64 xdp_pass;
	424	+ u64 xdp_drop;
	425	+ u64 xdp_xmit;
	426	+ u64 xdp_xmit_err;
	427	+ u64 xdp_tx;
	428	+ u64 xdp_tx_err;
	429	+};
	430	+
	431	+struct mvneta_ethtool_stats {
	432	+ struct mvneta_stats ps;
	433	+ u64 skb_alloc_error;
	434	+ u64 refill_error;
382	435	};
383	436
384	437	struct mvneta_pcpu_stats {
385		- struct u64_stats_sync syncp;
386		- u64 rx_packets;
387		- u64 rx_bytes;
	438	+ struct u64_stats_sync syncp;
	439	+
	440	+ struct mvneta_ethtool_stats es;
388	441	u64 rx_dropped;
389	442	u64 rx_errors;
390		- u64 tx_packets;
391		- u64 tx_bytes;
392	443	};
393	444
394	445	struct mvneta_pcpu_port {
..	..	@@ -402,10 +453,16 @@
402	453	u32 cause_rx_tx;
403	454	};
404	455
	456	+enum {
	457	+ __MVNETA_DOWN,
	458	+};
	459	+
405	460	struct mvneta_port {
406	461	u8 id;
407	462	struct mvneta_pcpu_port __percpu *ports;
408	463	struct mvneta_pcpu_stats __percpu *stats;
	464	+
	465	+ unsigned long state;
409	466
410	467	int pkt_size;
411	468	void __iomem *base;
..	..	@@ -424,6 +481,8 @@
424	481	u32 cause_rx_tx;
425	482	struct napi_struct napi;
426	483
	484	+ struct bpf_prog *xdp_prog;
	485	+
427	486	/* Core clock */
428	487	struct clk *clk;
429	488	/* AXI clock */
..	..	@@ -436,6 +495,8 @@
436	495	struct device_node *dn;
437	496	unsigned int tx_csum_limit;
438	497	struct phylink *phylink;
	498	+ struct phylink_config phylink_config;
	499	+ struct phy *comphy;
439	500
440	501	struct mvneta_bm *bm_priv;
441	502	struct mvneta_bm_pool *pool_long;
..	..	@@ -493,7 +554,7 @@
493	554	#if defined(__LITTLE_ENDIAN)
494	555	struct mvneta_tx_desc {
495	556	u32 command; /* Options used by HW for packet transmitting.*/
496		- u16 reserverd1; /* csum_l4 (for future use) */
	557	+ u16 reserved1; /* csum_l4 (for future use) */
497	558	u16 data_size; /* Data size of transmitted packet in bytes */
498	559	u32 buf_phys_addr; /* Physical addr of transmitted buffer */
499	560	u32 reserved2; /* hw_cmd - (for future use, PMT) */
..	..	@@ -518,7 +579,7 @@
518	579	#else
519	580	struct mvneta_tx_desc {
520	581	u16 data_size; /* Data size of transmitted packet in bytes */
521		- u16 reserverd1; /* csum_l4 (for future use) */
	582	+ u16 reserved1; /* csum_l4 (for future use) */
522	583	u32 command; /* Options used by HW for packet transmitting.*/
523	584	u32 reserved2; /* hw_cmd - (for future use, PMT) */
524	585	u32 buf_phys_addr; /* Physical addr of transmitted buffer */
..	..	@@ -542,6 +603,20 @@
542	603	};
543	604	#endif
544	605
	606	+enum mvneta_tx_buf_type {
	607	+ MVNETA_TYPE_SKB,
	608	+ MVNETA_TYPE_XDP_TX,
	609	+ MVNETA_TYPE_XDP_NDO,
	610	+};
	611	+
	612	+struct mvneta_tx_buf {
	613	+ enum mvneta_tx_buf_type type;
	614	+ union {
	615	+ struct xdp_frame *xdpf;
	616	+ struct sk_buff *skb;
	617	+ };
	618	+};
	619	+
545	620	struct mvneta_tx_queue {
546	621	/* Number of this TX queue, in the range 0-7 */
547	622	u8 id;
..	..	@@ -557,8 +632,8 @@
557	632	int tx_stop_threshold;
558	633	int tx_wake_threshold;
559	634
560		- /* Array of transmitted skb */
561		- struct sk_buff **tx_skb;
	635	+ /* Array of transmitted buffers */
	636	+ struct mvneta_tx_buf *buf;
562	637
563	638	/* Index of last TX DMA descriptor that was inserted */
564	639	int txq_put_index;
..	..	@@ -600,6 +675,10 @@
600	675	u32 pkts_coal;
601	676	u32 time_coal;
602	677
	678	+ /* page_pool */
	679	+ struct page_pool *page_pool;
	680	+ struct xdp_rxq_info xdp_rxq;
	681	+
603	682	/* Virtual address of the RX buffer */
604	683	void **buf_virt_addr;
605	684
..	..	@@ -618,14 +697,6 @@
618	697	/* Index of first RX DMA descriptor to refill */
619	698	int first_to_refill;
620	699	u32 refill_num;
621		-
622		- /* pointer to uncomplete skb buffer */
623		- struct sk_buff *skb;
624		- int left_size;
625		-
626		- /* error counters */
627		- u32 skb_alloc_err;
628		- u32 refill_err;
629	700	};
630	701
631	702	static enum cpuhp_state online_hpstate;
..	..	@@ -638,7 +709,6 @@
638	709	static int rxq_def;
639	710
640	711	static int rx_copybreak __read_mostly = 256;
641		-static int rx_header_size __read_mostly = 128;
642	712
643	713	/* HW BM need that each port be identify by a unique ID */
644	714	static int global_port_id;
..	..	@@ -681,13 +751,12 @@
681	751	static void mvneta_mib_counters_clear(struct mvneta_port *pp)
682	752	{
683	753	int i;
684		- u32 dummy;
685	754
686	755	/* Perform dummy reads from MIB counters */
687	756	for (i = 0; i < MVNETA_MIB_LATE_COLLISION; i += 4)
688		- dummy = mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i));
689		- dummy = mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT);
690		- dummy = mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT);
	757	+ mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i));
	758	+ mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT);
	759	+ mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT);
691	760	}
692	761
693	762	/* Get System Network Statistics */
..	..	@@ -711,12 +780,12 @@
711	780	cpu_stats = per_cpu_ptr(pp->stats, cpu);
712	781	do {
713	782	start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
714		- rx_packets = cpu_stats->rx_packets;
715		- rx_bytes = cpu_stats->rx_bytes;
	783	+ rx_packets = cpu_stats->es.ps.rx_packets;
	784	+ rx_bytes = cpu_stats->es.ps.rx_bytes;
716	785	rx_dropped = cpu_stats->rx_dropped;
717	786	rx_errors = cpu_stats->rx_errors;
718		- tx_packets = cpu_stats->tx_packets;
719		- tx_bytes = cpu_stats->tx_bytes;
	787	+ tx_packets = cpu_stats->es.ps.tx_packets;
	788	+ tx_bytes = cpu_stats->es.ps.tx_bytes;
720	789	} while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
721	790
722	791	stats->rx_packets += rx_packets;
..	..	@@ -1119,7 +1188,7 @@
1119	1188	SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(bm_pool->pkt_size));
1120	1189
1121	1190	/* Fill entire long pool */
1122		- num = hwbm_pool_add(hwbm_pool, hwbm_pool->size, GFP_ATOMIC);
	1191	+ num = hwbm_pool_add(hwbm_pool, hwbm_pool->size);
1123	1192	if (num != hwbm_pool->size) {
1124	1193	WARN(1, "pool %d: %d of %d allocated\n",
1125	1194	bm_pool->id, num, hwbm_pool->size);
..	..	@@ -1134,6 +1203,7 @@
1134	1203	mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 1 << pp->id);
1135	1204
1136	1205	pp->bm_priv = NULL;
	1206	+ pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
1137	1207	mvreg_write(pp, MVNETA_ACC_MODE, MVNETA_ACC_MODE_EXT1);
1138	1208	netdev_info(pp->dev, "fail to update MTU, fall back to software BM\n");
1139	1209	}
..	..	@@ -1761,30 +1831,34 @@
1761	1831	/* Free tx queue skbuffs */
1762	1832	static void mvneta_txq_bufs_free(struct mvneta_port *pp,
1763	1833	struct mvneta_tx_queue *txq, int num,
1764		- struct netdev_queue *nq)
	1834	+ struct netdev_queue *nq, bool napi)
1765	1835	{
1766	1836	unsigned int bytes_compl = 0, pkts_compl = 0;
1767	1837	int i;
1768	1838
1769	1839	for (i = 0; i < num; i++) {
	1840	+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_get_index];
1770	1841	struct mvneta_tx_desc *tx_desc = txq->descs +
1771	1842	txq->txq_get_index;
1772		- struct sk_buff *skb = txq->tx_skb[txq->txq_get_index];
1773		-
1774		- if (skb) {
1775		- bytes_compl += skb->len;
1776		- pkts_compl++;
1777		- }
1778	1843
1779	1844	mvneta_txq_inc_get(txq);
1780	1845
1781		- if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr))
	1846	+ if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr) &&
	1847	+ buf->type != MVNETA_TYPE_XDP_TX)
1782	1848	dma_unmap_single(pp->dev->dev.parent,
1783	1849	tx_desc->buf_phys_addr,
1784	1850	tx_desc->data_size, DMA_TO_DEVICE);
1785		- if (!skb)
1786		- continue;
1787		- dev_kfree_skb_any(skb);
	1851	+ if (buf->type == MVNETA_TYPE_SKB && buf->skb) {
	1852	+ bytes_compl += buf->skb->len;
	1853	+ pkts_compl++;
	1854	+ dev_kfree_skb_any(buf->skb);
	1855	+ } else if (buf->type == MVNETA_TYPE_XDP_TX \|\|
	1856	+ buf->type == MVNETA_TYPE_XDP_NDO) {
	1857	+ if (napi && buf->type == MVNETA_TYPE_XDP_TX)
	1858	+ xdp_return_frame_rx_napi(buf->xdpf);
	1859	+ else
	1860	+ xdp_return_frame(buf->xdpf);
	1861	+ }
1788	1862	}
1789	1863
1790	1864	netdev_tx_completed_queue(nq, pkts_compl, bytes_compl);
..	..	@@ -1801,7 +1875,7 @@
1801	1875	if (!tx_done)
1802	1876	return;
1803	1877
1804		- mvneta_txq_bufs_free(pp, txq, tx_done, nq);
	1878	+ mvneta_txq_bufs_free(pp, txq, tx_done, nq, true);
1805	1879
1806	1880	txq->count -= tx_done;
1807	1881
..	..	@@ -1821,20 +1895,14 @@
1821	1895	dma_addr_t phys_addr;
1822	1896	struct page *page;
1823	1897
1824		- page = __dev_alloc_page(gfp_mask);
	1898	+ page = page_pool_alloc_pages(rxq->page_pool,
	1899	+ gfp_mask \| __GFP_NOWARN);
1825	1900	if (!page)
1826	1901	return -ENOMEM;
1827	1902
1828		- /* map page for use */
1829		- phys_addr = dma_map_page(pp->dev->dev.parent, page, 0, PAGE_SIZE,
1830		- DMA_FROM_DEVICE);
1831		- if (unlikely(dma_mapping_error(pp->dev->dev.parent, phys_addr))) {
1832		- __free_page(page);
1833		- return -ENOMEM;
1834		- }
1835		-
1836		- phys_addr += pp->rx_offset_correction;
	1903	+ phys_addr = page_pool_get_dma_addr(page) + pp->rx_offset_correction;
1837	1904	mvneta_rx_desc_fill(rx_desc, phys_addr, page, rxq);
	1905	+
1838	1906	return 0;
1839	1907	}
1840	1908
..	..	@@ -1900,10 +1968,28 @@
1900	1968	if (!data \|\| !(rx_desc->buf_phys_addr))
1901	1969	continue;
1902	1970
1903		- dma_unmap_page(pp->dev->dev.parent, rx_desc->buf_phys_addr,
1904		- PAGE_SIZE, DMA_FROM_DEVICE);
1905		- __free_page(data);
	1971	+ page_pool_put_full_page(rxq->page_pool, data, false);
1906	1972	}
	1973	+ if (xdp_rxq_info_is_reg(&rxq->xdp_rxq))
	1974	+ xdp_rxq_info_unreg(&rxq->xdp_rxq);
	1975	+ page_pool_destroy(rxq->page_pool);
	1976	+ rxq->page_pool = NULL;
	1977	+}
	1978	+
	1979	+static void
	1980	+mvneta_update_stats(struct mvneta_port *pp,
	1981	+ struct mvneta_stats *ps)
	1982	+{
	1983	+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
	1984	+
	1985	+ u64_stats_update_begin(&stats->syncp);
	1986	+ stats->es.ps.rx_packets += ps->rx_packets;
	1987	+ stats->es.ps.rx_bytes += ps->rx_bytes;
	1988	+ /* xdp */
	1989	+ stats->es.ps.xdp_redirect += ps->xdp_redirect;
	1990	+ stats->es.ps.xdp_pass += ps->xdp_pass;
	1991	+ stats->es.ps.xdp_drop += ps->xdp_drop;
	1992	+ u64_stats_update_end(&stats->syncp);
1907	1993	}
1908	1994
1909	1995	static inline
..	..	@@ -1917,9 +2003,15 @@
1917	2003	rx_desc = rxq->descs + curr_desc;
1918	2004	if (!(rx_desc->buf_phys_addr)) {
1919	2005	if (mvneta_rx_refill(pp, rx_desc, rxq, GFP_ATOMIC)) {
	2006	+ struct mvneta_pcpu_stats *stats;
	2007	+
1920	2008	pr_err("Can't refill queue %d. Done %d from %d\n",
1921	2009	rxq->id, i, rxq->refill_num);
1922		- rxq->refill_err++;
	2010	+
	2011	+ stats = this_cpu_ptr(pp->stats);
	2012	+ u64_stats_update_begin(&stats->syncp);
	2013	+ stats->es.refill_error++;
	2014	+ u64_stats_update_end(&stats->syncp);
1923	2015	break;
1924	2016	}
1925	2017	}
..	..	@@ -1931,38 +2023,346 @@
1931	2023	return i;
1932	2024	}
1933	2025
	2026	+static void
	2027	+mvneta_xdp_put_buff(struct mvneta_port pp, struct mvneta_rx_queue rxq,
	2028	+ struct xdp_buff *xdp, int sync_len, bool napi)
	2029	+{
	2030	+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
	2031	+ int i;
	2032	+
	2033	+ for (i = 0; i < sinfo->nr_frags; i++)
	2034	+ page_pool_put_full_page(rxq->page_pool,
	2035	+ skb_frag_page(&sinfo->frags[i]), napi);
	2036	+ page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data),
	2037	+ sync_len, napi);
	2038	+}
	2039	+
	2040	+static int
	2041	+mvneta_xdp_submit_frame(struct mvneta_port pp, struct mvneta_tx_queue txq,
	2042	+ struct xdp_frame *xdpf, bool dma_map)
	2043	+{
	2044	+ struct mvneta_tx_desc *tx_desc;
	2045	+ struct mvneta_tx_buf *buf;
	2046	+ dma_addr_t dma_addr;
	2047	+
	2048	+ if (txq->count >= txq->tx_stop_threshold)
	2049	+ return MVNETA_XDP_DROPPED;
	2050	+
	2051	+ tx_desc = mvneta_txq_next_desc_get(txq);
	2052	+
	2053	+ buf = &txq->buf[txq->txq_put_index];
	2054	+ if (dma_map) {
	2055	+ /* ndo_xdp_xmit */
	2056	+ dma_addr = dma_map_single(pp->dev->dev.parent, xdpf->data,
	2057	+ xdpf->len, DMA_TO_DEVICE);
	2058	+ if (dma_mapping_error(pp->dev->dev.parent, dma_addr)) {
	2059	+ mvneta_txq_desc_put(txq);
	2060	+ return MVNETA_XDP_DROPPED;
	2061	+ }
	2062	+ buf->type = MVNETA_TYPE_XDP_NDO;
	2063	+ } else {
	2064	+ struct page *page = virt_to_page(xdpf->data);
	2065	+
	2066	+ dma_addr = page_pool_get_dma_addr(page) +
	2067	+ sizeof(*xdpf) + xdpf->headroom;
	2068	+ dma_sync_single_for_device(pp->dev->dev.parent, dma_addr,
	2069	+ xdpf->len, DMA_BIDIRECTIONAL);
	2070	+ buf->type = MVNETA_TYPE_XDP_TX;
	2071	+ }
	2072	+ buf->xdpf = xdpf;
	2073	+
	2074	+ tx_desc->command = MVNETA_TXD_FLZ_DESC;
	2075	+ tx_desc->buf_phys_addr = dma_addr;
	2076	+ tx_desc->data_size = xdpf->len;
	2077	+
	2078	+ mvneta_txq_inc_put(txq);
	2079	+ txq->pending++;
	2080	+ txq->count++;
	2081	+
	2082	+ return MVNETA_XDP_TX;
	2083	+}
	2084	+
	2085	+static int
	2086	+mvneta_xdp_xmit_back(struct mvneta_port pp, struct xdp_buff xdp)
	2087	+{
	2088	+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
	2089	+ struct mvneta_tx_queue *txq;
	2090	+ struct netdev_queue *nq;
	2091	+ struct xdp_frame *xdpf;
	2092	+ int cpu;
	2093	+ u32 ret;
	2094	+
	2095	+ xdpf = xdp_convert_buff_to_frame(xdp);
	2096	+ if (unlikely(!xdpf))
	2097	+ return MVNETA_XDP_DROPPED;
	2098	+
	2099	+ cpu = smp_processor_id();
	2100	+ txq = &pp->txqs[cpu % txq_number];
	2101	+ nq = netdev_get_tx_queue(pp->dev, txq->id);
	2102	+
	2103	+ __netif_tx_lock(nq, cpu);
	2104	+ ret = mvneta_xdp_submit_frame(pp, txq, xdpf, false);
	2105	+ if (ret == MVNETA_XDP_TX) {
	2106	+ u64_stats_update_begin(&stats->syncp);
	2107	+ stats->es.ps.tx_bytes += xdpf->len;
	2108	+ stats->es.ps.tx_packets++;
	2109	+ stats->es.ps.xdp_tx++;
	2110	+ u64_stats_update_end(&stats->syncp);
	2111	+
	2112	+ mvneta_txq_pend_desc_add(pp, txq, 0);
	2113	+ } else {
	2114	+ u64_stats_update_begin(&stats->syncp);
	2115	+ stats->es.ps.xdp_tx_err++;
	2116	+ u64_stats_update_end(&stats->syncp);
	2117	+ }
	2118	+ __netif_tx_unlock(nq);
	2119	+
	2120	+ return ret;
	2121	+}
	2122	+
	2123	+static int
	2124	+mvneta_xdp_xmit(struct net_device *dev, int num_frame,
	2125	+ struct xdp_frame **frames, u32 flags)
	2126	+{
	2127	+ struct mvneta_port *pp = netdev_priv(dev);
	2128	+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
	2129	+ int i, nxmit_byte = 0, nxmit = num_frame;
	2130	+ int cpu = smp_processor_id();
	2131	+ struct mvneta_tx_queue *txq;
	2132	+ struct netdev_queue *nq;
	2133	+ u32 ret;
	2134	+
	2135	+ if (unlikely(test_bit(__MVNETA_DOWN, &pp->state)))
	2136	+ return -ENETDOWN;
	2137	+
	2138	+ if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
	2139	+ return -EINVAL;
	2140	+
	2141	+ txq = &pp->txqs[cpu % txq_number];
	2142	+ nq = netdev_get_tx_queue(pp->dev, txq->id);
	2143	+
	2144	+ __netif_tx_lock(nq, cpu);
	2145	+ for (i = 0; i < num_frame; i++) {
	2146	+ ret = mvneta_xdp_submit_frame(pp, txq, frames[i], true);
	2147	+ if (ret == MVNETA_XDP_TX) {
	2148	+ nxmit_byte += frames[i]->len;
	2149	+ } else {
	2150	+ xdp_return_frame_rx_napi(frames[i]);
	2151	+ nxmit--;
	2152	+ }
	2153	+ }
	2154	+
	2155	+ if (unlikely(flags & XDP_XMIT_FLUSH))
	2156	+ mvneta_txq_pend_desc_add(pp, txq, 0);
	2157	+ __netif_tx_unlock(nq);
	2158	+
	2159	+ u64_stats_update_begin(&stats->syncp);
	2160	+ stats->es.ps.tx_bytes += nxmit_byte;
	2161	+ stats->es.ps.tx_packets += nxmit;
	2162	+ stats->es.ps.xdp_xmit += nxmit;
	2163	+ stats->es.ps.xdp_xmit_err += num_frame - nxmit;
	2164	+ u64_stats_update_end(&stats->syncp);
	2165	+
	2166	+ return nxmit;
	2167	+}
	2168	+
	2169	+static int
	2170	+mvneta_run_xdp(struct mvneta_port pp, struct mvneta_rx_queue rxq,
	2171	+ struct bpf_prog prog, struct xdp_buff xdp,
	2172	+ u32 frame_sz, struct mvneta_stats *stats)
	2173	+{
	2174	+ unsigned int len, data_len, sync;
	2175	+ u32 ret, act;
	2176	+
	2177	+ len = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction;
	2178	+ data_len = xdp->data_end - xdp->data;
	2179	+ act = bpf_prog_run_xdp(prog, xdp);
	2180	+
	2181	+ /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */
	2182	+ sync = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction;
	2183	+ sync = max(sync, len);
	2184	+
	2185	+ switch (act) {
	2186	+ case XDP_PASS:
	2187	+ stats->xdp_pass++;
	2188	+ return MVNETA_XDP_PASS;
	2189	+ case XDP_REDIRECT: {
	2190	+ int err;
	2191	+
	2192	+ err = xdp_do_redirect(pp->dev, xdp, prog);
	2193	+ if (unlikely(err)) {
	2194	+ mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
	2195	+ ret = MVNETA_XDP_DROPPED;
	2196	+ } else {
	2197	+ ret = MVNETA_XDP_REDIR;
	2198	+ stats->xdp_redirect++;
	2199	+ }
	2200	+ break;
	2201	+ }
	2202	+ case XDP_TX:
	2203	+ ret = mvneta_xdp_xmit_back(pp, xdp);
	2204	+ if (ret != MVNETA_XDP_TX)
	2205	+ mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
	2206	+ break;
	2207	+ default:
	2208	+ bpf_warn_invalid_xdp_action(act);
	2209	+ fallthrough;
	2210	+ case XDP_ABORTED:
	2211	+ trace_xdp_exception(pp->dev, prog, act);
	2212	+ fallthrough;
	2213	+ case XDP_DROP:
	2214	+ mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
	2215	+ ret = MVNETA_XDP_DROPPED;
	2216	+ stats->xdp_drop++;
	2217	+ break;
	2218	+ }
	2219	+
	2220	+ stats->rx_bytes += frame_sz + xdp->data_end - xdp->data - data_len;
	2221	+ stats->rx_packets++;
	2222	+
	2223	+ return ret;
	2224	+}
	2225	+
	2226	+static void
	2227	+mvneta_swbm_rx_frame(struct mvneta_port *pp,
	2228	+ struct mvneta_rx_desc *rx_desc,
	2229	+ struct mvneta_rx_queue *rxq,
	2230	+ struct xdp_buff xdp, int size,
	2231	+ struct page *page)
	2232	+{
	2233	+ unsigned char *data = page_address(page);
	2234	+ int data_len = -MVNETA_MH_SIZE, len;
	2235	+ struct net_device *dev = pp->dev;
	2236	+ enum dma_data_direction dma_dir;
	2237	+ struct skb_shared_info *sinfo;
	2238	+
	2239	+ if (*size > MVNETA_MAX_RX_BUF_SIZE) {
	2240	+ len = MVNETA_MAX_RX_BUF_SIZE;
	2241	+ data_len += len;
	2242	+ } else {
	2243	+ len = *size;
	2244	+ data_len += len - ETH_FCS_LEN;
	2245	+ }
	2246	+ size = size - len;
	2247	+
	2248	+ dma_dir = page_pool_get_dma_dir(rxq->page_pool);
	2249	+ dma_sync_single_for_cpu(dev->dev.parent,
	2250	+ rx_desc->buf_phys_addr,
	2251	+ len, dma_dir);
	2252	+
	2253	+ rx_desc->buf_phys_addr = 0;
	2254	+
	2255	+ /* Prefetch header */
	2256	+ prefetch(data);
	2257	+
	2258	+ xdp->data_hard_start = data;
	2259	+ xdp->data = data + pp->rx_offset_correction + MVNETA_MH_SIZE;
	2260	+ xdp->data_end = xdp->data + data_len;
	2261	+ xdp_set_data_meta_invalid(xdp);
	2262	+
	2263	+ sinfo = xdp_get_shared_info_from_buff(xdp);
	2264	+ sinfo->nr_frags = 0;
	2265	+}
	2266	+
	2267	+static void
	2268	+mvneta_swbm_add_rx_fragment(struct mvneta_port *pp,
	2269	+ struct mvneta_rx_desc *rx_desc,
	2270	+ struct mvneta_rx_queue *rxq,
	2271	+ struct xdp_buff xdp, int size,
	2272	+ struct page *page)
	2273	+{
	2274	+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
	2275	+ struct net_device *dev = pp->dev;
	2276	+ enum dma_data_direction dma_dir;
	2277	+ int data_len, len;
	2278	+
	2279	+ if (*size > MVNETA_MAX_RX_BUF_SIZE) {
	2280	+ len = MVNETA_MAX_RX_BUF_SIZE;
	2281	+ data_len = len;
	2282	+ } else {
	2283	+ len = *size;
	2284	+ data_len = len - ETH_FCS_LEN;
	2285	+ }
	2286	+ dma_dir = page_pool_get_dma_dir(rxq->page_pool);
	2287	+ dma_sync_single_for_cpu(dev->dev.parent,
	2288	+ rx_desc->buf_phys_addr,
	2289	+ len, dma_dir);
	2290	+ rx_desc->buf_phys_addr = 0;
	2291	+
	2292	+ if (data_len > 0 && sinfo->nr_frags < MAX_SKB_FRAGS) {
	2293	+ skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags];
	2294	+
	2295	+ skb_frag_off_set(frag, pp->rx_offset_correction);
	2296	+ skb_frag_size_set(frag, data_len);
	2297	+ __skb_frag_set_page(frag, page);
	2298	+ sinfo->nr_frags++;
	2299	+ } else {
	2300	+ page_pool_put_full_page(rxq->page_pool, page, true);
	2301	+ }
	2302	+ *size -= len;
	2303	+}
	2304	+
	2305	+static struct sk_buff *
	2306	+mvneta_swbm_build_skb(struct mvneta_port pp, struct mvneta_rx_queue rxq,
	2307	+ struct xdp_buff *xdp, u32 desc_status)
	2308	+{
	2309	+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
	2310	+ int i, num_frags = sinfo->nr_frags;
	2311	+ struct sk_buff *skb;
	2312	+
	2313	+ skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
	2314	+ if (!skb)
	2315	+ return ERR_PTR(-ENOMEM);
	2316	+
	2317	+ page_pool_release_page(rxq->page_pool, virt_to_page(xdp->data));
	2318	+
	2319	+ skb_reserve(skb, xdp->data - xdp->data_hard_start);
	2320	+ skb_put(skb, xdp->data_end - xdp->data);
	2321	+ mvneta_rx_csum(pp, desc_status, skb);
	2322	+
	2323	+ for (i = 0; i < num_frags; i++) {
	2324	+ skb_frag_t *frag = &sinfo->frags[i];
	2325	+
	2326	+ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
	2327	+ skb_frag_page(frag), skb_frag_off(frag),
	2328	+ skb_frag_size(frag), PAGE_SIZE);
	2329	+ page_pool_release_page(rxq->page_pool, skb_frag_page(frag));
	2330	+ }
	2331	+
	2332	+ return skb;
	2333	+}
	2334	+
1934	2335	/* Main rx processing when using software buffer management */
1935	2336	static int mvneta_rx_swbm(struct napi_struct *napi,
1936	2337	struct mvneta_port *pp, int budget,
1937	2338	struct mvneta_rx_queue *rxq)
1938	2339	{
	2340	+ int rx_proc = 0, rx_todo, refill, size = 0;
1939	2341	struct net_device *dev = pp->dev;
1940		- int rx_todo, rx_proc;
1941		- int refill = 0;
1942		- u32 rcvd_pkts = 0;
1943		- u32 rcvd_bytes = 0;
	2342	+ struct xdp_buff xdp_buf = {
	2343	+ .frame_sz = PAGE_SIZE,
	2344	+ .rxq = &rxq->xdp_rxq,
	2345	+ };
	2346	+ struct mvneta_stats ps = {};
	2347	+ struct bpf_prog *xdp_prog;
	2348	+ u32 desc_status, frame_sz;
1944	2349
1945	2350	/* Get number of received packets */
1946	2351	rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);
1947		- rx_proc = 0;
	2352	+
	2353	+ rcu_read_lock();
	2354	+ xdp_prog = READ_ONCE(pp->xdp_prog);
1948	2355
1949	2356	/* Fairness NAPI loop */
1950		- while ((rcvd_pkts < budget) && (rx_proc < rx_todo)) {
	2357	+ while (rx_proc < budget && rx_proc < rx_todo) {
1951	2358	struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
1952		- unsigned char *data;
1953		- struct page *page;
1954		- dma_addr_t phys_addr;
1955	2359	u32 rx_status, index;
1956		- int rx_bytes, skb_size, copy_size;
1957		- int frag_num, frag_size, frag_offset;
	2360	+ struct sk_buff *skb;
	2361	+ struct page *page;
1958	2362
1959	2363	index = rx_desc - rxq->descs;
1960	2364	page = (struct page *)rxq->buf_virt_addr[index];
1961		- data = page_address(page);
1962		- /* Prefetch header */
1963		- prefetch(data);
1964	2365
1965		- phys_addr = rx_desc->buf_phys_addr;
1966	2366	rx_status = rx_desc->status;
1967	2367	rx_proc++;
1968	2368	rxq->refill_num++;
..	..	@@ -1971,132 +2371,72 @@
1971	2371	/* Check errors only for FIRST descriptor */
1972	2372	if (rx_status & MVNETA_RXD_ERR_SUMMARY) {
1973	2373	mvneta_rx_error(pp, rx_desc);
1974		- /* leave the descriptor untouched */
1975		- continue;
	2374	+ goto next;
1976	2375	}
1977		- rx_bytes = rx_desc->data_size -
1978		- (ETH_FCS_LEN + MVNETA_MH_SIZE);
1979	2376
1980		- /* Allocate small skb for each new packet */
1981		- skb_size = max(rx_copybreak, rx_header_size);
1982		- rxq->skb = netdev_alloc_skb_ip_align(dev, skb_size);
1983		- if (unlikely(!rxq->skb)) {
1984		- struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
	2377	+ size = rx_desc->data_size;
	2378	+ frame_sz = size - ETH_FCS_LEN;
	2379	+ desc_status = rx_status;
1985	2380
1986		- netdev_err(dev,
1987		- "Can't allocate skb on queue %d\n",
1988		- rxq->id);
1989		-
1990		- rxq->skb_alloc_err++;
1991		-
1992		- u64_stats_update_begin(&stats->syncp);
1993		- stats->rx_dropped++;
1994		- u64_stats_update_end(&stats->syncp);
1995		- continue;
1996		- }
1997		- copy_size = min(skb_size, rx_bytes);
1998		-
1999		- /* Copy data from buffer to SKB, skip Marvell header */
2000		- memcpy(rxq->skb->data, data + MVNETA_MH_SIZE,
2001		- copy_size);
2002		- skb_put(rxq->skb, copy_size);
2003		- rxq->left_size = rx_bytes - copy_size;
2004		-
2005		- mvneta_rx_csum(pp, rx_status, rxq->skb);
2006		- if (rxq->left_size == 0) {
2007		- int size = copy_size + MVNETA_MH_SIZE;
2008		-
2009		- dma_sync_single_range_for_cpu(dev->dev.parent,
2010		- phys_addr, 0,
2011		- size,
2012		- DMA_FROM_DEVICE);
2013		-
2014		- /* leave the descriptor and buffer untouched */
2015		- } else {
2016		- /* refill descriptor with new buffer later */
2017		- rx_desc->buf_phys_addr = 0;
2018		-
2019		- frag_num = 0;
2020		- frag_offset = copy_size + MVNETA_MH_SIZE;
2021		- frag_size = min(rxq->left_size,
2022		- (int)(PAGE_SIZE - frag_offset));
2023		- skb_add_rx_frag(rxq->skb, frag_num, page,
2024		- frag_offset, frag_size,
2025		- PAGE_SIZE);
2026		- dma_unmap_page(dev->dev.parent, phys_addr,
2027		- PAGE_SIZE, DMA_FROM_DEVICE);
2028		- rxq->left_size -= frag_size;
2029		- }
	2381	+ mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,
	2382	+ &size, page);
2030	2383	} else {
2031		- /* Middle or Last descriptor */
2032		- if (unlikely(!rxq->skb)) {
2033		- pr_debug("no skb for rx_status 0x%x\n",
2034		- rx_status);
	2384	+ if (unlikely(!xdp_buf.data_hard_start)) {
	2385	+ rx_desc->buf_phys_addr = 0;
	2386	+ page_pool_put_full_page(rxq->page_pool, page,
	2387	+ true);
2035	2388	continue;
2036	2389	}
2037		- if (!rxq->left_size) {
2038		- /* last descriptor has only FCS */
2039		- /* and can be discarded */
2040		- dma_sync_single_range_for_cpu(dev->dev.parent,
2041		- phys_addr, 0,
2042		- ETH_FCS_LEN,
2043		- DMA_FROM_DEVICE);
2044		- /* leave the descriptor and buffer untouched */
2045		- } else {
2046		- /* refill descriptor with new buffer later */
2047		- rx_desc->buf_phys_addr = 0;
2048	2390
2049		- frag_num = skb_shinfo(rxq->skb)->nr_frags;
2050		- frag_offset = 0;
2051		- frag_size = min(rxq->left_size,
2052		- (int)(PAGE_SIZE - frag_offset));
2053		- skb_add_rx_frag(rxq->skb, frag_num, page,
2054		- frag_offset, frag_size,
2055		- PAGE_SIZE);
2056		-
2057		- dma_unmap_page(dev->dev.parent, phys_addr,
2058		- PAGE_SIZE, DMA_FROM_DEVICE);
2059		-
2060		- rxq->left_size -= frag_size;
2061		- }
	2391	+ mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf,
	2392	+ &size, page);
2062	2393	} /* Middle or Last descriptor */
2063	2394
2064	2395	if (!(rx_status & MVNETA_RXD_LAST_DESC))
2065	2396	/* no last descriptor this time */
2066	2397	continue;
2067	2398
2068		- if (rxq->left_size) {
2069		- pr_err("get last desc, but left_size (%d) != 0\n",
2070		- rxq->left_size);
2071		- dev_kfree_skb_any(rxq->skb);
2072		- rxq->left_size = 0;
2073		- rxq->skb = NULL;
2074		- continue;
	2399	+ if (size) {
	2400	+ mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
	2401	+ goto next;
2075	2402	}
2076		- rcvd_pkts++;
2077		- rcvd_bytes += rxq->skb->len;
2078	2403
2079		- /* Linux processing */
2080		- rxq->skb->protocol = eth_type_trans(rxq->skb, dev);
	2404	+ if (xdp_prog &&
	2405	+ mvneta_run_xdp(pp, rxq, xdp_prog, &xdp_buf, frame_sz, &ps))
	2406	+ goto next;
2081	2407
2082		- if (dev->features & NETIF_F_GRO)
2083		- napi_gro_receive(napi, rxq->skb);
2084		- else
2085		- netif_receive_skb(rxq->skb);
	2408	+ skb = mvneta_swbm_build_skb(pp, rxq, &xdp_buf, desc_status);
	2409	+ if (IS_ERR(skb)) {
	2410	+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2086	2411
2087		- /* clean uncomplete skb pointer in queue */
2088		- rxq->skb = NULL;
2089		- rxq->left_size = 0;
	2412	+ mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
	2413	+
	2414	+ u64_stats_update_begin(&stats->syncp);
	2415	+ stats->es.skb_alloc_error++;
	2416	+ stats->rx_dropped++;
	2417	+ u64_stats_update_end(&stats->syncp);
	2418	+
	2419	+ goto next;
	2420	+ }
	2421	+
	2422	+ ps.rx_bytes += skb->len;
	2423	+ ps.rx_packets++;
	2424	+
	2425	+ skb->protocol = eth_type_trans(skb, dev);
	2426	+ napi_gro_receive(napi, skb);
	2427	+next:
	2428	+ xdp_buf.data_hard_start = NULL;
2090	2429	}
	2430	+ rcu_read_unlock();
2091	2431
2092		- if (rcvd_pkts) {
2093		- struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
	2432	+ if (xdp_buf.data_hard_start)
	2433	+ mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
2094	2434
2095		- u64_stats_update_begin(&stats->syncp);
2096		- stats->rx_packets += rcvd_pkts;
2097		- stats->rx_bytes += rcvd_bytes;
2098		- u64_stats_update_end(&stats->syncp);
2099		- }
	2435	+ if (ps.xdp_redirect)
	2436	+ xdp_do_flush_map();
	2437	+
	2438	+ if (ps.rx_packets)
	2439	+ mvneta_update_stats(pp, &ps);
2100	2440
2101	2441	/* return some buffers to hardware queue, one at a time is too slow */
2102	2442	refill = mvneta_rx_refill_queue(pp, rxq);
..	..	@@ -2104,7 +2444,7 @@
2104	2444	/* Update rxq management counters */
2105	2445	mvneta_rxq_desc_num_update(pp, rxq, rx_proc, refill);
2106	2446
2107		- return rcvd_pkts;
	2447	+ return ps.rx_packets;
2108	2448	}
2109	2449
2110	2450	/* Main rx processing when using hardware buffer management */
..	..	@@ -2188,8 +2528,15 @@
2188	2528	/* Refill processing */
2189	2529	err = hwbm_pool_refill(&bm_pool->hwbm_pool, GFP_ATOMIC);
2190	2530	if (err) {
	2531	+ struct mvneta_pcpu_stats *stats;
	2532	+
2191	2533	netdev_err(dev, "Linux processing - Can't refill\n");
2192		- rxq->refill_err++;
	2534	+
	2535	+ stats = this_cpu_ptr(pp->stats);
	2536	+ u64_stats_update_begin(&stats->syncp);
	2537	+ stats->es.refill_error++;
	2538	+ u64_stats_update_end(&stats->syncp);
	2539	+
2193	2540	goto err_drop_frame_ret_pool;
2194	2541	}
2195	2542
..	..	@@ -2223,8 +2570,8 @@
2223	2570	struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2224	2571
2225	2572	u64_stats_update_begin(&stats->syncp);
2226		- stats->rx_packets += rcvd_pkts;
2227		- stats->rx_bytes += rcvd_bytes;
	2573	+ stats->es.ps.rx_packets += rcvd_pkts;
	2574	+ stats->es.ps.rx_bytes += rcvd_bytes;
2228	2575	u64_stats_update_end(&stats->syncp);
2229	2576	}
2230	2577
..	..	@@ -2238,16 +2585,19 @@
2238	2585	mvneta_tso_put_hdr(struct sk_buff *skb,
2239	2586	struct mvneta_port pp, struct mvneta_tx_queue txq)
2240	2587	{
2241		- struct mvneta_tx_desc *tx_desc;
2242	2588	int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
	2589	+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
	2590	+ struct mvneta_tx_desc *tx_desc;
2243	2591
2244		- txq->tx_skb[txq->txq_put_index] = NULL;
2245	2592	tx_desc = mvneta_txq_next_desc_get(txq);
2246	2593	tx_desc->data_size = hdr_len;
2247	2594	tx_desc->command = mvneta_skb_tx_csum(pp, skb);
2248	2595	tx_desc->command \|= MVNETA_TXD_F_DESC;
2249	2596	tx_desc->buf_phys_addr = txq->tso_hdrs_phys +
2250	2597	txq->txq_put_index * TSO_HEADER_SIZE;
	2598	+ buf->type = MVNETA_TYPE_SKB;
	2599	+ buf->skb = NULL;
	2600	+
2251	2601	mvneta_txq_inc_put(txq);
2252	2602	}
2253	2603
..	..	@@ -2256,6 +2606,7 @@
2256	2606	struct sk_buff skb, char data, int size,
2257	2607	bool last_tcp, bool is_last)
2258	2608	{
	2609	+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
2259	2610	struct mvneta_tx_desc *tx_desc;
2260	2611
2261	2612	tx_desc = mvneta_txq_next_desc_get(txq);
..	..	@@ -2269,7 +2620,8 @@
2269	2620	}
2270	2621
2271	2622	tx_desc->command = 0;
2272		- txq->tx_skb[txq->txq_put_index] = NULL;
	2623	+ buf->type = MVNETA_TYPE_SKB;
	2624	+ buf->skb = NULL;
2273	2625
2274	2626	if (last_tcp) {
2275	2627	/* last descriptor in the TCP packet */
..	..	@@ -2277,7 +2629,7 @@
2277	2629
2278	2630	/* last descriptor in SKB */
2279	2631	if (is_last)
2280		- txq->tx_skb[txq->txq_put_index] = skb;
	2632	+ buf->skb = skb;
2281	2633	}
2282	2634	mvneta_txq_inc_put(txq);
2283	2635	return 0;
..	..	@@ -2286,11 +2638,10 @@
2286	2638	static int mvneta_tx_tso(struct sk_buff skb, struct net_device dev,
2287	2639	struct mvneta_tx_queue *txq)
2288	2640	{
2289		- int total_len, data_left;
	2641	+ int hdr_len, total_len, data_left;
2290	2642	int desc_count = 0;
2291	2643	struct mvneta_port *pp = netdev_priv(dev);
2292	2644	struct tso_t tso;
2293		- int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2294	2645	int i;
2295	2646
2296	2647	/* Count needed descriptors */
..	..	@@ -2303,7 +2654,7 @@
2303	2654	}
2304	2655
2305	2656	/* Initialize the TSO handler, and prepare the first payload */
2306		- tso_start(skb, &tso);
	2657	+ hdr_len = tso_start(skb, &tso);
2307	2658
2308	2659	total_len = skb->len - hdr_len;
2309	2660	while (total_len > 0) {
..	..	@@ -2362,11 +2713,12 @@
2362	2713	int i, nr_frags = skb_shinfo(skb)->nr_frags;
2363	2714
2364	2715	for (i = 0; i < nr_frags; i++) {
	2716	+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
2365	2717	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2366		- void *addr = page_address(frag->page.p) + frag->page_offset;
	2718	+ void *addr = skb_frag_address(frag);
2367	2719
2368	2720	tx_desc = mvneta_txq_next_desc_get(txq);
2369		- tx_desc->data_size = frag->size;
	2721	+ tx_desc->data_size = skb_frag_size(frag);
2370	2722
2371	2723	tx_desc->buf_phys_addr =
2372	2724	dma_map_single(pp->dev->dev.parent, addr,
..	..	@@ -2381,12 +2733,13 @@
2381	2733	if (i == nr_frags - 1) {
2382	2734	/* Last descriptor */
2383	2735	tx_desc->command = MVNETA_TXD_L_DESC \| MVNETA_TXD_Z_PAD;
2384		- txq->tx_skb[txq->txq_put_index] = skb;
	2736	+ buf->skb = skb;
2385	2737	} else {
2386	2738	/* Descriptor in the middle: Not First, Not Last */
2387	2739	tx_desc->command = 0;
2388		- txq->tx_skb[txq->txq_put_index] = NULL;
	2740	+ buf->skb = NULL;
2389	2741	}
	2742	+ buf->type = MVNETA_TYPE_SKB;
2390	2743	mvneta_txq_inc_put(txq);
2391	2744	}
2392	2745
..	..	@@ -2414,6 +2767,7 @@
2414	2767	struct mvneta_port *pp = netdev_priv(dev);
2415	2768	u16 txq_id = skb_get_queue_mapping(skb);
2416	2769	struct mvneta_tx_queue *txq = &pp->txqs[txq_id];
	2770	+ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
2417	2771	struct mvneta_tx_desc *tx_desc;
2418	2772	int len = skb->len;
2419	2773	int frags = 0;
..	..	@@ -2446,16 +2800,17 @@
2446	2800	goto out;
2447	2801	}
2448	2802
	2803	+ buf->type = MVNETA_TYPE_SKB;
2449	2804	if (frags == 1) {
2450	2805	/* First and Last descriptor */
2451	2806	tx_cmd \|= MVNETA_TXD_FLZ_DESC;
2452	2807	tx_desc->command = tx_cmd;
2453		- txq->tx_skb[txq->txq_put_index] = skb;
	2808	+ buf->skb = skb;
2454	2809	mvneta_txq_inc_put(txq);
2455	2810	} else {
2456	2811	/* First but not Last */
2457	2812	tx_cmd \|= MVNETA_TXD_F_DESC;
2458		- txq->tx_skb[txq->txq_put_index] = NULL;
	2813	+ buf->skb = NULL;
2459	2814	mvneta_txq_inc_put(txq);
2460	2815	tx_desc->command = tx_cmd;
2461	2816	/* Continue with other skb fragments */
..	..	@@ -2472,8 +2827,8 @@
2472	2827
2473	2828	out:
2474	2829	if (frags > 0) {
2475		- struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2476	2830	struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id);
	2831	+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2477	2832
2478	2833	netdev_tx_sent_queue(nq, len);
2479	2834
..	..	@@ -2481,15 +2836,15 @@
2481	2836	if (txq->count >= txq->tx_stop_threshold)
2482	2837	netif_tx_stop_queue(nq);
2483	2838
2484		- if (!skb->xmit_more \|\| netif_xmit_stopped(nq) \|\|
	2839	+ if (!netdev_xmit_more() \|\| netif_xmit_stopped(nq) \|\|
2485	2840	txq->pending + frags > MVNETA_TXQ_DEC_SENT_MASK)
2486	2841	mvneta_txq_pend_desc_add(pp, txq, frags);
2487	2842	else
2488	2843	txq->pending += frags;
2489	2844
2490	2845	u64_stats_update_begin(&stats->syncp);
2491		- stats->tx_packets++;
2492		- stats->tx_bytes += len;
	2846	+ stats->es.ps.tx_bytes += len;
	2847	+ stats->es.ps.tx_packets++;
2493	2848	u64_stats_update_end(&stats->syncp);
2494	2849	} else {
2495	2850	dev->stats.tx_dropped++;
..	..	@@ -2508,7 +2863,7 @@
2508	2863	struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id);
2509	2864	int tx_done = txq->count;
2510	2865
2511		- mvneta_txq_bufs_free(pp, txq, tx_done, nq);
	2866	+ mvneta_txq_bufs_free(pp, txq, tx_done, nq, false);
2512	2867
2513	2868	/* reset txq */
2514	2869	txq->count = 0;
..	..	@@ -2523,12 +2878,13 @@
2523	2878	{
2524	2879	struct mvneta_tx_queue *txq;
2525	2880	struct netdev_queue *nq;
	2881	+ int cpu = smp_processor_id();
2526	2882
2527	2883	while (cause_tx_done) {
2528	2884	txq = mvneta_tx_done_policy(pp, cause_tx_done);
2529	2885
2530	2886	nq = netdev_get_tx_queue(pp->dev, txq->id);
2531		- __netif_tx_lock(nq, smp_processor_id());
	2887	+ __netif_tx_lock(nq, cpu);
2532	2888
2533	2889	if (txq->count)
2534	2890	mvneta_txq_done(pp, txq);
..	..	@@ -2841,11 +3197,57 @@
2841	3197	return rx_done;
2842	3198	}
2843	3199
	3200	+static int mvneta_create_page_pool(struct mvneta_port *pp,
	3201	+ struct mvneta_rx_queue *rxq, int size)
	3202	+{
	3203	+ struct bpf_prog *xdp_prog = READ_ONCE(pp->xdp_prog);
	3204	+ struct page_pool_params pp_params = {
	3205	+ .order = 0,
	3206	+ .flags = PP_FLAG_DMA_MAP \| PP_FLAG_DMA_SYNC_DEV,
	3207	+ .pool_size = size,
	3208	+ .nid = NUMA_NO_NODE,
	3209	+ .dev = pp->dev->dev.parent,
	3210	+ .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE,
	3211	+ .offset = pp->rx_offset_correction,
	3212	+ .max_len = MVNETA_MAX_RX_BUF_SIZE,
	3213	+ };
	3214	+ int err;
	3215	+
	3216	+ rxq->page_pool = page_pool_create(&pp_params);
	3217	+ if (IS_ERR(rxq->page_pool)) {
	3218	+ err = PTR_ERR(rxq->page_pool);
	3219	+ rxq->page_pool = NULL;
	3220	+ return err;
	3221	+ }
	3222	+
	3223	+ err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id);
	3224	+ if (err < 0)
	3225	+ goto err_free_pp;
	3226	+
	3227	+ err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL,
	3228	+ rxq->page_pool);
	3229	+ if (err)
	3230	+ goto err_unregister_rxq;
	3231	+
	3232	+ return 0;
	3233	+
	3234	+err_unregister_rxq:
	3235	+ xdp_rxq_info_unreg(&rxq->xdp_rxq);
	3236	+err_free_pp:
	3237	+ page_pool_destroy(rxq->page_pool);
	3238	+ rxq->page_pool = NULL;
	3239	+ return err;
	3240	+}
	3241	+
2844	3242	/* Handle rxq fill: allocates rxq skbs; called when initializing a port */
2845	3243	static int mvneta_rxq_fill(struct mvneta_port pp, struct mvneta_rx_queue rxq,
2846	3244	int num)
2847	3245	{
2848		- int i;
	3246	+ int i, err;
	3247	+
	3248	+ err = mvneta_create_page_pool(pp, rxq, num);
	3249	+ if (err < 0)
	3250	+ return err;
2849	3251
2850	3252	for (i = 0; i < num; i++) {
2851	3253	memset(rxq->descs + i, 0, sizeof(struct mvneta_rx_desc));
..	..	@@ -2919,7 +3321,7 @@
2919	3321	/* Set Offset */
2920	3322	mvneta_rxq_offset_set(pp, rxq, 0);
2921	3323	mvneta_rxq_buf_size_set(pp, rxq, PAGE_SIZE < SZ_64K ?
2922		- PAGE_SIZE :
	3324	+ MVNETA_MAX_RX_BUF_SIZE :
2923	3325	MVNETA_RX_BUF_SIZE(pp->pkt_size));
2924	3326	mvneta_rxq_bm_disable(pp, rxq);
2925	3327	mvneta_rxq_fill(pp, rxq, rxq->size);
..	..	@@ -2958,9 +3360,6 @@
2958	3360	{
2959	3361	mvneta_rxq_drop_pkts(pp, rxq);
2960	3362
2961		- if (rxq->skb)
2962		- dev_kfree_skb_any(rxq->skb);
2963		-
2964	3363	if (rxq->descs)
2965	3364	dma_free_coherent(pp->dev->dev.parent,
2966	3365	rxq->size * MVNETA_DESC_ALIGNED_SIZE,
..	..	@@ -2973,8 +3372,6 @@
2973	3372	rxq->descs_phys = 0;
2974	3373	rxq->first_to_refill = 0;
2975	3374	rxq->refill_num = 0;
2976		- rxq->skb = NULL;
2977		- rxq->left_size = 0;
2978	3375	}
2979	3376
2980	3377	static int mvneta_txq_sw_init(struct mvneta_port *pp,
..	..	@@ -3000,26 +3397,16 @@
3000	3397
3001	3398	txq->last_desc = txq->size - 1;
3002	3399
3003		- txq->tx_skb = kmalloc_array(txq->size, sizeof(*txq->tx_skb),
3004		- GFP_KERNEL);
3005		- if (!txq->tx_skb) {
3006		- dma_free_coherent(pp->dev->dev.parent,
3007		- txq->size * MVNETA_DESC_ALIGNED_SIZE,
3008		- txq->descs, txq->descs_phys);
	3400	+ txq->buf = kmalloc_array(txq->size, sizeof(*txq->buf), GFP_KERNEL);
	3401	+ if (!txq->buf)
3009	3402	return -ENOMEM;
3010		- }
3011	3403
3012	3404	/* Allocate DMA buffers for TSO MAC/IP/TCP headers */
3013	3405	txq->tso_hdrs = dma_alloc_coherent(pp->dev->dev.parent,
3014	3406	txq->size * TSO_HEADER_SIZE,
3015	3407	&txq->tso_hdrs_phys, GFP_KERNEL);
3016		- if (!txq->tso_hdrs) {
3017		- kfree(txq->tx_skb);
3018		- dma_free_coherent(pp->dev->dev.parent,
3019		- txq->size * MVNETA_DESC_ALIGNED_SIZE,
3020		- txq->descs, txq->descs_phys);
	3408	+ if (!txq->tso_hdrs)
3021	3409	return -ENOMEM;
3022		- }
3023	3410
3024	3411	/* Setup XPS mapping */
3025	3412	if (pp->neta_armada3700)
..	..	@@ -3069,7 +3456,7 @@
3069	3456	{
3070	3457	struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id);
3071	3458
3072		- kfree(txq->tx_skb);
	3459	+ kfree(txq->buf);
3073	3460
3074	3461	if (txq->tso_hdrs)
3075	3462	dma_free_coherent(pp->dev->dev.parent,
..	..	@@ -3163,9 +3550,60 @@
3163	3550	return 0;
3164	3551	}
3165	3552
	3553	+static int mvneta_comphy_init(struct mvneta_port *pp, phy_interface_t interface)
	3554	+{
	3555	+ int ret;
	3556	+
	3557	+ ret = phy_set_mode_ext(pp->comphy, PHY_MODE_ETHERNET, interface);
	3558	+ if (ret)
	3559	+ return ret;
	3560	+
	3561	+ return phy_power_on(pp->comphy);
	3562	+}
	3563	+
	3564	+static int mvneta_config_interface(struct mvneta_port *pp,
	3565	+ phy_interface_t interface)
	3566	+{
	3567	+ int ret = 0;
	3568	+
	3569	+ if (pp->comphy) {
	3570	+ if (interface == PHY_INTERFACE_MODE_SGMII \|\|
	3571	+ interface == PHY_INTERFACE_MODE_1000BASEX \|\|
	3572	+ interface == PHY_INTERFACE_MODE_2500BASEX) {
	3573	+ ret = mvneta_comphy_init(pp, interface);
	3574	+ }
	3575	+ } else {
	3576	+ switch (interface) {
	3577	+ case PHY_INTERFACE_MODE_QSGMII:
	3578	+ mvreg_write(pp, MVNETA_SERDES_CFG,
	3579	+ MVNETA_QSGMII_SERDES_PROTO);
	3580	+ break;
	3581	+
	3582	+ case PHY_INTERFACE_MODE_SGMII:
	3583	+ case PHY_INTERFACE_MODE_1000BASEX:
	3584	+ mvreg_write(pp, MVNETA_SERDES_CFG,
	3585	+ MVNETA_SGMII_SERDES_PROTO);
	3586	+ break;
	3587	+
	3588	+ case PHY_INTERFACE_MODE_2500BASEX:
	3589	+ mvreg_write(pp, MVNETA_SERDES_CFG,
	3590	+ MVNETA_HSGMII_SERDES_PROTO);
	3591	+ break;
	3592	+ default:
	3593	+ break;
	3594	+ }
	3595	+ }
	3596	+
	3597	+ pp->phy_interface = interface;
	3598	+
	3599	+ return ret;
	3600	+}
	3601	+
3166	3602	static void mvneta_start_dev(struct mvneta_port *pp)
3167	3603	{
3168	3604	int cpu;
	3605	+
	3606	+ WARN_ON(mvneta_config_interface(pp, pp->phy_interface));
3169	3607
3170	3608	mvneta_max_rx_size_set(pp, pp->pkt_size);
3171	3609	mvneta_txq_max_tx_size_set(pp, pp->pkt_size);
..	..	@@ -3193,12 +3631,23 @@
3193	3631	MVNETA_CAUSE_LINK_CHANGE);
3194	3632
3195	3633	phylink_start(pp->phylink);
	3634	+
	3635	+ /* We may have called phylink_speed_down before */
	3636	+ phylink_speed_up(pp->phylink);
	3637	+
3196	3638	netif_tx_start_all_queues(pp->dev);
	3639	+
	3640	+ clear_bit(__MVNETA_DOWN, &pp->state);
3197	3641	}
3198	3642
3199	3643	static void mvneta_stop_dev(struct mvneta_port *pp)
3200	3644	{
3201	3645	unsigned int cpu;
	3646	+
	3647	+ set_bit(__MVNETA_DOWN, &pp->state);
	3648	+
	3649	+ if (device_may_wakeup(&pp->dev->dev))
	3650	+ phylink_speed_down(pp->phylink, false);
3202	3651
3203	3652	phylink_stop(pp->phylink);
3204	3653
..	..	@@ -3229,6 +3678,8 @@
3229	3678
3230	3679	mvneta_tx_reset(pp);
3231	3680	mvneta_rx_reset(pp);
	3681	+
	3682	+ WARN_ON(phy_power_off(pp->comphy));
3232	3683	}
3233	3684
3234	3685	static void mvneta_percpu_enable(void *arg)
..	..	@@ -3255,6 +3706,11 @@
3255	3706	netdev_info(dev, "Illegal MTU value %d, rounding to %d\n",
3256	3707	mtu, ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8));
3257	3708	mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8);
	3709	+ }
	3710	+
	3711	+ if (pp->xdp_prog && mtu > MVNETA_MAX_RX_BUF_SIZE) {
	3712	+ netdev_info(dev, "Illegal MTU value %d for XDP mode\n", mtu);
	3713	+ return -EINVAL;
3258	3714	}
3259	3715
3260	3716	dev->mtu = mtu;
..	..	@@ -3351,9 +3807,12 @@
3351	3807	return 0;
3352	3808	}
3353	3809
3354		-static void mvneta_validate(struct net_device ndev, unsigned long supported,
	3810	+static void mvneta_validate(struct phylink_config *config,
	3811	+ unsigned long *supported,
3355	3812	struct phylink_link_state *state)
3356	3813	{
	3814	+ struct net_device *ndev = to_net_dev(config->dev);
	3815	+ struct mvneta_port *pp = netdev_priv(ndev);
3357	3816	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
3358	3817
3359	3818	/* We only support QSGMII, SGMII, 802.3z and RGMII modes */
..	..	@@ -3372,9 +3831,16 @@
3372	3831
3373	3832	/* Asymmetric pause is unsupported */
3374	3833	phylink_set(mask, Pause);
	3834	+
3375	3835	/* Half-duplex at speeds higher than 100Mbit is unsupported */
3376		- phylink_set(mask, 1000baseT_Full);
3377		- phylink_set(mask, 1000baseX_Full);
	3836	+ if (pp->comphy \|\| state->interface != PHY_INTERFACE_MODE_2500BASEX) {
	3837	+ phylink_set(mask, 1000baseT_Full);
	3838	+ phylink_set(mask, 1000baseX_Full);
	3839	+ }
	3840	+ if (pp->comphy \|\| state->interface == PHY_INTERFACE_MODE_2500BASEX) {
	3841	+ phylink_set(mask, 2500baseT_Full);
	3842	+ phylink_set(mask, 2500baseX_Full);
	3843	+ }
3378	3844
3379	3845	if (!phy_interface_mode_is_8023z(state->interface)) {
3380	3846	/* 10M and 100M are only supported in non-802.3z mode */
..	..	@@ -3388,18 +3854,26 @@
3388	3854	__ETHTOOL_LINK_MODE_MASK_NBITS);
3389	3855	bitmap_and(state->advertising, state->advertising, mask,
3390	3856	__ETHTOOL_LINK_MODE_MASK_NBITS);
	3857	+
	3858	+ /* We can only operate at 2500BaseX or 1000BaseX. If requested
	3859	+ * to advertise both, only report advertising at 2500BaseX.
	3860	+ */
	3861	+ phylink_helper_basex_speed(state);
3391	3862	}
3392	3863
3393		-static int mvneta_mac_link_state(struct net_device *ndev,
3394		- struct phylink_link_state *state)
	3864	+static void mvneta_mac_pcs_get_state(struct phylink_config *config,
	3865	+ struct phylink_link_state *state)
3395	3866	{
	3867	+ struct net_device *ndev = to_net_dev(config->dev);
3396	3868	struct mvneta_port *pp = netdev_priv(ndev);
3397	3869	u32 gmac_stat;
3398	3870
3399	3871	gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS);
3400	3872
3401	3873	if (gmac_stat & MVNETA_GMAC_SPEED_1000)
3402		- state->speed = SPEED_1000;
	3874	+ state->speed =
	3875	+ state->interface == PHY_INTERFACE_MODE_2500BASEX ?
	3876	+ SPEED_2500 : SPEED_1000;
3403	3877	else if (gmac_stat & MVNETA_GMAC_SPEED_100)
3404	3878	state->speed = SPEED_100;
3405	3879	else
..	..	@@ -3414,12 +3888,11 @@
3414	3888	state->pause \|= MLO_PAUSE_RX;
3415	3889	if (gmac_stat & MVNETA_GMAC_TX_FLOW_CTRL_ENABLE)
3416	3890	state->pause \|= MLO_PAUSE_TX;
3417		-
3418		- return 1;
3419	3891	}
3420	3892
3421		-static void mvneta_mac_an_restart(struct net_device *ndev)
	3893	+static void mvneta_mac_an_restart(struct phylink_config *config)
3422	3894	{
	3895	+ struct net_device *ndev = to_net_dev(config->dev);
3423	3896	struct mvneta_port *pp = netdev_priv(ndev);
3424	3897	u32 gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
3425	3898
..	..	@@ -3429,28 +3902,27 @@
3429	3902	gmac_an & ~MVNETA_GMAC_INBAND_RESTART_AN);
3430	3903	}
3431	3904
3432		-static void mvneta_mac_config(struct net_device *ndev, unsigned int mode,
3433		- const struct phylink_link_state *state)
	3905	+static void mvneta_mac_config(struct phylink_config *config, unsigned int mode,
	3906	+ const struct phylink_link_state *state)
3434	3907	{
	3908	+ struct net_device *ndev = to_net_dev(config->dev);
3435	3909	struct mvneta_port *pp = netdev_priv(ndev);
3436	3910	u32 new_ctrl0, gmac_ctrl0 = mvreg_read(pp, MVNETA_GMAC_CTRL_0);
3437	3911	u32 new_ctrl2, gmac_ctrl2 = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
	3912	+ u32 new_ctrl4, gmac_ctrl4 = mvreg_read(pp, MVNETA_GMAC_CTRL_4);
3438	3913	u32 new_clk, gmac_clk = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER);
3439	3914	u32 new_an, gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
3440	3915
3441	3916	new_ctrl0 = gmac_ctrl0 & ~MVNETA_GMAC0_PORT_1000BASE_X;
3442	3917	new_ctrl2 = gmac_ctrl2 & ~(MVNETA_GMAC2_INBAND_AN_ENABLE \|
3443	3918	MVNETA_GMAC2_PORT_RESET);
	3919	+ new_ctrl4 = gmac_ctrl4 & ~(MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE);
3444	3920	new_clk = gmac_clk & ~MVNETA_GMAC_1MS_CLOCK_ENABLE;
3445	3921	new_an = gmac_an & ~(MVNETA_GMAC_INBAND_AN_ENABLE \|
3446	3922	MVNETA_GMAC_INBAND_RESTART_AN \|
3447		- MVNETA_GMAC_CONFIG_MII_SPEED \|
3448		- MVNETA_GMAC_CONFIG_GMII_SPEED \|
3449	3923	MVNETA_GMAC_AN_SPEED_EN \|
3450	3924	MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL \|
3451		- MVNETA_GMAC_CONFIG_FLOW_CTRL \|
3452	3925	MVNETA_GMAC_AN_FLOW_CTRL_EN \|
3453		- MVNETA_GMAC_CONFIG_FULL_DUPLEX \|
3454	3926	MVNETA_GMAC_AN_DUPLEX_EN);
3455	3927
3456	3928	/* Even though it might look weird, when we're configured in
..	..	@@ -3465,24 +3937,20 @@
3465	3937
3466	3938	if (phylink_test(state->advertising, Pause))
3467	3939	new_an \|= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL;
3468		- if (state->pause & MLO_PAUSE_TXRX_MASK)
3469		- new_an \|= MVNETA_GMAC_CONFIG_FLOW_CTRL;
3470	3940
3471	3941	if (!phylink_autoneg_inband(mode)) {
3472		- /* Phy or fixed speed */
3473		- if (state->duplex)
3474		- new_an \|= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
3475		-
3476		- if (state->speed == SPEED_1000)
3477		- new_an \|= MVNETA_GMAC_CONFIG_GMII_SPEED;
3478		- else if (state->speed == SPEED_100)
3479		- new_an \|= MVNETA_GMAC_CONFIG_MII_SPEED;
	3942	+ /* Phy or fixed speed - nothing to do, leave the
	3943	+ * configured speed, duplex and flow control as-is.
	3944	+ */
3480	3945	} else if (state->interface == PHY_INTERFACE_MODE_SGMII) {
3481	3946	/* SGMII mode receives the state from the PHY */
3482	3947	new_ctrl2 \|= MVNETA_GMAC2_INBAND_AN_ENABLE;
3483	3948	new_clk \|= MVNETA_GMAC_1MS_CLOCK_ENABLE;
3484	3949	new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN \|
3485		- MVNETA_GMAC_FORCE_LINK_PASS)) \|
	3950	+ MVNETA_GMAC_FORCE_LINK_PASS \|
	3951	+ MVNETA_GMAC_CONFIG_MII_SPEED \|
	3952	+ MVNETA_GMAC_CONFIG_GMII_SPEED \|
	3953	+ MVNETA_GMAC_CONFIG_FULL_DUPLEX)) \|
3486	3954	MVNETA_GMAC_INBAND_AN_ENABLE \|
3487	3955	MVNETA_GMAC_AN_SPEED_EN \|
3488	3956	MVNETA_GMAC_AN_DUPLEX_EN;
..	..	@@ -3491,7 +3959,8 @@
3491	3959	new_ctrl0 \|= MVNETA_GMAC0_PORT_1000BASE_X;
3492	3960	new_clk \|= MVNETA_GMAC_1MS_CLOCK_ENABLE;
3493	3961	new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN \|
3494		- MVNETA_GMAC_FORCE_LINK_PASS)) \|
	3962	+ MVNETA_GMAC_FORCE_LINK_PASS \|
	3963	+ MVNETA_GMAC_CONFIG_MII_SPEED)) \|
3495	3964	MVNETA_GMAC_INBAND_AN_ENABLE \|
3496	3965	MVNETA_GMAC_CONFIG_GMII_SPEED \|
3497	3966	/* The MAC only supports FD mode */
..	..	@@ -3512,10 +3981,25 @@
3512	3981	MVNETA_GMAC_FORCE_LINK_DOWN);
3513	3982	}
3514	3983
	3984	+
	3985	+ /* When at 2.5G, the link partner can send frames with shortened
	3986	+ * preambles.
	3987	+ */
	3988	+ if (state->interface == PHY_INTERFACE_MODE_2500BASEX)
	3989	+ new_ctrl4 \|= MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE;
	3990	+
	3991	+ if (pp->phy_interface != state->interface) {
	3992	+ if (pp->comphy)
	3993	+ WARN_ON(phy_power_off(pp->comphy));
	3994	+ WARN_ON(mvneta_config_interface(pp, state->interface));
	3995	+ }
	3996	+
3515	3997	if (new_ctrl0 != gmac_ctrl0)
3516	3998	mvreg_write(pp, MVNETA_GMAC_CTRL_0, new_ctrl0);
3517	3999	if (new_ctrl2 != gmac_ctrl2)
3518	4000	mvreg_write(pp, MVNETA_GMAC_CTRL_2, new_ctrl2);
	4001	+ if (new_ctrl4 != gmac_ctrl4)
	4002	+ mvreg_write(pp, MVNETA_GMAC_CTRL_4, new_ctrl4);
3519	4003	if (new_clk != gmac_clk)
3520	4004	mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, new_clk);
3521	4005	if (new_an != gmac_an)
..	..	@@ -3540,9 +4024,10 @@
3540	4024	mvreg_write(pp, MVNETA_LPI_CTRL_1, lpi_ctl1);
3541	4025	}
3542	4026
3543		-static void mvneta_mac_link_down(struct net_device *ndev, unsigned int mode,
3544		- phy_interface_t interface)
	4027	+static void mvneta_mac_link_down(struct phylink_config *config,
	4028	+ unsigned int mode, phy_interface_t interface)
3545	4029	{
	4030	+ struct net_device *ndev = to_net_dev(config->dev);
3546	4031	struct mvneta_port *pp = netdev_priv(ndev);
3547	4032	u32 val;
3548	4033
..	..	@@ -3559,17 +4044,48 @@
3559	4044	mvneta_set_eee(pp, false);
3560	4045	}
3561	4046
3562		-static void mvneta_mac_link_up(struct net_device *ndev, unsigned int mode,
3563		- phy_interface_t interface,
3564		- struct phy_device *phy)
	4047	+static void mvneta_mac_link_up(struct phylink_config *config,
	4048	+ struct phy_device *phy,
	4049	+ unsigned int mode, phy_interface_t interface,
	4050	+ int speed, int duplex,
	4051	+ bool tx_pause, bool rx_pause)
3565	4052	{
	4053	+ struct net_device *ndev = to_net_dev(config->dev);
3566	4054	struct mvneta_port *pp = netdev_priv(ndev);
3567	4055	u32 val;
3568	4056
3569	4057	if (!phylink_autoneg_inband(mode)) {
3570	4058	val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
3571		- val &= ~MVNETA_GMAC_FORCE_LINK_DOWN;
	4059	+ val &= ~(MVNETA_GMAC_FORCE_LINK_DOWN \|
	4060	+ MVNETA_GMAC_CONFIG_MII_SPEED \|
	4061	+ MVNETA_GMAC_CONFIG_GMII_SPEED \|
	4062	+ MVNETA_GMAC_CONFIG_FLOW_CTRL \|
	4063	+ MVNETA_GMAC_CONFIG_FULL_DUPLEX);
3572	4064	val \|= MVNETA_GMAC_FORCE_LINK_PASS;
	4065	+
	4066	+ if (speed == SPEED_1000 \|\| speed == SPEED_2500)
	4067	+ val \|= MVNETA_GMAC_CONFIG_GMII_SPEED;
	4068	+ else if (speed == SPEED_100)
	4069	+ val \|= MVNETA_GMAC_CONFIG_MII_SPEED;
	4070	+
	4071	+ if (duplex == DUPLEX_FULL)
	4072	+ val \|= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
	4073	+
	4074	+ if (tx_pause \|\| rx_pause)
	4075	+ val \|= MVNETA_GMAC_CONFIG_FLOW_CTRL;
	4076	+
	4077	+ mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
	4078	+ } else {
	4079	+ /* When inband doesn't cover flow control or flow control is
	4080	+ * disabled, we need to manually configure it. This bit will
	4081	+ * only have effect if MVNETA_GMAC_AN_FLOW_CTRL_EN is unset.
	4082	+ */
	4083	+ val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
	4084	+ val &= ~MVNETA_GMAC_CONFIG_FLOW_CTRL;
	4085	+
	4086	+ if (tx_pause \|\| rx_pause)
	4087	+ val \|= MVNETA_GMAC_CONFIG_FLOW_CTRL;
	4088	+
3573	4089	mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
3574	4090	}
3575	4091
..	..	@@ -3583,7 +4099,7 @@
3583	4099
3584	4100	static const struct phylink_mac_ops mvneta_phylink_ops = {
3585	4101	.validate = mvneta_validate,
3586		- .mac_link_state = mvneta_mac_link_state,
	4102	+ .mac_pcs_get_state = mvneta_mac_pcs_get_state,
3587	4103	.mac_an_restart = mvneta_mac_an_restart,
3588	4104	.mac_config = mvneta_mac_config,
3589	4105	.mac_link_down = mvneta_mac_link_down,
..	..	@@ -3600,6 +4116,10 @@
3600	4116
3601	4117	phylink_ethtool_get_wol(pp->phylink, &wol);
3602	4118	device_set_wakeup_capable(&pp->dev->dev, !!wol.supported);
	4119	+
	4120	+ /* PHY WoL may be enabled but device wakeup disabled */
	4121	+ if (wol.supported)
	4122	+ device_set_wakeup_enable(&pp->dev->dev, !!wol.wolopts);
3603	4123
3604	4124	return err;
3605	4125	}
..	..	@@ -3620,7 +4140,7 @@
3620	4140	/* Use the cpu associated to the rxq when it is online, in all
3621	4141	* the other cases, use the cpu 0 which can't be offline.
3622	4142	*/
3623		- if (cpu_online(pp->rxq_def))
	4143	+ if (pp->rxq_def < nr_cpu_ids && cpu_online(pp->rxq_def))
3624	4144	elected_cpu = pp->rxq_def;
3625	4145
3626	4146	max_cpu = num_present_cpus();
..	..	@@ -3813,9 +4333,6 @@
3813	4333	goto err_free_online_hp;
3814	4334	}
3815	4335
3816		- /* In default link is down */
3817		- netif_carrier_off(pp->dev);
3818		-
3819	4336	ret = mvneta_mdio_probe(pp);
3820	4337	if (ret < 0) {
3821	4338	netdev_err(dev, "cannot probe MDIO bus\n");
..	..	@@ -3889,6 +4406,48 @@
3889	4406	struct mvneta_port *pp = netdev_priv(dev);
3890	4407
3891	4408	return phylink_mii_ioctl(pp->phylink, ifr, cmd);
	4409	+}
	4410	+
	4411	+static int mvneta_xdp_setup(struct net_device dev, struct bpf_prog prog,
	4412	+ struct netlink_ext_ack *extack)
	4413	+{
	4414	+ bool need_update, running = netif_running(dev);
	4415	+ struct mvneta_port *pp = netdev_priv(dev);
	4416	+ struct bpf_prog *old_prog;
	4417	+
	4418	+ if (prog && dev->mtu > MVNETA_MAX_RX_BUF_SIZE) {
	4419	+ NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP");
	4420	+ return -EOPNOTSUPP;
	4421	+ }
	4422	+
	4423	+ if (pp->bm_priv) {
	4424	+ NL_SET_ERR_MSG_MOD(extack,
	4425	+ "Hardware Buffer Management not supported on XDP");
	4426	+ return -EOPNOTSUPP;
	4427	+ }
	4428	+
	4429	+ need_update = !!pp->xdp_prog != !!prog;
	4430	+ if (running && need_update)
	4431	+ mvneta_stop(dev);
	4432	+
	4433	+ old_prog = xchg(&pp->xdp_prog, prog);
	4434	+ if (old_prog)
	4435	+ bpf_prog_put(old_prog);
	4436	+
	4437	+ if (running && need_update)
	4438	+ return mvneta_open(dev);
	4439	+
	4440	+ return 0;
	4441	+}
	4442	+
	4443	+static int mvneta_xdp(struct net_device dev, struct netdev_bpf xdp)
	4444	+{
	4445	+ switch (xdp->command) {
	4446	+ case XDP_SETUP_PROG:
	4447	+ return mvneta_xdp_setup(dev, xdp->prog, xdp->extack);
	4448	+ default:
	4449	+ return -EINVAL;
	4450	+ }
3892	4451	}
3893	4452
3894	4453	/* Ethtool methods */
..	..	@@ -4037,45 +4596,112 @@
4037	4596	}
4038	4597	}
4039	4598
	4599	+static void
	4600	+mvneta_ethtool_update_pcpu_stats(struct mvneta_port *pp,
	4601	+ struct mvneta_ethtool_stats *es)
	4602	+{
	4603	+ unsigned int start;
	4604	+ int cpu;
	4605	+
	4606	+ for_each_possible_cpu(cpu) {
	4607	+ struct mvneta_pcpu_stats *stats;
	4608	+ u64 skb_alloc_error;
	4609	+ u64 refill_error;
	4610	+ u64 xdp_redirect;
	4611	+ u64 xdp_xmit_err;
	4612	+ u64 xdp_tx_err;
	4613	+ u64 xdp_pass;
	4614	+ u64 xdp_drop;
	4615	+ u64 xdp_xmit;
	4616	+ u64 xdp_tx;
	4617	+
	4618	+ stats = per_cpu_ptr(pp->stats, cpu);
	4619	+ do {
	4620	+ start = u64_stats_fetch_begin_irq(&stats->syncp);
	4621	+ skb_alloc_error = stats->es.skb_alloc_error;
	4622	+ refill_error = stats->es.refill_error;
	4623	+ xdp_redirect = stats->es.ps.xdp_redirect;
	4624	+ xdp_pass = stats->es.ps.xdp_pass;
	4625	+ xdp_drop = stats->es.ps.xdp_drop;
	4626	+ xdp_xmit = stats->es.ps.xdp_xmit;
	4627	+ xdp_xmit_err = stats->es.ps.xdp_xmit_err;
	4628	+ xdp_tx = stats->es.ps.xdp_tx;
	4629	+ xdp_tx_err = stats->es.ps.xdp_tx_err;
	4630	+ } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
	4631	+
	4632	+ es->skb_alloc_error += skb_alloc_error;
	4633	+ es->refill_error += refill_error;
	4634	+ es->ps.xdp_redirect += xdp_redirect;
	4635	+ es->ps.xdp_pass += xdp_pass;
	4636	+ es->ps.xdp_drop += xdp_drop;
	4637	+ es->ps.xdp_xmit += xdp_xmit;
	4638	+ es->ps.xdp_xmit_err += xdp_xmit_err;
	4639	+ es->ps.xdp_tx += xdp_tx;
	4640	+ es->ps.xdp_tx_err += xdp_tx_err;
	4641	+ }
	4642	+}
	4643	+
4040	4644	static void mvneta_ethtool_update_stats(struct mvneta_port *pp)
4041	4645	{
	4646	+ struct mvneta_ethtool_stats stats = {};
4042	4647	const struct mvneta_statistic *s;
4043	4648	void __iomem *base = pp->base;
4044	4649	u32 high, low;
4045	4650	u64 val;
4046	4651	int i;
4047	4652
	4653	+ mvneta_ethtool_update_pcpu_stats(pp, &stats);
4048	4654	for (i = 0, s = mvneta_statistics;
4049	4655	s < mvneta_statistics + ARRAY_SIZE(mvneta_statistics);
4050	4656	s++, i++) {
4051		- val = 0;
4052		-
4053	4657	switch (s->type) {
4054	4658	case T_REG_32:
4055	4659	val = readl_relaxed(base + s->offset);
	4660	+ pp->ethtool_stats[i] += val;
4056	4661	break;
4057	4662	case T_REG_64:
4058	4663	/* Docs say to read low 32-bit then high */
4059	4664	low = readl_relaxed(base + s->offset);
4060	4665	high = readl_relaxed(base + s->offset + 4);
4061	4666	val = (u64)high << 32 \| low;
	4667	+ pp->ethtool_stats[i] += val;
4062	4668	break;
4063	4669	case T_SW:
4064	4670	switch (s->offset) {
4065	4671	case ETHTOOL_STAT_EEE_WAKEUP:
4066	4672	val = phylink_get_eee_err(pp->phylink);
	4673	+ pp->ethtool_stats[i] += val;
4067	4674	break;
4068	4675	case ETHTOOL_STAT_SKB_ALLOC_ERR:
4069		- val = pp->rxqs[0].skb_alloc_err;
	4676	+ pp->ethtool_stats[i] = stats.skb_alloc_error;
4070	4677	break;
4071	4678	case ETHTOOL_STAT_REFILL_ERR:
4072		- val = pp->rxqs[0].refill_err;
	4679	+ pp->ethtool_stats[i] = stats.refill_error;
	4680	+ break;
	4681	+ case ETHTOOL_XDP_REDIRECT:
	4682	+ pp->ethtool_stats[i] = stats.ps.xdp_redirect;
	4683	+ break;
	4684	+ case ETHTOOL_XDP_PASS:
	4685	+ pp->ethtool_stats[i] = stats.ps.xdp_pass;
	4686	+ break;
	4687	+ case ETHTOOL_XDP_DROP:
	4688	+ pp->ethtool_stats[i] = stats.ps.xdp_drop;
	4689	+ break;
	4690	+ case ETHTOOL_XDP_TX:
	4691	+ pp->ethtool_stats[i] = stats.ps.xdp_tx;
	4692	+ break;
	4693	+ case ETHTOOL_XDP_TX_ERR:
	4694	+ pp->ethtool_stats[i] = stats.ps.xdp_tx_err;
	4695	+ break;
	4696	+ case ETHTOOL_XDP_XMIT:
	4697	+ pp->ethtool_stats[i] = stats.ps.xdp_xmit;
	4698	+ break;
	4699	+ case ETHTOOL_XDP_XMIT_ERR:
	4700	+ pp->ethtool_stats[i] = stats.ps.xdp_xmit_err;
4073	4701	break;
4074	4702	}
4075	4703	break;
4076	4704	}
4077		-
4078		- pp->ethtool_stats[i] += val;
4079	4705	}
4080	4706	}
4081	4707
..	..	@@ -4261,8 +4887,7 @@
4261	4887
4262	4888	/* The Armada 37x documents do not give limits for this other than
4263	4889	* it being an 8-bit register. */
4264		- if (eee->tx_lpi_enabled &&
4265		- (eee->tx_lpi_timer < 0 \|\| eee->tx_lpi_timer > 255))
	4890	+ if (eee->tx_lpi_enabled && eee->tx_lpi_timer > 255)
4266	4891	return -EINVAL;
4267	4892
4268	4893	lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0);
..	..	@@ -4288,9 +4913,13 @@
4288	4913	.ndo_fix_features = mvneta_fix_features,
4289	4914	.ndo_get_stats64 = mvneta_get_stats64,
4290	4915	.ndo_do_ioctl = mvneta_ioctl,
	4916	+ .ndo_bpf = mvneta_xdp,
	4917	+ .ndo_xdp_xmit = mvneta_xdp_xmit,
4291	4918	};
4292	4919
4293	4920	static const struct ethtool_ops mvneta_eth_tool_ops = {
	4921	+ .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS \|
	4922	+ ETHTOOL_COALESCE_MAX_FRAMES,
4294	4923	.nway_reset = mvneta_ethtool_nway_reset,
4295	4924	.get_link = ethtool_op_get_link,
4296	4925	.set_coalesce = mvneta_ethtool_set_coalesce,
..	..	@@ -4415,12 +5044,10 @@
4415	5044	/* MAC Cause register should be cleared */
4416	5045	mvreg_write(pp, MVNETA_UNIT_INTR_CAUSE, 0);
4417	5046
4418		- if (phy_mode == PHY_INTERFACE_MODE_QSGMII)
4419		- mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_QSGMII_SERDES_PROTO);
4420		- else if (phy_mode == PHY_INTERFACE_MODE_SGMII \|\|
4421		- phy_mode == PHY_INTERFACE_MODE_1000BASEX)
4422		- mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_SGMII_SERDES_PROTO);
4423		- else if (!phy_interface_mode_is_rgmii(phy_mode))
	5047	+ if (phy_mode != PHY_INTERFACE_MODE_QSGMII &&
	5048	+ phy_mode != PHY_INTERFACE_MODE_SGMII &&
	5049	+ !phy_interface_mode_is_8023z(phy_mode) &&
	5050	+ !phy_interface_mode_is_rgmii(phy_mode))
4424	5051	return -EINVAL;
4425	5052
4426	5053	return 0;
..	..	@@ -4429,39 +5056,51 @@
4429	5056	/* Device initialization routine */
4430	5057	static int mvneta_probe(struct platform_device *pdev)
4431	5058	{
4432		- struct resource *res;
4433	5059	struct device_node *dn = pdev->dev.of_node;
4434	5060	struct device_node *bm_node;
4435	5061	struct mvneta_port *pp;
4436	5062	struct net_device *dev;
4437	5063	struct phylink *phylink;
	5064	+ struct phy *comphy;
4438	5065	const char *dt_mac_addr;
4439	5066	char hw_mac_addr[ETH_ALEN];
	5067	+ phy_interface_t phy_mode;
4440	5068	const char *mac_from;
4441	5069	int tx_csum_limit;
4442		- int phy_mode;
4443	5070	int err;
4444	5071	int cpu;
4445	5072
4446		- dev = alloc_etherdev_mqs(sizeof(struct mvneta_port), txq_number, rxq_number);
	5073	+ dev = devm_alloc_etherdev_mqs(&pdev->dev, sizeof(struct mvneta_port),
	5074	+ txq_number, rxq_number);
4447	5075	if (!dev)
4448	5076	return -ENOMEM;
4449	5077
4450	5078	dev->irq = irq_of_parse_and_map(dn, 0);
4451		- if (dev->irq == 0) {
4452		- err = -EINVAL;
4453		- goto err_free_netdev;
4454		- }
	5079	+ if (dev->irq == 0)
	5080	+ return -EINVAL;
4455	5081
4456		- phy_mode = of_get_phy_mode(dn);
4457		- if (phy_mode < 0) {
	5082	+ err = of_get_phy_mode(dn, &phy_mode);
	5083	+ if (err) {
4458	5084	dev_err(&pdev->dev, "incorrect phy-mode\n");
4459		- err = -EINVAL;
4460	5085	goto err_free_irq;
4461	5086	}
4462	5087
4463		- phylink = phylink_create(dev, pdev->dev.fwnode, phy_mode,
4464		- &mvneta_phylink_ops);
	5088	+ comphy = devm_of_phy_get(&pdev->dev, dn, NULL);
	5089	+ if (comphy == ERR_PTR(-EPROBE_DEFER)) {
	5090	+ err = -EPROBE_DEFER;
	5091	+ goto err_free_irq;
	5092	+ } else if (IS_ERR(comphy)) {
	5093	+ comphy = NULL;
	5094	+ }
	5095	+
	5096	+ pp = netdev_priv(dev);
	5097	+ spin_lock_init(&pp->lock);
	5098	+
	5099	+ pp->phylink_config.dev = &dev->dev;
	5100	+ pp->phylink_config.type = PHYLINK_NETDEV;
	5101	+
	5102	+ phylink = phylink_create(&pp->phylink_config, pdev->dev.fwnode,
	5103	+ phy_mode, &mvneta_phylink_ops);
4465	5104	if (IS_ERR(phylink)) {
4466	5105	err = PTR_ERR(phylink);
4467	5106	goto err_free_irq;
..	..	@@ -4473,9 +5112,8 @@
4473	5112
4474	5113	dev->ethtool_ops = &mvneta_eth_tool_ops;
4475	5114
4476		- pp = netdev_priv(dev);
4477		- spin_lock_init(&pp->lock);
4478	5115	pp->phylink = phylink;
	5116	+ pp->comphy = comphy;
4479	5117	pp->phy_interface = phy_mode;
4480	5118	pp->dn = dn;
4481	5119
..	..	@@ -4500,8 +5138,7 @@
4500	5138	if (!IS_ERR(pp->clk_bus))
4501	5139	clk_prepare_enable(pp->clk_bus);
4502	5140
4503		- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4504		- pp->base = devm_ioremap_resource(&pdev->dev, res);
	5141	+ pp->base = devm_platform_ioremap_resource(pdev, 0);
4505	5142	if (IS_ERR(pp->base)) {
4506	5143	err = PTR_ERR(pp->base);
4507	5144	goto err_clk;
..	..	@@ -4522,9 +5159,9 @@
4522	5159	}
4523	5160
4524	5161	dt_mac_addr = of_get_mac_address(dn);
4525		- if (dt_mac_addr) {
	5162	+ if (!IS_ERR(dt_mac_addr)) {
4526	5163	mac_from = "device tree";
4527		- memcpy(dev->dev_addr, dt_mac_addr, ETH_ALEN);
	5164	+ ether_addr_copy(dev->dev_addr, dt_mac_addr);
4528	5165	} else {
4529	5166	mvneta_get_mac_addr(pp, hw_mac_addr);
4530	5167	if (is_valid_ether_addr(hw_mac_addr)) {
..	..	@@ -4567,7 +5204,6 @@
4567	5204	SET_NETDEV_DEV(dev, &pdev->dev);
4568	5205
4569	5206	pp->id = global_port_id++;
4570		- pp->rx_offset_correction = 0; /* not relevant for SW BM */
4571	5207
4572	5208	/* Obtain access to BM resources if enabled and already initialized */
4573	5209	bm_node = of_parse_phandle(dn, "buffer-manager", 0);
..	..	@@ -4592,11 +5228,15 @@
4592	5228	}
4593	5229	of_node_put(bm_node);
4594	5230
	5231	+ /* sw buffer management */
	5232	+ if (!pp->bm_priv)
	5233	+ pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
	5234	+
4595	5235	err = mvneta_init(&pdev->dev, pp);
4596	5236	if (err < 0)
4597	5237	goto err_netdev;
4598	5238
4599		- err = mvneta_port_power_up(pp, phy_mode);
	5239	+ err = mvneta_port_power_up(pp, pp->phy_interface);
4600	5240	if (err < 0) {
4601	5241	dev_err(&pdev->dev, "can't power up port\n");
4602	5242	goto err_netdev;
..	..	@@ -4618,7 +5258,8 @@
4618	5258	}
4619	5259	}
4620	5260
4621		- dev->features = NETIF_F_SG \| NETIF_F_IP_CSUM \| NETIF_F_IPV6_CSUM \| NETIF_F_TSO;
	5261	+ dev->features = NETIF_F_SG \| NETIF_F_IP_CSUM \| NETIF_F_IPV6_CSUM \|
	5262	+ NETIF_F_TSO \| NETIF_F_RXCSUM;
4622	5263	dev->hw_features \|= dev->features;
4623	5264	dev->vlan_features \|= dev->features;
4624	5265	dev->priv_flags \|= IFF_LIVE_ADDR_CHANGE;
..	..	@@ -4660,8 +5301,6 @@
4660	5301	phylink_destroy(pp->phylink);
4661	5302	err_free_irq:
4662	5303	irq_dispose_mapping(dev->irq);
4663		-err_free_netdev:
4664		- free_netdev(dev);
4665	5304	return err;
4666	5305	}
4667	5306
..	..	@@ -4678,7 +5317,6 @@
4678	5317	free_percpu(pp->stats);
4679	5318	irq_dispose_mapping(dev->irq);
4680	5319	phylink_destroy(pp->phylink);
4681		- free_netdev(dev);
4682	5320
4683	5321	if (pp->bm_priv) {
4684	5322	mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
..	..	@@ -4751,6 +5389,7 @@
4751	5389	err = mvneta_bm_port_init(pdev, pp);
4752	5390	if (err < 0) {
4753	5391	dev_info(&pdev->dev, "use SW buffer management\n");
	5392	+ pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
4754	5393	pp->bm_priv = NULL;
4755	5394	}
4756	5395	}
..	..	@@ -4823,7 +5462,7 @@
4823	5462	{
4824	5463	int ret;
4825	5464
4826		- ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/mvmeta:online",
	5465	+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/mvneta:online",
4827	5466	mvneta_cpu_online,
4828	5467	mvneta_cpu_down_prepare);
4829	5468	if (ret < 0)