~hc/RK356X_SDK_RELEASE.git

..	..	@@ -36,7 +36,6 @@
36	36	#include <linux/etherdevice.h>
37	37	#include <linux/timecounter.h>
38	38	#include <linux/net_tstamp.h>
39		-#include <linux/ptp_clock_kernel.h>
40	39	#include <linux/crash_dump.h>
41	40	#include <linux/mlx5/driver.h>
42	41	#include <linux/mlx5/qp.h>
..	..	@@ -46,13 +45,17 @@
46	45	#include <linux/mlx5/transobj.h>
47	46	#include <linux/mlx5/fs.h>
48	47	#include <linux/rhashtable.h>
	48	+#include <net/udp_tunnel.h>
49	49	#include <net/switchdev.h>
50	50	#include <net/xdp.h>
51		-#include <linux/net_dim.h>
	51	+#include <linux/dim.h>
	52	+#include <linux/bits.h>
52	53	#include "wq.h"
53	54	#include "mlx5_core.h"
54	55	#include "en_stats.h"
	56	+#include "en/dcbnl.h"
55	57	#include "en/fs.h"
	58	+#include "lib/hv_vhca.h"
56	59
57	60	extern const struct net_device_ops mlx5e_netdev_ops;
58	61	struct page_pool;
..	..	@@ -67,35 +70,38 @@
67	70	#define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu))
68	71	#define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu))
69	72
70		-#define MLX5E_MAX_PRIORITY 8
71		-#define MLX5E_MAX_DSCP 64
72	73	#define MLX5E_MAX_NUM_TC 8
73	74
74	75	#define MLX5_RX_HEADROOM NET_SKB_PAD
75	76	#define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
76	77	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
77	78
	79	+#define MLX5E_RX_MAX_HEAD (256)
	80	+
78	81	#define MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev) \
79	82	(6 + MLX5_CAP_GEN(mdev, cache_line_128byte)) /* HW restriction */
80	83	#define MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, req) \
81	84	max_t(u32, MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev), req)
82		-#define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, 6)
83		-#define MLX5_MPWRQ_CQE_CMPRS_LOG_STRIDE_SZ(mdev) MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, 8)
84		-#define MLX5E_MPWQE_STRIDE_SZ(mdev, cqe_cmprs) \
85		- (cqe_cmprs ? MLX5_MPWRQ_CQE_CMPRS_LOG_STRIDE_SZ(mdev) : \
86		- MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev))
	85	+#define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \
	86	+ MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD))
87	87
88	88	#define MLX5_MPWRQ_LOG_WQE_SZ 18
89	89	#define MLX5_MPWRQ_WQE_PAGE_ORDER (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \
90	90	MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT : 0)
91	91	#define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER)
92	92
93		-#define MLX5_MTT_OCTW(npages) (ALIGN(npages, 8) / 2)
94		-#define MLX5E_REQUIRED_WQE_MTTS (ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8))
95		-#define MLX5E_LOG_ALIGNED_MPWQE_PPW (ilog2(MLX5E_REQUIRED_WQE_MTTS))
	93	+#define MLX5_ALIGN_MTTS(mtts) (ALIGN(mtts, 8))
	94	+#define MLX5_ALIGNED_MTTS_OCTW(mtts) ((mtts) / 2)
	95	+#define MLX5_MTT_OCTW(mtts) (MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts)))
	96	+/* Add another page to MLX5E_REQUIRED_WQE_MTTS as a buffer between
	97	+ * WQEs, This page will absorb write overflow by the hardware, when
	98	+ * receiving packets larger than MTU. These oversize packets are
	99	+ * dropped by the driver at a later stage.
	100	+ */
	101	+#define MLX5E_REQUIRED_WQE_MTTS (MLX5_ALIGN_MTTS(MLX5_MPWRQ_PAGES_PER_WQE + 1))
96	102	#define MLX5E_REQUIRED_MTTS(wqes) (wqes * MLX5E_REQUIRED_WQE_MTTS)
97	103	#define MLX5E_MAX_RQ_NUM_MTTS \
98		- ((1 << 16) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */
	104	+ (ALIGN_DOWN(U16_MAX, 4) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */
99	105	#define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024))
100	106	#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW \
101	107	(ilog2(MLX5E_MAX_RQ_NUM_MTTS / MLX5E_REQUIRED_WQE_MTTS))
..	..	@@ -118,8 +124,6 @@
118	124
119	125	#define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2
120	126
121		-#define MLX5E_RX_MAX_HEAD (256)
122		-
123	127	#define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024)
124	128	#define MLX5E_DEFAULT_LRO_TIMEOUT 32
125	129	#define MLX5E_LRO_TIMEOUT_ARR_SIZE 4
..	..	@@ -136,9 +140,10 @@
136	140	#define MLX5E_LOG_INDIR_RQT_SIZE 0x7
137	141	#define MLX5E_INDIR_RQT_SIZE BIT(MLX5E_LOG_INDIR_RQT_SIZE)
138	142	#define MLX5E_MIN_NUM_CHANNELS 0x1
139		-#define MLX5E_MAX_NUM_CHANNELS (MLX5E_INDIR_RQT_SIZE >> 1)
	143	+#define MLX5E_MAX_NUM_CHANNELS MLX5E_INDIR_RQT_SIZE
140	144	#define MLX5E_MAX_NUM_SQS (MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC)
141	145	#define MLX5E_TX_CQ_POLL_BUDGET 128
	146	+#define MLX5E_TX_XSK_POLL_BUDGET 64
142	147	#define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */
143	148
144	149	#define MLX5E_UMR_WQE_INLINE_SZ \
..	..	@@ -147,9 +152,6 @@
147	152	MLX5_UMR_MTT_ALIGNMENT))
148	153	#define MLX5E_UMR_WQEBBS \
149	154	(DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB))
150		-#define MLX5E_ICOSQ_MAX_WQEBBS MLX5E_UMR_WQEBBS
151		-
152		-#define MLX5E_NUM_MAIN_GROUPS 9
153	155
154	156	#define MLX5E_MSG_LEVEL NETIF_MSG_LINK
155	157
..	..	@@ -160,6 +162,19 @@
160	162	##__VA_ARGS__); \
161	163	} while (0)
162	164
	165	+enum mlx5e_rq_group {
	166	+ MLX5E_RQ_GROUP_REGULAR,
	167	+ MLX5E_RQ_GROUP_XSK,
	168	+#define MLX5E_NUM_RQ_GROUPS(g) (1 + MLX5E_RQ_GROUP_##g)
	169	+};
	170	+
	171	+static inline u8 mlx5e_get_num_lag_ports(struct mlx5_core_dev *mdev)
	172	+{
	173	+ if (mlx5_lag_is_lacp_owner(mdev))
	174	+ return 1;
	175	+
	176	+ return clamp_t(u8, MLX5_CAP_GEN(mdev, num_lag_ports), 1, MLX5_MAX_PORTS);
	177	+}
163	178
164	179	static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
165	180	{
..	..	@@ -173,12 +188,12 @@
173	188	}
174	189	}
175	190
	191	+/* Use this function to get max num channels (rxqs/txqs) only to create netdev */
176	192	static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
177	193	{
178	194	return is_kdump_kernel() ?
179	195	MLX5E_MIN_NUM_CHANNELS :
180		- min_t(int, mdev->priv.eq_table.num_comp_vectors,
181		- MLX5E_MAX_NUM_CHANNELS);
	196	+ min_t(int, mlx5_comp_vectors_count(mdev), MLX5E_MAX_NUM_CHANNELS);
182	197	}
183	198
184	199	struct mlx5e_tx_wqe {
..	..	@@ -189,7 +204,7 @@
189	204
190	205	struct mlx5e_rx_wqe_ll {
191	206	struct mlx5_wqe_srq_next_seg next;
192		- struct mlx5_wqe_data_seg data[0];
	207	+ struct mlx5_wqe_data_seg data[];
193	208	};
194	209
195	210	struct mlx5e_rx_wqe_cyc {
..	..	@@ -205,35 +220,26 @@
205	220
206	221	extern const char mlx5e_self_tests[][ETH_GSTRING_LEN];
207	222
208		-static const char mlx5e_priv_flags[][ETH_GSTRING_LEN] = {
209		- "rx_cqe_moder",
210		- "tx_cqe_moder",
211		- "rx_cqe_compress",
212		- "rx_striding_rq",
213		- "rx_no_csum_complete",
214		-};
215		-
216	223	enum mlx5e_priv_flag {
217		- MLX5E_PFLAG_RX_CQE_BASED_MODER = (1 << 0),
218		- MLX5E_PFLAG_TX_CQE_BASED_MODER = (1 << 1),
219		- MLX5E_PFLAG_RX_CQE_COMPRESS = (1 << 2),
220		- MLX5E_PFLAG_RX_STRIDING_RQ = (1 << 3),
221		- MLX5E_PFLAG_RX_NO_CSUM_COMPLETE = (1 << 4),
	224	+ MLX5E_PFLAG_RX_CQE_BASED_MODER,
	225	+ MLX5E_PFLAG_TX_CQE_BASED_MODER,
	226	+ MLX5E_PFLAG_RX_CQE_COMPRESS,
	227	+ MLX5E_PFLAG_RX_STRIDING_RQ,
	228	+ MLX5E_PFLAG_RX_NO_CSUM_COMPLETE,
	229	+ MLX5E_PFLAG_XDP_TX_MPWQE,
	230	+ MLX5E_PFLAG_SKB_TX_MPWQE,
	231	+ MLX5E_NUM_PFLAGS, /* Keep last */
222	232	};
223	233
224	234	#define MLX5E_SET_PFLAG(params, pflag, enable) \
225	235	do { \
226	236	if (enable) \
227		- (params)->pflags \|= (pflag); \
	237	+ (params)->pflags \|= BIT(pflag); \
228	238	else \
229		- (params)->pflags &= ~(pflag); \
	239	+ (params)->pflags &= ~(BIT(pflag)); \
230	240	} while (0)
231	241
232		-#define MLX5E_GET_PFLAG(params, pflag) (!!((params)->pflags & (pflag)))
233		-
234		-#ifdef CONFIG_MLX5_CORE_EN_DCB
235		-#define MLX5E_MAX_BW_ALLOC 100 /* Max percentage of BW allocation */
236		-#endif
	242	+#define MLX5E_GET_PFLAG(params, pflag) (!!((params)->pflags & (BIT(pflag))))
237	243
238	244	struct mlx5e_params {
239	245	u8 log_sq_size;
..	..	@@ -242,14 +248,11 @@
242	248	u16 num_channels;
243	249	u8 num_tc;
244	250	bool rx_cqe_compress_def;
245		- struct net_dim_cq_moder rx_cq_moderation;
246		- struct net_dim_cq_moder tx_cq_moderation;
	251	+ bool tunneled_offload_en;
	252	+ struct dim_cq_moder rx_cq_moderation;
	253	+ struct dim_cq_moder tx_cq_moderation;
247	254	bool lro_en;
248		- u32 lro_wqe_sz;
249	255	u8 tx_min_inline_mode;
250		- u8 rss_hfunc;
251		- u8 toeplitz_hash_key[40];
252		- u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE];
253	256	bool vlan_strip_disable;
254	257	bool scatter_fcs_en;
255	258	bool rx_dim_enabled;
..	..	@@ -257,50 +260,19 @@
257	260	u32 lro_timeout;
258	261	u32 pflags;
259	262	struct bpf_prog *xdp_prog;
	263	+ struct mlx5e_xsk *xsk;
260	264	unsigned int sw_mtu;
261	265	int hard_mtu;
262	266	};
263	267
264		-#ifdef CONFIG_MLX5_CORE_EN_DCB
265		-struct mlx5e_cee_config {
266		- /* bw pct for priority group */
267		- u8 pg_bw_pct[CEE_DCBX_MAX_PGS];
268		- u8 prio_to_pg_map[CEE_DCBX_MAX_PRIO];
269		- bool pfc_setting[CEE_DCBX_MAX_PRIO];
270		- bool pfc_enable;
271		-};
272		-
273		-enum {
274		- MLX5_DCB_CHG_RESET,
275		- MLX5_DCB_NO_CHG,
276		- MLX5_DCB_CHG_NO_RESET,
277		-};
278		-
279		-struct mlx5e_dcbx {
280		- enum mlx5_dcbx_oper_mode mode;
281		- struct mlx5e_cee_config cee_cfg; /* pending configuration */
282		- u8 dscp_app_cnt;
283		-
284		- /* The only setting that cannot be read from FW */
285		- u8 tc_tsa[IEEE_8021QAZ_MAX_TCS];
286		- u8 cap;
287		-
288		- /* Buffer configuration */
289		- bool manual_buffer;
290		- u32 cable_len;
291		- u32 xoff;
292		-};
293		-
294		-struct mlx5e_dcbx_dp {
295		- u8 dscp2prio[MLX5E_MAX_DSCP];
296		- u8 trust_state;
297		-};
298		-#endif
299		-
300	268	enum {
301	269	MLX5E_RQ_STATE_ENABLED,
	270	+ MLX5E_RQ_STATE_RECOVERING,
302	271	MLX5E_RQ_STATE_AM,
303	272	MLX5E_RQ_STATE_NO_CSUM_COMPLETE,
	273	+ MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */
	274	+ MLX5E_RQ_STATE_FPGA_TLS, /* FPGA TLS enabled */
	275	+ MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX /* set when mini_cqe_resp_stride_index cap is used */
304	276	};
305	277
306	278	struct mlx5e_cq {
..	..	@@ -313,24 +285,19 @@
313	285	struct mlx5_core_cq mcq;
314	286	struct mlx5e_channel *channel;
315	287
316		- /* cqe decompression */
317		- struct mlx5_cqe64 title;
318		- struct mlx5_mini_cqe8 mini_arr[MLX5_MINI_CQE_ARRAY_SIZE];
319		- u8 mini_arr_idx;
320		- u16 decmprs_left;
321		- u16 decmprs_wqe_counter;
322		-
323	288	/* control */
324	289	struct mlx5_core_dev *mdev;
325	290	struct mlx5_wq_ctrl wq_ctrl;
326	291	} ____cacheline_aligned_in_smp;
327	292
328		-struct mlx5e_tx_wqe_info {
329		- struct sk_buff *skb;
330		- u32 num_bytes;
331		- u8 num_wqebbs;
332		- u8 num_dma;
333		-};
	293	+struct mlx5e_cq_decomp {
	294	+ /* cqe decompression */
	295	+ struct mlx5_cqe64 title;
	296	+ struct mlx5_mini_cqe8 mini_arr[MLX5_MINI_CQE_ARRAY_SIZE];
	297	+ u8 mini_arr_idx;
	298	+ u16 left;
	299	+ u16 wqe_counter;
	300	+} ____cacheline_aligned_in_smp;
334	301
335	302	enum mlx5e_dma_map_type {
336	303	MLX5E_DMA_MAP_SINGLE,
..	..	@@ -345,15 +312,22 @@
345	312
346	313	enum {
347	314	MLX5E_SQ_STATE_ENABLED,
	315	+ MLX5E_SQ_STATE_MPWQE,
348	316	MLX5E_SQ_STATE_RECOVERING,
349	317	MLX5E_SQ_STATE_IPSEC,
350	318	MLX5E_SQ_STATE_AM,
351	319	MLX5E_SQ_STATE_TLS,
352		- MLX5E_SQ_STATE_REDIRECT,
	320	+ MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE,
	321	+ MLX5E_SQ_STATE_PENDING_XSK_TX,
353	322	};
354	323
355		-struct mlx5e_sq_wqe_info {
356		- u8 opcode;
	324	+struct mlx5e_tx_mpwqe {
	325	+ /* Current MPWQE session */
	326	+ struct mlx5e_tx_wqe *wqe;
	327	+ u32 bytes_count;
	328	+ u8 ds_count;
	329	+ u8 pkt_count;
	330	+ u8 inline_on;
357	331	};
358	332
359	333	struct mlx5e_txqsq {
..	..	@@ -361,73 +335,136 @@
361	335
362	336	/* dirtied @completion */
363	337	u16 cc;
	338	+ u16 skb_fifo_cc;
364	339	u32 dma_fifo_cc;
365		- struct net_dim dim; /* Adaptive Moderation */
	340	+ struct dim dim; /* Adaptive Moderation */
366	341
367	342	/* dirtied @xmit */
368	343	u16 pc ____cacheline_aligned_in_smp;
	344	+ u16 skb_fifo_pc;
369	345	u32 dma_fifo_pc;
	346	+ struct mlx5e_tx_mpwqe mpwqe;
370	347
371	348	struct mlx5e_cq cq;
372	349
373	350	/* read only */
374	351	struct mlx5_wq_cyc wq;
375	352	u32 dma_fifo_mask;
	353	+ u16 skb_fifo_mask;
376	354	struct mlx5e_sq_stats *stats;
377	355	struct {
378	356	struct mlx5e_sq_dma *dma_fifo;
	357	+ struct sk_buff **skb_fifo;
379	358	struct mlx5e_tx_wqe_info *wqe_info;
380	359	} db;
381	360	void __iomem *uar_map;
382	361	struct netdev_queue *txq;
383	362	u32 sqn;
	363	+ u16 stop_room;
384	364	u8 min_inline_mode;
385	365	struct device *pdev;
386	366	__be32 mkey_be;
387	367	unsigned long state;
	368	+ unsigned int hw_mtu;
388	369	struct hwtstamp_config *tstamp;
389	370	struct mlx5_clock *clock;
390	371
391	372	/* control path */
392	373	struct mlx5_wq_ctrl wq_ctrl;
393	374	struct mlx5e_channel *channel;
	375	+ int ch_ix;
394	376	int txq_ix;
395	377	u32 rate_limit;
396		- struct mlx5e_txqsq_recover {
397		- struct work_struct recover_work;
398		- u64 last_recover;
399		- } recover;
	378	+ struct work_struct recover_work;
400	379	} ____cacheline_aligned_in_smp;
401	380
402	381	struct mlx5e_dma_info {
403		- struct page *page;
404		- dma_addr_t addr;
	382	+ dma_addr_t addr;
	383	+ union {
	384	+ struct page *page;
	385	+ struct xdp_buff *xsk;
	386	+ };
	387	+};
	388	+
	389	+/* XDP packets can be transmitted in different ways. On completion, we need to
	390	+ * distinguish between them to clean up things in a proper way.
	391	+ */
	392	+enum mlx5e_xdp_xmit_mode {
	393	+ /* An xdp_frame was transmitted due to either XDP_REDIRECT from another
	394	+ * device or XDP_TX from an XSK RQ. The frame has to be unmapped and
	395	+ * returned.
	396	+ */
	397	+ MLX5E_XDP_XMIT_MODE_FRAME,
	398	+
	399	+ /* The xdp_frame was created in place as a result of XDP_TX from a
	400	+ * regular RQ. No DMA remapping happened, and the page belongs to us.
	401	+ */
	402	+ MLX5E_XDP_XMIT_MODE_PAGE,
	403	+
	404	+ /* No xdp_frame was created at all, the transmit happened from a UMEM
	405	+ * page. The UMEM Completion Ring producer pointer has to be increased.
	406	+ */
	407	+ MLX5E_XDP_XMIT_MODE_XSK,
405	408	};
406	409
407	410	struct mlx5e_xdp_info {
408		- struct xdp_frame *xdpf;
409		- dma_addr_t dma_addr;
410		- struct mlx5e_dma_info di;
	411	+ enum mlx5e_xdp_xmit_mode mode;
	412	+ union {
	413	+ struct {
	414	+ struct xdp_frame *xdpf;
	415	+ dma_addr_t dma_addr;
	416	+ } frame;
	417	+ struct {
	418	+ struct mlx5e_rq *rq;
	419	+ struct mlx5e_dma_info di;
	420	+ } page;
	421	+ };
411	422	};
	423	+
	424	+struct mlx5e_xmit_data {
	425	+ dma_addr_t dma_addr;
	426	+ void *data;
	427	+ u32 len;
	428	+};
	429	+
	430	+struct mlx5e_xdp_info_fifo {
	431	+ struct mlx5e_xdp_info *xi;
	432	+ u32 *cc;
	433	+ u32 *pc;
	434	+ u32 mask;
	435	+};
	436	+
	437	+struct mlx5e_xdpsq;
	438	+typedef int (mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq );
	439	+typedef bool (mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq ,
	440	+ struct mlx5e_xmit_data *,
	441	+ struct mlx5e_xdp_info *,
	442	+ int);
412	443
413	444	struct mlx5e_xdpsq {
414	445	/* data path */
415	446
416	447	/* dirtied @completion */
	448	+ u32 xdpi_fifo_cc;
417	449	u16 cc;
418		- bool redirect_flush;
419	450
420	451	/* dirtied @xmit */
421		- u16 pc ____cacheline_aligned_in_smp;
422		- bool doorbell;
	452	+ u32 xdpi_fifo_pc ____cacheline_aligned_in_smp;
	453	+ u16 pc;
	454	+ struct mlx5_wqe_ctrl_seg *doorbell_cseg;
	455	+ struct mlx5e_tx_mpwqe mpwqe;
423	456
424	457	struct mlx5e_cq cq;
425	458
426	459	/* read only */
	460	+ struct xsk_buff_pool *xsk_pool;
427	461	struct mlx5_wq_cyc wq;
428	462	struct mlx5e_xdpsq_stats *stats;
	463	+ mlx5e_fp_xmit_xdp_frame_check xmit_xdp_frame_check;
	464	+ mlx5e_fp_xmit_xdp_frame xmit_xdp_frame;
429	465	struct {
430		- struct mlx5e_xdp_info *xdpi;
	466	+ struct mlx5e_xdp_wqe_info *wqe_info;
	467	+ struct mlx5e_xdp_info_fifo xdpi_fifo;
431	468	} db;
432	469	void __iomem *uar_map;
433	470	u32 sqn;
..	..	@@ -444,15 +481,15 @@
444	481
445	482	struct mlx5e_icosq {
446	483	/* data path */
	484	+ u16 cc;
	485	+ u16 pc;
447	486
448		- /* dirtied @xmit */
449		- u16 pc ____cacheline_aligned_in_smp;
450		-
	487	+ struct mlx5_wqe_ctrl_seg *doorbell_cseg;
451	488	struct mlx5e_cq cq;
452	489
453	490	/* write@xmit, read@completion */
454	491	struct {
455		- struct mlx5e_sq_wqe_info *ico_wqe;
	492	+ struct mlx5e_icosq_wqe_info *wqe_info;
456	493	} db;
457	494
458	495	/* read only */
..	..	@@ -464,13 +501,9 @@
464	501	/* control path */
465	502	struct mlx5_wq_ctrl wq_ctrl;
466	503	struct mlx5e_channel *channel;
467		-} ____cacheline_aligned_in_smp;
468	504
469		-static inline bool
470		-mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n)
471		-{
472		- return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) \|\| (cc == pc);
473		-}
	505	+ struct work_struct recover_work;
	506	+} ____cacheline_aligned_in_smp;
474	507
475	508	struct mlx5e_wqe_frag_info {
476	509	struct mlx5e_dma_info *di;
..	..	@@ -513,8 +546,11 @@
513	546	typedef bool (mlx5e_fp_post_rx_wqes)(struct mlx5e_rq rq);
514	547	typedef void (mlx5e_fp_dealloc_wqe)(struct mlx5e_rq, u16);
515	548
	549	+int mlx5e_rq_set_handlers(struct mlx5e_rq rq, struct mlx5e_params params, bool xsk);
	550	+
516	551	enum mlx5e_rq_flag {
517		- MLX5E_RQ_FLAG_XDP_XMIT = BIT(0),
	552	+ MLX5E_RQ_FLAG_XDP_XMIT,
	553	+ MLX5E_RQ_FLAG_XDP_REDIRECT,
518	554	};
519	555
520	556	struct mlx5e_rq_frag_info {
..	..	@@ -545,12 +581,16 @@
545	581	struct mlx5e_mpw_info *info;
546	582	mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;
547	583	u16 num_strides;
	584	+ u16 actual_wq_head;
548	585	u8 log_stride_sz;
549		- bool umr_in_progress;
	586	+ u8 umr_in_progress;
	587	+ u8 umr_last_bulk;
	588	+ u8 umr_completed;
550	589	} mpwqe;
551	590	};
552	591	struct {
553	592	u16 headroom;
	593	+ u32 frame0_sz;
554	594	u8 map_dir; /* dma map direction */
555	595	} buff;
556	596
..	..	@@ -559,6 +599,7 @@
559	599	struct net_device *netdev;
560	600	struct mlx5e_rq_stats *stats;
561	601	struct mlx5e_cq cq;
	602	+ struct mlx5e_cq_decomp cqd;
562	603	struct mlx5e_page_cache page_cache;
563	604	struct hwtstamp_config *tstamp;
564	605	struct mlx5_clock *clock;
..	..	@@ -571,13 +612,18 @@
571	612	int ix;
572	613	unsigned int hw_mtu;
573	614
574		- struct net_dim dim; /* Dynamic Interrupt Moderation */
	615	+ struct dim dim; /* Dynamic Interrupt Moderation */
575	616
576	617	/* XDP */
577		- struct bpf_prog *xdp_prog;
578		- struct mlx5e_xdpsq xdpsq;
	618	+ struct bpf_prog __rcu *xdp_prog;
	619	+ struct mlx5e_xdpsq *xdpsq;
579	620	DECLARE_BITMAP(flags, 8);
580	621	struct page_pool *page_pool;
	622	+
	623	+ /* AF_XDP zero-copy */
	624	+ struct xsk_buff_pool *xsk_pool;
	625	+
	626	+ struct work_struct recover_work;
581	627
582	628	/* control */
583	629	struct mlx5_wq_ctrl wq_ctrl;
..	..	@@ -586,14 +632,21 @@
586	632	u32 rqn;
587	633	struct mlx5_core_dev *mdev;
588	634	struct mlx5_core_mkey umr_mkey;
	635	+ struct mlx5e_dma_info wqe_overflow;
589	636
590	637	/* XDP read-mostly */
591	638	struct xdp_rxq_info xdp_rxq;
592	639	} ____cacheline_aligned_in_smp;
593	640
	641	+enum mlx5e_channel_state {
	642	+ MLX5E_CHANNEL_STATE_XSK,
	643	+ MLX5E_CHANNEL_NUM_STATES
	644	+};
	645	+
594	646	struct mlx5e_channel {
595	647	/* data path */
596	648	struct mlx5e_rq rq;
	649	+ struct mlx5e_xdpsq rq_xdpsq;
597	650	struct mlx5e_txqsq sq[MLX5E_MAX_NUM_TC];
598	651	struct mlx5e_icosq icosq; /* internal control operations */
599	652	bool xdp;
..	..	@@ -602,9 +655,19 @@
602	655	struct net_device *netdev;
603	656	__be32 mkey_be;
604	657	u8 num_tc;
	658	+ u8 lag_port;
605	659
606	660	/* XDP_REDIRECT */
607	661	struct mlx5e_xdpsq xdpsq;
	662	+
	663	+ /* AF_XDP zero-copy */
	664	+ struct mlx5e_rq xskrq;
	665	+ struct mlx5e_xdpsq xsksq;
	666	+
	667	+ /* Async ICOSQ */
	668	+ struct mlx5e_icosq async_icosq;
	669	+ /* async_icosq can be accessed from any CPU - the spinlock protects it. */
	670	+ spinlock_t async_icosq_lock;
608	671
609	672	/* data path - accessed per napi poll */
610	673	struct irq_desc *irq_desc;
..	..	@@ -614,6 +677,7 @@
614	677	struct mlx5e_priv *priv;
615	678	struct mlx5_core_dev *mdev;
616	679	struct hwtstamp_config *tstamp;
	680	+ DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);
617	681	int ix;
618	682	int cpu;
619	683	};
..	..	@@ -628,15 +692,17 @@
628	692	struct mlx5e_ch_stats ch;
629	693	struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];
630	694	struct mlx5e_rq_stats rq;
	695	+ struct mlx5e_rq_stats xskrq;
631	696	struct mlx5e_xdpsq_stats rq_xdpsq;
632	697	struct mlx5e_xdpsq_stats xdpsq;
	698	+ struct mlx5e_xdpsq_stats xsksq;
633	699	} ____cacheline_aligned_in_smp;
634	700
635	701	enum {
636		- MLX5E_STATE_ASYNC_EVENTS_ENABLED,
637	702	MLX5E_STATE_OPENED,
638	703	MLX5E_STATE_DESTROYING,
639	704	MLX5E_STATE_XDP_TX_ENABLED,
	705	+ MLX5E_STATE_XDP_ACTIVE,
640	706	};
641	707
642	708	struct mlx5e_rqt {
..	..	@@ -655,10 +721,54 @@
655	721	MLX5E_NIC_PRIO
656	722	};
657	723
	724	+struct mlx5e_rss_params {
	725	+ u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE];
	726	+ u32 rx_hash_fields[MLX5E_NUM_INDIR_TIRS];
	727	+ u8 toeplitz_hash_key[40];
	728	+ u8 hfunc;
	729	+};
	730	+
	731	+struct mlx5e_modify_sq_param {
	732	+ int curr_state;
	733	+ int next_state;
	734	+ int rl_update;
	735	+ int rl_index;
	736	+};
	737	+
	738	+#if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
	739	+struct mlx5e_hv_vhca_stats_agent {
	740	+ struct mlx5_hv_vhca_agent *agent;
	741	+ struct delayed_work work;
	742	+ u16 delay;
	743	+ void *buf;
	744	+};
	745	+#endif
	746	+
	747	+struct mlx5e_xsk {
	748	+ /* XSK buffer pools are stored separately from channels,
	749	+ * because we don't want to lose them when channels are
	750	+ * recreated. The kernel also stores buffer pool, but it doesn't
	751	+ * distinguish between zero-copy and non-zero-copy UMEMs, so
	752	+ * rely on our mechanism.
	753	+ */
	754	+ struct xsk_buff_pool **pools;
	755	+ u16 refcnt;
	756	+ bool ever_used;
	757	+};
	758	+
	759	+/* Temporary storage for variables that are allocated when struct mlx5e_priv is
	760	+ * initialized, and used where we can't allocate them because that functions
	761	+ * must not fail. Use with care and make sure the same variable is not used
	762	+ * simultaneously by multiple users.
	763	+ */
	764	+struct mlx5e_scratchpad {
	765	+ cpumask_var_t cpumask;
	766	+};
	767	+
658	768	struct mlx5e_priv {
659	769	/* priv data path fields - start */
660	770	struct mlx5e_txqsq txq2sq[MLX5E_MAX_NUM_CHANNELS MLX5E_MAX_NUM_TC];
661		- int channel_tc2txq[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC];
	771	+ int channel_tc2realtxq[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC];
662	772	#ifdef CONFIG_MLX5_CORE_EN_DCB
663	773	struct mlx5e_dcbx_dp dcbx_dp;
664	774	#endif
..	..	@@ -670,11 +780,13 @@
670	780	struct mlx5e_rq drop_rq;
671	781
672	782	struct mlx5e_channels channels;
673		- u32 tisn[MLX5E_MAX_NUM_TC];
	783	+ u32 tisn[MLX5_MAX_PORTS][MLX5E_MAX_NUM_TC];
674	784	struct mlx5e_rqt indir_rqt;
675	785	struct mlx5e_tir indir_tir[MLX5E_NUM_INDIR_TIRS];
676	786	struct mlx5e_tir inner_indir_tir[MLX5E_NUM_INDIR_TIRS];
677	787	struct mlx5e_tir direct_tir[MLX5E_MAX_NUM_CHANNELS];
	788	+ struct mlx5e_tir xsk_tir[MLX5E_MAX_NUM_CHANNELS];
	789	+ struct mlx5e_rss_params rss_params;
678	790	u32 tx_rates[MLX5E_MAX_NUM_SQS];
679	791
680	792	struct mlx5e_flow_steering fs;
..	..	@@ -683,16 +795,22 @@
683	795	struct work_struct update_carrier_work;
684	796	struct work_struct set_rx_mode_work;
685	797	struct work_struct tx_timeout_work;
686		- struct delayed_work update_stats_work;
	798	+ struct work_struct update_stats_work;
	799	+ struct work_struct monitor_counters_work;
	800	+ struct mlx5_nb monitor_counters_nb;
687	801
688	802	struct mlx5_core_dev *mdev;
689	803	struct net_device *netdev;
690	804	struct mlx5e_stats stats;
691	805	struct mlx5e_channel_stats channel_stats[MLX5E_MAX_NUM_CHANNELS];
	806	+ u16 max_nch;
692	807	u8 max_opened_tc;
693	808	struct hwtstamp_config tstamp;
694	809	u16 q_counter;
695	810	u16 drop_rq_q_counter;
	811	+ struct notifier_block events_nb;
	812	+
	813	+ struct udp_tunnel_nic_info nic_info;
696	814	#ifdef CONFIG_MLX5_CORE_EN_DCB
697	815	struct mlx5e_dcbx dcbx;
698	816	#endif
..	..	@@ -705,10 +823,25 @@
705	823	#ifdef CONFIG_MLX5_EN_TLS
706	824	struct mlx5e_tls *tls;
707	825	#endif
	826	+ struct devlink_health_reporter *tx_reporter;
	827	+ struct devlink_health_reporter *rx_reporter;
	828	+ struct devlink_port dl_port;
	829	+ struct mlx5e_xsk xsk;
	830	+#if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
	831	+ struct mlx5e_hv_vhca_stats_agent stats_agent;
	832	+#endif
	833	+ struct mlx5e_scratchpad scratchpad;
708	834	};
709	835
	836	+struct mlx5e_rx_handlers {
	837	+ mlx5e_fp_handle_rx_cqe handle_rx_cqe;
	838	+ mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe;
	839	+};
	840	+
	841	+extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic;
	842	+
710	843	struct mlx5e_profile {
711		- void (init)(struct mlx5_core_dev mdev,
	844	+ int (init)(struct mlx5_core_dev mdev,
712	845	struct net_device *netdev,
713	846	const struct mlx5e_profile profile, void ppriv);
714	847	void (cleanup)(struct mlx5e_priv priv);
..	..	@@ -718,59 +851,24 @@
718	851	void (cleanup_tx)(struct mlx5e_priv priv);
719	852	void (enable)(struct mlx5e_priv priv);
720	853	void (disable)(struct mlx5e_priv priv);
	854	+ int (update_rx)(struct mlx5e_priv priv);
721	855	void (update_stats)(struct mlx5e_priv priv);
722	856	void (update_carrier)(struct mlx5e_priv priv);
723		- int (max_nch)(struct mlx5_core_dev mdev);
724		- struct {
725		- mlx5e_fp_handle_rx_cqe handle_rx_cqe;
726		- mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe;
727		- } rx_handlers;
	857	+ unsigned int (stats_grps_num)(struct mlx5e_priv priv);
	858	+ mlx5e_stats_grp_t *stats_grps;
	859	+ const struct mlx5e_rx_handlers *rx_handlers;
728	860	int max_tc;
	861	+ u8 rq_groups;
729	862	};
730	863
731	864	void mlx5e_build_ptys2ethtool_map(void);
732		-
733		-u16 mlx5e_select_queue(struct net_device dev, struct sk_buff skb,
734		- struct net_device *sb_dev,
735		- select_queue_fallback_t fallback);
736		-netdev_tx_t mlx5e_xmit(struct sk_buff skb, struct net_device dev);
737		-netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq sq, struct sk_buff skb,
738		- struct mlx5e_tx_wqe *wqe, u16 pi);
739		-
740		-void mlx5e_completion_event(struct mlx5_core_cq *mcq);
741		-void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
742		-int mlx5e_napi_poll(struct napi_struct *napi, int budget);
743		-bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
744		-int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
745		-void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
746	865
747	866	bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev);
748	867	bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
749	868	struct mlx5e_params *params);
750	869
751		-void mlx5e_page_dma_unmap(struct mlx5e_rq rq, struct mlx5e_dma_info dma_info);
752		-void mlx5e_page_release(struct mlx5e_rq rq, struct mlx5e_dma_info dma_info,
753		- bool recycle);
754		-void mlx5e_handle_rx_cqe(struct mlx5e_rq rq, struct mlx5_cqe64 cqe);
755		-void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq rq, struct mlx5_cqe64 cqe);
756		-bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq);
757		-bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);
758		-void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
759		-void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
760		-struct sk_buff *
761		-mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq rq, struct mlx5e_mpw_info wi,
762		- u16 cqe_bcnt, u32 head_offset, u32 page_idx);
763		-struct sk_buff *
764		-mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq rq, struct mlx5e_mpw_info wi,
765		- u16 cqe_bcnt, u32 head_offset, u32 page_idx);
766		-struct sk_buff *
767		-mlx5e_skb_from_cqe_linear(struct mlx5e_rq rq, struct mlx5_cqe64 cqe,
768		- struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt);
769		-struct sk_buff *
770		-mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq rq, struct mlx5_cqe64 cqe,
771		- struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt);
772		-
773		-void mlx5e_update_stats(struct mlx5e_priv *priv);
	870	+void mlx5e_get_stats(struct net_device dev, struct rtnl_link_stats64 stats);
	871	+void mlx5e_fold_sw_stats64(struct mlx5e_priv priv, struct rtnl_link_stats64 s);
774	872
775	873	void mlx5e_init_l2_addr(struct mlx5e_priv *priv);
776	874	int mlx5e_self_test_num(struct mlx5e_priv *priv);
..	..	@@ -801,9 +899,32 @@
801	899
802	900	int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
803	901	struct mlx5e_redirect_rqt_param rrp);
804		-void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_params *params,
805		- enum mlx5e_traffic_types tt,
	902	+void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params,
	903	+ const struct mlx5e_tirc_config *ttconfig,
806	904	void *tirc, bool inner);
	905	+void mlx5e_modify_tirs_hash(struct mlx5e_priv priv, void in);
	906	+struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt);
	907	+
	908	+struct mlx5e_xsk_param;
	909	+
	910	+struct mlx5e_rq_param;
	911	+int mlx5e_open_rq(struct mlx5e_channel c, struct mlx5e_params params,
	912	+ struct mlx5e_rq_param param, struct mlx5e_xsk_param xsk,
	913	+ struct xsk_buff_pool xsk_pool, struct mlx5e_rq rq);
	914	+int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
	915	+void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
	916	+void mlx5e_close_rq(struct mlx5e_rq *rq);
	917	+
	918	+struct mlx5e_sq_param;
	919	+int mlx5e_open_xdpsq(struct mlx5e_channel c, struct mlx5e_params params,
	920	+ struct mlx5e_sq_param param, struct xsk_buff_pool xsk_pool,
	921	+ struct mlx5e_xdpsq *sq, bool is_redirect);
	922	+void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq);
	923	+
	924	+struct mlx5e_cq_param;
	925	+int mlx5e_open_cq(struct mlx5e_channel *c, struct dim_cq_moder moder,
	926	+ struct mlx5e_cq_param param, struct mlx5e_cq cq);
	927	+void mlx5e_close_cq(struct mlx5e_cq *cq);
807	928
808	929	int mlx5e_open_locked(struct net_device *netdev);
809	930	int mlx5e_close_locked(struct net_device *netdev);
..	..	@@ -812,131 +933,101 @@
812	933	struct mlx5e_channels *chs);
813	934	void mlx5e_close_channels(struct mlx5e_channels *chs);
814	935
815		-/* Function pointer to be used to modify WH settings while
	936	+/* Function pointer to be used to modify HW or kernel settings while
816	937	* switching channels
817	938	*/
818		-typedef int (mlx5e_fp_hw_modify)(struct mlx5e_priv priv);
819		-void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
820		- struct mlx5e_channels *new_chs,
821		- mlx5e_fp_hw_modify hw_modify);
	939	+typedef int (mlx5e_fp_preactivate)(struct mlx5e_priv priv, void *context);
	940	+#define MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(fn) \
	941	+int fn##_ctx(struct mlx5e_priv priv, void context) \
	942	+{ \
	943	+ return fn(priv); \
	944	+}
	945	+int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv);
	946	+int mlx5e_safe_switch_channels(struct mlx5e_priv *priv,
	947	+ struct mlx5e_channels *new_chs,
	948	+ mlx5e_fp_preactivate preactivate,
	949	+ void *context);
	950	+int mlx5e_num_channels_changed(struct mlx5e_priv *priv);
	951	+int mlx5e_num_channels_changed_ctx(struct mlx5e_priv priv, void context);
822	952	void mlx5e_activate_priv_channels(struct mlx5e_priv *priv);
823	953	void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);
824	954
825	955	void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
826	956	int num_channels);
827		-void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params,
828		- u8 cq_period_mode);
829		-void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params,
830		- u8 cq_period_mode);
	957	+
	958	+void mlx5e_reset_tx_moderation(struct mlx5e_params *params, u8 cq_period_mode);
	959	+void mlx5e_reset_rx_moderation(struct mlx5e_params *params, u8 cq_period_mode);
	960	+void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode);
	961	+void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode);
	962	+
831	963	void mlx5e_set_rq_type(struct mlx5_core_dev mdev, struct mlx5e_params params);
832	964	void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
833	965	struct mlx5e_params *params);
	966	+int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state);
	967	+void mlx5e_activate_rq(struct mlx5e_rq *rq);
	968	+void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
	969	+void mlx5e_activate_icosq(struct mlx5e_icosq *icosq);
	970	+void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq);
834	971
835		-static inline bool mlx5e_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev)
	972	+int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
	973	+ struct mlx5e_modify_sq_param *p);
	974	+void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq);
	975	+void mlx5e_tx_disable_queue(struct netdev_queue *txq);
	976	+
	977	+static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev)
836	978	{
837		- return (MLX5_CAP_ETH(mdev, tunnel_stateless_gre) &&
838		- MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.inner_ip_version));
839		-}
840		-
841		-static inline void mlx5e_sq_fetch_wqe(struct mlx5e_txqsq *sq,
842		- struct mlx5e_tx_wqe **wqe,
843		- u16 *pi)
844		-{
845		- struct mlx5_wq_cyc *wq = &sq->wq;
846		-
847		- *pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
848		- wqe = mlx5_wq_cyc_get_wqe(wq, pi);
849		- memset(wqe, 0, sizeof(*wqe));
850		-}
851		-
852		-static inline
853		-struct mlx5e_tx_wqe mlx5e_post_nop(struct mlx5_wq_cyc wq, u32 sqn, u16 *pc)
854		-{
855		- u16 pi = mlx5_wq_cyc_ctr2ix(wq, *pc);
856		- struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
857		- struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
858		-
859		- memset(cseg, 0, sizeof(*cseg));
860		-
861		- cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) \| MLX5_OPCODE_NOP);
862		- cseg->qpn_ds = cpu_to_be32((sqn << 8) \| 0x01);
863		-
864		- (*pc)++;
865		-
866		- return wqe;
867		-}
868		-
869		-static inline
870		-void mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc,
871		- void __iomem *uar_map,
872		- struct mlx5_wqe_ctrl_seg *ctrl)
873		-{
874		- ctrl->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
875		- /* ensure wqe is visible to device before updating doorbell record */
876		- dma_wmb();
877		-
878		- *wq->db = cpu_to_be32(pc);
879		-
880		- /* ensure doorbell record is visible to device before ringing the
881		- * doorbell
882		- */
883		- wmb();
884		-
885		- mlx5_write64((__be32 *)ctrl, uar_map, NULL);
886		-}
887		-
888		-static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
889		-{
890		- struct mlx5_core_cq *mcq;
891		-
892		- mcq = &cq->mcq;
893		- mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
	979	+ return MLX5_CAP_ETH(mdev, swp) &&
	980	+ MLX5_CAP_ETH(mdev, swp_csum) && MLX5_CAP_ETH(mdev, swp_lso);
894	981	}
895	982
896	983	extern const struct ethtool_ops mlx5e_ethtool_ops;
897		-#ifdef CONFIG_MLX5_CORE_EN_DCB
898		-extern const struct dcbnl_rtnl_ops mlx5e_dcbnl_ops;
899		-int mlx5e_dcbnl_ieee_setets_core(struct mlx5e_priv priv, struct ieee_ets ets);
900		-void mlx5e_dcbnl_initialize(struct mlx5e_priv *priv);
901		-void mlx5e_dcbnl_init_app(struct mlx5e_priv *priv);
902		-void mlx5e_dcbnl_delete_app(struct mlx5e_priv *priv);
903		-#endif
904	984
905		-int mlx5e_create_tir(struct mlx5_core_dev *mdev,
906		- struct mlx5e_tir tir, u32 in, int inlen);
	985	+int mlx5e_create_tir(struct mlx5_core_dev mdev, struct mlx5e_tir tir,
	986	+ u32 *in);
907	987	void mlx5e_destroy_tir(struct mlx5_core_dev *mdev,
908	988	struct mlx5e_tir *tir);
909	989	int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev);
910	990	void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev);
911		-int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb);
	991	+int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb,
	992	+ bool enable_mc_lb);
	993	+void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev mdev, void mkc);
912	994
913	995	/* common netdev helpers */
	996	+void mlx5e_create_q_counters(struct mlx5e_priv *priv);
	997	+void mlx5e_destroy_q_counters(struct mlx5e_priv *priv);
	998	+int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
	999	+ struct mlx5e_rq *drop_rq);
	1000	+void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq);
	1001	+
914	1002	int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv);
915	1003
916		-int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv);
	1004	+int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc);
917	1005	void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv);
918	1006
919		-int mlx5e_create_direct_rqts(struct mlx5e_priv *priv);
920		-void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv);
921		-int mlx5e_create_direct_tirs(struct mlx5e_priv *priv);
922		-void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv);
	1007	+int mlx5e_create_direct_rqts(struct mlx5e_priv priv, struct mlx5e_tir tirs);
	1008	+void mlx5e_destroy_direct_rqts(struct mlx5e_priv priv, struct mlx5e_tir tirs);
	1009	+int mlx5e_create_direct_tirs(struct mlx5e_priv priv, struct mlx5e_tir tirs);
	1010	+void mlx5e_destroy_direct_tirs(struct mlx5e_priv priv, struct mlx5e_tir tirs);
923	1011	void mlx5e_destroy_rqt(struct mlx5e_priv priv, struct mlx5e_rqt rqt);
924	1012
925		-int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc,
926		- u32 underlay_qpn, u32 *tisn);
	1013	+int mlx5e_create_tis(struct mlx5_core_dev mdev, void in, u32 *tisn);
927	1014	void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn);
928	1015
929	1016	int mlx5e_create_tises(struct mlx5e_priv *priv);
930		-void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv);
	1017	+void mlx5e_destroy_tises(struct mlx5e_priv *priv);
	1018	+int mlx5e_update_nic_rx(struct mlx5e_priv *priv);
	1019	+void mlx5e_update_carrier(struct mlx5e_priv *priv);
931	1020	int mlx5e_close(struct net_device *netdev);
932	1021	int mlx5e_open(struct net_device *netdev);
933		-void mlx5e_update_stats_work(struct work_struct *work);
934	1022
	1023	+void mlx5e_queue_update_stats(struct mlx5e_priv *priv);
935	1024	int mlx5e_bits_invert(unsigned long a, int size);
936	1025
937		-typedef int (change_hw_mtu_cb)(struct mlx5e_priv priv);
	1026	+int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv);
	1027	+int mlx5e_set_dev_port_mtu_ctx(struct mlx5e_priv priv, void context);
938	1028	int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
939		- change_hw_mtu_cb set_mtu_cb);
	1029	+ mlx5e_fp_preactivate preactivate);
	1030	+void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv);
940	1031
941	1032	/* ethtool helpers */
942	1033	void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
..	..	@@ -958,22 +1049,61 @@
958	1049	struct ethtool_coalesce *coal);
959	1050	int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
960	1051	struct ethtool_coalesce *coal);
	1052	+int mlx5e_ethtool_get_link_ksettings(struct mlx5e_priv *priv,
	1053	+ struct ethtool_link_ksettings *link_ksettings);
	1054	+int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv,
	1055	+ const struct ethtool_link_ksettings *link_ksettings);
	1056	+int mlx5e_get_rxfh(struct net_device netdev, u32 indir, u8 key, u8 hfunc);
	1057	+int mlx5e_set_rxfh(struct net_device dev, const u32 indir, const u8 *key,
	1058	+ const u8 hfunc);
	1059	+int mlx5e_get_rxnfc(struct net_device dev, struct ethtool_rxnfc info,
	1060	+ u32 *rule_locs);
	1061	+int mlx5e_set_rxnfc(struct net_device dev, struct ethtool_rxnfc cmd);
	1062	+u32 mlx5e_ethtool_get_rxfh_key_size(struct mlx5e_priv *priv);
	1063	+u32 mlx5e_ethtool_get_rxfh_indir_size(struct mlx5e_priv *priv);
961	1064	int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv,
962	1065	struct ethtool_ts_info *info);
963	1066	int mlx5e_ethtool_flash_device(struct mlx5e_priv *priv,
964	1067	struct ethtool_flash *flash);
	1068	+void mlx5e_ethtool_get_pauseparam(struct mlx5e_priv *priv,
	1069	+ struct ethtool_pauseparam *pauseparam);
	1070	+int mlx5e_ethtool_set_pauseparam(struct mlx5e_priv *priv,
	1071	+ struct ethtool_pauseparam *pauseparam);
965	1072
966	1073	/* mlx5e generic netdev management API */
	1074	+int mlx5e_netdev_init(struct net_device *netdev,
	1075	+ struct mlx5e_priv *priv,
	1076	+ struct mlx5_core_dev *mdev,
	1077	+ const struct mlx5e_profile *profile,
	1078	+ void *ppriv);
	1079	+void mlx5e_netdev_cleanup(struct net_device netdev, struct mlx5e_priv priv);
967	1080	struct net_device*
968	1081	mlx5e_create_netdev(struct mlx5_core_dev mdev, const struct mlx5e_profile profile,
969		- void *ppriv);
	1082	+ int nch, void *ppriv);
970	1083	int mlx5e_attach_netdev(struct mlx5e_priv *priv);
971	1084	void mlx5e_detach_netdev(struct mlx5e_priv *priv);
972	1085	void mlx5e_destroy_netdev(struct mlx5e_priv *priv);
973		-void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
	1086	+void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv);
	1087	+void mlx5e_build_nic_params(struct mlx5e_priv *priv,
	1088	+ struct mlx5e_xsk *xsk,
	1089	+ struct mlx5e_rss_params *rss_params,
974	1090	struct mlx5e_params *params,
975		- u16 max_channels, u16 mtu);
976		-u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev);
	1091	+ u16 mtu);
	1092	+void mlx5e_build_rq_params(struct mlx5_core_dev *mdev,
	1093	+ struct mlx5e_params *params);
	1094	+void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params,
	1095	+ u16 num_channels);
977	1096	void mlx5e_rx_dim_work(struct work_struct *work);
978	1097	void mlx5e_tx_dim_work(struct work_struct *work);
	1098	+
	1099	+netdev_features_t mlx5e_features_check(struct sk_buff *skb,
	1100	+ struct net_device *netdev,
	1101	+ netdev_features_t features);
	1102	+int mlx5e_set_features(struct net_device *netdev, netdev_features_t features);
	1103	+#ifdef CONFIG_MLX5_ESWITCH
	1104	+int mlx5e_set_vf_mac(struct net_device dev, int vf, u8 mac);
	1105	+int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_tx_rate);
	1106	+int mlx5e_get_vf_config(struct net_device dev, int vf, struct ifla_vf_info ivi);
	1107	+int mlx5e_get_vf_stats(struct net_device dev, int vf, struct ifla_vf_stats vf_stats);
	1108	+#endif
979	1109	#endif /* __MLX5_EN_H__ */