~hc/RK356X_SDK_RELEASE.git

..	..	@@ -32,56 +32,13 @@
32	32
33	33	#include <linux/tcp.h>
34	34	#include <linux/if_vlan.h>
	35	+#include <net/geneve.h>
35	36	#include <net/dsfield.h>
36	37	#include "en.h"
	38	+#include "en/txrx.h"
37	39	#include "ipoib/ipoib.h"
38	40	#include "en_accel/en_accel.h"
39	41	#include "lib/clock.h"
40		-
41		-#define MLX5E_SQ_NOPS_ROOM MLX5_SEND_WQE_MAX_WQEBBS
42		-
43		-#ifndef CONFIG_MLX5_EN_TLS
44		-#define MLX5E_SQ_STOP_ROOM (MLX5_SEND_WQE_MAX_WQEBBS +\
45		- MLX5E_SQ_NOPS_ROOM)
46		-#else
47		-/* TLS offload requires MLX5E_SQ_STOP_ROOM to have
48		- * enough room for a resync SKB, a normal SKB and a NOP
49		- */
50		-#define MLX5E_SQ_STOP_ROOM (2 * MLX5_SEND_WQE_MAX_WQEBBS +\
51		- MLX5E_SQ_NOPS_ROOM)
52		-#endif
53		-
54		-static inline void mlx5e_tx_dma_unmap(struct device *pdev,
55		- struct mlx5e_sq_dma *dma)
56		-{
57		- switch (dma->type) {
58		- case MLX5E_DMA_MAP_SINGLE:
59		- dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
60		- break;
61		- case MLX5E_DMA_MAP_PAGE:
62		- dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
63		- break;
64		- default:
65		- WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n");
66		- }
67		-}
68		-
69		-static inline struct mlx5e_sq_dma mlx5e_dma_get(struct mlx5e_txqsq sq, u32 i)
70		-{
71		- return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
72		-}
73		-
74		-static inline void mlx5e_dma_push(struct mlx5e_txqsq *sq,
75		- dma_addr_t addr,
76		- u32 size,
77		- enum mlx5e_dma_map_type map_type)
78		-{
79		- struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++);
80		-
81		- dma->addr = addr;
82		- dma->size = size;
83		- dma->type = map_type;
84		-}
85	42
86	43	static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma)
87	44	{
..	..	@@ -110,16 +67,15 @@
110	67	#endif
111	68
112	69	u16 mlx5e_select_queue(struct net_device dev, struct sk_buff skb,
113		- struct net_device *sb_dev,
114		- select_queue_fallback_t fallback)
	70	+ struct net_device *sb_dev)
115	71	{
	72	+ int txq_ix = netdev_pick_tx(dev, skb, NULL);
116	73	struct mlx5e_priv *priv = netdev_priv(dev);
117		- int channel_ix = fallback(dev, skb, NULL);
118		- u16 num_channels;
119	74	int up = 0;
	75	+ int ch_ix;
120	76
121	77	if (!netdev_get_num_tc(dev))
122		- return channel_ix;
	78	+ return txq_ix;
123	79
124	80	#ifdef CONFIG_MLX5_CORE_EN_DCB
125	81	if (priv->dcbx_dp.trust_state == MLX5_QPTS_TRUST_DSCP)
..	..	@@ -127,16 +83,15 @@
127	83	else
128	84	#endif
129	85	if (skb_vlan_tag_present(skb))
130		- up = skb->vlan_tci >> VLAN_PRIO_SHIFT;
	86	+ up = skb_vlan_tag_get_prio(skb);
131	87
132		- /* channel_ix can be larger than num_channels since
133		- * dev->num_real_tx_queues = num_channels * num_tc
	88	+ /* Normalize any picked txq_ix to [0, num_channels),
	89	+ * So we can return a txq_ix that matches the channel and
	90	+ * packet UP.
134	91	*/
135		- num_channels = priv->channels.params.num_channels;
136		- if (channel_ix >= num_channels)
137		- channel_ix = reciprocal_scale(channel_ix, num_channels);
	92	+ ch_ix = priv->txq2sq[txq_ix]->ch_ix;
138	93
139		- return priv->channel_tc2txq[channel_ix][up];
	94	+ return priv->channel_tc2realtxq[ch_ix][up];
140	95	}
141	96
142	97	static inline int mlx5e_skb_l2_header_offset(struct sk_buff *skb)
..	..	@@ -148,12 +103,8 @@
148	103
149	104	static inline int mlx5e_skb_l3_header_offset(struct sk_buff *skb)
150	105	{
151		- struct flow_keys keys;
152		-
153	106	if (skb_transport_header_was_set(skb))
154	107	return skb_transport_offset(skb);
155		- else if (skb_flow_dissect_flow_keys(skb, &keys, 0))
156		- return keys.control.thoff;
157	108	else
158	109	return mlx5e_skb_l2_header_offset(skb);
159	110	}
..	..	@@ -167,20 +118,13 @@
167	118	case MLX5_INLINE_MODE_NONE:
168	119	return 0;
169	120	case MLX5_INLINE_MODE_TCP_UDP:
170		- hlen = eth_get_headlen(skb->data, skb_headlen(skb));
	121	+ hlen = eth_get_headlen(skb->dev, skb->data, skb_headlen(skb));
171	122	if (hlen == ETH_HLEN && !skb_vlan_tag_present(skb))
172	123	hlen += VLAN_HLEN;
173	124	break;
174	125	case MLX5_INLINE_MODE_IP:
175		- /* When transport header is set to zero, it means no transport
176		- * header. When transport header is set to 0xff's, it means
177		- * transport header wasn't set.
178		- */
179		- if (skb_transport_offset(skb)) {
180		- hlen = mlx5e_skb_l3_header_offset(skb);
181		- break;
182		- }
183		- /* fall through */
	126	+ hlen = mlx5e_skb_l3_header_offset(skb);
	127	+ break;
184	128	case MLX5_INLINE_MODE_L2:
185	129	default:
186	130	hlen = mlx5e_skb_l2_header_offset(skb);
..	..	@@ -200,8 +144,26 @@
200	144	memcpy(&vhdr->h_vlan_encapsulated_proto, skb->data + cpy1_sz, cpy2_sz);
201	145	}
202	146
	147	+/* If packet is not IP's CHECKSUM_PARTIAL (e.g. icmd packet),
	148	+ * need to set L3 checksum flag for IPsec
	149	+ */
	150	+static void
	151	+ipsec_txwqe_build_eseg_csum(struct mlx5e_txqsq sq, struct sk_buff skb,
	152	+ struct mlx5_wqe_eth_seg *eseg)
	153	+{
	154	+ eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM;
	155	+ if (skb->encapsulation) {
	156	+ eseg->cs_flags \|= MLX5_ETH_WQE_L3_INNER_CSUM;
	157	+ sq->stats->csum_partial_inner++;
	158	+ } else {
	159	+ sq->stats->csum_partial++;
	160	+ }
	161	+}
	162	+
203	163	static inline void
204		-mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq sq, struct sk_buff skb, struct mlx5_wqe_eth_seg *eseg)
	164	+mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq sq, struct sk_buff skb,
	165	+ struct mlx5e_accel_tx_state *accel,
	166	+ struct mlx5_wqe_eth_seg *eseg)
205	167	{
206	168	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
207	169	eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM;
..	..	@@ -213,6 +175,14 @@
213	175	eseg->cs_flags \|= MLX5_ETH_WQE_L4_CSUM;
214	176	sq->stats->csum_partial++;
215	177	}
	178	+#ifdef CONFIG_MLX5_EN_TLS
	179	+ } else if (unlikely(accel && accel->tls.tls_tisn)) {
	180	+ eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM \| MLX5_ETH_WQE_L4_CSUM;
	181	+ sq->stats->csum_partial++;
	182	+#endif
	183	+ } else if (unlikely(eseg->flow_table_metadata & cpu_to_be32(MLX5_ETH_WQE_FT_META_IPSEC))) {
	184	+ ipsec_txwqe_build_eseg_csum(sq, skb, eseg);
	185	+
216	186	} else
217	187	sq->stats->csum_none++;
218	188	}
..	..	@@ -264,7 +234,7 @@
264	234	}
265	235
266	236	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
267		- struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
	237	+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
268	238	int fsz = skb_frag_size(frag);
269	239
270	240	dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
..	..	@@ -288,114 +258,169 @@
288	258	return -ENOMEM;
289	259	}
290	260
291		-static inline void mlx5e_fill_sq_frag_edge(struct mlx5e_txqsq *sq,
292		- struct mlx5_wq_cyc *wq,
293		- u16 pi, u16 nnops)
	261	+struct mlx5e_tx_attr {
	262	+ u32 num_bytes;
	263	+ u16 headlen;
	264	+ u16 ihs;
	265	+ __be16 mss;
	266	+ u16 insz;
	267	+ u8 opcode;
	268	+};
	269	+
	270	+struct mlx5e_tx_wqe_attr {
	271	+ u16 ds_cnt;
	272	+ u16 ds_cnt_inl;
	273	+ u16 ds_cnt_ids;
	274	+ u8 num_wqebbs;
	275	+};
	276	+
	277	+static u8
	278	+mlx5e_tx_wqe_inline_mode(struct mlx5e_txqsq sq, struct sk_buff skb,
	279	+ struct mlx5e_accel_tx_state *accel)
294	280	{
295		- struct mlx5e_tx_wqe_info edge_wi, wi = &sq->db.wqe_info[pi];
	281	+ u8 mode;
296	282
297		- edge_wi = wi + nnops;
	283	+#ifdef CONFIG_MLX5_EN_TLS
	284	+ if (accel && accel->tls.tls_tisn)
	285	+ return MLX5_INLINE_MODE_TCP_UDP;
	286	+#endif
298	287
299		- /* fill sq frag edge with nops to avoid wqe wrapping two pages */
300		- for (; wi < edge_wi; wi++) {
301		- wi->skb = NULL;
302		- wi->num_wqebbs = 1;
303		- mlx5e_post_nop(wq, sq->sqn, &sq->pc);
	288	+ mode = sq->min_inline_mode;
	289	+
	290	+ if (skb_vlan_tag_present(skb) &&
	291	+ test_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state))
	292	+ mode = max_t(u8, MLX5_INLINE_MODE_L2, mode);
	293	+
	294	+ return mode;
	295	+}
	296	+
	297	+static void mlx5e_sq_xmit_prepare(struct mlx5e_txqsq sq, struct sk_buff skb,
	298	+ struct mlx5e_accel_tx_state *accel,
	299	+ struct mlx5e_tx_attr *attr)
	300	+{
	301	+ struct mlx5e_sq_stats *stats = sq->stats;
	302	+
	303	+ if (skb_is_gso(skb)) {
	304	+ u16 ihs = mlx5e_tx_get_gso_ihs(sq, skb);
	305	+
	306	+ *attr = (struct mlx5e_tx_attr) {
	307	+ .opcode = MLX5_OPCODE_LSO,
	308	+ .mss = cpu_to_be16(skb_shinfo(skb)->gso_size),
	309	+ .ihs = ihs,
	310	+ .num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs,
	311	+ .headlen = skb_headlen(skb) - ihs,
	312	+ };
	313	+
	314	+ stats->packets += skb_shinfo(skb)->gso_segs;
	315	+ } else {
	316	+ u8 mode = mlx5e_tx_wqe_inline_mode(sq, skb, accel);
	317	+ u16 ihs = mlx5e_calc_min_inline(mode, skb);
	318	+
	319	+ *attr = (struct mlx5e_tx_attr) {
	320	+ .opcode = MLX5_OPCODE_SEND,
	321	+ .mss = cpu_to_be16(0),
	322	+ .ihs = ihs,
	323	+ .num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN),
	324	+ .headlen = skb_headlen(skb) - ihs,
	325	+ };
	326	+
	327	+ stats->packets++;
304	328	}
305		- sq->stats->nop += nnops;
	329	+
	330	+ attr->insz = mlx5e_accel_tx_ids_len(sq, accel);
	331	+ stats->bytes += attr->num_bytes;
	332	+}
	333	+
	334	+static void mlx5e_sq_calc_wqe_attr(struct sk_buff skb, const struct mlx5e_tx_attr attr,
	335	+ struct mlx5e_tx_wqe_attr *wqe_attr)
	336	+{
	337	+ u16 ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT;
	338	+ u16 ds_cnt_inl = 0;
	339	+ u16 ds_cnt_ids = 0;
	340	+
	341	+ if (attr->insz)
	342	+ ds_cnt_ids = DIV_ROUND_UP(sizeof(struct mlx5_wqe_inline_seg) + attr->insz,
	343	+ MLX5_SEND_WQE_DS);
	344	+
	345	+ ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags + ds_cnt_ids;
	346	+ if (attr->ihs) {
	347	+ u16 inl = attr->ihs - INL_HDR_START_SZ;
	348	+
	349	+ if (skb_vlan_tag_present(skb))
	350	+ inl += VLAN_HLEN;
	351	+
	352	+ ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS);
	353	+ ds_cnt += ds_cnt_inl;
	354	+ }
	355	+
	356	+ *wqe_attr = (struct mlx5e_tx_wqe_attr) {
	357	+ .ds_cnt = ds_cnt,
	358	+ .ds_cnt_inl = ds_cnt_inl,
	359	+ .ds_cnt_ids = ds_cnt_ids,
	360	+ .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS),
	361	+ };
	362	+}
	363	+
	364	+static void mlx5e_tx_skb_update_hwts_flags(struct sk_buff *skb)
	365	+{
	366	+ if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
	367	+ skb_shinfo(skb)->tx_flags \|= SKBTX_IN_PROGRESS;
	368	+}
	369	+
	370	+static void mlx5e_tx_check_stop(struct mlx5e_txqsq *sq)
	371	+{
	372	+ if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room))) {
	373	+ netif_tx_stop_queue(sq->txq);
	374	+ sq->stats->stopped++;
	375	+ }
306	376	}
307	377
308	378	static inline void
309	379	mlx5e_txwqe_complete(struct mlx5e_txqsq sq, struct sk_buff skb,
310		- u8 opcode, u16 ds_cnt, u8 num_wqebbs, u32 num_bytes, u8 num_dma,
311		- struct mlx5e_tx_wqe_info wi, struct mlx5_wqe_ctrl_seg cseg)
	380	+ const struct mlx5e_tx_attr *attr,
	381	+ const struct mlx5e_tx_wqe_attr *wqe_attr, u8 num_dma,
	382	+ struct mlx5e_tx_wqe_info wi, struct mlx5_wqe_ctrl_seg cseg,
	383	+ bool xmit_more)
312	384	{
313	385	struct mlx5_wq_cyc *wq = &sq->wq;
	386	+ bool send_doorbell;
314	387
315		- wi->num_bytes = num_bytes;
316		- wi->num_dma = num_dma;
317		- wi->num_wqebbs = num_wqebbs;
318		- wi->skb = skb;
	388	+ *wi = (struct mlx5e_tx_wqe_info) {
	389	+ .skb = skb,
	390	+ .num_bytes = attr->num_bytes,
	391	+ .num_dma = num_dma,
	392	+ .num_wqebbs = wqe_attr->num_wqebbs,
	393	+ .num_fifo_pkts = 0,
	394	+ };
319	395
320		- cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| opcode);
321		- cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) \| ds_cnt);
	396	+ cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| attr->opcode);
	397	+ cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) \| wqe_attr->ds_cnt);
322	398
323		- netdev_tx_sent_queue(sq->txq, num_bytes);
324		-
325		- if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
326		- skb_shinfo(skb)->tx_flags \|= SKBTX_IN_PROGRESS;
	399	+ mlx5e_tx_skb_update_hwts_flags(skb);
327	400
328	401	sq->pc += wi->num_wqebbs;
329		- if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, MLX5E_SQ_STOP_ROOM))) {
330		- netif_tx_stop_queue(sq->txq);
331		- sq->stats->stopped++;
332		- }
333	402
334		- if (!skb->xmit_more \|\| netif_xmit_stopped(sq->txq))
	403	+ mlx5e_tx_check_stop(sq);
	404	+
	405	+ send_doorbell = __netdev_tx_sent_queue(sq->txq, attr->num_bytes, xmit_more);
	406	+ if (send_doorbell)
335	407	mlx5e_notify_hw(wq, sq->pc, sq->uar_map, cseg);
336	408	}
337	409
338		-#define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start))
339		-
340		-netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq sq, struct sk_buff skb,
341		- struct mlx5e_tx_wqe *wqe, u16 pi)
	410	+static void
	411	+mlx5e_sq_xmit_wqe(struct mlx5e_txqsq sq, struct sk_buff skb,
	412	+ const struct mlx5e_tx_attr attr, const struct mlx5e_tx_wqe_attr wqe_attr,
	413	+ struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more)
342	414	{
343		- struct mlx5_wq_cyc *wq = &sq->wq;
344	415	struct mlx5_wqe_ctrl_seg *cseg;
345	416	struct mlx5_wqe_eth_seg *eseg;
346	417	struct mlx5_wqe_data_seg *dseg;
347	418	struct mlx5e_tx_wqe_info *wi;
348	419
349	420	struct mlx5e_sq_stats *stats = sq->stats;
350		- u16 headlen, ihs, contig_wqebbs_room;
351		- u16 ds_cnt, ds_cnt_inl = 0;
352		- u8 num_wqebbs, opcode;
353		- u32 num_bytes;
354	421	int num_dma;
355		- __be16 mss;
356	422
357		- /* Calc ihs and ds cnt, no writes to wqe yet */
358		- ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
359		- if (skb_is_gso(skb)) {
360		- opcode = MLX5_OPCODE_LSO;
361		- mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
362		- ihs = mlx5e_tx_get_gso_ihs(sq, skb);
363		- num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs;
364		- stats->packets += skb_shinfo(skb)->gso_segs;
365		- } else {
366		- opcode = MLX5_OPCODE_SEND;
367		- mss = 0;
368		- ihs = mlx5e_calc_min_inline(sq->min_inline_mode, skb);
369		- num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
370		- stats->packets++;
371		- }
372		-
373		- stats->bytes += num_bytes;
374		- stats->xmit_more += skb->xmit_more;
375		-
376		- headlen = skb->len - ihs - skb->data_len;
377		- ds_cnt += !!headlen;
378		- ds_cnt += skb_shinfo(skb)->nr_frags;
379		-
380		- if (ihs) {
381		- ihs += !!skb_vlan_tag_present(skb) * VLAN_HLEN;
382		-
383		- ds_cnt_inl = DIV_ROUND_UP(ihs - INL_HDR_START_SZ, MLX5_SEND_WQE_DS);
384		- ds_cnt += ds_cnt_inl;
385		- }
386		-
387		- num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
388		- contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
389		- if (unlikely(contig_wqebbs_room < num_wqebbs)) {
390		-#ifdef CONFIG_MLX5_EN_IPSEC
391		- struct mlx5_wqe_eth_seg cur_eth = wqe->eth;
392		-#endif
393		- mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room);
394		- mlx5e_sq_fetch_wqe(sq, &wqe, &pi);
395		-#ifdef CONFIG_MLX5_EN_IPSEC
396		- wqe->eth = cur_eth;
397		-#endif
398		- }
	423	+ stats->xmit_more += xmit_more;
399	424
400	425	/* fill wqe */
401	426	wi = &sq->db.wqe_info[pi];
..	..	@@ -403,20 +428,18 @@
403	428	eseg = &wqe->eth;
404	429	dseg = wqe->data;
405	430
406		- mlx5e_txwqe_build_eseg_csum(sq, skb, eseg);
	431	+ eseg->mss = attr->mss;
407	432
408		- eseg->mss = mss;
409		-
410		- if (ihs) {
411		- eseg->inline_hdr.sz = cpu_to_be16(ihs);
	433	+ if (attr->ihs) {
412	434	if (skb_vlan_tag_present(skb)) {
413		- ihs -= VLAN_HLEN;
414		- mlx5e_insert_vlan(eseg->inline_hdr.start, skb, ihs);
	435	+ eseg->inline_hdr.sz \|= cpu_to_be16(attr->ihs + VLAN_HLEN);
	436	+ mlx5e_insert_vlan(eseg->inline_hdr.start, skb, attr->ihs);
415	437	stats->added_vlan_packets++;
416	438	} else {
417		- memcpy(eseg->inline_hdr.start, skb->data, ihs);
	439	+ eseg->inline_hdr.sz \|= cpu_to_be16(attr->ihs);
	440	+ memcpy(eseg->inline_hdr.start, skb->data, attr->ihs);
418	441	}
419		- dseg += ds_cnt_inl;
	442	+ dseg += wqe_attr->ds_cnt_inl;
420	443	} else if (skb_vlan_tag_present(skb)) {
421	444	eseg->insert.type = cpu_to_be16(MLX5_ETH_WQE_INSERT_VLAN);
422	445	if (skb->vlan_proto == cpu_to_be16(ETH_P_8021AD))
..	..	@@ -425,53 +448,282 @@
425	448	stats->added_vlan_packets++;
426	449	}
427	450
428		- num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + ihs, headlen, dseg);
	451	+ dseg += wqe_attr->ds_cnt_ids;
	452	+ num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + attr->ihs,
	453	+ attr->headlen, dseg);
429	454	if (unlikely(num_dma < 0))
430	455	goto err_drop;
431	456
432		- mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt, num_wqebbs, num_bytes,
433		- num_dma, wi, cseg);
	457	+ mlx5e_txwqe_complete(sq, skb, attr, wqe_attr, num_dma, wi, cseg, xmit_more);
434	458
435		- return NETDEV_TX_OK;
	459	+ return;
436	460
437	461	err_drop:
438	462	stats->dropped++;
439	463	dev_kfree_skb_any(skb);
	464	+}
440	465
441		- return NETDEV_TX_OK;
	466	+static bool mlx5e_tx_skb_supports_mpwqe(struct sk_buff skb, struct mlx5e_tx_attr attr)
	467	+{
	468	+ return !skb_is_nonlinear(skb) && !skb_vlan_tag_present(skb) && !attr->ihs &&
	469	+ !attr->insz;
	470	+}
	471	+
	472	+static bool mlx5e_tx_mpwqe_same_eseg(struct mlx5e_txqsq sq, struct mlx5_wqe_eth_seg eseg)
	473	+{
	474	+ struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
	475	+
	476	+ /* Assumes the session is already running and has at least one packet. */
	477	+ return !memcmp(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN);
	478	+}
	479	+
	480	+static void mlx5e_tx_mpwqe_session_start(struct mlx5e_txqsq *sq,
	481	+ struct mlx5_wqe_eth_seg *eseg)
	482	+{
	483	+ struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
	484	+ struct mlx5e_tx_wqe *wqe;
	485	+ u16 pi;
	486	+
	487	+ pi = mlx5e_txqsq_get_next_pi(sq, MLX5E_TX_MPW_MAX_WQEBBS);
	488	+ wqe = MLX5E_TX_FETCH_WQE(sq, pi);
	489	+ net_prefetchw(wqe->data);
	490	+
	491	+ *session = (struct mlx5e_tx_mpwqe) {
	492	+ .wqe = wqe,
	493	+ .bytes_count = 0,
	494	+ .ds_count = MLX5E_TX_WQE_EMPTY_DS_COUNT,
	495	+ .pkt_count = 0,
	496	+ .inline_on = 0,
	497	+ };
	498	+
	499	+ memcpy(&session->wqe->eth, eseg, MLX5E_ACCEL_ESEG_LEN);
	500	+
	501	+ sq->stats->mpwqe_blks++;
	502	+}
	503	+
	504	+static bool mlx5e_tx_mpwqe_session_is_active(struct mlx5e_txqsq *sq)
	505	+{
	506	+ return sq->mpwqe.wqe;
	507	+}
	508	+
	509	+static void mlx5e_tx_mpwqe_add_dseg(struct mlx5e_txqsq sq, struct mlx5e_xmit_data txd)
	510	+{
	511	+ struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
	512	+ struct mlx5_wqe_data_seg *dseg;
	513	+
	514	+ dseg = (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count;
	515	+
	516	+ session->pkt_count++;
	517	+ session->bytes_count += txd->len;
	518	+
	519	+ dseg->addr = cpu_to_be64(txd->dma_addr);
	520	+ dseg->byte_count = cpu_to_be32(txd->len);
	521	+ dseg->lkey = sq->mkey_be;
	522	+ session->ds_count++;
	523	+
	524	+ sq->stats->mpwqe_pkts++;
	525	+}
	526	+
	527	+static struct mlx5_wqe_ctrl_seg mlx5e_tx_mpwqe_session_complete(struct mlx5e_txqsq sq)
	528	+{
	529	+ struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
	530	+ u8 ds_count = session->ds_count;
	531	+ struct mlx5_wqe_ctrl_seg *cseg;
	532	+ struct mlx5e_tx_wqe_info *wi;
	533	+ u16 pi;
	534	+
	535	+ cseg = &session->wqe->ctrl;
	536	+ cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| MLX5_OPCODE_ENHANCED_MPSW);
	537	+ cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) \| ds_count);
	538	+
	539	+ pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
	540	+ wi = &sq->db.wqe_info[pi];
	541	+ *wi = (struct mlx5e_tx_wqe_info) {
	542	+ .skb = NULL,
	543	+ .num_bytes = session->bytes_count,
	544	+ .num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS),
	545	+ .num_dma = session->pkt_count,
	546	+ .num_fifo_pkts = session->pkt_count,
	547	+ };
	548	+
	549	+ sq->pc += wi->num_wqebbs;
	550	+
	551	+ session->wqe = NULL;
	552	+
	553	+ mlx5e_tx_check_stop(sq);
	554	+
	555	+ return cseg;
	556	+}
	557	+
	558	+static void
	559	+mlx5e_sq_xmit_mpwqe(struct mlx5e_txqsq sq, struct sk_buff skb,
	560	+ struct mlx5_wqe_eth_seg *eseg, bool xmit_more)
	561	+{
	562	+ struct mlx5_wqe_ctrl_seg *cseg;
	563	+ struct mlx5e_xmit_data txd;
	564	+
	565	+ if (!mlx5e_tx_mpwqe_session_is_active(sq)) {
	566	+ mlx5e_tx_mpwqe_session_start(sq, eseg);
	567	+ } else if (!mlx5e_tx_mpwqe_same_eseg(sq, eseg)) {
	568	+ mlx5e_tx_mpwqe_session_complete(sq);
	569	+ mlx5e_tx_mpwqe_session_start(sq, eseg);
	570	+ }
	571	+
	572	+ sq->stats->xmit_more += xmit_more;
	573	+
	574	+ txd.data = skb->data;
	575	+ txd.len = skb->len;
	576	+
	577	+ txd.dma_addr = dma_map_single(sq->pdev, txd.data, txd.len, DMA_TO_DEVICE);
	578	+ if (unlikely(dma_mapping_error(sq->pdev, txd.dma_addr)))
	579	+ goto err_unmap;
	580	+ mlx5e_dma_push(sq, txd.dma_addr, txd.len, MLX5E_DMA_MAP_SINGLE);
	581	+
	582	+ mlx5e_skb_fifo_push(sq, skb);
	583	+
	584	+ mlx5e_tx_mpwqe_add_dseg(sq, &txd);
	585	+
	586	+ mlx5e_tx_skb_update_hwts_flags(skb);
	587	+
	588	+ if (unlikely(mlx5e_tx_mpwqe_is_full(&sq->mpwqe))) {
	589	+ /* Might stop the queue and affect the retval of __netdev_tx_sent_queue. */
	590	+ cseg = mlx5e_tx_mpwqe_session_complete(sq);
	591	+
	592	+ if (__netdev_tx_sent_queue(sq->txq, txd.len, xmit_more))
	593	+ mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
	594	+ } else if (__netdev_tx_sent_queue(sq->txq, txd.len, xmit_more)) {
	595	+ /* Might stop the queue, but we were asked to ring the doorbell anyway. */
	596	+ cseg = mlx5e_tx_mpwqe_session_complete(sq);
	597	+
	598	+ mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
	599	+ }
	600	+
	601	+ return;
	602	+
	603	+err_unmap:
	604	+ mlx5e_dma_unmap_wqe_err(sq, 1);
	605	+ sq->stats->dropped++;
	606	+ dev_kfree_skb_any(skb);
	607	+}
	608	+
	609	+void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq)
	610	+{
	611	+ /* Unlikely in non-MPWQE workloads; not important in MPWQE workloads. */
	612	+ if (unlikely(mlx5e_tx_mpwqe_session_is_active(sq)))
	613	+ mlx5e_tx_mpwqe_session_complete(sq);
	614	+}
	615	+
	616	+static bool mlx5e_txwqe_build_eseg(struct mlx5e_priv priv, struct mlx5e_txqsq sq,
	617	+ struct sk_buff skb, struct mlx5e_accel_tx_state accel,
	618	+ struct mlx5_wqe_eth_seg *eseg, u16 ihs)
	619	+{
	620	+ if (unlikely(!mlx5e_accel_tx_eseg(priv, skb, eseg, ihs)))
	621	+ return false;
	622	+
	623	+ mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg);
	624	+
	625	+ return true;
442	626	}
443	627
444	628	netdev_tx_t mlx5e_xmit(struct sk_buff skb, struct net_device dev)
445	629	{
446	630	struct mlx5e_priv *priv = netdev_priv(dev);
	631	+ struct mlx5e_accel_tx_state accel = {};
	632	+ struct mlx5e_tx_wqe_attr wqe_attr;
	633	+ struct mlx5e_tx_attr attr;
447	634	struct mlx5e_tx_wqe *wqe;
448	635	struct mlx5e_txqsq *sq;
449	636	u16 pi;
450	637
451	638	sq = priv->txq2sq[skb_get_queue_mapping(skb)];
452		- mlx5e_sq_fetch_wqe(sq, &wqe, &pi);
453	639
454		- /* might send skbs and update wqe and pi */
455		- skb = mlx5e_accel_handle_tx(skb, sq, dev, &wqe, &pi);
456		- if (unlikely(!skb))
	640	+ /* May send SKBs and WQEs. */
	641	+ if (unlikely(!mlx5e_accel_tx_begin(dev, sq, skb, &accel)))
457	642	return NETDEV_TX_OK;
458	643
459		- return mlx5e_sq_xmit(sq, skb, wqe, pi);
	644	+ mlx5e_sq_xmit_prepare(sq, skb, &accel, &attr);
	645	+
	646	+ if (test_bit(MLX5E_SQ_STATE_MPWQE, &sq->state)) {
	647	+ if (mlx5e_tx_skb_supports_mpwqe(skb, &attr)) {
	648	+ struct mlx5_wqe_eth_seg eseg = {};
	649	+
	650	+ if (unlikely(!mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &eseg,
	651	+ attr.ihs)))
	652	+ return NETDEV_TX_OK;
	653	+
	654	+ mlx5e_sq_xmit_mpwqe(sq, skb, &eseg, netdev_xmit_more());
	655	+ return NETDEV_TX_OK;
	656	+ }
	657	+
	658	+ mlx5e_tx_mpwqe_ensure_complete(sq);
	659	+ }
	660	+
	661	+ mlx5e_sq_calc_wqe_attr(skb, &attr, &wqe_attr);
	662	+ pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs);
	663	+ wqe = MLX5E_TX_FETCH_WQE(sq, pi);
	664	+
	665	+ /* May update the WQE, but may not post other WQEs. */
	666	+ mlx5e_accel_tx_finish(sq, wqe, &accel,
	667	+ (struct mlx5_wqe_inline_seg *)(wqe->data + wqe_attr.ds_cnt_inl));
	668	+ if (unlikely(!mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &wqe->eth, attr.ihs)))
	669	+ return NETDEV_TX_OK;
	670	+
	671	+ mlx5e_sq_xmit_wqe(sq, skb, &attr, &wqe_attr, wqe, pi, netdev_xmit_more());
	672	+
	673	+ return NETDEV_TX_OK;
460	674	}
461	675
462		-static void mlx5e_dump_error_cqe(struct mlx5e_txqsq *sq,
463		- struct mlx5_err_cqe *err_cqe)
	676	+void mlx5e_sq_xmit_simple(struct mlx5e_txqsq sq, struct sk_buff skb, bool xmit_more)
464	677	{
465		- struct mlx5_cqwq *wq = &sq->cq.wq;
466		- u32 ci;
	678	+ struct mlx5e_tx_wqe_attr wqe_attr;
	679	+ struct mlx5e_tx_attr attr;
	680	+ struct mlx5e_tx_wqe *wqe;
	681	+ u16 pi;
467	682
468		- ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1);
	683	+ mlx5e_sq_xmit_prepare(sq, skb, NULL, &attr);
	684	+ mlx5e_sq_calc_wqe_attr(skb, &attr, &wqe_attr);
	685	+ pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs);
	686	+ wqe = MLX5E_TX_FETCH_WQE(sq, pi);
	687	+ mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, &wqe->eth);
	688	+ mlx5e_sq_xmit_wqe(sq, skb, &attr, &wqe_attr, wqe, pi, xmit_more);
	689	+}
469	690
470		- netdev_err(sq->channel->netdev,
471		- "Error cqe on cqn 0x%x, ci 0x%x, sqn 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n",
472		- sq->cq.mcq.cqn, ci, sq->sqn, err_cqe->syndrome,
473		- err_cqe->vendor_err_synd);
474		- mlx5_dump_err_cqe(sq->cq.mdev, err_cqe);
	691	+static void mlx5e_tx_wi_dma_unmap(struct mlx5e_txqsq sq, struct mlx5e_tx_wqe_info wi,
	692	+ u32 *dma_fifo_cc)
	693	+{
	694	+ int i;
	695	+
	696	+ for (i = 0; i < wi->num_dma; i++) {
	697	+ struct mlx5e_sq_dma dma = mlx5e_dma_get(sq, (dma_fifo_cc)++);
	698	+
	699	+ mlx5e_tx_dma_unmap(sq->pdev, dma);
	700	+ }
	701	+}
	702	+
	703	+static void mlx5e_consume_skb(struct mlx5e_txqsq sq, struct sk_buff skb,
	704	+ struct mlx5_cqe64 *cqe, int napi_budget)
	705	+{
	706	+ if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
	707	+ struct skb_shared_hwtstamps hwts = {};
	708	+ u64 ts = get_cqe_ts(cqe);
	709	+
	710	+ hwts.hwtstamp = mlx5_timecounter_cyc2time(sq->clock, ts);
	711	+ skb_tstamp_tx(skb, &hwts);
	712	+ }
	713	+
	714	+ napi_consume_skb(skb, napi_budget);
	715	+}
	716	+
	717	+static void mlx5e_tx_wi_consume_fifo_skbs(struct mlx5e_txqsq sq, struct mlx5e_tx_wqe_info wi,
	718	+ struct mlx5_cqe64 *cqe, int napi_budget)
	719	+{
	720	+ int i;
	721	+
	722	+ for (i = 0; i < wi->num_fifo_pkts; i++) {
	723	+ struct sk_buff *skb = mlx5e_skb_fifo_pop(sq);
	724	+
	725	+ mlx5e_consume_skb(sq, skb, cqe, napi_budget);
	726	+ }
475	727	}
476	728
477	729	bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
..	..	@@ -509,63 +761,56 @@
509	761
510	762	i = 0;
511	763	do {
	764	+ struct mlx5e_tx_wqe_info *wi;
512	765	u16 wqe_counter;
513	766	bool last_wqe;
	767	+ u16 ci;
514	768
515	769	mlx5_cqwq_pop(&cq->wq);
516	770
517	771	wqe_counter = be16_to_cpu(cqe->wqe_counter);
518	772
519		- if (unlikely(cqe->op_own >> 4 == MLX5_CQE_REQ_ERR)) {
520		- if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING,
521		- &sq->state)) {
522		- mlx5e_dump_error_cqe(sq,
523		- (struct mlx5_err_cqe *)cqe);
524		- queue_work(cq->channel->priv->wq,
525		- &sq->recover.recover_work);
526		- }
527		- stats->cqe_err++;
528		- }
529		-
530	773	do {
531		- struct mlx5e_tx_wqe_info *wi;
532		- struct sk_buff *skb;
533		- u16 ci;
534		- int j;
535		-
536	774	last_wqe = (sqcc == wqe_counter);
537	775
538	776	ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
539	777	wi = &sq->db.wqe_info[ci];
540		- skb = wi->skb;
541	778
542		- if (unlikely(!skb)) { /* nop */
543		- sqcc++;
	779	+ sqcc += wi->num_wqebbs;
	780	+
	781	+ if (likely(wi->skb)) {
	782	+ mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc);
	783	+ mlx5e_consume_skb(sq, wi->skb, cqe, napi_budget);
	784	+
	785	+ npkts++;
	786	+ nbytes += wi->num_bytes;
544	787	continue;
545	788	}
546	789
547		- if (unlikely(skb_shinfo(skb)->tx_flags &
548		- SKBTX_HW_TSTAMP)) {
549		- struct skb_shared_hwtstamps hwts = {};
	790	+ if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi,
	791	+ &dma_fifo_cc)))
	792	+ continue;
550	793
551		- hwts.hwtstamp =
552		- mlx5_timecounter_cyc2time(sq->clock,
553		- get_cqe_ts(cqe));
554		- skb_tstamp_tx(skb, &hwts);
	794	+ if (wi->num_fifo_pkts) {
	795	+ mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc);
	796	+ mlx5e_tx_wi_consume_fifo_skbs(sq, wi, cqe, napi_budget);
	797	+
	798	+ npkts += wi->num_fifo_pkts;
	799	+ nbytes += wi->num_bytes;
555	800	}
556		-
557		- for (j = 0; j < wi->num_dma; j++) {
558		- struct mlx5e_sq_dma *dma =
559		- mlx5e_dma_get(sq, dma_fifo_cc++);
560		-
561		- mlx5e_tx_dma_unmap(sq->pdev, dma);
562		- }
563		-
564		- npkts++;
565		- nbytes += wi->num_bytes;
566		- sqcc += wi->num_wqebbs;
567		- napi_consume_skb(skb, napi_budget);
568	801	} while (!last_wqe);
	802	+
	803	+ if (unlikely(get_cqe_opcode(cqe) == MLX5_CQE_REQ_ERR)) {
	804	+ if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING,
	805	+ &sq->state)) {
	806	+ mlx5e_dump_error_cqe(&sq->cq, sq->sqn,
	807	+ (struct mlx5_err_cqe *)cqe);
	808	+ mlx5_wq_cyc_wqe_dump(&sq->wq, ci, wi->num_wqebbs);
	809	+ queue_work(cq->channel->priv->wq,
	810	+ &sq->recover_work);
	811	+ }
	812	+ stats->cqe_err++;
	813	+ }
569	814
570	815	} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
571	816
..	..	@@ -582,8 +827,7 @@
582	827	netdev_tx_completed_queue(sq->txq, npkts, nbytes);
583	828
584	829	if (netif_tx_queue_stopped(sq->txq) &&
585		- mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc,
586		- MLX5E_SQ_STOP_ROOM) &&
	830	+ mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room) &&
587	831	!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) {
588	832	netif_tx_wake_queue(sq->txq);
589	833	stats->wake++;
..	..	@@ -592,36 +836,53 @@
592	836	return (i == MLX5E_TX_CQ_POLL_BUDGET);
593	837	}
594	838
	839	+static void mlx5e_tx_wi_kfree_fifo_skbs(struct mlx5e_txqsq sq, struct mlx5e_tx_wqe_info wi)
	840	+{
	841	+ int i;
	842	+
	843	+ for (i = 0; i < wi->num_fifo_pkts; i++)
	844	+ dev_kfree_skb_any(mlx5e_skb_fifo_pop(sq));
	845	+}
	846	+
595	847	void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq)
596	848	{
597	849	struct mlx5e_tx_wqe_info *wi;
598		- u32 nbytes = 0;
599		- u16 ci, npkts = 0;
600		- struct sk_buff *skb;
601		- int i;
	850	+ u32 dma_fifo_cc, nbytes = 0;
	851	+ u16 ci, sqcc, npkts = 0;
602	852
603		- while (sq->cc != sq->pc) {
604		- ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
	853	+ sqcc = sq->cc;
	854	+ dma_fifo_cc = sq->dma_fifo_cc;
	855	+
	856	+ while (sqcc != sq->pc) {
	857	+ ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
605	858	wi = &sq->db.wqe_info[ci];
606		- skb = wi->skb;
607	859
608		- if (!skb) { /* nop */
609		- sq->cc++;
	860	+ sqcc += wi->num_wqebbs;
	861	+
	862	+ if (likely(wi->skb)) {
	863	+ mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc);
	864	+ dev_kfree_skb_any(wi->skb);
	865	+
	866	+ npkts++;
	867	+ nbytes += wi->num_bytes;
610	868	continue;
611	869	}
612	870
613		- for (i = 0; i < wi->num_dma; i++) {
614		- struct mlx5e_sq_dma *dma =
615		- mlx5e_dma_get(sq, sq->dma_fifo_cc++);
	871	+ if (unlikely(mlx5e_ktls_tx_try_handle_resync_dump_comp(sq, wi, &dma_fifo_cc)))
	872	+ continue;
616	873
617		- mlx5e_tx_dma_unmap(sq->pdev, dma);
	874	+ if (wi->num_fifo_pkts) {
	875	+ mlx5e_tx_wi_dma_unmap(sq, wi, &dma_fifo_cc);
	876	+ mlx5e_tx_wi_kfree_fifo_skbs(sq, wi);
	877	+
	878	+ npkts += wi->num_fifo_pkts;
	879	+ nbytes += wi->num_bytes;
618	880	}
619		-
620		- dev_kfree_skb_any(skb);
621		- npkts++;
622		- nbytes += wi->num_bytes;
623		- sq->cc += wi->num_wqebbs;
624	881	}
	882	+
	883	+ sq->dma_fifo_cc = dma_fifo_cc;
	884	+ sq->cc = sqcc;
	885	+
625	886	netdev_tx_completed_queue(sq->txq, npkts, nbytes);
626	887	}
627	888
..	..	@@ -635,10 +896,34 @@
635	896	dseg->av.key.qkey.qkey = cpu_to_be32(dqkey);
636	897	}
637	898
638		-netdev_tx_t mlx5i_sq_xmit(struct mlx5e_txqsq sq, struct sk_buff skb,
639		- struct mlx5_av *av, u32 dqpn, u32 dqkey)
	899	+static void mlx5i_sq_calc_wqe_attr(struct sk_buff *skb,
	900	+ const struct mlx5e_tx_attr *attr,
	901	+ struct mlx5e_tx_wqe_attr *wqe_attr)
640	902	{
641		- struct mlx5_wq_cyc *wq = &sq->wq;
	903	+ u16 ds_cnt = sizeof(struct mlx5i_tx_wqe) / MLX5_SEND_WQE_DS;
	904	+ u16 ds_cnt_inl = 0;
	905	+
	906	+ ds_cnt += !!attr->headlen + skb_shinfo(skb)->nr_frags;
	907	+
	908	+ if (attr->ihs) {
	909	+ u16 inl = attr->ihs - INL_HDR_START_SZ;
	910	+
	911	+ ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS);
	912	+ ds_cnt += ds_cnt_inl;
	913	+ }
	914	+
	915	+ *wqe_attr = (struct mlx5e_tx_wqe_attr) {
	916	+ .ds_cnt = ds_cnt,
	917	+ .ds_cnt_inl = ds_cnt_inl,
	918	+ .num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS),
	919	+ };
	920	+}
	921	+
	922	+void mlx5i_sq_xmit(struct mlx5e_txqsq sq, struct sk_buff skb,
	923	+ struct mlx5_av *av, u32 dqpn, u32 dqkey, bool xmit_more)
	924	+{
	925	+ struct mlx5e_tx_wqe_attr wqe_attr;
	926	+ struct mlx5e_tx_attr attr;
642	927	struct mlx5i_tx_wqe *wqe;
643	928
644	929	struct mlx5_wqe_datagram_seg *datagram;
..	..	@@ -648,50 +933,16 @@
648	933	struct mlx5e_tx_wqe_info *wi;
649	934
650	935	struct mlx5e_sq_stats *stats = sq->stats;
651		- u16 headlen, ihs, pi, contig_wqebbs_room;
652		- u16 ds_cnt, ds_cnt_inl = 0;
653		- u8 num_wqebbs, opcode;
654		- u32 num_bytes;
655	936	int num_dma;
656		- __be16 mss;
	937	+ u16 pi;
657	938
658		- /* Calc ihs and ds cnt, no writes to wqe yet */
659		- ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
660		- if (skb_is_gso(skb)) {
661		- opcode = MLX5_OPCODE_LSO;
662		- mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
663		- ihs = mlx5e_tx_get_gso_ihs(sq, skb);
664		- num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs;
665		- stats->packets += skb_shinfo(skb)->gso_segs;
666		- } else {
667		- opcode = MLX5_OPCODE_SEND;
668		- mss = 0;
669		- ihs = mlx5e_calc_min_inline(sq->min_inline_mode, skb);
670		- num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
671		- stats->packets++;
672		- }
	939	+ mlx5e_sq_xmit_prepare(sq, skb, NULL, &attr);
	940	+ mlx5i_sq_calc_wqe_attr(skb, &attr, &wqe_attr);
673	941
674		- stats->bytes += num_bytes;
675		- stats->xmit_more += skb->xmit_more;
	942	+ pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs);
	943	+ wqe = MLX5I_SQ_FETCH_WQE(sq, pi);
676	944
677		- headlen = skb->len - ihs - skb->data_len;
678		- ds_cnt += !!headlen;
679		- ds_cnt += skb_shinfo(skb)->nr_frags;
680		-
681		- if (ihs) {
682		- ds_cnt_inl = DIV_ROUND_UP(ihs - INL_HDR_START_SZ, MLX5_SEND_WQE_DS);
683		- ds_cnt += ds_cnt_inl;
684		- }
685		-
686		- num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
687		- pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
688		- contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
689		- if (unlikely(contig_wqebbs_room < num_wqebbs)) {
690		- mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room);
691		- pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
692		- }
693		-
694		- mlx5i_sq_fetch_wqe(sq, &wqe, pi);
	945	+ stats->xmit_more += xmit_more;
695	946
696	947	/* fill wqe */
697	948	wi = &sq->db.wqe_info[pi];
..	..	@@ -702,29 +953,27 @@
702	953
703	954	mlx5i_txwqe_build_datagram(av, dqpn, dqkey, datagram);
704	955
705		- mlx5e_txwqe_build_eseg_csum(sq, skb, eseg);
	956	+ mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, eseg);
706	957
707		- eseg->mss = mss;
	958	+ eseg->mss = attr.mss;
708	959
709		- if (ihs) {
710		- memcpy(eseg->inline_hdr.start, skb->data, ihs);
711		- eseg->inline_hdr.sz = cpu_to_be16(ihs);
712		- dseg += ds_cnt_inl;
	960	+ if (attr.ihs) {
	961	+ memcpy(eseg->inline_hdr.start, skb->data, attr.ihs);
	962	+ eseg->inline_hdr.sz = cpu_to_be16(attr.ihs);
	963	+ dseg += wqe_attr.ds_cnt_inl;
713	964	}
714	965
715		- num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + ihs, headlen, dseg);
	966	+ num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + attr.ihs,
	967	+ attr.headlen, dseg);
716	968	if (unlikely(num_dma < 0))
717	969	goto err_drop;
718	970
719		- mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt, num_wqebbs, num_bytes,
720		- num_dma, wi, cseg);
	971	+ mlx5e_txwqe_complete(sq, skb, &attr, &wqe_attr, num_dma, wi, cseg, xmit_more);
721	972
722		- return NETDEV_TX_OK;
	973	+ return;
723	974
724	975	err_drop:
725	976	stats->dropped++;
726	977	dev_kfree_skb_any(skb);
727		-
728		- return NETDEV_TX_OK;
729	978	}
730	979	#endif